zwitschi/gig-line
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(character): ComfyUI blueprints and a render-vs-blueprint evaluation

Browse Source 
Two tools that turn character work into something measurable.

- Tools/comfyui_character_blueprint.py generates a four-view sheet (front, side, back,
  head) plus a contact sheet from one description, with fixed seeds and a JSON record
  of prompts and prompt ids. First sheet: 'GigLine walker', lean pedestrian in a navy
  tee, jeans and sneakers.
- Tools/character_match.py compares a rendered character against a sheet: figure
  aspect, fill and coverage deltas, mean CIE76 colour distance, dominant-palette
  distance, and a side-by-side montage. The deltas stay separate on purpose - a single
  similarity score between a photoreal sheet and a low-detail body would be a number
  with no meaning.
- comfyui_generate.py grew wait_for_image() so both tools share the polling path
  instead of duplicating it.
- 13 unit tests for the colour maths, masks and palette; 95 tool tests pass.

First real result, recorded rather than smoothed over: aspect delta 1.24, fill delta
0.12, mean deltaE 31.9, palette deltaE 25.4. The montage shows the reason - the model
is a nude body and the blueprint wears a tee and jeans, so clothing dominates every
metric. Figure segmentation is explicit (--render-box): auto-detection picked backdrop
blobs on every attempt, and a background plate is the proper fix.
This commit is contained in:
zwitschi committed 2026-11-09 13:36:28 +01:00
1 parent 1fc5835327
commit 85a8437cbc
56 files changed
+841 -170
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Tools/character_match.py
+368
View File
@@ -0,0 +1,368 @@
#!/usr/bin/env python3
"""Evaluate a rendered character against a generated blueprint.
The blueprint comes from `Tools/comfyui_character_blueprint.py`; the render comes from
the review pipeline. This tool isolates the figure in both, then reports how far the
model has drifted from the intended character:
python3 Tools/character_match.py \
--render Screenshots/demo/review-demo-third-person-front.png \
--blueprint Art/characters/blueprints/front.png \
--out /tmp/gigline/character-match.json \
--montage /tmp/gigline/character-match.png
Reported per pair:
aspect_delta |height/width difference| between the two figures, in ratio units
fill_delta |figure area / bbox area| - bulk, catches a thin or bloated build
coverage_delta |figure area / frame area| - how much of the frame the figure takes
mean_delta_e mean CIE76 colour distance over the figure
palette_delta_e mean distance between the five dominant colours of each figure
Why these and not a similarity score: an absolute "match percentage" between a
photoreal generated sheet and a low-detail game body would be a number with no
meaning. The deltas are separable, so a report says *what* drifted - build, framing
or colour - instead of blending all three into one figure.
The figure mask is a heuristic and the tool says so in its report: for the blueprint
it is "differs from the corner colour", for the render it is "more saturated or darker
than the frame median", largest connected component. Pair the tool with a human look
at the montage; it narrows where to look, it does not replace the review.
"""
from __future__ import annotations
import argparse
import json
import math
import os
import sys
from PIL import Image
# ---------------------------------------------------------------- colour maths
def srgb_to_linear(value: float) -> float:
value /= 255.0
return value / 12.92 if value <= 0.04045 else ((value + 0.055) / 1.055) ** 2.4
def rgb_to_lab(rgb) -> tuple[float, float, float]:
"""sRGB (0-255) to CIELAB under D65."""
r, g, b = (srgb_to_linear(v) for v in rgb[:3])
x = r * 0.4124 + g * 0.3576 + b * 0.1805
y = r * 0.2126 + g * 0.7152 + b * 0.0722
z = r * 0.0193 + g * 0.1192 + b * 0.9505
# D65 white point
xn, yn, zn = 0.95047, 1.0, 1.08883
def f(t: float) -> float:
return t ** (1.0 / 3.0) if t > 0.008856 else (7.787 * t) + (16.0 / 116.0)
fx, fy, fz = f(x / xn), f(y / yn), f(z / zn)
return (116.0 * fy - 16.0, 500.0 * (fx - fy), 200.0 * (fy - fz))
def delta_e76(lab_a, lab_b) -> float:
return math.sqrt(sum((a - b) ** 2 for a, b in zip(lab_a, lab_b)))
# ------------------------------------------------------------------- masking
def luminance(rgb) -> float:
return 0.2126 * rgb[0] + 0.7152 * rgb[1] + 0.0722 * rgb[2]
def _subject_mask(image: Image.Image, saturation_threshold: int = 20,
dark_ratio: float = 0.62) -> Image.Image:
"""Mask the figure as "more saturated than the backdrop, or notably darker".
Both inputs have a desaturated backdrop - grey studio sweep in the blueprint,
beige pavement and pale walls in the render - and a figure that carries colour
(skin, denim, hair). Saturation separates them without needing a clean plate.
The previous version masked the blueprint by "differs from the corner colour",
which swallowed a gradient backdrop whole.
"""
rgb = image.convert("RGB")
pixels = rgb.load()
lums = [luminance(pixels[x, y])
for y in range(0, rgb.height, 4) for x in range(0, rgb.width, 4)]
lums.sort()
median = lums[len(lums) // 2] if lums else 128.0
mask = Image.new("L", rgb.size, 0)
out = mask.load()
for y in range(rgb.height):
for x in range(rgb.width):
r, g, b = pixels[x, y]
saturation = max(r, g, b) - min(r, g, b)
if saturation > saturation_threshold or luminance((r, g, b)) < median * dark_ratio:
out[x, y] = 255
return mask
def figure_mask_from_background(image: Image.Image, tolerance: int = 28):
"""Figure mask for a blueprint sheet: a coloured subject on a grey sweep."""
return _largest_component(_subject_mask(image, saturation_threshold=18,
dark_ratio=0.55))
def figure_mask_from_frame(image: Image.Image):
"""Figure mask for a rendered frame: the body against pavement and stucco."""
return _largest_component(_subject_mask(image, saturation_threshold=26,
dark_ratio=0.72))
def _largest_component(mask: Image.Image) -> Image.Image:
"""Keep the largest plausible figure blob.
"Largest" alone is wrong twice over: in both inputs the backdrop is one huge
connected region, so a plain largest-blob pick returned the background and the
first report measured the whole frame. The filter is size-based instead - a figure
occupies between 0.2 % and 55 % of the frame - which rejects the backdrop on one
side and road markings on the other.
"""
width, height = mask.size
frame_area = width * height
source = mask.load()
seen = bytearray(width * height)
best = [] # largest blob fully inside the frame
usable = [] # largest plausible blob, border or not
fallback = [] # largest blob at all, used when nothing is plausible
best_score = 0
for start_y in range(0, height, 3):
for start_x in range(0, width, 3):
index = start_y * width + start_x
if seen[index] or source[start_x, start_y] == 0:
continue
stack = [(start_x, start_y)]
blob = []
seen[index] = 1
while stack:
x, y = stack.pop()
blob.append((x, y))
for nx, ny in ((x + 1, y), (x - 1, y), (x, y + 1), (x, y - 1)):
if 0 <= nx < width and 0 <= ny < height:
nindex = ny * width + nx
if not seen[nindex] and source[nx, ny] != 0:
seen[nindex] = 1
stack.append((nx, ny))
if len(blob) > len(fallback):
fallback = blob
share = len(blob) / frame_area
if not (0.002 < share < 0.55):
continue
# Blobs glued to the frame edge are backdrop: the sky band in a render and
# the studio sweep in a sheet both touch the border and both beat the
# figure on raw area. Prefer fully enclosed blobs, fall back by area.
xs = [p[0] for p in blob]
ys = [p[1] for p in blob]
enclosed = (min(xs) > 1 and min(ys) > 1
and max(xs) < width - 2 and max(ys) < height - 2)
if enclosed and len(blob) > best_score:
best_score = len(blob)
best = blob
elif len(blob) > len(usable):
usable = blob
keep = Image.new("L", mask.size, 0)
out = keep.load()
for x, y in (best or usable or fallback):
out[x, y] = 255
return keep
def bbox(mask: Image.Image):
return mask.getbbox()
def mask_stats(mask: Image.Image) -> dict:
box = mask.getbbox()
if not box:
return {"area": 0, "bbox": None, "aspect": None, "fill": None, "coverage": None}
x0, y0, x1, y1 = box
histogram = mask.histogram()
area = sum(histogram[1:])
box_area = max(1, (x1 - x0) * (y1 - y0))
return {
"area": area,
"bbox": [x0, y0, x1, y1],
"aspect": (y1 - y0) / max(1, (x1 - x0)),
"fill": area / box_area,
"coverage": area / (mask.width * mask.height),
}
def masked_colours(image: Image.Image, mask: Image.Image, limit=20000):
rgb = image.convert("RGB")
pixels = rgb.load()
values = mask.load()
collected = []
step = max(1, (rgb.width * rgb.height) // limit)
index = 0
for y in range(rgb.height):
for x in range(rgb.width):
index += 1
if index % step:
continue
if values[x, y]:
collected.append(pixels[x, y])
return collected
def dominant(counts: dict, count: int = 5):
"""Coarse palette: quantise to 32 levels per channel, then take the top bins."""
bins: dict[tuple, int] = {}
for colour in counts:
key = (colour[0] // 32, colour[1] // 32, colour[2] // 32)
bins[key] = bins.get(key, 0) + 1
ranked = sorted(bins.items(), key=lambda item: -item[1])[:count]
return [((k[0] * 32) + 16, (k[1] * 32) + 16, (k[2] * 32) + 16) for k, _ in ranked]
def palette_delta_e(palette_a, palette_b) -> float:
if not palette_a or not palette_b:
return float("nan")
labs_b = [rgb_to_lab(c) for c in palette_b]
total = 0.0
for colour in palette_a:
lab = rgb_to_lab(colour)
total += min(delta_e76(lab, other) for other in labs_b)
return total / len(palette_a)
# -------------------------------------------------------------------- report
def evaluate(render_path: str, blueprint_path: str, montage_path: str | None = None,
render_box: tuple[float, float, float, float] | None = None) -> dict:
render_full = Image.open(render_path).convert("RGB")
blueprint = Image.open(blueprint_path).convert("RGB")
# The figure box is explicit by default. Auto-segmentation on these shots failed
# in a way worth recording rather than papering over: the backdrop (sky band, pale
# wall, dark pavement) is more saturated and far larger than the subject, so every
# size-based rule picked a backdrop blob. A box read off the shot is honest and
# cheap; a background plate rendered without the character would be the proper fix.
width, height = render_full.size
box = render_box or (0.15, 0.02, 0.85, 0.99)
region = (int(width * box[0]), int(height * box[1]),
int(width * box[2]), int(height * box[3]))
render = render_full.crop(region)
render_mask = figure_mask_from_frame(render)
blueprint_mask = figure_mask_from_background(blueprint)
render_stats = mask_stats(render_mask)
blueprint_stats = mask_stats(blueprint_mask)
render_colours = masked_colours(render, render_mask)
blueprint_colours = masked_colours(blueprint, blueprint_mask)
def mean_lab(colours):
labs = [rgb_to_lab(c) for c in colours]
if not labs:
return None
return tuple(sum(l[i] for l in labs) / len(labs) for i in range(3))
lab_render = mean_lab(render_colours)
lab_blueprint = mean_lab(blueprint_colours)
mean_de = (delta_e76(lab_render, lab_blueprint)
if lab_render and lab_blueprint else float("nan"))
palette_render = dominant(render_colours)
palette_blueprint = dominant(blueprint_colours)
def delta(a, b):
if a is None or b is None:
return None
return abs(a - b)
report = {
"render": os.path.abspath(render_path),
"blueprint": os.path.abspath(blueprint_path),
"render_mask": "saturated-or-dark vs frame median, largest component",
"blueprint_mask": "differs from corner colour",
"render_region": list(region),
"render_figure": render_stats,
"blueprint_figure": blueprint_stats,
"aspect_delta": delta(render_stats["aspect"], blueprint_stats["aspect"]),
"fill_delta": delta(render_stats["fill"], blueprint_stats["fill"]),
"coverage_delta": delta(render_stats["coverage"], blueprint_stats["coverage"]),
"mean_lab_render": lab_render,
"mean_lab_blueprint": lab_blueprint,
"mean_delta_e": mean_de,
"palette_render": palette_render,
"palette_blueprint": palette_blueprint,
"palette_delta_e": palette_delta_e(palette_render, palette_blueprint),
}
if montage_path:
report["montage"] = write_montage(render, render_mask, blueprint, blueprint_mask,
montage_path)
return report
def write_montage(render, render_mask, blueprint, blueprint_mask, path, height=640):
"""Render crop and blueprint crop side by side, same height, for a human look."""
def crop(image, mask):
box = mask.getbbox()
return image.crop(box) if box else image
left = crop(render, render_mask)
right = crop(blueprint, blueprint_mask)
left = left.resize((max(1, int(left.width * height / max(1, left.height))), height))
right = right.resize((max(1, int(right.width * height / max(1, right.height))), height))
sheet = Image.new("RGB", (left.width + right.width + 12, height), (20, 20, 20))
sheet.paste(left, (0, 0))
sheet.paste(right, (left.width + 12, 0))
os.makedirs(os.path.dirname(os.path.abspath(path)) or ".", exist_ok=True)
sheet.save(path)
return os.path.abspath(path)
def main(argv=None):
ap = argparse.ArgumentParser(description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--render", required=True)
ap.add_argument("--blueprint", required=True)
ap.add_argument("--out", default="/tmp/gigline/character-match.json")
ap.add_argument("--montage", default=None)
ap.add_argument("--render-box", default=None,
help="figure box as x0,y0,x1,y1 fractions of the frame (default: central 70 percent)")
args = ap.parse_args(argv)
for path in (args.render, args.blueprint):
if not os.path.exists(path):
print(f"error: no such image: {path}", file=sys.stderr)
return 2
box = None
if args.render_box:
box = tuple(float(v) for v in args.render_box.split(","))
if len(box) != 4:
print("error: --render-box needs four comma-separated fractions", file=sys.stderr)
return 2
report = evaluate(args.render, args.blueprint, args.montage, box)
os.makedirs(os.path.dirname(os.path.abspath(args.out)) or ".", exist_ok=True)
with open(args.out, "w") as handle:
json.dump(report, handle, indent=2)
def show(value, digits=3):
return "n/a" if value is None or (isinstance(value, float) and math.isnan(value)) \
else f"{value:.{digits}f}"
print(f"render {report['render_figure']}")
print(f"blueprint {report['blueprint_figure']}")
print(f"aspect delta {show(report['aspect_delta'])}")
print(f"fill delta {show(report['fill_delta'])}")
print(f"coverage delta {show(report['coverage_delta'])}")
print(f"mean deltaE {show(report['mean_delta_e'], 1)}")
print(f"palette deltaE {show(report['palette_delta_e'], 1)}")
print(f"report -> {args.out}")
return 0
if __name__ == "__main__":
raise SystemExit(main())