feat(tools): every image judged by two models, every generated texture drawn by two checkpoints

A single vision model has been wrong in both directions on this project - it missed real defects and
disputed frames that were correct - so agreement between different model families is worth more than one
confident opinion.

judge_screenshot.py now runs two models by default (openai/gpt-4o-mini + qwen/qwen3.7-flash, both served
on OpenRouter; claude-3.5-haiku is gone, which the first attempt discovered with a 404). combine()
reconciles them: agreement carries the verdict with the mean confidence, disagreement reports 'uncertain'
and keeps both reasons rather than letting the more confident model win. --model still forces a single
judgement, and every line prints which model said what.

comfyui_generate.py generates the same prompt and seed with two checkpoints, scores both by
high-frequency detail (mean absolute Laplacian, which catches blurry or flat results), keeps the loser as
<out>__<model>.png with its sidecar, and moves the winner onto --out. --alt-model  restores the single
run.

Tests cover the reconciliation - agreement, disagreement, unknown verdicts, and no verdicts at all - and the
suite is at 103.
This commit is contained in:
zwitschi committed 2026-11-10 09:12:26 +01:00
1 parent bf7dfe1052
commit abde390045
4 files changed
+158 -10

No files matched your search

+61
View File
@@ -40,6 +40,26 @@ import urllib.request
HOST = os.environ.get("COMFYUI_HOST", "http://127.0.0.1:8188")
DEFAULT_MODEL = "juggernautXL_ragnarok.safetensors"
# A second checkpoint by default: one model's texture is a lottery ticket, two is a comparison. Both are
# generated from the identical prompt and seed, then scored, and only the winner takes the output path.
DEFAULT_ALT_MODEL = "sd_xl_base_1.0.safetensors"
def score_image(path):
"""A deterministic, offline 'how much detail did this texture actually get' score.
Mean absolute Laplacian: high-frequency energy. Generated textures that are blurry or flat score low,
which is the failure this comparison exists to catch. Pillow is used only here, so a host without it
still generates and simply skips scoring.
"""
try:
from PIL import Image, ImageFilter
except ImportError: # pragma: no cover - host without Pillow
return None
with Image.open(path) as image:
grey = image.convert("L").filter(ImageFilter.FIND_EDGES)
pixels = list(grey.getdata())
return sum(pixels) / max(1, len(pixels))
def api(path: str) -> dict:
@@ -156,6 +176,42 @@ def generate(args) -> int:
def generate_pair(args) -> int:
"""Generate the same prompt with two checkpoints and keep the better one at `--out`.
Both are written beside the output as `<out stem>__<model>.png` with their own sidecars, so the losing
candidate is kept for inspection instead of thrown away. The winner is chosen by `score_image()`; when
Pillow is missing or the scores tie, the primary model keeps the output and the run says so.
"""
import dataclasses
stem, ext = os.path.splitext(args.out)
candidates = []
for label, model in (("primary", args.model), ("alternate", args.alt_model)):
out = f"{stem}__{model.replace('/', '_')[:40]}{ext}"
sub_args = dataclasses.replace(args, model=model, out=out)
if generate(sub_args) != 0:
print(f"{label} model {model} failed; continuing with what worked")
continue
candidates.append((model, out, score_image(out)))
if not candidates:
return 1
scored = [c for c in candidates if c[2] is not None]
if not scored:
print(f"no scoring available; keeping {candidates[0][1]} as the output")
winner = candidates[0]
else:
winner = max(scored, key=lambda c: c[2])
for model, path, score in candidates:
print(f" {model}: {score if score is None else round(score, 2)} {path}")
os.replace(winner[1], args.out)
sidecar = winner[1] + ".json"
if os.path.exists(sidecar):
os.replace(sidecar, args.out + ".json")
print(f"chose {winner[0]} -> {args.out}")
return 0
def main(argv=None) -> int:
argv = sys.argv[1:] if argv is None else argv
ap = argparse.ArgumentParser(description=__doc__,
@@ -166,6 +222,9 @@ def main(argv=None) -> int:
t.add_argument("--prompt", required=True)
t.add_argument("--negative", default="text, watermark, signature, blurry, low quality")
t.add_argument("--model", default=DEFAULT_MODEL)
t.add_argument("--alt-model", default=DEFAULT_ALT_MODEL,
help=f"second checkpoint generated from the same prompt and seed, then compared "
f"(default {DEFAULT_ALT_MODEL}); pass an empty string to generate one image only")
t.add_argument("--size", default="1024x1024")
t.add_argument("--seed", type=int, default=None)
t.add_argument("--steps", type=int, default=28)
@@ -181,6 +240,8 @@ def main(argv=None) -> int:
for name in checkpoints():
print(name)
return 0
if args.alt_model and args.alt_model != args.model:
return generate_pair(args)
return generate(args)