feat(tools): every image judged by two models, every generated texture drawn by two checkpoints
A single vision model has been wrong in both directions on this project - it missed real defects and disputed frames that were correct - so agreement between different model families is worth more than one confident opinion. judge_screenshot.py now runs two models by default (openai/gpt-4o-mini + qwen/qwen3.7-flash, both served on OpenRouter; claude-3.5-haiku is gone, which the first attempt discovered with a 404). combine() reconciles them: agreement carries the verdict with the mean confidence, disagreement reports 'uncertain' and keeps both reasons rather than letting the more confident model win. --model still forces a single judgement, and every line prints which model said what. comfyui_generate.py generates the same prompt and seed with two checkpoints, scores both by high-frequency detail (mean absolute Laplacian, which catches blurry or flat results), keeps the loser as <out>__<model>.png with its sidecar, and moves the winner onto --out. --alt-model restores the single run. Tests cover the reconciliation - agreement, disagreement, unknown verdicts, and no verdicts at all - and the suite is at 103.
This commit is contained in:
1 parent
bf7dfe1052
commit
abde390045
4 files changed
+158
-10
No files matched your search
@@ -40,6 +40,26 @@ import urllib.request
|
||||
|
||||
HOST = os.environ.get("COMFYUI_HOST", "http://127.0.0.1:8188")
|
||||
DEFAULT_MODEL = "juggernautXL_ragnarok.safetensors"
|
||||
# A second checkpoint by default: one model's texture is a lottery ticket, two is a comparison. Both are
|
||||
# generated from the identical prompt and seed, then scored, and only the winner takes the output path.
|
||||
DEFAULT_ALT_MODEL = "sd_xl_base_1.0.safetensors"
|
||||
|
||||
|
||||
def score_image(path):
|
||||
"""A deterministic, offline 'how much detail did this texture actually get' score.
|
||||
|
||||
Mean absolute Laplacian: high-frequency energy. Generated textures that are blurry or flat score low,
|
||||
which is the failure this comparison exists to catch. Pillow is used only here, so a host without it
|
||||
still generates and simply skips scoring.
|
||||
"""
|
||||
try:
|
||||
from PIL import Image, ImageFilter
|
||||
except ImportError: # pragma: no cover - host without Pillow
|
||||
return None
|
||||
with Image.open(path) as image:
|
||||
grey = image.convert("L").filter(ImageFilter.FIND_EDGES)
|
||||
pixels = list(grey.getdata())
|
||||
return sum(pixels) / max(1, len(pixels))
|
||||
|
||||
|
||||
def api(path: str) -> dict:
|
||||
@@ -156,6 +176,42 @@ def generate(args) -> int:
|
||||
|
||||
|
||||
|
||||
def generate_pair(args) -> int:
|
||||
"""Generate the same prompt with two checkpoints and keep the better one at `--out`.
|
||||
|
||||
Both are written beside the output as `<out stem>__<model>.png` with their own sidecars, so the losing
|
||||
candidate is kept for inspection instead of thrown away. The winner is chosen by `score_image()`; when
|
||||
Pillow is missing or the scores tie, the primary model keeps the output and the run says so.
|
||||
"""
|
||||
import dataclasses
|
||||
|
||||
stem, ext = os.path.splitext(args.out)
|
||||
candidates = []
|
||||
for label, model in (("primary", args.model), ("alternate", args.alt_model)):
|
||||
out = f"{stem}__{model.replace('/', '_')[:40]}{ext}"
|
||||
sub_args = dataclasses.replace(args, model=model, out=out)
|
||||
if generate(sub_args) != 0:
|
||||
print(f"{label} model {model} failed; continuing with what worked")
|
||||
continue
|
||||
candidates.append((model, out, score_image(out)))
|
||||
if not candidates:
|
||||
return 1
|
||||
scored = [c for c in candidates if c[2] is not None]
|
||||
if not scored:
|
||||
print(f"no scoring available; keeping {candidates[0][1]} as the output")
|
||||
winner = candidates[0]
|
||||
else:
|
||||
winner = max(scored, key=lambda c: c[2])
|
||||
for model, path, score in candidates:
|
||||
print(f" {model}: {score if score is None else round(score, 2)} {path}")
|
||||
os.replace(winner[1], args.out)
|
||||
sidecar = winner[1] + ".json"
|
||||
if os.path.exists(sidecar):
|
||||
os.replace(sidecar, args.out + ".json")
|
||||
print(f"chose {winner[0]} -> {args.out}")
|
||||
return 0
|
||||
|
||||
|
||||
def main(argv=None) -> int:
|
||||
argv = sys.argv[1:] if argv is None else argv
|
||||
ap = argparse.ArgumentParser(description=__doc__,
|
||||
@@ -166,6 +222,9 @@ def main(argv=None) -> int:
|
||||
t.add_argument("--prompt", required=True)
|
||||
t.add_argument("--negative", default="text, watermark, signature, blurry, low quality")
|
||||
t.add_argument("--model", default=DEFAULT_MODEL)
|
||||
t.add_argument("--alt-model", default=DEFAULT_ALT_MODEL,
|
||||
help=f"second checkpoint generated from the same prompt and seed, then compared "
|
||||
f"(default {DEFAULT_ALT_MODEL}); pass an empty string to generate one image only")
|
||||
t.add_argument("--size", default="1024x1024")
|
||||
t.add_argument("--seed", type=int, default=None)
|
||||
t.add_argument("--steps", type=int, default=28)
|
||||
@@ -181,6 +240,8 @@ def main(argv=None) -> int:
|
||||
for name in checkpoints():
|
||||
print(name)
|
||||
return 0
|
||||
if args.alt_model and args.alt_model != args.model:
|
||||
return generate_pair(args)
|
||||
return generate(args)
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user