feat(tools): every image judged by two models, every generated texture drawn by two checkpoints

A single vision model has been wrong in both directions on this project - it missed real defects and
disputed frames that were correct - so agreement between different model families is worth more than one
confident opinion.

judge_screenshot.py now runs two models by default (openai/gpt-4o-mini + qwen/qwen3.7-flash, both served
on OpenRouter; claude-3.5-haiku is gone, which the first attempt discovered with a 404). combine()
reconciles them: agreement carries the verdict with the mean confidence, disagreement reports 'uncertain'
and keeps both reasons rather than letting the more confident model win. --model still forces a single
judgement, and every line prints which model said what.

comfyui_generate.py generates the same prompt and seed with two checkpoints, scores both by
high-frequency detail (mean absolute Laplacian, which catches blurry or flat results), keeps the loser as
<out>__<model>.png with its sidecar, and moves the winner onto --out. --alt-model  restores the single
run.

Tests cover the reconciliation - agreement, disagreement, unknown verdicts, and no verdicts at all - and the
suite is at 103.
This commit is contained in:
zwitschi committed 2026-11-10 09:12:26 +01:00
1 parent bf7dfe1052
commit abde390045
4 files changed
+158 -10

No files matched your search

+55 -8
View File
@@ -35,6 +35,39 @@ import requests
API_URL = "https://openrouter.ai/api/v1/chat/completions"
DEFAULT_MODEL = "openai/gpt-4o-mini"
# Two models by default. A single vision model is advisory and has been wrong in both directions on this
# project, so every image is judged twice by different families and the pair is reconciled below.
DEFAULT_MODELS = (
"openai/gpt-4o-mini",
"qwen/qwen3.7-flash",
)
def combine(judgements):
"""Reconcile two or more model judgements into one verdict.
Agreement carries the verdict and the mean confidence. Disagreement is reported as `uncertain` with
both reasons kept, because picking the more confident model would hide exactly the case worth looking
at - the models seeing different things.
"""
usable = [j for j in judgements if j.get("verdict") in ("yes", "no", "uncertain")]
if not usable:
return {"verdict": "unknown", "confidence": 0.0,
"reason": "no usable verdicts", "models": {}}
verdicts = {j["verdict"] for j in usable}
by_model = {j.get("model", "?"): {"verdict": j.get("verdict"),
"confidence": j.get("confidence", 0.0),
"reason": j.get("reason", "")} for j in usable}
if len(verdicts) == 1:
verdict = verdicts.pop()
confidence = sum(j.get("confidence", 0.0) for j in usable) / len(usable)
reason = usable[0].get("reason", "")
else:
verdict = "uncertain"
confidence = min(j.get("confidence", 0.0) for j in usable)
reason = "models disagree: " + " | ".join(
f"{j.get('model')}={j.get('verdict')}: {j.get('reason', '')[:120]}" for j in usable)
return {"verdict": verdict, "confidence": confidence, "reason": reason, "models": by_model}
# Project root = parent of the Tools/ directory the script lives in.
PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
@@ -87,6 +120,7 @@ def build_messages(image_path, expect):
def judge(api_key, image_path, expect, model, timeout=60):
"""One model's judgement. `combine()` reconciles several of these."""
payload = {
"model": model,
"messages": build_messages(image_path, expect),
@@ -109,7 +143,9 @@ def judge(api_key, image_path, expect, model, timeout=60):
m = re.search(r"\{.*\}", content, re.S)
if not m:
sys.exit(f"error: model returned non-JSON:\n{content[:500]}")
return json.loads(m.group(0))
verdict = json.loads(m.group(0))
verdict["model"] = model
return verdict
def parse_shots_from_targets():
@@ -144,19 +180,24 @@ def main():
g.add_argument("--shots", action="store_true",
help="evaluate every review shot listed in Screenshots/README.md")
ap.add_argument("--expect", help="what the shot is expected to show (required with --image)")
ap.add_argument("--model", default=DEFAULT_MODEL, help=f"OpenRouter model (default {DEFAULT_MODEL})")
ap.add_argument("--models", default=",".join(DEFAULT_MODELS),
help=f"comma-separated OpenRouter models; every image is judged by all of them "
f"and the results reconciled (default {','.join(DEFAULT_MODELS)})")
ap.add_argument("--model", default=None,
help="single model, overriding --models (one judgement, no reconciliation)")
ap.add_argument("--verbose", action="store_true")
ap.add_argument("--timeout", type=int, default=60)
args = ap.parse_args()
api_key = load_api_key()
models = [args.model] if args.model else [m.strip() for m in args.models.split(",") if m.strip()]
if args.shots:
shots = parse_shots_from_targets()
if not shots:
sys.exit("error: no shots found in Tools/visual_targets.json")
n = len(shots)
print(f"Judging {n} screenshots with {args.model} ...")
print(f"Judging {n} screenshots with {' + '.join(models)} ...")
ok = 0
for i, (fname, expect) in enumerate(shots, 1):
path = os.path.join(PROJECT_ROOT, fname)
@@ -164,10 +205,11 @@ def main():
print(f"[{i}/{n}] {fname}: MISSING (skip)")
continue
try:
res = judge(api_key, path, expect, args.model, args.timeout)
judged = [judge(api_key, path, expect, model, args.timeout) for model in models]
except SystemExit as e:
print(f"[{i}/{n}] {fname}: ERROR {e}")
continue
res = combine(judged) if len(judged) > 1 else judged[0]
verdict = res.get("verdict", "unknown")
conf = res.get("confidence", 0.0)
reason = res.get("reason", "")
@@ -181,10 +223,15 @@ def main():
if not args.expect:
ap.error("--expect is required with --image")
res = judge(api_key, args.image, args.expect, args.model, args.timeout)
print(json.dumps(res, indent=2) if args.verbose
else f"verdict={res.get('verdict')} confidence={res.get('confidence')} "
f"reason={res.get('reason', '')}")
judged = [judge(api_key, args.image, args.expect, model, args.timeout) for model in models]
res = combine(judged) if len(judged) > 1 else judged[0]
if args.verbose:
print(json.dumps(res, indent=2))
else:
print(f"verdict={res.get('verdict')} confidence={res.get('confidence')} "
f"reason={res.get('reason', '')}")
for model, detail in res.get("models", {}).items():
print(f" {model}: {detail['verdict']} ({detail['confidence']:.2f})")
if __name__ == "__main__":