feat(tools): every image judged by two models, every generated texture drawn by two checkpoints
A single vision model has been wrong in both directions on this project - it missed real defects and disputed frames that were correct - so agreement between different model families is worth more than one confident opinion. judge_screenshot.py now runs two models by default (openai/gpt-4o-mini + qwen/qwen3.7-flash, both served on OpenRouter; claude-3.5-haiku is gone, which the first attempt discovered with a 404). combine() reconciles them: agreement carries the verdict with the mean confidence, disagreement reports 'uncertain' and keeps both reasons rather than letting the more confident model win. --model still forces a single judgement, and every line prints which model said what. comfyui_generate.py generates the same prompt and seed with two checkpoints, scores both by high-frequency detail (mean absolute Laplacian, which catches blurry or flat results), keeps the loser as <out>__<model>.png with its sidecar, and moves the winner onto --out. --alt-model restores the single run. Tests cover the reconciliation - agreement, disagreement, unknown verdicts, and no verdicts at all - and the suite is at 103.
This commit is contained in:
1 parent
bf7dfe1052
commit
abde390045
4 files changed
+158
-10
No files matched your search
@@ -35,6 +35,39 @@ import requests
|
||||
|
||||
API_URL = "https://openrouter.ai/api/v1/chat/completions"
|
||||
DEFAULT_MODEL = "openai/gpt-4o-mini"
|
||||
# Two models by default. A single vision model is advisory and has been wrong in both directions on this
|
||||
# project, so every image is judged twice by different families and the pair is reconciled below.
|
||||
DEFAULT_MODELS = (
|
||||
"openai/gpt-4o-mini",
|
||||
"qwen/qwen3.7-flash",
|
||||
)
|
||||
|
||||
|
||||
def combine(judgements):
|
||||
"""Reconcile two or more model judgements into one verdict.
|
||||
|
||||
Agreement carries the verdict and the mean confidence. Disagreement is reported as `uncertain` with
|
||||
both reasons kept, because picking the more confident model would hide exactly the case worth looking
|
||||
at - the models seeing different things.
|
||||
"""
|
||||
usable = [j for j in judgements if j.get("verdict") in ("yes", "no", "uncertain")]
|
||||
if not usable:
|
||||
return {"verdict": "unknown", "confidence": 0.0,
|
||||
"reason": "no usable verdicts", "models": {}}
|
||||
verdicts = {j["verdict"] for j in usable}
|
||||
by_model = {j.get("model", "?"): {"verdict": j.get("verdict"),
|
||||
"confidence": j.get("confidence", 0.0),
|
||||
"reason": j.get("reason", "")} for j in usable}
|
||||
if len(verdicts) == 1:
|
||||
verdict = verdicts.pop()
|
||||
confidence = sum(j.get("confidence", 0.0) for j in usable) / len(usable)
|
||||
reason = usable[0].get("reason", "")
|
||||
else:
|
||||
verdict = "uncertain"
|
||||
confidence = min(j.get("confidence", 0.0) for j in usable)
|
||||
reason = "models disagree: " + " | ".join(
|
||||
f"{j.get('model')}={j.get('verdict')}: {j.get('reason', '')[:120]}" for j in usable)
|
||||
return {"verdict": verdict, "confidence": confidence, "reason": reason, "models": by_model}
|
||||
|
||||
# Project root = parent of the Tools/ directory the script lives in.
|
||||
PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
|
||||
@@ -87,6 +120,7 @@ def build_messages(image_path, expect):
|
||||
|
||||
|
||||
def judge(api_key, image_path, expect, model, timeout=60):
|
||||
"""One model's judgement. `combine()` reconciles several of these."""
|
||||
payload = {
|
||||
"model": model,
|
||||
"messages": build_messages(image_path, expect),
|
||||
@@ -109,7 +143,9 @@ def judge(api_key, image_path, expect, model, timeout=60):
|
||||
m = re.search(r"\{.*\}", content, re.S)
|
||||
if not m:
|
||||
sys.exit(f"error: model returned non-JSON:\n{content[:500]}")
|
||||
return json.loads(m.group(0))
|
||||
verdict = json.loads(m.group(0))
|
||||
verdict["model"] = model
|
||||
return verdict
|
||||
|
||||
|
||||
def parse_shots_from_targets():
|
||||
@@ -144,19 +180,24 @@ def main():
|
||||
g.add_argument("--shots", action="store_true",
|
||||
help="evaluate every review shot listed in Screenshots/README.md")
|
||||
ap.add_argument("--expect", help="what the shot is expected to show (required with --image)")
|
||||
ap.add_argument("--model", default=DEFAULT_MODEL, help=f"OpenRouter model (default {DEFAULT_MODEL})")
|
||||
ap.add_argument("--models", default=",".join(DEFAULT_MODELS),
|
||||
help=f"comma-separated OpenRouter models; every image is judged by all of them "
|
||||
f"and the results reconciled (default {','.join(DEFAULT_MODELS)})")
|
||||
ap.add_argument("--model", default=None,
|
||||
help="single model, overriding --models (one judgement, no reconciliation)")
|
||||
ap.add_argument("--verbose", action="store_true")
|
||||
ap.add_argument("--timeout", type=int, default=60)
|
||||
args = ap.parse_args()
|
||||
|
||||
api_key = load_api_key()
|
||||
models = [args.model] if args.model else [m.strip() for m in args.models.split(",") if m.strip()]
|
||||
|
||||
if args.shots:
|
||||
shots = parse_shots_from_targets()
|
||||
if not shots:
|
||||
sys.exit("error: no shots found in Tools/visual_targets.json")
|
||||
n = len(shots)
|
||||
print(f"Judging {n} screenshots with {args.model} ...")
|
||||
print(f"Judging {n} screenshots with {' + '.join(models)} ...")
|
||||
ok = 0
|
||||
for i, (fname, expect) in enumerate(shots, 1):
|
||||
path = os.path.join(PROJECT_ROOT, fname)
|
||||
@@ -164,10 +205,11 @@ def main():
|
||||
print(f"[{i}/{n}] {fname}: MISSING (skip)")
|
||||
continue
|
||||
try:
|
||||
res = judge(api_key, path, expect, args.model, args.timeout)
|
||||
judged = [judge(api_key, path, expect, model, args.timeout) for model in models]
|
||||
except SystemExit as e:
|
||||
print(f"[{i}/{n}] {fname}: ERROR {e}")
|
||||
continue
|
||||
res = combine(judged) if len(judged) > 1 else judged[0]
|
||||
verdict = res.get("verdict", "unknown")
|
||||
conf = res.get("confidence", 0.0)
|
||||
reason = res.get("reason", "")
|
||||
@@ -181,10 +223,15 @@ def main():
|
||||
if not args.expect:
|
||||
ap.error("--expect is required with --image")
|
||||
|
||||
res = judge(api_key, args.image, args.expect, args.model, args.timeout)
|
||||
print(json.dumps(res, indent=2) if args.verbose
|
||||
else f"verdict={res.get('verdict')} confidence={res.get('confidence')} "
|
||||
f"reason={res.get('reason', '')}")
|
||||
judged = [judge(api_key, args.image, args.expect, model, args.timeout) for model in models]
|
||||
res = combine(judged) if len(judged) > 1 else judged[0]
|
||||
if args.verbose:
|
||||
print(json.dumps(res, indent=2))
|
||||
else:
|
||||
print(f"verdict={res.get('verdict')} confidence={res.get('confidence')} "
|
||||
f"reason={res.get('reason', '')}")
|
||||
for model, detail in res.get("models", {}).items():
|
||||
print(f" {model}: {detail['verdict']} ({detail['confidence']:.2f})")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in new issue
Block a user