Make the vision judge discover cost-effective OpenRouter models
Full build / Import + headless tests (push) Successful in 40s
Full build / Full desktop build + optional release (push) Successful in 4m6s

Add openrouter_judge.py to rank live image+JSON models by advertised price,
with ordered fallbacks, auth-error handling and auto selection. Wire both judges
to --model auto, record the model used, and add fake-catalogue/mocked-call
tests. Re-run the judges with the valid key: 13/13 screen designs match, 0 diff;
the Forward+ A/B report records the selected model and ~$0.006 spend. Headless
231/231, Python 94/94.
This commit is contained in:
zwitschi committed 2026-11-01 19:55:43 +01:00
1 parent 28f2a48811
commit 659b0e35b2
15 files changed
+535 -65

No files matched your search

+29 -9
View File
@@ -51,6 +51,7 @@ from pathlib import Path
from PIL import Image, ImageChops, ImageDraw, ImageFilter, ImageOps, ImageStat
import generate_textures
import openrouter_judge
ROOT = Path(__file__).resolve().parent.parent
## Runtime reference art (ADR-0014); docs/screens/designs/ is the reference-only
@@ -68,7 +69,7 @@ ASPECT = 16 / 9
DEFAULT_MODEL = "openai/gpt-5-image-mini"
# Vision model used by `judge`. Cheap, multimodal, and good at "same screen?".
DEFAULT_JUDGE_MODEL = "google/gemini-2.5-flash-lite"
DEFAULT_JUDGE_MODEL = openrouter_judge.DEFAULT_MODEL
OPENROUTER_CHAT_URL = "https://openrouter.ai/api/v1/chat/completions"
# Shared art direction, appended to every prompt so the generated frames read
@@ -965,6 +966,10 @@ def judge(args):
refs = {screen["name"]: screen["ref"]
for screen in SCREENS if screen["ref"]}
by_name = {screen["name"]: screen for screen in SCREENS}
models = [args.model]
if not args.dry_run:
models = openrouter_judge.resolve_models(args.model)
print(f"judge models: {', '.join(models)}", file=sys.stderr)
results = []
for capture in sorted(SCREENSHOTS_DIR.glob("*.png")):
if args.only and args.only not in capture.name:
@@ -982,23 +987,36 @@ def judge(args):
"match": None, "confidence": 0.0,
"differences": ["missing reference design; run generate"]})
continue
body = build_judge_request(args.model, by_name[design_name],
capture, reference)
if args.dry_run:
body = build_judge_request(args.model, by_name[design_name],
capture, reference)
results.append({
"capture": capture.name, "screen": design_name, "match": None,
"confidence": 0.0, "model": body["model"], "class": "unknown",
"differences": [f"dry run: {len(body['messages'][0]['content'])} parts"],
})
continue
# Providers occasionally answer a valid key with a transient HTTP 400;
# one retry keeps a whole judge run from dying on a single hiccup.
try:
verdict = parse_judge_response(_call_openrouter(body))
model, payload = openrouter_judge.call_with_fallback(
models,
lambda selected: build_judge_request(
selected, by_name[design_name], capture, reference),
_call_openrouter)
except ValueError as exc:
print(f"retrying {capture.name}: {exc}", file=sys.stderr)
verdict = parse_judge_response(_call_openrouter(body))
if openrouter_judge.is_auth_error(exc):
print(f"error: OpenRouter authentication failed: {exc}",
file=sys.stderr)
return 2
print(f"error judging {capture.name}: {exc}", file=sys.stderr)
results.append({
"capture": capture.name, "screen": design_name, "match": None,
"confidence": 0.0, "model": "",
"class": "unknown", "differences": [f"judge failed: {exc}"],
})
continue
verdict = parse_judge_response(payload)
verdict.update({"capture": capture.name, "screen": design_name,
"model": model,
"class": classify(verdict, args.fail_confidence)})
results.append(verdict)
@@ -1047,7 +1065,9 @@ def build_parser():
judge_cmd = commands.add_parser(
"judge", help="ask a vision model whether captures match the designs")
judge_cmd.add_argument("--model", default=DEFAULT_JUDGE_MODEL)
judge_cmd.add_argument("--model", default=DEFAULT_JUDGE_MODEL,
help="OpenRouter model id, or 'auto' to pick the cheapest "
"available image+JSON model with fallbacks")
judge_cmd.add_argument("--json", action="store_true")
judge_cmd.add_argument("--dry-run", action="store_true",
help="print requests instead of calling the API")