Make the vision judge discover cost-effective OpenRouter models
Add openrouter_judge.py to rank live image+JSON models by advertised price, with ordered fallbacks, auth-error handling and auto selection. Wire both judges to --model auto, record the model used, and add fake-catalogue/mocked-call tests. Re-run the judges with the valid key: 13/13 screen designs match, 0 diff; the Forward+ A/B report records the selected model and ~$0.006 spend. Headless 231/231, Python 94/94.
This commit is contained in:
1 parent
28f2a48811
commit
659b0e35b2
15 files changed
+535
-65
No files matched your search
+29
-9
@@ -51,6 +51,7 @@ from pathlib import Path
|
||||
from PIL import Image, ImageChops, ImageDraw, ImageFilter, ImageOps, ImageStat
|
||||
|
||||
import generate_textures
|
||||
import openrouter_judge
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
## Runtime reference art (ADR-0014); docs/screens/designs/ is the reference-only
|
||||
@@ -68,7 +69,7 @@ ASPECT = 16 / 9
|
||||
DEFAULT_MODEL = "openai/gpt-5-image-mini"
|
||||
|
||||
# Vision model used by `judge`. Cheap, multimodal, and good at "same screen?".
|
||||
DEFAULT_JUDGE_MODEL = "google/gemini-2.5-flash-lite"
|
||||
DEFAULT_JUDGE_MODEL = openrouter_judge.DEFAULT_MODEL
|
||||
OPENROUTER_CHAT_URL = "https://openrouter.ai/api/v1/chat/completions"
|
||||
|
||||
# Shared art direction, appended to every prompt so the generated frames read
|
||||
@@ -965,6 +966,10 @@ def judge(args):
|
||||
refs = {screen["name"]: screen["ref"]
|
||||
for screen in SCREENS if screen["ref"]}
|
||||
by_name = {screen["name"]: screen for screen in SCREENS}
|
||||
models = [args.model]
|
||||
if not args.dry_run:
|
||||
models = openrouter_judge.resolve_models(args.model)
|
||||
print(f"judge models: {', '.join(models)}", file=sys.stderr)
|
||||
results = []
|
||||
for capture in sorted(SCREENSHOTS_DIR.glob("*.png")):
|
||||
if args.only and args.only not in capture.name:
|
||||
@@ -982,23 +987,36 @@ def judge(args):
|
||||
"match": None, "confidence": 0.0,
|
||||
"differences": ["missing reference design; run generate"]})
|
||||
continue
|
||||
body = build_judge_request(args.model, by_name[design_name],
|
||||
capture, reference)
|
||||
if args.dry_run:
|
||||
body = build_judge_request(args.model, by_name[design_name],
|
||||
capture, reference)
|
||||
results.append({
|
||||
"capture": capture.name, "screen": design_name, "match": None,
|
||||
"confidence": 0.0, "model": body["model"], "class": "unknown",
|
||||
"differences": [f"dry run: {len(body['messages'][0]['content'])} parts"],
|
||||
})
|
||||
continue
|
||||
# Providers occasionally answer a valid key with a transient HTTP 400;
|
||||
# one retry keeps a whole judge run from dying on a single hiccup.
|
||||
try:
|
||||
verdict = parse_judge_response(_call_openrouter(body))
|
||||
model, payload = openrouter_judge.call_with_fallback(
|
||||
models,
|
||||
lambda selected: build_judge_request(
|
||||
selected, by_name[design_name], capture, reference),
|
||||
_call_openrouter)
|
||||
except ValueError as exc:
|
||||
print(f"retrying {capture.name}: {exc}", file=sys.stderr)
|
||||
verdict = parse_judge_response(_call_openrouter(body))
|
||||
if openrouter_judge.is_auth_error(exc):
|
||||
print(f"error: OpenRouter authentication failed: {exc}",
|
||||
file=sys.stderr)
|
||||
return 2
|
||||
print(f"error judging {capture.name}: {exc}", file=sys.stderr)
|
||||
results.append({
|
||||
"capture": capture.name, "screen": design_name, "match": None,
|
||||
"confidence": 0.0, "model": "",
|
||||
"class": "unknown", "differences": [f"judge failed: {exc}"],
|
||||
})
|
||||
continue
|
||||
verdict = parse_judge_response(payload)
|
||||
verdict.update({"capture": capture.name, "screen": design_name,
|
||||
"model": model,
|
||||
"class": classify(verdict, args.fail_confidence)})
|
||||
results.append(verdict)
|
||||
|
||||
@@ -1047,7 +1065,9 @@ def build_parser():
|
||||
|
||||
judge_cmd = commands.add_parser(
|
||||
"judge", help="ask a vision model whether captures match the designs")
|
||||
judge_cmd.add_argument("--model", default=DEFAULT_JUDGE_MODEL)
|
||||
judge_cmd.add_argument("--model", default=DEFAULT_JUDGE_MODEL,
|
||||
help="OpenRouter model id, or 'auto' to pick the cheapest "
|
||||
"available image+JSON model with fallbacks")
|
||||
judge_cmd.add_argument("--json", action="store_true")
|
||||
judge_cmd.add_argument("--dry-run", action="store_true",
|
||||
help="print requests instead of calling the API")
|
||||
|
||||
Reference in new issue
Block a user