Make the vision judge discover cost-effective OpenRouter models
Full build / Import + headless tests (push) Successful in 40s
Full build / Full desktop build + optional release (push) Successful in 4m6s

Add openrouter_judge.py to rank live image+JSON models by advertised price,
with ordered fallbacks, auth-error handling and auto selection. Wire both judges
to --model auto, record the model used, and add fake-catalogue/mocked-call
tests. Re-run the judges with the valid key: 13/13 screen designs match, 0 diff;
the Forward+ A/B report records the selected model and ~$0.006 spend. Headless
231/231, Python 94/94.
This commit is contained in:
zwitschi committed 2026-11-01 19:55:43 +01:00
1 parent 28f2a48811
commit 659b0e35b2
15 files changed
+535 -65

No files matched your search

+37 -16
View File
@@ -38,6 +38,7 @@ from PIL import Image, ImageFilter, ImageOps, ImageStat
import asset_pipeline
import generate_textures
import openrouter_judge
ROOT = Path(__file__).resolve().parent.parent
RAW_DIR = ROOT / "assets" / "textures" / "ai" / "v2" / "raw"
@@ -46,7 +47,7 @@ CONCEPT_DIR = ROOT / "docs" / "screens" / "concepts"
REALISM_DIR = ROOT / "docs" / "screens" / "realism"
DEFAULT_MODEL = "openai/gpt-5-image-mini"
DEFAULT_JUDGE_MODEL = "google/gemini-2.5-flash-lite"
DEFAULT_JUDGE_MODEL = openrouter_judge.DEFAULT_MODEL
OPENROUTER_CHAT_URL = "https://openrouter.ai/api/v1/chat/completions"
SCENE_W, SCENE_H = 1280, 720
@@ -758,25 +759,32 @@ def _call_openrouter(body: dict, timeout: int = 180) -> dict:
raise ValueError(f"OpenRouter HTTP {exc.code}: {exc.read().decode('utf-8', 'replace')[:300]}")
def judge_capture(model: str, target: Path, capture: Path, attempts: int = 2,
def judge_capture(models, target: Path, capture: Path, attempts: int = 2,
repeats: int = 1) -> dict:
"""Vision-judge one capture; retry transient errors, average repeat scores.
`models` is a fallback-ordered list from `openrouter_judge.resolve_models`.
A single score from a small vision model swings by ±1-2, so `repeats` runs
are averaged and the spread is kept (`score_range`) for the report.
"""
if isinstance(models, str):
models = [models]
runs = []
error = None
used = set()
for _ in range(max(1, repeats)):
for _attempt in range(attempts):
try:
runs.append(parse_realism_response(_call_openrouter(
build_realism_request(model, target, capture))))
break
except (ValueError, OSError) as exc:
error = exc
try:
model, payload = openrouter_judge.call_with_fallback(
models, lambda selected: build_realism_request(selected, target, capture),
_call_openrouter, attempts=attempts)
used.add(model)
runs.append(parse_realism_response(payload))
except (ValueError, OSError) as exc:
if openrouter_judge.is_auth_error(exc):
raise
error = exc
if not runs:
return {"scores": {}, "fixes": [f"judge failed: {error}"]}
return {"scores": {}, "fixes": [f"judge failed: {error}"], "model": ""}
keys = {key for run in runs for key in run["scores"]}
scores = {}
spread = {}
@@ -790,7 +798,7 @@ def judge_capture(model: str, target: Path, capture: Path, attempts: int = 2,
if fix not in fixes:
fixes.append(fix)
return {"scores": scores, "score_range": spread, "runs": len(runs),
"fixes": fixes[:5]}
"model": ", ".join(sorted(used)), "fixes": fixes[:5]}
def cmd_realism(args) -> int:
@@ -800,10 +808,14 @@ def cmd_realism(args) -> int:
if not sets:
print(f"no capture sets under {_rel(run)}", file=sys.stderr)
return 2
judge_models = []
if args.judge:
_load_env_file()
report = {"run": _rel(run), "judge_model": args.model if args.judge else None,
"sets": {}}
judge_models = openrouter_judge.resolve_models(args.model)
print(f"judge models: {', '.join(judge_models)}", file=sys.stderr)
report = {"run": _rel(run),
"judge_model": (judge_models[0] if judge_models else None),
"judge_fallbacks": judge_models, "sets": {}}
for capture_set in sets:
rows = {}
for capture in sorted(capture_set.glob("*.png")):
@@ -815,8 +827,15 @@ def cmd_realism(args) -> int:
row = {"target": _rel(target), "metrics": cap_m, "target_metrics": tgt_m,
"metric_gap": metric_gap(cap_m, tgt_m)}
if args.judge:
row["judge"] = judge_capture(args.model, target, capture,
repeats=args.repeats)
try:
row["judge"] = judge_capture(judge_models, target, capture,
repeats=args.repeats)
except ValueError as exc:
if openrouter_judge.is_auth_error(exc):
print(f"error: OpenRouter authentication failed: {exc}",
file=sys.stderr)
return 2
raise
rows[capture.stem] = row
report["sets"][capture_set.name] = rows
json_path = next_free_path(run / "report.json")
@@ -975,7 +994,9 @@ def build_parser():
real = sub.add_parser("realism")
real.add_argument("--run", required=True)
real.add_argument("--judge", action="store_true")
real.add_argument("--model", default=DEFAULT_JUDGE_MODEL)
real.add_argument("--model", default=DEFAULT_JUDGE_MODEL,
help="OpenRouter model id, or 'auto' to pick the cheapest "
"available image+JSON model with fallbacks")
real.add_argument("--repeats", type=int, default=1,
help="average N judge runs per shot (scores are noisy)")
real.set_defaults(handler=cmd_realism)