Make the vision judge discover cost-effective OpenRouter models

Add openrouter_judge.py to rank live image+JSON models by advertised price,
with ordered fallbacks, auth-error handling and auto selection. Wire both judges
to --model auto, record the model used, and add fake-catalogue/mocked-call
tests. Re-run the judges with the valid key: 13/13 screen designs match, 0 diff;
the Forward+ A/B report records the selected model and ~$0.006 spend. Headless
231/231, Python 94/94.

tools/openrouter_judge.py

@@ -0,0 +1,127 @@
+"""Pick affordable OpenRouter vision models for the screenshot judges.
+
+The model catalogue is fetched from OpenRouter (no auth needed) and filtered to
+models that accept image input and advertise `response_format`/structured output.
+The ranking uses the advertised per-token prices, so it follows the account's
+current catalogue instead of relying on one hard-coded model name. If the
+catalogue cannot be fetched, a small known-good list keeps QA working offline.
+"""
+
+from __future__ import annotations
+
+import os
+import urllib.error
+import urllib.request
+
+MODELS_URL = "https://openrouter.ai/api/v1/models"
+DEFAULT_MODEL = "auto"
+
+# Used only when the catalogue is unavailable. Ordered by price/known quality.
+FALLBACK_MODELS = [
+    "z-ai/glm-5.3-flash",
+    "qwen/qwen3.7-flash",
+    "google/gemma-3-4b-it",
+    "openai/gpt-4.1-nano",
+    "google/gemini-2.5-flash-lite",
+]
+
+
+def fetch_models(timeout: int = 30) -> list[dict]:
+    """Return the OpenRouter model catalogue, or [] when it is unreachable."""
+    req = urllib.request.Request(MODELS_URL, headers={"Accept": "application/json"})
+    try:
+        with urllib.request.urlopen(req, timeout=timeout) as resp:
+            payload = __import__("json").loads(resp.read().decode("utf-8"))
+    except (OSError, ValueError, urllib.error.HTTPError):
+        return []
+    return list(payload.get("data") or [])
+
+
+def _price(model: dict) -> float:
+    pricing = model.get("pricing") or {}
+    try:
+        prompt = float(pricing.get("prompt") or 0.0)
+        completion = float(pricing.get("completion") or 0.0)
+        image = float(pricing.get("image") or 0.0)
+    except (TypeError, ValueError):
+        return float("inf")
+    # A judge request is image-heavy and emits a short JSON verdict.
+    return prompt + image + completion * 0.25
+
+
+def vision_candidates(catalog: list[dict], limit: int = 6) -> list[str]:
+    """Cheap, image-capable models that can return structured JSON."""
+    ranked: list[tuple[float, str]] = []
+    for model in catalog:
+        model_id = str(model.get("id") or "")
+        if not model_id or model_id.startswith("~") or ":batch" in model_id:
+            continue
+        architecture = model.get("architecture") or {}
+        inputs = architecture.get("input_modalities") or []
+        outputs = architecture.get("output_modalities") or []
+        if "image" not in inputs or "text" not in outputs:
+            continue
+        params = model.get("supported_parameters") or []
+        # Our request uses response_format={"type": "json_object"}; models that
+        # only advertise structured_outputs may reject that envelope.
+        if "response_format" not in params:
+            continue
+        price = _price(model)
+        if price <= 0.0 or price == float("inf"):
+            continue
+        ranked.append((price, model_id))
+    ranked.sort()
+    out: list[str] = []
+    for _price_value, model_id in ranked:
+        if model_id not in out:
+            out.append(model_id)
+        if len(out) >= max(1, limit):
+            break
+    return out
+
+
+def resolve_models(requested: str = DEFAULT_MODEL, *, catalog: list[dict] | None = None,
+                   limit: int = 6) -> list[str]:
+    """Resolve ``auto``/an explicit model to a fallback-ordered model list.
+
+    `GIG_LINE_JUDGE_MODEL` overrides the automatic choice for reproducible runs;
+    an explicit CLI `--model` still wins.
+    """
+    requested = str(requested or "").strip()
+    if requested and requested != DEFAULT_MODEL:
+        return [requested]
+    env_model = os.environ.get("GIG_LINE_JUDGE_MODEL", "").strip()
+    if env_model:
+        return [env_model]
+    models = vision_candidates(catalog if catalog is not None else fetch_models(), limit)
+    if not models:
+        models = FALLBACK_MODELS[:max(1, limit)]
+    return models
+
+
+def is_auth_error(exc: BaseException) -> bool:
+    text = str(exc)
+    return "HTTP 401" in text or "HTTP 403" in text or "API key not valid" in text
+
+
+def call_with_fallback(models: list[str], make_body, call, attempts: int = 2):
+    """Try each model, retrying provider errors, and return (model, response).
+
+    Authentication errors abort immediately: switching models cannot repair a
+    bad key. Model/provider errors move to the next candidate.
+    """
+    last: BaseException | None = None
+    for model in models:
+        body = make_body(model)
+        for attempt in range(max(1, attempts)):
+            try:
+                return model, call(body)
+            except (ValueError, OSError) as exc:
+                last = exc
+                if is_auth_error(exc):
+                    raise
+                if attempt + 1 >= max(1, attempts):
+                    break
+    if last is not None:
+        raise last
+    raise ValueError("no judge model was available")

tools/scene_concepts.py

@@ -38,6 +38,7 @@ from PIL import Image, ImageFilter, ImageOps, ImageStat
 
 import asset_pipeline
 import generate_textures
+import openrouter_judge
 
 ROOT = Path(__file__).resolve().parent.parent
 RAW_DIR = ROOT / "assets" / "textures" / "ai" / "v2" / "raw"
@@ -46,7 +47,7 @@ CONCEPT_DIR = ROOT / "docs" / "screens" / "concepts"
 REALISM_DIR = ROOT / "docs" / "screens" / "realism"
 
 DEFAULT_MODEL = "openai/gpt-5-image-mini"
-DEFAULT_JUDGE_MODEL = "google/gemini-2.5-flash-lite"
+DEFAULT_JUDGE_MODEL = openrouter_judge.DEFAULT_MODEL
 OPENROUTER_CHAT_URL = "https://openrouter.ai/api/v1/chat/completions"
 
 SCENE_W, SCENE_H = 1280, 720
@@ -758,25 +759,32 @@ def _call_openrouter(body: dict, timeout: int = 180) -> dict:
         raise ValueError(f"OpenRouter HTTP {exc.code}: {exc.read().decode('utf-8', 'replace')[:300]}")
 
 
-def judge_capture(model: str, target: Path, capture: Path, attempts: int = 2,
+def judge_capture(models, target: Path, capture: Path, attempts: int = 2,
                   repeats: int = 1) -> dict:
     """Vision-judge one capture; retry transient errors, average repeat scores.
 
+    `models` is a fallback-ordered list from `openrouter_judge.resolve_models`.
     A single score from a small vision model swings by ±1-2, so `repeats` runs
     are averaged and the spread is kept (`score_range`) for the report.
     """
+    if isinstance(models, str):
+        models = [models]
     runs = []
     error = None
+    used = set()
     for _ in range(max(1, repeats)):
-        for _attempt in range(attempts):
-            try:
-                runs.append(parse_realism_response(_call_openrouter(
-                    build_realism_request(model, target, capture))))
-                break
-            except (ValueError, OSError) as exc:
-                error = exc
+        try:
+            model, payload = openrouter_judge.call_with_fallback(
+                models, lambda selected: build_realism_request(selected, target, capture),
+                _call_openrouter, attempts=attempts)
+            used.add(model)
+            runs.append(parse_realism_response(payload))
+        except (ValueError, OSError) as exc:
+            if openrouter_judge.is_auth_error(exc):
+                raise
+            error = exc
     if not runs:
-        return {"scores": {}, "fixes": [f"judge failed: {error}"]}
+        return {"scores": {}, "fixes": [f"judge failed: {error}"], "model": ""}
     keys = {key for run in runs for key in run["scores"]}
     scores = {}
     spread = {}
@@ -790,7 +798,7 @@ def judge_capture(model: str, target: Path, capture: Path, attempts: int = 2,
             if fix not in fixes:
                 fixes.append(fix)
     return {"scores": scores, "score_range": spread, "runs": len(runs),
-            "fixes": fixes[:5]}
+            "model": ", ".join(sorted(used)), "fixes": fixes[:5]}
 
 
 def cmd_realism(args) -> int:
@@ -800,10 +808,14 @@ def cmd_realism(args) -> int:
     if not sets:
         print(f"no capture sets under {_rel(run)}", file=sys.stderr)
         return 2
+    judge_models = []
     if args.judge:
         _load_env_file()
-    report = {"run": _rel(run), "judge_model": args.model if args.judge else None,
-              "sets": {}}
+        judge_models = openrouter_judge.resolve_models(args.model)
+        print(f"judge models: {', '.join(judge_models)}", file=sys.stderr)
+    report = {"run": _rel(run),
+              "judge_model": (judge_models[0] if judge_models else None),
+              "judge_fallbacks": judge_models, "sets": {}}
     for capture_set in sets:
         rows = {}
         for capture in sorted(capture_set.glob("*.png")):
@@ -815,8 +827,15 @@ def cmd_realism(args) -> int:
             row = {"target": _rel(target), "metrics": cap_m, "target_metrics": tgt_m,
                    "metric_gap": metric_gap(cap_m, tgt_m)}
             if args.judge:
-                row["judge"] = judge_capture(args.model, target, capture,
-                                             repeats=args.repeats)
+                try:
+                    row["judge"] = judge_capture(judge_models, target, capture,
+                                                 repeats=args.repeats)
+                except ValueError as exc:
+                    if openrouter_judge.is_auth_error(exc):
+                        print(f"error: OpenRouter authentication failed: {exc}",
+                              file=sys.stderr)
+                        return 2
+                    raise
             rows[capture.stem] = row
         report["sets"][capture_set.name] = rows
     json_path = next_free_path(run / "report.json")
@@ -975,7 +994,9 @@ def build_parser():
     real = sub.add_parser("realism")
     real.add_argument("--run", required=True)
     real.add_argument("--judge", action="store_true")
-    real.add_argument("--model", default=DEFAULT_JUDGE_MODEL)
+    real.add_argument("--model", default=DEFAULT_JUDGE_MODEL,
+                      help="OpenRouter model id, or 'auto' to pick the cheapest "
+                           "available image+JSON model with fallbacks")
     real.add_argument("--repeats", type=int, default=1,
                       help="average N judge runs per shot (scores are noisy)")
     real.set_defaults(handler=cmd_realism)

tools/screen_designs.py

@@ -51,6 +51,7 @@ from pathlib import Path
 from PIL import Image, ImageChops, ImageDraw, ImageFilter, ImageOps, ImageStat
 
 import generate_textures
+import openrouter_judge
 
 ROOT = Path(__file__).resolve().parent.parent
 ## Runtime reference art (ADR-0014); docs/screens/designs/ is the reference-only
@@ -68,7 +69,7 @@ ASPECT = 16 / 9
 DEFAULT_MODEL = "openai/gpt-5-image-mini"
 
 # Vision model used by `judge`. Cheap, multimodal, and good at "same screen?".
-DEFAULT_JUDGE_MODEL = "google/gemini-2.5-flash-lite"
+DEFAULT_JUDGE_MODEL = openrouter_judge.DEFAULT_MODEL
 OPENROUTER_CHAT_URL = "https://openrouter.ai/api/v1/chat/completions"
 
 # Shared art direction, appended to every prompt so the generated frames read
@@ -965,6 +966,10 @@ def judge(args):
     refs = {screen["name"]: screen["ref"]
             for screen in SCREENS if screen["ref"]}
     by_name = {screen["name"]: screen for screen in SCREENS}
+    models = [args.model]
+    if not args.dry_run:
+        models = openrouter_judge.resolve_models(args.model)
+        print(f"judge models: {', '.join(models)}", file=sys.stderr)
     results = []
     for capture in sorted(SCREENSHOTS_DIR.glob("*.png")):
         if args.only and args.only not in capture.name:
@@ -982,23 +987,36 @@ def judge(args):
                             "match": None, "confidence": 0.0,
                             "differences": ["missing reference design; run generate"]})
             continue
-        body = build_judge_request(args.model, by_name[design_name],
-                                   capture, reference)
         if args.dry_run:
+            body = build_judge_request(args.model, by_name[design_name],
+                                       capture, reference)
             results.append({
                 "capture": capture.name, "screen": design_name, "match": None,
                 "confidence": 0.0, "model": body["model"], "class": "unknown",
                 "differences": [f"dry run: {len(body['messages'][0]['content'])} parts"],
             })
             continue
-        # Providers occasionally answer a valid key with a transient HTTP 400;
-        # one retry keeps a whole judge run from dying on a single hiccup.
         try:
-            verdict = parse_judge_response(_call_openrouter(body))
+            model, payload = openrouter_judge.call_with_fallback(
+                models,
+                lambda selected: build_judge_request(
+                    selected, by_name[design_name], capture, reference),
+                _call_openrouter)
         except ValueError as exc:
-            print(f"retrying {capture.name}: {exc}", file=sys.stderr)
-            verdict = parse_judge_response(_call_openrouter(body))
+            if openrouter_judge.is_auth_error(exc):
+                print(f"error: OpenRouter authentication failed: {exc}",
+                      file=sys.stderr)
+                return 2
+            print(f"error judging {capture.name}: {exc}", file=sys.stderr)
+            results.append({
+                "capture": capture.name, "screen": design_name, "match": None,
+                "confidence": 0.0, "model": "",
+                "class": "unknown", "differences": [f"judge failed: {exc}"],
+            })
+            continue
+        verdict = parse_judge_response(payload)
         verdict.update({"capture": capture.name, "screen": design_name,
+                        "model": model,
                         "class": classify(verdict, args.fail_confidence)})
         results.append(verdict)
 
@@ -1047,7 +1065,9 @@ def build_parser():
 
     judge_cmd = commands.add_parser(
         "judge", help="ask a vision model whether captures match the designs")
-    judge_cmd.add_argument("--model", default=DEFAULT_JUDGE_MODEL)
+    judge_cmd.add_argument("--model", default=DEFAULT_JUDGE_MODEL,
+                           help="OpenRouter model id, or 'auto' to pick the cheapest "
+                                "available image+JSON model with fallbacks")
     judge_cmd.add_argument("--json", action="store_true")
     judge_cmd.add_argument("--dry-run", action="store_true",
                            help="print requests instead of calling the API")

tools/test_openrouter_judge.py

@@ -0,0 +1,89 @@
+import os
+import unittest
+from unittest import mock
+
+import openrouter_judge as oj
+
+
+def _model(model_id, prompt, completion, *, image=True, response_format=True, batch=False):
+    return {
+        "id": model_id,
+        "architecture": {
+            "input_modalities": ["text", "image"] if image else ["text"],
+            "output_modalities": ["text"],
+        },
+        "supported_parameters": ["response_format"] if response_format else ["temperature"],
+        "pricing": {"prompt": str(prompt), "completion": str(completion)},
+    }
+
+
+class ModelSelectionTests(unittest.TestCase):
+    def test_candidates_are_image_json_models_sorted_by_price(self):
+        catalog = [
+            _model("expensive/model", 0.10, 0.40),
+            _model("cheap/model", 0.01, 0.02),
+            _model("text-only", 0.001, 0.001, image=False),
+            _model("no-json", 0.001, 0.001, response_format=False),
+            _model("batch/model:batch", 0.0001, 0.0001),
+        ]
+        self.assertEqual(oj.vision_candidates(catalog), ["cheap/model", "expensive/model"])
+
+    def test_explicit_model_wins_over_auto_and_env(self):
+        with mock.patch.dict(os.environ, {"GIG_LINE_JUDGE_MODEL": "env/model"}):
+            self.assertEqual(oj.resolve_models("explicit/model", catalog=[]),
+                             ["explicit/model"])
+
+    def test_auto_uses_env_override(self):
+        with mock.patch.dict(os.environ, {"GIG_LINE_JUDGE_MODEL": "env/model"}):
+            self.assertEqual(oj.resolve_models("auto", catalog=[]), ["env/model"])
+
+    def test_auto_uses_catalog_and_falls_back_offline(self):
+        catalog = [_model("cheap/model", 0.01, 0.02)]
+        self.assertEqual(oj.resolve_models("auto", catalog=catalog), ["cheap/model"])
+        self.assertEqual(oj.resolve_models("auto", catalog=[]), oj.FALLBACK_MODELS[:6])
+
+
+class FallbackTests(unittest.TestCase):
+    def test_fallback_switches_model_after_provider_error(self):
+        seen = []
+
+        def call(body):
+            seen.append(body["model"])
+            if body["model"] == "bad/model":
+                raise ValueError("OpenRouter HTTP 400: provider rejected image")
+            return {"ok": True}
+
+        model, payload = oj.call_with_fallback(
+            ["bad/model", "good/model"], lambda m: {"model": m}, call, attempts=1)
+        self.assertEqual(model, "good/model")
+        self.assertEqual(payload, {"ok": True})
+        self.assertEqual(seen, ["bad/model", "good/model"])
+
+    def test_auth_error_does_not_try_another_model(self):
+        seen = []
+
+        def call(body):
+            seen.append(body["model"])
+            raise ValueError("OpenRouter HTTP 401: API key not valid")
+
+        with self.assertRaises(ValueError):
+            oj.call_with_fallback(["one", "two"], lambda m: {"model": m}, call)
+        self.assertEqual(seen, ["one"])
+
+    def test_transient_error_retries_same_model(self):
+        seen = []
+
+        def call(body):
+            seen.append(body["model"])
+            if len(seen) == 1:
+                raise ValueError("OpenRouter HTTP 500")
+            return {"ok": True}
+
+        model, _ = oj.call_with_fallback(
+            ["one"], lambda m: {"model": m}, call, attempts=2)
+        self.assertEqual(model, "one")
+        self.assertEqual(seen, ["one", "one"])
+
+
+if __name__ == "__main__":
+    unittest.main()