Make the vision judge discover cost-effective OpenRouter models
Add openrouter_judge.py to rank live image+JSON models by advertised price, with ordered fallbacks, auth-error handling and auto selection. Wire both judges to --model auto, record the model used, and add fake-catalogue/mocked-call tests. Re-run the judges with the valid key: 13/13 screen designs match, 0 diff; the Forward+ A/B report records the selected model and ~$0.006 spend. Headless 231/231, Python 94/94.
This commit is contained in:
1 parent
28f2a48811
commit
659b0e35b2
15 files changed
+535
-65
No files matched your search
+37
-16
@@ -38,6 +38,7 @@ from PIL import Image, ImageFilter, ImageOps, ImageStat
|
||||
|
||||
import asset_pipeline
|
||||
import generate_textures
|
||||
import openrouter_judge
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
RAW_DIR = ROOT / "assets" / "textures" / "ai" / "v2" / "raw"
|
||||
@@ -46,7 +47,7 @@ CONCEPT_DIR = ROOT / "docs" / "screens" / "concepts"
|
||||
REALISM_DIR = ROOT / "docs" / "screens" / "realism"
|
||||
|
||||
DEFAULT_MODEL = "openai/gpt-5-image-mini"
|
||||
DEFAULT_JUDGE_MODEL = "google/gemini-2.5-flash-lite"
|
||||
DEFAULT_JUDGE_MODEL = openrouter_judge.DEFAULT_MODEL
|
||||
OPENROUTER_CHAT_URL = "https://openrouter.ai/api/v1/chat/completions"
|
||||
|
||||
SCENE_W, SCENE_H = 1280, 720
|
||||
@@ -758,25 +759,32 @@ def _call_openrouter(body: dict, timeout: int = 180) -> dict:
|
||||
raise ValueError(f"OpenRouter HTTP {exc.code}: {exc.read().decode('utf-8', 'replace')[:300]}")
|
||||
|
||||
|
||||
def judge_capture(model: str, target: Path, capture: Path, attempts: int = 2,
|
||||
def judge_capture(models, target: Path, capture: Path, attempts: int = 2,
|
||||
repeats: int = 1) -> dict:
|
||||
"""Vision-judge one capture; retry transient errors, average repeat scores.
|
||||
|
||||
`models` is a fallback-ordered list from `openrouter_judge.resolve_models`.
|
||||
A single score from a small vision model swings by ±1-2, so `repeats` runs
|
||||
are averaged and the spread is kept (`score_range`) for the report.
|
||||
"""
|
||||
if isinstance(models, str):
|
||||
models = [models]
|
||||
runs = []
|
||||
error = None
|
||||
used = set()
|
||||
for _ in range(max(1, repeats)):
|
||||
for _attempt in range(attempts):
|
||||
try:
|
||||
runs.append(parse_realism_response(_call_openrouter(
|
||||
build_realism_request(model, target, capture))))
|
||||
break
|
||||
except (ValueError, OSError) as exc:
|
||||
error = exc
|
||||
try:
|
||||
model, payload = openrouter_judge.call_with_fallback(
|
||||
models, lambda selected: build_realism_request(selected, target, capture),
|
||||
_call_openrouter, attempts=attempts)
|
||||
used.add(model)
|
||||
runs.append(parse_realism_response(payload))
|
||||
except (ValueError, OSError) as exc:
|
||||
if openrouter_judge.is_auth_error(exc):
|
||||
raise
|
||||
error = exc
|
||||
if not runs:
|
||||
return {"scores": {}, "fixes": [f"judge failed: {error}"]}
|
||||
return {"scores": {}, "fixes": [f"judge failed: {error}"], "model": ""}
|
||||
keys = {key for run in runs for key in run["scores"]}
|
||||
scores = {}
|
||||
spread = {}
|
||||
@@ -790,7 +798,7 @@ def judge_capture(model: str, target: Path, capture: Path, attempts: int = 2,
|
||||
if fix not in fixes:
|
||||
fixes.append(fix)
|
||||
return {"scores": scores, "score_range": spread, "runs": len(runs),
|
||||
"fixes": fixes[:5]}
|
||||
"model": ", ".join(sorted(used)), "fixes": fixes[:5]}
|
||||
|
||||
|
||||
def cmd_realism(args) -> int:
|
||||
@@ -800,10 +808,14 @@ def cmd_realism(args) -> int:
|
||||
if not sets:
|
||||
print(f"no capture sets under {_rel(run)}", file=sys.stderr)
|
||||
return 2
|
||||
judge_models = []
|
||||
if args.judge:
|
||||
_load_env_file()
|
||||
report = {"run": _rel(run), "judge_model": args.model if args.judge else None,
|
||||
"sets": {}}
|
||||
judge_models = openrouter_judge.resolve_models(args.model)
|
||||
print(f"judge models: {', '.join(judge_models)}", file=sys.stderr)
|
||||
report = {"run": _rel(run),
|
||||
"judge_model": (judge_models[0] if judge_models else None),
|
||||
"judge_fallbacks": judge_models, "sets": {}}
|
||||
for capture_set in sets:
|
||||
rows = {}
|
||||
for capture in sorted(capture_set.glob("*.png")):
|
||||
@@ -815,8 +827,15 @@ def cmd_realism(args) -> int:
|
||||
row = {"target": _rel(target), "metrics": cap_m, "target_metrics": tgt_m,
|
||||
"metric_gap": metric_gap(cap_m, tgt_m)}
|
||||
if args.judge:
|
||||
row["judge"] = judge_capture(args.model, target, capture,
|
||||
repeats=args.repeats)
|
||||
try:
|
||||
row["judge"] = judge_capture(judge_models, target, capture,
|
||||
repeats=args.repeats)
|
||||
except ValueError as exc:
|
||||
if openrouter_judge.is_auth_error(exc):
|
||||
print(f"error: OpenRouter authentication failed: {exc}",
|
||||
file=sys.stderr)
|
||||
return 2
|
||||
raise
|
||||
rows[capture.stem] = row
|
||||
report["sets"][capture_set.name] = rows
|
||||
json_path = next_free_path(run / "report.json")
|
||||
@@ -975,7 +994,9 @@ def build_parser():
|
||||
real = sub.add_parser("realism")
|
||||
real.add_argument("--run", required=True)
|
||||
real.add_argument("--judge", action="store_true")
|
||||
real.add_argument("--model", default=DEFAULT_JUDGE_MODEL)
|
||||
real.add_argument("--model", default=DEFAULT_JUDGE_MODEL,
|
||||
help="OpenRouter model id, or 'auto' to pick the cheapest "
|
||||
"available image+JSON model with fallbacks")
|
||||
real.add_argument("--repeats", type=int, default=1,
|
||||
help="average N judge runs per shot (scores are noisy)")
|
||||
real.set_defaults(handler=cmd_realism)
|
||||
|
||||
Reference in new issue
Block a user