zwitschi
91768a9f74
- Tools/judge_screenshot.py: judges a screenshot against an expected-scene
description via OpenRouter multimodal image understanding, reading
OPENROUTER_API_KEY from the ignored .env. --image+--expect for one shot,
--shots to run every review shot in Screenshots/README.md. Returns a
{verdict, confidence, reason, missing} JSON.
- Documented in AGENTS.md (Visual review pre-check step) and Tools/README.md;
recorded the OpenRouter payload format gotcha in MEMORY.md.
192 lines
7.6 KiB
Python
Executable File
192 lines
7.6 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Judge a screenshot against an expected-scene description using OpenRouter
|
|
multimodal image understanding.
|
|
|
|
Reads OPENROUTER_API_KEY from the project .env (ignored, never committed) and
|
|
sends the screenshot as a base64 data-URI image alongside a description of what
|
|
the shot is expected to show. Returns a verdict (yes / no / uncertain), a
|
|
confidence, and the model's reasoning, so a screenshot can be pre-checked before
|
|
a human reviews it.
|
|
|
|
API: OpenRouter chat/completions, OpenAI-compatible image_url content part:
|
|
{"type": "image_url", "image_url": {"url": "data:image/png;base64,<...>"}}
|
|
Docs: https://openrouter.ai/docs/guides/overview/multimodal/image-understanding
|
|
|
|
Usage:
|
|
judge_screenshot.py --image Screenshots/review-overview.png \
|
|
--expect "a character and two 1 m cubes on a large grey floor"
|
|
judge_screenshot.py --image Screenshots/review-character.png \
|
|
--expect "the idle upright human figure with warm skin, feet on the plane" \
|
|
--model openai/gpt-4o-mini --verbose
|
|
judge_screenshot.py --shots # evaluate every review-NN shot in README
|
|
|
|
.env format:
|
|
OPENROUTER_API_KEY=sk-or-v1-...
|
|
"""
|
|
|
|
import argparse
|
|
import base64
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
|
|
import requests
|
|
|
|
API_URL = "https://openrouter.ai/api/v1/chat/completions"
|
|
DEFAULT_MODEL = "openai/gpt-4o-mini"
|
|
|
|
# Project root = parent of the Tools/ directory the script lives in.
|
|
PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
|
|
ENV_PATH = os.path.join(PROJECT_ROOT, ".env")
|
|
|
|
|
|
def load_api_key():
|
|
"""Read OPENROUTER_API_KEY from the project .env."""
|
|
if not os.path.exists(ENV_PATH):
|
|
sys.exit(f"error: no {ENV_PATH} — create it with OPENROUTER_API_KEY=sk-or-v1-...")
|
|
with open(ENV_PATH, encoding="utf-8") as f:
|
|
for line in f:
|
|
line = line.strip()
|
|
if line and not line.startswith("#"):
|
|
key, _, val = line.partition("=")
|
|
if key.strip() == "OPENROUTER_API_KEY":
|
|
return val.strip()
|
|
sys.exit("error: OPENROUTER_API_KEY not found in .env")
|
|
|
|
|
|
def image_data_uri(path):
|
|
"""Return a base64 data: URI for a local image file."""
|
|
if not os.path.isfile(path):
|
|
sys.exit(f"error: image not found: {path}")
|
|
ext = os.path.splitext(path)[1].lower().lstrip(".") or "png"
|
|
mime = {"png": "image/png", "jpg": "image/jpeg", "jpeg": "image/jpeg",
|
|
"webp": "image/webp", "gif": "image/gif"}.get(ext, "image/png")
|
|
with open(path, "rb") as fh:
|
|
b64 = base64.b64encode(fh.read()).decode("ascii")
|
|
return f"data:{mime};base64,{b64}"
|
|
|
|
|
|
def build_messages(image_path, expect):
|
|
user = (
|
|
f"You are an objective QA reviewer for a 3D game scene. Look at the "
|
|
f"rendered screenshot and judge whether it shows what is expected.\n\n"
|
|
f"Expected scene:\n{expect}\n\n"
|
|
f"Answer STRICTLY in this JSON shape, with no other text:\n"
|
|
f'{{"verdict": "yes"|"no"|"uncertain", "confidence": 0.0-1.0, '
|
|
f'"reason": "<short reason, up to 3 sentences>", '
|
|
f'"missing": ["<brief gap, if any>"]}}'
|
|
)
|
|
return [{
|
|
"role": "user",
|
|
"content": [
|
|
{"type": "text", "text": user},
|
|
{"type": "image_url", "image_url": {"url": image_data_uri(image_path)}},
|
|
],
|
|
}]
|
|
|
|
|
|
def judge(api_key, image_path, expect, model, timeout=60):
|
|
payload = {
|
|
"model": model,
|
|
"messages": build_messages(image_path, expect),
|
|
"temperature": 0,
|
|
}
|
|
headers = {
|
|
"Authorization": f"Bearer {api_key}",
|
|
"Content-Type": "application/json",
|
|
# Identifies the caller to OpenRouter; HTTP-Referer/X-Title are optional.
|
|
"HTTP-Referer": "https://github.com/zwitschi/gig-line",
|
|
"X-Title": "gig-line screenshot judge",
|
|
}
|
|
resp = requests.post(API_URL, headers=headers, json=payload, timeout=timeout)
|
|
if resp.status_code != 200:
|
|
body = resp.text[:500]
|
|
sys.exit(f"error: OpenRouter returned {resp.status_code}: {body}")
|
|
data = resp.json()
|
|
content = data["choices"][0]["message"]["content"]
|
|
# Trim any markdown fences and take the first JSON object we can parse.
|
|
m = re.search(r"\{.*\}", content, re.S)
|
|
if not m:
|
|
sys.exit(f"error: model returned non-JSON:\n{content[:500]}")
|
|
return json.loads(m.group(0))
|
|
|
|
|
|
def parse_shots_from_readme():
|
|
"""Parse the Screenshots/README.md tables into (path, expected) pairs."""
|
|
readme = os.path.join(PROJECT_ROOT, "Screenshots", "README.md")
|
|
if not os.path.isfile(readme):
|
|
sys.exit(f"error: no {readme}")
|
|
text = open(readme, encoding="utf-8").read()
|
|
rows = []
|
|
# Each table row: | File | ... | ... | What it shows |
|
|
for line in text.splitlines():
|
|
if not line.strip().startswith("|"):
|
|
continue
|
|
cells = [c.strip() for c in line.strip().strip("|").split("|")]
|
|
if not cells or "`review-" not in cells[0]:
|
|
continue
|
|
fname = re.search(r"`(review-[^`]+\.png)`", cells[0])
|
|
if not fname:
|
|
continue
|
|
# the last cell is the "What it shows" description
|
|
expect = cells[-1]
|
|
rows.append((fname.group(1), expect))
|
|
return rows
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser(description=__doc__,
|
|
formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
g = ap.add_mutually_exclusive_group(required=True)
|
|
g.add_argument("--image", help="path to a screenshot PNG to judge")
|
|
g.add_argument("--shots", action="store_true",
|
|
help="evaluate every review shot listed in Screenshots/README.md")
|
|
ap.add_argument("--expect", help="what the shot is expected to show (required with --image)")
|
|
ap.add_argument("--model", default=DEFAULT_MODEL, help=f"OpenRouter model (default {DEFAULT_MODEL})")
|
|
ap.add_argument("--verbose", action="store_true")
|
|
ap.add_argument("--timeout", type=int, default=60)
|
|
args = ap.parse_args()
|
|
|
|
api_key = load_api_key()
|
|
|
|
if args.shots:
|
|
shots = parse_shots_from_readme()
|
|
if not shots:
|
|
sys.exit("error: no review shots found in Screenshots/README.md")
|
|
# Only review-map shots (skip the demo/ ones which have a preview column too).
|
|
shots = [s for s in shots if "/demo/" not in s[0]]
|
|
n = len(shots)
|
|
print(f"Judging {n} screenshots with {args.model} ...")
|
|
ok = 0
|
|
for i, (fname, expect) in enumerate(shots, 1):
|
|
path = os.path.join(PROJECT_ROOT, "Screenshots", fname.replace("Screenshots/", ""))
|
|
if not os.path.isfile(path):
|
|
print(f"[{i}/{n}] {fname}: MISSING (skip)")
|
|
continue
|
|
try:
|
|
res = judge(api_key, path, expect, args.model, args.timeout)
|
|
except SystemExit as e:
|
|
print(f"[{i}/{n}] {fname}: ERROR {e}")
|
|
continue
|
|
verdict = res.get("verdict", "unknown")
|
|
conf = res.get("confidence", 0.0)
|
|
reason = res.get("reason", "")
|
|
mark = {"yes": "OK", "no": "FAIL", "uncertain": "?"}.get(verdict, "?")
|
|
print(f"[{i}/{n}] {fname}: {mark} ({conf:.2f}) {reason}")
|
|
if verdict == "yes":
|
|
ok += 1
|
|
print(f"\nSummary: {ok}/{n} judged yes")
|
|
return
|
|
|
|
if not args.expect:
|
|
ap.error("--expect is required with --image")
|
|
|
|
res = judge(api_key, args.image, args.expect, args.model, args.timeout)
|
|
print(json.dumps(res, indent=2) if args.verbose
|
|
else f"verdict={res.get('verdict')} confidence={res.get('confidence')} "
|
|
f"reason={res.get('reason', '')}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main() |