From 91768a9f74cd2bb118402d19897a27929d557cd9 Mon Sep 17 00:00:00 2001 From: Zwitschi Date: Sat, 7 Nov 2026 15:57:49 +0100 Subject: [PATCH] tools: add screenshot judge script using OpenRouter vision - Tools/judge_screenshot.py: judges a screenshot against an expected-scene description via OpenRouter multimodal image understanding, reading OPENROUTER_API_KEY from the ignored .env. --image+--expect for one shot, --shots to run every review shot in Screenshots/README.md. Returns a {verdict, confidence, reason, missing} JSON. - Documented in AGENTS.md (Visual review pre-check step) and Tools/README.md; recorded the OpenRouter payload format gotcha in MEMORY.md. --- AGENTS.md | 7 +- MEMORY.md | 5 + Tools/README.md | 1 + Tools/judge_screenshot.py | 192 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 202 insertions(+), 3 deletions(-) create mode 100755 Tools/judge_screenshot.py diff --git a/AGENTS.md b/AGENTS.md index 473577b..267533d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -99,9 +99,10 @@ Always use a consistent scale test, named export presets, and repeatable import Any change that affects the game's appearance — meshes, materials, lighting, levels, character, camera, post-processing — must be rendered for human review: 1. Run `Tools/render_screenshots.sh` (rebuilds `Game/GigLine/Content/Test/ReviewMap.umap` from current content, runs the game on it, captures the fixed camera list, copies the PNGs into `Screenshots/`). -2. Update `Screenshots/README.md`: one row per screenshot with what it shows and an approval state; describe any new shots or scene changes in the same change. -3. The PNGs **are committed** so reviewers see exactly what the build rendered (`.gitignore` does not exclude them). Keep them in sync with this README: drop retired shots in the same change. -4. Wait for the user's approval before treating a visual change as done; record approval by ticking the table in `Screenshots/README.md`. +2. Optional pre-check with a vision model: `Tools/judge_screenshot.py --image --expect ""` (or `--shots` to run every shot in the README) flags likely-missing content before a human looks. It reads `OPENROUTER_API_KEY` from `.env` (ignored). Small helper only — it never replaces the human verdict. +3. Update `Screenshots/README.md`: one row per screenshot with what it shows and an approval state; describe any new shots or scene changes in the same change. +4. The PNGs **are committed** so reviewers see exactly what the build rendered (`.gitignore` does not exclude them). Keep them in sync with this README: drop retired shots in the same change. +5. Wait for the user's approval before treating a visual change as done; record approval by ticking the table in `Screenshots/README.md`. New content that needs a different camera angle gets a new entry in the shot list in `Tools/ue_build_review_map.py`. diff --git a/MEMORY.md b/MEMORY.md index 1989645..736abf8 100644 --- a/MEMORY.md +++ b/MEMORY.md @@ -147,3 +147,8 @@ Scope: facts and traps that cost time to establish. Planned work lives in [TODO. - **A per-pixel `struct.pack` loop over `img.pixels` is catastrophic in headless Blender** (256² took 25 s). Blender ships numpy: read `img.pixels[:]` once into a float array, reshape `(h,w,4)`, clip→uint8, reverse rows for PNG top-down, then zlib-compress. Whole 1024² encode ≈ 0.3 s. - **MPFB's v2-skin node tree does not resolve to a simple base colour**, so a Cycles DIFFUSE bake returns black. Since the FBX export material replaces the node tree anyway, synthesize the diffuse directly (`make_diffuse`): warm skin body + a darker shorts-suggestive bottom band. - **A read-only `.umap` silently makes `save_map` return `False` with no error** (already recorded); re-hit in the review pipeline — `chmod u+w` the maps before rebuilding, otherwise the new character material never appears in shots. + +## 2026-11-07 — screenshot judge tool + +- `Tools/judge_screenshot.py` — runs an OpenRouter vision model over a screenshot vs an expected-scene description. Reads `OPENROUTER_API_KEY` from the repo `.env` (ignored). Base64 data-URI image under `image_url.url`; OpenAI-compatible body → `POST /api/v1/chat/completions`. `openai/gpt-4o-mini` returns a `{verdict, confidence, reason, missing}` JSON. It flags genuinely missing content (e.g. `main`'s pre-warm-skin character shot judged "no"); it is a pre-check, not the human verdict. +- Test one: `Tools/judge_screenshot.py --image Screenshots/review-01-character.png --expect "" --verbose`. Bulk: `--shots` (parses the README rows). diff --git a/Tools/README.md b/Tools/README.md index 29279ac..26c5939 100644 --- a/Tools/README.md +++ b/Tools/README.md @@ -50,3 +50,4 @@ Geographic tools live in [`Geo/tools/`](../Geo/README.md) and the Blender-side b | Tool | Purpose | | --- | --- | | `render_screenshots.py` / `render_screenshots.sh` | Rebuilds the review map, runs the game, captures the shot list into `Screenshots/` | +| `judge_screenshot.py` | Vision-model pre-check: judges a screenshot against an expected-scene description (`--image` + `--expect`, or `--shots` for every shot in the README). Reads `OPENROUTER_API_KEY` from `.env` (ignored). Optional helper — a human must still approve. | diff --git a/Tools/judge_screenshot.py b/Tools/judge_screenshot.py new file mode 100755 index 0000000..e13f877 --- /dev/null +++ b/Tools/judge_screenshot.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python3 +"""Judge a screenshot against an expected-scene description using OpenRouter +multimodal image understanding. + +Reads OPENROUTER_API_KEY from the project .env (ignored, never committed) and +sends the screenshot as a base64 data-URI image alongside a description of what +the shot is expected to show. Returns a verdict (yes / no / uncertain), a +confidence, and the model's reasoning, so a screenshot can be pre-checked before +a human reviews it. + +API: OpenRouter chat/completions, OpenAI-compatible image_url content part: + {"type": "image_url", "image_url": {"url": "data:image/png;base64,<...>"}} +Docs: https://openrouter.ai/docs/guides/overview/multimodal/image-understanding + +Usage: + judge_screenshot.py --image Screenshots/review-overview.png \ + --expect "a character and two 1 m cubes on a large grey floor" + judge_screenshot.py --image Screenshots/review-character.png \ + --expect "the idle upright human figure with warm skin, feet on the plane" \ + --model openai/gpt-4o-mini --verbose + judge_screenshot.py --shots # evaluate every review-NN shot in README + +.env format: + OPENROUTER_API_KEY=sk-or-v1-... +""" + +import argparse +import base64 +import json +import os +import re +import sys + +import requests + +API_URL = "https://openrouter.ai/api/v1/chat/completions" +DEFAULT_MODEL = "openai/gpt-4o-mini" + +# Project root = parent of the Tools/ directory the script lives in. +PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) +ENV_PATH = os.path.join(PROJECT_ROOT, ".env") + + +def load_api_key(): + """Read OPENROUTER_API_KEY from the project .env.""" + if not os.path.exists(ENV_PATH): + sys.exit(f"error: no {ENV_PATH} — create it with OPENROUTER_API_KEY=sk-or-v1-...") + with open(ENV_PATH, encoding="utf-8") as f: + for line in f: + line = line.strip() + if line and not line.startswith("#"): + key, _, val = line.partition("=") + if key.strip() == "OPENROUTER_API_KEY": + return val.strip() + sys.exit("error: OPENROUTER_API_KEY not found in .env") + + +def image_data_uri(path): + """Return a base64 data: URI for a local image file.""" + if not os.path.isfile(path): + sys.exit(f"error: image not found: {path}") + ext = os.path.splitext(path)[1].lower().lstrip(".") or "png" + mime = {"png": "image/png", "jpg": "image/jpeg", "jpeg": "image/jpeg", + "webp": "image/webp", "gif": "image/gif"}.get(ext, "image/png") + with open(path, "rb") as fh: + b64 = base64.b64encode(fh.read()).decode("ascii") + return f"data:{mime};base64,{b64}" + + +def build_messages(image_path, expect): + user = ( + f"You are an objective QA reviewer for a 3D game scene. Look at the " + f"rendered screenshot and judge whether it shows what is expected.\n\n" + f"Expected scene:\n{expect}\n\n" + f"Answer STRICTLY in this JSON shape, with no other text:\n" + f'{{"verdict": "yes"|"no"|"uncertain", "confidence": 0.0-1.0, ' + f'"reason": "", ' + f'"missing": [""]}}' + ) + return [{ + "role": "user", + "content": [ + {"type": "text", "text": user}, + {"type": "image_url", "image_url": {"url": image_data_uri(image_path)}}, + ], + }] + + +def judge(api_key, image_path, expect, model, timeout=60): + payload = { + "model": model, + "messages": build_messages(image_path, expect), + "temperature": 0, + } + headers = { + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + # Identifies the caller to OpenRouter; HTTP-Referer/X-Title are optional. + "HTTP-Referer": "https://github.com/zwitschi/gig-line", + "X-Title": "gig-line screenshot judge", + } + resp = requests.post(API_URL, headers=headers, json=payload, timeout=timeout) + if resp.status_code != 200: + body = resp.text[:500] + sys.exit(f"error: OpenRouter returned {resp.status_code}: {body}") + data = resp.json() + content = data["choices"][0]["message"]["content"] + # Trim any markdown fences and take the first JSON object we can parse. + m = re.search(r"\{.*\}", content, re.S) + if not m: + sys.exit(f"error: model returned non-JSON:\n{content[:500]}") + return json.loads(m.group(0)) + + +def parse_shots_from_readme(): + """Parse the Screenshots/README.md tables into (path, expected) pairs.""" + readme = os.path.join(PROJECT_ROOT, "Screenshots", "README.md") + if not os.path.isfile(readme): + sys.exit(f"error: no {readme}") + text = open(readme, encoding="utf-8").read() + rows = [] + # Each table row: | File | ... | ... | What it shows | + for line in text.splitlines(): + if not line.strip().startswith("|"): + continue + cells = [c.strip() for c in line.strip().strip("|").split("|")] + if not cells or "`review-" not in cells[0]: + continue + fname = re.search(r"`(review-[^`]+\.png)`", cells[0]) + if not fname: + continue + # the last cell is the "What it shows" description + expect = cells[-1] + rows.append((fname.group(1), expect)) + return rows + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + g = ap.add_mutually_exclusive_group(required=True) + g.add_argument("--image", help="path to a screenshot PNG to judge") + g.add_argument("--shots", action="store_true", + help="evaluate every review shot listed in Screenshots/README.md") + ap.add_argument("--expect", help="what the shot is expected to show (required with --image)") + ap.add_argument("--model", default=DEFAULT_MODEL, help=f"OpenRouter model (default {DEFAULT_MODEL})") + ap.add_argument("--verbose", action="store_true") + ap.add_argument("--timeout", type=int, default=60) + args = ap.parse_args() + + api_key = load_api_key() + + if args.shots: + shots = parse_shots_from_readme() + if not shots: + sys.exit("error: no review shots found in Screenshots/README.md") + # Only review-map shots (skip the demo/ ones which have a preview column too). + shots = [s for s in shots if "/demo/" not in s[0]] + n = len(shots) + print(f"Judging {n} screenshots with {args.model} ...") + ok = 0 + for i, (fname, expect) in enumerate(shots, 1): + path = os.path.join(PROJECT_ROOT, "Screenshots", fname.replace("Screenshots/", "")) + if not os.path.isfile(path): + print(f"[{i}/{n}] {fname}: MISSING (skip)") + continue + try: + res = judge(api_key, path, expect, args.model, args.timeout) + except SystemExit as e: + print(f"[{i}/{n}] {fname}: ERROR {e}") + continue + verdict = res.get("verdict", "unknown") + conf = res.get("confidence", 0.0) + reason = res.get("reason", "") + mark = {"yes": "OK", "no": "FAIL", "uncertain": "?"}.get(verdict, "?") + print(f"[{i}/{n}] {fname}: {mark} ({conf:.2f}) {reason}") + if verdict == "yes": + ok += 1 + print(f"\nSummary: {ok}/{n} judged yes") + return + + if not args.expect: + ap.error("--expect is required with --image") + + res = judge(api_key, args.image, args.expect, args.model, args.timeout) + print(json.dumps(res, indent=2) if args.verbose + else f"verdict={res.get('verdict')} confidence={res.get('confidence')} " + f"reason={res.get('reason', '')}") + + +if __name__ == "__main__": + main() \ No newline at end of file -- 2.47.3