From 72b9378e69e327974969d526ff6c9e6fd0229468 Mon Sep 17 00:00:00 2001 From: Zwitschi Date: Sat, 31 Oct 2026 22:11:45 +0100 Subject: [PATCH] Add a vision-model screenshot judge to the QA tooling `tools/screen_designs.py judge` sends each capture together with its concept mockup to a vision model and asks whether the capture realises the same screen and layout intent. This covers what pixel/perceptual metrics cannot judge. - Prompt explicitly ignores art style, fidelity, fonts, and placeholder text, since the references are stylised mockups. - Verdicts are classed match / review (mismatch below --fail-confidence, default 0.6) / diff (confident mismatch, exits non-zero) / unknown; --json, --only, --dry-run, --model supported. - Reads the repo .env for OPENROUTER_API_KEY without overriding real env vars. - Default judge model google/gemini-2.5-flash-lite; openai/gpt-4o-mini verified too. Tests/docs - 7 new offline tests in tools/test_screen_designs.py (request payload embeds both images, JSON/fenced/parts/non-JSON parsing, threshold classification, .env loading); 30 Python tests pass. - Live run: 3 captures match, 3 review, 0 confident diffs. - README, TODO section moved to DONE, MEMORY. Version 0.2.15. --- DONE.md | 15 +++ MEMORY.md | 12 +- README.md | 19 +++- TODO.md | 18 +-- project.godot | 2 +- tools/screen_designs.py | 210 +++++++++++++++++++++++++++++++++++ tools/test_screen_designs.py | 92 +++++++++++++++ 7 files changed, 347 insertions(+), 21 deletions(-) diff --git a/DONE.md b/DONE.md index 4817483..f4e944a 100644 --- a/DONE.md +++ b/DONE.md @@ -568,3 +568,18 @@ Completed 2026-10-31 (working tree). Headless **153/153**. - [x] Keep `%` placeholders stable in `Strings.fmt` when adding keys (positional formatting). + +## Screen design references (QA tooling) + +Completed 2026-10-31 (working tree). 30 Python tooling tests pass; the live judge run classed 3 captures as match and 3 as review (concept mockups are more stylised), with no confident mismatch. + + +- [x] `tools/screen_designs.py generate` writes one reference design per screen + (from the `docs/` descriptions) to `docs/screens/designs/`, with an offline + `--draft` wireframe mode and an OpenRouter `--ai` mockup mode. +- [x] `tools/screen_designs.py compare` checks `docs/screenshots/*.png` against the + designs (aspect ratio, size, blank-frame, and layout/palette/detail drift). +- [x] Vision-model judge for semantic screenshot comparison: `judge` sends the + capture + reference design to a vision model, classes verdicts as + match/review/diff with a confidence threshold, supports `--dry-run`, `--json`, + `--only`, and `--model`, and reads the repo `.env` for the API key. diff --git a/MEMORY.md b/MEMORY.md index 4d5a293..9ee822b 100644 --- a/MEMORY.md +++ b/MEMORY.md @@ -21,8 +21,8 @@ Session memory for durable decisions, state, and gotchas. Keep concise, remove o Windowed control smoke: `godot --path . --script res://src/test/smoke_focus_radial.gd` → exit 0. - `main` is in sync with origin. **v0.2.2** tags the session-shell + world-follow-up commit; CI builds the portable binaries, NSIS installer, and AppImage on the tag push. -- Next work: the two remaining TODO sections — the vision-model screenshot judge - (`tools/screen_designs.py`, OpenRouter key in `.env`) and editor tooling (GUT / `.tres` venues). +- Next work: only the optional editor-tooling section remains (GUT plugin, `.tres` venues); + everything else in `TODO.md` is complete. - Do not re-open `override_access`. Do not bind wand to `W`. Do not remove the radio easter egg. ## Current State @@ -95,6 +95,14 @@ Session memory for durable decisions, state, and gotchas. Keep concise, remove o `_begin_incident()` now starts one per focused NPC (`has_open_incident()` guard), verbs call `_ensure_incident()`, and `ShiftCoordinator._finish_shift()` closes leftovers as `unresolved` before building the review. +- **Vision judge (2026-10-31):** `python3 tools/screen_designs.py judge` sends the capture plus + its concept mockup to a vision model (`DEFAULT_JUDGE_MODEL = google/gemini-2.5-flash-lite`, + override with `--model`; `openai/gpt-4o-mini` also works). Reads the repo `.env` via + `_load_env_file()` (never overrides real env). Verdicts are classed `match` / `review` + (mismatch below `--fail-confidence`, default 0.6) / `diff` (confident mismatch → exit 1) / + `unknown`. `--dry-run`, `--json`, `--only` supported. The prompt explicitly ignores art style, + because the references are stylised mockups; the live run gave 3 match, 3 review, 0 diff. + Python tests: `cd tools && python3 -m unittest test_asset_pipeline test_screen_designs`. - **String formatting safety (2026-10-31):** `Strings.placeholder_count()` counts `%` specs (ignoring `%%`), `fmt()` warns and returns the template when the arg count does not match (instead of raising an engine error), and `validate()` lints the table for trailing `%`, diff --git a/README.md b/README.md index 4f8e55e..8a570db 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ The central design goal is simple: > Downloads are public (no sign-in required) since the host's instance-wide > `REQUIRE_SIGNIN_VIEW` was disabled on 2026-10-31. > -> **Latest: v0.2.14** — string-formatting safety, plus public downloads (no sign-in) on the host. +> **Latest: v0.2.15** — vision-model screenshot judge for the QA tooling. > Releases are built automatically when a `vX.Y.Z` tag is pushed > (see [`docs/RELEASING.md`](docs/RELEASING.md)); manual builds use > `./tools/build_releases.sh`. Installer/AppImage packaging runs in the same pipeline @@ -205,13 +205,26 @@ python3 tools/screen_designs.py generate # Compare docs/screenshots/*.png against the designs python3 tools/screen_designs.py compare python3 tools/screen_designs.py compare --json + +# Ask a vision model whether each capture realises its design's screen and layout +# (requires OPENROUTER_API_KEY; the repo .env is read automatically) +python3 tools/screen_designs.py judge +python3 tools/screen_designs.py judge --model openai/gpt-4o-mini --json +python3 tools/screen_designs.py judge --only 03- # one capture +python3 tools/screen_designs.py judge --dry-run # build requests, no network ``` `compare` flags gross drift: wrong aspect ratio, undersized frame, near-blank frame, and large layout/palette/detail distance. It is a regression gate, not a semantic judge — pixel and perceptual metrics cannot decide whether a frame's -composition is *right*, so a vision-model pass is the documented follow-up (see -`TODO.md`). Regenerate the designs after changing screen layout, then re-run +composition is *right*. + +`judge` covers that gap: it sends each capture together with its concept mockup to +a vision model and asks whether the capture realises the same screen and layout +intent. Art style, fidelity, fonts, and placeholder text are explicitly ignored. +Verdicts are classed as `match`, `review` (a mismatch below `--fail-confidence`, +default 0.6 — expected when a concept mockup is more stylised than the game), or +`diff` (a confident mismatch, which exits non-zero). Regenerate the designs after changing screen layout, then re-run `compare` after regenerating the game screenshots with `src/test/capture_screenshots.gd`. diff --git a/TODO.md b/TODO.md index 9e4cad8..69f14bd 100644 --- a/TODO.md +++ b/TODO.md @@ -1,20 +1,8 @@ # TODO -Godot 4.3 prototype. Latest release **v0.2.14**. Headless suite **153/153**, plus the session, -PLAN-verb, crowd, emergency, timeline, audio, scenario-content, and windowed focus/radial smokes. -Anonymous downloads are enabled on the host. - -## Screen design references (QA tooling) - -- [x] `tools/screen_designs.py generate` writes one reference design per screen - (from the `docs/` descriptions) to `docs/screens/designs/`, with an offline - `--draft` wireframe mode and an OpenRouter `--ai` mockup mode. -- [x] `tools/screen_designs.py compare` checks `docs/screenshots/*.png` against the - designs (aspect ratio, size, blank-frame, and layout/palette/detail drift). -- [ ] Vision-model judge for semantic screenshot comparison. Pixel and perceptual - metrics cannot decide whether a frame's composition is correct; a follow-up - should send the capture + reference design to a vision model and ask whether - they depict the same screen. +Godot 4.3 prototype. Latest release **v0.2.15**. Headless suite **153/153** plus 30 Python +tooling tests; the session, PLAN-verb, crowd, emergency, timeline, audio, scenario-content, and +windowed focus/radial smokes pass. Anonymous downloads are enabled on the host. ## Tooling (when an editor workflow exists) diff --git a/project.godot b/project.godot index 1a5bf34..0667bb2 100644 --- a/project.godot +++ b/project.godot @@ -8,7 +8,7 @@ config_version=5 config/name="Step Line" config/description="Workplace security simulation about observation, access verification, and proportionate escalation." -config/version="0.2.14" +config/version="0.2.15" run/main_scene="res://scenes/main.tscn" config/features=PackedStringArray("4.3", "GL Compatibility") diff --git a/tools/screen_designs.py b/tools/screen_designs.py index 0973fd1..0aa479f 100644 --- a/tools/screen_designs.py +++ b/tools/screen_designs.py @@ -17,6 +17,12 @@ Two commands: --draft deterministic wireframe sketches drawn with Pillow (no network or API key; good for CI and layout checks). + judge send each capture together with its reference design to a vision + model and ask whether they depict the same screen. Pixel and + perceptual metrics cannot judge composition; this closes that gap. + Requires OPENROUTER_API_KEY (falls back to the repo .env). + --dry-run build and print the requests without calling the API. + compare compare docs/screenshots/*.png against the reference designs and flag gross drift. Semantic composition is judged with coarse, Pillow-only signals (aspect ratio, frame size, blank-frame @@ -33,6 +39,7 @@ Run from the repo root: """ import argparse +import json import os import sys from pathlib import Path @@ -54,6 +61,10 @@ ASPECT = 16 / 9 # the model asset_pipeline.py already trusts for non-alpha roles). DEFAULT_MODEL = "openai/gpt-5-image-mini" +# Vision model used by `judge`. Cheap, multimodal, and good at "same screen?". +DEFAULT_JUDGE_MODEL = "google/gemini-2.5-flash-lite" +OPENROUTER_CHAT_URL = "https://openrouter.ai/api/v1/chat/completions" + # Shared art direction, appended to every prompt so the generated frames read # as one coherent game. "scene" entries are first-person 3D world views; # "ui" entries are in-game panels overlaid on a softened scene. @@ -582,6 +593,193 @@ def compare(args): # --------------------------------------------------------------------------- +# --------------------------------------------------------------------------- +# Vision judge — semantic "same screen?" check against the reference designs. +# --------------------------------------------------------------------------- + +JUDGE_INSTRUCTIONS = ( + "You are reviewing screenshots of a low-poly 3D security-guard game. " + "IMAGE 1 is a CONCEPT MOCKUP of the intended screen (it may be painted in a " + "different art style and show different actors); IMAGE 2 is the ACTUAL " + "capture from the game. Different art style, fidelity, fonts, and character " + "models are EXPECTED and must be ignored.\n\n" + "Judge only whether IMAGE 2 realises the same SCREEN and LAYOUT INTENT as " + "IMAGE 1: is it the same kind of screen (world view, HUD overlay, modal " + "panel), are the main interface regions present in roughly the same places " + "(target line, inspection card, report form, on-screen buttons), and is the " + "subject of the shot comparable (entrance, street, person)?\n\n" + "Answer with JSON only: {\"match\": true|false, \"confidence\": 0..1, " + "\"differences\": [\"short point\", ...]}. Set match=true when the capture " + "realises the mockup's screen even if it looks much simpler." +) + + +def _load_env_file(path=None): + """Populate os.environ from a .env file without overriding real variables.""" + path = Path(path) if path else ROOT / ".env" + if not path.is_file(): + return + for raw in path.read_text(encoding="utf-8").splitlines(): + line = raw.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, value = line.split("=", 1) + os.environ.setdefault(key.strip(), value.strip().strip("\"'")) + + +def _data_url(path): + import base64 + payload = Path(path).read_bytes() + return "data:image/png;base64," + base64.b64encode(payload).decode("ascii") + + +def build_judge_request(model, screen, capture_path, design_path): + """Build the OpenRouter chat payload for one capture/design pair.""" + prompt = (f"{JUDGE_INSTRUCTIONS}\n\nScreen: {screen['title']} " + f"({screen['source']}). Design notes: {screen['description']}") + return { + "model": model, + "messages": [{ + "role": "user", + "content": [ + {"type": "text", "text": prompt}, + {"type": "image_url", "image_url": {"url": _data_url(design_path)}}, + {"type": "image_url", "image_url": {"url": _data_url(capture_path)}}, + ], + }], + "temperature": 0, + "response_format": {"type": "json_object"}, + } + + +def parse_judge_response(payload): + """Extract a verdict dict from an OpenRouter chat response.""" + try: + text = payload["choices"][0]["message"]["content"] + except (KeyError, IndexError, TypeError): + return {"match": None, "confidence": 0.0, + "differences": ["unreadable model response"]} + if isinstance(text, list): # some providers return content parts + text = "".join(part.get("text", "") for part in text + if isinstance(part, dict)) + text = str(text).strip() + start, end = text.find("{"), text.rfind("}") + if start >= 0 and end > start: + text = text[start:end + 1] + import json + try: + data = json.loads(text) + except ValueError: + return {"match": None, "confidence": 0.0, + "differences": [f"non-JSON response: {text[:120]}"]} + differences = data.get("differences") or [] + if not isinstance(differences, list): + differences = [str(differences)] + return { + "match": data.get("match") if isinstance(data.get("match"), bool) else None, + "confidence": float(data.get("confidence", 0.0) or 0.0), + "differences": [str(item) for item in differences], + } + + +def classify(verdict, threshold=0.6): + """Turn a model verdict into a QA class. + + The reference images are concept mockups, so a low-confidence disagreement is + a review note, not a failure. Only a confident mismatch fails the check. + """ + if verdict.get("match") is True: + return "match" + if verdict.get("match") is False: + return "bad" if float(verdict.get("confidence", 0.0)) >= threshold else "review" + return "unknown" + + +def _call_openrouter(body, timeout=180): + import urllib.error + import urllib.request + key = os.environ.get("OPENROUTER_API_KEY", "").strip() + if not key: + raise ValueError("OPENROUTER_API_KEY is not set (repo .env is read " + "automatically); use --dry-run to inspect requests") + req = urllib.request.Request( + OPENROUTER_CHAT_URL, + data=json.dumps(body).encode("utf-8"), + headers={ + "Authorization": f"Bearer {key}", + "Content-Type": "application/json", + "HTTP-Referer": "https://git.allucanget.biz/zwitschi/security-game", + "X-Title": "Step Line", + }, + method="POST", + ) + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + return json.loads(resp.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + detail = exc.read().decode("utf-8", "replace") + raise ValueError(f"OpenRouter HTTP {exc.code}: {detail[:300]}") + + +def judge(args): + _load_env_file() + if not SCREENSHOTS_DIR.is_dir(): + sys.exit(f"error: no screenshots directory at {SCREENSHOTS_DIR}") + refs = {screen["name"]: screen["ref"] for screen in SCREENS if screen["ref"]} + by_name = {screen["name"]: screen for screen in SCREENS} + results = [] + for capture in sorted(SCREENSHOTS_DIR.glob("*.png")): + if args.only and args.only not in capture.name: + continue + design_name = next((n for n, ref in refs.items() + if ref == capture.name), None) + if design_name is None: + results.append({"capture": capture.name, "screen": None, + "match": None, "confidence": 0.0, + "differences": ["no design reference for this capture"]}) + continue + reference = _design_path(design_name) + if not reference.is_file(): + results.append({"capture": capture.name, "screen": design_name, + "match": None, "confidence": 0.0, + "differences": ["missing reference design; run generate"]}) + continue + body = build_judge_request(args.model, by_name[design_name], + capture, reference) + if args.dry_run: + results.append({ + "capture": capture.name, "screen": design_name, "match": None, + "confidence": 0.0, "model": body["model"], + "differences": [f"dry run: {len(body['messages'][0]['content'])} parts"], + }) + continue + verdict = parse_judge_response(_call_openrouter(body)) + verdict.update({"capture": capture.name, "screen": design_name, + "class": classify(verdict, args.fail_confidence)}) + results.append(verdict) + + if args.json: + print(json.dumps(results, indent=2)) + else: + labels = {"match": "MATCH", "review": "REVIEW", "bad": "DIFF", + "unknown": "UNKNOWN"} + for result in results: + print(f"{labels[result['class']]:7} {result['capture']} " + f"(design: {result['screen']}) " + f"confidence={result['confidence']:.2f}") + for difference in result["differences"]: + print(f" - {difference}") + counts = {key: sum(1 for r in results if r["class"] == key) + for key in labels} + print(f"Summary: {counts['match']}/{len(results)} match, " + f"{counts['review']} review, {counts['bad']} diff " + f"(fail threshold confidence >= {args.fail_confidence:.2f})") + + if args.dry_run: + return 0 + return 1 if any(r["class"] == "bad" for r in results) else 0 + + def build_parser(): parser = argparse.ArgumentParser( description="Generate and compare Step Line screen-design references.") @@ -599,6 +797,18 @@ def build_parser(): default=generate_textures.GEN_SIZE) gen.set_defaults(handler=generate, draft=False) + judge_cmd = commands.add_parser( + "judge", help="ask a vision model whether captures match the designs") + judge_cmd.add_argument("--model", default=DEFAULT_JUDGE_MODEL) + judge_cmd.add_argument("--json", action="store_true") + judge_cmd.add_argument("--dry-run", action="store_true", + help="print requests instead of calling the API") + judge_cmd.add_argument("--only", default="", + help="only judge captures whose name contains this text") + judge_cmd.add_argument("--fail-confidence", type=float, default=0.6, + help="a mismatch at or above this confidence fails") + judge_cmd.set_defaults(handler=judge) + cmp = commands.add_parser( "compare", help="compare docs/screenshots/*.png against the designs") cmp.add_argument("--json", action="store_true") diff --git a/tools/test_screen_designs.py b/tools/test_screen_designs.py index a2bbff3..51839c8 100644 --- a/tools/test_screen_designs.py +++ b/tools/test_screen_designs.py @@ -86,5 +86,97 @@ class ScreenDesignsTests(unittest.TestCase): self.assertTrue(any("aspect" in e for e in report["errors"])) +class VisionJudgeTests(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.TemporaryDirectory() + self.root = Path(self.temp_dir.name) + self.capture = self.root / "capture.png" + self.design = self.root / "design.png" + Image.new("RGB", (64, 36), (20, 30, 40)).save(self.capture) + Image.new("RGB", (64, 36), (200, 190, 180)).save(self.design) + self.screen = screen_designs.SCREENS[0] + + def tearDown(self): + self.temp_dir.cleanup() + + def test_judge_request_embeds_both_images(self): + body = screen_designs.build_judge_request( + "model-x", self.screen, self.capture, self.design) + self.assertEqual(body["model"], "model-x") + self.assertEqual(body["temperature"], 0) + self.assertEqual(body["response_format"], {"type": "json_object"}) + parts = body["messages"][0]["content"] + self.assertEqual(parts[0]["type"], "text") + images = [p for p in parts if p["type"] == "image_url"] + self.assertEqual(len(images), 2, "design and capture are both attached") + for image in images: + self.assertTrue(image["image_url"]["url"].startswith( + "data:image/png;base64,")) + self.assertIn(self.screen["title"], parts[0]["text"]) + self.assertIn("mockup", parts[0]["text"].lower()) + + def test_parse_plain_json_verdict(self): + payload = {"choices": [{"message": {"content": + '{"match": true, "confidence": 0.9, "differences": ["style"]}'}}]} + verdict = screen_designs.parse_judge_response(payload) + self.assertTrue(verdict["match"]) + self.assertAlmostEqual(verdict["confidence"], 0.9) + self.assertEqual(verdict["differences"], ["style"]) + + def test_parse_fenced_json_verdict(self): + payload = {"choices": [{"message": {"content": + 'Sure:\n```json\n{"match": false, "confidence": 0.7}\n```'}}]} + verdict = screen_designs.parse_judge_response(payload) + self.assertFalse(verdict["match"]) + self.assertAlmostEqual(verdict["confidence"], 0.7) + + def test_parse_content_parts(self): + payload = {"choices": [{"message": {"content": [ + {"type": "text", "text": '{"match": true, "confidence": 0.5}'}]}}]} + verdict = screen_designs.parse_judge_response(payload) + self.assertTrue(verdict["match"]) + + def test_parse_non_json_is_unknown(self): + payload = {"choices": [{"message": {"content": "no idea"}}]} + verdict = screen_designs.parse_judge_response(payload) + self.assertIsNone(verdict["match"]) + self.assertTrue(verdict["differences"]) + + def test_parse_missing_choices_is_unknown(self): + verdict = screen_designs.parse_judge_response({}) + self.assertIsNone(verdict["match"]) + + def test_classify_thresholds(self): + self.assertEqual(screen_designs.classify( + {"match": True, "confidence": 0.1}), "match") + self.assertEqual(screen_designs.classify( + {"match": False, "confidence": 0.9}), "bad") + self.assertEqual(screen_designs.classify( + {"match": False, "confidence": 0.3}), "review") + self.assertEqual(screen_designs.classify( + {"match": None, "confidence": 0.0}), "unknown") + self.assertEqual(screen_designs.classify( + {"match": False, "confidence": 0.55}, threshold=0.5), "bad") + + def test_env_file_does_not_override_existing(self): + env_path = self.root / ".env" + env_path.write_text('OPENROUTER_API_KEY=from-file\nOTHER=1\n', + encoding="utf-8") + import os + old = os.environ.get("OPENROUTER_API_KEY") + os.environ["OPENROUTER_API_KEY"] = "from-env" + os.environ.pop("OTHER", None) + try: + screen_designs._load_env_file(env_path) + self.assertEqual(os.environ["OPENROUTER_API_KEY"], "from-env") + self.assertEqual(os.environ.get("OTHER"), "1") + finally: + if old is None: + os.environ.pop("OPENROUTER_API_KEY", None) + else: + os.environ["OPENROUTER_API_KEY"] = old + os.environ.pop("OTHER", None) + + if __name__ == "__main__": unittest.main()