gig-line/Tools/judge_screenshot.py

#!/usr/bin/env python3
"""Judge a screenshot against an expected-scene description using OpenRouter
multimodal image understanding.

Reads OPENROUTER_API_KEY from the project .env (ignored, never committed) and
sends the screenshot as a base64 data-URI image alongside a description of what
the shot is expected to show. Returns a verdict (yes / no / uncertain), a
confidence, and the model's reasoning, so a screenshot can be pre-checked before
a human reviews it.

API: OpenRouter chat/completions, OpenAI-compatible image_url content part:
    {"type": "image_url", "image_url": {"url": "data:image/png;base64,<...>"}}
Docs: https://openrouter.ai/docs/guides/overview/multimodal/image-understanding

Usage:
    judge_screenshot.py --image Screenshots/review-overview.png \
        --expect "a character and two 1 m cubes on a large grey floor"
    judge_screenshot.py --image Screenshots/review-character.png \
        --expect "the idle upright human figure with warm skin, feet on the plane" \
        --model openai/gpt-4o-mini --verbose
    judge_screenshot.py --shots            # evaluate every review-NN shot in README

.env format:
    OPENROUTER_API_KEY=sk-or-v1-...
"""

import argparse
import base64
import json
import os
import re
import sys

import requests

API_URL = "https://openrouter.ai/api/v1/chat/completions"
DEFAULT_MODEL = "openai/gpt-4o-mini"

# Project root = parent of the Tools/ directory the script lives in.
PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
ENV_PATH = os.path.join(PROJECT_ROOT, ".env")


def load_api_key():
    """Read OPENROUTER_API_KEY from the project .env."""
    if not os.path.exists(ENV_PATH):
        sys.exit(f"error: no {ENV_PATH} — create it with OPENROUTER_API_KEY=sk-or-v1-...")
    with open(ENV_PATH, encoding="utf-8") as f:
        for line in f:
            line = line.strip()
            if line and not line.startswith("#"):
                key, _, val = line.partition("=")
                if key.strip() == "OPENROUTER_API_KEY":
                    return val.strip()
    sys.exit("error: OPENROUTER_API_KEY not found in .env")


def image_data_uri(path):
    """Return a base64 data: URI for a local image file."""
    if not os.path.isfile(path):
        sys.exit(f"error: image not found: {path}")
    ext = os.path.splitext(path)[1].lower().lstrip(".") or "png"
    mime = {"png": "image/png", "jpg": "image/jpeg", "jpeg": "image/jpeg",
            "webp": "image/webp", "gif": "image/gif"}.get(ext, "image/png")
    with open(path, "rb") as fh:
        b64 = base64.b64encode(fh.read()).decode("ascii")
    return f"data:{mime};base64,{b64}"


def build_messages(image_path, expect):
    user = (
        f"You are an objective QA reviewer for a 3D game scene. Look at the "
        f"rendered screenshot and judge whether it shows what is expected.\n\n"
        f"Expected scene:\n{expect}\n\n"
        f"Answer STRICTLY in this JSON shape, with no other text:\n"
        f'{{"verdict": "yes"|"no"|"uncertain", "confidence": 0.0-1.0, '
        f'"reason": "<short reason, up to 3 sentences>", '
        f'"missing": ["<brief gap, if any>"]}}'
    )
    return [{
        "role": "user",
        "content": [
            {"type": "text", "text": user},
            {"type": "image_url", "image_url": {"url": image_data_uri(image_path)}},
        ],
    }]


def judge(api_key, image_path, expect, model, timeout=60):
    payload = {
        "model": model,
        "messages": build_messages(image_path, expect),
        "temperature": 0,
    }
    headers = {
        "Authorization": f"Bearer {api_key}",
        "Content-Type": "application/json",
        # Identifies the caller to OpenRouter; HTTP-Referer/X-Title are optional.
        "HTTP-Referer": "https://github.com/zwitschi/gig-line",
        "X-Title": "gig-line screenshot judge",
    }
    resp = requests.post(API_URL, headers=headers, json=payload, timeout=timeout)
    if resp.status_code != 200:
        body = resp.text[:500]
        sys.exit(f"error: OpenRouter returned {resp.status_code}: {body}")
    data = resp.json()
    content = data["choices"][0]["message"]["content"]
    # Trim any markdown fences and take the first JSON object we can parse.
    m = re.search(r"\{.*\}", content, re.S)
    if not m:
        sys.exit(f"error: model returned non-JSON:\n{content[:500]}")
    return json.loads(m.group(0))


def parse_shots_from_targets():
    """Read the review shots and their expectations from Tools/visual_targets.json.

    Parsing the README's tables broke when the README was restructured into per-shot
    sections with the comparison table appended: the old parser looked for a table whose
    first cell held a backticked `review-` name. The targets file is the same source the
    render gate uses, so it is both stable and the thing the expectations are written to.
    """
    targets_path = os.path.join(PROJECT_ROOT, "Tools", "visual_targets.json")
    if not os.path.isfile(targets_path):
        sys.exit(f"error: no {targets_path}")
    import json
    with open(targets_path) as handle:
        data = json.load(handle)
    rows = []
    for target in data.get("targets", []):
        shot = target.get("shot") or ""
        expect = target.get("expect") or ""
        if not shot or not expect:
            continue
        rows.append((shot, expect))
    return rows


def main():
    ap = argparse.ArgumentParser(description=__doc__,
                                 formatter_class=argparse.RawDescriptionHelpFormatter)
    g = ap.add_mutually_exclusive_group(required=True)
    g.add_argument("--image", help="path to a screenshot PNG to judge")
    g.add_argument("--shots", action="store_true",
                   help="evaluate every review shot listed in Screenshots/README.md")
    ap.add_argument("--expect", help="what the shot is expected to show (required with --image)")
    ap.add_argument("--model", default=DEFAULT_MODEL, help=f"OpenRouter model (default {DEFAULT_MODEL})")
    ap.add_argument("--verbose", action="store_true")
    ap.add_argument("--timeout", type=int, default=60)
    args = ap.parse_args()

    api_key = load_api_key()

    if args.shots:
        shots = parse_shots_from_targets()
        if not shots:
            sys.exit("error: no shots found in Tools/visual_targets.json")
        n = len(shots)
        print(f"Judging {n} screenshots with {args.model} ...")
        ok = 0
        for i, (fname, expect) in enumerate(shots, 1):
            path = os.path.join(PROJECT_ROOT, fname)
            if not os.path.isfile(path):
                print(f"[{i}/{n}] {fname}: MISSING (skip)")
                continue
            try:
                res = judge(api_key, path, expect, args.model, args.timeout)
            except SystemExit as e:
                print(f"[{i}/{n}] {fname}: ERROR {e}")
                continue
            verdict = res.get("verdict", "unknown")
            conf = res.get("confidence", 0.0)
            reason = res.get("reason", "")
            mark = {"yes": "OK", "no": "FAIL", "uncertain": "?"}.get(verdict, "?")
            print(f"[{i}/{n}] {fname}: {mark} ({conf:.2f}) {reason}")
            if verdict == "yes":
                ok += 1
        print(f"\nSummary: {ok}/{n} judged yes")
        return

    if not args.expect:
        ap.error("--expect is required with --image")

    res = judge(api_key, args.image, args.expect, args.model, args.timeout)
    print(json.dumps(res, indent=2) if args.verbose
          else f"verdict={res.get('verdict')} confidence={res.get('confidence')} "
               f"reason={res.get('reason', '')}")


if __name__ == "__main__":
    main()