refactor(tools): group the sixty scripts by what they talk to
One directory with sixty scripts stopped being navigable, and the grouping is also the sandbox boundary:
Tools/unreal/ commandlets and the two map builders, with ue_common.py and ue_placement.py as the
shared imports (they need the engine)
Tools/blender/ builders, exporters, the FBX preset, the MPFB character pipeline, blender_screenshots
(they need the Blender Flatpak)
Tools/review/ the visual loop: render_screenshots.sh, judge_screenshot.py, hero_compare.py,
build_comparison_md.py, e2e_asset_check.py, hero_iterate.sh, visual_loop.py,
visual_targets.json
Tools/gen/ image and texture generation: the ComfyUI driver, deterministic texture writers,
reference fetching
Tools/input/ OS-level input injection
root _common.py, check-lfs.sh, addon_compat_probe.py, sibling_asset_audit.py, tests/
Every moved script now resolves the repository root as ../.. and puts the tools root plus its siblings on
sys.path, so 'import ue_common' still works from unreal/. The test suite bootstrap does the same for the five
folders. render_screenshots.sh takes the folder as a third argument to run_cmdlet, and the e2e driver,
hero_iterate.sh and the comparison generator were repointed.
51 README rows now link to their new location, the README gains a Layout section naming each folder's
sandbox, and the architecture notes and Blender README follow the moves.
Verified: 108 tests pass, prettier is clean, judge_screenshot.py and hero_compare.py run from their new
homes, and the e2e run was aborted mid-way (it had been rebuilding an empty DemoMap for 15 minutes) and is
being rerun against the new layout.
This commit is contained in:
1 parent
f681d83736
commit
4d27109668
168 files changed
+799
-407
No files matched your search
Executable
+238
@@ -0,0 +1,238 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Judge a screenshot against an expected-scene description using OpenRouter
|
||||
multimodal image understanding.
|
||||
|
||||
Reads OPENROUTER_API_KEY from the project .env (ignored, never committed) and
|
||||
sends the screenshot as a base64 data-URI image alongside a description of what
|
||||
the shot is expected to show. Returns a verdict (yes / no / uncertain), a
|
||||
confidence, and the model's reasoning, so a screenshot can be pre-checked before
|
||||
a human reviews it.
|
||||
|
||||
API: OpenRouter chat/completions, OpenAI-compatible image_url content part:
|
||||
{"type": "image_url", "image_url": {"url": "data:image/png;base64,<...>"}}
|
||||
Docs: https://openrouter.ai/docs/guides/overview/multimodal/image-understanding
|
||||
|
||||
Usage:
|
||||
judge_screenshot.py --image Screenshots/review-overview.png \
|
||||
--expect "a character and two 1 m cubes on a large grey floor"
|
||||
judge_screenshot.py --image Screenshots/review-character.png \
|
||||
--expect "the idle upright human figure with warm skin, feet on the plane" \
|
||||
--model openai/gpt-4o-mini --verbose
|
||||
judge_screenshot.py --shots # evaluate every review-NN shot in README
|
||||
|
||||
.env format:
|
||||
OPENROUTER_API_KEY=sk-or-v1-...
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
import requests
|
||||
|
||||
API_URL = "https://openrouter.ai/api/v1/chat/completions"
|
||||
DEFAULT_MODEL = "openai/gpt-4o-mini"
|
||||
# Two models by default. A single vision model is advisory and has been wrong in both directions on this
|
||||
# project, so every image is judged twice by different families and the pair is reconciled below.
|
||||
DEFAULT_MODELS = (
|
||||
"openai/gpt-4o-mini",
|
||||
"qwen/qwen3.7-flash",
|
||||
)
|
||||
|
||||
|
||||
def combine(judgements):
|
||||
"""Reconcile two or more model judgements into one verdict.
|
||||
|
||||
Agreement carries the verdict and the mean confidence. Disagreement is reported as `uncertain` with
|
||||
both reasons kept, because picking the more confident model would hide exactly the case worth looking
|
||||
at - the models seeing different things.
|
||||
"""
|
||||
usable = [j for j in judgements if j.get("verdict") in ("yes", "no", "uncertain")]
|
||||
if not usable:
|
||||
return {"verdict": "unknown", "confidence": 0.0,
|
||||
"reason": "no usable verdicts", "models": {}}
|
||||
verdicts = {j["verdict"] for j in usable}
|
||||
by_model = {j.get("model", "?"): {"verdict": j.get("verdict"),
|
||||
"confidence": j.get("confidence", 0.0),
|
||||
"reason": j.get("reason", "")} for j in usable}
|
||||
if len(verdicts) == 1:
|
||||
verdict = verdicts.pop()
|
||||
confidence = sum(j.get("confidence", 0.0) for j in usable) / len(usable)
|
||||
reason = usable[0].get("reason", "")
|
||||
else:
|
||||
verdict = "uncertain"
|
||||
confidence = min(j.get("confidence", 0.0) for j in usable)
|
||||
reason = "models disagree: " + " | ".join(
|
||||
f"{j.get('model')}={j.get('verdict')}: {j.get('reason', '')[:120]}" for j in usable)
|
||||
return {"verdict": verdict, "confidence": confidence, "reason": reason, "models": by_model}
|
||||
|
||||
# Project root = parent of the Tools/ directory the script lives in.
|
||||
PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
|
||||
ENV_PATH = os.path.join(PROJECT_ROOT, ".env")
|
||||
|
||||
|
||||
def load_api_key():
|
||||
"""Read OPENROUTER_API_KEY from the project .env."""
|
||||
if not os.path.exists(ENV_PATH):
|
||||
sys.exit(f"error: no {ENV_PATH} — create it with OPENROUTER_API_KEY=sk-or-v1-...")
|
||||
with open(ENV_PATH, encoding="utf-8") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if line and not line.startswith("#"):
|
||||
key, _, val = line.partition("=")
|
||||
if key.strip() == "OPENROUTER_API_KEY":
|
||||
return val.strip()
|
||||
sys.exit("error: OPENROUTER_API_KEY not found in .env")
|
||||
|
||||
|
||||
def image_data_uri(path):
|
||||
"""Return a base64 data: URI for a local image file."""
|
||||
if not os.path.isfile(path):
|
||||
sys.exit(f"error: image not found: {path}")
|
||||
ext = os.path.splitext(path)[1].lower().lstrip(".") or "png"
|
||||
mime = {"png": "image/png", "jpg": "image/jpeg", "jpeg": "image/jpeg",
|
||||
"webp": "image/webp", "gif": "image/gif"}.get(ext, "image/png")
|
||||
with open(path, "rb") as fh:
|
||||
b64 = base64.b64encode(fh.read()).decode("ascii")
|
||||
return f"data:{mime};base64,{b64}"
|
||||
|
||||
|
||||
def build_messages(image_path, expect):
|
||||
user = (
|
||||
f"You are an objective QA reviewer for a 3D game scene. Look at the "
|
||||
f"rendered screenshot and judge whether it shows what is expected.\n\n"
|
||||
f"Expected scene:\n{expect}\n\n"
|
||||
f"Answer STRICTLY in this JSON shape, with no other text:\n"
|
||||
f'{{"verdict": "yes"|"no"|"uncertain", "confidence": 0.0-1.0, '
|
||||
f'"reason": "<short reason, up to 3 sentences>", '
|
||||
f'"missing": ["<brief gap, if any>"]}}'
|
||||
)
|
||||
return [{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": user},
|
||||
{"type": "image_url", "image_url": {"url": image_data_uri(image_path)}},
|
||||
],
|
||||
}]
|
||||
|
||||
|
||||
def judge(api_key, image_path, expect, model, timeout=60):
|
||||
"""One model's judgement. `combine()` reconciles several of these."""
|
||||
payload = {
|
||||
"model": model,
|
||||
"messages": build_messages(image_path, expect),
|
||||
"temperature": 0,
|
||||
}
|
||||
headers = {
|
||||
"Authorization": f"Bearer {api_key}",
|
||||
"Content-Type": "application/json",
|
||||
# Identifies the caller to OpenRouter; HTTP-Referer/X-Title are optional.
|
||||
"HTTP-Referer": "https://github.com/zwitschi/gig-line",
|
||||
"X-Title": "gig-line screenshot judge",
|
||||
}
|
||||
resp = requests.post(API_URL, headers=headers, json=payload, timeout=timeout)
|
||||
if resp.status_code != 200:
|
||||
body = resp.text[:500]
|
||||
sys.exit(f"error: OpenRouter returned {resp.status_code}: {body}")
|
||||
data = resp.json()
|
||||
content = data["choices"][0]["message"]["content"]
|
||||
# Trim any markdown fences and take the first JSON object we can parse.
|
||||
m = re.search(r"\{.*\}", content, re.S)
|
||||
if not m:
|
||||
sys.exit(f"error: model returned non-JSON:\n{content[:500]}")
|
||||
verdict = json.loads(m.group(0))
|
||||
verdict["model"] = model
|
||||
return verdict
|
||||
|
||||
|
||||
def parse_shots_from_targets():
|
||||
"""Read the review shots and their expectations from Tools/review/visual_targets.json.
|
||||
|
||||
Parsing the README's tables broke when the README was restructured into per-shot
|
||||
sections with the comparison table appended: the old parser looked for a table whose
|
||||
first cell held a backticked `review-` name. The targets file is the same source the
|
||||
render gate uses, so it is both stable and the thing the expectations are written to.
|
||||
"""
|
||||
targets_path = os.path.join(PROJECT_ROOT, "Tools", "review", "visual_targets.json")
|
||||
if not os.path.isfile(targets_path):
|
||||
sys.exit(f"error: no {targets_path}")
|
||||
import json
|
||||
with open(targets_path) as handle:
|
||||
data = json.load(handle)
|
||||
rows = []
|
||||
for target in data.get("targets", []):
|
||||
shot = target.get("shot") or ""
|
||||
expect = target.get("expect") or ""
|
||||
if not shot or not expect:
|
||||
continue
|
||||
rows.append((shot, expect))
|
||||
return rows
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
g = ap.add_mutually_exclusive_group(required=True)
|
||||
g.add_argument("--image", help="path to a screenshot PNG to judge")
|
||||
g.add_argument("--shots", action="store_true",
|
||||
help="evaluate every review shot listed in Screenshots/README.md")
|
||||
ap.add_argument("--expect", help="what the shot is expected to show (required with --image)")
|
||||
ap.add_argument("--models", default=",".join(DEFAULT_MODELS),
|
||||
help=f"comma-separated OpenRouter models; every image is judged by all of them "
|
||||
f"and the results reconciled (default {','.join(DEFAULT_MODELS)})")
|
||||
ap.add_argument("--model", default=None,
|
||||
help="single model, overriding --models (one judgement, no reconciliation)")
|
||||
ap.add_argument("--verbose", action="store_true")
|
||||
ap.add_argument("--timeout", type=int, default=60)
|
||||
args = ap.parse_args()
|
||||
|
||||
api_key = load_api_key()
|
||||
models = [args.model] if args.model else [m.strip() for m in args.models.split(",") if m.strip()]
|
||||
|
||||
if args.shots:
|
||||
shots = parse_shots_from_targets()
|
||||
if not shots:
|
||||
sys.exit("error: no shots found in Tools/review/visual_targets.json")
|
||||
n = len(shots)
|
||||
print(f"Judging {n} screenshots with {' + '.join(models)} ...")
|
||||
ok = 0
|
||||
for i, (fname, expect) in enumerate(shots, 1):
|
||||
path = os.path.join(PROJECT_ROOT, fname)
|
||||
if not os.path.isfile(path):
|
||||
print(f"[{i}/{n}] {fname}: MISSING (skip)")
|
||||
continue
|
||||
try:
|
||||
judged = [judge(api_key, path, expect, model, args.timeout) for model in models]
|
||||
except SystemExit as e:
|
||||
print(f"[{i}/{n}] {fname}: ERROR {e}")
|
||||
continue
|
||||
res = combine(judged) if len(judged) > 1 else judged[0]
|
||||
verdict = res.get("verdict", "unknown")
|
||||
conf = res.get("confidence", 0.0)
|
||||
reason = res.get("reason", "")
|
||||
mark = {"yes": "OK", "no": "FAIL", "uncertain": "?"}.get(verdict, "?")
|
||||
print(f"[{i}/{n}] {fname}: {mark} ({conf:.2f}) {reason}")
|
||||
if verdict == "yes":
|
||||
ok += 1
|
||||
print(f"\nSummary: {ok}/{n} judged yes")
|
||||
return
|
||||
|
||||
if not args.expect:
|
||||
ap.error("--expect is required with --image")
|
||||
|
||||
judged = [judge(api_key, args.image, args.expect, model, args.timeout) for model in models]
|
||||
res = combine(judged) if len(judged) > 1 else judged[0]
|
||||
if args.verbose:
|
||||
print(json.dumps(res, indent=2))
|
||||
else:
|
||||
print(f"verdict={res.get('verdict')} confidence={res.get('confidence')} "
|
||||
f"reason={res.get('reason', '')}")
|
||||
for model, detail in res.get("models", {}).items():
|
||||
print(f" {model}: {detail['verdict']} ({detail['confidence']:.2f})")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user