test(review): judge every shot from the targets file, and record that all nine fail
--shots was parsing a table shape the README no longer has, so it silently found nothing. It now reads Tools/visual_targets.json, the same file the render gate uses, and reports one line per shot with the model's verdict and reasoning. The result is not good news and is committed as such: 0 of 9 shots judged 'yes'. Shot 7 shows the scene itself wrong - a disassembled car floating over a blank surface with no street or buildings in frame - and the others fail for missing detail rather than for a broken frame. visual_loop --verify still reports 9/9, because it only measures exposure and size. A warning now sits at the top of Screenshots/README.md: treat every image there as stale until the scene is fixed. The cause is not yet identified - the map builder's recent changes are 15 lines of vehicle staging, which does not explain a missing street.
This commit is contained in:
1 parent
58095b6a33
commit
fd2ad4b391
38 files changed
+70
-69
No files matched your search
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -12,6 +12,8 @@ Tools/render_screenshots.sh --verify
|
||||
|
||||
Stages the demo shot cameras, runs the game on `/Game/Hollywood/DemoMap`, copies the PNGs into `Screenshots/demo/` and then runs [`Tools/visual_loop.py`](../Tools/visual_loop.py) `--verify` to check that each shot still shows the content its expectation describes. About a minute.
|
||||
|
||||
> **WARNING — the current capture set is broken (2026-11-09).** All nine shots were judged `no` by the vision model, and shot 7 shows the scene itself wrong: a disassembled car floating over a blank surface with no street or buildings in frame. `--verify` still reports 9/9 because it only measures exposure and size. Treat every image in this file as stale until this is fixed; the cause is not yet identified.
|
||||
|
||||
## Status at a glance
|
||||
|
||||
Every shot is embedded below, in order, so the whole review happens in this file.
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
+22
-23
@@ -112,26 +112,27 @@ def judge(api_key, image_path, expect, model, timeout=60):
|
||||
return json.loads(m.group(0))
|
||||
|
||||
|
||||
def parse_shots_from_readme():
|
||||
"""Parse the Screenshots/README.md tables into (path, expected) pairs."""
|
||||
readme = os.path.join(PROJECT_ROOT, "Screenshots", "README.md")
|
||||
if not os.path.isfile(readme):
|
||||
sys.exit(f"error: no {readme}")
|
||||
text = open(readme, encoding="utf-8").read()
|
||||
def parse_shots_from_targets():
|
||||
"""Read the review shots and their expectations from Tools/visual_targets.json.
|
||||
|
||||
Parsing the README's tables broke when the README was restructured into per-shot
|
||||
sections with the comparison table appended: the old parser looked for a table whose
|
||||
first cell held a backticked `review-` name. The targets file is the same source the
|
||||
render gate uses, so it is both stable and the thing the expectations are written to.
|
||||
"""
|
||||
targets_path = os.path.join(PROJECT_ROOT, "Tools", "visual_targets.json")
|
||||
if not os.path.isfile(targets_path):
|
||||
sys.exit(f"error: no {targets_path}")
|
||||
import json
|
||||
with open(targets_path) as handle:
|
||||
data = json.load(handle)
|
||||
rows = []
|
||||
# Each table row: | File | ... | ... | What it shows |
|
||||
for line in text.splitlines():
|
||||
if not line.strip().startswith("|"):
|
||||
for target in data.get("targets", []):
|
||||
shot = target.get("shot") or ""
|
||||
expect = target.get("expect") or ""
|
||||
if not shot or not expect:
|
||||
continue
|
||||
cells = [c.strip() for c in line.strip().strip("|").split("|")]
|
||||
if not cells or "`review-" not in cells[0]:
|
||||
continue
|
||||
fname = re.search(r"`(review-[^`]+\.png)`", cells[0])
|
||||
if not fname:
|
||||
continue
|
||||
# the last cell is the "What it shows" description
|
||||
expect = cells[-1]
|
||||
rows.append((fname.group(1), expect))
|
||||
rows.append((shot, expect))
|
||||
return rows
|
||||
|
||||
|
||||
@@ -151,16 +152,14 @@ def main():
|
||||
api_key = load_api_key()
|
||||
|
||||
if args.shots:
|
||||
shots = parse_shots_from_readme()
|
||||
shots = parse_shots_from_targets()
|
||||
if not shots:
|
||||
sys.exit("error: no review shots found in Screenshots/README.md")
|
||||
# Only review-map shots (skip the demo/ ones which have a preview column too).
|
||||
shots = [s for s in shots if "/demo/" not in s[0]]
|
||||
sys.exit("error: no shots found in Tools/visual_targets.json")
|
||||
n = len(shots)
|
||||
print(f"Judging {n} screenshots with {args.model} ...")
|
||||
ok = 0
|
||||
for i, (fname, expect) in enumerate(shots, 1):
|
||||
path = os.path.join(PROJECT_ROOT, "Screenshots", fname.replace("Screenshots/", ""))
|
||||
path = os.path.join(PROJECT_ROOT, fname)
|
||||
if not os.path.isfile(path):
|
||||
print(f"[{i}/{n}] {fname}: MISSING (skip)")
|
||||
continue
|
||||
|
||||
Reference in new issue
Block a user