test(review): judge every shot from the targets file, and record that all nine fail

--shots was parsing a table shape the README no longer has, so it silently found nothing.
It now reads Tools/visual_targets.json, the same file the render gate uses, and reports one
line per shot with the model's verdict and reasoning.

The result is not good news and is committed as such: 0 of 9 shots judged 'yes'. Shot 7 shows
the scene itself wrong - a disassembled car floating over a blank surface with no street or
buildings in frame - and the others fail for missing detail rather than for a broken frame.
visual_loop --verify still reports 9/9, because it only measures exposure and size.

A warning now sits at the top of Screenshots/README.md: treat every image there as stale until
the scene is fixed. The cause is not yet identified - the map builder's recent changes are 15
lines of vehicle staging, which does not explain a missing street.
This commit is contained in:
zwitschi committed 2026-11-09 16:55:59 +01:00
1 parent 58095b6a33
commit fd2ad4b391
38 files changed
+70 -69

No files matched your search

+22 -23
View File
@@ -112,26 +112,27 @@ def judge(api_key, image_path, expect, model, timeout=60):
return json.loads(m.group(0))
def parse_shots_from_readme():
"""Parse the Screenshots/README.md tables into (path, expected) pairs."""
readme = os.path.join(PROJECT_ROOT, "Screenshots", "README.md")
if not os.path.isfile(readme):
sys.exit(f"error: no {readme}")
text = open(readme, encoding="utf-8").read()
def parse_shots_from_targets():
"""Read the review shots and their expectations from Tools/visual_targets.json.
Parsing the README's tables broke when the README was restructured into per-shot
sections with the comparison table appended: the old parser looked for a table whose
first cell held a backticked `review-` name. The targets file is the same source the
render gate uses, so it is both stable and the thing the expectations are written to.
"""
targets_path = os.path.join(PROJECT_ROOT, "Tools", "visual_targets.json")
if not os.path.isfile(targets_path):
sys.exit(f"error: no {targets_path}")
import json
with open(targets_path) as handle:
data = json.load(handle)
rows = []
# Each table row: | File | ... | ... | What it shows |
for line in text.splitlines():
if not line.strip().startswith("|"):
for target in data.get("targets", []):
shot = target.get("shot") or ""
expect = target.get("expect") or ""
if not shot or not expect:
continue
cells = [c.strip() for c in line.strip().strip("|").split("|")]
if not cells or "`review-" not in cells[0]:
continue
fname = re.search(r"`(review-[^`]+\.png)`", cells[0])
if not fname:
continue
# the last cell is the "What it shows" description
expect = cells[-1]
rows.append((fname.group(1), expect))
rows.append((shot, expect))
return rows
@@ -151,16 +152,14 @@ def main():
api_key = load_api_key()
if args.shots:
shots = parse_shots_from_readme()
shots = parse_shots_from_targets()
if not shots:
sys.exit("error: no review shots found in Screenshots/README.md")
# Only review-map shots (skip the demo/ ones which have a preview column too).
shots = [s for s in shots if "/demo/" not in s[0]]
sys.exit("error: no shots found in Tools/visual_targets.json")
n = len(shots)
print(f"Judging {n} screenshots with {args.model} ...")
ok = 0
for i, (fname, expect) in enumerate(shots, 1):
path = os.path.join(PROJECT_ROOT, "Screenshots", fname.replace("Screenshots/", ""))
path = os.path.join(PROJECT_ROOT, fname)
if not os.path.isfile(path):
print(f"[{i}/{n}] {fname}: MISSING (skip)")
continue