test(review): judge every shot from the targets file, and record that all nine fail
--shots was parsing a table shape the README no longer has, so it silently found nothing. It now reads Tools/visual_targets.json, the same file the render gate uses, and reports one line per shot with the model's verdict and reasoning. The result is not good news and is committed as such: 0 of 9 shots judged 'yes'. Shot 7 shows the scene itself wrong - a disassembled car floating over a blank surface with no street or buildings in frame - and the others fail for missing detail rather than for a broken frame. visual_loop --verify still reports 9/9, because it only measures exposure and size. A warning now sits at the top of Screenshots/README.md: treat every image there as stale until the scene is fixed. The cause is not yet identified - the map builder's recent changes are 15 lines of vehicle staging, which does not explain a missing street.
This commit is contained in:
1 parent
58095b6a33
commit
fd2ad4b391
38 files changed
+70
-69
No files matched your search
+22
-23
@@ -112,26 +112,27 @@ def judge(api_key, image_path, expect, model, timeout=60):
|
||||
return json.loads(m.group(0))
|
||||
|
||||
|
||||
def parse_shots_from_readme():
|
||||
"""Parse the Screenshots/README.md tables into (path, expected) pairs."""
|
||||
readme = os.path.join(PROJECT_ROOT, "Screenshots", "README.md")
|
||||
if not os.path.isfile(readme):
|
||||
sys.exit(f"error: no {readme}")
|
||||
text = open(readme, encoding="utf-8").read()
|
||||
def parse_shots_from_targets():
|
||||
"""Read the review shots and their expectations from Tools/visual_targets.json.
|
||||
|
||||
Parsing the README's tables broke when the README was restructured into per-shot
|
||||
sections with the comparison table appended: the old parser looked for a table whose
|
||||
first cell held a backticked `review-` name. The targets file is the same source the
|
||||
render gate uses, so it is both stable and the thing the expectations are written to.
|
||||
"""
|
||||
targets_path = os.path.join(PROJECT_ROOT, "Tools", "visual_targets.json")
|
||||
if not os.path.isfile(targets_path):
|
||||
sys.exit(f"error: no {targets_path}")
|
||||
import json
|
||||
with open(targets_path) as handle:
|
||||
data = json.load(handle)
|
||||
rows = []
|
||||
# Each table row: | File | ... | ... | What it shows |
|
||||
for line in text.splitlines():
|
||||
if not line.strip().startswith("|"):
|
||||
for target in data.get("targets", []):
|
||||
shot = target.get("shot") or ""
|
||||
expect = target.get("expect") or ""
|
||||
if not shot or not expect:
|
||||
continue
|
||||
cells = [c.strip() for c in line.strip().strip("|").split("|")]
|
||||
if not cells or "`review-" not in cells[0]:
|
||||
continue
|
||||
fname = re.search(r"`(review-[^`]+\.png)`", cells[0])
|
||||
if not fname:
|
||||
continue
|
||||
# the last cell is the "What it shows" description
|
||||
expect = cells[-1]
|
||||
rows.append((fname.group(1), expect))
|
||||
rows.append((shot, expect))
|
||||
return rows
|
||||
|
||||
|
||||
@@ -151,16 +152,14 @@ def main():
|
||||
api_key = load_api_key()
|
||||
|
||||
if args.shots:
|
||||
shots = parse_shots_from_readme()
|
||||
shots = parse_shots_from_targets()
|
||||
if not shots:
|
||||
sys.exit("error: no review shots found in Screenshots/README.md")
|
||||
# Only review-map shots (skip the demo/ ones which have a preview column too).
|
||||
shots = [s for s in shots if "/demo/" not in s[0]]
|
||||
sys.exit("error: no shots found in Tools/visual_targets.json")
|
||||
n = len(shots)
|
||||
print(f"Judging {n} screenshots with {args.model} ...")
|
||||
ok = 0
|
||||
for i, (fname, expect) in enumerate(shots, 1):
|
||||
path = os.path.join(PROJECT_ROOT, "Screenshots", fname.replace("Screenshots/", ""))
|
||||
path = os.path.join(PROJECT_ROOT, fname)
|
||||
if not os.path.isfile(path):
|
||||
print(f"[{i}/{n}] {fname}: MISSING (skip)")
|
||||
continue
|
||||
|
||||
Reference in new issue
Block a user