Add a vision-model screenshot judge to the QA tooling
`tools/screen_designs.py judge` sends each capture together with its concept mockup to a vision model and asks whether the capture realises the same screen and layout intent. This covers what pixel/perceptual metrics cannot judge. - Prompt explicitly ignores art style, fidelity, fonts, and placeholder text, since the references are stylised mockups. - Verdicts are classed match / review (mismatch below --fail-confidence, default 0.6) / diff (confident mismatch, exits non-zero) / unknown; --json, --only, --dry-run, --model supported. - Reads the repo .env for OPENROUTER_API_KEY without overriding real env vars. - Default judge model google/gemini-2.5-flash-lite; openai/gpt-4o-mini verified too. Tests/docs - 7 new offline tests in tools/test_screen_designs.py (request payload embeds both images, JSON/fenced/parts/non-JSON parsing, threshold classification, .env loading); 30 Python tests pass. - Live run: 3 captures match, 3 review, 0 confident diffs. - README, TODO section moved to DONE, MEMORY. Version 0.2.15.
This commit is contained in:
1 parent
74f14014b2
commit
72b9378e69
7 files changed
+347
-21
No files matched your search
@@ -86,5 +86,97 @@ class ScreenDesignsTests(unittest.TestCase):
|
||||
self.assertTrue(any("aspect" in e for e in report["errors"]))
|
||||
|
||||
|
||||
class VisionJudgeTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp_dir = tempfile.TemporaryDirectory()
|
||||
self.root = Path(self.temp_dir.name)
|
||||
self.capture = self.root / "capture.png"
|
||||
self.design = self.root / "design.png"
|
||||
Image.new("RGB", (64, 36), (20, 30, 40)).save(self.capture)
|
||||
Image.new("RGB", (64, 36), (200, 190, 180)).save(self.design)
|
||||
self.screen = screen_designs.SCREENS[0]
|
||||
|
||||
def tearDown(self):
|
||||
self.temp_dir.cleanup()
|
||||
|
||||
def test_judge_request_embeds_both_images(self):
|
||||
body = screen_designs.build_judge_request(
|
||||
"model-x", self.screen, self.capture, self.design)
|
||||
self.assertEqual(body["model"], "model-x")
|
||||
self.assertEqual(body["temperature"], 0)
|
||||
self.assertEqual(body["response_format"], {"type": "json_object"})
|
||||
parts = body["messages"][0]["content"]
|
||||
self.assertEqual(parts[0]["type"], "text")
|
||||
images = [p for p in parts if p["type"] == "image_url"]
|
||||
self.assertEqual(len(images), 2, "design and capture are both attached")
|
||||
for image in images:
|
||||
self.assertTrue(image["image_url"]["url"].startswith(
|
||||
"data:image/png;base64,"))
|
||||
self.assertIn(self.screen["title"], parts[0]["text"])
|
||||
self.assertIn("mockup", parts[0]["text"].lower())
|
||||
|
||||
def test_parse_plain_json_verdict(self):
|
||||
payload = {"choices": [{"message": {"content":
|
||||
'{"match": true, "confidence": 0.9, "differences": ["style"]}'}}]}
|
||||
verdict = screen_designs.parse_judge_response(payload)
|
||||
self.assertTrue(verdict["match"])
|
||||
self.assertAlmostEqual(verdict["confidence"], 0.9)
|
||||
self.assertEqual(verdict["differences"], ["style"])
|
||||
|
||||
def test_parse_fenced_json_verdict(self):
|
||||
payload = {"choices": [{"message": {"content":
|
||||
'Sure:\n```json\n{"match": false, "confidence": 0.7}\n```'}}]}
|
||||
verdict = screen_designs.parse_judge_response(payload)
|
||||
self.assertFalse(verdict["match"])
|
||||
self.assertAlmostEqual(verdict["confidence"], 0.7)
|
||||
|
||||
def test_parse_content_parts(self):
|
||||
payload = {"choices": [{"message": {"content": [
|
||||
{"type": "text", "text": '{"match": true, "confidence": 0.5}'}]}}]}
|
||||
verdict = screen_designs.parse_judge_response(payload)
|
||||
self.assertTrue(verdict["match"])
|
||||
|
||||
def test_parse_non_json_is_unknown(self):
|
||||
payload = {"choices": [{"message": {"content": "no idea"}}]}
|
||||
verdict = screen_designs.parse_judge_response(payload)
|
||||
self.assertIsNone(verdict["match"])
|
||||
self.assertTrue(verdict["differences"])
|
||||
|
||||
def test_parse_missing_choices_is_unknown(self):
|
||||
verdict = screen_designs.parse_judge_response({})
|
||||
self.assertIsNone(verdict["match"])
|
||||
|
||||
def test_classify_thresholds(self):
|
||||
self.assertEqual(screen_designs.classify(
|
||||
{"match": True, "confidence": 0.1}), "match")
|
||||
self.assertEqual(screen_designs.classify(
|
||||
{"match": False, "confidence": 0.9}), "bad")
|
||||
self.assertEqual(screen_designs.classify(
|
||||
{"match": False, "confidence": 0.3}), "review")
|
||||
self.assertEqual(screen_designs.classify(
|
||||
{"match": None, "confidence": 0.0}), "unknown")
|
||||
self.assertEqual(screen_designs.classify(
|
||||
{"match": False, "confidence": 0.55}, threshold=0.5), "bad")
|
||||
|
||||
def test_env_file_does_not_override_existing(self):
|
||||
env_path = self.root / ".env"
|
||||
env_path.write_text('OPENROUTER_API_KEY=from-file\nOTHER=1\n',
|
||||
encoding="utf-8")
|
||||
import os
|
||||
old = os.environ.get("OPENROUTER_API_KEY")
|
||||
os.environ["OPENROUTER_API_KEY"] = "from-env"
|
||||
os.environ.pop("OTHER", None)
|
||||
try:
|
||||
screen_designs._load_env_file(env_path)
|
||||
self.assertEqual(os.environ["OPENROUTER_API_KEY"], "from-env")
|
||||
self.assertEqual(os.environ.get("OTHER"), "1")
|
||||
finally:
|
||||
if old is None:
|
||||
os.environ.pop("OPENROUTER_API_KEY", None)
|
||||
else:
|
||||
os.environ["OPENROUTER_API_KEY"] = old
|
||||
os.environ.pop("OTHER", None)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user