zwitschi
abde390045
A single vision model has been wrong in both directions on this project - it missed real defects and disputed frames that were correct - so agreement between different model families is worth more than one confident opinion. judge_screenshot.py now runs two models by default (openai/gpt-4o-mini + qwen/qwen3.7-flash, both served on OpenRouter; claude-3.5-haiku is gone, which the first attempt discovered with a 404). combine() reconciles them: agreement carries the verdict with the mean confidence, disagreement reports 'uncertain' and keeps both reasons rather than letting the more confident model win. --model still forces a single judgement, and every line prints which model said what. comfyui_generate.py generates the same prompt and seed with two checkpoints, scores both by high-frequency detail (mean absolute Laplacian, which catches blurry or flat results), keeps the loser as <out>__<model>.png with its sidecar, and moves the winner onto --out. --alt-model restores the single run. Tests cover the reconciliation - agreement, disagreement, unknown verdicts, and no verdicts at all - and the suite is at 103.
41 lines
1.5 KiB
Python
41 lines
1.5 KiB
Python
"""Tests for the multi-model reconciliation in judge_screenshot.combine()."""
|
|
|
|
import unittest
|
|
|
|
from judge_screenshot import combine
|
|
|
|
|
|
class CombineTests(unittest.TestCase):
|
|
def test_agreement_takes_the_verdict_and_mean_confidence(self):
|
|
result = combine([
|
|
{"model": "a", "verdict": "yes", "confidence": 0.8, "reason": "ok"},
|
|
{"model": "b", "verdict": "yes", "confidence": 0.6, "reason": "fine"},
|
|
])
|
|
self.assertEqual(result["verdict"], "yes")
|
|
self.assertAlmostEqual(result["confidence"], 0.7)
|
|
|
|
def test_disagreement_becomes_uncertain_and_keeps_both_reasons(self):
|
|
result = combine([
|
|
{"model": "a", "verdict": "yes", "confidence": 0.9, "reason": "looks right"},
|
|
{"model": "b", "verdict": "no", "confidence": 0.4, "reason": "missing banner"},
|
|
])
|
|
self.assertEqual(result["verdict"], "uncertain")
|
|
self.assertAlmostEqual(result["confidence"], 0.4)
|
|
self.assertIn("a=yes", result["reason"])
|
|
self.assertIn("b=no", result["reason"])
|
|
|
|
def test_unknown_verdicts_are_ignored(self):
|
|
result = combine([
|
|
{"model": "a", "verdict": "no", "confidence": 0.5, "reason": "x"},
|
|
{"model": "b", "verdict": "garbage", "confidence": 0.9, "reason": "y"},
|
|
])
|
|
self.assertEqual(result["verdict"], "no")
|
|
|
|
def test_no_usable_verdicts(self):
|
|
result = combine([{"model": "a", "verdict": "garbage"}])
|
|
self.assertEqual(result["verdict"], "unknown")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|