From 43c17ce3c099febe49630a1420283cf8aa213b4a Mon Sep 17 00:00:00 2001 From: Zwitschi Date: Tue, 3 Nov 2026 17:12:18 +0100 Subject: [PATCH] Add the local-only DuckDuckGo reference collector --- MEMORY.md | 4 + TODO.md | 10 ++- tools/refcollect.py | 176 +++++++++++++++++++++++++++++++++++++++ tools/test_refcollect.py | 51 ++++++++++++ 4 files changed, 237 insertions(+), 4 deletions(-) create mode 100644 tools/refcollect.py create mode 100644 tools/test_refcollect.py diff --git a/MEMORY.md b/MEMORY.md index 643dfa0..c10e34e 100644 --- a/MEMORY.md +++ b/MEMORY.md @@ -4,6 +4,10 @@ Session memory for durable decisions, state, and gotchas. Keep concise, remove o ## Decisions +- **DuckDuckGo collection tool (2026-11-03):** `tools/refcollect.py` wraps the `duckduckgo_search` + package (installed locally; renamed `ddgs` upstream) and retries DDG's HTTP 202 rate limit with + backoff. DDG rate-limits by IP for a while after probing, so the per-venue collections should be + run in small spaced batches. Downloads are `local_only`/`unchecked-local` and never committed. - **Local-only reference override (2026-11-03, owner decision):** for development-only photo research the licence gate does **not** apply. Downloads for this track live in `tmp/research/` (gitignored) and may come from DuckDuckGo image search, street-view frames, neighbouring diff --git a/TODO.md b/TODO.md index 581762a..10b0475 100644 --- a/TODO.md +++ b/TODO.md @@ -41,10 +41,12 @@ never committed, shipped, or used as a committed asset. The moment a frame is pr `docs/screens/refs//` it must pass the sidecar licence + privacy gate; local research is the licence-free zone only. -- [ ] Add a DuckDuckGo image-search fetcher to the reference tooling (e.g. `tools/refboard.py - --source ddg` or a small `tools/refcollect.py`): query terms per venue, download the top - results into `tmp/research//` with a source-URL manifest, no licence filter, spaced - requests. This is local tooling only โ€” no CI dependency. +- [x] Add a DuckDuckGo image-search fetcher: `tools/refcollect.py search --slug + --limit N` downloads top results into `tmp/research//` with a source-URL manifest + (no licence filter, marked `local_only`/`unchecked-local`), and `sheet` composes a local + contact sheet. Retries DDG's HTTP 202 rate limit with backoff. Tests in + `tools/test_refcollect.py`. **Run note:** DDG rate-limits aggressively; space batches and + run the per-venue collection below when the limit clears. - [ ] Collect Hollywood block reference: El Capitan facade (day/night/close-up), the El Capitan Theatre marquee + blade sign neighbour, the east retail neighbour, Ovation Hollywood and other buildings across the boulevard, the Walk of Fame terrazzo and stars, and the premiere diff --git a/tools/refcollect.py b/tools/refcollect.py new file mode 100644 index 0000000..326077c --- /dev/null +++ b/tools/refcollect.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +"""Local-only reference collector โ€” licence-free, gitignored research material. + +Downloads images from DuckDuckGo image search into `tmp/research//` for +development reference only. This tool deliberately applies **no licence gate**: +everything it fetches stays under the gitignored `tmp/` tree and must never be +committed, shipped, or promoted into `docs/screens/refs/` without passing the +normal sidecar licence + privacy gate. The owner authorised this local-only +override on 2026-11-03 (see TODO.md ยง3D model re-evaluation โ†’ A). + + python3 tools/refcollect.py search "El Capitan Entertainment Centre facade" \ + --slug elcapitan --limit 20 + python3 tools/refcollect.py sheet --slug elcapitan + +Requires the `duckduckgo_search` package (pip install duckduckgo-search, or the +renamed `ddgs`). DuckDuckGo rate-limits aggressive queries (HTTP 202): requests +are spaced and retried with backoff, so a large batch can take a while. +""" + +import argparse +import json +import re +import sys +import time +import urllib.error +import urllib.request +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +RESEARCH_DIR = ROOT / "tmp" / "research" +UA = "Mozilla/5.0 (X11; Linux x86_64; rv:120.0) Gecko/20100101 Firefox/120.0" +DOWNLOAD_SPACING_S = 2.0 + + +def _rel(path: Path) -> str: + return str(path.relative_to(ROOT)) if path.is_relative_to(ROOT) else str(path) + + +def _slug(text: str) -> str: + return re.sub(r"[^a-z0-9]+", "-", text.strip().lower()).strip("-") or "block" + + +def _search_images(query: str, limit: int) -> list[dict]: + """Return DDG image results as small dicts; retries on 202 rate limits.""" + try: + from duckduckgo_search import DDGS # legacy import name + except ImportError: + try: + from ddgs import DDGS # current package name + except ImportError as exc: + sys.exit("error: install the DuckDuckGo search package first: " + "pip install duckduckgo-search") + last_exc = None + for attempt in range(5): + try: + with DDGS() as ddgs: + rows = ddgs.images(query, max_results=limit) + return [{ + "title": (r.get("title") or "")[:200], + "image": r.get("image") or "", + "thumbnail": r.get("thumbnail") or "", + "source": r.get("url") or "", + "width": r.get("width") or 0, + "height": r.get("height") or 0, + } for r in rows if r.get("image")] + except Exception as exc: # noqa: BLE001 - package raises several types + last_exc = exc + wait = 8 * (attempt + 1) + print(f" DDG rate limit or error ({exc.__class__.__name__}); " + f"retrying in {wait}s ...", file=sys.stderr) + time.sleep(wait) + sys.exit(f"error: DuckDuckGo image search kept failing: {last_exc}") + + +def _download(url: str, path: Path) -> None: + req = urllib.request.Request(url, headers={"User-Agent": UA}) + with urllib.request.urlopen(req, timeout=60) as resp: + path.write_bytes(resp.read()) + + +def _manifest_rows(rows: list[dict]) -> list[dict]: + out = [] + for row in rows: + out.append({**row, "local_only": True, "licence": "unchecked-local"}) + return out + + +def cmd_search(args) -> int: + slug = _slug(args.slug or args.query) + out_dir = RESEARCH_DIR / slug + out_dir.mkdir(parents=True, exist_ok=True) + print(f"searching DDG: {args.query!r} (limit {args.limit})", flush=True) + rows = _search_images(args.query, args.limit) + manifest = [] + for index, row in enumerate(rows, start=1): + stem = f"{index:02d}" + suffix = Path(row["image"].split("?")[0]).suffix.lower() or ".jpg" + if suffix not in (".jpg", ".jpeg", ".png", ".webp", ".gif"): + suffix = ".jpg" + path = out_dir / f"{stem}{suffix}" + if path.exists() and not args.force: + print(f" skip existing {path.name}") + else: + try: + _download(row["image"], path) + print(f" {path.name} <- {row['image'][:70]}") + except Exception as exc: # noqa: BLE001 - some hosts block hotlinking + print(f" FAILED {path.name}: {exc.__class__.__name__}", file=sys.stderr) + continue + time.sleep(DOWNLOAD_SPACING_S) + manifest.append({**row, "file": path.name}) + (out_dir / "manifest.json").write_text( + json.dumps(_manifest_rows(manifest), indent=2) + "\n") + print(f"manifest -> {out_dir / 'manifest.json'} ({len(manifest)} entries)") + return 0 + + +def cmd_sheet(args) -> int: + """Compose a local contact sheet (never committed) from a manifest folder.""" + from PIL import Image + slug = _slug(args.slug) + out_dir = RESEARCH_DIR / slug + manifest_path = out_dir / "manifest.json" + if not manifest_path.is_file(): + sys.exit(f"error: no manifest at {manifest_path}; run `search` first") + rows = json.loads(manifest_path.read_text()) + images = [] + for row in rows: + path = out_dir / row["file"] + if path.is_file(): + with Image.open(path) as img: + img.load() + images.append(img.convert("RGB")) + if not images: + sys.exit("error: no downloaded frames in this folder") + cell = 320 + cols = 4 + rows_count = (len(images) + cols - 1) // cols + sheet = Image.new("RGB", (cols * cell, rows_count * cell), (18, 18, 18)) + for i, img in enumerate(images): + thumb = img.copy() + thumb.thumbnail((cell, cell), Image.Resampling.LANCZOS) + x = (i % cols) * cell + y = (i // cols) * cell + sheet.paste(thumb, (x, y)) + out = out_dir / "sheet.jpg" + sheet.save(out, quality=82) + print(f"wrote {_rel(out)} {sheet.size[0]}x{sheet.size[1]} " + f"({len(images)} frames)") + return 0 + + +def build_parser(): + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + sub = parser.add_subparsers(dest="cmd", required=True) + search = sub.add_parser("search", help="DDG image search into tmp/research//") + search.add_argument("query") + search.add_argument("--slug", default="", help="folder slug (default: from query)") + search.add_argument("--limit", type=int, default=20) + search.add_argument("--force", action="store_true") + search.set_defaults(handler=cmd_search) + sheet = sub.add_parser("sheet", help="compose a local contact sheet") + sheet.add_argument("--slug", required=True) + sheet.set_defaults(handler=cmd_sheet) + return parser + + +def main(argv=None) -> int: + args = build_parser().parse_args(argv) + return args.handler(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/test_refcollect.py b/tools/test_refcollect.py new file mode 100644 index 0000000..cb530eb --- /dev/null +++ b/tools/test_refcollect.py @@ -0,0 +1,51 @@ +"""Tests for the local-only reference collector (no network).""" + +import json +import unittest +from pathlib import Path +from tempfile import TemporaryDirectory +from unittest import mock + +import refcollect as rc + + +class SlugTests(unittest.TestCase): + def test_slug_folds_and_strips(self): + self.assertEqual(rc._slug("El Capitan Entertainment Centre"), + "el-capitan-entertainment-centre") + self.assertEqual(rc._slug(" !!! "), "block") + + +class ManifestTests(unittest.TestCase): + def test_manifest_rows_are_marked_local_only_and_never_licenced(self): + rows = [{"title": "t", "image": "https://x/i.jpg", "thumbnail": "", + "source": "https://x/", "width": 640, "height": 480}] + out = rc._manifest_rows(rows) + self.assertTrue(out[0]["local_only"]) + self.assertEqual(out[0]["licence"], "unchecked-local") + + +class SheetTests(unittest.TestCase): + def test_sheet_requires_a_manifest(self): + with TemporaryDirectory() as tmp: + with mock.patch.object(rc, "RESEARCH_DIR", Path(tmp)): + with self.assertRaises(SystemExit): + rc.cmd_sheet(mock.Mock(slug="nope")) + + def test_sheet_composes_from_downloaded_frames(self): + from PIL import Image + with TemporaryDirectory() as tmp: + root = Path(tmp) + slug_dir = root / "block" + slug_dir.mkdir() + for i in range(3): + Image.new("RGB", (640, 400), (i * 60, 0, 0)).save(slug_dir / f"{i:02d}.jpg") + (slug_dir / "manifest.json").write_text(json.dumps([ + {"file": "00.jpg"}, {"file": "01.jpg"}, {"file": "02.jpg"}])) + with mock.patch.object(rc, "RESEARCH_DIR", root): + rc.cmd_sheet(mock.Mock(slug="block")) + self.assertTrue((slug_dir / "sheet.jpg").is_file()) + + +if __name__ == "__main__": + unittest.main()