refs: Flickr + web search for the hero entrance
common_fetch was out of material, so an OpenRouter web-search call (gpt-5.2, openrouter:web_search) returned candidate photo pages and page_image_fetch.py resolved each og:image and downloaded it, recording page URL and hash locally. Four frames kept, including the clearest entrance view yet. The frames overturn the inherited 'eight stone steps, 12 m wide': the entrance is three portals with individual short flights. Recorded as a resolved conflict with the photographs as evidence.
This commit is contained in:
zwitschi committed
1 parent
b8f5a062ac
commit
0dab597dc1
3 files changed
+162
No files matched your search
@@ -17,6 +17,7 @@ Repeatable import/export and processing helpers. Anything that was done twice by
|
||||
- `street_mesh.py` — road, kerbs, and sidewalks on the DEM, in the project frame; kerbs flush at the three crossings.
|
||||
- `terrain_from_dem.py` — terrain grid from the committed DEM, with building footprints cut out.
|
||||
- `blosm_blockout.py` — headless Blosm import of the committed `.osm` extract into a `.blend`, with a per-building scale report. Exact invocation in `Art/blender/README.md`.
|
||||
- `page_image_fetch.py` — resolves a web page's `og:image` and downloads it, with page URL and hash in a local sidecar. Works for Flickr, Commons file pages, and most photo blogs.
|
||||
- `commons_fetch.py` — Wikimedia Commons reference fetcher. Search, exact `File:` titles, or a whole Commons category (`--category`), thumbnails, provenance `sidecar.json` per folder. Wikimedia throttles `upload.wikimedia.org` (HTTP 429), so keep batches to six or fewer files and pass `--delay 3`; a throttled file is skipped and recorded rather than aborting the run. Writes only into the gitignored `Art/reference/`; reference images are never committed.
|
||||
|
||||
## Planned helpers
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Download the main photograph from a web page by reading its Open Graph image.
|
||||
|
||||
For reference gathering where the source is a gallery or photo page rather than
|
||||
an API: the page's `og:image` (or `twitter:image`) is the image the site itself
|
||||
publishes as representative, so it is a defensible pick without scraping every
|
||||
`<img>` on the page.
|
||||
|
||||
python3 Tools/page_image_fetch.py \
|
||||
--page "https://www.flickr.com/photos/…/in/…" --slug hero-masonic --name entry-01
|
||||
|
||||
Writes into `Art/reference/<slug>/` (gitignored) and appends provenance to
|
||||
`sidecar-web.json` there: page URL, image URL, timestamp, byte size, SHA-256.
|
||||
|
||||
Flickr note: the og:image is served from `live.staticflickr.com` at ~1024 px.
|
||||
The suffix is rewritten to the `_k` (2048 px) variant when the page shows one;
|
||||
if that 404s the original is used.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
UA = ("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/130.0 Safari/537.36 reference-fetch/0.1")
|
||||
META_PATTERNS = [
|
||||
r'<meta[^>]+property=["\']og:image(?::url)?["\'][^>]+content=["\']([^"\']+)',
|
||||
r'<meta[^>]+content=["\']([^"\']+)["\'][^>]+property=["\']og:image',
|
||||
r'<meta[^>]+name=["\']twitter:image["\'][^>]+content=["\']([^"\']+)',
|
||||
]
|
||||
|
||||
|
||||
def fetch(url, timeout=45, retries=3):
|
||||
for attempt in range(retries):
|
||||
req = urllib.request.Request(url, headers={"User-Agent": UA,
|
||||
"Accept": "*/*",
|
||||
"Accept-Language": "en-US,en;q=0.9"})
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
return resp.read(), resp.headers.get("Content-Type", "")
|
||||
except urllib.error.HTTPError as exc:
|
||||
if attempt == retries - 1:
|
||||
raise
|
||||
time.sleep(2 * (attempt + 1))
|
||||
except Exception:
|
||||
if attempt == retries - 1:
|
||||
raise
|
||||
time.sleep(2 * (attempt + 1))
|
||||
return b"", ""
|
||||
|
||||
|
||||
def og_image(html: str):
|
||||
for pattern in META_PATTERNS:
|
||||
match = re.search(pattern, html, flags=re.I)
|
||||
if match:
|
||||
return urllib.parse.unquote(match.group(1))
|
||||
return None
|
||||
|
||||
|
||||
def upgrade_flickr(url: str):
|
||||
"""Try the large-size suffix for a live.staticflickr.com URL."""
|
||||
if "staticflickr.com" not in url:
|
||||
return None
|
||||
candidate = re.sub(r"_[a-z]\.jpg$", "_k.jpg", url)
|
||||
return candidate if candidate != url else None
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--page", required=True)
|
||||
ap.add_argument("--slug", required=True)
|
||||
ap.add_argument("--out", default="Art/reference")
|
||||
ap.add_argument("--name", default=None, help="output file stem; defaults to a hash of the URL")
|
||||
ap.add_argument("--note", default="")
|
||||
args = ap.parse_args()
|
||||
|
||||
out_dir = os.path.join(args.out, args.slug)
|
||||
os.makedirs(out_dir, exist_ok=True)
|
||||
|
||||
print(f"page: {args.page}")
|
||||
raw, ctype = fetch(args.page)
|
||||
if "html" not in ctype.lower():
|
||||
print(f" not HTML (content-type {ctype}); skipping")
|
||||
return 1
|
||||
html = raw.decode("utf-8", errors="replace")
|
||||
image_url = og_image(html)
|
||||
if not image_url:
|
||||
print(" no og:image found; skipping")
|
||||
return 1
|
||||
print(f" og:image: {image_url[:110]}")
|
||||
|
||||
upgraded = upgrade_flickr(image_url)
|
||||
if upgraded:
|
||||
try:
|
||||
blob, _ = fetch(upgraded)
|
||||
if len(blob) > 10000:
|
||||
image_url = upgraded
|
||||
print(f" upgraded to large: {upgraded[:110]}")
|
||||
except Exception as exc:
|
||||
print(f" large variant failed ({exc}); using original")
|
||||
|
||||
blob, ctype = fetch(image_url)
|
||||
if len(blob) < 5000:
|
||||
print(f" too small ({len(blob)} bytes); skipping")
|
||||
return 1
|
||||
|
||||
stem = args.name or hashlib.sha256(args.page.encode()).hexdigest()[:10]
|
||||
ext = ".jpg" if "jpeg" in ctype or "jpg" in ctype else ".png"
|
||||
filename = f"{stem}{ext}"
|
||||
path = os.path.join(out_dir, filename)
|
||||
with open(path, "wb") as handle:
|
||||
handle.write(blob)
|
||||
digest = hashlib.sha256(blob).hexdigest()
|
||||
print(f" saved {filename} ({len(blob)//1024} KB)")
|
||||
|
||||
sidecar_path = os.path.join(out_dir, "sidecar-web.json")
|
||||
data = {"source": "web page og:image", "tool": "Tools/page_image_fetch.py",
|
||||
"note": "local development reference only; not committed", "images": []}
|
||||
if os.path.exists(sidecar_path):
|
||||
try:
|
||||
data = json.load(open(sidecar_path))
|
||||
except Exception:
|
||||
pass
|
||||
data["images"].append({
|
||||
"local_file": filename,
|
||||
"page_url": args.page,
|
||||
"image_url": image_url,
|
||||
"bytes": len(blob),
|
||||
"sha256": digest,
|
||||
"note": args.note,
|
||||
"fetched_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||||
})
|
||||
with open(sidecar_path, "w", encoding="utf-8") as handle:
|
||||
json.dump(data, handle, indent=2)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in new issue
Block a user