diff --git a/.tmp-search1.txt b/.tmp-search1.txt new file mode 100644 index 0000000..4b825e4 --- /dev/null +++ b/.tmp-search1.txt @@ -0,0 +1,15 @@ +https://www.flickr.com/photos/sminor/2242555618 | Street-level photo of the Hollywood Masonic Temple/El Capitan Entertainment Centre façade, including the entrance steps and recessed doorway area facing Hollywood Blvd + +https://www.flickr.com/photos/snuffy/50743379941/ | Street-level exterior view of the Hollywood Masonic Temple (6840 Hollywood Blvd) showing the front façade from the sidewalk with the entrance/steps area + +https://www.flickr.com/photos/jp1958/34677173634/ | Street-level view of the Jimmy Kimmel Live / Hollywood Masonic Temple frontage at 6840 Hollywood Blvd, with the sidewalk and entrance area + +https://www.flickr.com/photos/faasdant/55130458397 | Photo taken from the Hollywood Walk of Fame sidewalk with the El Capitan Entertainment Centre (Hollywood Masonic Temple) entrance frontage in view + +https://commons.wikimedia.org/wiki/File:Hollywood_Masonic_Temple.JPG | Street-level exterior photo of the Hollywood Masonic Temple (El Capitan Entertainment Center), including the front entrance zone facing the sidewalk + +https://commons.wikimedia.org/wiki/File:MasonicTemple01.jpg | Street-level exterior photo of the Hollywood Masonic Temple on Hollywood Blvd, showing the entrance portion of the façade from the sidewalk + +https://commons.wikimedia.org/wiki/File:Hollywood_Masonic_Temple_2013-10-13_10-02-25.jpg | Street-level exterior photo of the Hollywood Masonic Temple façade (2013), with the entrance/steps area visible from the sidewalk + +https://commons.wikimedia.org/wiki/File:El_Capitan_Theater_frontal_fachade-The_Masonic_Temple.jpg | Frontal street-level view of the Masonic Temple building frontage on Hollywood Blvd, showing the recessed entrance area and the sidewalk/street context \ No newline at end of file diff --git a/Tools/README.md b/Tools/README.md index 9a3fb5a..64500a1 100644 --- a/Tools/README.md +++ b/Tools/README.md @@ -17,6 +17,7 @@ Repeatable import/export and processing helpers. Anything that was done twice by - `street_mesh.py` — road, kerbs, and sidewalks on the DEM, in the project frame; kerbs flush at the three crossings. - `terrain_from_dem.py` — terrain grid from the committed DEM, with building footprints cut out. - `blosm_blockout.py` — headless Blosm import of the committed `.osm` extract into a `.blend`, with a per-building scale report. Exact invocation in `Art/blender/README.md`. +- `page_image_fetch.py` — resolves a web page's `og:image` and downloads it, with page URL and hash in a local sidecar. Works for Flickr, Commons file pages, and most photo blogs. - `commons_fetch.py` — Wikimedia Commons reference fetcher. Search, exact `File:` titles, or a whole Commons category (`--category`), thumbnails, provenance `sidecar.json` per folder. Wikimedia throttles `upload.wikimedia.org` (HTTP 429), so keep batches to six or fewer files and pass `--delay 3`; a throttled file is skipped and recorded rather than aborting the run. Writes only into the gitignored `Art/reference/`; reference images are never committed. ## Planned helpers diff --git a/Tools/page_image_fetch.py b/Tools/page_image_fetch.py new file mode 100644 index 0000000..cad4ca9 --- /dev/null +++ b/Tools/page_image_fetch.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +"""Download the main photograph from a web page by reading its Open Graph image. + +For reference gathering where the source is a gallery or photo page rather than +an API: the page's `og:image` (or `twitter:image`) is the image the site itself +publishes as representative, so it is a defensible pick without scraping every +`` on the page. + + python3 Tools/page_image_fetch.py \ + --page "https://www.flickr.com/photos/…/in/…" --slug hero-masonic --name entry-01 + +Writes into `Art/reference//` (gitignored) and appends provenance to +`sidecar-web.json` there: page URL, image URL, timestamp, byte size, SHA-256. + +Flickr note: the og:image is served from `live.staticflickr.com` at ~1024 px. +The suffix is rewritten to the `_k` (2048 px) variant when the page shows one; +if that 404s the original is used. +""" + +import argparse +import hashlib +import json +import os +import re +import sys +import time +import urllib.error +import urllib.parse +import urllib.request + +UA = ("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/130.0 Safari/537.36 reference-fetch/0.1") +META_PATTERNS = [ + r']+property=["\']og:image(?::url)?["\'][^>]+content=["\']([^"\']+)', + r']+content=["\']([^"\']+)["\'][^>]+property=["\']og:image', + r']+name=["\']twitter:image["\'][^>]+content=["\']([^"\']+)', +] + + +def fetch(url, timeout=45, retries=3): + for attempt in range(retries): + req = urllib.request.Request(url, headers={"User-Agent": UA, + "Accept": "*/*", + "Accept-Language": "en-US,en;q=0.9"}) + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + return resp.read(), resp.headers.get("Content-Type", "") + except urllib.error.HTTPError as exc: + if attempt == retries - 1: + raise + time.sleep(2 * (attempt + 1)) + except Exception: + if attempt == retries - 1: + raise + time.sleep(2 * (attempt + 1)) + return b"", "" + + +def og_image(html: str): + for pattern in META_PATTERNS: + match = re.search(pattern, html, flags=re.I) + if match: + return urllib.parse.unquote(match.group(1)) + return None + + +def upgrade_flickr(url: str): + """Try the large-size suffix for a live.staticflickr.com URL.""" + if "staticflickr.com" not in url: + return None + candidate = re.sub(r"_[a-z]\.jpg$", "_k.jpg", url) + return candidate if candidate != url else None + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--page", required=True) + ap.add_argument("--slug", required=True) + ap.add_argument("--out", default="Art/reference") + ap.add_argument("--name", default=None, help="output file stem; defaults to a hash of the URL") + ap.add_argument("--note", default="") + args = ap.parse_args() + + out_dir = os.path.join(args.out, args.slug) + os.makedirs(out_dir, exist_ok=True) + + print(f"page: {args.page}") + raw, ctype = fetch(args.page) + if "html" not in ctype.lower(): + print(f" not HTML (content-type {ctype}); skipping") + return 1 + html = raw.decode("utf-8", errors="replace") + image_url = og_image(html) + if not image_url: + print(" no og:image found; skipping") + return 1 + print(f" og:image: {image_url[:110]}") + + upgraded = upgrade_flickr(image_url) + if upgraded: + try: + blob, _ = fetch(upgraded) + if len(blob) > 10000: + image_url = upgraded + print(f" upgraded to large: {upgraded[:110]}") + except Exception as exc: + print(f" large variant failed ({exc}); using original") + + blob, ctype = fetch(image_url) + if len(blob) < 5000: + print(f" too small ({len(blob)} bytes); skipping") + return 1 + + stem = args.name or hashlib.sha256(args.page.encode()).hexdigest()[:10] + ext = ".jpg" if "jpeg" in ctype or "jpg" in ctype else ".png" + filename = f"{stem}{ext}" + path = os.path.join(out_dir, filename) + with open(path, "wb") as handle: + handle.write(blob) + digest = hashlib.sha256(blob).hexdigest() + print(f" saved {filename} ({len(blob)//1024} KB)") + + sidecar_path = os.path.join(out_dir, "sidecar-web.json") + data = {"source": "web page og:image", "tool": "Tools/page_image_fetch.py", + "note": "local development reference only; not committed", "images": []} + if os.path.exists(sidecar_path): + try: + data = json.load(open(sidecar_path)) + except Exception: + pass + data["images"].append({ + "local_file": filename, + "page_url": args.page, + "image_url": image_url, + "bytes": len(blob), + "sha256": digest, + "note": args.note, + "fetched_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + }) + with open(sidecar_path, "w", encoding="utf-8") as handle: + json.dump(data, handle, indent=2) + return 0 + + +if __name__ == "__main__": + sys.exit(main())