gig-line/Tools/page_image_fetch.py

#!/usr/bin/env python3
"""Download the main photograph from a web page by reading its Open Graph image.

For reference gathering where the source is a gallery or photo page rather than
an API: the page's `og:image` (or `twitter:image`) is the image the site itself
publishes as representative, so it is a defensible pick without scraping every
`<img>` on the page.

    python3 Tools/page_image_fetch.py \
        --page "https://www.flickr.com/photos/…/in/…" --slug hero-masonic --name entry-01

Writes into `Art/reference/<slug>/` (gitignored) and appends provenance to
`sidecar-web.json` there: page URL, image URL, timestamp, byte size, SHA-256.

Flickr note: the og:image is served from `live.staticflickr.com` at ~1024 px.
The suffix is rewritten to the `_k` (2048 px) variant when the page shows one;
if that 404s the original is used.
"""

import argparse
import hashlib
import json
import os
import re
import sys
import time
import urllib.error
import urllib.parse
import urllib.request

UA = ("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) "
      "Chrome/130.0 Safari/537.36 reference-fetch/0.1")
META_PATTERNS = [
    r'<meta[^>]+property=["\']og:image(?::url)?["\'][^>]+content=["\']([^"\']+)',
    r'<meta[^>]+content=["\']([^"\']+)["\'][^>]+property=["\']og:image',
    r'<meta[^>]+name=["\']twitter:image["\'][^>]+content=["\']([^"\']+)',
]


def fetch(url, timeout=45, retries=3):
    for attempt in range(retries):
        req = urllib.request.Request(url, headers={"User-Agent": UA,
                                                   "Accept": "*/*",
                                                   "Accept-Language": "en-US,en;q=0.9"})
        try:
            with urllib.request.urlopen(req, timeout=timeout) as resp:
                return resp.read(), resp.headers.get("Content-Type", "")
        except urllib.error.HTTPError as exc:
            if attempt == retries - 1:
                raise
            time.sleep(2 * (attempt + 1))
        except Exception:
            if attempt == retries - 1:
                raise
            time.sleep(2 * (attempt + 1))
    return b"", ""


def og_image(html: str):
    for pattern in META_PATTERNS:
        match = re.search(pattern, html, flags=re.I)
        if match:
            return urllib.parse.unquote(match.group(1))
    return None


def upgrade_flickr(url: str):
    """Try the large-size suffix for a live.staticflickr.com URL."""
    if "staticflickr.com" not in url:
        return None
    candidate = re.sub(r"_[a-z]\.jpg$", "_k.jpg", url)
    return candidate if candidate != url else None


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--page", required=True)
    ap.add_argument("--slug", required=True)
    ap.add_argument("--out", default="Art/reference")
    ap.add_argument("--name", default=None, help="output file stem; defaults to a hash of the URL")
    ap.add_argument("--note", default="")
    args = ap.parse_args()

    out_dir = os.path.join(args.out, args.slug)
    os.makedirs(out_dir, exist_ok=True)

    print(f"page: {args.page}")
    raw, ctype = fetch(args.page)
    if "html" not in ctype.lower():
        print(f"  not HTML (content-type {ctype}); skipping")
        return 1
    html = raw.decode("utf-8", errors="replace")
    image_url = og_image(html)
    if not image_url:
        print("  no og:image found; skipping")
        return 1
    print(f"  og:image: {image_url[:110]}")

    upgraded = upgrade_flickr(image_url)
    if upgraded:
        try:
            blob, _ = fetch(upgraded)
            if len(blob) > 10000:
                image_url = upgraded
                print(f"  upgraded to large: {upgraded[:110]}")
        except Exception as exc:
            print(f"  large variant failed ({exc}); using original")

    blob, ctype = fetch(image_url)
    if len(blob) < 5000:
        print(f"  too small ({len(blob)} bytes); skipping")
        return 1

    stem = args.name or hashlib.sha256(args.page.encode()).hexdigest()[:10]
    ext = ".jpg" if "jpeg" in ctype or "jpg" in ctype else ".png"
    filename = f"{stem}{ext}"
    path = os.path.join(out_dir, filename)
    with open(path, "wb") as handle:
        handle.write(blob)
    digest = hashlib.sha256(blob).hexdigest()
    print(f"  saved {filename} ({len(blob)//1024} KB)")

    sidecar_path = os.path.join(out_dir, "sidecar-web.json")
    data = {"source": "web page og:image", "tool": "Tools/page_image_fetch.py",
            "note": "local development reference only; not committed", "images": []}
    if os.path.exists(sidecar_path):
        try:
            data = json.load(open(sidecar_path))
        except Exception:
            pass
    data["images"].append({
        "local_file": filename,
        "page_url": args.page,
        "image_url": image_url,
        "bytes": len(blob),
        "sha256": digest,
        "note": args.note,
        "fetched_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
    })
    with open(sidecar_path, "w", encoding="utf-8") as handle:
        json.dump(data, handle, indent=2)
    return 0


if __name__ == "__main__":
    sys.exit(main())