294 lines
14 KiB
Python
294 lines
14 KiB
Python
#!/usr/bin/env python3
|
|
"""Wikimedia Commons reference-board helper for venue documentation.
|
|
|
|
Lists and fetches licence-cleared photos of a real location and writes the
|
|
`SIDECAR.md` rows and the folder `README.md` gallery rows used by
|
|
`docs/screens/refs/<venue>/`. Research downloads stay under `tmp/` unless
|
|
`--stage refs` copies them into the board folder (still uncommitted; the git
|
|
commit is a deliberate human step).
|
|
|
|
python3 tools/refboard.py search "El Capitan Entertainment Centre" \
|
|
[--months 12] [--radius 500] [--at coord] [--json]
|
|
python3 tools/refboard.py fetch --from search.json --dir tmp/research/elcapitan \
|
|
[--max N]
|
|
python3 tools/refboard.py rows --from meta.json --licence-note ... [--append-sidecar PATH]
|
|
|
|
Licence gate: CC0 / public domain / CC BY are listed; CC BY-SA rows are marked
|
|
`(review)` and must record the share-alike decision; anything else is skipped.
|
|
|
|
Rate limits: the Commons API needs a descriptive User-Agent, metadata is fetched
|
|
in one batched `titles=` query, and file downloads are spaced (default 3 s) to
|
|
avoid HTTP 429.
|
|
"""
|
|
|
|
import argparse
|
|
import json
|
|
import re
|
|
import sys
|
|
import time
|
|
import urllib.parse
|
|
import urllib.request
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
USER_AGENT = "gig-line-dev-refboard/1.0 (development reference collection; contact: repo)"
|
|
API = "https://commons.wikimedia.org/w/api.php"
|
|
DOWNLOAD_SPACING_S = 3.0
|
|
#Licences that can be committed with the stated attribution; anything else is
|
|
#research-only.
|
|
COMMITTABLE = {
|
|
"Public domain": "Public domain",
|
|
"CC0": "CC0",
|
|
"CC BY 2.0": "CC BY 2.0",
|
|
"CC BY 3.0": "CC BY 3.0",
|
|
"CC BY 4.0": "CC BY 4.0",
|
|
}
|
|
# CC BY-SA requires a share-alike review before a file enters the repository, so
|
|
# it stays research-only until `--allow-ccbysa` records the decision.
|
|
REVIEW_ONLY = {
|
|
"CC BY-SA 2.0": "CC BY-SA 2.0 (review)",
|
|
"CC BY-SA 3.0": "CC BY-SA 3.0 (review)",
|
|
"CC BY-SA 4.0": "CC BY-SA 4.0 (review)",
|
|
}
|
|
def _strip_html(text: str) -> str:
|
|
return re.sub("<[^>]+>", "", text or "").strip()
|
|
|
|
|
|
def normalise_licence(raw: str, allow_ccbysa: bool = False):
|
|
"""Return (committed label | None, research-only label | None).
|
|
|
|
`clean` is set when the file may enter the repository; `review` is set for
|
|
share-alike licences that stay under `tmp/` until a review records otherwise.
|
|
"""
|
|
if raw in COMMITTABLE:
|
|
return raw, None
|
|
if raw in REVIEW_ONLY:
|
|
return (REVIEW_ONLY[raw], None) if allow_ccbysa else (None, REVIEW_ONLY[raw])
|
|
return None, raw or None
|
|
class Commons:
|
|
def __init__(self, user_agent: str = USER_AGENT):
|
|
self.user_agent = user_agent
|
|
self.last_request = 0.0
|
|
def _get(self, params: dict, binary: bool = False, spacing: float = 0.0):
|
|
if spacing:
|
|
wait = spacing - (time.monotonic() - self.last_request)
|
|
if wait > 0:
|
|
time.sleep(wait)
|
|
self.last_request = time.monotonic()
|
|
req = urllib.request.Request(
|
|
API if not binary else params["url"],
|
|
data=None if binary else urllib.parse.urlencode(params).encode(),
|
|
headers={"User-Agent": self.user_agent}, method="GET" if binary else "POST")
|
|
with urllib.request.urlopen(req, timeout=90) as resp:
|
|
return resp.read() if binary else json.loads(resp.read().decode("utf-8"))
|
|
def meta_for_titles(self, titles: list[str], allow_ccbysa: bool = False) -> dict:
|
|
rows = {}
|
|
for chunk in [titles[i:i + 20] for i in range(0, len(titles), 20)]:
|
|
data = self._get({
|
|
"action": "query", "format": "json", "titles": "|".join(chunk),
|
|
"prop": "imageinfo", "iiprop": "url|size|extmetadata",
|
|
"iiurlwidth": "1600", "formatversion": "2",
|
|
})
|
|
for page in data.get("query", {}).get("pages", []):
|
|
if "missing" in page or not page.get("imageinfo"):
|
|
continue
|
|
info = page["imageinfo"][0]
|
|
meta = info.get("extmetadata", {})
|
|
def val(key):
|
|
return _strip_html(meta.get(key, {}).get("value", ""))
|
|
|
|
def first_date():
|
|
for key in ("DateTimeOriginal", "DateTime", "DateTimeDigitized"):
|
|
text = _strip_html(meta.get(key, {}).get("value", ""))
|
|
match = re.search(r"\d{4}-\d{2}-\d{2}", text)
|
|
if match:
|
|
return match.group(0)
|
|
return ""
|
|
|
|
raw_licence = val("LicenseShortName")
|
|
clean, review = normalise_licence(raw_licence, allow_ccbysa)
|
|
rows[page["title"]] = {
|
|
"title": page["title"],
|
|
"licence": raw_licence,
|
|
"licence_url": val("LicenseUrl"),
|
|
"artist": val("Artist") or "unknown",
|
|
"credit": val("Credit")[:120],
|
|
"date": first_date(),
|
|
"description": val("ImageDescription")[:300],
|
|
"page": info.get("descriptionurl", ""),
|
|
"width": info.get("thumbwidth") or info.get("width", 0),
|
|
"height": info.get("thumbheight") or info.get("height", 0),
|
|
"url": info.get("thumburl") or info.get("url", ""),
|
|
"clean": clean,
|
|
"review": review,
|
|
}
|
|
return rows
|
|
def search(self, query: str, limit: int = 12) -> list[str]:
|
|
data = self._get({"action": "query", "format": "json", "generator": "search",
|
|
"gsrsearch": query, "gsrnamespace": "6", "gsrlimit": str(limit),
|
|
"formatversion": "2"})
|
|
return [p["title"] for p in data.get("query", {}).get("pages", [])
|
|
if not p.get("missing")]
|
|
def geosearch(self, coord: str, radius_m: int = 500, limit: int = 60) -> list[str]:
|
|
data = self._get({"action": "query", "format": "json", "generator": "geosearch",
|
|
"ggscoord": coord, "ggsradius": str(radius_m),
|
|
"ggslimit": str(limit), "ggsnamespace": "6",
|
|
"formatversion": "2"})
|
|
return [p["title"] for p in data.get("query", {}).get("pages", [])
|
|
if not p.get("missing")]
|
|
def category(self, category: str, limit: int = 50) -> list[str]:
|
|
data = self._get({"action": "query", "format": "json", "generator": "categorymembers",
|
|
"gcmtitle": category, "gcmtype": "file", "gcmlimit": str(limit),
|
|
"formatversion": "2"})
|
|
return [p["title"] for p in data.get("query", {}).get("pages", [])
|
|
if not p.get("missing")]
|
|
def collect(source: str, argument: str, limit: int) -> list[str]:
|
|
api = Commons()
|
|
if source == "search":
|
|
return api.search(argument, limit)
|
|
if source == "geo":
|
|
coord, radius = (argument.split(",") + [str(500)])[:2] if "," in argument \
|
|
else (argument, "500")
|
|
return api.geosearch(argument if "," in argument else coord, int(radius), limit)
|
|
if source == "category":
|
|
return api.category(argument if argument.startswith("Category:")
|
|
else f"Category:{argument}", limit)
|
|
raise SystemExit(f"unknown source {source}")
|
|
def _age_months(date: str, today: str) -> int | None:
|
|
if not re.match(r"^\d{4}-\d{2}", date):
|
|
return None
|
|
years = int(today[:4]) - int(date[:4])
|
|
months = int(today[5:7]) - int(date[5:7])
|
|
return years * 12 + months
|
|
def cmd_search(args) -> int:
|
|
titles = collect(args.source, args.query, args.limit)
|
|
api = Commons()
|
|
rows = api.meta_for_titles(titles, allow_ccbysa=args.allow_ccbysa)
|
|
today = time.strftime("%Y-%m-%d")
|
|
result = []
|
|
for title in titles:
|
|
row = rows.get(title)
|
|
if not row:
|
|
continue
|
|
row["age_months"] = _age_months(row["date"], today)
|
|
row["recent"] = (args.months <= 0 or
|
|
(row["age_months"] is not None and row["age_months"] <= args.months))
|
|
row["usable"] = bool(row["clean"] and row["url"]
|
|
and int(row["width"] or 0) >= args.min_width)
|
|
row["research_only"] = not row["usable"] and bool(row["review"] and row["url"]
|
|
and int(row["width"] or 0) >= args.min_width)
|
|
result.append(row)
|
|
result.sort(key=lambda r: (not r["recent"], r["date"]), reverse=False)
|
|
result.sort(key=lambda r: r["date"], reverse=True)
|
|
if args.json:
|
|
print(json.dumps(result, indent=2))
|
|
return 0
|
|
for row in result:
|
|
age = "?" if row["age_months"] is None else row["age_months"]
|
|
flags = "recent" if row["recent"] else " "
|
|
flags += " commit" if row["usable"] else " review" if row["research_only"] else " skip"
|
|
print(f"{row['date'] or '?':10s} {age:>4}mo {flags:12s} "
|
|
f"{row['licence'][:20]:20s} {row['title'][:56]}")
|
|
return 0
|
|
def cmd_fetch(args) -> int:
|
|
rows = json.loads(Path(args.manifest).read_text(encoding="utf-8"))
|
|
usable = [r for r in rows if r.get("usable")
|
|
or (args.include_review and r.get("research_only"))]
|
|
if args.only_recent:
|
|
usable = [r for r in usable if r.get("recent")]
|
|
out = Path(args.dir)
|
|
out.mkdir(parents=True, exist_ok=True)
|
|
api = Commons()
|
|
manifest = []
|
|
for row in usable[:args.max]:
|
|
name = re.sub(r"^File:", "", row["title"])
|
|
name = re.sub(r"[^A-Za-z0-9._-]+", "_", name)
|
|
path = out / name
|
|
if path.exists() and not args.force:
|
|
print(f"skip existing {path.name}")
|
|
else:
|
|
data = api._get({"url": row["url"]}, binary=True,
|
|
spacing=DOWNLOAD_SPACING_S)
|
|
path.write_bytes(data)
|
|
print(f"fetched {path.name} {len(data) // 1024} KB")
|
|
manifest.append({**row, "file": path.name})
|
|
(out / "meta.json").write_text(json.dumps(manifest, indent=2) + "\n")
|
|
print(f"meta -> {out / 'meta.json'} ({len(manifest)} entries)")
|
|
return 0
|
|
def _markdown_filename(title: str, date: str = "") -> str:
|
|
stem = re.sub(r"^File:", "", title)
|
|
stem = re.sub(r"\.[A-Za-z0-9]+$", "", stem)
|
|
slug = re.sub(r"[^A-Za-z0-9]+", "-", stem).strip("-").lower()
|
|
return f"{slug}-{date or 'unknown'}.jpg"
|
|
def cmd_rows(args) -> int:
|
|
rows = json.loads(Path(args.manifest).read_text(encoding="utf-8"))
|
|
sidecar = []
|
|
readme = []
|
|
for row in rows:
|
|
name = _markdown_filename(row["title"], row["date"] or "unknown")
|
|
licence = row["clean"] or row["licence"]
|
|
sidecar.append(
|
|
f"| `{name}` | {_strip_html(row['description'])[:80] or row['title']} | "
|
|
f"view | <{row['page']}> | {row['artist'][:40]} | {licence} | "
|
|
f"{row['date'] or 'n.d.'} | purpose | no |")
|
|
readme.append(
|
|
f"| ![{row['title']}]({name}) | **{row['title'][:40]}** |")
|
|
print("== SIDECAR rows (paste into the Photographs table)\n")
|
|
print("\n".join(sidecar))
|
|
print("\n== README rows (paste into the board gallery)\n")
|
|
print("\n".join(readme))
|
|
if args.append_sidecar:
|
|
target = Path(args.append_sidecar)
|
|
if not target.is_file():
|
|
return 1
|
|
text = target.read_text(encoding="utf-8")
|
|
marker = "| `"
|
|
additions = []
|
|
existing = {m.group(1) for m in re.finditer(r"^\| `([^`]+)`", text, re.M)}
|
|
for row, line in zip(rows, sidecar):
|
|
name = line.split("|")[1].strip().strip("`")
|
|
if name not in existing:
|
|
additions.append(line)
|
|
if additions:
|
|
last = [l for l in text.splitlines() if l.startswith(marker)][-1]
|
|
text = text.replace(last, last + "\n" + "\n".join(additions))
|
|
target.write_text(text)
|
|
print(f"appended {len(additions)} row(s) to {target}")
|
|
return 0
|
|
def build_parser():
|
|
parser = argparse.ArgumentParser(description=__doc__,
|
|
formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
sub = parser.add_subparsers(dest="cmd", required=True)
|
|
search = sub.add_parser("search", help="list Commons candidates with licence/date")
|
|
search.add_argument("query", help="search text, coord 'lat,lon' with --source geo, "
|
|
"or category with --source category")
|
|
search.add_argument("--source", choices=("search", "geo", "category"), default="search")
|
|
search.add_argument("--limit", type=int, default=20)
|
|
search.add_argument("--months", type=int, default=0,
|
|
help="flag files captured within this many months (0 = all)")
|
|
search.add_argument("--min-width", type=int, default=1200)
|
|
search.add_argument("--allow-ccbysa", action="store_true",
|
|
help="treat CC BY-SA as committable after a recorded review "
|
|
"(default: research-only)")
|
|
search.add_argument("--json", action="store_true")
|
|
search.set_defaults(handler=cmd_search)
|
|
fetch = sub.add_parser("fetch", help="download the usable rows of a manifest")
|
|
fetch.add_argument("--from", dest="manifest", required=True)
|
|
fetch.add_argument("--dir", required=True)
|
|
fetch.add_argument("--max", type=int, default=12)
|
|
fetch.add_argument("--only-recent", action="store_true")
|
|
fetch.add_argument("--force", action="store_true")
|
|
fetch.add_argument("--include-review", action="store_true",
|
|
help="also download review-only (CC BY-SA) rows for study under tmp/")
|
|
fetch.set_defaults(handler=cmd_fetch)
|
|
rows = sub.add_parser("rows", help="emit SIDECAR/README table rows from a manifest")
|
|
rows.add_argument("--from", dest="manifest", required=True)
|
|
rows.add_argument("--append-sidecar", default="")
|
|
rows.set_defaults(handler=cmd_rows)
|
|
return parser
|
|
def main(argv=None) -> int:
|
|
args = build_parser().parse_args(argv)
|
|
return args.handler(args)
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|