refactor(tools): group the sixty scripts by what they talk to

One directory with sixty scripts stopped being navigable, and the grouping is also the sandbox boundary:

  Tools/unreal/    commandlets and the two map builders, with ue_common.py and ue_placement.py as the
                   shared imports (they need the engine)
  Tools/blender/   builders, exporters, the FBX preset, the MPFB character pipeline, blender_screenshots
                   (they need the Blender Flatpak)
  Tools/review/    the visual loop: render_screenshots.sh, judge_screenshot.py, hero_compare.py,
                   build_comparison_md.py, e2e_asset_check.py, hero_iterate.sh, visual_loop.py,
                   visual_targets.json
  Tools/gen/       image and texture generation: the ComfyUI driver, deterministic texture writers,
                   reference fetching
  Tools/input/     OS-level input injection
  root             _common.py, check-lfs.sh, addon_compat_probe.py, sibling_asset_audit.py, tests/

Every moved script now resolves the repository root as ../.. and puts the tools root plus its siblings on
sys.path, so 'import ue_common' still works from unreal/. The test suite bootstrap does the same for the five
folders. render_screenshots.sh takes the folder as a third argument to run_cmdlet, and the e2e driver,
hero_iterate.sh and the comparison generator were repointed.

51 README rows now link to their new location, the README gains a Layout section naming each folder's
sandbox, and the architecture notes and Blender README follow the moves.

Verified: 108 tests pass, prettier is clean, judge_screenshot.py and hero_compare.py run from their new
homes, and the e2e run was aborted mid-way (it had been rebuilding an empty DemoMap for 15 minutes) and is
being rerun against the new layout.
This commit is contained in:
zwitschi committed 2026-11-10 12:40:30 +01:00
1 parent f681d83736
commit 4d27109668
168 files changed
+799 -407

No files matched your search

+146
View File
@@ -0,0 +1,146 @@
#!/usr/bin/env python3
"""Download the main photograph from a web page by reading its Open Graph image.
For reference gathering where the source is a gallery or photo page rather than
an API: the page's `og:image` (or `twitter:image`) is the image the site itself
publishes as representative, so it is a defensible pick without scraping every
`<img>` on the page.
python3 Tools/gen/page_image_fetch.py \
--page "https://www.flickr.com/photos/…/in/…" --slug hero-masonic --name entry-01
Writes into `Art/reference/<slug>/` (gitignored) and appends provenance to
`sidecar-web.json` there: page URL, image URL, timestamp, byte size, SHA-256.
Flickr note: the og:image is served from `live.staticflickr.com` at ~1024 px.
The suffix is rewritten to the `_k` (2048 px) variant when the page shows one;
if that 404s the original is used.
"""
import argparse
import hashlib
import json
import os
import re
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
UA = ("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/130.0 Safari/537.36 reference-fetch/0.1")
META_PATTERNS = [
r'<meta[^>]+property=["\']og:image(?::url)?["\'][^>]+content=["\']([^"\']+)',
r'<meta[^>]+content=["\']([^"\']+)["\'][^>]+property=["\']og:image',
r'<meta[^>]+name=["\']twitter:image["\'][^>]+content=["\']([^"\']+)',
]
def fetch(url, timeout=45, retries=3):
for attempt in range(retries):
req = urllib.request.Request(url, headers={"User-Agent": UA,
"Accept": "*/*",
"Accept-Language": "en-US,en;q=0.9"})
try:
with urllib.request.urlopen(req, timeout=timeout) as resp:
return resp.read(), resp.headers.get("Content-Type", "")
except urllib.error.HTTPError as exc:
if attempt == retries - 1:
raise
time.sleep(2 * (attempt + 1))
except Exception:
if attempt == retries - 1:
raise
time.sleep(2 * (attempt + 1))
return b"", ""
def og_image(html: str):
for pattern in META_PATTERNS:
match = re.search(pattern, html, flags=re.I)
if match:
return urllib.parse.unquote(match.group(1))
return None
def upgrade_flickr(url: str):
"""Try the large-size suffix for a live.staticflickr.com URL."""
if "staticflickr.com" not in url:
return None
candidate = re.sub(r"_[a-z]\.jpg$", "_k.jpg", url)
return candidate if candidate != url else None
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--page", required=True)
ap.add_argument("--slug", required=True)
ap.add_argument("--out", default="Art/reference")
ap.add_argument("--name", default=None, help="output file stem; defaults to a hash of the URL")
ap.add_argument("--note", default="")
args = ap.parse_args()
out_dir = os.path.join(args.out, args.slug)
os.makedirs(out_dir, exist_ok=True)
print(f"page: {args.page}")
raw, ctype = fetch(args.page)
if "html" not in ctype.lower():
print(f" not HTML (content-type {ctype}); skipping")
return 1
html = raw.decode("utf-8", errors="replace")
image_url = og_image(html)
if not image_url:
print(" no og:image found; skipping")
return 1
print(f" og:image: {image_url[:110]}")
upgraded = upgrade_flickr(image_url)
if upgraded:
try:
blob, _ = fetch(upgraded)
if len(blob) > 10000:
image_url = upgraded
print(f" upgraded to large: {upgraded[:110]}")
except Exception as exc:
print(f" large variant failed ({exc}); using original")
blob, ctype = fetch(image_url)
if len(blob) < 5000:
print(f" too small ({len(blob)} bytes); skipping")
return 1
stem = args.name or hashlib.sha256(args.page.encode()).hexdigest()[:10]
ext = ".jpg" if "jpeg" in ctype or "jpg" in ctype else ".png"
filename = f"{stem}{ext}"
path = os.path.join(out_dir, filename)
with open(path, "wb") as handle:
handle.write(blob)
digest = hashlib.sha256(blob).hexdigest()
print(f" saved {filename} ({len(blob)//1024} KB)")
sidecar_path = os.path.join(out_dir, "sidecar-web.json")
data = {"source": "web page og:image", "tool": "Tools/gen/page_image_fetch.py",
"note": "local development reference only; not committed", "images": []}
if os.path.exists(sidecar_path):
try:
data = json.load(open(sidecar_path))
except Exception:
pass
data["images"].append({
"local_file": filename,
"page_url": args.page,
"image_url": image_url,
"bytes": len(blob),
"sha256": digest,
"note": args.note,
"fetched_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
})
with open(sidecar_path, "w", encoding="utf-8") as handle:
json.dump(data, handle, indent=2)
return 0
if __name__ == "__main__":
sys.exit(main())