refactor(tools): group the sixty scripts by what they talk to
One directory with sixty scripts stopped being navigable, and the grouping is also the sandbox boundary:
Tools/unreal/ commandlets and the two map builders, with ue_common.py and ue_placement.py as the
shared imports (they need the engine)
Tools/blender/ builders, exporters, the FBX preset, the MPFB character pipeline, blender_screenshots
(they need the Blender Flatpak)
Tools/review/ the visual loop: render_screenshots.sh, judge_screenshot.py, hero_compare.py,
build_comparison_md.py, e2e_asset_check.py, hero_iterate.sh, visual_loop.py,
visual_targets.json
Tools/gen/ image and texture generation: the ComfyUI driver, deterministic texture writers,
reference fetching
Tools/input/ OS-level input injection
root _common.py, check-lfs.sh, addon_compat_probe.py, sibling_asset_audit.py, tests/
Every moved script now resolves the repository root as ../.. and puts the tools root plus its siblings on
sys.path, so 'import ue_common' still works from unreal/. The test suite bootstrap does the same for the five
folders. render_screenshots.sh takes the folder as a third argument to run_cmdlet, and the e2e driver,
hero_iterate.sh and the comparison generator were repointed.
51 README rows now link to their new location, the README gains a Layout section naming each folder's
sandbox, and the architecture notes and Blender README follow the moves.
Verified: 108 tests pass, prettier is clean, judge_screenshot.py and hero_compare.py run from their new
homes, and the e2e run was aborted mid-way (it had been rebuilding an empty DemoMap for 15 minutes) and is
being rerun against the new layout.
This commit is contained in:
1 parent
f681d83736
commit
4d27109668
168 files changed
+799
-407
No files matched your search
@@ -0,0 +1,146 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Download the main photograph from a web page by reading its Open Graph image.
|
||||
|
||||
For reference gathering where the source is a gallery or photo page rather than
|
||||
an API: the page's `og:image` (or `twitter:image`) is the image the site itself
|
||||
publishes as representative, so it is a defensible pick without scraping every
|
||||
`<img>` on the page.
|
||||
|
||||
python3 Tools/gen/page_image_fetch.py \
|
||||
--page "https://www.flickr.com/photos/…/in/…" --slug hero-masonic --name entry-01
|
||||
|
||||
Writes into `Art/reference/<slug>/` (gitignored) and appends provenance to
|
||||
`sidecar-web.json` there: page URL, image URL, timestamp, byte size, SHA-256.
|
||||
|
||||
Flickr note: the og:image is served from `live.staticflickr.com` at ~1024 px.
|
||||
The suffix is rewritten to the `_k` (2048 px) variant when the page shows one;
|
||||
if that 404s the original is used.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
UA = ("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/130.0 Safari/537.36 reference-fetch/0.1")
|
||||
META_PATTERNS = [
|
||||
r'<meta[^>]+property=["\']og:image(?::url)?["\'][^>]+content=["\']([^"\']+)',
|
||||
r'<meta[^>]+content=["\']([^"\']+)["\'][^>]+property=["\']og:image',
|
||||
r'<meta[^>]+name=["\']twitter:image["\'][^>]+content=["\']([^"\']+)',
|
||||
]
|
||||
|
||||
|
||||
def fetch(url, timeout=45, retries=3):
|
||||
for attempt in range(retries):
|
||||
req = urllib.request.Request(url, headers={"User-Agent": UA,
|
||||
"Accept": "*/*",
|
||||
"Accept-Language": "en-US,en;q=0.9"})
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
return resp.read(), resp.headers.get("Content-Type", "")
|
||||
except urllib.error.HTTPError as exc:
|
||||
if attempt == retries - 1:
|
||||
raise
|
||||
time.sleep(2 * (attempt + 1))
|
||||
except Exception:
|
||||
if attempt == retries - 1:
|
||||
raise
|
||||
time.sleep(2 * (attempt + 1))
|
||||
return b"", ""
|
||||
|
||||
|
||||
def og_image(html: str):
|
||||
for pattern in META_PATTERNS:
|
||||
match = re.search(pattern, html, flags=re.I)
|
||||
if match:
|
||||
return urllib.parse.unquote(match.group(1))
|
||||
return None
|
||||
|
||||
|
||||
def upgrade_flickr(url: str):
|
||||
"""Try the large-size suffix for a live.staticflickr.com URL."""
|
||||
if "staticflickr.com" not in url:
|
||||
return None
|
||||
candidate = re.sub(r"_[a-z]\.jpg$", "_k.jpg", url)
|
||||
return candidate if candidate != url else None
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--page", required=True)
|
||||
ap.add_argument("--slug", required=True)
|
||||
ap.add_argument("--out", default="Art/reference")
|
||||
ap.add_argument("--name", default=None, help="output file stem; defaults to a hash of the URL")
|
||||
ap.add_argument("--note", default="")
|
||||
args = ap.parse_args()
|
||||
|
||||
out_dir = os.path.join(args.out, args.slug)
|
||||
os.makedirs(out_dir, exist_ok=True)
|
||||
|
||||
print(f"page: {args.page}")
|
||||
raw, ctype = fetch(args.page)
|
||||
if "html" not in ctype.lower():
|
||||
print(f" not HTML (content-type {ctype}); skipping")
|
||||
return 1
|
||||
html = raw.decode("utf-8", errors="replace")
|
||||
image_url = og_image(html)
|
||||
if not image_url:
|
||||
print(" no og:image found; skipping")
|
||||
return 1
|
||||
print(f" og:image: {image_url[:110]}")
|
||||
|
||||
upgraded = upgrade_flickr(image_url)
|
||||
if upgraded:
|
||||
try:
|
||||
blob, _ = fetch(upgraded)
|
||||
if len(blob) > 10000:
|
||||
image_url = upgraded
|
||||
print(f" upgraded to large: {upgraded[:110]}")
|
||||
except Exception as exc:
|
||||
print(f" large variant failed ({exc}); using original")
|
||||
|
||||
blob, ctype = fetch(image_url)
|
||||
if len(blob) < 5000:
|
||||
print(f" too small ({len(blob)} bytes); skipping")
|
||||
return 1
|
||||
|
||||
stem = args.name or hashlib.sha256(args.page.encode()).hexdigest()[:10]
|
||||
ext = ".jpg" if "jpeg" in ctype or "jpg" in ctype else ".png"
|
||||
filename = f"{stem}{ext}"
|
||||
path = os.path.join(out_dir, filename)
|
||||
with open(path, "wb") as handle:
|
||||
handle.write(blob)
|
||||
digest = hashlib.sha256(blob).hexdigest()
|
||||
print(f" saved {filename} ({len(blob)//1024} KB)")
|
||||
|
||||
sidecar_path = os.path.join(out_dir, "sidecar-web.json")
|
||||
data = {"source": "web page og:image", "tool": "Tools/gen/page_image_fetch.py",
|
||||
"note": "local development reference only; not committed", "images": []}
|
||||
if os.path.exists(sidecar_path):
|
||||
try:
|
||||
data = json.load(open(sidecar_path))
|
||||
except Exception:
|
||||
pass
|
||||
data["images"].append({
|
||||
"local_file": filename,
|
||||
"page_url": args.page,
|
||||
"image_url": image_url,
|
||||
"bytes": len(blob),
|
||||
"sha256": digest,
|
||||
"note": args.note,
|
||||
"fetched_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||||
})
|
||||
with open(sidecar_path, "w", encoding="utf-8") as handle:
|
||||
json.dump(data, handle, indent=2)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in new issue
Block a user