geo: one tool directory, one unit boundary, per-asset provenance
Geo/tools/ is now every geographic script, with Geo/README.md documenting each one; the Blender-side builders moved beside their outputs in Art/blender/*/ and import their helpers from Geo/tools/osm_common.py. That module gains to_unreal_local() and to_blender_local(): the only place in the repository that multiplies metres by 100, so a second silent scale cannot be introduced. block_height_report.py adds height provenance for the block — 3 of 15 buildings have no height data and render at a placeholder 6.4 m. asset_metadata.py writes and audits <asset>.geo.json sidecars; four now exist (hero-facade, terrain, street, blockout) and the audit recomputes each centre from the origin, failing anything that disagrees.
This commit is contained in:
zwitschi committed
1 parent
5e5e5ae9b4
commit
386d4bee78
26 files changed
+471
-57
No files matched your search
@@ -39,7 +39,7 @@ The initial release is **`0.1.0`**: a walkable exterior demo around **6840 Holly
|
||||
- Track `.uasset`, `.umap`, `.blend`, and large source binaries with LFS.
|
||||
- Ignore `Binaries/`, `Intermediate/`, `Saved/`, and Derived Data Cache.
|
||||
- Never commit Unreal Engine source, credentials, or non-redistributable assets.
|
||||
- Planned remote repository: `https://git.allucanget.biz/zwitschi/gig-line.git`. Do not assume it is configured or available.
|
||||
- Active remote repository (`origin`): `https://git.allucanget.biz/zwitschi/gig-line.git`. Verify connectivity before use.
|
||||
|
||||
## Release Policy
|
||||
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"schema": 1,
|
||||
"asset": "blockout",
|
||||
"kind": "building",
|
||||
"generated_utc": "2026-11-07T11:27:43+00:00",
|
||||
"geographic": {
|
||||
"crs": "EPSG:26911",
|
||||
"source": "Geo/raw/osm-api-map-2026-11-06.osm, demo bounds per Geo/verification/2026-11-06-el-capitan-block.md",
|
||||
"centre_easting_m": 376380.0,
|
||||
"centre_northing_m": 3774200.0,
|
||||
"elevation_navd88_m": 117.0,
|
||||
"origin": {
|
||||
"easting_m": 376515.389,
|
||||
"northing_m": 3774224.083,
|
||||
"reference": "Geo/manifests/local-origin.md"
|
||||
}
|
||||
},
|
||||
"unreal": {
|
||||
"centre_cm": [
|
||||
-13538.9,
|
||||
2408.3,
|
||||
108.9
|
||||
],
|
||||
"conversion": "100 uu per metre, applied once in osm_common.to_unreal_local",
|
||||
"axes": "X east, Y south (FBX preset flips north), Z up"
|
||||
},
|
||||
"dims_report": null,
|
||||
"notes": [
|
||||
"86 meshes from Blosm; affine-corrected from the spherical TM frame into EPSG:26911 (0.0 cm residual)"
|
||||
]
|
||||
}
|
||||
File renamed without changes.
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"schema": 1,
|
||||
"asset": "hero-facade",
|
||||
"kind": "building",
|
||||
"generated_utc": "2026-11-07T11:26:11+00:00",
|
||||
"geographic": {
|
||||
"crs": "EPSG:26911",
|
||||
"source": "OSM way 424024147 (El Capitan Entertainment Centre), 25.75 m frontage",
|
||||
"centre_easting_m": 376387.944,
|
||||
"centre_northing_m": 3774185.293,
|
||||
"elevation_navd88_m": 116.6,
|
||||
"origin": {
|
||||
"easting_m": 376515.389,
|
||||
"northing_m": 3774224.083,
|
||||
"reference": "Geo/manifests/local-origin.md"
|
||||
}
|
||||
},
|
||||
"unreal": {
|
||||
"centre_cm": [
|
||||
-12744.5,
|
||||
3879.0,
|
||||
68.9
|
||||
],
|
||||
"conversion": "100 uu per metre, applied once in osm_common.to_unreal_local",
|
||||
"axes": "X east, Y south (FBX preset flips north), Z up"
|
||||
},
|
||||
"dims_report": "Art/blender/hero/hero-facade-2026-11-07.json",
|
||||
"notes": [
|
||||
"frontage 25.75 m and height 14.5 m measured; column bay 4.96 m from the sibling notes; portal flights from the photographs; everything else assumption"
|
||||
]
|
||||
}
|
||||
@@ -26,7 +26,7 @@ import sys
|
||||
import bpy
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
sys.path.insert(0, os.path.join(HERE, "..", "Geo", "scripts"))
|
||||
sys.path.insert(0, os.path.join(HERE, "..", "..", "..", "Geo", "tools"))
|
||||
import osm_common as oc # noqa: E402
|
||||
|
||||
ORIGIN_E = 376515.389
|
||||
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"schema": 1,
|
||||
"asset": "street",
|
||||
"kind": "street",
|
||||
"generated_utc": "2026-11-07T11:27:43+00:00",
|
||||
"geographic": {
|
||||
"crs": "EPSG:26911",
|
||||
"source": "Hollywood Blvd centreline between Orange Dr and Highland Ave, Geo/verification/2026-11-06-street-profile.md",
|
||||
"centre_easting_m": 376375.6,
|
||||
"centre_northing_m": 3774226.0,
|
||||
"elevation_navd88_m": 117.3,
|
||||
"origin": {
|
||||
"easting_m": 376515.389,
|
||||
"northing_m": 3774224.083,
|
||||
"reference": "Geo/manifests/local-origin.md"
|
||||
}
|
||||
},
|
||||
"unreal": {
|
||||
"centre_cm": [
|
||||
-13978.9,
|
||||
-191.7,
|
||||
138.9
|
||||
],
|
||||
"conversion": "100 uu per metre, applied once in osm_common.to_unreal_local",
|
||||
"axes": "X east, Y south (FBX preset flips north), Z up"
|
||||
},
|
||||
"dims_report": null,
|
||||
"notes": [
|
||||
"carriageway -10.5..+10.5 m, kerbs 0.15 m assumed, sidewalks 4.0 m south / 5.3 m north"
|
||||
]
|
||||
}
|
||||
@@ -38,7 +38,7 @@ import sys
|
||||
import bpy
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
sys.path.insert(0, os.path.join(HERE, "..", "Geo", "scripts"))
|
||||
sys.path.insert(0, os.path.join(HERE, "..", "..", "..", "Geo", "tools"))
|
||||
import dem_stats # noqa: E402
|
||||
import osm_common as oc # noqa: E402
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"schema": 1,
|
||||
"asset": "terrain",
|
||||
"kind": "terrain",
|
||||
"generated_utc": "2026-11-07T11:27:43+00:00",
|
||||
"geographic": {
|
||||
"crs": "EPSG:26911",
|
||||
"source": "3DEP 1 m DEM window, Geo/manifests/2026-11-06-3dep-dem.md",
|
||||
"centre_easting_m": 376375.0,
|
||||
"centre_northing_m": 3774225.0,
|
||||
"elevation_navd88_m": 115.911,
|
||||
"origin": {
|
||||
"easting_m": 376515.389,
|
||||
"northing_m": 3774224.083,
|
||||
"reference": "Geo/manifests/local-origin.md"
|
||||
}
|
||||
},
|
||||
"unreal": {
|
||||
"centre_cm": [
|
||||
-14038.9,
|
||||
-91.7,
|
||||
0.0
|
||||
],
|
||||
"conversion": "100 uu per metre, applied once in osm_common.to_unreal_local",
|
||||
"axes": "X east, Y south (FBX preset flips north), Z up"
|
||||
},
|
||||
"dims_report": null,
|
||||
"notes": [
|
||||
"2 m grid, 191 x 106, building footprints cut out; Z relative to origin ground 115.911 m NAVD88"
|
||||
]
|
||||
}
|
||||
@@ -31,7 +31,7 @@ import subprocess
|
||||
|
||||
import bpy
|
||||
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "Geo", "scripts"))
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..", "..", "Geo", "tools"))
|
||||
import dem_stats # noqa: E402
|
||||
import osm_common as oc # noqa: E402
|
||||
|
||||
+1
-1
@@ -61,7 +61,7 @@ The archive on disk is **Unreal Engine 5.2.0**, released May 2023, three and a h
|
||||
- `[x]` Blender 5.2 LTS (Flatpak) and MPFB 2.0.18 recorded.
|
||||
- `[x]` Install QGIS and record its version. QGIS **4.2.3**, Flatpak `stable`, runtime `org.kde.Platform 6.11`, bundled PROJ `9.9.0`. The stale `lts` 3.28.9 install was uninstalled 2026-11-07; only `stable` remains.
|
||||
- `[x]` Install Blosm in Blender, record its version, and confirm it loads on Blender 5.2. Blosm **2.7.28** imports the block extract headless in `3Dsimple` mode; the exact command and the two required headless workarounds are in `Art/blender/README.md`. No second Blender version was needed.
|
||||
- `[ ]` **Optional.** Confirm the geo tool Python environment (venv with `pyproj`, `GDAL`, `shapely`). Not needed so far: every geo script in `Geo/scripts/` is standard-library only, and PROJ 9.9.0 is invoked from the QGIS Flatpak (`LD_LIBRARY_PATH` trick in `Geo/manifests/local-origin.md`). Add the venv only when a step genuinely needs `pyproj` or GDAL in-process.
|
||||
- `[ ]` **Optional.** Confirm the geo tool Python environment (venv with `pyproj`, `GDAL`, `shapely`). Not needed so far: every geo script in `Geo/tools/` is standard-library only, and PROJ 9.9.0 is invoked from the QGIS Flatpak (`LD_LIBRARY_PATH` trick in `Geo/manifests/local-origin.md`). Add the venv only when a step genuinely needs `pyproj` or GDAL in-process.
|
||||
|
||||
## 4. Project bootstrap
|
||||
|
||||
|
||||
+26
-29
@@ -1,48 +1,45 @@
|
||||
# Geo
|
||||
|
||||
Geographic sources, manifests, and the scripts that turn them into Blender imports.
|
||||
Geographic sources, manifests, and the tools that turn them into Blender and Unreal content.
|
||||
|
||||
OSM is layout data only. It is not authoritative for façade detail, height, or pavement, and it is never sampled as a texture or panorama source.
|
||||
|
||||
The OSM source files are © OpenStreetMap contributors, available under the [Open Database License 1.0](https://opendatacommons.org/licenses/odbl/1-0/); see [OpenStreetMap attribution](https://www.openstreetmap.org/copyright). Public use of a derivative database can require sharing that database or the changes needed to recreate it under ODbL. Review the applicable obligations for each distributed product.
|
||||
|
||||
## Layout
|
||||
|
||||
| Path | Contents |
|
||||
| --- | --- |
|
||||
| `Geo/raw/` | Verbatim source responses, checksummed. Never edited by hand. |
|
||||
| `Geo/verification/` | Per-pass verification notes: address, footprint, bounds, and what remains unresolved |
|
||||
| `Geo/manifests/` | One manifest per saved extract (see below) |
|
||||
| `Geo/scripts/` | Repeatable fetch and conversion helpers |
|
||||
| `Geo/raw/` | Verbatim source responses and downloads, checksummed. Never edited by hand. |
|
||||
| `Geo/manifests/` | One manifest per source: extract, DEM, and the local origin |
|
||||
| `Geo/verification/` | Per-pass notes: address, footprints, bounds, street profile, cross-sections, elevation |
|
||||
| `Geo/tools/` | Every geographic tool (Python, standard library only unless noted) |
|
||||
|
||||
Current state: the extract, both El Capitan footprints, the demo bounds, the building sets, the street profile, and the designated crossing are recorded in [`verification/`](verification/). The local CRS and origin are fixed in [`manifests/local-origin.md`](manifests/local-origin.md), with the projected easting/northing still to be computed. Elevation is measured and the 3DEP 1 m DEM window is committed (EPSG:26911, NAVD88, 984 KB). The street cross-sections (crown ~2 %, gutters at ±10–11 m) are in `verification/`. The hero façade width is resolved at 25.75 m. The geo pipeline is ready for the Blender blockout.
|
||||
Tools that build 3D content from these sources live next to their output in `Art/blender/*/` (`terrain_from_dem.py`, `street_mesh.py`, `site_assembly.py`, `blosm_blockout.py`) because they need Blender; their shared geo helpers and the unit conversion come from `Geo/tools/osm_common.py`.
|
||||
|
||||
## Required manifest
|
||||
## Tools
|
||||
|
||||
Every extract gets a manifest file `Geo/manifests/<source>-<date>.md` recording:
|
||||
|
||||
| Field | Example |
|
||||
| Tool | Purpose |
|
||||
| --- | --- |
|
||||
| Source | Overpass API / Geofabrik / OSM extract provider |
|
||||
| Extract date | date the query ran or the file was downloaded |
|
||||
| Query or file URL | the exact query text or download URL |
|
||||
| Data timestamp | OSM data date reported by the source (may differ from the extract date) |
|
||||
| Bounds | west, south, east, north in WGS84 decimal degrees |
|
||||
| Licence | ODbL 1.0, attribution string |
|
||||
| Checksum | SHA-256 of the committed file |
|
||||
| CRS and local origin | projected CRS plus the chosen origin, with its WGS84 coordinates |
|
||||
| Files | every file produced from this extract |
|
||||
| `osm_common.py` | Shared helpers: parse `.osm`, stitch and clip a street, project to UTM through PROJ, **and the single metres→centimetres conversion** (`to_unreal_local`) |
|
||||
| `block_report.py` | Streets, boundary intersections, and the building set for a block, split by side |
|
||||
| `block_height_report.py` | Every block building with its height **source** — tag, levels × 3.2 m, or a placeholder guess |
|
||||
| `street_profile.py` | Carriageway width from crossings, sidewalk offsets, per-building setbacks |
|
||||
| `street_section.py` | Cross-sections from the DEM: crown, gutters, kerb detection |
|
||||
| `elevation_probe.py` | Samples 3DEP for relief and records the baseline for the release checks |
|
||||
| `dem_stats.py` | Stdlib reader for the tiled GeoTIFF the 3DEP service returns; GDAL does not run on this host |
|
||||
| `asset_metadata.py` | Writes and audits `<asset>.geo.json` sidecars: position, provenance, conversion |
|
||||
|
||||
## Demo bounds
|
||||
## Unit boundary
|
||||
|
||||
`0.1.0` covers 6840 Hollywood Blvd / El Capitan Entertainment Centre, buildings on the south side of Hollywood Boulevard between Highland Avenue and Orange Drive, and north-side buildings whose footprint lies within 50 m of the north frontage. Verify address, footprints, and boundaries against source data before modeling; anything unverifiable is recorded here as unresolved and is not modeled.
|
||||
There is exactly one conversion in the repository:
|
||||
|
||||
## Coordinate convention
|
||||
```
|
||||
WGS84 (lat, lon) → EPSG:26911 (E, N metres) → project-local metres → Unreal centimetres
|
||||
```
|
||||
|
||||
- Work in a local projected CRS (EPSG:26911, NAD83 / UTM zone 11N, is the default candidate) and record the local origin explicitly.
|
||||
- Blender and Unreal both take metres → Unreal units at 1 m = 100 uu. The conversion is applied once, in one place, and recorded in the manifest. Never apply a second silent scale on import.
|
||||
- Preserve the original geographic coordinates in metadata so any asset can be traced back to its position.
|
||||
`Geo/tools/osm_common.py::to_unreal_local` is the only place that multiplies by 100. Per-asset sidecars record the result, and `asset_metadata.py audit` checks each saved sidecar against that function, so a file that drifts is caught rather than trusted.
|
||||
|
||||
## Unresolved items
|
||||
## Current state
|
||||
|
||||
Record here any structure whose address, footprint, or boundary could not be verified from permitted sources, with the sources checked. It does not get modeled.
|
||||
The extract, both El Capitan footprints, the demo bounds, the building sets, the street profile, cross-sections, and the 3DEP DEM window are recorded in `Geo/verification/` and `Geo/manifests/`. The local CRS and origin are fixed in `manifests/local-origin.md`.
|
||||
|
||||
Open items live in `ROADMAP.md` under **Geographic data**.
|
||||
@@ -35,7 +35,7 @@ Min 110.46 m, max 127.73 m, mean 117.43 m over the 550 × 350 m window — a 17.
|
||||
|
||||
## Reading the file
|
||||
|
||||
`Geo/scripts/dem_stats.py` parses it with the standard library. GDAL cannot run on the host (its flatpak libraries do not resolve outside the sandbox), and the exported TIFF is **tiled**, not strip-organised — a strip reader silently scrambles the rows. If another tool misreads it, suspect the tile layout before the data.
|
||||
`Geo/tools/dem_stats.py` parses it with the standard library. GDAL cannot run on the host (its flatpak libraries do not resolve outside the sandbox), and the exported TIFF is **tiled**, not strip-organised — a strip reader silently scrambles the rows. If another tool misreads it, suspect the tile layout before the data.
|
||||
|
||||
## Open
|
||||
|
||||
|
||||
Executable
+132
@@ -0,0 +1,132 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Read, write and audit per-asset geographic metadata — the provenance sidecar.
|
||||
|
||||
Every exported asset gets `<asset>.geo.json` next to it carrying the boundary it
|
||||
was modelled against, its position in EPSG:26911 and in Unreal centimetres, the
|
||||
reasoning behind every non-measured dimension, and the unit conversion used. The
|
||||
file makes three things checkable after the fact:
|
||||
|
||||
* where the asset belongs in the world, without re-deriving it;
|
||||
* which measurements are evidence and which are assumptions;
|
||||
* that the metres-to-centimetres conversion was applied once, consistently.
|
||||
|
||||
Geo/tools/asset_metadata.py write --asset hero-facade --kind building \\
|
||||
--bounds "OSM way 424024147" --centre-easting 376381.9 --centre-northing 3774184 \\
|
||||
--out Art/blender/hero
|
||||
Geo/tools/asset_metadata.py audit Art/blender/hero
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import datetime
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
import osm_common as oc
|
||||
|
||||
SCHEMA = 1
|
||||
|
||||
|
||||
def build(asset, kind, bounds, easting, northing, elevation, report_path, extra):
|
||||
x, y, z = oc.to_unreal_local(easting, northing, elevation if elevation else 0.0)
|
||||
doc = {
|
||||
"schema": SCHEMA,
|
||||
"asset": asset,
|
||||
"kind": kind,
|
||||
"generated_utc": datetime.datetime.now(datetime.timezone.utc).isoformat(timespec="seconds"),
|
||||
"geographic": {
|
||||
"crs": "EPSG:26911",
|
||||
"source": bounds,
|
||||
"centre_easting_m": easting,
|
||||
"centre_northing_m": northing,
|
||||
"elevation_navd88_m": elevation,
|
||||
"origin": {
|
||||
"easting_m": oc.ORIGIN_E,
|
||||
"northing_m": oc.ORIGIN_N,
|
||||
"reference": "Geo/manifests/local-origin.md",
|
||||
},
|
||||
},
|
||||
"unreal": {
|
||||
"centre_cm": [round(x, 1), round(y, 1), None if z is None else round(z, 1)],
|
||||
"conversion": f"{oc.METRES_TO_UNREAL:g} uu per metre, applied once in osm_common.to_unreal_local",
|
||||
"axes": "X east, Y south (FBX preset flips north), Z up",
|
||||
},
|
||||
"dims_report": report_path,
|
||||
"notes": extra,
|
||||
}
|
||||
return doc
|
||||
|
||||
|
||||
def cmd_write(args):
|
||||
doc = build(args.asset, args.kind, args.bounds, args.centre_easting, args.centre_northing,
|
||||
args.elevation, args.report, args.note or [])
|
||||
path = os.path.join(args.out, args.asset + ".geo.json")
|
||||
os.makedirs(args.out, exist_ok=True)
|
||||
with open(path, "w") as f:
|
||||
json.dump(doc, f, indent=2)
|
||||
f.write("\n")
|
||||
print(f"wrote {path}")
|
||||
|
||||
|
||||
def cmd_audit(targets):
|
||||
files = []
|
||||
for t in targets:
|
||||
if os.path.isdir(t):
|
||||
# assets keep their sidecars either in the same folder as the .blend
|
||||
# or one level down, so a directory target must recurse
|
||||
files += glob.glob(os.path.join(t, "**", "*.geo.json"), recursive=True)
|
||||
else:
|
||||
files.append(t)
|
||||
bad = 0
|
||||
for path in sorted(files):
|
||||
try:
|
||||
doc = json.load(open(path))
|
||||
except Exception as exc: # noqa: BLE001
|
||||
print(f"UNREADABLE {path}: {exc}")
|
||||
bad += 1
|
||||
continue
|
||||
missing = [k for k in ("asset", "kind", "geographic", "unreal") if k not in doc]
|
||||
if missing:
|
||||
print(f"MISSING {path}: {missing}")
|
||||
bad += 1
|
||||
continue
|
||||
u = doc["unreal"]["centre_cm"]
|
||||
geo = doc["geographic"]
|
||||
expect_x = (geo["centre_easting_m"] - oc.ORIGIN_E) * oc.METRES_TO_UNREAL
|
||||
expect_y = -(geo["centre_northing_m"] - oc.ORIGIN_N) * oc.METRES_TO_UNREAL
|
||||
if abs(u[0] - expect_x) > 1.0 or abs(u[1] - expect_y) > 1.0:
|
||||
print(f"STALE {path}: centre_cm {u} disagrees with the origin conversion "
|
||||
f"({expect_x:.1f}, {expect_y:.1f})")
|
||||
bad += 1
|
||||
continue
|
||||
print(f"ok {doc['asset']:24} {doc['kind']:10} centre_cm {u[0]:10.1f} {u[1]:10.1f} "
|
||||
f"src {geo['source'][:40]}")
|
||||
print(f"\n{len(files)} sidecars, {bad} problem(s)")
|
||||
return 1 if bad else 0
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
sub = ap.add_subparsers(dest="cmd", required=True)
|
||||
w = sub.add_parser("write")
|
||||
w.add_argument("--asset", required=True)
|
||||
w.add_argument("--kind", required=True, choices=("building", "terrain", "street", "prop", "character"))
|
||||
w.add_argument("--bounds", required=True, help="what the asset was modelled against")
|
||||
w.add_argument("--centre-easting", type=float, required=True)
|
||||
w.add_argument("--centre-northing", type=float, required=True)
|
||||
w.add_argument("--elevation", type=float, default=None, help="NAVD88 metres")
|
||||
w.add_argument("--report", default=None, help="dimension report produced by the build script")
|
||||
w.add_argument("--note", action="append")
|
||||
w.add_argument("--out", required=True)
|
||||
a = sub.add_parser("audit")
|
||||
a.add_argument("targets", nargs="+")
|
||||
args = ap.parse_args()
|
||||
if args.cmd == "write":
|
||||
cmd_write(args)
|
||||
return 0
|
||||
return cmd_audit(args.targets)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Executable
+75
@@ -0,0 +1,75 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Report which block buildings have real height data and which are guesses.
|
||||
|
||||
Usage:
|
||||
Geo/tools/block_height_report.py Geo/raw/osm-api-map-2026-11-06.osm
|
||||
|
||||
`height` and `building:levels` are the only height evidence OSM carries. Where
|
||||
neither exists, Blosm falls back to `defaultLevels * levelHeight`, which is a
|
||||
placeholder: the demo renders a constant-height block instead of a real skyline.
|
||||
This tool lists every building in the demo bounds with its height source, so the
|
||||
substitutions are visible and measurable rather than buried in the model.
|
||||
|
||||
Exit code is 0 either way; the count of guessed buildings is the deliverable.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import math
|
||||
import sys
|
||||
|
||||
import osm_common as oc
|
||||
|
||||
LEVEL_HEIGHT = 3.2
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("osm")
|
||||
args = ap.parse_args()
|
||||
nodes, ways = oc.load(args.osm)
|
||||
|
||||
line, lat0, lon0, lengths, block_len = oc.block_extent(
|
||||
nodes, ways, "Hollywood Boulevard", "North Orange Drive", "North Highland Avenue")
|
||||
|
||||
def in_block(pts):
|
||||
if min(p[1] for p in pts) > line[-1][1] or max(p[1] for p in pts) < line[0][1]:
|
||||
return False
|
||||
return True
|
||||
|
||||
rows = []
|
||||
for wid, tags, refs in ways:
|
||||
if "building" not in tags:
|
||||
continue
|
||||
pts = oc.way_points(nodes, refs)
|
||||
if not pts or not in_block(pts):
|
||||
continue
|
||||
tag_height = tags.get("height")
|
||||
levels = tags.get("building:levels")
|
||||
if tag_height:
|
||||
height, source = float(tag_height), "height tag"
|
||||
elif levels:
|
||||
height, source = float(levels) * LEVEL_HEIGHT, f"levels x {LEVEL_HEIGHT} m"
|
||||
else:
|
||||
height, source = 2 * LEVEL_HEIGHT, f"GUESS: default 2 levels x {LEVEL_HEIGHT} m"
|
||||
nearest = min(oc.project(p, line, lat0, lon0, lengths)[0] for p in pts)
|
||||
side = "S" if oc.project(pts[0], line, lat0, lon0, lengths)[2] < 0 else "N"
|
||||
rows.append((side, nearest, wid, tags.get("name", "") or tags.get("addr:housenumber", ""),
|
||||
height, source))
|
||||
|
||||
rows.sort(key=lambda r: (r[0], r[1]))
|
||||
guessed = [r for r in rows if r[5].startswith("GUESS")]
|
||||
print(f"{len(rows)} buildings in the block, {len(guessed)} with a guessed height\n")
|
||||
for side, setback, wid, name, height, source in rows:
|
||||
flag = " <-- guessed" if source.startswith("GUESS") else ""
|
||||
print(f"{side} way {wid:>10} setback {setback:5.1f} m height {height:5.1f} m "
|
||||
f"({source:26}) {name[:34]}{flag}")
|
||||
|
||||
if guessed:
|
||||
print("\nGuessed heights to replace from the reference photography:")
|
||||
for side, setback, wid, name, height, source in guessed:
|
||||
print(f" {side} way {wid} {name}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -2,7 +2,7 @@
|
||||
"""Report streets, boundary intersections and the demo building set for an OSM extract.
|
||||
|
||||
Usage:
|
||||
Geo/scripts/block_report.py Geo/raw/osm-api-map-2026-11-06.osm \
|
||||
Geo/tools/block_report.py Geo/raw/osm-api-map-2026-11-06.osm \
|
||||
--blvd "Hollywood Boulevard" --west "North Orange Drive" --east "North Highland Avenue"
|
||||
|
||||
Reads a plain .osm XML file (the output of the OSM API `map` endpoint or an
|
||||
@@ -2,8 +2,8 @@
|
||||
"""Read a plain float32 GeoTIFF, report its georeferencing and elevation stats.
|
||||
|
||||
Usage:
|
||||
Geo/scripts/dem_stats.py Geo/raw/dem-3dep-1m-26911-2026-11-06.tif
|
||||
Geo/scripts/dem_stats.py <tif> --sample 376515.389,3774224.083
|
||||
Geo/tools/dem_stats.py Geo/raw/dem-3dep-1m-26911-2026-11-06.tif
|
||||
Geo/tools/dem_stats.py <tif> --sample 376515.389,3774224.083
|
||||
|
||||
Why this exists: GDAL is not runnable on the host (its flatpak libraries do not
|
||||
resolve outside the sandbox), so this reads the strip-based GeoTIFF that the
|
||||
@@ -2,7 +2,7 @@
|
||||
"""Probe ground elevation across the demo block with the USGS 3DEP point service.
|
||||
|
||||
Usage:
|
||||
Geo/scripts/elevation_probe.py Geo/raw/osm-api-map-2026-11-06.osm
|
||||
Geo/tools/elevation_probe.py Geo/raw/osm-api-map-2026-11-06.osm
|
||||
|
||||
Samples a grid along the boulevard (centreline and both sidewalk centrelines),
|
||||
the hero footprint corners, and the crossing midpoints, then reports the relief
|
||||
@@ -1,17 +1,92 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Shared helpers for the Geo scripts: parse .osm, stitch a street, project points.
|
||||
"""Shared geo helpers: parse .osm, stitch a street, project points, convert to Unreal.
|
||||
|
||||
Standard library only. Imported by `block_report.py`, `street_profile.py`, and
|
||||
`elevation_probe.py` when run from this directory.
|
||||
Standard library only. Imported by every script in `Geo/tools/`.
|
||||
|
||||
The unit boundary is deliberate and lives here, once:
|
||||
|
||||
WGS84 (lat, lon) -> EPSG:26911 (E, N metres) -> project local metres
|
||||
-> Unreal centimetres
|
||||
|
||||
`to_unreal` is the only place that multiplies by 100. Nothing else in the
|
||||
repository may convert metres to centimetres, so a second silent scale cannot be
|
||||
introduced by accident.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import os
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
EARTH_R = 6378137.0
|
||||
|
||||
# Project origin, EPSG:26911. See Geo/manifests/local-origin.md.
|
||||
ORIGIN_E = 376515.389
|
||||
ORIGIN_N = 3774224.083
|
||||
ORIGIN_Z_NAVD88 = 115.911
|
||||
METRES_TO_UNREAL = 100.0
|
||||
|
||||
|
||||
def to_unreal_local(easting, northing, elevation=None):
|
||||
"""EPSG:26911 metres -> Unreal centimetres in the project frame.
|
||||
|
||||
X = east of the origin, Y = south of the origin (the FBX preset maps Blender
|
||||
+Y/north to Unreal -Y), Z = height above the origin ground. This is the single
|
||||
meters->centimetres conversion for the repository.
|
||||
"""
|
||||
x_cm = (easting - ORIGIN_E) * METRES_TO_UNREAL
|
||||
y_cm = -(northing - ORIGIN_N) * METRES_TO_UNREAL
|
||||
if elevation is None:
|
||||
return x_cm, y_cm, None
|
||||
return x_cm, y_cm, (elevation - ORIGIN_Z_NAVD88) * METRES_TO_UNREAL
|
||||
|
||||
|
||||
def to_blender_local(easting, northing, elevation=None):
|
||||
"""EPSG:26911 metres -> Blender metres in the project frame (X east, Y north, Z up)."""
|
||||
x = easting - ORIGIN_E
|
||||
y = northing - ORIGIN_N
|
||||
z = None if elevation is None else elevation - ORIGIN_Z_NAVD88
|
||||
return x, y, z
|
||||
|
||||
|
||||
def find_proj():
|
||||
"""The PROJ binary: environment override, PATH, then the QGIS Flatpak."""
|
||||
import glob
|
||||
import os
|
||||
import shutil
|
||||
|
||||
env = os.environ.get("PROJ_BIN")
|
||||
if env and os.path.exists(env):
|
||||
return env
|
||||
which = shutil.which("proj")
|
||||
if which:
|
||||
return which
|
||||
matches = sorted(glob.glob(os.path.expanduser(
|
||||
"~/.local/share/flatpak/app/org.qgis.qgis/x86_64/stable/*/files/bin/proj")))
|
||||
if not matches:
|
||||
raise SystemExit("no PROJ binary found; set PROJ_BIN or install pyproj")
|
||||
return matches[0]
|
||||
|
||||
|
||||
def project_wgs84_to_utm(points, proj_bin=None):
|
||||
"""WGS84 (lat, lon) -> EPSG:26911 (E, N) for a list, in one PROJ call."""
|
||||
import subprocess
|
||||
|
||||
proj_bin = proj_bin or find_proj()
|
||||
app_files = os.path.dirname(os.path.dirname(proj_bin))
|
||||
env = dict(os.environ)
|
||||
env["LD_LIBRARY_PATH"] = f"{app_files}/lib:{env.get('LD_LIBRARY_PATH', '')}"
|
||||
payload = "\n".join(f"{lon:.9f} {lat:.9f}" for lat, lon in points) + "\n"
|
||||
out = subprocess.run(
|
||||
[proj_bin, "+proj=utm", "+zone=11", "+datum=NAD83", "-f", "%.3f"],
|
||||
input=payload, capture_output=True, text=True, env=env, check=True,
|
||||
).stdout.strip().splitlines()
|
||||
result = [tuple(float(v) for v in line.split()) for line in out]
|
||||
if len(result) != len(points):
|
||||
raise SystemExit(f"PROJ returned {len(result)} points for {len(points)} inputs")
|
||||
return result
|
||||
|
||||
|
||||
def to_xy(lat: float, lon: float, lat0: float, lon0: float) -> tuple[float, float]:
|
||||
"""Local planar metres, east/north, centred on (lat0, lon0)."""
|
||||
@@ -2,7 +2,7 @@
|
||||
"""Street profile for a block: frontage setbacks, carriageway width, crossings.
|
||||
|
||||
Usage:
|
||||
Geo/scripts/street_profile.py Geo/raw/osm-api-map-2026-11-06.osm \
|
||||
Geo/tools/street_profile.py Geo/raw/osm-api-map-2026-11-06.osm \
|
||||
--blvd "Hollywood Boulevard" --west "North Orange Drive" --east "North Highland Avenue"
|
||||
|
||||
Measures, in metres, from a plain .osm extract (standard library only):
|
||||
@@ -2,7 +2,7 @@
|
||||
"""Cross-sections of the street from the local 3DEP DEM.
|
||||
|
||||
Usage:
|
||||
Geo/scripts/street_section.py Geo/raw/osm-api-map-2026-11-06.osm \
|
||||
Geo/tools/street_section.py Geo/raw/osm-api-map-2026-11-06.osm \
|
||||
Geo/raw/dem-3dep-1m-26911-2026-11-06.tif --step 0.5
|
||||
|
||||
For several positions along the block it samples the DEM on a line perpendicular
|
||||
@@ -1,6 +1,6 @@
|
||||
# Cross-sections — crowned road, gutters, sidewalk shelves
|
||||
|
||||
Sampled 2026-11-06 from the local 3DEP DEM (`Geo/raw/dem-3dep-1m-26911-2026-11-06.tif`) with `Geo/scripts/street_section.py`. Seven sections at 40 m spacing, offsets −18 m to +18 m from the boulevard centreline (negative = south), 1 m sampling step.
|
||||
Sampled 2026-11-06 from the local 3DEP DEM (`Geo/raw/dem-3dep-1m-26911-2026-11-06.tif`) with `Geo/tools/street_section.py`. Seven sections at 40 m spacing, offsets −18 m to +18 m from the boulevard centreline (negative = south), 1 m sampling step.
|
||||
|
||||
## What the profile shows
|
||||
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
# Elevation check — Hollywood Blvd block
|
||||
|
||||
Measured 2026-11-06 with `Geo/scripts/elevation_probe.py` against the USGS Elevation Point Query Service (3DEP, 1 m seamless DEM, raster id 129943). Raw responses: `Geo/raw/epqs-2026-11-06.json`. Reproduce with:
|
||||
Measured 2026-11-06 with `Geo/tools/elevation_probe.py` against the USGS Elevation Point Query Service (3DEP, 1 m seamless DEM, raster id 129943). Raw responses: `Geo/raw/epqs-2026-11-06.json`. Reproduce with:
|
||||
|
||||
```bash
|
||||
python3 Geo/scripts/elevation_probe.py Geo/raw/osm-api-map-2026-11-06.osm --workers 5
|
||||
python3 Geo/tools/elevation_probe.py Geo/raw/osm-api-map-2026-11-06.osm --workers 5
|
||||
```
|
||||
|
||||
Heights are **orthometric, NAVD88**, as published for 3DEP CONUS. They are not ellipsoidal heights and must not be mixed with GPS/ellipsoid values.
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
# Street profile — Hollywood Blvd block
|
||||
|
||||
Measured 2026-11-06 from the block extract (`Geo/raw/osm-api-map-2026-11-06.osm`) with `Geo/scripts/street_profile.py`. Reproduce with:
|
||||
Measured 2026-11-06 from the block extract (`Geo/raw/osm-api-map-2026-11-06.osm`) with `Geo/tools/street_profile.py`. Reproduce with:
|
||||
|
||||
```bash
|
||||
python3 Geo/scripts/street_profile.py Geo/raw/osm-api-map-2026-11-06.osm
|
||||
python3 Geo/tools/street_profile.py Geo/raw/osm-api-map-2026-11-06.osm
|
||||
```
|
||||
|
||||
All distances are metres, computed on a local planar approximation at the block's mean latitude (error well under 1 cm over 300 m). OSM is layout data; these are geometry measurements, not a survey.
|
||||
|
||||
@@ -52,7 +52,7 @@ Scope: facts and traps that cost time to establish. Planned work lives in [ROADM
|
||||
|
||||
## 2026-11-06 — Geo data traps
|
||||
|
||||
- **The 3DEP export is a tiled TIFF, not strips.** A strip reader silently scrambles rows and produces fake 0 m / 127 m bands. `Geo/scripts/dem_stats.py` handles tiles.
|
||||
- **The 3DEP export is a tiled TIFF, not strips.** A strip reader silently scrambles rows and produces fake 0 m / 127 m bands. `Geo/tools/dem_stats.py` handles tiles.
|
||||
- **Sample walk routes through PROJ.** Placing samples with the OSM spherical approximation put them metres off the UTM meshes and reported 91 phantom holes. `/tmp` is not shared between tool calls here either — regenerate instead of saving intermediate files there.
|
||||
- **3DEP cannot resolve kerbs.** Sharpest break near the kerb line is 0.09–0.12 m/m, a gutter transition, not a 0.15 m upstand. Kerb height stays a stated assumption.
|
||||
- **Building interiors are not ground.** Points inside the hero footprint returned 114.5–117.1 m — structure, not terrain. Excluded from the relief figure.
|
||||
|
||||
+15
-3
@@ -42,15 +42,27 @@ Status detail in [Docs/setup.md](Docs/setup.md); pins in [Docs/toolchain.md](Doc
|
||||
- [x] Export one MPFB character with baked materials and deform skeleton; verify import. 2026-11-07: MPFB 2.0.18 generated human (19 158 verts, 1.66 m tall) + standard rig with weights (137 bones) + v2 skin baked to a 1024² diffuse texture (`Tools/mpfb_character_export.py`, export preset `ue_character_v1`). Imported headless (`Tools/ue_character_import.py`) into `/Game/Test/Character`: `SkeletalMesh` + 138-bone `Skeleton` + `Material` with the baked `Texture2D` assigned to slot 0. Quirks recorded: 5.8 Interchange skips materials for skeletal FBX imports (finished in script); Blender 5.2 dropped the FBX `armature_nla` option; the baked image needs `image.filepath` set for FBX to reference it.
|
||||
- [x] Document initial setup in `Docs/`: engine install, project checkout, fresh-checkout build procedure. 2026-11-07: `Docs/setup.md` reconciled with reality (all engine/bootstrap items marked done with evidence); `Docs/build.md` completed with the fresh-checkout build, packaging, and headless-content-check procedures plus known engine quirks.
|
||||
|
||||
### Geographic data
|
||||
### Geographic data — active — gig-line-geo
|
||||
|
||||
Tools ported 2026-11-07: every geographic script now lives in `Geo/tools/` with `Geo/README.md` listing its purpose, and the Blender-side builders moved next to their outputs in `Art/blender/*/`, importing the shared helpers and the single unit conversion from `Geo/tools/osm_common.py`.
|
||||
|
||||
| Tool | Purpose |
|
||||
| --- | --- |
|
||||
| `osm_common.py` | parse `.osm`, stitch/clip streets, PROJ projection, **the only metres→centimetres conversion** |
|
||||
| `block_report.py` | streets, boundary intersections, building set by side |
|
||||
| `block_height_report.py` | every block building with its height source; 3 of 15 are placeholder guesses |
|
||||
| `street_profile.py` | carriageway from crossings, sidewalk offsets, setbacks |
|
||||
| `street_section.py` | DEM cross-sections, crown and gutters |
|
||||
| `elevation_probe.py` | 3DEP relief sampling |
|
||||
| `dem_stats.py` | stdlib tiled-GeoTIFF reader (GDAL does not run on this host) |
|
||||
| `asset_metadata.py` | writes and audits the `<asset>.geo.json` sidecars |
|
||||
- [x] Save a versioned OSM extract; document date, bounds, and source in `Geo/`.
|
||||
- [x] Acquire separately licensed elevation data if needed. Needed and acquired: USGS 3DEP 1 m DEM in EPSG:26911, public domain, NAVD88, 984 KB committed (`Geo/manifests/2026-11-06-3dep-dem.md`); measured relief 3.94 m.
|
||||
- [x] Download the 3DEP 1 m DEM for the demo bounds, reproject it to EPSG:26911 with the recorded origin and vertical datum, and record the source, clip bounds, and checksum in `Geo/`.
|
||||
- [x] In QGIS, set a suitable local projected CRS (e.g. EPSG:26911) and record a local origin. EPSG:26911, origin and projected easting/northing are recorded in `Geo/manifests/local-origin.md` with PROJ 9.9.0 control points; QGIS 4.2.3 is installed for the DEM reprojection step.
|
||||
- [x] Verify the address, building footprints, and demo boundaries for 6840 Hollywood Blvd against source data.
|
||||
- [-] Import OSM blockouts through Blosm; clean and detail them in Blender. Imported 2026-11-06 (`Art/blender/blockout/blockout-2026-11-06.blend`, 86 meshes, scale 1:1 against OSM, re-framed into EPSG:26911 with a 0.0 cm affine residual); cleaning and detailing not started.
|
||||
- [-] Preserve geographic coordinates in metadata; define explicit meter→centimeter conversion to local Unreal coordinates. The frame is defined (`Geo/manifests/local-origin.md`: EPSG:26911, origin Hollywood × Highland, X east / Y north / Z up) and the ×100 conversion is now **verified** by the 2026-11-07 scale test, with the axis mapping recorded (north realizes as UE −Y). Left: build the production export preset for the actual site meshes using these settings.
|
||||
- [~] Import OSM blockouts through Blosm; clean and detail them in Blender. Imported 2026-11-06 and re-framed to EPSG:26911. Tooling for the cleaning pass is now in `Geo/tools/` (height provenance, block report) and `Art/blender/blockout/blosm_blockout.py`. **Still open:** 3 of 15 block buildings carry no height data and render at a placeholder 6.4 m, roofs and facades are untextured, and Blosm `setupScript` façade styles are unused. Earlier: (`Art/blender/blockout/blockout-2026-11-06.blend`, 86 meshes, scale 1:1 against OSM, re-framed into EPSG:26911 with a 0.0 cm affine residual); cleaning and detailing not started.
|
||||
- [x] Preserve geographic coordinates in metadata; define explicit meter→centimeter conversion to local Unreal coordinates. Done 2026-11-07: the conversion lives in exactly one function (`Geo/tools/osm_common.py::to_unreal_local`) and every exporter must call it; `<asset>.geo.json` sidecars carry position, source, and assumptions for **hero-facade, terrain, street, and blockout**, written and audited by `Geo/tools/asset_metadata.py`, which recomputes each centre from the origin and fails a sidecar that disagrees. Props follow the same schema when they are imported. The frame is defined (`Geo/manifests/local-origin.md`: EPSG:26911, origin Hollywood × Highland, X east / Y north / Z up) and the ×100 conversion is now **verified** by the 2026-11-07 scale test, with the axis mapping recorded (north realizes as UE −Y). Left: build the production export preset for the actual site meshes using these settings.
|
||||
|
||||
### Review scene and player placement - active - gig-line-review-placement
|
||||
|
||||
|
||||
+3
-4
@@ -14,10 +14,9 @@ Repeatable import/export and processing helpers. Anything that was done twice by
|
||||
|
||||
- `render_screenshots.sh` — visual-review pipeline: rebuilds `/Game/Test/ReviewMap` from current content (`ue_build_review_map.py`), runs the game on it in `-game` mode, lets `AGigLineShotActor` (`Game/GigLine/Source/GigLine/`) capture the fixed camera list, and copies the PNGs into the gitignored `Screenshots/`. See `Screenshots/README.md`.
|
||||
- `check-lfs.sh` — repository guard: fails if `git-lfs` or the `lfs` filter is missing, or if an LFS-tracked file was committed as a plain blob.
|
||||
- `site_assembly.py` — assembles terrain + street + blockout into one file and raycasts the walking routes for holes and steps.
|
||||
- `street_mesh.py` — road, kerbs, and sidewalks on the DEM, in the project frame; kerbs flush at the three crossings.
|
||||
- `terrain_from_dem.py` — terrain grid from the committed DEM, with building footprints cut out.
|
||||
- `blosm_blockout.py` — headless Blosm import of the committed `.osm` extract into a `.blend`, with a per-building scale report. Exact invocation in `Art/blender/README.md`.
|
||||
- Geographic tools and their shared helpers live in `Geo/tools/` (see `Geo/README.md`).
|
||||
- Blender-side builders that turn geo data into meshes live beside their output: `Art/blender/terrain/terrain_from_dem.py`, `Art/blender/street/street_mesh.py`, `Art/blender/site/site_assembly.py`, `Art/blender/blockout/blosm_blockout.py`. They import their helpers from `Geo/tools/osm_common.py`, which is also the only place that converts metres to Unreal centimetres.
|
||||
- `sibling_asset_audit.py` — reads the sibling project's prop library straight from GLB headers and writes `Art/props/reuse-manifest.json`; copies nothing.
|
||||
- `page_image_fetch.py` — resolves a web page's `og:image` and downloads it, with page URL and hash in a local sidecar. Works for Flickr, Commons file pages, and most photo blogs.
|
||||
- `commons_fetch.py` — Wikimedia Commons reference fetcher. Search, exact `File:` titles, or a whole Commons category (`--category`), thumbnails, provenance `sidecar.json` per folder. Wikimedia throttles `upload.wikimedia.org` (HTTP 429), so keep batches to six or fewer files and pass `--delay 3`; a throttled file is skipped and recorded rather than aborting the run. Writes only into the gitignored `Art/reference/`; reference images are never committed.
|
||||
|
||||
|
||||
Reference in new issue
Block a user