From 1c8adb36fef3434893633150d379efe6d3a783b2 Mon Sep 17 00:00:00 2001 From: zwitschi Date: Sat, 11 Oct 2025 20:21:14 +0200 Subject: [PATCH] feat: Add OSM refresh script and update loading scripts for improved database handling --- README.md | 131 ++++++++++--------- backend/scripts/osm_refresh.py | 207 +++++++++++++++++++++++++++++++ backend/scripts/stations_load.py | 11 +- backend/scripts/tracks_load.py | 11 +- 4 files changed, 297 insertions(+), 63 deletions(-) create mode 100644 backend/scripts/osm_refresh.py diff --git a/README.md b/README.md index 08b2fc9..21cd81f 100644 --- a/README.md +++ b/README.md @@ -4,56 +4,6 @@ A browser-based railway simulation game using real world railway maps from OpenS ## Features -- Real world railway maps -- Interactive Leaflet map preview of the demo network snapshot -- Build and manage your own railway network -- Dynamic train schedules - -## Architecture - -The project is built using the following technologies: - -- Frontend: HTML5, CSS3, JavaScript, React -- Backend: Python, FastAPI, Flask, SQLAlchemy -- Database: PostgreSQL with PostGIS extension -- Mapping: Leaflet, OpenStreetMap - -## Project Structure - -Planned structure for code and assets (folders created as needed): - -```text -rail-game/ -|-- backend/ -| |-- app/ -| | |-- api/ # FastAPI/Flask route handlers -| | |-- core/ # Config, startup, shared utilities -| | |-- models/ # SQLAlchemy models and schemas -| | |-- services/ # Domain logic and service layer -| | `-- websocket/ # Real-time communication handlers -| |-- tests/ # Backend unit and integration tests -| `-- requirements/ # Backend dependency lockfiles -|-- frontend/ -| |-- public/ # Static assets served as-is -| |-- src/ -| | |-- components/ # Reusable React components -| | |-- hooks/ # Custom React hooks -| | |-- pages/ # Top-level routed views -| | |-- state/ # Redux/Context stores and slices -| | |-- styles/ # Global and modular stylesheets -| | `-- utils/ # Frontend helpers and formatters -| `-- tests/ # Frontend unit and integration tests -|-- docs/ # Architecture docs, ADRs, guides -|-- infra/ # Deployment, IaC, Docker, CI workflows -|-- scripts/ # Tooling for setup, linting, migrations -|-- data/ # Seed data, fixtures, import/export tooling -`-- tests/ # End-to-end and cross-cutting tests -``` - -Use `infra/` to capture deployment assets (Dockerfiles, compose files, Terraform) and `.github/` for automation. Shared code that crosses layers should live in the respective service directories or dedicated packages under `backend/`. - -## Installation - 1. Clone the repository: ```bash @@ -83,6 +33,60 @@ Use `infra/` to capture deployment assets (Dockerfiles, compose files, Terraform copy .env.example .env # PowerShell: Copy-Item .env.example .env ``` + `DATABASE_URL`, `TEST_DATABASE_URL`, and `ALEMBIC_DATABASE_URL` control the runtime, test, and migration connections respectively. + +5. (Optional) Point Git to the bundled hooks: `pwsh scripts/setup_hooks.ps1`. + +6. Run database migrations to set up the schema: + + ```bash + cd backend + alembic upgrade head + cd .. + ``` + +7. Refresh OpenStreetMap fixtures (stations + tracks) into the local database: + + ```bash + python -m backend.scripts.osm_refresh --region all + ``` + + Use `--no-commit` to dry-run the loader against existing data, or skip specific steps with the `--skip-*` flags. + +8. Start the development servers from separate terminals: + + - Backend: `uvicorn backend.app.main:app --reload --port 8000` + - Frontend: `cd frontend && npm run dev` + +9. Open your browser: frontend runs at `http://localhost:5173`, backend API at `http://localhost:8000`. + +10. Run quality checks: + + - Backend unit tests: `pytest` + - Backend formatters: `black backend/` and `isort backend/` + - Frontend lint: `cd frontend && npm run lint` + - Frontend type/build check: `cd frontend && npm run build` + +11. Build for production: + + - Frontend bundle: `cd frontend && npm run build` + - Backend container: `docker build -t rail-game-backend backend/` + +12. Run containers: + + - Backend: `docker run -p 8000:8000 rail-game-backend` + - Frontend: Serve `frontend/dist` with any static file host. + cd frontend + npm install + cd .. + ``` + +11. Build for production: + + ```bash + copy .env.example .env # PowerShell: Copy-Item .env.example .env +12. Run containers: + `DATABASE_URL`, `TEST_DATABASE_URL`, and `ALEMBIC_DATABASE_URL` control the runtime, test, and migration connections respectively. 5. (Optional) Point Git to the bundled hooks: `pwsh scripts/setup_hooks.ps1`. 6. Run database migrations to set up the schema: @@ -93,27 +97,36 @@ Use `infra/` to capture deployment assets (Dockerfiles, compose files, Terraform cd .. ``` -7. Start the development servers from separate terminals: +7. Refresh OpenStreetMap fixtures (stations + tracks) into the local database: + + ```bash + python -m backend.scripts.osm_refresh --region all + ``` + + Use `--no-commit` to dry-run the loader against existing data, or skip specific steps with the `--skip-*` flags. + +8. Start the development servers from separate terminals: - Backend: `uvicorn backend.app.main:app --reload --port 8000` - Frontend: `cd frontend && npm run dev` -8. Open your browser: frontend runs at `http://localhost:5173`, backend API at `http://localhost:8000`. -9. Run quality checks: +9. Open your browser: frontend runs at `http://localhost:5173`, backend API at `http://localhost:8000`. +10. Run quality checks: - Backend unit tests: `pytest` - Backend formatters: `black backend/` and `isort backend/` - Frontend lint: `cd frontend && npm run lint` - Frontend type/build check: `cd frontend && npm run build` -10. Build for production: +11. Build for production: - - Frontend bundle: `cd frontend && npm run build` - - Backend container: `docker build -t rail-game-backend backend/` + - Frontend bundle: `cd frontend && npm run build` + - Backend container: `docker build -t rail-game-backend backend/` -11. Run containers: - - Backend: `docker run -p 8000:8000 rail-game-backend` - - Frontend: Serve `frontend/dist` with any static file host. +12. Run containers: + + - Backend: `docker run -p 8000:8000 rail-game-backend` + - Frontend: Serve `frontend/dist` with any static file host. ## Database Migrations diff --git a/backend/scripts/osm_refresh.py b/backend/scripts/osm_refresh.py new file mode 100644 index 0000000..7d6c0d3 --- /dev/null +++ b/backend/scripts/osm_refresh.py @@ -0,0 +1,207 @@ +from __future__ import annotations + +"""Orchestrate the OSM station/track import and load pipeline.""" + +import argparse +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Callable, Sequence + +from backend.app.core.osm_config import DEFAULT_REGIONS +from backend.scripts import ( + stations_import, + stations_load, + tracks_import, + tracks_load, +) + + +@dataclass(slots=True) +class Stage: + label: str + runner: Callable[[list[str] | None], int] + args: list[str] + input_path: Path | None = None + output_path: Path | None = None + + +def build_argument_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Run the station and track import/load workflow in sequence.", + ) + parser.add_argument( + "--region", + choices=[region.name for region in DEFAULT_REGIONS] + ["all"], + default="all", + help="Region selector forwarded to the import scripts (default: all).", + ) + parser.add_argument( + "--output-dir", + type=Path, + default=Path("data"), + help="Directory where intermediate JSON payloads are stored (default: data/).", + ) + parser.add_argument( + "--stations-json", + type=Path, + help="Existing station JSON file to load; defaults to /osm_stations.json.", + ) + parser.add_argument( + "--tracks-json", + type=Path, + help="Existing track JSON file to load; defaults to /osm_tracks.json.", + ) + parser.add_argument( + "--skip-station-import", + action="store_true", + help="Skip the station import step (expects --stations-json to point to data).", + ) + parser.add_argument( + "--skip-station-load", + action="store_true", + help="Skip loading stations into PostGIS.", + ) + parser.add_argument( + "--skip-track-import", + action="store_true", + help="Skip the track import step (expects --tracks-json to point to data).", + ) + parser.add_argument( + "--skip-track-load", + action="store_true", + help="Skip loading tracks into PostGIS.", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Print the planned stages without invoking Overpass or mutating the database.", + ) + parser.add_argument( + "--commit", + dest="commit", + action="store_true", + default=True, + help="Commit database changes produced by the load steps (default).", + ) + parser.add_argument( + "--no-commit", + dest="commit", + action="store_false", + help="Rollback database changes after load steps (dry run).", + ) + return parser + + +def _build_stage_plan(args: argparse.Namespace) -> tuple[list[Stage], Path, Path]: + station_json = args.stations_json or args.output_dir / "osm_stations.json" + track_json = args.tracks_json or args.output_dir / "osm_tracks.json" + + stages: list[Stage] = [] + + if not args.skip_station_import: + stages.append( + Stage( + label="Import stations", + runner=stations_import.main, + args=["--output", str(station_json), "--region", args.region], + output_path=station_json, + ) + ) + + if not args.skip_station_load: + load_args = [str(station_json)] + if not args.commit: + load_args.append("--no-commit") + stages.append( + Stage( + label="Load stations", + runner=stations_load.main, + args=load_args, + input_path=station_json, + ) + ) + + if not args.skip_track_import: + stages.append( + Stage( + label="Import tracks", + runner=tracks_import.main, + args=["--output", str(track_json), "--region", args.region], + output_path=track_json, + ) + ) + + if not args.skip_track_load: + load_args = [str(track_json)] + if not args.commit: + load_args.append("--no-commit") + stages.append( + Stage( + label="Load tracks", + runner=tracks_load.main, + args=load_args, + input_path=track_json, + ) + ) + + return stages, station_json, track_json + + +def _describe_plan(stages: Sequence[Stage]) -> None: + if not stages: + print("No stages selected; nothing to do.") + return + + print("Selected stages:") + for stage in stages: + detail = " ".join(stage.args) if stage.args else "" + print(f" - {stage.label}: {detail}") + + +def _execute_stage(stage: Stage) -> None: + print(f"\n>>> {stage.label}") + + if stage.output_path is not None: + stage.output_path.parent.mkdir(parents=True, exist_ok=True) + + if stage.input_path is not None and not stage.input_path.exists(): + raise RuntimeError( + f"Expected input file {stage.input_path} for {stage.label}; run the import step first or provide an existing file." + ) + + try: + exit_code = stage.runner(stage.args) + except SystemExit as exc: # argparse.error exits via SystemExit + exit_code = int(exc.code or 0) + + if exit_code: + raise RuntimeError(f"{stage.label} failed with exit code {exit_code}.") + + +def main(argv: list[str] | None = None) -> int: + parser = build_argument_parser() + args = parser.parse_args(argv) + + stages, station_json, track_json = _build_stage_plan(args) + + if args.dry_run: + print("Dry run: the following stages would run in order.") + _describe_plan(stages) + return 0 + + # Ensure parent directories exist when we plan to write files. + if not args.skip_station_import: + station_json.parent.mkdir(parents=True, exist_ok=True) + if not args.skip_track_import: + track_json.parent.mkdir(parents=True, exist_ok=True) + + for stage in stages: + _execute_stage(stage) + + print("\nOSM refresh pipeline completed successfully.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/backend/scripts/stations_load.py b/backend/scripts/stations_load.py index 5f3e566..531a587 100644 --- a/backend/scripts/stations_load.py +++ b/backend/scripts/stations_load.py @@ -23,10 +23,17 @@ def build_argument_parser() -> argparse.ArgumentParser: help="Path to the normalized station JSON file produced by stations_import.py", ) parser.add_argument( - "--commit/--no-commit", + "--commit", dest="commit", + action="store_true", default=True, - help="Commit the transaction (default: commit). Use --no-commit for dry runs.", + help="Commit the transaction after loading (default).", + ) + parser.add_argument( + "--no-commit", + dest="commit", + action="store_false", + help="Rollback the transaction after loading (useful for dry runs).", ) return parser diff --git a/backend/scripts/tracks_load.py b/backend/scripts/tracks_load.py index cda1484..11d37fc 100644 --- a/backend/scripts/tracks_load.py +++ b/backend/scripts/tracks_load.py @@ -45,10 +45,17 @@ def build_argument_parser() -> argparse.ArgumentParser: help="Path to the normalized track JSON file produced by tracks_import.py", ) parser.add_argument( - "--commit/--no-commit", + "--commit", dest="commit", + action="store_true", default=True, - help="Commit the transaction (default: commit). Use --no-commit for dry runs.", + help="Commit the transaction after loading (default).", + ) + parser.add_argument( + "--no-commit", + dest="commit", + action="store_false", + help="Rollback the transaction after loading (useful for dry runs).", ) return parser