diff --git a/main.py b/main.py index 999b17c..6aab622 100644 --- a/main.py +++ b/main.py @@ -17,6 +17,7 @@ from models import ( from routes.auth import router as auth_router from routes.dashboard import router as dashboard_router from routes.imports import router as imports_router +from routes.exports import router as exports_router from routes.projects import router as projects_router from routes.scenarios import router as scenarios_router from services.bootstrap import bootstrap_admin @@ -63,6 +64,7 @@ async def ensure_admin_bootstrap() -> None: app.include_router(dashboard_router) app.include_router(auth_router) app.include_router(imports_router) +app.include_router(exports_router) app.include_router(projects_router) app.include_router(scenarios_router) diff --git a/routes/dashboard.py b/routes/dashboard.py index dad24b6..d6ecdd3 100644 --- a/routes/dashboard.py +++ b/routes/dashboard.py @@ -7,8 +7,7 @@ from fastapi.responses import HTMLResponse from fastapi.templating import Jinja2Templates from dependencies import get_unit_of_work, require_authenticated_user -from models import User -from models import ScenarioStatus +from models import ScenarioStatus, User from services.unit_of_work import UnitOfWork router = APIRouter(tags=["Dashboard"]) @@ -120,5 +119,9 @@ def dashboard_home( "recent_projects": _load_recent_projects(uow), "simulation_updates": _load_simulation_updates(uow), "scenario_alerts": _load_scenario_alerts(request, uow), + "export_modals": { + "projects": request.url_for("exports.modal", dataset="projects"), + "scenarios": request.url_for("exports.modal", dataset="scenarios"), + }, } return templates.TemplateResponse(request, "dashboard.html", context) diff --git a/routes/exports.py b/routes/exports.py new file mode 100644 index 0000000..e896d37 --- /dev/null +++ b/routes/exports.py @@ -0,0 +1,132 @@ +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Annotated + +from fastapi import APIRouter, Depends, HTTPException, Request, Response, status +from fastapi.responses import HTMLResponse, StreamingResponse +from fastapi.templating import Jinja2Templates + +from dependencies import get_unit_of_work, require_any_role +from schemas.exports import ( + ExportFormat, + ProjectExportRequest, + ScenarioExportRequest, +) +from services.export_serializers import ( + export_projects_to_excel, + export_scenarios_to_excel, + stream_projects_to_csv, + stream_scenarios_to_csv, +) +from services.unit_of_work import UnitOfWork + +router = APIRouter(prefix="/exports", tags=["exports"]) + + +@router.get( + "/modal/{dataset}", + response_model=None, + response_class=HTMLResponse, + include_in_schema=False, + name="exports.modal", +) +async def export_modal( + dataset: str, + request: Request, +) -> HTMLResponse: + dataset = dataset.lower() + if dataset not in {"projects", "scenarios"}: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail="Unknown dataset") + + submit_url = request.url_for( + "export_projects" if dataset == "projects" else "export_scenarios" + ) + templates = Jinja2Templates(directory="templates") + return templates.TemplateResponse( + request, + "exports/modal.html", + { + "dataset": dataset, + "submit_url": submit_url, + }, + ) + + +def _timestamp_suffix() -> str: + return datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S") + + +def _ensure_repository(repo, name: str): + if repo is None: + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"{name} repository unavailable") + return repo + + +@router.post( + "/projects", + status_code=status.HTTP_200_OK, + response_class=StreamingResponse, + dependencies=[Depends(require_any_role( + "admin", "project_manager", "analyst"))], +) +async def export_projects( + request: ProjectExportRequest, + uow: Annotated[UnitOfWork, Depends(get_unit_of_work)], +) -> Response: + project_repo = _ensure_repository( + getattr(uow, "projects", None), "Project") + projects = project_repo.filtered_for_export(request.filters) + + filename = f"projects-{_timestamp_suffix()}" + + if request.format == ExportFormat.CSV: + stream = stream_projects_to_csv(projects) + response = StreamingResponse(stream, media_type="text/csv") + response.headers["Content-Disposition"] = f"attachment; filename={filename}.csv" + return response + + data = export_projects_to_excel(projects) + return StreamingResponse( + iter([data]), + media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + headers={ + "Content-Disposition": f"attachment; filename={filename}.xlsx", + }, + ) + + +@router.post( + "/scenarios", + status_code=status.HTTP_200_OK, + response_class=StreamingResponse, + dependencies=[Depends(require_any_role( + "admin", "project_manager", "analyst"))], +) +async def export_scenarios( + request: ScenarioExportRequest, + uow: Annotated[UnitOfWork, Depends(get_unit_of_work)], +) -> Response: + scenario_repo = _ensure_repository( + getattr(uow, "scenarios", None), "Scenario") + scenarios = scenario_repo.filtered_for_export( + request.filters, include_project=True) + + filename = f"scenarios-{_timestamp_suffix()}" + + if request.format == ExportFormat.CSV: + stream = stream_scenarios_to_csv(scenarios) + response = StreamingResponse(stream, media_type="text/csv") + response.headers["Content-Disposition"] = f"attachment; filename={filename}.csv" + return response + + data = export_scenarios_to_excel(scenarios) + return StreamingResponse( + iter([data]), + media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + headers={ + "Content-Disposition": f"attachment; filename={filename}.xlsx", + }, + ) diff --git a/schemas/exports.py b/schemas/exports.py new file mode 100644 index 0000000..f1a6105 --- /dev/null +++ b/schemas/exports.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +from enum import Enum +from typing import Literal + +from pydantic import BaseModel, ConfigDict, field_validator + +from services.export_query import ProjectExportFilters, ScenarioExportFilters + + +class ExportFormat(str, Enum): + CSV = "csv" + XLSX = "xlsx" + + +class BaseExportRequest(BaseModel): + format: ExportFormat = ExportFormat.CSV + include_metadata: bool = False + + model_config = ConfigDict(extra="forbid") + + +class ProjectExportRequest(BaseExportRequest): + filters: ProjectExportFilters | None = None + + @field_validator("filters", mode="before") + @classmethod + def validate_filters(cls, value: ProjectExportFilters | None) -> ProjectExportFilters | None: + if value is None: + return None + if isinstance(value, ProjectExportFilters): + return value + return ProjectExportFilters(**value) + + +class ScenarioExportRequest(BaseExportRequest): + filters: ScenarioExportFilters | None = None + + @field_validator("filters", mode="before") + @classmethod + def validate_filters(cls, value: ScenarioExportFilters | None) -> ScenarioExportFilters | None: + if value is None: + return None + if isinstance(value, ScenarioExportFilters): + return value + return ScenarioExportFilters(**value) + + +class ExportTicket(BaseModel): + token: str + format: ExportFormat + resource: Literal["projects", "scenarios"] + + model_config = ConfigDict(extra="forbid") + + +class ExportResponse(BaseModel): + ticket: ExportTicket + + model_config = ConfigDict(extra="forbid") + + +__all__ = [ + "ExportFormat", + "ProjectExportRequest", + "ScenarioExportRequest", + "ExportTicket", + "ExportResponse", +] diff --git a/static/css/imports.css b/static/css/imports.css new file mode 100644 index 0000000..799ccec --- /dev/null +++ b/static/css/imports.css @@ -0,0 +1,86 @@ +.import-upload { + background-color: var(--surface-color); + border: 1px dashed var(--border-color); + border-radius: var(--radius-md); + padding: 1.5rem; + margin-bottom: 1.5rem; +} + +.import-upload__header { + margin-bottom: 1rem; +} + +.import-upload__dropzone { + border: 2px dashed var(--border-color); + border-radius: var(--radius-sm); + padding: 2rem; + text-align: center; + transition: border-color 0.2s ease, background-color 0.2s ease; +} + +.import-upload__dropzone.dragover { + border-color: var(--primary-color); + background-color: rgba(0, 123, 255, 0.05); +} + +.import-upload__actions { + display: flex; + gap: 0.75rem; + margin-top: 1rem; +} + +.table-cell-actions { + display: flex; + align-items: center; + gap: 0.5rem; +} + +.btn-ghost { + background: transparent; + border: none; + cursor: pointer; + padding: 0.25rem 0.5rem; + color: var(--text-muted); +} + +.btn-ghost:hover { + color: var(--primary-color); +} + +.toast { + position: fixed; + right: 1rem; + bottom: 1rem; + display: flex; + align-items: center; + gap: 0.75rem; + padding: 1rem 1.25rem; + border-radius: var(--radius-md); + color: #fff; + box-shadow: var(--shadow-lg); + z-index: 1000; +} + +.toast.hidden { + display: none; +} + +.toast--success { + background-color: #198754; +} + +.toast--error { + background-color: #dc3545; +} + +.toast--info { + background-color: #0d6efd; +} + +.toast__close { + background: none; + border: none; + color: inherit; + cursor: pointer; + font-size: 1.1rem; +} diff --git a/static/js/exports.js b/static/js/exports.js new file mode 100644 index 0000000..5c2a6de --- /dev/null +++ b/static/js/exports.js @@ -0,0 +1,97 @@ +document.addEventListener("DOMContentLoaded", () => { + const modalContainer = document.createElement("div"); + modalContainer.id = "export-modal-container"; + document.body.appendChild(modalContainer); + + async function loadModal(dataset) { + const response = await fetch(`/exports/modal/${dataset}`); + if (!response.ok) { + throw new Error(`Failed to load export modal (${response.status})`); + } + const html = await response.text(); + modalContainer.innerHTML = html; + const modal = modalContainer.querySelector(".modal"); + if (!modal) return; + modal.classList.add("is-active"); + + const closeButtons = modal.querySelectorAll("[data-dismiss='modal']"); + closeButtons.forEach((btn) => + btn.addEventListener("click", () => closeModal(modal)) + ); + + const form = modal.querySelector("[data-export-form]"); + if (form) { + form.addEventListener("submit", handleSubmit); + } + } + + function closeModal(modal) { + modal.classList.remove("is-active"); + setTimeout(() => { + modalContainer.innerHTML = ""; + }, 200); + } + + async function handleSubmit(event) { + event.preventDefault(); + const form = event.currentTarget; + const submitUrl = form.action; + const formData = new FormData(form); + const format = formData.get("format") || "csv"; + + const response = await fetch(submitUrl, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + format, + include_metadata: formData.get("include_metadata") === "true", + filters: null, + }), + }); + + if (!response.ok) { + alert("Export failed. Please try again."); + return; + } + + const blob = await response.blob(); + const disposition = response.headers.get("Content-Disposition"); + let filename = "export"; + if (disposition) { + const match = disposition.match(/filename=([^;]+)/i); + if (match) { + filename = match[1].replace(/"/g, ""); + } + } + + const url = window.URL.createObjectURL(blob); + const link = document.createElement("a"); + link.href = url; + link.download = filename; + document.body.appendChild(link); + link.click(); + link.remove(); + window.URL.revokeObjectURL(url); + + const modal = modalContainer.querySelector(".modal"); + if (modal) { + closeModal(modal); + } + } + + document.querySelectorAll("[data-export-trigger]").forEach((button) => { + button.addEventListener("click", async (event) => { + event.preventDefault(); + const dataset = button.getAttribute("data-export-target"); + if (!dataset) return; + try { + await loadModal(dataset); + } catch (error) { + console.error(error); + alert("Unable to open export dialog."); + } + }); + }); +}); diff --git a/static/js/imports.js b/static/js/imports.js new file mode 100644 index 0000000..f64480a --- /dev/null +++ b/static/js/imports.js @@ -0,0 +1,100 @@ +document.addEventListener("DOMContentLoaded", () => { + const dropzones = document.querySelectorAll("[data-import-dropzone]"); + const uploadButtons = document.querySelectorAll( + "[data-import-upload-trigger]" + ); + const resetButtons = document.querySelectorAll("[data-import-reset]"); + const feedbackEl = document.querySelector("#import-upload-feedback"); + + function showFeedback(message, type = "info") { + if (!feedbackEl) return; + feedbackEl.textContent = message; + feedbackEl.classList.remove("hidden", "success", "error", "info"); + feedbackEl.classList.add(type); + } + + function hideFeedback() { + if (!feedbackEl) return; + feedbackEl.textContent = ""; + feedbackEl.classList.add("hidden"); + } + + dropzones.forEach((zone) => { + const input = zone.querySelector("input[type='file']"); + const uploadButton = zone + .closest("[data-import-upload]") + .querySelector("[data-import-upload-trigger]"); + const resetButton = zone + .closest("[data-import-upload]") + .querySelector("[data-import-reset]"); + + function enableUpload() { + if (uploadButton) { + uploadButton.disabled = false; + } + if (resetButton) { + resetButton.hidden = false; + } + } + + function disableUpload() { + if (uploadButton) { + uploadButton.disabled = true; + } + if (resetButton) { + resetButton.hidden = true; + } + } + + zone.addEventListener("dragover", (event) => { + event.preventDefault(); + zone.classList.add("dragover"); + }); + + zone.addEventListener("dragleave", () => { + zone.classList.remove("dragover"); + }); + + zone.addEventListener("drop", (event) => { + event.preventDefault(); + zone.classList.remove("dragover"); + if (!event.dataTransfer?.files?.length) { + return; + } + input.files = event.dataTransfer.files; + enableUpload(); + hideFeedback(); + }); + + input.addEventListener("change", () => { + if (input.files?.length) { + enableUpload(); + hideFeedback(); + } else { + disableUpload(); + } + }); + + resetButton?.addEventListener("click", () => { + input.value = ""; + disableUpload(); + hideFeedback(); + }); + + uploadButton?.addEventListener("click", () => { + if (!input.files?.length) { + showFeedback( + "Please select a CSV or XLSX file before uploading.", + "error" + ); + return; + } + + showFeedback("Uploading…", "info"); + uploadButton.disabled = true; + uploadButton.classList.add("loading"); + + // Actual upload logic handled separately (e.g., fetch). + }); + }); +}); diff --git a/templates/Dashboard.html b/templates/Dashboard.html index d843aa0..9eea868 100644 --- a/templates/Dashboard.html +++ b/templates/Dashboard.html @@ -1,132 +1,178 @@ -{% extends "base.html" %} -{% block title %}Dashboard · CalMiner{% endblock %} +{% extends "base.html" %} {% block title %}Dashboard · CalMiner{% endblock %} {% +block head_extra %} + +{% endblock %} {% block content %} + -{% block head_extra %} - -{% endblock %} - -{% block content %} - - -
-
-

Total Projects

-

{{ metrics.total_projects }}

- Across all operation types -
-
-

Active Scenarios

-

{{ metrics.active_scenarios }}

- Ready for analysis -
-
-

Pending Simulations

-

{{ metrics.pending_simulations }}

- Awaiting execution -
-
-

Last Data Import

-

{{ metrics.last_import or '—' }}

- UTC timestamp -
-
- -
-
-
-
-

Recent Projects

- View all -
- {% if recent_projects %} - - - - - - - - - - {% for project in recent_projects %} - - - - - - {% endfor %} - -
ProjectOperationUpdated
- {{ project.name }} - {{ project.operation_type.value.replace('_', ' ') | title }}{{ project.updated_at.strftime('%Y-%m-%d') if project.updated_at else '—' }}
- {% else %} -

No recent projects. Create one now.

- {% endif %} -
- -
-
-

Simulation Pipeline

-
- {% if simulation_updates %} -
    - {% for update in simulation_updates %} -
  • - {{ update.timestamp_label or '—' }} -
    - {{ update.title }} -

    {{ update.description }}

    -
    -
  • - {% endfor %} -
- {% else %} -

No simulation runs yet. Configure a scenario to start simulations.

- {% endif %} -
-
- - -
+
+
+

Total Projects

+

{{ metrics.total_projects }}

+ Across all operation types +
+
+

Active Scenarios

+

{{ metrics.active_scenarios }}

+ Ready for analysis +
+
+

Pending Simulations

+

{{ metrics.pending_simulations }}

+ Awaiting execution +
+
+

Last Data Import

+

{{ metrics.last_import or '—' }}

+ UTC timestamp +
+
+ +
+
+
+
+

Recent Projects

+ View all +
+ {% if recent_projects %} + + + + + + + + + + {% for project in recent_projects %} + + + + + + {% endfor %} + +
ProjectOperationUpdated
+ {{ project.name }} + + + {{ project.operation_type.value.replace('_', ' ') | title }} + + {{ project.updated_at.strftime('%Y-%m-%d') if project.updated_at + else '—' }} +
+ {% else %} +

+ No recent projects. + Create one now. +

+ {% endif %} +
+ +
+
+

Simulation Pipeline

+
+ {% if simulation_updates %} +
    + {% for update in simulation_updates %} +
  • + {{ update.timestamp_label or '—' }} +
    + {{ update.title }} +

    {{ update.description }}

    +
    +
  • + {% endfor %} +
+ {% else %} +

+ No simulation runs yet. Configure a scenario to start simulations. +

+ {% endif %} +
+
+ + +
{% endblock %} diff --git a/templates/base.html b/templates/base.html index b159e20..a7c9700 100644 --- a/templates/base.html +++ b/templates/base.html @@ -5,6 +5,7 @@ {% block title %}CalMiner{% endblock %} + {% block head_extra %}{% endblock %} @@ -20,7 +21,9 @@ {% block scripts %}{% endblock %} - + + + diff --git a/templates/exports/modal.html b/templates/exports/modal.html new file mode 100644 index 0000000..6da58db --- /dev/null +++ b/templates/exports/modal.html @@ -0,0 +1,51 @@ + diff --git a/templates/partials/alerts.html b/templates/partials/alerts.html new file mode 100644 index 0000000..9315d62 --- /dev/null +++ b/templates/partials/alerts.html @@ -0,0 +1,10 @@ +{% macro toast(id, hidden=True, level="info", message="") %} + +{% endmacro %} diff --git a/templates/partials/import_preview_table.html b/templates/partials/import_preview_table.html new file mode 100644 index 0000000..86b582f --- /dev/null +++ b/templates/partials/import_preview_table.html @@ -0,0 +1,17 @@ +{% from "partials/components.html" import table_container %} + +{% call table_container("import-preview-container", hidden=True, aria_label="Import preview table", heading=table_heading or "Preview Rows") %} + + + Row + Status + Issues + {% for column in columns %} + {{ column }} + {% endfor %} + + + + + +{% endcall %} diff --git a/templates/partials/import_upload.html b/templates/partials/import_upload.html new file mode 100644 index 0000000..e0879a1 --- /dev/null +++ b/templates/partials/import_upload.html @@ -0,0 +1,25 @@ +{% from "partials/components.html" import feedback %} + +
+
+

{{ title or "Bulk Import" }}

+ {% if description %}

{{ description }}

{% endif %} +
+ +
+ +

Drag & drop CSV/XLSX files here or

+ +

Maximum size {{ max_size_hint or "10 MB" }}. UTF-8 encoding required.

+
+ +
+ + +
+ + {{ feedback("import-upload-feedback", hidden=True, role="alert") }} +
diff --git a/tests/conftest.py b/tests/conftest.py index 1b96454..631125c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -18,6 +18,7 @@ from routes.dashboard import router as dashboard_router from routes.projects import router as projects_router from routes.scenarios import router as scenarios_router from routes.imports import router as imports_router +from routes.exports import router as exports_router from services.importers import ImportIngestionService from services.unit_of_work import UnitOfWork from services.session import AuthSession, SessionTokens @@ -54,6 +55,7 @@ def app(session_factory: sessionmaker) -> FastAPI: application.include_router(projects_router) application.include_router(scenarios_router) application.include_router(imports_router) + application.include_router(exports_router) def _override_uow() -> Iterator[UnitOfWork]: with UnitOfWork(session_factory=session_factory) as uow: diff --git a/tests/test_export_routes.py b/tests/test_export_routes.py new file mode 100644 index 0000000..e103b13 --- /dev/null +++ b/tests/test_export_routes.py @@ -0,0 +1,130 @@ +from __future__ import annotations + +import csv +from io import BytesIO, StringIO +from zipfile import ZipFile + +from fastapi.testclient import TestClient +from sqlalchemy.orm import Session + +from models import Project, Scenario, ScenarioStatus +from services.unit_of_work import UnitOfWork + + +def _seed_projects(session: Session) -> None: + project = Project(name="Alpha", operation_type="open_pit") + project.updated_at = project.created_at + session.add(project) + session.commit() + + +def _seed_scenarios(session: Session, project: Project) -> Scenario: + scenario = Scenario( + name="Scenario A", + project_id=project.id, + status=ScenarioStatus.ACTIVE, + ) + session.add(scenario) + session.commit() + session.refresh(scenario) + return scenario + + +def test_projects_export_modal(client: TestClient) -> None: + response = client.get("/exports/modal/projects") + assert response.status_code == 200 + assert "Export Projects" in response.text + + +def test_scenarios_export_modal(client: TestClient) -> None: + response = client.get("/exports/modal/scenarios") + assert response.status_code == 200 + assert "Export Scenarios" in response.text + + +def test_project_export_csv(client: TestClient, unit_of_work_factory) -> None: + with unit_of_work_factory() as uow: + project = Project(name="CSV Project", operation_type="open_pit") + uow.projects.create(project) + + response = client.post( + "/exports/projects", + json={"format": "csv"}, + ) + + assert response.status_code == 200 + assert response.headers["Content-Type"].startswith("text/csv") + assert "attachment; filename=" in response.headers["Content-Disposition"] + + content = response.content.decode("utf-8") + reader = csv.reader(StringIO(content)) + rows = list(reader) + assert rows[0][:3] == ["name", "location", "operation_type"] + assert any(row[0] == "CSV Project" for row in rows[1:]) + + +def test_project_export_excel(client: TestClient, unit_of_work_factory) -> None: + with unit_of_work_factory() as uow: + project = Project(name="XLSX Project", operation_type="open_pit") + uow.projects.create(project) + + response = client.post( + "/exports/projects", + json={"format": "xlsx"}, + ) + + assert response.status_code == 200 + assert response.headers["Content-Type"].startswith( + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + ) + + with ZipFile(BytesIO(response.content)) as archive: + assert "xl/workbook.xml" in archive.namelist() + + +def test_scenario_export_csv(client: TestClient, unit_of_work_factory) -> None: + with unit_of_work_factory() as uow: + project = Project(name="Scenario Project", operation_type="open_pit") + uow.projects.create(project) + scenario = Scenario( + name="Scenario CSV", + project_id=project.id, + status=ScenarioStatus.ACTIVE, + ) + uow.scenarios.create(scenario) + + response = client.post( + "/exports/scenarios", + json={"format": "csv"}, + ) + + assert response.status_code == 200 + reader = csv.reader(StringIO(response.content.decode("utf-8"))) + rows = list(reader) + assert rows[0][0] == "project_name" + assert any(row[0] == "Scenario Project" for row in rows[1:]) + + +def test_scenario_export_excel(client: TestClient, unit_of_work_factory) -> None: + with unit_of_work_factory() as uow: + project = Project(name="Scenario Excel", operation_type="open_pit") + uow.projects.create(project) + scenario = Scenario( + name="Scenario XLSX", + project_id=project.id, + status=ScenarioStatus.ACTIVE, + ) + uow.scenarios.create(scenario) + + response = client.post( + "/exports/scenarios", + json={"format": "xlsx"}, + ) + + assert response.status_code == 200 + assert response.headers["Content-Type"].startswith( + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + ) + + with ZipFile(BytesIO(response.content)) as archive: + assert "xl/workbook.xml" in archive.namelist()