allucanget/calminer
2
0
Fork 0
Code Pull Requests Actions Packages Releases Activity

v2 init

Browse Source
This commit is contained in:
zwitschi
2025-11-09 16:49:27 +01:00
parent 22ddfb671d
commit d807a50f77
96 changed files with 3 additions and 9689 deletions
Download Patch File Download Diff File Expand all files Collapse all files

157
scripts/backfill_currency.py
View File

@@ -1,157 +0,0 @@
"""
Backfill script to populate currency_id for capex and opex rows using existing currency_code.
Usage:
python scripts/backfill_currency.py --dry-run
python scripts/backfill_currency.py --create-missing
This script is intentionally cautious: it defaults to dry-run mode and will refuse to run
if database connection settings are missing. It supports creating missing currency rows when `--create-missing`
is provided. Always run against a development/staging database first.
"""
from __future__ import annotations
import argparse
import importlib
import sys
from pathlib import Path
from sqlalchemy import text, create_engine
PROJECT_ROOT = Path(__file__).resolve().parent.parent
if str(PROJECT_ROOT) not in sys.path:
sys.path.insert(0, str(PROJECT_ROOT))
def load_database_url() -> str:
try:
db_module = importlib.import_module("config.database")
except RuntimeError as exc:
raise RuntimeError(
"Database configuration missing: set DATABASE_URL or provide granular "
"variables (DATABASE_DRIVER, DATABASE_HOST, DATABASE_PORT, DATABASE_USER, "
"DATABASE_PASSWORD, DATABASE_NAME, optional DATABASE_SCHEMA)."
) from exc
return getattr(db_module, "DATABASE_URL")
def backfill(
db_url: str, dry_run: bool = True, create_missing: bool = False
) -> None:
engine = create_engine(db_url)
with engine.begin() as conn:
# Ensure currency table exists
if db_url.startswith("sqlite:"):
conn.execute(
text(
"SELECT name FROM sqlite_master WHERE type='table' AND name='currency';"
)
)
else:
conn.execute(text("SELECT to_regclass('public.currency');"))
# Note: we don't strictly depend on the above - we assume migration was already applied
# Helper: find or create currency by code
def find_currency_id(code: str):
r = conn.execute(
text("SELECT id FROM currency WHERE code = :code"),
{"code": code},
).fetchone()
if r:
return r[0]
if create_missing:
# insert and return id
conn.execute(
text(
"INSERT INTO currency (code, name, symbol, is_active) VALUES (:c, :n, NULL, TRUE)"
),
{"c": code, "n": code},
)
r2 = conn.execute(
text("SELECT id FROM currency WHERE code = :code"),
{"code": code},
).fetchone()
if not r2:
raise RuntimeError(
f"Unable to determine currency ID for '{code}' after insert"
)
return r2[0]
return None
# Process tables capex and opex
for table in ("capex", "opex"):
# Check if currency_id column exists
try:
cols = (
conn.execute(
text(
f"SELECT 1 FROM information_schema.columns WHERE table_name = '{table}' AND column_name = 'currency_id'"
)
)
if not db_url.startswith("sqlite:")
else [(1,)]
)
except Exception:
cols = [(1,)]
if not cols:
print(f"Skipping {table}: no currency_id column found")
continue
# Find rows where currency_id IS NULL but currency_code exists
rows = conn.execute(
text(
f"SELECT id, currency_code FROM {table} WHERE currency_id IS NULL OR currency_id = ''"
)
)
changed = 0
for r in rows:
rid = r[0]
code = (r[1] or "USD").strip().upper()
cid = find_currency_id(code)
if cid is None:
print(
f"Row {table}:{rid} has unknown currency code '{code}' and create_missing=False; skipping"
)
continue
if dry_run:
print(
f"[DRY RUN] Would set {table}.currency_id = {cid} for row id={rid} (code={code})"
)
else:
conn.execute(
text(
f"UPDATE {table} SET currency_id = :cid WHERE id = :rid"
),
{"cid": cid, "rid": rid},
)
changed += 1
print(f"{table}: processed, changed={changed} (dry_run={dry_run})")
def main() -> None:
parser = argparse.ArgumentParser(
description="Backfill currency_id from currency_code for capex/opex tables"
)
parser.add_argument(
"--dry-run",
action="store_true",
default=True,
help="Show actions without writing",
)
parser.add_argument(
"--create-missing",
action="store_true",
help="Create missing currency rows in the currency table",
)
args = parser.parse_args()
db = load_database_url()
backfill(db, dry_run=args.dry_run, create_missing=args.create_missing)
if __name__ == "__main__":
main()

50
scripts/check_docs_links.py
View File

@@ -1,50 +0,0 @@
"""Simple Markdown link checker for local docs/ files.
Checks only local file links (relative paths) and reports missing targets.
Run from the repository root using the project's Python environment.
"""
import re
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
DOCS = ROOT / "docs"
MD_LINK_RE = re.compile(r"\[([^\]]+)\]\(([^)]+)\)")
errors = []
for md in DOCS.rglob("*.md"):
text = md.read_text(encoding="utf-8")
for m in MD_LINK_RE.finditer(text):
label, target = m.groups()
# skip URLs
if (
target.startswith("http://")
or target.startswith("https://")
or target.startswith("#")
):
continue
# strip anchors
target_path = target.split("#")[0]
# if link is to a directory index, allow
candidate = (md.parent / target_path).resolve()
if candidate.exists():
continue
# check common implicit index: target/ -> target/README.md or target/index.md
candidate_dir = md.parent / target_path
if candidate_dir.is_dir():
if (candidate_dir / "README.md").exists() or (
candidate_dir / "index.md"
).exists():
continue
errors.append((str(md.relative_to(ROOT)), target, label))
if errors:
print("Broken local links found:")
for src, tgt, label in errors:
print(f"- {src} -> {tgt} ({label})")
exit(2)
print("No broken local links detected.")

92
scripts/format_docs_md.py
View File

@@ -1,92 +0,0 @@
"""Lightweight Markdown formatter: normalizes first-line H1, adds code-fence language hints for common shebangs, trims trailing whitespace.
This is intentionally small and non-destructive; it touches only files under docs/ and makes safe changes.
"""
import re
from pathlib import Path
DOCS = Path(__file__).resolve().parents[1] / "docs"
CODE_LANG_HINTS = {
"powershell": ("powershell",),
"bash": ("bash", "sh"),
"sql": ("sql",),
"python": ("python",),
}
def add_code_fence_language(match):
fence = match.group(0)
inner = match.group(1)
# If language already present, return unchanged
if fence.startswith("```") and len(fence.splitlines()[0].strip()) > 3:
return fence
# Try to infer language from the code content
code = inner.strip().splitlines()[0] if inner.strip() else ""
lang = ""
if (
code.startswith("$")
or code.startswith("PS")
or code.lower().startswith("powershell")
):
lang = "powershell"
elif (
code.startswith("#")
or code.startswith("import")
or code.startswith("from")
):
lang = "python"
elif re.match(r"^(select|insert|update|create)\b", code.strip(), re.I):
lang = "sql"
elif (
code.startswith("git")
or code.startswith("./")
or code.startswith("sudo")
):
lang = "bash"
if lang:
return f"```{lang}\n{inner}\n```"
return fence
def normalize_file(path: Path):
text = path.read_text(encoding="utf-8")
orig = text
# Trim trailing whitespace and ensure single trailing newline
text = "\n".join(line.rstrip() for line in text.splitlines()) + "\n"
# Ensure first non-empty line is H1
lines = text.splitlines()
for i, ln in enumerate(lines):
if ln.strip():
if not ln.startswith("#"):
lines[i] = "# " + ln
break
text = "\n".join(lines) + "\n"
# Add basic code fence languages where missing (simple heuristic)
text = re.sub(r"```\n([\s\S]*?)\n```", add_code_fence_language, text)
if text != orig:
path.write_text(text, encoding="utf-8")
return True
return False
def main():
changed = []
for p in DOCS.rglob("*.md"):
if p.is_file():
try:
if normalize_file(p):
changed.append(str(p.relative_to(Path.cwd())))
except Exception as e:
print(f"Failed to format {p}: {e}")
if changed:
print("Formatted files:")
for c in changed:
print(" -", c)
else:
print("No formatting changes required.")
if __name__ == "__main__":
main()

189
scripts/migrations/000_base.sql
View File

@@ -1,189 +0,0 @@
-- Baseline migration for CalMiner database schema
-- Date: 2025-10-25
-- Purpose: Consolidate foundational tables and reference data
BEGIN;
-- Currency reference table
CREATE TABLE IF NOT EXISTS currency (
id SERIAL PRIMARY KEY,
code VARCHAR(3) NOT NULL UNIQUE,
name VARCHAR(128) NOT NULL,
symbol VARCHAR(8),
is_active BOOLEAN NOT NULL DEFAULT TRUE
);
INSERT INTO currency (code, name, symbol, is_active)
VALUES
('USD', 'United States Dollar', 'USD$', TRUE),
('EUR', 'Euro', 'EUR', TRUE),
('CLP', 'Chilean Peso', 'CLP$', TRUE),
('RMB', 'Chinese Yuan', 'RMB', TRUE),
('GBP', 'British Pound', 'GBP', TRUE),
('CAD', 'Canadian Dollar', 'CAD$', TRUE),
('AUD', 'Australian Dollar', 'AUD$', TRUE)
ON CONFLICT (code) DO UPDATE
SET name = EXCLUDED.name,
symbol = EXCLUDED.symbol,
is_active = EXCLUDED.is_active;
-- Application-level settings table
CREATE TABLE IF NOT EXISTS application_setting (
id SERIAL PRIMARY KEY,
key VARCHAR(128) NOT NULL UNIQUE,
value TEXT NOT NULL,
value_type VARCHAR(32) NOT NULL DEFAULT 'string',
category VARCHAR(32) NOT NULL DEFAULT 'general',
description TEXT,
is_editable BOOLEAN NOT NULL DEFAULT TRUE,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE UNIQUE INDEX IF NOT EXISTS ux_application_setting_key
ON application_setting (key);
CREATE INDEX IF NOT EXISTS ix_application_setting_category
ON application_setting (category);
-- Measurement unit reference table
CREATE TABLE IF NOT EXISTS measurement_unit (
id SERIAL PRIMARY KEY,
code VARCHAR(64) NOT NULL UNIQUE,
name VARCHAR(128) NOT NULL,
symbol VARCHAR(16),
unit_type VARCHAR(32) NOT NULL,
is_active BOOLEAN NOT NULL DEFAULT TRUE,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
INSERT INTO measurement_unit (code, name, symbol, unit_type, is_active)
VALUES
('tonnes', 'Tonnes', 't', 'mass', TRUE),
('kilograms', 'Kilograms', 'kg', 'mass', TRUE),
('pounds', 'Pounds', 'lb', 'mass', TRUE),
('liters', 'Liters', 'L', 'volume', TRUE),
('cubic_meters', 'Cubic Meters', 'm3', 'volume', TRUE),
('kilowatt_hours', 'Kilowatt Hours', 'kWh', 'energy', TRUE)
ON CONFLICT (code) DO UPDATE
SET name = EXCLUDED.name,
symbol = EXCLUDED.symbol,
unit_type = EXCLUDED.unit_type,
is_active = EXCLUDED.is_active;
-- Consumption and production measurement metadata
ALTER TABLE consumption
ADD COLUMN IF NOT EXISTS unit_name VARCHAR(64);
ALTER TABLE consumption
ADD COLUMN IF NOT EXISTS unit_symbol VARCHAR(16);
ALTER TABLE production_output
ADD COLUMN IF NOT EXISTS unit_name VARCHAR(64);
ALTER TABLE production_output
ADD COLUMN IF NOT EXISTS unit_symbol VARCHAR(16);
-- Currency integration for CAPEX and OPEX
ALTER TABLE capex
ADD COLUMN IF NOT EXISTS currency_id INTEGER;
ALTER TABLE opex
ADD COLUMN IF NOT EXISTS currency_id INTEGER;
DO $$
DECLARE
usd_id INTEGER;
BEGIN
-- Ensure currency_id columns align with legacy currency_code values when present
IF EXISTS (
SELECT 1 FROM information_schema.columns
WHERE table_name = 'capex' AND column_name = 'currency_code'
) THEN
UPDATE capex AS c
SET currency_id = cur.id
FROM currency AS cur
WHERE c.currency_code = cur.code
AND (c.currency_id IS DISTINCT FROM cur.id);
END IF;
IF EXISTS (
SELECT 1 FROM information_schema.columns
WHERE table_name = 'opex' AND column_name = 'currency_code'
) THEN
UPDATE opex AS o
SET currency_id = cur.id
FROM currency AS cur
WHERE o.currency_code = cur.code
AND (o.currency_id IS DISTINCT FROM cur.id);
END IF;
SELECT id INTO usd_id FROM currency WHERE code = 'USD';
IF usd_id IS NOT NULL THEN
UPDATE capex SET currency_id = usd_id WHERE currency_id IS NULL;
UPDATE opex SET currency_id = usd_id WHERE currency_id IS NULL;
END IF;
END $$;
ALTER TABLE capex
ALTER COLUMN currency_id SET NOT NULL;
ALTER TABLE opex
ALTER COLUMN currency_id SET NOT NULL;
DO $$
BEGIN
IF NOT EXISTS (
SELECT 1 FROM information_schema.table_constraints
WHERE table_schema = current_schema()
AND table_name = 'capex'
AND constraint_name = 'fk_capex_currency'
) THEN
ALTER TABLE capex
ADD CONSTRAINT fk_capex_currency FOREIGN KEY (currency_id)
REFERENCES currency (id) ON DELETE RESTRICT;
END IF;
IF NOT EXISTS (
SELECT 1 FROM information_schema.table_constraints
WHERE table_schema = current_schema()
AND table_name = 'opex'
AND constraint_name = 'fk_opex_currency'
) THEN
ALTER TABLE opex
ADD CONSTRAINT fk_opex_currency FOREIGN KEY (currency_id)
REFERENCES currency (id) ON DELETE RESTRICT;
END IF;
END $$;
ALTER TABLE capex
DROP COLUMN IF EXISTS currency_code;
ALTER TABLE opex
DROP COLUMN IF EXISTS currency_code;
-- Role-based access control tables
CREATE TABLE IF NOT EXISTS roles (
id SERIAL PRIMARY KEY,
name VARCHAR(255) UNIQUE NOT NULL
);
CREATE TABLE IF NOT EXISTS users (
id SERIAL PRIMARY KEY,
username VARCHAR(255) UNIQUE NOT NULL,
email VARCHAR(255) UNIQUE NOT NULL,
hashed_password VARCHAR(255) NOT NULL,
role_id INTEGER NOT NULL REFERENCES roles (id) ON DELETE RESTRICT
);
CREATE INDEX IF NOT EXISTS ix_users_username ON users (username);
CREATE INDEX IF NOT EXISTS ix_users_email ON users (email);
-- Theme settings configuration table
CREATE TABLE IF NOT EXISTS theme_settings (
id SERIAL PRIMARY KEY,
theme_name VARCHAR(255) UNIQUE NOT NULL,
primary_color VARCHAR(7) NOT NULL,
secondary_color VARCHAR(7) NOT NULL,
accent_color VARCHAR(7) NOT NULL,
background_color VARCHAR(7) NOT NULL,
text_color VARCHAR(7) NOT NULL
);
COMMIT;

268
scripts/seed_data.py
View File

@@ -1,268 +0,0 @@
"""Seed baseline data for CalMiner in an idempotent manner.
Usage examples
--------------
```powershell
# Use existing environment variables (or load from setup_test.env.example)
python scripts/seed_data.py --currencies --units --defaults
# Dry-run to preview actions
python scripts/seed_data.py --currencies --dry-run
```
"""
from __future__ import annotations
import argparse
import logging
from typing import Optional
import psycopg2
from psycopg2 import errors
from psycopg2.extras import execute_values
from scripts.setup_database import DatabaseConfig
logger = logging.getLogger(__name__)
CURRENCY_SEEDS = (
("USD", "United States Dollar", "USD$", True),
("EUR", "Euro", "EUR", True),
("CLP", "Chilean Peso", "CLP$", True),
("RMB", "Chinese Yuan", "RMB", True),
("GBP", "British Pound", "GBP", True),
("CAD", "Canadian Dollar", "CAD$", True),
("AUD", "Australian Dollar", "AUD$", True),
)
MEASUREMENT_UNIT_SEEDS = (
("tonnes", "Tonnes", "t", "mass", True),
("kilograms", "Kilograms", "kg", "mass", True),
("pounds", "Pounds", "lb", "mass", True),
("liters", "Liters", "L", "volume", True),
("cubic_meters", "Cubic Meters", "m3", "volume", True),
("kilowatt_hours", "Kilowatt Hours", "kWh", "energy", True),
)
THEME_SETTING_SEEDS = (
("--color-background", "#f4f5f7", "color",
"theme", "CSS variable --color-background", True),
("--color-surface", "#ffffff", "color",
"theme", "CSS variable --color-surface", True),
("--color-text-primary", "#2a1f33", "color",
"theme", "CSS variable --color-text-primary", True),
("--color-text-secondary", "#624769", "color",
"theme", "CSS variable --color-text-secondary", True),
("--color-text-muted", "#64748b", "color",
"theme", "CSS variable --color-text-muted", True),
("--color-text-subtle", "#94a3b8", "color",
"theme", "CSS variable --color-text-subtle", True),
("--color-text-invert", "#ffffff", "color",
"theme", "CSS variable --color-text-invert", True),
("--color-text-dark", "#0f172a", "color",
"theme", "CSS variable --color-text-dark", True),
("--color-text-strong", "#111827", "color",
"theme", "CSS variable --color-text-strong", True),
("--color-primary", "#5f320d", "color",
"theme", "CSS variable --color-primary", True),
("--color-primary-strong", "#7e4c13", "color",
"theme", "CSS variable --color-primary-strong", True),
("--color-primary-stronger", "#837c15", "color",
"theme", "CSS variable --color-primary-stronger", True),
("--color-accent", "#bff838", "color",
"theme", "CSS variable --color-accent", True),
("--color-border", "#e2e8f0", "color",
"theme", "CSS variable --color-border", True),
("--color-border-strong", "#cbd5e1", "color",
"theme", "CSS variable --color-border-strong", True),
("--color-highlight", "#eef2ff", "color",
"theme", "CSS variable --color-highlight", True),
("--color-panel-shadow", "rgba(15, 23, 42, 0.08)", "color",
"theme", "CSS variable --color-panel-shadow", True),
("--color-panel-shadow-deep", "rgba(15, 23, 42, 0.12)", "color",
"theme", "CSS variable --color-panel-shadow-deep", True),
("--color-surface-alt", "#f8fafc", "color",
"theme", "CSS variable --color-surface-alt", True),
("--color-success", "#047857", "color",
"theme", "CSS variable --color-success", True),
("--color-error", "#b91c1c", "color",
"theme", "CSS variable --color-error", True),
)
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Seed baseline CalMiner data")
parser.add_argument(
"--currencies", action="store_true", help="Seed currency table"
)
parser.add_argument("--units", action="store_true", help="Seed unit table")
parser.add_argument(
"--theme", action="store_true", help="Seed theme settings"
)
parser.add_argument(
"--defaults", action="store_true", help="Seed default records"
)
parser.add_argument(
"--dry-run", action="store_true", help="Print actions without executing"
)
parser.add_argument(
"--verbose",
"-v",
action="count",
default=0,
help="Increase logging verbosity",
)
return parser.parse_args()
def _configure_logging(args: argparse.Namespace) -> None:
level = logging.WARNING - (10 * min(args.verbose, 2))
logging.basicConfig(
level=max(level, logging.INFO), format="%(levelname)s %(message)s"
)
def main() -> None:
args = parse_args()
run_with_namespace(args)
def run_with_namespace(
args: argparse.Namespace,
*,
config: Optional[DatabaseConfig] = None,
) -> None:
if not hasattr(args, "verbose"):
args.verbose = 0
if not hasattr(args, "dry_run"):
args.dry_run = False
_configure_logging(args)
currencies = bool(getattr(args, "currencies", False))
units = bool(getattr(args, "units", False))
theme = bool(getattr(args, "theme", False))
defaults = bool(getattr(args, "defaults", False))
dry_run = bool(getattr(args, "dry_run", False))
if not any((currencies, units, theme, defaults)):
logger.info("No seeding options provided; exiting")
return
config = config or DatabaseConfig.from_env()
with psycopg2.connect(config.application_dsn()) as conn:
conn.autocommit = True
with conn.cursor() as cursor:
if currencies:
_seed_currencies(cursor, dry_run=dry_run)
if units:
_seed_units(cursor, dry_run=dry_run)
if theme:
_seed_theme(cursor, dry_run=dry_run)
if defaults:
_seed_defaults(cursor, dry_run=dry_run)
def _seed_currencies(cursor, *, dry_run: bool) -> None:
logger.info("Seeding currency table (%d rows)", len(CURRENCY_SEEDS))
if dry_run:
for code, name, symbol, active in CURRENCY_SEEDS:
logger.info("Dry run: would upsert currency %s (%s)", code, name)
return
execute_values(
cursor,
"""
INSERT INTO currency (code, name, symbol, is_active)
VALUES %s
ON CONFLICT (code) DO UPDATE
SET name = EXCLUDED.name,
symbol = EXCLUDED.symbol,
is_active = EXCLUDED.is_active
""",
CURRENCY_SEEDS,
)
logger.info("Currency seed complete")
def _seed_units(cursor, *, dry_run: bool) -> None:
total = len(MEASUREMENT_UNIT_SEEDS)
logger.info("Seeding measurement_unit table (%d rows)", total)
if dry_run:
for code, name, symbol, unit_type, _ in MEASUREMENT_UNIT_SEEDS:
logger.info(
"Dry run: would upsert measurement unit %s (%s - %s)",
code,
name,
unit_type,
)
return
try:
execute_values(
cursor,
"""
INSERT INTO measurement_unit (code, name, symbol, unit_type, is_active)
VALUES %s
ON CONFLICT (code) DO UPDATE
SET name = EXCLUDED.name,
symbol = EXCLUDED.symbol,
unit_type = EXCLUDED.unit_type,
is_active = EXCLUDED.is_active
""",
MEASUREMENT_UNIT_SEEDS,
)
except errors.UndefinedTable:
logger.warning(
"measurement_unit table does not exist; skipping unit seeding."
)
cursor.connection.rollback()
return
logger.info("Measurement unit seed complete")
def _seed_theme(cursor, *, dry_run: bool) -> None:
logger.info("Seeding theme settings (%d rows)", len(THEME_SETTING_SEEDS))
if dry_run:
for key, value, _, _, _, _ in THEME_SETTING_SEEDS:
logger.info(
"Dry run: would upsert theme setting %s = %s", key, value)
return
try:
execute_values(
cursor,
"""
INSERT INTO application_setting (key, value, value_type, category, description, is_editable)
VALUES %s
ON CONFLICT (key) DO UPDATE
SET value = EXCLUDED.value,
value_type = EXCLUDED.value_type,
category = EXCLUDED.category,
description = EXCLUDED.description,
is_editable = EXCLUDED.is_editable
""",
THEME_SETTING_SEEDS,
)
except errors.UndefinedTable:
logger.warning(
"application_setting table does not exist; skipping theme seeding."
)
cursor.connection.rollback()
return
logger.info("Theme settings seed complete")
def _seed_defaults(cursor, *, dry_run: bool) -> None:
logger.info("Seeding default records")
_seed_theme(cursor, dry_run=dry_run)
logger.info("Default records seed complete")
if __name__ == "__main__":
main()

1233
scripts/setup_database.py
View File

File diff suppressed because it is too large Load Diff