fluvilog 0.1.0__py3-none-any.whl

fluvilog/__init__.py

@@ -0,0 +1,5 @@
+"""fluvilog — near-real-time readings from Hamburg's water quality network (WGMN)."""
+
+from .wgmn import fetch, fetch_history
+
+__all__ = ["fetch", "fetch_history"]

fluvilog/__main__.py

@@ -0,0 +1,6 @@
+"""Entry point for `python -m fluvilog`."""
+
+from .cli import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())

fluvilog/api/__init__.py

@@ -0,0 +1,9 @@
+"""Optional HTTP read API (requires the [api] extra: FastAPI + uvicorn).
+
+Importing this package pulls in FastAPI; the base CLI keeps it out of the import
+graph and only loads it inside the serve-api handler.
+"""
+
+from .app import create_app
+
+__all__ = ["create_app"]

fluvilog/api/app.py

@@ -0,0 +1,107 @@
+"""FastAPI application factory for the optional HTTP read API.
+
+Three GET endpoints over stored readings plus the station catalogue. Build with
+create_app; the `fluvilog serve-api` subcommand runs it under uvicorn. Each
+request gets its own read-only Storage, so reads never touch the poller's writer
+connection or its schema.
+"""
+
+# Route handlers are registered by the @app.get decorators' side effect, which
+# the type checker can't see; without this it flags them as unused.
+# pyright: reportUnusedFunction=false
+
+from collections.abc import Iterator
+from dataclasses import asdict
+from datetime import datetime, timedelta
+from typing import Annotated
+from zoneinfo import ZoneInfo
+
+from fastapi import Depends, FastAPI, HTTPException, Query
+from fastapi.middleware.cors import CORSMiddleware
+
+from .. import catalogue
+from ..constants import BERLIN_TZ, MAX_WINDOW_DAYS, PARAMETERS, STATIONS
+from ..storage import SqliteStorage
+from .schemas import ReadingOut, StationOut
+
+_BERLIN = ZoneInfo(BERLIN_TZ)
+
+
+def _as_berlin(value: datetime) -> datetime:
+    """Make a query datetime tz-aware in Europe/Berlin (naive ⇒ interpreted as)."""
+    if value.tzinfo is None:
+        return value.replace(tzinfo=_BERLIN)
+    return value.astimezone(_BERLIN)
+
+
+def _validate_filters(station: list[str] | None, parameter: list[str] | None) -> None:
+    """Reject unknown station codes or parameter names with HTTP 422."""
+    unknown_st = [s for s in station or [] if s not in STATIONS]
+    if unknown_st:
+        raise HTTPException(422, f"unknown station code(s): {unknown_st}")
+    unknown_pa = [p for p in parameter or [] if p not in PARAMETERS]
+    if unknown_pa:
+        raise HTTPException(422, f"unknown parameter name(s): {unknown_pa}")
+
+
+def _validate_window(start: datetime, end: datetime) -> None:
+    """Reject reversed or over-wide windows with HTTP 422."""
+    if start > end:
+        raise HTTPException(422, "'from' must not be after 'to'")
+    if end - start > timedelta(days=MAX_WINDOW_DAYS):
+        raise HTTPException(422, f"window exceeds the {MAX_WINDOW_DAYS}-day limit")
+
+
+def create_app(*, db_path: str, allowed_origins: list[str]) -> FastAPI:
+    """Build the read-only FastAPI app bound to a SQLite database path.
+
+    allowed_origins seeds CORS (GET only); an empty list permits no cross-origin
+    request. db_path is opened read-only per request; the schema is never touched.
+    """
+    app = FastAPI(title="fluvilog API", version="0.1.0")
+    app.add_middleware(
+        CORSMiddleware,
+        allow_origins=allowed_origins,
+        allow_methods=["GET"],
+        allow_headers=["*"],
+    )
+
+    def get_storage() -> Iterator[SqliteStorage]:
+        store = SqliteStorage.open_readonly(db_path)
+        try:
+            yield store
+        finally:
+            store.close()
+
+    @app.get("/api/stations")
+    def get_stations() -> list[StationOut]:
+        return [StationOut(**asdict(s)) for s in catalogue.stations()]
+
+    @app.get("/api/readings/latest")
+    def get_latest(
+        store: Annotated[SqliteStorage, Depends(get_storage)],
+        station: Annotated[list[str] | None, Query()] = None,
+        parameter: Annotated[list[str] | None, Query()] = None,
+    ) -> list[ReadingOut]:
+        _validate_filters(station, parameter)
+        rows = store.latest_readings(station_codes=station, parameters=parameter)
+        return [ReadingOut(**asdict(r)) for r in rows]
+
+    @app.get("/api/readings")
+    def get_readings(
+        store: Annotated[SqliteStorage, Depends(get_storage)],
+        start: Annotated[datetime, Query(alias="from")],
+        end: Annotated[datetime | None, Query(alias="to")] = None,
+        station: Annotated[list[str] | None, Query()] = None,
+        parameter: Annotated[list[str] | None, Query()] = None,
+    ) -> list[ReadingOut]:
+        _validate_filters(station, parameter)
+        start = _as_berlin(start)
+        end = datetime.now(_BERLIN) if end is None else _as_berlin(end)
+        _validate_window(start, end)
+        rows = store.readings_in_window(
+            start, end, station_codes=station, parameters=parameter
+        )
+        return [ReadingOut(**asdict(r)) for r in rows]
+
+    return app

fluvilog/api/schemas.py

@@ -0,0 +1,29 @@
+"""Public, validated HTTP response models.
+
+Pydantic shapes for the wire contract, kept separate from records.py so the HTTP
+surface can evolve independently of the internal storage records.
+"""
+
+from datetime import datetime
+
+from pydantic import BaseModel
+
+
+class StationOut(BaseModel):
+    """A station in the catalogue. latitude/longitude are WGS84 degrees."""
+
+    code: str
+    name: str
+    water_body: str
+    latitude: float
+    longitude: float
+
+
+class ReadingOut(BaseModel):
+    """A single reading. timestamp serialises as ISO 8601 with offset."""
+
+    station_code: str
+    parameter: str
+    unit: str
+    timestamp: datetime
+    value: float | None

fluvilog/catalogue.py

@@ -0,0 +1,34 @@
+"""Station reference data: the catalogue joining names with coordinates.
+
+Source of truth is constants (STATIONS + STATION_COORDS), not the DB stations
+table — that table exists only for the readings foreign key.
+"""
+
+from .constants import STATION_COORDS, STATIONS
+from .records import StationRecord
+
+
+def _check_coords() -> None:
+    """Raise ValueError if any STATIONS code lacks a STATION_COORDS entry."""
+    missing = set(STATIONS) - set(STATION_COORDS)
+    if missing:
+        raise ValueError(
+            f"STATION_COORDS is missing coordinates for: {sorted(missing)}"
+        )
+
+
+_check_coords()
+
+
+def stations() -> list[StationRecord]:
+    """Return all WGMN stations sorted by code, each with WGS84 coordinates."""
+    return [
+        StationRecord(
+            code=code,
+            name=name,
+            water_body=water_body,
+            latitude=STATION_COORDS[code][0],
+            longitude=STATION_COORDS[code][1],
+        )
+        for code, (name, water_body) in sorted(STATIONS.items())
+    ]

fluvilog/cli.py

@@ -0,0 +1,214 @@
+"""Near-real-time readings from Hamburg's water quality network (WGMN).
+
+Polls the public HamburgService platform (Wassergüte-Auskunft) at
+serviceportal.hamburg.de and, by default, stores readings continuously.
+
+Usage:
+    fluvilog                          # collect: poll and store (default subcommand)
+    fluvilog collect --station BL SH  # collect only specific stations
+    fluvilog collect --db water.db --interval 10m
+    fluvilog once                     # one-shot fetch and print
+    fluvilog once --csv values.csv    # ... and write to CSV
+    fluvilog list                     # list known stations
+    fluvilog serve-api                # serve the HTTP read API (needs [api] extra)
+"""
+
+import argparse
+import logging
+import sqlite3
+import sys
+
+import pandas as pd
+import requests
+
+from .constants import (
+    DEFAULT_API_HOST,
+    DEFAULT_API_PORT,
+    DEFAULT_DB_PATH,
+    DEFAULT_INTERVAL,
+    DEFAULT_PARAMETERS,
+    STATIONS,
+)
+from .service import collect, parse_interval
+from .storage import IncompatibleSchemaError, SqliteStorage
+from .wgmn import fetch
+
+_COMMANDS = {"collect", "once", "list", "serve-api"}
+
+
+def resolve_codes(selectors: list[str] | None) -> list[str]:
+    """Translate --station arguments (code or name, case-insensitive) to codes."""
+    if not selectors:
+        return list(STATIONS)
+    by_name = {name.casefold(): code for code, (name, _) in STATIONS.items()}
+    codes: list[str] = []
+    for sel in selectors:
+        if sel.upper() in STATIONS:
+            codes.append(sel.upper())
+        elif sel.casefold() in by_name:
+            codes.append(by_name[sel.casefold()])
+        else:
+            print(f"  ! unknown station: {sel!r} (list shows all)", file=sys.stderr)
+    return codes
+
+
+def _run_list() -> int:
+    """Print the station catalogue and exit."""
+    print("# Known WGMN stations:")
+    for code, (name, water_body) in STATIONS.items():
+        print(f"  {code}  {name} ({water_body})")
+    return 0
+
+
+def _run_once(args: argparse.Namespace) -> int:
+    """Fetch the latest values once, print them, and optionally write CSV."""
+    codes = resolve_codes(args.station)
+    if not codes:
+        return 2
+
+    try:
+        df = fetch(codes, DEFAULT_PARAMETERS)
+    except requests.RequestException as e:
+        print(f"Network/HTTP error: {e}", file=sys.stderr)
+        return 1
+
+    if df.empty:
+        print("No measurements available.", file=sys.stderr)
+        return 1
+
+    pd.set_option("display.max_rows", None)
+    pd.set_option("display.width", 200)
+    print(df.to_string(index=False))
+    latest = df["timestamp"].max()
+    print(
+        f"\n{len(df)} measurement(s) from {df['station'].nunique()} station(s), "
+        f"latest: {latest:%d.%m.%Y %H:%M}."
+    )
+
+    if args.csv:
+        df.to_csv(args.csv, index=False)
+        print(f"Saved: {args.csv}")
+    return 0
+
+
+def _run_collect(args: argparse.Namespace) -> int:
+    """Run the continuous poll-and-store loop."""
+    codes = resolve_codes(args.station)
+    if not codes:
+        return 2
+
+    try:
+        with SqliteStorage(args.db) as store:
+            return collect(codes, DEFAULT_PARAMETERS, store, args.interval)
+    except IncompatibleSchemaError as e:
+        print(str(e), file=sys.stderr)
+        return 1
+    except sqlite3.Error as e:
+        print(f"Database error: {e}", file=sys.stderr)
+        return 1
+
+
+def _run_serve_api(args: argparse.Namespace) -> int:
+    """Serve the HTTP read API under uvicorn (requires the optional [api] extra).
+
+    Imports the web stack lazily so the base CLI works without [api] installed.
+    """
+    try:
+        import uvicorn
+
+        from .api import create_app
+    except ImportError:
+        print(
+            "The HTTP API needs the optional dependencies. "
+            "Install them with: pip install 'fluvilog[api]'",
+            file=sys.stderr,
+        )
+        return 1
+
+    app = create_app(db_path=args.db, allowed_origins=args.cors_origin)
+    uvicorn.run(app, host=args.host, port=args.port)
+    return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    """Entry point. Bare invocation (no subcommand) runs `collect`."""
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(asctime)s %(levelname)s %(message)s",
+        datefmt="%Y-%m-%d %H:%M:%S",
+        stream=sys.stderr,
+    )
+    argv = sys.argv[1:] if argv is None else list(argv)
+    if not argv or (argv[0] not in _COMMANDS and argv[0] not in {"-h", "--help"}):
+        argv = ["collect", *argv]
+
+    ap = argparse.ArgumentParser(
+        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
+    )
+    sub = ap.add_subparsers(dest="command", required=True)
+
+    p_collect = sub.add_parser("collect", help="Continuously fetch and store (default)")
+    p_collect.add_argument(
+        "--station", nargs="+", metavar="CODE/NAME", help="Only these stations"
+    )
+    p_collect.add_argument(
+        "--db",
+        metavar="PATH",
+        default=DEFAULT_DB_PATH,
+        help=f"SQLite database path (default: {DEFAULT_DB_PATH})",
+    )
+    p_collect.add_argument(
+        "--interval",
+        type=parse_interval,
+        default=float(DEFAULT_INTERVAL),
+        metavar="DURATION",
+        help=f"Poll interval, e.g. 30s/10m/1h (default: {DEFAULT_INTERVAL}s)",
+    )
+
+    p_once = sub.add_parser("once", help="One-shot fetch and print")
+    p_once.add_argument(
+        "--station", nargs="+", metavar="CODE/NAME", help="Only these stations"
+    )
+    p_once.add_argument("--csv", metavar="PATH", help="Write result to CSV")
+
+    sub.add_parser("list", help="List known stations and exit")
+
+    p_api = sub.add_parser(
+        "serve-api", help="Serve the HTTP read API (needs the [api] extra)"
+    )
+    p_api.add_argument(
+        "--db",
+        metavar="PATH",
+        default=DEFAULT_DB_PATH,
+        help=f"SQLite database path (default: {DEFAULT_DB_PATH})",
+    )
+    p_api.add_argument(
+        "--host",
+        metavar="HOST",
+        default=DEFAULT_API_HOST,
+        help=f"Bind host (default: {DEFAULT_API_HOST})",
+    )
+    p_api.add_argument(
+        "--port",
+        type=int,
+        metavar="PORT",
+        default=DEFAULT_API_PORT,
+        help=f"Bind port (default: {DEFAULT_API_PORT})",
+    )
+    p_api.add_argument(
+        "--cors-origin",
+        action="append",
+        default=[],
+        metavar="ORIGIN",
+        help="Allowed CORS origin; repeatable (default: none)",
+    )
+
+    args = ap.parse_args(argv)
+
+    if args.command == "list":
+        return _run_list()
+    if args.command == "once":
+        return _run_once(args)
+    if args.command == "serve-api":
+        return _run_serve_api(args)
+    return _run_collect(args)

fluvilog/constants.py

@@ -0,0 +1,87 @@
+"""Static configuration: endpoint, request limits, and the station/parameter
+catalogues of the HamburgService water quality endpoint."""
+
+# Service endpoint (ASP.NET WebForms "Wassergüte-Auskunft").
+BASE = "https://serviceportal.hamburg.de/HamburgGateway"
+START = f"{BASE}/Service/StartService/WGMN?linkId=0&ars=020000000000"
+PFX = "GatewayMaster:ContentSection:wucStationenAuswahlListe1:"
+ENCODING = "windows-1252"
+TIMEOUT = 60
+USER_AGENT = (
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
+    "AppleWebKit/605.1.15 (KHTML, like Gecko) Safari/605.1.15"
+)
+
+# Service limit per request.
+MAX_STATIONS = 5
+MAX_PARAMETERS = 5
+
+# Station code -> (station name, body of water). The 9 official WGMN stations.
+# The form's checkbox order (cblStationen:0..8) is not encoded here; it is read
+# from the live form at runtime (see _station_index_map in wgmn.py).
+STATIONS: dict[str, tuple[str, str]] = {
+    "BU": ("Bunthaus", "Elbe"),
+    "SH": ("Seemannshöft", "Elbe"),
+    "BL": ("Blankenese", "Elbe"),
+    "LB": ("Lombardsbrücke", "Alster"),
+    "HA": ("Haselknick", "Alster"),
+    "TA": ("Rosenbrook", "Tarpenbek"),
+    "BK": ("Brügkamp", "Ammersbek"),
+    "FH": ("Fischerhof", "Bille"),
+    "WA": ("Wandsbeker Allee", "Wandse"),
+}
+
+# WGS84 (latitude, longitude) per station code. Static reference data: the 9
+# WGMN stations are fixed, so coordinates live here rather than in the DB schema.
+# Order is (lat, lon) for human sanity; GeoJSON's [lon, lat] is the API's job.
+# Keys must stay in sync with STATIONS (enforced by catalogue.stations()).
+STATION_COORDS: dict[str, tuple[float, float]] = {
+    "BU": (53.46166, 10.06434),  # Bunthaus (Elbe)
+    "SH": (53.54024, 9.87984),  # Seemannshöft (Elbe)
+    "BL": (53.55587, 9.80545),  # Blankenese (Elbe)
+    "LB": (53.55725, 9.99797),  # Lombardsbrücke (Alster)
+    "HA": (53.69548, 10.12075),  # Haselknick (Alster)
+    "TA": (53.60223, 9.98624),  # Rosenbrook (Tarpenbek)
+    "BK": (53.71191, 10.16285),  # Brügkamp (Ammersbek)
+    "FH": (53.48927, 10.21090),  # Fischerhof (Bille)
+    "WA": (53.57616, 10.06895),  # Wandsbeker Allee (Wandse)
+}
+
+# Order of the parameter checkboxes (clbMesswerte:0..13) in the form.
+PARAMETERS = [
+    "Lufttemperatur",
+    "Wassertemperatur",
+    "Sauerstoffkonzentration",
+    "Sauerstoffsättigung",
+    "pH-Wert",
+    "Leitfähigkeit Kappa 25",
+    "Trübung",
+    "Gesamtchlorophyll",
+    "Chlorophyll Blaualgen",
+    "Chlorophyll Grünalgen",
+    "Chlorophyll Kieselalgen",
+    "Chlorophyll Cryptophyceen",
+    "UV-Absorption",
+    "AlarmIndex",
+]
+# Core parameters (water temp, O2 conc., O2 sat., pH, conductivity, turbidity).
+DEFAULT_PARAMETERS = [1, 2, 3, 4, 5, 6]
+
+# Collect-mode defaults.
+DEFAULT_INTERVAL = 600  # seconds between polls (source updates ~every 10 min)
+MIN_INTERVAL = 30  # floor enforced by --interval parsing
+DEFAULT_DB_PATH = "fluvilog.db"  # SQLite file, relative to cwd
+
+# Timezone of stored reading timestamps; also the offset all read APIs emit.
+BERLIN_TZ = "Europe/Berlin"
+
+# HTTP API tier (optional [api] extra; see fluvilog.api).
+MAX_WINDOW_DAYS = 30  # largest /api/readings window before a 422
+DEFAULT_API_HOST = "127.0.0.1"
+DEFAULT_API_PORT = 8000
+
+# Schema object names (DDL itself lives in storage.py).
+TABLE_READINGS = "readings"  # fact table
+TABLE_STATIONS = "stations"  # station dimension
+TABLE_PARAMETERS = "parameters"  # parameter dimension
+VIEW_READINGS_FULL = "readings_full"  # denormalized join of the three

fluvilog/records.py

@@ -0,0 +1,39 @@
+"""Internal read records: the contract between storage/catalogue and callers.
+
+Plain dataclasses, deliberately not Pydantic, so the persistence and read
+layers carry no dependency on the optional API tier. See fluvilog.api.schemas
+for the public HTTP shapes.
+"""
+
+from dataclasses import dataclass
+from datetime import datetime
+
+
+@dataclass(frozen=True, slots=True)
+class StationRecord:
+    """A WGMN station with its WGS84 position.
+
+    latitude/longitude are decimal degrees (EPSG:4326).
+    """
+
+    code: str
+    name: str
+    water_body: str
+    latitude: float
+    longitude: float
+
+
+@dataclass(frozen=True, slots=True)
+class ReadingRecord:
+    """A single stored reading.
+
+    parameter is the display name (e.g. "Wassertemperatur"). timestamp is
+    tz-aware Europe/Berlin. value is None when the sensor reported no value;
+    None is preserved, not dropped.
+    """
+
+    station_code: str
+    parameter: str
+    unit: str
+    timestamp: datetime
+    value: float | None

fluvilog/service.py

@@ -0,0 +1,83 @@
+"""Continuous polling loop that persists WGMN readings to a Storage backend."""
+
+import datetime as dt
+import logging
+import signal
+import sqlite3
+import threading
+import time
+
+import requests
+
+from .constants import MIN_INTERVAL
+from .storage import Storage
+from .wgmn import fetch_history
+
+_UNITS = {"s": 1, "m": 60, "h": 3600}
+
+
+def parse_interval(text: str) -> float:
+    """Parse '600', '30s', '10m', or '1h' to seconds.
+
+    A bare number is seconds; a trailing s/m/h scales accordingly. The result
+    must be at least MIN_INTERVAL. Raises ValueError on malformed or too-small
+    input.
+    """
+    text = text.strip().lower()
+    if text and text[-1] in _UNITS:
+        seconds = float(text[:-1]) * _UNITS[text[-1]]
+    else:
+        seconds = float(text)
+    if seconds < MIN_INTERVAL:
+        raise ValueError(f"interval must be >= {MIN_INTERVAL}s")
+    return seconds
+
+
+def collect(
+    station_codes: list[str],
+    parameter_idx: list[int],
+    storage: Storage,
+    interval: float,
+    *,
+    log: logging.Logger | None = None,
+) -> int:
+    """Poll the selected stations/parameters every `interval` seconds.
+
+    Each poll fetches the full window and persists it idempotently. Network and
+    storage errors in one iteration are logged and skipped; the loop continues.
+    The interval is measured from each iteration's start, so a slow fetch
+    shortens the following wait rather than accumulating drift. Runs until
+    SIGINT or SIGTERM, then returns 0.
+    """
+    log = log or logging.getLogger(__name__)
+    stop = threading.Event()
+
+    def _handle(signum: int, frame: object) -> None:
+        log.info("received %s, shutting down", signal.Signals(signum).name)
+        stop.set()
+
+    signal.signal(signal.SIGINT, _handle)
+    signal.signal(signal.SIGTERM, _handle)
+
+    log.info(
+        "collecting %d station(s) × %d parameter(s), every %.0fs",
+        len(station_codes),
+        len(parameter_idx),
+        interval,
+    )
+    while not stop.is_set():
+        started = time.monotonic()
+        fetched_at = dt.datetime.now(dt.UTC)
+        try:
+            df = fetch_history(station_codes, parameter_idx)
+            inserted = storage.write(df, fetched_at)
+            log.info("inserted %d new row(s)", inserted)
+        except requests.RequestException as e:
+            log.warning("fetch failed, skipping iteration: %s", e)
+        except sqlite3.Error as e:
+            log.warning("storage error, skipping iteration: %s", e)
+        delay = max(0.0, interval - (time.monotonic() - started))
+        log.info("next poll in %.0fs", delay)
+        if stop.wait(timeout=delay):
+            break
+    return 0