microsegments 0.1.0__py3-none-any.whl

microsegments/io/__init__.py

@@ -0,0 +1,9 @@
+"""Input adapters: any source -> ``schema.OBSERVATION`` / ``SNAPSHOT`` / ``STOP_EVENT``, and feed coverage."""
+from __future__ import annotations
+
+from .coverage import coverage, coverage_from_config, snapshots_from_observations
+from .tabular import read_observations, read_snapshots
+from .timeutil import add_local_time
+
+__all__ = ["read_observations", "read_snapshots", "add_local_time", "coverage", "coverage_from_config",
+           "snapshots_from_observations"]

microsegments/io/coverage.py

@@ -0,0 +1,114 @@
+"""Feed health per local service date x hour (``schema.COVERAGE``).
+
+Each non-frozen poll at t_i covers [t_i, t_i + min(t_next - t_i, gap_cap_s)), t_next being the next
+non-frozen poll (the last poll covers min(tick_s, gap_cap_s)). Each frozen snapshot covers
+[t_i, t_i + min(t_next_any - t_i, gap_cap_s)) of ``frozen_s`` (t_next_any: next snapshot of any kind).
+Intervals are split exactly at local hour boundaries and credited to the hour they fall in.
+``coverage = covered_s / 3600`` clipped to 1. Frozen snapshots never count as polls.
+"""
+from __future__ import annotations
+
+import datetime as dt
+
+import numpy as np
+import polars as pl
+
+from ..schema import COVERAGE, SNAPSHOT, UTC, Flag
+from .timeutil import add_local_time, parse_hhmm, utc_offset_s
+
+
+def snapshots_from_observations(obs: pl.DataFrame | pl.LazyFrame) -> pl.DataFrame:
+    """Distinct observation times as polls (``frozen`` False, ``n_rows`` = rows at that ts)."""
+    lf = obs.lazy() if isinstance(obs, pl.DataFrame) else obs
+    return (lf.group_by("ts").agg(pl.len().cast(pl.UInt32).alias("n_rows"))
+            .with_columns(pl.lit(False).alias("frozen"))
+            .select(pl.col("ts").cast(UTC), "n_rows", "frozen").sort("ts").collect())
+
+
+def _split_by_hour(a: np.ndarray, b: np.ndarray, tz: str) -> tuple[np.ndarray, np.ndarray]:
+    """Split [a, b) epoch-second intervals at local hour boundaries -> (piece start, piece length)."""
+    starts, lens = [], []
+    a = a.astype(np.float64).copy()
+    b = b.astype(np.float64)
+    keep = b > a
+    a, b = a[keep], b[keep]
+    while a.size:
+        off = utc_offset_s(a, tz)
+        nxt = np.floor((a + off) / 3600.0) * 3600.0 + 3600.0 - off
+        nxt = np.where(nxt <= a, a + 3600.0, nxt)   # guard (DST edge)
+        end = np.minimum(b, nxt)
+        starts.append(a.copy())
+        lens.append(end - a)
+        more = end < b
+        a, b = end[more], b[more]
+    if not starts:
+        return np.zeros(0), np.zeros(0)
+    return np.concatenate(starts), np.concatenate(lens)
+
+
+def _hour_table(t: np.ndarray, value: np.ndarray, name: str, tz: str, sds: str) -> pl.DataFrame:
+    df = pl.DataFrame({"ts": (np.floor(t) * 1_000_000).astype(np.int64), name: value}).with_columns(
+        pl.col("ts").cast(pl.Datetime("us")).dt.replace_time_zone("UTC"))
+    return (add_local_time(df, tz, sds).group_by("service_date", "hour")
+            .agg(pl.col(name).sum()))
+
+
+def coverage(snapshots: pl.DataFrame | pl.LazyFrame, tz: str = "Europe/Brussels",
+             service_day_start: str = "04:00", tick_s: float = 20.0, gap_cap_s: float = 40.0,
+             max_frozen_s: float = 300.0, min_coverage: float = 0.8,
+             dates: tuple[dt.date, dt.date] | None = None) -> pl.DataFrame:
+    """SNAPSHOT -> COVERAGE, one row per service date x hour of the service day (24 rows per date,
+    from the service-day start hour, e.g. 4..27), zero-filled where the feed was silent.
+
+    ``dates``: (first, last) service dates of the grid; default: span of the snapshots."""
+    df = snapshots.collect() if isinstance(snapshots, pl.LazyFrame) else snapshots
+    if "frozen" not in df.columns:
+        df = df.with_columns(pl.lit(False).alias("frozen"))
+    df = (df.select(pl.col("ts").cast(UTC), pl.col("frozen").cast(pl.Boolean).fill_null(False))
+          .sort("ts", "frozen").unique("ts", keep="first", maintain_order=True))
+    t = df["ts"].dt.epoch("us").to_numpy().astype(np.float64) / 1e6
+    fr = df["frozen"].to_numpy().astype(bool)
+    cap = float(gap_cap_s) if gap_cap_s is not None else np.inf
+
+    # polls
+    tp = t[~fr]
+    gap = np.diff(tp, append=tp[-1] + tick_s if tp.size else np.zeros(0))
+    ps, pl_ = _split_by_hour(tp, tp + np.minimum(gap, cap), tz)
+    # frozen snapshots
+    nxt_any = np.append(t[1:], t[-1] + tick_s) if t.size else t
+    tf, gf = t[fr], (nxt_any - t)[fr]
+    fs, fl = _split_by_hour(tf, tf + np.minimum(gf, cap), tz)
+
+    sds = service_day_start
+    cov = _hour_table(ps, pl_, "covered_s", tz, sds)
+    frz = _hour_table(fs, fl, "frozen_s", tz, sds)
+    nsn = _hour_table(tp, np.ones_like(tp), "n_snapshots", tz, sds)
+
+    if dates is None:
+        if t.size == 0:
+            return pl.DataFrame(schema=COVERAGE)
+        sd = add_local_time(df.select("ts"), tz, sds)["service_date"]
+        dates = (sd.min(), sd.max())
+    h0 = parse_hhmm(sds) // 3600
+    grid = pl.DataFrame({"service_date": pl.date_range(dates[0], dates[1], "1d", eager=True)}).join(
+        pl.DataFrame({"hour": pl.Series(range(h0, h0 + 24), dtype=pl.Int8)}), how="cross")
+    out = (grid.join(cov, on=["service_date", "hour"], how="left")
+           .join(frz, on=["service_date", "hour"], how="left")
+           .join(nsn, on=["service_date", "hour"], how="left")
+           .with_columns(pl.col("covered_s", "frozen_s", "n_snapshots").fill_null(0.0)))
+    c = (pl.col("covered_s") / 3600.0).clip(0.0, 1.0)
+    flags = (pl.when(c < min_coverage).then(Flag.LOW_COVERAGE).otherwise(0)
+             + pl.when(pl.col("frozen_s") > max_frozen_s).then(Flag.FROZEN).otherwise(0))
+    out = out.with_columns(c.alias("coverage"), flags.alias("flags"))
+    return out.select([pl.col(k).cast(v) for k, v in COVERAGE.items()]).sort("service_date", "hour")
+
+
+def coverage_from_config(snapshots, cfg, dates=None) -> pl.DataFrame:
+    """``coverage`` with the parameters of a ``Config``."""
+    return coverage(snapshots, tz=cfg.input.timezone, service_day_start=cfg.input.service_day_start,
+                    tick_s=cfg.params.tick_s, gap_cap_s=cfg.params.gap_cap_s,
+                    max_frozen_s=cfg.quality.max_frozen_s, min_coverage=cfg.quality.min_hour_coverage,
+                    dates=dates)
+
+
+__all__ = ["coverage", "coverage_from_config", "snapshots_from_observations", "SNAPSHOT"]

microsegments/io/events.py

@@ -0,0 +1,98 @@
+"""Stop-call events -> ``schema.STOP_EVENT``.
+
+STIB punctuality parquet (MobilityTwin ``stib/punctuality``): one row per trip x stop with
+``journey_id, trip_id, start_date (YYYYMMDD), stop_sequence, stop_id ("5272F"), route_id ("55"),
+arrival_time, departure_time`` (naive UTC), ``observed`` (else inferred).
+Generic CSV / parquet: any column names, mapped canonical -> source.
+"""
+from __future__ import annotations
+
+import polars as pl
+
+from .. import schema
+from ..schema import STOP_EVENT, UTC
+from .ids import normalise_expr
+from .tabular import expand_paths, parse_ts, scan_table
+from .timeutil import add_local_time
+
+_PUNCT_COLS = ["journey_id", "trip_id", "start_date", "stop_sequence", "stop_id", "route_id",
+               "arrival_time", "departure_time", "observed"]
+
+
+def read_stib_punctuality(paths, route: str | None = None, id_normaliser: str = "leading_digits",
+                          only_observed: bool = False) -> pl.LazyFrame:
+    """STIB punctuality parquet files -> STOP_EVENT. ``trip_key`` = ``journey_id:trip_id``."""
+    files = [f for f in expand_paths(paths) if f.endswith(".parquet")]
+    if not files:
+        raise FileNotFoundError(f"no punctuality parquet in {paths}")
+    lf = pl.concat([pl.scan_parquet(f) for f in files], how="diagonal_relaxed")
+    names = lf.collect_schema().names()
+    if route is not None:
+        lf = lf.filter(pl.col("route_id").cast(pl.Utf8) == str(route))
+    if only_observed:
+        lf = lf.filter(pl.col("observed"))
+
+    def utc(c):
+        dtype = lf.collect_schema()[c]
+        e = pl.col(c)
+        if isinstance(dtype, pl.Datetime) and dtype.time_zone is None:
+            e = e.dt.replace_time_zone("UTC")
+        return e.dt.convert_time_zone("UTC").cast(UTC)
+
+    jid = pl.col("journey_id").cast(pl.Utf8) if "journey_id" in names else pl.lit("")
+    out = lf.select(
+        pl.col("start_date").cast(pl.Utf8).str.strptime(pl.Date, "%Y%m%d", strict=False).alias("service_date"),
+        pl.col("route_id").cast(pl.Utf8),
+        pl.concat_str([jid.fill_null(""), pl.col("trip_id").cast(pl.Utf8)], separator=":").alias("trip_key"),
+        normalise_expr(pl.col("stop_id"), id_normaliser).alias("stop_id"),
+        pl.col("stop_sequence").cast(pl.Int32),
+        utc("arrival_time").alias("arrival"),
+        utc("departure_time").alias("departure"),
+        (pl.col("observed") if "observed" in names else pl.lit(True)).cast(pl.Boolean).alias("observed"),
+    )
+    return out.select(list(STOP_EVENT))
+
+
+def read_events(paths, columns: dict[str, str] | None = None, ts_unit: str = "s",
+                tz: str = "Europe/Brussels", service_day_start: str = "04:00",
+                id_normaliser: str = "none", route: str | None = None) -> pl.LazyFrame:
+    """Generic CSV / parquet stop events -> STOP_EVENT.
+
+    ``columns``: canonical -> source names. Needs ``trip_key`` (or ``trip_id``), ``stop_id`` and
+    ``arrival`` and/or ``departure``. Missing ``service_date``: the service date of the trip's first
+    event; missing ``stop_sequence``: rank by time within the trip; missing ``observed``: True."""
+    lf = scan_table(expand_paths(paths))
+    names = lf.collect_schema().names()
+    ren = {src: can for can, src in (columns or {}).items() if src in names and src != can}
+    lf = lf.rename(ren)
+    names = lf.collect_schema().names()
+    if "trip_key" not in names:
+        if "trip_id" not in names:
+            raise ValueError("stop events need a trip_key or trip_id column")
+        lf = lf.with_columns(pl.col("trip_id").cast(pl.Utf8).alias("trip_key"))
+    for c in ("arrival", "departure"):
+        if c in names:
+            lf = parse_ts(lf, c, ts_unit, tz)
+        else:
+            lf = lf.with_columns(pl.lit(None, dtype=UTC).alias(c))
+    if "arrival" not in names and "departure" not in names:
+        raise ValueError("stop events need arrival and/or departure")
+    t = pl.coalesce("departure", "arrival")
+    if "service_date" not in names:
+        lf = add_local_time(lf.with_columns(t.min().over("trip_key").alias("_t0")), tz, service_day_start,
+                            ts_col="_t0").drop("_t0", "hour", "dow")
+    elif lf.collect_schema()["service_date"] == pl.Utf8:
+        sd = pl.col("service_date").str.replace_all("-", "")
+        lf = lf.with_columns(sd.str.strptime(pl.Date, "%Y%m%d", strict=False).alias("service_date"))
+    if "stop_sequence" not in names:
+        lf = lf.with_columns(t.rank("ordinal").over("trip_key").cast(pl.Int32).alias("stop_sequence"))
+    if "observed" not in names:
+        lf = lf.with_columns(pl.lit(True).alias("observed"))
+    lf = lf.with_columns(normalise_expr(pl.col("stop_id"), id_normaliser).alias("stop_id"))
+    if "route_id" in names:
+        lf = lf.with_columns(pl.col("route_id").cast(pl.Utf8))
+        if route is not None:
+            lf = lf.filter(pl.col("route_id") == str(route))
+    elif route is not None:
+        lf = lf.with_columns(pl.lit(str(route)).alias("route_id"))
+    return schema.conform(lf, STOP_EVENT).select(list(STOP_EVENT))

microsegments/io/gtfsrt.py

@@ -0,0 +1,168 @@
+"""GTFS-RT VehiclePositions -> ``schema.OBSERVATION``.
+
+Flattened tables (CSV / parquet) are read with their column names auto-detected: protobuf paths
+(``vehicle.position.latitude``, ``vehicle.trip.trip_id``, ``vehicle.vehicle.id``,
+``vehicle.timestamp`` ...), snake / camel variants, or plain ``latitude`` / ``lon`` / ``timestamp``
+/ ``trip_id`` / ``route_id`` / ``vehicle_id`` / ``direction_id`` / ``bearing``.
+Raw ``.pb`` FeedMessages need the ``gtfsrt`` extra (``gtfs-realtime-bindings``).
+"""
+from __future__ import annotations
+
+import gzip
+import re
+from pathlib import Path
+
+import polars as pl
+
+from .. import schema
+from ..schema import OBSERVATION
+
+try:  # optional extra
+    from google.transit import gtfs_realtime_pb2  # type: ignore
+    HAS_BINDINGS = True
+except Exception:  # pragma: no cover - depends on the environment
+    gtfs_realtime_pb2 = None
+    HAS_BINDINGS = False
+
+# canonical -> candidate suffixes of the normalised column name (dot separated, lower case)
+_CANDIDATES: dict[str, list[str]] = {
+    "ts": ["vehicle.timestamp", "position.timestamp", "timestamp", "ts", "time", "datetime", "fix.time"],
+    "lat": ["position.latitude", "latitude", "lat"],
+    "lon": ["position.longitude", "longitude", "lon", "lng", "long"],
+    "bearing": ["position.bearing", "bearing", "heading"],
+    "trip_id": ["trip.trip.id", "trip.id"],
+    "route_id": ["trip.route.id", "route.id", "route"],
+    "direction_id": ["trip.direction.id", "direction.id", "direction"],
+    "vehicle_id": ["vehicle.vehicle.id", "vehicle.id", "vehicle.label", "vehicle"],
+}
+
+
+def _norm(name: str) -> str:
+    n = re.sub(r"([a-z0-9])([A-Z])", r"\1.\2", name)       # camelCase -> camel.Case
+    return re.sub(r"[^a-z0-9]+", ".", n.lower()).strip(".")
+
+
+def detect_columns(names: list[str]) -> dict[str, str]:
+    """Map canonical OBSERVATION names to source columns (canonical -> source)."""
+    normed = {c: _norm(c) for c in names if not _norm(c).startswith("header.")}
+    out: dict[str, str] = {}
+    used: set[str] = set()
+    for can, cands in _CANDIDATES.items():
+        for cand in cands:
+            hits = [c for c, n in normed.items() if c not in used and (n == cand or n.endswith("." + cand))]
+            if hits:
+                best = min(hits, key=lambda c: len(normed[c]))
+                out[can] = best
+                used.add(best)
+                break
+    return out
+
+
+def _guess_unit(lf: pl.LazyFrame, col: str) -> str:
+    dtype = lf.collect_schema()[col]
+    if dtype == pl.Utf8:
+        v = lf.select(pl.col(col)).drop_nulls().head(1).collect()[col]
+        if len(v) and not re.fullmatch(r"\s*\d+(\.\d+)?\s*", v[0]):
+            return "iso"
+        mx = lf.select(pl.col(col).cast(pl.Float64, strict=False).max()).collect().item()
+    elif dtype.is_numeric():
+        mx = lf.select(pl.col(col).max()).collect().item()
+    else:
+        return "s"
+    if mx is None:
+        return "s"
+    return "us" if mx > 1e14 else "ms" if mx > 1e11 else "s"
+
+
+def read_flat(paths, route: str | None = None, tz: str = "Europe/Brussels",
+              columns: dict[str, str] | None = None, ts_unit: str | None = None) -> pl.LazyFrame:
+    """Flattened VehiclePositions (CSV / parquet files or globs) -> OBSERVATION LazyFrame.
+    ``columns`` overrides the detection (canonical -> source); ``ts_unit`` None = guessed."""
+    from .tabular import expand_paths, normalise_frame, scan_table
+    lf = scan_table(expand_paths(paths))
+    cols = detect_columns(lf.collect_schema().names())
+    cols.update(columns or {})
+    if "ts" not in cols:
+        raise ValueError("no timestamp column detected")
+    unit = ts_unit or _guess_unit(lf, cols["ts"])
+    return normalise_frame(lf, cols, unit, tz, "none", route)
+
+
+def parse_feed(content: bytes, route: str | None = None) -> pl.DataFrame:
+    """One serialized FeedMessage -> OBSERVATION rows (vehicle entities only)."""
+    if not HAS_BINDINGS:
+        raise ImportError("parsing .pb needs the 'gtfsrt' extra: pip install 'microsegments[gtfsrt]'")
+    msg = gtfs_realtime_pb2.FeedMessage()
+    msg.ParseFromString(content)
+    head_ts = msg.header.timestamp if msg.header.HasField("timestamp") else None
+    rows = []
+    for ent in msg.entity:
+        if not ent.HasField("vehicle"):
+            continue
+        v = ent.vehicle
+        if not v.HasField("position"):
+            continue
+        ts = v.timestamp if v.HasField("timestamp") else head_ts
+        if ts is None:
+            continue
+        tr = v.trip if v.HasField("trip") else None
+        rid = tr.route_id if tr is not None and tr.route_id else None
+        if route is not None and rid != str(route):
+            continue
+        p = v.position
+        rows.append({
+            "ts": int(ts) * 1_000_000,
+            "route_id": rid,
+            "vehicle_id": (v.vehicle.id or v.vehicle.label or ent.id) if v.HasField("vehicle") else ent.id,
+            "trip_id": (tr.trip_id or None) if tr is not None else None,
+            "direction_id": tr.direction_id if tr is not None and tr.HasField("direction_id") else None,
+            "lat": p.latitude, "lon": p.longitude,
+            "bearing": p.bearing if p.HasField("bearing") else None,
+        })
+    df = pl.DataFrame(rows, schema={"ts": pl.Int64, "route_id": pl.Utf8, "vehicle_id": pl.Utf8,
+                                    "trip_id": pl.Utf8, "direction_id": pl.Int8, "lat": pl.Float64,
+                                    "lon": pl.Float64, "bearing": pl.Float32})
+    df = df.with_columns(pl.col("ts").cast(pl.Datetime("us")).dt.replace_time_zone("UTC"))
+    return schema.conform(df, OBSERVATION, schema.OBSERVATION_REQUIRED)
+
+
+def read_pb(paths, route: str | None = None) -> pl.DataFrame:
+    """``.pb`` / ``.pb.gz`` FeedMessage files -> OBSERVATION, duplicates (same vehicle and ts) dropped."""
+    from .tabular import expand_paths
+    frames = []
+    for f in expand_paths(paths):
+        b = Path(f).read_bytes()
+        if f.endswith(".gz"):
+            b = gzip.decompress(b)
+        frames.append(parse_feed(b, route))
+    if not frames:
+        return schema.empty(OBSERVATION)
+    return pl.concat(frames).unique(["ts", "vehicle_id", "lat", "lon"], maintain_order=True).sort("ts")
+
+
+def build_feed(obs: pl.DataFrame) -> bytes:
+    """OBSERVATION (lat/lon) -> serialized FeedMessage (for tests and examples)."""
+    if not HAS_BINDINGS:
+        raise ImportError("needs the 'gtfsrt' extra")
+    msg = gtfs_realtime_pb2.FeedMessage()
+    msg.header.gtfs_realtime_version = "2.0"
+    if len(obs):
+        msg.header.timestamp = int(obs["ts"].dt.epoch("s").max())
+    for i, r in enumerate(obs.iter_rows(named=True)):
+        e = msg.entity.add()
+        e.id = str(i)
+        v = e.vehicle
+        v.timestamp = int(r["ts"].timestamp())
+        v.position.latitude = r["lat"]
+        v.position.longitude = r["lon"]
+        if r.get("bearing") is not None:
+            v.position.bearing = r["bearing"]
+        if r.get("vehicle_id") is not None:
+            v.vehicle.id = r["vehicle_id"]
+        if r.get("trip_id") is not None:
+            v.trip.trip_id = r["trip_id"]
+        if r.get("route_id") is not None:
+            v.trip.route_id = r["route_id"]
+        if r.get("direction_id") is not None:
+            v.trip.direction_id = int(r["direction_id"])
+    return msg.SerializeToString()

microsegments/io/ids.py

@@ -0,0 +1,34 @@
+"""Stop / terminus id normalisation, so feed ids match GTFS ids.
+
+``"leading_digits"`` mirrors the prototype's ``digits()``: the leading digits without their
+leading zeros ("0626" -> "626", "5272F" -> "5272", "0" -> "0"); not starting with a digit -> null.
+``"none"`` keeps the id as text. Any other string is a regex whose first group is kept.
+"""
+from __future__ import annotations
+
+import re
+
+import polars as pl
+
+
+def normalise_expr(e: pl.Expr, normaliser: str = "leading_digits") -> pl.Expr:
+    e = e.cast(pl.Utf8).str.strip_chars()
+    if normaliser in (None, "", "none"):
+        return e
+    if normaliser == "leading_digits":
+        return e.str.extract(r"^0*(\d+)", 1)
+    if re.compile(normaliser).groups < 1:
+        raise ValueError(f"id normaliser regex needs one group: {normaliser!r}")
+    return e.str.extract(normaliser, 1)
+
+
+def normalise_py(value, normaliser: str = "leading_digits") -> str | None:
+    """Scalar version of ``normalise_expr``."""
+    if value is None:
+        return None
+    s = str(value).strip()
+    if normaliser in (None, "", "none"):
+        return s
+    pat = r"^0*(\d+)" if normaliser == "leading_digits" else normaliser
+    m = re.search(pat, s)
+    return m.group(1) if m else None

microsegments/io/stib.py

@@ -0,0 +1,217 @@
+"""STIB ``vehicle-distance`` feed (MobilityTwin ``stib/vehicle-distance``).
+
+Raw snapshot (one every ~20 s, all lines)::
+
+    [{"lineId": "55", "directionId": "9943", "pointId": "5766", "distanceFromPoint": 95}, ...]
+
+(or the same list under a ``data`` / ``results`` key). A row is a vehicle: line, terminus code
+(``directionId``, a stop code, sometimes not a GTFS stop such as 9943), last stop passed and metres
+travelled since it. No vehicle id.
+
+Compact per-day parquet (prototype ``data2025/stib/vd55`` and the platform's ``raw/vd``)::
+
+    <day>.parquet        ts:int64 epoch s, [line], dir, point, dist
+    <day>.snaps.parquet  ts:int64 epoch s, same|frozen:bool, [n_rows]
+
+Rows of frozen snapshots (content identical to the previous one) are not stored: the feed did not
+move, so they carry no new observation. Snapshots keep them, flagged ``frozen``.
+"""
+from __future__ import annotations
+
+import glob
+import gzip
+import hashlib
+import json
+import re
+from collections.abc import Iterable
+from pathlib import Path
+
+import polars as pl
+
+from .. import schema
+from ..schema import OBSERVATION, SNAPSHOT, UTC
+from .ids import normalise_expr
+
+_FNAME_TS = re.compile(r"(\d{4})-(\d{2})-(\d{2})T(\d{2})-?:?(\d{2})-?:?(\d{2})")
+
+
+def _records(data) -> list[dict]:
+    if isinstance(data, (bytes, str)):
+        data = json.loads(data)
+    if isinstance(data, dict):
+        data = data.get("data") or data.get("results") or []
+    return list(data or [])
+
+
+def _epoch_us(ts) -> int:
+    """int/float epoch seconds, datetime or ISO string -> epoch microseconds."""
+    import datetime as dt
+    if isinstance(ts, (int, float)):
+        return int(round(float(ts) * 1_000_000))
+    if isinstance(ts, str):
+        ts = dt.datetime.fromisoformat(ts.replace("Z", "+00:00"))
+    if isinstance(ts, dt.datetime):
+        if ts.tzinfo is None:
+            ts = ts.replace(tzinfo=dt.timezone.utc)
+        return int(round(ts.timestamp() * 1_000_000))
+    raise TypeError(f"unsupported timestamp {ts!r}")
+
+
+def _rows_frame(recs: list[dict], ts_us: int) -> pl.DataFrame:
+    return pl.DataFrame({
+        "ts": [ts_us] * len(recs),
+        "line": [None if r.get("lineId") is None else str(r.get("lineId")) for r in recs],
+        "dir": [None if r.get("directionId") is None else str(r.get("directionId")) for r in recs],
+        "point": [None if r.get("pointId") is None else str(r.get("pointId")) for r in recs],
+        "dist": [r.get("distanceFromPoint") for r in recs],
+    }, schema={"ts": pl.Int64, "line": pl.Utf8, "dir": pl.Utf8, "point": pl.Utf8, "dist": pl.Float64})
+
+
+def parse_snapshots(snapshots: Iterable[tuple[object, object]], route: str | list[str] | None = None,
+                    id_normaliser: str = "leading_digits") -> tuple[pl.DataFrame, pl.DataFrame]:
+    """Parse raw snapshots ``(ts, data)`` in time order -> (OBSERVATION, SNAPSHOT) frames.
+
+    ``ts``: epoch seconds, datetime (naive = UTC) or ISO string; ``data``: decoded JSON, bytes or str.
+    ``route``: keep these lineIds only (None: all lines). ``n_rows`` counts every row of the snapshot.
+    A snapshot whose content equals the previous one is ``frozen`` and contributes no rows."""
+    routes = None if route is None else {str(route)} if isinstance(route, (str, int)) else {str(r) for r in route}
+    frames, snaps = [], []
+    prev = None
+    for ts, data in snapshots:
+        raw = data if isinstance(data, (bytes, str)) else json.dumps(data, sort_keys=True)
+        digest = hashlib.sha1(raw.encode() if isinstance(raw, str) else raw).digest()
+        recs = _records(data)
+        ts_us = _epoch_us(ts)
+        frozen = digest == prev
+        prev = digest
+        snaps.append((ts_us, len(recs), frozen))
+        if frozen:
+            continue
+        if routes is not None:
+            recs = [r for r in recs if str(r.get("lineId")) in routes]
+        if recs:
+            frames.append(_rows_frame(recs, ts_us))
+    rows = pl.concat(frames) if frames else _rows_frame([], 0)
+    obs = compact_to_observations(rows.lazy(), id_normaliser=id_normaliser, ts_unit="us").collect()
+    sn = pl.DataFrame(snaps, schema={"ts": pl.Int64, "n_rows": pl.UInt32, "frozen": pl.Boolean}, orient="row")
+    sn = sn.with_columns(pl.col("ts").cast(pl.Datetime("us")).dt.replace_time_zone("UTC")).cast(SNAPSHOT)
+    return obs, sn
+
+
+def ts_from_filename(path: str | Path) -> float | None:
+    """MobilityTwin download name ``2025-02-17T04-00-00Z.json`` (UTC) -> epoch seconds."""
+    import datetime as dt
+    m = _FNAME_TS.search(Path(path).name)
+    if not m:
+        return None
+    y, mo, d, h, mi, s = map(int, m.groups())
+    return dt.datetime(y, mo, d, h, mi, s, tzinfo=dt.timezone.utc).timestamp()
+
+
+def read_json_snapshots(paths: str | Path | list, route: str | list[str] | None = None,
+                        id_normaliser: str = "leading_digits") -> tuple[pl.DataFrame, pl.DataFrame]:
+    """Raw JSON snapshot files (``.json`` / ``.json.gz``, globs allowed), time from the file name
+    (MobilityTwin layout) -> (OBSERVATION, SNAPSHOT)."""
+    files = _expand(paths)
+    stamped = []
+    for f in files:
+        t = ts_from_filename(f)
+        if t is None:
+            raise ValueError(f"no UTC timestamp in file name {f} (expected e.g. 2025-02-17T04-00-00Z.json)")
+        stamped.append((t, f))
+    stamped.sort()
+
+    def gen():
+        for t, f in stamped:
+            b = Path(f).read_bytes()
+            if str(f).endswith(".gz"):
+                b = gzip.decompress(b)
+            yield t, b
+    return parse_snapshots(gen(), route=route, id_normaliser=id_normaliser)
+
+
+def compact_to_observations(lf: pl.LazyFrame, route_id: str | None = None,
+                            id_normaliser: str = "leading_digits", ts_unit: str = "s") -> pl.LazyFrame:
+    """Compact rows (ts, [line], dir, point, dist) -> OBSERVATION. ``route_id`` fills a missing
+    ``line`` column (prototype files hold one line) or filters an existing one."""
+    cols = lf.collect_schema().names()
+    factor = {"s": 1_000_000, "ms": 1_000, "us": 1}[ts_unit]
+    if "line" in cols:
+        route = pl.col("line").cast(pl.Utf8)
+        if route_id is not None:
+            lf = lf.filter(pl.col("line").cast(pl.Utf8) == str(route_id))
+    elif route_id is not None:
+        route = pl.lit(str(route_id), dtype=pl.Utf8)
+    else:
+        raise ValueError("compact STIB rows have no 'line' column: pass route_id")
+    out = lf.select(
+        (pl.col("ts").cast(pl.Int64) * factor).cast(pl.Datetime("us")).dt.replace_time_zone("UTC").alias("ts"),
+        route.alias("route_id"),
+        normalise_expr(pl.col("dir"), id_normaliser).alias("terminus_id"),
+        normalise_expr(pl.col("point"), id_normaliser).alias("stop_id"),
+        pl.col("dist").cast(pl.Float32).alias("dist_m"),
+    )
+    return schema.conform(out, OBSERVATION, schema.OBSERVATION_REQUIRED)
+
+
+def observations_to_compact(obs: pl.DataFrame | pl.LazyFrame) -> pl.DataFrame | pl.LazyFrame:
+    """OBSERVATION (linear) -> compact columns ts:int64 s, line, dir, point, dist:int64."""
+    return obs.select(
+        pl.col("ts").dt.epoch("s").alias("ts"),
+        pl.col("route_id").alias("line"),
+        pl.col("terminus_id").alias("dir"),
+        pl.col("stop_id").alias("point"),
+        pl.col("dist_m").round(0).cast(pl.Int64).alias("dist"),
+    )
+
+
+def snapshots_to_compact(snaps: pl.DataFrame | pl.LazyFrame) -> pl.DataFrame | pl.LazyFrame:
+    return snaps.select(pl.col("ts").dt.epoch("s").alias("ts"), pl.col("frozen"), pl.col("n_rows"))
+
+
+def _expand(paths) -> list[str]:
+    if isinstance(paths, (str, Path)):
+        paths = [paths]
+    out: list[str] = []
+    for p in paths:
+        p = str(p)
+        if Path(p).is_dir():
+            out += sorted(str(x) for x in Path(p).rglob("*") if x.is_file())
+        elif any(ch in p for ch in "*?["):
+            out += sorted(glob.glob(p, recursive=True))
+        else:
+            out.append(p)
+    return out
+
+
+def split_compact_paths(paths) -> tuple[list[str], list[str]]:
+    """Separate row files from snapshot files (``*.snaps.parquet`` or ``snaps.parquet``)."""
+    files = [f for f in _expand(paths) if f.endswith(".parquet")]
+    snaps = [f for f in files if Path(f).name.endswith("snaps.parquet")]
+    rows = [f for f in files if f not in set(snaps)]
+    return rows, snaps
+
+
+def read_compact(paths, route_id: str | None = None, id_normaliser: str = "leading_digits",
+                 ) -> tuple[pl.LazyFrame, pl.LazyFrame | None]:
+    """Compact per-day parquet (files, directories or globs; row and ``snaps`` files may be mixed)
+    -> (OBSERVATION LazyFrame, SNAPSHOT LazyFrame or None if no snaps file)."""
+    rows, snaps = split_compact_paths(paths)
+    if not rows:
+        raise FileNotFoundError(f"no STIB compact parquet in {paths}")
+    lf = pl.concat([pl.scan_parquet(f) for f in rows], how="diagonal_relaxed")
+    obs = compact_to_observations(lf, route_id=route_id, id_normaliser=id_normaliser)
+    return obs, (read_compact_snapshots(snaps) if snaps else None)
+
+
+def read_compact_snapshots(paths) -> pl.LazyFrame:
+    files = _expand(paths)
+    lf = pl.concat([pl.scan_parquet(f) for f in files], how="diagonal_relaxed")
+    cols = lf.collect_schema().names()
+    frozen = pl.col("frozen") if "frozen" in cols else pl.col("same") if "same" in cols else pl.lit(False)
+    n_rows = pl.col("n_rows") if "n_rows" in cols else pl.lit(None)
+    return lf.select(
+        (pl.col("ts").cast(pl.Int64) * 1_000_000).cast(pl.Datetime("us")).dt.replace_time_zone("UTC").cast(UTC).alias("ts"),
+        n_rows.cast(pl.UInt32).alias("n_rows"),
+        frozen.cast(pl.Boolean).fill_null(False).alias("frozen"),
+    ).unique("ts", keep="first").sort("ts")