microsegments 0.1.0__tar.gz → 0.2.0__tar.gz

{microsegments-0.1.0 → microsegments-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: microsegments
-Version: 0.1.0
+Version: 0.2.0
 Summary: Where do transit vehicles linger? Count vehicle position observations per micro-segment of a line, from GTFS + AVL / GTFS-RT positions.
 Project-URL: Homepage, https://github.com/GaspardMerten/microsegments
 Project-URL: Repository, https://github.com/GaspardMerten/microsegments
@@ -132,7 +132,9 @@ Stages are usable on their own: `read_observations`, `build_network`, `segment`,
 microsegments run CONFIG -o OUT [--lang en] [--segment-m 20] [--dates A..B] [--source "credit"]
 microsegments tune CONFIG [--lengths 10,20,30,50] [--sensitivity] [-o OUT]
 microsegments hotspots CONFIG [-o hotspots.geojson|.csv|.parquet]
-microsegments inspect CONFIG        # coverage per day, GTFS versions, placement / drop counts
+microsegments inspect CONFIG        # coverage per day, GTFS versions, placement, evening vs line reference
+microsegments compare CONFIG --a 2025-02-17..2025-03-31 --b 2025-04-01..2025-05-15 -o OUT
+microsegments sensitivity-table CONFIG [CONFIG ...] -o OUT    # one tidy table + markdown over several lines
 ```
 
 ## What the numbers mean
@@ -169,7 +171,8 @@ fixed per metric (p98 over 6–21 h), so hours and weekdays compare directly.
 split-half reliability, hotspot localisation spread and Jaccard) and picks the smallest L with reliability
 ≥ 0.8, spread ≤ 30 m and deviance within one standard error of the minimum. 30 m is a good default for
 20 s polling at urban speeds; go shorter only with dense GTFS-RT fixes and many days. `--sensitivity`
-re-runs phase offsets, gap caps, stop zones, references and passage sources.
+re-runs phase offsets, gap caps (30, 40, 60, ∞ s), stop zones, references, passage sources and segment
+lengths 15 / 30 / 60 m around the baseline.
 
 ## Hotspots
 
@@ -179,6 +182,15 @@ Adjacent bins merge, never across a stop / running border. Classes: *infrastruct
 *congestion* (peak only), *mixed*. Needs at least 5 included days. Export as GeoJSON with
 `microsegments hotspots CONFIG -o hotspots.geojson`.
 
+## Comparing two periods
+
+`microsegments compare` (or `ms.compare(analysis_a, analysis_b)`) compares observations per passage
+segment by segment and hour by hour, each period with its own days and references, with a day bootstrap
+of the difference. Segments are matched by key (or by stop pair when a new GTFS version moved the shape a
+little); the others are listed and left out. Adjacent significant segments form stretches, ranked by the
+change in vehicle time per day; stretches whose frequency changed by more than 20 % are flagged, since a
+timetable change can explain part of the difference. The HTML page gets a "compare two periods" view.
+
 ## License
 
 MIT. The bundled Leaflet CSS is BSD-2-Clause.

{microsegments-0.1.0 → microsegments-0.2.0}/README.md

@@ -94,7 +94,9 @@ Stages are usable on their own: `read_observations`, `build_network`, `segment`,
 microsegments run CONFIG -o OUT [--lang en] [--segment-m 20] [--dates A..B] [--source "credit"]
 microsegments tune CONFIG [--lengths 10,20,30,50] [--sensitivity] [-o OUT]
 microsegments hotspots CONFIG [-o hotspots.geojson|.csv|.parquet]
-microsegments inspect CONFIG        # coverage per day, GTFS versions, placement / drop counts
+microsegments inspect CONFIG        # coverage per day, GTFS versions, placement, evening vs line reference
+microsegments compare CONFIG --a 2025-02-17..2025-03-31 --b 2025-04-01..2025-05-15 -o OUT
+microsegments sensitivity-table CONFIG [CONFIG ...] -o OUT    # one tidy table + markdown over several lines
 ```
 
 ## What the numbers mean
@@ -131,7 +133,8 @@ fixed per metric (p98 over 6–21 h), so hours and weekdays compare directly.
 split-half reliability, hotspot localisation spread and Jaccard) and picks the smallest L with reliability
 ≥ 0.8, spread ≤ 30 m and deviance within one standard error of the minimum. 30 m is a good default for
 20 s polling at urban speeds; go shorter only with dense GTFS-RT fixes and many days. `--sensitivity`
-re-runs phase offsets, gap caps, stop zones, references and passage sources.
+re-runs phase offsets, gap caps (30, 40, 60, ∞ s), stop zones, references, passage sources and segment
+lengths 15 / 30 / 60 m around the baseline.
 
 ## Hotspots
 
@@ -141,6 +144,15 @@ Adjacent bins merge, never across a stop / running border. Classes: *infrastruct
 *congestion* (peak only), *mixed*. Needs at least 5 included days. Export as GeoJSON with
 `microsegments hotspots CONFIG -o hotspots.geojson`.
 
+## Comparing two periods
+
+`microsegments compare` (or `ms.compare(analysis_a, analysis_b)`) compares observations per passage
+segment by segment and hour by hour, each period with its own days and references, with a day bootstrap
+of the difference. Segments are matched by key (or by stop pair when a new GTFS version moved the shape a
+little); the others are listed and left out. Adjacent significant segments form stretches, ranked by the
+change in vehicle time per day; stretches whose frequency changed by more than 20 % are flagged, since a
+timetable change can explain part of the difference. The HTML page gets a "compare two periods" view.
+
 ## License
 
 MIT. The bundled Leaflet CSS is BSD-2-Clause.

microsegments-0.2.0/examples/stib1_platform.toml

@@ -0,0 +1,23 @@
+# LOCAL EXAMPLE (not shipped data): STIB line 1 from the StibMicrosegments platform's local ingest
+# (raw/vd/date=.../{vd,snaps}.parquet with a line column, raw/punctuality, gtfs/index.parquet + gtfs/feeds/<sha>).
+#   microsegments sensitivity-table examples/stib55_full.toml examples/stib71_platform.toml examples/stib1_platform.toml -o out/sens
+
+[input]
+paths = ["/home/gaspard/Documents/Dev/CoDE/StibMicrosegments/data/ms/raw/vd/date=2025-0[2-5]-*/*.parquet"]
+kind = "stib"
+events = "/home/gaspard/Documents/Dev/CoDE/StibMicrosegments/data/ms/raw/punctuality/date=2025-0[2-5]-*/p.parquet"
+
+[gtfs]
+index = "/home/gaspard/Documents/Dev/CoDE/StibMicrosegments/data/ms/gtfs/index.parquet"
+
+[select]
+route = "1"
+dates = "2025-03-03..2025-03-28"
+weekdays = [0, 1, 2, 3, 4]
+
+[params]
+segment_m = 30
+bootstrap = 200
+
+[report]
+names = "title"

microsegments-0.2.0/examples/stib55_full.toml

@@ -0,0 +1,32 @@
+# LOCAL EXAMPLE (not shipped data): STIB tram 55, weekdays 17 Feb - 15 May 2025, on the author's machine.
+# Paths point at ~/Documents/Dev/CoDE/BrusselsTransitStuck/data2025 (MobilityTwin stib/vehicle-distance as
+# compact daily parquet, STIB punctuality, one GTFS feed); adapt them to your copy of the data.
+#   microsegments run examples/stib55_full.toml -o out/stib55_full
+#   microsegments tune examples/stib55_full.toml --sensitivity
+
+[input]
+paths = ["/home/gaspard/Documents/Dev/CoDE/BrusselsTransitStuck/data2025/stib/vd55/*.parquet"]
+kind = "stib"
+events = "/home/gaspard/Documents/Dev/CoDE/BrusselsTransitStuck/data2025/stib/punctuality/2025-0[2-5]-*.parquet"
+
+[input.linear]
+direction_kind = "terminus_stop"
+id_normaliser = "leading_digits"
+feed_length = "auto"
+
+[gtfs]
+# one feed (calendar 14 Apr - 11 May 2025); other dates take the patterns of the same weekday
+path = "/home/gaspard/Documents/Dev/CoDE/BrusselsTransitStuck/data2025/gtfs"
+
+[select]
+route = "55"
+dates = "2025-02-17..2025-05-15"
+weekdays = [0, 1, 2, 3, 4]
+exclude_dates = ["2025-04-21", "2025-05-01"]     # Easter Monday, Labour Day
+
+[params]
+segment_m = 30
+bootstrap = 200
+
+[report]
+names = "title"                                 # "GARE DU NORD" -> "Gare du Nord"

microsegments-0.2.0/examples/stib71_platform.toml

@@ -0,0 +1,23 @@
+# LOCAL EXAMPLE (not shipped data): STIB line 71 from the StibMicrosegments platform's local ingest
+# (raw/vd/date=.../{vd,snaps}.parquet with a line column, raw/punctuality, gtfs/index.parquet + gtfs/feeds/<sha>).
+#   microsegments sensitivity-table examples/stib55_full.toml examples/stib71_platform.toml examples/stib1_platform.toml -o out/sens
+
+[input]
+paths = ["/home/gaspard/Documents/Dev/CoDE/StibMicrosegments/data/ms/raw/vd/date=2025-0[2-5]-*/*.parquet"]
+kind = "stib"
+events = "/home/gaspard/Documents/Dev/CoDE/StibMicrosegments/data/ms/raw/punctuality/date=2025-0[2-5]-*/p.parquet"
+
+[gtfs]
+index = "/home/gaspard/Documents/Dev/CoDE/StibMicrosegments/data/ms/gtfs/index.parquet"
+
+[select]
+route = "71"
+dates = "2025-03-03..2025-03-28"
+weekdays = [0, 1, 2, 3, 4]
+
+[params]
+segment_m = 30
+bootstrap = 200
+
+[report]
+names = "title"

{microsegments-0.1.0 → microsegments-0.2.0}/src/microsegments/__init__.py

@@ -15,6 +15,7 @@ except ImportError:  # pragma: no cover
     __version__ = "0.0.0"
 
 from .aggregate import count
+from .compare import Comparison, compare, run_compare
 from .config import Config
 from .hotspots import hotspots
 from .html import export
@@ -28,8 +29,8 @@ from .segments import segment
 from .simulate import simulate
 from . import tune
 
-__all__ = ["Analysis", "Config", "Flag", "RunResult", "__version__", "analyse", "build_network", "count", "export",
-           "hotspots", "plot", "prepare", "read_observations", "run", "segment", "simulate", "to_contract", "tune"]
+__all__ = ["Analysis", "Comparison", "Config", "Flag", "RunResult", "__version__", "analyse", "build_network", "compare", "count", "export",
+           "hotspots", "plot", "prepare", "read_observations", "run", "run_compare", "segment", "simulate", "to_contract", "tune"]
 
 
 def __getattr__(name: str):

{microsegments-0.1.0 → microsegments-0.2.0}/src/microsegments/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.1.0'
-__version_tuple__ = version_tuple = (0, 1, 0)
+__version__ = version = '0.2.0'
+__version_tuple__ = version_tuple = (0, 2, 0)
 
 __commit_id__ = commit_id = None

{microsegments-0.1.0 → microsegments-0.2.0}/src/microsegments/aggregate.py

@@ -6,9 +6,11 @@ upper bound closed, as in the prototype: a vehicle standing at stop B is at the
 A row at ``pos_m <= 0`` of link i > 0 goes to the last segment of link i - 1, i.e. the end of the
 link arriving at that stop. Positions beyond the link end are clipped to its last segment.
 
-Per-vehicle feeds (GTFS-RT with irregular / dense fixes) are thinned to at most one observation
-per ``(track_id, floor(ts / tick_s))`` (the last fix of each tick), so counts stay comparable with
-a snapshot feed polled every ``tick_s``.
+Per-vehicle feeds (GTFS-RT with irregular / dense fixes) are put on a ``tick_s`` grid so counts stay
+comparable with a snapshot feed polled every ``tick_s``: by default ``locate.resample.resample`` (each
+track sampled at the grid instants between its fixes, nearer fix; unbiased at track ends and for
+sparse fixes), or ``method="thin"``: at most one observation per ``(track_id, floor(ts / tick_s))``
+(the last fix of each tick; under-counts when fixes are sparser than the tick).
 """
 from __future__ import annotations
 
@@ -74,15 +76,34 @@ def assign(placed: pl.DataFrame, segments: pl.DataFrame) -> pl.DataFrame:
     )
 
 
+def per_vehicle_grid(placed: pl.DataFrame, tick_s: float = 20.0, method: str = "resample") -> pl.DataFrame:
+    """Counted rows of a per-vehicle feed on the ``tick_s`` grid (``method`` "resample" or "thin")."""
+    if method == "thin":
+        return thin(placed.filter(pl.col("count").fill_null(True)), tick_s)
+    if method != "resample":
+        raise ValueError(f"unknown per-vehicle method {method!r}")
+    from .locate.resample import resample
+    df = placed
+    if "s_m" not in df.columns:
+        df = df.with_columns(pl.col("pos_m").alias("s_m"))
+    elif "pos_m" in df.columns and df["s_m"].null_count():
+        df = df.with_columns(pl.coalesce("s_m", pl.col("pos_m").cast(df.schema["s_m"])).alias("s_m"))
+    if "count" not in df.columns:
+        df = df.with_columns(pl.lit(True).alias("count"))
+    # resample first (non-counted fixes still bound the intervals), then keep counted instants
+    return resample(df, tick_s).filter(pl.col("count").fill_null(True))
+
+
 def count(placed: pl.DataFrame, segments: pl.DataFrame, tick_s: float = 20.0,
-          per_vehicle: bool = False) -> pl.DataFrame:
+          per_vehicle: bool = False, method: str = "resample") -> pl.DataFrame:
     """Placed -> Cube (service_date, hour, pattern_uid, seg_key, obs).
 
-    Σ obs equals the number of counted (and, for per-vehicle feeds, thinned) rows whose link has
-    segments."""
-    df = placed.filter(pl.col("count").fill_null(True))
+    Σ obs equals the number of counted (for per-vehicle feeds: resampled, or thinned with
+    ``method="thin"``) rows whose link has segments."""
     if per_vehicle:
-        df = thin(df, tick_s)
+        df = per_vehicle_grid(placed, tick_s, method)
+    else:
+        df = placed.filter(pl.col("count").fill_null(True))
     df = assign(df.select("ts", "service_date", "hour", "pattern_uid", "link_idx", "pos_m", "track_id"), segments)
     cube = (df.filter(pl.col("seg_key").is_not_null())
             .group_by("service_date", "hour", "pattern_uid", "seg_key")

{microsegments-0.1.0 → microsegments-0.2.0}/src/microsegments/cli.py

@@ -1,4 +1,5 @@
-"""Command line: ``microsegments run | tune | hotspots | inspect CONFIG``."""
+"""Command line: ``microsegments run | tune | hotspots | inspect | compare CONFIG``, ``microsegments
+sensitivity-table CONFIG...``."""
 from __future__ import annotations
 
 import argparse
@@ -25,13 +26,16 @@ def _log(args):
     return (lambda s: print(s, file=sys.stderr)) if not getattr(args, "quiet", False) else None
 
 
-def _hotspot_table(hs: pl.DataFrame, tick_s: float) -> pl.DataFrame:
+def _hotspot_table(hs: pl.DataFrame, tick_s: float, names: dict[str, str] | None = None) -> pl.DataFrame:
     if hs is None or hs.height == 0:
         return pl.DataFrame()
+    nm = names or {}
+    fix = lambda c: pl.col(c).replace(nm) if nm else pl.col(c)  # noqa: E731
+    crit = pl.col("criterion") if "criterion" in hs.columns else pl.lit("peak")
     return hs.select(
         "direction_id", "rank",
-        pl.concat_str([pl.col("from_stop_name"), pl.lit(" -> "), pl.col("to_stop_name")]).alias("stretch"),
-        pl.col("x0_m").round(0), pl.col("x1_m").round(0), "zone", "kind",
+        pl.concat_str([fix("from_stop_name"), pl.lit(" -> "), fix("to_stop_name")]).alias("stretch"),
+        pl.col("x0_m").round(0), pl.col("x1_m").round(0), "zone", "kind", crit.alias("criterion"),
         pl.col("hours").cast(pl.List(pl.Utf8)).list.join(",").alias("hours"),
         pl.col("excess_per_passage").round(2).alias("excess_obs_per_veh"),
         (pl.col("excess_per_passage") * tick_s).round(0).alias("approx_s_per_veh"),
@@ -57,9 +61,10 @@ def cmd_hotspots(args) -> int:
     from .pipeline import run
     cfg = _cfg(args)
     res = run(cfg, log=_log(args))
-    t = _hotspot_table(res.hotspots, cfg.params.tick_s)
+    from .report import name_map
+    t = _hotspot_table(res.hotspots, cfg.params.tick_s, name_map(cfg.report.names, res.network))
     if t.height == 0:
-        print("no hotspot")
+        print(f"no hotspot ({res.hotspots_status})")
         return 0
     with pl.Config(tbl_rows=200, tbl_cols=20, fmt_str_lengths=60, tbl_width_chars=200):
         print(t)
@@ -95,7 +100,9 @@ def cmd_tune(args) -> int:
         tr.table.write_parquet(out / "tune.parquet")
     sr = None
     if args.sensitivity:
-        sr = tune.sensitivity(pre.placed, pre.segment_fn(), tr.recommended, B=args.bootstrap, **kw)
+        kw["params"] = replace(cfg.params, segment_m=float(tr.recommended))
+        sr = tune.sensitivity(pre.placed, pre.segment_fn(), tr.recommended, B=args.bootstrap,
+                              coverage_fn=pre.coverage_fn(), **kw)
         with pl.Config(tbl_rows=50, tbl_cols=20, tbl_width_chars=200, float_precision=3):
             print(sr.summary)
             print(sr.hotspots)
@@ -152,6 +159,70 @@ def cmd_inspect(args) -> int:
         p = pre.passages
         print(f"\npassages: {p['n'].sum():,.0f} link passages"
               + (f" (feed {p['n_feed'].sum():,.0f}, events {p['n_events'].sum():,.0f})" if p["n_events"].null_count() < p.height else ""))
+    if not args.no_agreement:
+        from .metrics import reference_agreement
+        from .pipeline import run
+        res = run(cfg, prepared=pre, hotspot_kw={"B": min(cfg.params.bootstrap, 100)})
+        ag = reference_agreement(res.analysis, res.hotspots)
+        print(f"\nreferences: evening {cfg.params.reference_hours[0]}-{cfg.params.reference_hours[1]} h vs line level "
+              "(median running obs / passage / m, 6-23 h); day-band excess profiles:")
+        with pl.Config(tbl_rows=10, tbl_cols=20, tbl_width_chars=200, float_precision=3):
+            print(ag)
+    return 0
+
+
+def _period(s: str) -> str:
+    a, b = s.split("..")
+    import datetime as dt
+    dt.date.fromisoformat(a), dt.date.fromisoformat(b)
+    return s
+
+
+def cmd_compare(args) -> int:
+    from .compare import run_compare
+    cfg = _cfg(args)
+    res = run_compare(cfg, args.a, args.b, B=args.bootstrap, log=_log(args))
+    paths = res.save(args.out, html=not args.no_html, lang=args.lang, title=args.title, source=args.source)
+    st = res.comparison.stretches
+    if st.height:
+        from .report import name_map
+        nm = name_map(cfg.report.names, res.prepared.network)
+        fix = lambda c: pl.col(c).replace(nm) if nm else pl.col(c)  # noqa: E731
+        t = st.select("direction_id", "rank",
+                      pl.concat_str([fix("from_stop_name"), pl.lit(" -> "), fix("to_stop_name")]).alias("stretch"),
+                      pl.col("x0_m").round(0), pl.col("x1_m").round(0), "zone",
+                      pl.col("hours").cast(pl.List(pl.Utf8)).list.join(",").alias("hours"),
+                      pl.col("delta_s").round(0).alias("delta_s_per_veh"),
+                      (pl.col("veh_time_s_per_day") / 60).round(1).alias("veh_min_per_day"),
+                      pl.col("passages_per_h_a").round(1).alias("veh_h_a"), pl.col("passages_per_h_b").round(1).alias("veh_h_b"),
+                      "frequency_changed")
+        with pl.Config(tbl_rows=60, tbl_cols=20, fmt_str_lengths=60, tbl_width_chars=220):
+            print(t.head(args.top) if args.top else t)
+    else:
+        print("no significant change")
+    for p in paths.values():
+        print(p)
+    return 0
+
+
+def cmd_sensitivity_table(args) -> int:
+    from . import tune
+    configs = [Config.from_toml(c) for c in args.configs]
+    if args.segment_m:
+        configs = [replace(c, params=replace(c.params, segment_m=float(args.segment_m))) for c in configs]
+    table, md = tune.sensitivity_table(configs, B=args.bootstrap, dates=args.dates, log=_log(args))
+    with pl.Config(tbl_rows=200, tbl_cols=20, tbl_width_chars=200, float_precision=3):
+        print(table)
+    if args.out:
+        out = Path(args.out)
+        out.mkdir(parents=True, exist_ok=True)
+        table.write_parquet(out / "sensitivity_table.parquet")
+        table.write_csv(out / "sensitivity_table.csv")
+        (out / "sensitivity_table.md").write_text(md)
+        print(out / "sensitivity_table.parquet")
+        print(out / "sensitivity_table.md")
+    else:
+        print(md)
     return 0
 
 
@@ -189,10 +260,33 @@ def main(argv=None) -> int:
     p.add_argument("-o", "--out", help="write .parquet, .csv or .geojson")
     p.set_defaults(fn=cmd_hotspots)
 
-    p = sub.add_parser("inspect", help="coverage, GTFS versions, placement / drop counts")
+    p = sub.add_parser("inspect", help="coverage, GTFS versions, placement / drop counts, reference agreement")
     common(p)
+    p.add_argument("--no-agreement", action="store_true", help="skip the analysis (evening vs line-level reference)")
     p.set_defaults(fn=cmd_inspect)
 
+    p = sub.add_parser("compare", help="compare two periods of the same line: parquet + JSON + HTML")
+    common(p)
+    p.add_argument("--a", required=True, type=_period, help="period A, before (YYYY-MM-DD..YYYY-MM-DD)")
+    p.add_argument("--b", required=True, type=_period, help="period B, after (YYYY-MM-DD..YYYY-MM-DD)")
+    p.add_argument("-o", "--out", default="out_compare", help="output directory (default: out_compare)")
+    p.add_argument("--bootstrap", type=int, help="bootstrap draws (default: params.bootstrap)")
+    p.add_argument("--top", type=int, default=20, help="changes printed (0: all)")
+    p.add_argument("--lang", default="fr", choices=["fr", "en"])
+    p.add_argument("--title")
+    p.add_argument("--source", help="data credit shown in the page footer")
+    p.add_argument("--no-html", action="store_true")
+    p.set_defaults(fn=cmd_compare)
+
+    p = sub.add_parser("sensitivity-table", help="sensitivity suite over several configs -> one tidy table + markdown")
+    p.add_argument("configs", nargs="+", help="TOML configurations (one per line)")
+    p.add_argument("--segment-m", type=float, help="baseline segment length (default: each config's params.segment_m)")
+    p.add_argument("--dates", help="override select.dates for every config")
+    p.add_argument("--bootstrap", type=int, default=100)
+    p.add_argument("-o", "--out", help="directory for sensitivity_table.{parquet,csv,md}")
+    p.add_argument("-q", "--quiet", action="store_true")
+    p.set_defaults(fn=cmd_sensitivity_table)
+
     args = ap.parse_args(argv)
     return args.fn(args)