microsegments 0.1.1__tar.gz → 0.2.0__tar.gz

{microsegments-0.1.1 → microsegments-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: microsegments
-Version: 0.1.1
+Version: 0.2.0
 Summary: Where do transit vehicles linger? Count vehicle position observations per micro-segment of a line, from GTFS + AVL / GTFS-RT positions.
 Project-URL: Homepage, https://github.com/GaspardMerten/microsegments
 Project-URL: Repository, https://github.com/GaspardMerten/microsegments
@@ -132,7 +132,9 @@ Stages are usable on their own: `read_observations`, `build_network`, `segment`,
 microsegments run CONFIG -o OUT [--lang en] [--segment-m 20] [--dates A..B] [--source "credit"]
 microsegments tune CONFIG [--lengths 10,20,30,50] [--sensitivity] [-o OUT]
 microsegments hotspots CONFIG [-o hotspots.geojson|.csv|.parquet]
-microsegments inspect CONFIG        # coverage per day, GTFS versions, placement / drop counts
+microsegments inspect CONFIG        # coverage per day, GTFS versions, placement, evening vs line reference
+microsegments compare CONFIG --a 2025-02-17..2025-03-31 --b 2025-04-01..2025-05-15 -o OUT
+microsegments sensitivity-table CONFIG [CONFIG ...] -o OUT    # one tidy table + markdown over several lines
 ```
 
 ## What the numbers mean
@@ -169,7 +171,8 @@ fixed per metric (p98 over 6–21 h), so hours and weekdays compare directly.
 split-half reliability, hotspot localisation spread and Jaccard) and picks the smallest L with reliability
 ≥ 0.8, spread ≤ 30 m and deviance within one standard error of the minimum. 30 m is a good default for
 20 s polling at urban speeds; go shorter only with dense GTFS-RT fixes and many days. `--sensitivity`
-re-runs phase offsets, gap caps, stop zones, references and passage sources.
+re-runs phase offsets, gap caps (30, 40, 60, ∞ s), stop zones, references, passage sources and segment
+lengths 15 / 30 / 60 m around the baseline.
 
 ## Hotspots
 
@@ -179,6 +182,15 @@ Adjacent bins merge, never across a stop / running border. Classes: *infrastruct
 *congestion* (peak only), *mixed*. Needs at least 5 included days. Export as GeoJSON with
 `microsegments hotspots CONFIG -o hotspots.geojson`.
 
+## Comparing two periods
+
+`microsegments compare` (or `ms.compare(analysis_a, analysis_b)`) compares observations per passage
+segment by segment and hour by hour, each period with its own days and references, with a day bootstrap
+of the difference. Segments are matched by key (or by stop pair when a new GTFS version moved the shape a
+little); the others are listed and left out. Adjacent significant segments form stretches, ranked by the
+change in vehicle time per day; stretches whose frequency changed by more than 20 % are flagged, since a
+timetable change can explain part of the difference. The HTML page gets a "compare two periods" view.
+
 ## License
 
 MIT. The bundled Leaflet CSS is BSD-2-Clause.

{microsegments-0.1.1 → microsegments-0.2.0}/README.md

@@ -94,7 +94,9 @@ Stages are usable on their own: `read_observations`, `build_network`, `segment`,
 microsegments run CONFIG -o OUT [--lang en] [--segment-m 20] [--dates A..B] [--source "credit"]
 microsegments tune CONFIG [--lengths 10,20,30,50] [--sensitivity] [-o OUT]
 microsegments hotspots CONFIG [-o hotspots.geojson|.csv|.parquet]
-microsegments inspect CONFIG        # coverage per day, GTFS versions, placement / drop counts
+microsegments inspect CONFIG        # coverage per day, GTFS versions, placement, evening vs line reference
+microsegments compare CONFIG --a 2025-02-17..2025-03-31 --b 2025-04-01..2025-05-15 -o OUT
+microsegments sensitivity-table CONFIG [CONFIG ...] -o OUT    # one tidy table + markdown over several lines
 ```
 
 ## What the numbers mean
@@ -131,7 +133,8 @@ fixed per metric (p98 over 6–21 h), so hours and weekdays compare directly.
 split-half reliability, hotspot localisation spread and Jaccard) and picks the smallest L with reliability
 ≥ 0.8, spread ≤ 30 m and deviance within one standard error of the minimum. 30 m is a good default for
 20 s polling at urban speeds; go shorter only with dense GTFS-RT fixes and many days. `--sensitivity`
-re-runs phase offsets, gap caps, stop zones, references and passage sources.
+re-runs phase offsets, gap caps (30, 40, 60, ∞ s), stop zones, references, passage sources and segment
+lengths 15 / 30 / 60 m around the baseline.
 
 ## Hotspots
 
@@ -141,6 +144,15 @@ Adjacent bins merge, never across a stop / running border. Classes: *infrastruct
 *congestion* (peak only), *mixed*. Needs at least 5 included days. Export as GeoJSON with
 `microsegments hotspots CONFIG -o hotspots.geojson`.
 
+## Comparing two periods
+
+`microsegments compare` (or `ms.compare(analysis_a, analysis_b)`) compares observations per passage
+segment by segment and hour by hour, each period with its own days and references, with a day bootstrap
+of the difference. Segments are matched by key (or by stop pair when a new GTFS version moved the shape a
+little); the others are listed and left out. Adjacent significant segments form stretches, ranked by the
+change in vehicle time per day; stretches whose frequency changed by more than 20 % are flagged, since a
+timetable change can explain part of the difference. The HTML page gets a "compare two periods" view.
+
 ## License
 
 MIT. The bundled Leaflet CSS is BSD-2-Clause.

microsegments-0.2.0/examples/stib1_platform.toml

@@ -0,0 +1,23 @@
+# LOCAL EXAMPLE (not shipped data): STIB line 1 from the StibMicrosegments platform's local ingest
+# (raw/vd/date=.../{vd,snaps}.parquet with a line column, raw/punctuality, gtfs/index.parquet + gtfs/feeds/<sha>).
+#   microsegments sensitivity-table examples/stib55_full.toml examples/stib71_platform.toml examples/stib1_platform.toml -o out/sens
+
+[input]
+paths = ["/home/gaspard/Documents/Dev/CoDE/StibMicrosegments/data/ms/raw/vd/date=2025-0[2-5]-*/*.parquet"]
+kind = "stib"
+events = "/home/gaspard/Documents/Dev/CoDE/StibMicrosegments/data/ms/raw/punctuality/date=2025-0[2-5]-*/p.parquet"
+
+[gtfs]
+index = "/home/gaspard/Documents/Dev/CoDE/StibMicrosegments/data/ms/gtfs/index.parquet"
+
+[select]
+route = "1"
+dates = "2025-03-03..2025-03-28"
+weekdays = [0, 1, 2, 3, 4]
+
+[params]
+segment_m = 30
+bootstrap = 200
+
+[report]
+names = "title"

microsegments-0.2.0/examples/stib71_platform.toml

@@ -0,0 +1,23 @@
+# LOCAL EXAMPLE (not shipped data): STIB line 71 from the StibMicrosegments platform's local ingest
+# (raw/vd/date=.../{vd,snaps}.parquet with a line column, raw/punctuality, gtfs/index.parquet + gtfs/feeds/<sha>).
+#   microsegments sensitivity-table examples/stib55_full.toml examples/stib71_platform.toml examples/stib1_platform.toml -o out/sens
+
+[input]
+paths = ["/home/gaspard/Documents/Dev/CoDE/StibMicrosegments/data/ms/raw/vd/date=2025-0[2-5]-*/*.parquet"]
+kind = "stib"
+events = "/home/gaspard/Documents/Dev/CoDE/StibMicrosegments/data/ms/raw/punctuality/date=2025-0[2-5]-*/p.parquet"
+
+[gtfs]
+index = "/home/gaspard/Documents/Dev/CoDE/StibMicrosegments/data/ms/gtfs/index.parquet"
+
+[select]
+route = "71"
+dates = "2025-03-03..2025-03-28"
+weekdays = [0, 1, 2, 3, 4]
+
+[params]
+segment_m = 30
+bootstrap = 200
+
+[report]
+names = "title"

{microsegments-0.1.1 → microsegments-0.2.0}/src/microsegments/__init__.py

@@ -15,6 +15,7 @@ except ImportError:  # pragma: no cover
     __version__ = "0.0.0"
 
 from .aggregate import count
+from .compare import Comparison, compare, run_compare
 from .config import Config
 from .hotspots import hotspots
 from .html import export
@@ -28,8 +29,8 @@ from .segments import segment
 from .simulate import simulate
 from . import tune
 
-__all__ = ["Analysis", "Config", "Flag", "RunResult", "__version__", "analyse", "build_network", "count", "export",
-           "hotspots", "plot", "prepare", "read_observations", "run", "segment", "simulate", "to_contract", "tune"]
+__all__ = ["Analysis", "Comparison", "Config", "Flag", "RunResult", "__version__", "analyse", "build_network", "compare", "count", "export",
+           "hotspots", "plot", "prepare", "read_observations", "run", "run_compare", "segment", "simulate", "to_contract", "tune"]
 
 
 def __getattr__(name: str):

{microsegments-0.1.1 → microsegments-0.2.0}/src/microsegments/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.1.1'
-__version_tuple__ = version_tuple = (0, 1, 1)
+__version__ = version = '0.2.0'
+__version_tuple__ = version_tuple = (0, 2, 0)
 
 __commit_id__ = commit_id = None

{microsegments-0.1.1 → microsegments-0.2.0}/src/microsegments/cli.py

@@ -1,4 +1,5 @@
-"""Command line: ``microsegments run | tune | hotspots | inspect CONFIG``."""
+"""Command line: ``microsegments run | tune | hotspots | inspect | compare CONFIG``, ``microsegments
+sensitivity-table CONFIG...``."""
 from __future__ import annotations
 
 import argparse
@@ -99,7 +100,9 @@ def cmd_tune(args) -> int:
         tr.table.write_parquet(out / "tune.parquet")
     sr = None
     if args.sensitivity:
-        sr = tune.sensitivity(pre.placed, pre.segment_fn(), tr.recommended, B=args.bootstrap, **kw)
+        kw["params"] = replace(cfg.params, segment_m=float(tr.recommended))
+        sr = tune.sensitivity(pre.placed, pre.segment_fn(), tr.recommended, B=args.bootstrap,
+                              coverage_fn=pre.coverage_fn(), **kw)
         with pl.Config(tbl_rows=50, tbl_cols=20, tbl_width_chars=200, float_precision=3):
             print(sr.summary)
             print(sr.hotspots)
@@ -156,6 +159,70 @@ def cmd_inspect(args) -> int:
         p = pre.passages
         print(f"\npassages: {p['n'].sum():,.0f} link passages"
               + (f" (feed {p['n_feed'].sum():,.0f}, events {p['n_events'].sum():,.0f})" if p["n_events"].null_count() < p.height else ""))
+    if not args.no_agreement:
+        from .metrics import reference_agreement
+        from .pipeline import run
+        res = run(cfg, prepared=pre, hotspot_kw={"B": min(cfg.params.bootstrap, 100)})
+        ag = reference_agreement(res.analysis, res.hotspots)
+        print(f"\nreferences: evening {cfg.params.reference_hours[0]}-{cfg.params.reference_hours[1]} h vs line level "
+              "(median running obs / passage / m, 6-23 h); day-band excess profiles:")
+        with pl.Config(tbl_rows=10, tbl_cols=20, tbl_width_chars=200, float_precision=3):
+            print(ag)
+    return 0
+
+
+def _period(s: str) -> str:
+    a, b = s.split("..")
+    import datetime as dt
+    dt.date.fromisoformat(a), dt.date.fromisoformat(b)
+    return s
+
+
+def cmd_compare(args) -> int:
+    from .compare import run_compare
+    cfg = _cfg(args)
+    res = run_compare(cfg, args.a, args.b, B=args.bootstrap, log=_log(args))
+    paths = res.save(args.out, html=not args.no_html, lang=args.lang, title=args.title, source=args.source)
+    st = res.comparison.stretches
+    if st.height:
+        from .report import name_map
+        nm = name_map(cfg.report.names, res.prepared.network)
+        fix = lambda c: pl.col(c).replace(nm) if nm else pl.col(c)  # noqa: E731
+        t = st.select("direction_id", "rank",
+                      pl.concat_str([fix("from_stop_name"), pl.lit(" -> "), fix("to_stop_name")]).alias("stretch"),
+                      pl.col("x0_m").round(0), pl.col("x1_m").round(0), "zone",
+                      pl.col("hours").cast(pl.List(pl.Utf8)).list.join(",").alias("hours"),
+                      pl.col("delta_s").round(0).alias("delta_s_per_veh"),
+                      (pl.col("veh_time_s_per_day") / 60).round(1).alias("veh_min_per_day"),
+                      pl.col("passages_per_h_a").round(1).alias("veh_h_a"), pl.col("passages_per_h_b").round(1).alias("veh_h_b"),
+                      "frequency_changed")
+        with pl.Config(tbl_rows=60, tbl_cols=20, fmt_str_lengths=60, tbl_width_chars=220):
+            print(t.head(args.top) if args.top else t)
+    else:
+        print("no significant change")
+    for p in paths.values():
+        print(p)
+    return 0
+
+
+def cmd_sensitivity_table(args) -> int:
+    from . import tune
+    configs = [Config.from_toml(c) for c in args.configs]
+    if args.segment_m:
+        configs = [replace(c, params=replace(c.params, segment_m=float(args.segment_m))) for c in configs]
+    table, md = tune.sensitivity_table(configs, B=args.bootstrap, dates=args.dates, log=_log(args))
+    with pl.Config(tbl_rows=200, tbl_cols=20, tbl_width_chars=200, float_precision=3):
+        print(table)
+    if args.out:
+        out = Path(args.out)
+        out.mkdir(parents=True, exist_ok=True)
+        table.write_parquet(out / "sensitivity_table.parquet")
+        table.write_csv(out / "sensitivity_table.csv")
+        (out / "sensitivity_table.md").write_text(md)
+        print(out / "sensitivity_table.parquet")
+        print(out / "sensitivity_table.md")
+    else:
+        print(md)
     return 0
 
 
@@ -193,10 +260,33 @@ def main(argv=None) -> int:
     p.add_argument("-o", "--out", help="write .parquet, .csv or .geojson")
     p.set_defaults(fn=cmd_hotspots)
 
-    p = sub.add_parser("inspect", help="coverage, GTFS versions, placement / drop counts")
+    p = sub.add_parser("inspect", help="coverage, GTFS versions, placement / drop counts, reference agreement")
     common(p)
+    p.add_argument("--no-agreement", action="store_true", help="skip the analysis (evening vs line-level reference)")
     p.set_defaults(fn=cmd_inspect)
 
+    p = sub.add_parser("compare", help="compare two periods of the same line: parquet + JSON + HTML")
+    common(p)
+    p.add_argument("--a", required=True, type=_period, help="period A, before (YYYY-MM-DD..YYYY-MM-DD)")
+    p.add_argument("--b", required=True, type=_period, help="period B, after (YYYY-MM-DD..YYYY-MM-DD)")
+    p.add_argument("-o", "--out", default="out_compare", help="output directory (default: out_compare)")
+    p.add_argument("--bootstrap", type=int, help="bootstrap draws (default: params.bootstrap)")
+    p.add_argument("--top", type=int, default=20, help="changes printed (0: all)")
+    p.add_argument("--lang", default="fr", choices=["fr", "en"])
+    p.add_argument("--title")
+    p.add_argument("--source", help="data credit shown in the page footer")
+    p.add_argument("--no-html", action="store_true")
+    p.set_defaults(fn=cmd_compare)
+
+    p = sub.add_parser("sensitivity-table", help="sensitivity suite over several configs -> one tidy table + markdown")
+    p.add_argument("configs", nargs="+", help="TOML configurations (one per line)")
+    p.add_argument("--segment-m", type=float, help="baseline segment length (default: each config's params.segment_m)")
+    p.add_argument("--dates", help="override select.dates for every config")
+    p.add_argument("--bootstrap", type=int, default=100)
+    p.add_argument("-o", "--out", help="directory for sensitivity_table.{parquet,csv,md}")
+    p.add_argument("-q", "--quiet", action="store_true")
+    p.set_defaults(fn=cmd_sensitivity_table)
+
     args = ap.parse_args(argv)
     return args.fn(args)