microsegments 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (96) hide show
  1. microsegments-0.1.0/.github/workflows/publish.yml +71 -0
  2. microsegments-0.1.0/.github/workflows/test.yml +28 -0
  3. microsegments-0.1.0/.gitignore +14 -0
  4. microsegments-0.1.0/LICENSE +21 -0
  5. microsegments-0.1.0/PKG-INFO +184 -0
  6. microsegments-0.1.0/README.md +146 -0
  7. microsegments-0.1.0/docs/report.jpg +0 -0
  8. microsegments-0.1.0/examples/gtfsrt_csv.toml +35 -0
  9. microsegments-0.1.0/examples/make_synth.py +29 -0
  10. microsegments-0.1.0/examples/stib55.toml +29 -0
  11. microsegments-0.1.0/examples/synth.toml +27 -0
  12. microsegments-0.1.0/pyproject.toml +62 -0
  13. microsegments-0.1.0/src/microsegments/__init__.py +42 -0
  14. microsegments-0.1.0/src/microsegments/_version.py +24 -0
  15. microsegments-0.1.0/src/microsegments/aggregate.py +101 -0
  16. microsegments-0.1.0/src/microsegments/cli.py +201 -0
  17. microsegments-0.1.0/src/microsegments/config.py +142 -0
  18. microsegments-0.1.0/src/microsegments/contract.py +42 -0
  19. microsegments-0.1.0/src/microsegments/hotspots.py +337 -0
  20. microsegments-0.1.0/src/microsegments/html/__init__.py +6 -0
  21. microsegments-0.1.0/src/microsegments/html/export.py +96 -0
  22. microsegments-0.1.0/src/microsegments/html/leaflet.min.css +2 -0
  23. microsegments-0.1.0/src/microsegments/html/template.html +863 -0
  24. microsegments-0.1.0/src/microsegments/io/__init__.py +9 -0
  25. microsegments-0.1.0/src/microsegments/io/coverage.py +114 -0
  26. microsegments-0.1.0/src/microsegments/io/events.py +98 -0
  27. microsegments-0.1.0/src/microsegments/io/gtfsrt.py +168 -0
  28. microsegments-0.1.0/src/microsegments/io/ids.py +34 -0
  29. microsegments-0.1.0/src/microsegments/io/stib.py +217 -0
  30. microsegments-0.1.0/src/microsegments/io/tabular.py +188 -0
  31. microsegments-0.1.0/src/microsegments/io/timeutil.py +84 -0
  32. microsegments-0.1.0/src/microsegments/locate/__init__.py +47 -0
  33. microsegments-0.1.0/src/microsegments/locate/common.py +149 -0
  34. microsegments-0.1.0/src/microsegments/locate/linear.py +317 -0
  35. microsegments-0.1.0/src/microsegments/locate/mapmatch.py +390 -0
  36. microsegments-0.1.0/src/microsegments/locate/resample.py +71 -0
  37. microsegments-0.1.0/src/microsegments/metrics.py +520 -0
  38. microsegments-0.1.0/src/microsegments/network/__init__.py +10 -0
  39. microsegments-0.1.0/src/microsegments/network/calendar.py +218 -0
  40. microsegments-0.1.0/src/microsegments/network/geometry.py +224 -0
  41. microsegments-0.1.0/src/microsegments/network/keys.py +97 -0
  42. microsegments-0.1.0/src/microsegments/network/patterns.py +328 -0
  43. microsegments-0.1.0/src/microsegments/pipeline.py +308 -0
  44. microsegments-0.1.0/src/microsegments/plot.py +476 -0
  45. microsegments-0.1.0/src/microsegments/py.typed +0 -0
  46. microsegments-0.1.0/src/microsegments/report.py +258 -0
  47. microsegments-0.1.0/src/microsegments/schema.py +257 -0
  48. microsegments-0.1.0/src/microsegments/segments.py +253 -0
  49. microsegments-0.1.0/src/microsegments/simulate.py +612 -0
  50. microsegments-0.1.0/src/microsegments/tracks.py +391 -0
  51. microsegments-0.1.0/src/microsegments/tune.py +518 -0
  52. microsegments-0.1.0/tests/fixtures/stib55/golden/buckets.parquet +0 -0
  53. microsegments-0.1.0/tests/fixtures/stib55/golden/drops.parquet +0 -0
  54. microsegments-0.1.0/tests/fixtures/stib55/golden/flen.parquet +0 -0
  55. microsegments-0.1.0/tests/fixtures/stib55/golden/obs_day.parquet +0 -0
  56. microsegments-0.1.0/tests/fixtures/stib55/golden/obs_dow.parquet +0 -0
  57. microsegments-0.1.0/tests/fixtures/stib55/golden/passages.parquet +0 -0
  58. microsegments-0.1.0/tests/fixtures/stib55/golden/summary.json +229 -0
  59. microsegments-0.1.0/tests/fixtures/stib55/golden/validation.parquet +0 -0
  60. microsegments-0.1.0/tests/fixtures/stib55/gtfs/agency.parquet +0 -0
  61. microsegments-0.1.0/tests/fixtures/stib55/gtfs/calendar.parquet +0 -0
  62. microsegments-0.1.0/tests/fixtures/stib55/gtfs/calendar_dates.parquet +0 -0
  63. microsegments-0.1.0/tests/fixtures/stib55/gtfs/routes.parquet +0 -0
  64. microsegments-0.1.0/tests/fixtures/stib55/gtfs/shapes.parquet +0 -0
  65. microsegments-0.1.0/tests/fixtures/stib55/gtfs/stop_times.parquet +0 -0
  66. microsegments-0.1.0/tests/fixtures/stib55/gtfs/stops.parquet +0 -0
  67. microsegments-0.1.0/tests/fixtures/stib55/gtfs/trips.parquet +0 -0
  68. microsegments-0.1.0/tests/fixtures/stib55/punctuality/route55.parquet +0 -0
  69. microsegments-0.1.0/tests/fixtures/stib55/speed55_ref.json +315 -0
  70. microsegments-0.1.0/tests/fixtures/stib55/vd55/20250318.parquet +0 -0
  71. microsegments-0.1.0/tests/fixtures/stib55/vd55/20250318.snaps.parquet +0 -0
  72. microsegments-0.1.0/tests/fixtures/stib55/vd55/20250319.parquet +0 -0
  73. microsegments-0.1.0/tests/fixtures/stib55/vd55/20250319.snaps.parquet +0 -0
  74. microsegments-0.1.0/tests/gtfs_fixtures.py +123 -0
  75. microsegments-0.1.0/tests/regression/make_golden.py +366 -0
  76. microsegments-0.1.0/tests/t4_fixtures.py +226 -0
  77. microsegments-0.1.0/tests/test_aggregate.py +104 -0
  78. microsegments-0.1.0/tests/test_coverage.py +129 -0
  79. microsegments-0.1.0/tests/test_hotspots.py +77 -0
  80. microsegments-0.1.0/tests/test_io_events.py +56 -0
  81. microsegments-0.1.0/tests/test_io_gtfsrt.py +50 -0
  82. microsegments-0.1.0/tests/test_io_stib.py +114 -0
  83. microsegments-0.1.0/tests/test_io_tabular.py +103 -0
  84. microsegments-0.1.0/tests/test_locate_linear.py +79 -0
  85. microsegments-0.1.0/tests/test_locate_mapmatch.py +81 -0
  86. microsegments-0.1.0/tests/test_metrics.py +155 -0
  87. microsegments-0.1.0/tests/test_network_geometry.py +62 -0
  88. microsegments-0.1.0/tests/test_network_patterns.py +209 -0
  89. microsegments-0.1.0/tests/test_pipeline_cli.py +69 -0
  90. microsegments-0.1.0/tests/test_regression_stib55.py +130 -0
  91. microsegments-0.1.0/tests/test_report_html.py +116 -0
  92. microsegments-0.1.0/tests/test_scaffold.py +25 -0
  93. microsegments-0.1.0/tests/test_segments.py +139 -0
  94. microsegments-0.1.0/tests/test_simulate.py +121 -0
  95. microsegments-0.1.0/tests/test_tracks.py +77 -0
  96. microsegments-0.1.0/tests/test_tune.py +69 -0
@@ -0,0 +1,71 @@
1
+ name: Publish to PyPI
2
+
3
+ on:
4
+ release:
5
+ types: [published]
6
+
7
+ permissions:
8
+ contents: write
9
+ id-token: write
10
+
11
+ jobs:
12
+ build:
13
+ name: Build distribution
14
+ runs-on: ubuntu-latest
15
+ steps:
16
+ - uses: actions/checkout@v4
17
+ - name: Set up Python
18
+ uses: actions/setup-python@v5
19
+ with:
20
+ python-version: "3.12"
21
+ - name: Install build
22
+ run: pip install build
23
+ - name: Build package
24
+ run: python -m build
25
+ - name: Upload dist artifacts
26
+ uses: actions/upload-artifact@v4
27
+ with:
28
+ name: python-package-distributions
29
+ path: dist/
30
+
31
+ publish-to-pypi:
32
+ name: Publish to PyPI
33
+ needs: build
34
+ runs-on: ubuntu-latest
35
+ environment:
36
+ name: pypi
37
+ url: https://pypi.org/p/microsegments
38
+ permissions:
39
+ id-token: write
40
+ steps:
41
+ - name: Download dist artifacts
42
+ uses: actions/download-artifact@v4
43
+ with:
44
+ name: python-package-distributions
45
+ path: dist/
46
+ - name: Publish to PyPI
47
+ uses: pypa/gh-action-pypi-publish@release/v1
48
+
49
+ github-release:
50
+ name: Sign and create GitHub Release
51
+ needs: publish-to-pypi
52
+ runs-on: ubuntu-latest
53
+ permissions:
54
+ contents: write
55
+ id-token: write
56
+ steps:
57
+ - name: Download dist artifacts
58
+ uses: actions/download-artifact@v4
59
+ with:
60
+ name: python-package-distributions
61
+ path: dist/
62
+ - name: Sign with Sigstore
63
+ uses: sigstore/gh-action-sigstore-python@v3.0.0
64
+ with:
65
+ inputs: >-
66
+ ./dist/*.tar.gz
67
+ ./dist/*.whl
68
+ - name: Upload to GitHub Release
69
+ env:
70
+ GITHUB_TOKEN: ${{ github.token }}
71
+ run: gh release upload '${{ github.ref_name }}' dist/** --repo '${{ github.repository }}' --clobber
@@ -0,0 +1,28 @@
1
+ name: Test
2
+
3
+ on:
4
+ push:
5
+ branches: [main]
6
+ pull_request:
7
+ branches: [main]
8
+
9
+ jobs:
10
+ test:
11
+ runs-on: ubuntu-latest
12
+ strategy:
13
+ fail-fast: false
14
+ matrix:
15
+ python-version: ["3.11", "3.12", "3.13"]
16
+ steps:
17
+ - uses: actions/checkout@v4
18
+ with:
19
+ fetch-depth: 0
20
+ - uses: actions/setup-python@v5
21
+ with:
22
+ python-version: ${{ matrix.python-version }}
23
+ - name: Install dependencies
24
+ run: |
25
+ python -m pip install --upgrade pip
26
+ pip install -e ".[dev]"
27
+ - name: Run tests
28
+ run: pytest
@@ -0,0 +1,14 @@
1
+ __pycache__/
2
+ *.py[cod]
3
+ *.egg-info/
4
+ dist/
5
+ build/
6
+ .venv/
7
+ .pytest_cache/
8
+ *.parquet
9
+ !tests/fixtures/**/*.parquet
10
+ .claude/
11
+ src/microsegments/_version.py
12
+ tests/regression/golden/
13
+ examples/data/
14
+ out/
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Gaspard Merten
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,184 @@
1
+ Metadata-Version: 2.5
2
+ Name: microsegments
3
+ Version: 0.1.0
4
+ Summary: Where do transit vehicles linger? Count vehicle position observations per micro-segment of a line, from GTFS + AVL / GTFS-RT positions.
5
+ Project-URL: Homepage, https://github.com/GaspardMerten/microsegments
6
+ Project-URL: Repository, https://github.com/GaspardMerten/microsegments
7
+ Project-URL: Issues, https://github.com/GaspardMerten/microsegments/issues
8
+ Author-email: Gaspard Merten <gaspard.mp.work@gmail.com>
9
+ License-Expression: MIT
10
+ License-File: LICENSE
11
+ Keywords: avl,bus,gtfs,gtfs-rt,hotspots,polars,tram,transit
12
+ Classifier: Development Status :: 3 - Alpha
13
+ Classifier: Intended Audience :: Science/Research
14
+ Classifier: License :: OSI Approved :: MIT License
15
+ Classifier: Operating System :: OS Independent
16
+ Classifier: Programming Language :: Python :: 3
17
+ Classifier: Programming Language :: Python :: 3.11
18
+ Classifier: Programming Language :: Python :: 3.12
19
+ Classifier: Programming Language :: Python :: 3.13
20
+ Classifier: Topic :: Scientific/Engineering :: GIS
21
+ Classifier: Typing :: Typed
22
+ Requires-Python: >=3.11
23
+ Requires-Dist: gtfs-parquet>=0.7
24
+ Requires-Dist: numpy>=1.26
25
+ Requires-Dist: polars>=1.0
26
+ Requires-Dist: shapely>=2.0
27
+ Provides-Extra: dev
28
+ Requires-Dist: hypothesis>=6.0; extra == 'dev'
29
+ Requires-Dist: matplotlib>=3.8; extra == 'dev'
30
+ Requires-Dist: pytest>=8.0; extra == 'dev'
31
+ Provides-Extra: gtfsrt
32
+ Requires-Dist: gtfs-realtime-bindings>=1.0; extra == 'gtfsrt'
33
+ Provides-Extra: mobilitytwin
34
+ Requires-Dist: requests>=2.31; extra == 'mobilitytwin'
35
+ Provides-Extra: plot
36
+ Requires-Dist: matplotlib>=3.8; extra == 'plot'
37
+ Description-Content-Type: text/markdown
38
+
39
+ # microsegments
40
+
41
+ Where do buses and trams linger? `microsegments` cuts each line into short stretches (30 m by default)
42
+ and counts how many times a vehicle position is observed in each one, per hour and per day of the week.
43
+ It works with GTFS plus any vehicle position feed: GTFS-RT VehiclePositions (lat/lon, as CSV, parquet or
44
+ `.pb`) or linear-referenced AVL (last stop + distance, like STIB).
45
+
46
+ ![Report page: hour player, map coloured by micro-segment, hotspots, hour x micro-segment matrix](docs/report.jpg)
47
+
48
+ ## Install
49
+
50
+ ```bash
51
+ pip install "microsegments[plot]" # plot extra = matplotlib
52
+ ```
53
+
54
+ Python ≥ 3.11. Core dependencies: polars, numpy, shapely, gtfs-parquet (no pandas).
55
+
56
+ ## Quick start (offline, synthetic data)
57
+
58
+ ```bash
59
+ python examples/make_synth.py # simulated tram line S1 -> examples/data/
60
+ microsegments run examples/synth.toml -o out/ # result.parquet, analysis.json, report.html, matrix_dir*.png
61
+ ```
62
+
63
+ ## Linear-referenced feeds (STIB vehicle-distance)
64
+
65
+ ```toml
66
+ [input]
67
+ paths = ["vd55/*.parquet"] # <YYYYMMDD>.parquet (ts, dir, point, dist) + <YYYYMMDD>.snaps.parquet
68
+ kind = "stib" # or "linear" for any "stop_id + dist_m" table (map columns below)
69
+ events = "punctuality/*.parquet" # optional stop events: passages from both sources, max taken
70
+
71
+ [input.linear]
72
+ id_normaliser = "leading_digits" # "05766" -> "5766"
73
+ feed_length = "auto" # rescale feed metres to GTFS link lengths
74
+
75
+ [gtfs]
76
+ path = "gtfs/" # zip, .txt directory or gtfs-parquet directory
77
+ # dated = "gtfs/{date}" # one feed per day instead
78
+
79
+ [select]
80
+ route = "55"
81
+ dates = "2025-02-17..2025-05-16"
82
+ weekdays = [0, 1, 2, 3, 4]
83
+ ```
84
+
85
+ See `examples/stib55.toml` (runs on the two-day test fixture).
86
+
87
+ ## GTFS-RT positions as CSV (lat/lon)
88
+
89
+ ```toml
90
+ [input]
91
+ paths = ["positions/*.csv"]
92
+ kind = "latlon"
93
+ ts_unit = "s" # "s" | "ms" | "us" | "iso"
94
+ [input.columns] # canonical name = your column
95
+ ts = "timestamp"
96
+ route_id = "route_id"
97
+ vehicle_id = "vehicle_id"
98
+ lat = "latitude"
99
+ lon = "longitude"
100
+ trip_id = "trip_id" # optional, helps map matching
101
+ ```
102
+
103
+ Per-vehicle fixes are thinned to one per `tick_s` (20 s) so counts stay comparable with a snapshot feed.
104
+ See `examples/gtfsrt_csv.toml` for every option.
105
+
106
+ ## Python
107
+
108
+ ```python
109
+ import microsegments as ms
110
+
111
+ res = ms.run(ms.Config.from_toml("ms.toml")) # obs -> coverage -> network -> place -> passages
112
+ # -> segments -> count -> analyse -> hotspots
113
+ res.result # polars: one row per segment x hour (bands as hour -1 am, -2 pm, -3 day, -4 evening)
114
+ res.hotspots # ranked stretches
115
+ ms.export(res, "report.html", lang="fr") # standalone page (fr / en)
116
+ ms.plot.matrix(res, direction_id=0, metric="obs_per_h", days=[0, 1, 2, 3, 4], stops=False)
117
+ ms.plot.map(res, 0, "pm", metric="excess_per_passage")
118
+ contract = res.contract() # the page JSON (also the platform's /api/analysis)
119
+
120
+ pre = ms.prepare(cfg) # reuse placement to tune the segment length
121
+ tr = ms.tune.segment_length(pre.placed, pre.segment_fn(), coverage=pre.coverage, passages=pre.passages,
122
+ pattern_days=pre.network.pattern_days)
123
+ ms.plot.tune_curves(tr)
124
+ ```
125
+
126
+ Stages are usable on their own: `read_observations`, `build_network`, `segment`, `count`, `analyse`,
127
+ `hotspots`, `simulate`.
128
+
129
+ ## CLI
130
+
131
+ ```bash
132
+ microsegments run CONFIG -o OUT [--lang en] [--segment-m 20] [--dates A..B] [--source "credit"]
133
+ microsegments tune CONFIG [--lengths 10,20,30,50] [--sensitivity] [-o OUT]
134
+ microsegments hotspots CONFIG [-o hotspots.geojson|.csv|.parquet]
135
+ microsegments inspect CONFIG # coverage per day, GTFS versions, placement / drop counts
136
+ ```
137
+
138
+ ## What the numbers mean
139
+
140
+ Every poll (about every 20 s) puts each vehicle in one micro-segment: one **observation**. One observation
141
+ is therefore about 20 s of presence.
142
+
143
+ | Metric | Definition | Read as |
144
+ |---|---|---|
145
+ | `obs_per_h` (default) | Σ observations / Σ covered hours | vehicles seen in the segment per hour of polling; `/len × 10` for per 10 m |
146
+ | `obs_per_passage` | Σ observations / Σ passages | × 20 s ≈ time each vehicle spends there |
147
+ | `excess_per_passage` | obs per passage − the same at 20–23 h, same days | extra observations (≈ seconds / 20) lost per vehicle vs free-flowing evening |
148
+ | `excess_obs_per_h`, `log2_ratio` | see `metrics.py` | |
149
+
150
+ All values are ratios of sums over the selected days (optionally post-stratified by weekday). Stop zones
151
+ (30 m before to 60 m after each stop) can be masked to bring out signals and junctions. Colour scales are
152
+ fixed per metric (p98 over 6–21 h), so hours and weekdays compare directly.
153
+
154
+ ## Missing data and GTFS versions
155
+
156
+ - **Hours** with < 80 % of polls, a frozen feed (> 5 min), the line absent, or abnormal vehicle counts are
157
+ dropped from numerator and denominator. Polling gaps are handled by the exposure (covered hours); counts
158
+ are never imputed.
159
+ - **Days** with more than 25 % of their 6–21 h dropped are excluded and listed with the reason
160
+ (`res.analysis.excluded_days`, coverage calendar in the page).
161
+ - **GTFS changes**: patterns are rebuilt per service date; `link_key` / `seg_key` are stable across
162
+ versions. Each segment is aggregated only over the days it belongs to the day's main pattern (the
163
+ reference too). The page shows the most frequent version; others are selectable. A single feed whose
164
+ calendar does not cover some dates lends them the patterns of the same weekday.
165
+
166
+ ## Choosing the segment length
167
+
168
+ `microsegments tune` scores L ∈ {10, 15, 20, 30, 40, 50, 75, 100} m (leave-one-day-out Poisson deviance,
169
+ split-half reliability, hotspot localisation spread and Jaccard) and picks the smallest L with reliability
170
+ ≥ 0.8, spread ≤ 30 m and deviance within one standard error of the minimum. 30 m is a good default for
171
+ 20 s polling at urban speeds; go shorter only with dense GTFS-RT fixes and many days. `--sensitivity`
172
+ re-runs phase offsets, gap caps, stop zones, references and passage sources.
173
+
174
+ ## Hotspots
175
+
176
+ Stretches where the excess over the evening has a 95 % bootstrap lower bound above 2 s per vehicle, is
177
+ positive on ≥ 60 % of days, survives Benjamini–Hochberg (q = 0.1) and lasts ≥ 2 hours or a peak band.
178
+ Adjacent bins merge, never across a stop / running border. Classes: *infrastructure* (all day),
179
+ *congestion* (peak only), *mixed*. Needs at least 5 included days. Export as GeoJSON with
180
+ `microsegments hotspots CONFIG -o hotspots.geojson`.
181
+
182
+ ## License
183
+
184
+ MIT. The bundled Leaflet CSS is BSD-2-Clause.
@@ -0,0 +1,146 @@
1
+ # microsegments
2
+
3
+ Where do buses and trams linger? `microsegments` cuts each line into short stretches (30 m by default)
4
+ and counts how many times a vehicle position is observed in each one, per hour and per day of the week.
5
+ It works with GTFS plus any vehicle position feed: GTFS-RT VehiclePositions (lat/lon, as CSV, parquet or
6
+ `.pb`) or linear-referenced AVL (last stop + distance, like STIB).
7
+
8
+ ![Report page: hour player, map coloured by micro-segment, hotspots, hour x micro-segment matrix](docs/report.jpg)
9
+
10
+ ## Install
11
+
12
+ ```bash
13
+ pip install "microsegments[plot]" # plot extra = matplotlib
14
+ ```
15
+
16
+ Python ≥ 3.11. Core dependencies: polars, numpy, shapely, gtfs-parquet (no pandas).
17
+
18
+ ## Quick start (offline, synthetic data)
19
+
20
+ ```bash
21
+ python examples/make_synth.py # simulated tram line S1 -> examples/data/
22
+ microsegments run examples/synth.toml -o out/ # result.parquet, analysis.json, report.html, matrix_dir*.png
23
+ ```
24
+
25
+ ## Linear-referenced feeds (STIB vehicle-distance)
26
+
27
+ ```toml
28
+ [input]
29
+ paths = ["vd55/*.parquet"] # <YYYYMMDD>.parquet (ts, dir, point, dist) + <YYYYMMDD>.snaps.parquet
30
+ kind = "stib" # or "linear" for any "stop_id + dist_m" table (map columns below)
31
+ events = "punctuality/*.parquet" # optional stop events: passages from both sources, max taken
32
+
33
+ [input.linear]
34
+ id_normaliser = "leading_digits" # "05766" -> "5766"
35
+ feed_length = "auto" # rescale feed metres to GTFS link lengths
36
+
37
+ [gtfs]
38
+ path = "gtfs/" # zip, .txt directory or gtfs-parquet directory
39
+ # dated = "gtfs/{date}" # one feed per day instead
40
+
41
+ [select]
42
+ route = "55"
43
+ dates = "2025-02-17..2025-05-16"
44
+ weekdays = [0, 1, 2, 3, 4]
45
+ ```
46
+
47
+ See `examples/stib55.toml` (runs on the two-day test fixture).
48
+
49
+ ## GTFS-RT positions as CSV (lat/lon)
50
+
51
+ ```toml
52
+ [input]
53
+ paths = ["positions/*.csv"]
54
+ kind = "latlon"
55
+ ts_unit = "s" # "s" | "ms" | "us" | "iso"
56
+ [input.columns] # canonical name = your column
57
+ ts = "timestamp"
58
+ route_id = "route_id"
59
+ vehicle_id = "vehicle_id"
60
+ lat = "latitude"
61
+ lon = "longitude"
62
+ trip_id = "trip_id" # optional, helps map matching
63
+ ```
64
+
65
+ Per-vehicle fixes are thinned to one per `tick_s` (20 s) so counts stay comparable with a snapshot feed.
66
+ See `examples/gtfsrt_csv.toml` for every option.
67
+
68
+ ## Python
69
+
70
+ ```python
71
+ import microsegments as ms
72
+
73
+ res = ms.run(ms.Config.from_toml("ms.toml")) # obs -> coverage -> network -> place -> passages
74
+ # -> segments -> count -> analyse -> hotspots
75
+ res.result # polars: one row per segment x hour (bands as hour -1 am, -2 pm, -3 day, -4 evening)
76
+ res.hotspots # ranked stretches
77
+ ms.export(res, "report.html", lang="fr") # standalone page (fr / en)
78
+ ms.plot.matrix(res, direction_id=0, metric="obs_per_h", days=[0, 1, 2, 3, 4], stops=False)
79
+ ms.plot.map(res, 0, "pm", metric="excess_per_passage")
80
+ contract = res.contract() # the page JSON (also the platform's /api/analysis)
81
+
82
+ pre = ms.prepare(cfg) # reuse placement to tune the segment length
83
+ tr = ms.tune.segment_length(pre.placed, pre.segment_fn(), coverage=pre.coverage, passages=pre.passages,
84
+ pattern_days=pre.network.pattern_days)
85
+ ms.plot.tune_curves(tr)
86
+ ```
87
+
88
+ Stages are usable on their own: `read_observations`, `build_network`, `segment`, `count`, `analyse`,
89
+ `hotspots`, `simulate`.
90
+
91
+ ## CLI
92
+
93
+ ```bash
94
+ microsegments run CONFIG -o OUT [--lang en] [--segment-m 20] [--dates A..B] [--source "credit"]
95
+ microsegments tune CONFIG [--lengths 10,20,30,50] [--sensitivity] [-o OUT]
96
+ microsegments hotspots CONFIG [-o hotspots.geojson|.csv|.parquet]
97
+ microsegments inspect CONFIG # coverage per day, GTFS versions, placement / drop counts
98
+ ```
99
+
100
+ ## What the numbers mean
101
+
102
+ Every poll (about every 20 s) puts each vehicle in one micro-segment: one **observation**. One observation
103
+ is therefore about 20 s of presence.
104
+
105
+ | Metric | Definition | Read as |
106
+ |---|---|---|
107
+ | `obs_per_h` (default) | Σ observations / Σ covered hours | vehicles seen in the segment per hour of polling; `/len × 10` for per 10 m |
108
+ | `obs_per_passage` | Σ observations / Σ passages | × 20 s ≈ time each vehicle spends there |
109
+ | `excess_per_passage` | obs per passage − the same at 20–23 h, same days | extra observations (≈ seconds / 20) lost per vehicle vs free-flowing evening |
110
+ | `excess_obs_per_h`, `log2_ratio` | see `metrics.py` | |
111
+
112
+ All values are ratios of sums over the selected days (optionally post-stratified by weekday). Stop zones
113
+ (30 m before to 60 m after each stop) can be masked to bring out signals and junctions. Colour scales are
114
+ fixed per metric (p98 over 6–21 h), so hours and weekdays compare directly.
115
+
116
+ ## Missing data and GTFS versions
117
+
118
+ - **Hours** with < 80 % of polls, a frozen feed (> 5 min), the line absent, or abnormal vehicle counts are
119
+ dropped from numerator and denominator. Polling gaps are handled by the exposure (covered hours); counts
120
+ are never imputed.
121
+ - **Days** with more than 25 % of their 6–21 h dropped are excluded and listed with the reason
122
+ (`res.analysis.excluded_days`, coverage calendar in the page).
123
+ - **GTFS changes**: patterns are rebuilt per service date; `link_key` / `seg_key` are stable across
124
+ versions. Each segment is aggregated only over the days it belongs to the day's main pattern (the
125
+ reference too). The page shows the most frequent version; others are selectable. A single feed whose
126
+ calendar does not cover some dates lends them the patterns of the same weekday.
127
+
128
+ ## Choosing the segment length
129
+
130
+ `microsegments tune` scores L ∈ {10, 15, 20, 30, 40, 50, 75, 100} m (leave-one-day-out Poisson deviance,
131
+ split-half reliability, hotspot localisation spread and Jaccard) and picks the smallest L with reliability
132
+ ≥ 0.8, spread ≤ 30 m and deviance within one standard error of the minimum. 30 m is a good default for
133
+ 20 s polling at urban speeds; go shorter only with dense GTFS-RT fixes and many days. `--sensitivity`
134
+ re-runs phase offsets, gap caps, stop zones, references and passage sources.
135
+
136
+ ## Hotspots
137
+
138
+ Stretches where the excess over the evening has a 95 % bootstrap lower bound above 2 s per vehicle, is
139
+ positive on ≥ 60 % of days, survives Benjamini–Hochberg (q = 0.1) and lasts ≥ 2 hours or a peak band.
140
+ Adjacent bins merge, never across a stop / running border. Classes: *infrastructure* (all day),
141
+ *congestion* (peak only), *mixed*. Needs at least 5 included days. Export as GeoJSON with
142
+ `microsegments hotspots CONFIG -o hotspots.geojson`.
143
+
144
+ ## License
145
+
146
+ MIT. The bundled Leaflet CSS is BSD-2-Clause.
Binary file
@@ -0,0 +1,35 @@
1
+ # GTFS-RT VehiclePositions flattened to CSV (one row per vehicle fix), matched on the GTFS shapes.
2
+ # microsegments run examples/gtfsrt_csv.toml -o out/bus
3
+
4
+ [input]
5
+ paths = ["positions/*.csv"] # or *.parquet; GTFS-RT .pb files also work (format = "gtfsrt_pb")
6
+ kind = "latlon"
7
+ format = "auto"
8
+ timezone = "Europe/Brussels"
9
+ service_day_start = "04:00" # 01:30 belongs to the previous service day
10
+ ts_unit = "s" # "s" | "ms" | "us" | "iso"
11
+
12
+ # canonical name = column in your file (only those that differ; unmapped well-known names are detected)
13
+ [input.columns]
14
+ ts = "timestamp" # fix time (vehicle.timestamp), epoch seconds here
15
+ route_id = "route_id" # must match the GTFS route_short_name (or route_id, see gtfs.route_key)
16
+ vehicle_id = "vehicle_id" # per-vehicle feed: fixes are thinned to one per tick_s
17
+ trip_id = "trip_id" # optional, helps map matching
18
+ direction_id = "direction_id" # optional
19
+ lat = "latitude"
20
+ lon = "longitude"
21
+ bearing = "bearing" # optional, degrees
22
+
23
+ [gtfs]
24
+ path = "gtfs.zip"
25
+ route_key = "route_short_name"
26
+
27
+ [select]
28
+ route = "71"
29
+ dates = "2025-02-17..2025-05-16"
30
+ weekdays = [0, 1, 2, 3, 4]
31
+
32
+ [params]
33
+ segment_m = 30
34
+ tick_s = 20 # nominal interval: counts stay comparable with a 20 s snapshot feed
35
+ max_lateral_m = 40 # fixes farther from the shape are dropped
@@ -0,0 +1,29 @@
1
+ """Generate the synthetic example data (offline) into examples/data/:
2
+
3
+ python examples/make_synth.py
4
+ microsegments run examples/synth.toml -o out/
5
+
6
+ A simulated tram line "S1" (two directions, 8 links), 3 weeks of STIB-like positions every ~20 s
7
+ with outages and a frozen feed, a GTFS change after two thirds of the period (the last two stops are cut), and three
8
+ injected hotspots (a signal, a peak-hour queue, an evening holding stop). See microsegments.simulate.
9
+ """
10
+ from __future__ import annotations
11
+
12
+ import sys
13
+ from pathlib import Path
14
+
15
+ from microsegments.simulate import simulate
16
+
17
+ HERE = Path(__file__).resolve().parent
18
+
19
+
20
+ def main(out: Path = HERE / "data", days: int = 21, seed: int = 0) -> Path:
21
+ sim = simulate(seed=seed, days=days, start="2025-03-03", mode="stib", out_dir=out / "gtfs", change_day=days * 2 // 3)
22
+ sim.write_stib(out / "vd")
23
+ sim.stop_events.write_parquet(out / "events.parquet")
24
+ print(f"{len(sim.observations):,} positions, {len(sim.dates)} days -> {out}")
25
+ return out
26
+
27
+
28
+ if __name__ == "__main__":
29
+ main(Path(sys.argv[1]) if len(sys.argv) > 1 else HERE / "data")
@@ -0,0 +1,29 @@
1
+ # STIB tram 55, two days of vehicle-distance positions (the test fixture), linear referencing.
2
+ # microsegments run examples/stib55.toml -o out/stib55
3
+ # Real use: point `paths` at the compact daily parquet files of a longer period (MobilityTwin
4
+ # stib/vehicle-distance), and `gtfs.dated` at one GTFS snapshot per day.
5
+
6
+ [input]
7
+ paths = ["../tests/fixtures/stib55/vd55/*.parquet"] # <YYYYMMDD>.parquet (ts, dir, point, dist) + .snaps.parquet
8
+ kind = "stib"
9
+ events = "../tests/fixtures/stib55/punctuality/route55.parquet" # STIB punctuality: passages from stop events
10
+
11
+ [input.linear]
12
+ direction_kind = "terminus_stop" # STIB directionId = destination stop code
13
+ id_normaliser = "leading_digits"
14
+ feed_length = "auto" # rescale feed metres to the GTFS link length (p99.5 per link)
15
+
16
+ [gtfs]
17
+ path = "../tests/fixtures/stib55/gtfs" # gtfs-parquet directory
18
+ # dated = "gtfs/{date}" # or one feed per day
19
+
20
+ [select]
21
+ route = "55"
22
+ dates = "2025-03-18..2025-03-19"
23
+ weekdays = [0, 1, 2, 3, 4]
24
+
25
+ [params]
26
+ segment_m = 30
27
+
28
+ [quality]
29
+ min_days = 2 # only two days in the fixture
@@ -0,0 +1,27 @@
1
+ # Synthetic tram line S1 (generate the data first: python examples/make_synth.py)
2
+ # microsegments run examples/synth.toml -o out/
3
+
4
+ [input]
5
+ paths = ["data/vd/*.parquet"] # compact STIB-like files: <YYYYMMDD>.parquet + .snaps.parquet
6
+ kind = "stib" # linear referencing: last stop passed + metres since
7
+ timezone = "Europe/Brussels"
8
+ events = "data/events.parquet" # stop events (arrival / departure per trip and stop), optional
9
+
10
+ [input.linear]
11
+ id_normaliser = "leading_digits" # "01003" -> "1003"
12
+
13
+ [gtfs]
14
+ path = "data/gtfs" # GTFS directory (.txt), zip or gtfs-parquet directory
15
+ route_key = "route_short_name"
16
+
17
+ [select]
18
+ route = "S1"
19
+ dates = "2025-03-03..2025-03-23"
20
+ weekdays = [0, 1, 2, 3, 4]
21
+ hours = [5, 24]
22
+
23
+ [params]
24
+ segment_m = 30
25
+ stop_zone = [30, 60] # metres before / after a stop masked as "stop zone"
26
+ reference_hours = [20, 23]
27
+ bootstrap = 200
@@ -0,0 +1,62 @@
1
+ [project]
2
+ name = "microsegments"
3
+ dynamic = ["version"]
4
+ description = "Where do transit vehicles linger? Count vehicle position observations per micro-segment of a line, from GTFS + AVL / GTFS-RT positions."
5
+ readme = "README.md"
6
+ license = "MIT"
7
+ requires-python = ">=3.11"
8
+ authors = [
9
+ { name = "Gaspard Merten", email = "gaspard.mp.work@gmail.com" },
10
+ ]
11
+ keywords = ["gtfs", "gtfs-rt", "avl", "transit", "bus", "tram", "hotspots", "polars"]
12
+ classifiers = [
13
+ "Development Status :: 3 - Alpha",
14
+ "Intended Audience :: Science/Research",
15
+ "License :: OSI Approved :: MIT License",
16
+ "Operating System :: OS Independent",
17
+ "Programming Language :: Python :: 3",
18
+ "Programming Language :: Python :: 3.11",
19
+ "Programming Language :: Python :: 3.12",
20
+ "Programming Language :: Python :: 3.13",
21
+ "Topic :: Scientific/Engineering :: GIS",
22
+ "Typing :: Typed",
23
+ ]
24
+ dependencies = [
25
+ "polars>=1.0",
26
+ "numpy>=1.26",
27
+ "shapely>=2.0",
28
+ "gtfs-parquet>=0.7",
29
+ ]
30
+
31
+ [project.optional-dependencies]
32
+ plot = ["matplotlib>=3.8"]
33
+ mobilitytwin = ["requests>=2.31"]
34
+ gtfsrt = ["gtfs-realtime-bindings>=1.0"]
35
+ dev = ["pytest>=8.0", "hypothesis>=6.0", "matplotlib>=3.8"]
36
+
37
+ [project.scripts]
38
+ microsegments = "microsegments.cli:main"
39
+
40
+ [project.urls]
41
+ Homepage = "https://github.com/GaspardMerten/microsegments"
42
+ Repository = "https://github.com/GaspardMerten/microsegments"
43
+ Issues = "https://github.com/GaspardMerten/microsegments/issues"
44
+
45
+ [build-system]
46
+ requires = ["hatchling", "hatch-vcs"]
47
+ build-backend = "hatchling.build"
48
+
49
+ [tool.hatch.version]
50
+ source = "vcs"
51
+ fallback-version = "0.0.0"
52
+
53
+ [tool.hatch.build.hooks.vcs]
54
+ version-file = "src/microsegments/_version.py"
55
+
56
+ [tool.hatch.build.targets.wheel]
57
+ packages = ["src/microsegments"]
58
+
59
+ [tool.pytest.ini_options]
60
+ testpaths = ["tests"]
61
+ markers = ["regression: needs the local BrusselsTransitStuck prototype data (MS_PROTOTYPE_DATA)"]
62
+ addopts = "-m 'not regression'"