microsegments 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- microsegments-0.1.0/.github/workflows/publish.yml +71 -0
- microsegments-0.1.0/.github/workflows/test.yml +28 -0
- microsegments-0.1.0/.gitignore +14 -0
- microsegments-0.1.0/LICENSE +21 -0
- microsegments-0.1.0/PKG-INFO +184 -0
- microsegments-0.1.0/README.md +146 -0
- microsegments-0.1.0/docs/report.jpg +0 -0
- microsegments-0.1.0/examples/gtfsrt_csv.toml +35 -0
- microsegments-0.1.0/examples/make_synth.py +29 -0
- microsegments-0.1.0/examples/stib55.toml +29 -0
- microsegments-0.1.0/examples/synth.toml +27 -0
- microsegments-0.1.0/pyproject.toml +62 -0
- microsegments-0.1.0/src/microsegments/__init__.py +42 -0
- microsegments-0.1.0/src/microsegments/_version.py +24 -0
- microsegments-0.1.0/src/microsegments/aggregate.py +101 -0
- microsegments-0.1.0/src/microsegments/cli.py +201 -0
- microsegments-0.1.0/src/microsegments/config.py +142 -0
- microsegments-0.1.0/src/microsegments/contract.py +42 -0
- microsegments-0.1.0/src/microsegments/hotspots.py +337 -0
- microsegments-0.1.0/src/microsegments/html/__init__.py +6 -0
- microsegments-0.1.0/src/microsegments/html/export.py +96 -0
- microsegments-0.1.0/src/microsegments/html/leaflet.min.css +2 -0
- microsegments-0.1.0/src/microsegments/html/template.html +863 -0
- microsegments-0.1.0/src/microsegments/io/__init__.py +9 -0
- microsegments-0.1.0/src/microsegments/io/coverage.py +114 -0
- microsegments-0.1.0/src/microsegments/io/events.py +98 -0
- microsegments-0.1.0/src/microsegments/io/gtfsrt.py +168 -0
- microsegments-0.1.0/src/microsegments/io/ids.py +34 -0
- microsegments-0.1.0/src/microsegments/io/stib.py +217 -0
- microsegments-0.1.0/src/microsegments/io/tabular.py +188 -0
- microsegments-0.1.0/src/microsegments/io/timeutil.py +84 -0
- microsegments-0.1.0/src/microsegments/locate/__init__.py +47 -0
- microsegments-0.1.0/src/microsegments/locate/common.py +149 -0
- microsegments-0.1.0/src/microsegments/locate/linear.py +317 -0
- microsegments-0.1.0/src/microsegments/locate/mapmatch.py +390 -0
- microsegments-0.1.0/src/microsegments/locate/resample.py +71 -0
- microsegments-0.1.0/src/microsegments/metrics.py +520 -0
- microsegments-0.1.0/src/microsegments/network/__init__.py +10 -0
- microsegments-0.1.0/src/microsegments/network/calendar.py +218 -0
- microsegments-0.1.0/src/microsegments/network/geometry.py +224 -0
- microsegments-0.1.0/src/microsegments/network/keys.py +97 -0
- microsegments-0.1.0/src/microsegments/network/patterns.py +328 -0
- microsegments-0.1.0/src/microsegments/pipeline.py +308 -0
- microsegments-0.1.0/src/microsegments/plot.py +476 -0
- microsegments-0.1.0/src/microsegments/py.typed +0 -0
- microsegments-0.1.0/src/microsegments/report.py +258 -0
- microsegments-0.1.0/src/microsegments/schema.py +257 -0
- microsegments-0.1.0/src/microsegments/segments.py +253 -0
- microsegments-0.1.0/src/microsegments/simulate.py +612 -0
- microsegments-0.1.0/src/microsegments/tracks.py +391 -0
- microsegments-0.1.0/src/microsegments/tune.py +518 -0
- microsegments-0.1.0/tests/fixtures/stib55/golden/buckets.parquet +0 -0
- microsegments-0.1.0/tests/fixtures/stib55/golden/drops.parquet +0 -0
- microsegments-0.1.0/tests/fixtures/stib55/golden/flen.parquet +0 -0
- microsegments-0.1.0/tests/fixtures/stib55/golden/obs_day.parquet +0 -0
- microsegments-0.1.0/tests/fixtures/stib55/golden/obs_dow.parquet +0 -0
- microsegments-0.1.0/tests/fixtures/stib55/golden/passages.parquet +0 -0
- microsegments-0.1.0/tests/fixtures/stib55/golden/summary.json +229 -0
- microsegments-0.1.0/tests/fixtures/stib55/golden/validation.parquet +0 -0
- microsegments-0.1.0/tests/fixtures/stib55/gtfs/agency.parquet +0 -0
- microsegments-0.1.0/tests/fixtures/stib55/gtfs/calendar.parquet +0 -0
- microsegments-0.1.0/tests/fixtures/stib55/gtfs/calendar_dates.parquet +0 -0
- microsegments-0.1.0/tests/fixtures/stib55/gtfs/routes.parquet +0 -0
- microsegments-0.1.0/tests/fixtures/stib55/gtfs/shapes.parquet +0 -0
- microsegments-0.1.0/tests/fixtures/stib55/gtfs/stop_times.parquet +0 -0
- microsegments-0.1.0/tests/fixtures/stib55/gtfs/stops.parquet +0 -0
- microsegments-0.1.0/tests/fixtures/stib55/gtfs/trips.parquet +0 -0
- microsegments-0.1.0/tests/fixtures/stib55/punctuality/route55.parquet +0 -0
- microsegments-0.1.0/tests/fixtures/stib55/speed55_ref.json +315 -0
- microsegments-0.1.0/tests/fixtures/stib55/vd55/20250318.parquet +0 -0
- microsegments-0.1.0/tests/fixtures/stib55/vd55/20250318.snaps.parquet +0 -0
- microsegments-0.1.0/tests/fixtures/stib55/vd55/20250319.parquet +0 -0
- microsegments-0.1.0/tests/fixtures/stib55/vd55/20250319.snaps.parquet +0 -0
- microsegments-0.1.0/tests/gtfs_fixtures.py +123 -0
- microsegments-0.1.0/tests/regression/make_golden.py +366 -0
- microsegments-0.1.0/tests/t4_fixtures.py +226 -0
- microsegments-0.1.0/tests/test_aggregate.py +104 -0
- microsegments-0.1.0/tests/test_coverage.py +129 -0
- microsegments-0.1.0/tests/test_hotspots.py +77 -0
- microsegments-0.1.0/tests/test_io_events.py +56 -0
- microsegments-0.1.0/tests/test_io_gtfsrt.py +50 -0
- microsegments-0.1.0/tests/test_io_stib.py +114 -0
- microsegments-0.1.0/tests/test_io_tabular.py +103 -0
- microsegments-0.1.0/tests/test_locate_linear.py +79 -0
- microsegments-0.1.0/tests/test_locate_mapmatch.py +81 -0
- microsegments-0.1.0/tests/test_metrics.py +155 -0
- microsegments-0.1.0/tests/test_network_geometry.py +62 -0
- microsegments-0.1.0/tests/test_network_patterns.py +209 -0
- microsegments-0.1.0/tests/test_pipeline_cli.py +69 -0
- microsegments-0.1.0/tests/test_regression_stib55.py +130 -0
- microsegments-0.1.0/tests/test_report_html.py +116 -0
- microsegments-0.1.0/tests/test_scaffold.py +25 -0
- microsegments-0.1.0/tests/test_segments.py +139 -0
- microsegments-0.1.0/tests/test_simulate.py +121 -0
- microsegments-0.1.0/tests/test_tracks.py +77 -0
- microsegments-0.1.0/tests/test_tune.py +69 -0
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
name: Publish to PyPI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
release:
|
|
5
|
+
types: [published]
|
|
6
|
+
|
|
7
|
+
permissions:
|
|
8
|
+
contents: write
|
|
9
|
+
id-token: write
|
|
10
|
+
|
|
11
|
+
jobs:
|
|
12
|
+
build:
|
|
13
|
+
name: Build distribution
|
|
14
|
+
runs-on: ubuntu-latest
|
|
15
|
+
steps:
|
|
16
|
+
- uses: actions/checkout@v4
|
|
17
|
+
- name: Set up Python
|
|
18
|
+
uses: actions/setup-python@v5
|
|
19
|
+
with:
|
|
20
|
+
python-version: "3.12"
|
|
21
|
+
- name: Install build
|
|
22
|
+
run: pip install build
|
|
23
|
+
- name: Build package
|
|
24
|
+
run: python -m build
|
|
25
|
+
- name: Upload dist artifacts
|
|
26
|
+
uses: actions/upload-artifact@v4
|
|
27
|
+
with:
|
|
28
|
+
name: python-package-distributions
|
|
29
|
+
path: dist/
|
|
30
|
+
|
|
31
|
+
publish-to-pypi:
|
|
32
|
+
name: Publish to PyPI
|
|
33
|
+
needs: build
|
|
34
|
+
runs-on: ubuntu-latest
|
|
35
|
+
environment:
|
|
36
|
+
name: pypi
|
|
37
|
+
url: https://pypi.org/p/microsegments
|
|
38
|
+
permissions:
|
|
39
|
+
id-token: write
|
|
40
|
+
steps:
|
|
41
|
+
- name: Download dist artifacts
|
|
42
|
+
uses: actions/download-artifact@v4
|
|
43
|
+
with:
|
|
44
|
+
name: python-package-distributions
|
|
45
|
+
path: dist/
|
|
46
|
+
- name: Publish to PyPI
|
|
47
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
48
|
+
|
|
49
|
+
github-release:
|
|
50
|
+
name: Sign and create GitHub Release
|
|
51
|
+
needs: publish-to-pypi
|
|
52
|
+
runs-on: ubuntu-latest
|
|
53
|
+
permissions:
|
|
54
|
+
contents: write
|
|
55
|
+
id-token: write
|
|
56
|
+
steps:
|
|
57
|
+
- name: Download dist artifacts
|
|
58
|
+
uses: actions/download-artifact@v4
|
|
59
|
+
with:
|
|
60
|
+
name: python-package-distributions
|
|
61
|
+
path: dist/
|
|
62
|
+
- name: Sign with Sigstore
|
|
63
|
+
uses: sigstore/gh-action-sigstore-python@v3.0.0
|
|
64
|
+
with:
|
|
65
|
+
inputs: >-
|
|
66
|
+
./dist/*.tar.gz
|
|
67
|
+
./dist/*.whl
|
|
68
|
+
- name: Upload to GitHub Release
|
|
69
|
+
env:
|
|
70
|
+
GITHUB_TOKEN: ${{ github.token }}
|
|
71
|
+
run: gh release upload '${{ github.ref_name }}' dist/** --repo '${{ github.repository }}' --clobber
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
name: Test
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
branches: [main]
|
|
8
|
+
|
|
9
|
+
jobs:
|
|
10
|
+
test:
|
|
11
|
+
runs-on: ubuntu-latest
|
|
12
|
+
strategy:
|
|
13
|
+
fail-fast: false
|
|
14
|
+
matrix:
|
|
15
|
+
python-version: ["3.11", "3.12", "3.13"]
|
|
16
|
+
steps:
|
|
17
|
+
- uses: actions/checkout@v4
|
|
18
|
+
with:
|
|
19
|
+
fetch-depth: 0
|
|
20
|
+
- uses: actions/setup-python@v5
|
|
21
|
+
with:
|
|
22
|
+
python-version: ${{ matrix.python-version }}
|
|
23
|
+
- name: Install dependencies
|
|
24
|
+
run: |
|
|
25
|
+
python -m pip install --upgrade pip
|
|
26
|
+
pip install -e ".[dev]"
|
|
27
|
+
- name: Run tests
|
|
28
|
+
run: pytest
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Gaspard Merten
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: microsegments
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Where do transit vehicles linger? Count vehicle position observations per micro-segment of a line, from GTFS + AVL / GTFS-RT positions.
|
|
5
|
+
Project-URL: Homepage, https://github.com/GaspardMerten/microsegments
|
|
6
|
+
Project-URL: Repository, https://github.com/GaspardMerten/microsegments
|
|
7
|
+
Project-URL: Issues, https://github.com/GaspardMerten/microsegments/issues
|
|
8
|
+
Author-email: Gaspard Merten <gaspard.mp.work@gmail.com>
|
|
9
|
+
License-Expression: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: avl,bus,gtfs,gtfs-rt,hotspots,polars,tram,transit
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Intended Audience :: Science/Research
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Operating System :: OS Independent
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Topic :: Scientific/Engineering :: GIS
|
|
21
|
+
Classifier: Typing :: Typed
|
|
22
|
+
Requires-Python: >=3.11
|
|
23
|
+
Requires-Dist: gtfs-parquet>=0.7
|
|
24
|
+
Requires-Dist: numpy>=1.26
|
|
25
|
+
Requires-Dist: polars>=1.0
|
|
26
|
+
Requires-Dist: shapely>=2.0
|
|
27
|
+
Provides-Extra: dev
|
|
28
|
+
Requires-Dist: hypothesis>=6.0; extra == 'dev'
|
|
29
|
+
Requires-Dist: matplotlib>=3.8; extra == 'dev'
|
|
30
|
+
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
31
|
+
Provides-Extra: gtfsrt
|
|
32
|
+
Requires-Dist: gtfs-realtime-bindings>=1.0; extra == 'gtfsrt'
|
|
33
|
+
Provides-Extra: mobilitytwin
|
|
34
|
+
Requires-Dist: requests>=2.31; extra == 'mobilitytwin'
|
|
35
|
+
Provides-Extra: plot
|
|
36
|
+
Requires-Dist: matplotlib>=3.8; extra == 'plot'
|
|
37
|
+
Description-Content-Type: text/markdown
|
|
38
|
+
|
|
39
|
+
# microsegments
|
|
40
|
+
|
|
41
|
+
Where do buses and trams linger? `microsegments` cuts each line into short stretches (30 m by default)
|
|
42
|
+
and counts how many times a vehicle position is observed in each one, per hour and per day of the week.
|
|
43
|
+
It works with GTFS plus any vehicle position feed: GTFS-RT VehiclePositions (lat/lon, as CSV, parquet or
|
|
44
|
+
`.pb`) or linear-referenced AVL (last stop + distance, like STIB).
|
|
45
|
+
|
|
46
|
+

|
|
47
|
+
|
|
48
|
+
## Install
|
|
49
|
+
|
|
50
|
+
```bash
|
|
51
|
+
pip install "microsegments[plot]" # plot extra = matplotlib
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
Python ≥ 3.11. Core dependencies: polars, numpy, shapely, gtfs-parquet (no pandas).
|
|
55
|
+
|
|
56
|
+
## Quick start (offline, synthetic data)
|
|
57
|
+
|
|
58
|
+
```bash
|
|
59
|
+
python examples/make_synth.py # simulated tram line S1 -> examples/data/
|
|
60
|
+
microsegments run examples/synth.toml -o out/ # result.parquet, analysis.json, report.html, matrix_dir*.png
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
## Linear-referenced feeds (STIB vehicle-distance)
|
|
64
|
+
|
|
65
|
+
```toml
|
|
66
|
+
[input]
|
|
67
|
+
paths = ["vd55/*.parquet"] # <YYYYMMDD>.parquet (ts, dir, point, dist) + <YYYYMMDD>.snaps.parquet
|
|
68
|
+
kind = "stib" # or "linear" for any "stop_id + dist_m" table (map columns below)
|
|
69
|
+
events = "punctuality/*.parquet" # optional stop events: passages from both sources, max taken
|
|
70
|
+
|
|
71
|
+
[input.linear]
|
|
72
|
+
id_normaliser = "leading_digits" # "05766" -> "5766"
|
|
73
|
+
feed_length = "auto" # rescale feed metres to GTFS link lengths
|
|
74
|
+
|
|
75
|
+
[gtfs]
|
|
76
|
+
path = "gtfs/" # zip, .txt directory or gtfs-parquet directory
|
|
77
|
+
# dated = "gtfs/{date}" # one feed per day instead
|
|
78
|
+
|
|
79
|
+
[select]
|
|
80
|
+
route = "55"
|
|
81
|
+
dates = "2025-02-17..2025-05-16"
|
|
82
|
+
weekdays = [0, 1, 2, 3, 4]
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
See `examples/stib55.toml` (runs on the two-day test fixture).
|
|
86
|
+
|
|
87
|
+
## GTFS-RT positions as CSV (lat/lon)
|
|
88
|
+
|
|
89
|
+
```toml
|
|
90
|
+
[input]
|
|
91
|
+
paths = ["positions/*.csv"]
|
|
92
|
+
kind = "latlon"
|
|
93
|
+
ts_unit = "s" # "s" | "ms" | "us" | "iso"
|
|
94
|
+
[input.columns] # canonical name = your column
|
|
95
|
+
ts = "timestamp"
|
|
96
|
+
route_id = "route_id"
|
|
97
|
+
vehicle_id = "vehicle_id"
|
|
98
|
+
lat = "latitude"
|
|
99
|
+
lon = "longitude"
|
|
100
|
+
trip_id = "trip_id" # optional, helps map matching
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
Per-vehicle fixes are thinned to one per `tick_s` (20 s) so counts stay comparable with a snapshot feed.
|
|
104
|
+
See `examples/gtfsrt_csv.toml` for every option.
|
|
105
|
+
|
|
106
|
+
## Python
|
|
107
|
+
|
|
108
|
+
```python
|
|
109
|
+
import microsegments as ms
|
|
110
|
+
|
|
111
|
+
res = ms.run(ms.Config.from_toml("ms.toml")) # obs -> coverage -> network -> place -> passages
|
|
112
|
+
# -> segments -> count -> analyse -> hotspots
|
|
113
|
+
res.result # polars: one row per segment x hour (bands as hour -1 am, -2 pm, -3 day, -4 evening)
|
|
114
|
+
res.hotspots # ranked stretches
|
|
115
|
+
ms.export(res, "report.html", lang="fr") # standalone page (fr / en)
|
|
116
|
+
ms.plot.matrix(res, direction_id=0, metric="obs_per_h", days=[0, 1, 2, 3, 4], stops=False)
|
|
117
|
+
ms.plot.map(res, 0, "pm", metric="excess_per_passage")
|
|
118
|
+
contract = res.contract() # the page JSON (also the platform's /api/analysis)
|
|
119
|
+
|
|
120
|
+
pre = ms.prepare(cfg) # reuse placement to tune the segment length
|
|
121
|
+
tr = ms.tune.segment_length(pre.placed, pre.segment_fn(), coverage=pre.coverage, passages=pre.passages,
|
|
122
|
+
pattern_days=pre.network.pattern_days)
|
|
123
|
+
ms.plot.tune_curves(tr)
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
Stages are usable on their own: `read_observations`, `build_network`, `segment`, `count`, `analyse`,
|
|
127
|
+
`hotspots`, `simulate`.
|
|
128
|
+
|
|
129
|
+
## CLI
|
|
130
|
+
|
|
131
|
+
```bash
|
|
132
|
+
microsegments run CONFIG -o OUT [--lang en] [--segment-m 20] [--dates A..B] [--source "credit"]
|
|
133
|
+
microsegments tune CONFIG [--lengths 10,20,30,50] [--sensitivity] [-o OUT]
|
|
134
|
+
microsegments hotspots CONFIG [-o hotspots.geojson|.csv|.parquet]
|
|
135
|
+
microsegments inspect CONFIG # coverage per day, GTFS versions, placement / drop counts
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
## What the numbers mean
|
|
139
|
+
|
|
140
|
+
Every poll (about every 20 s) puts each vehicle in one micro-segment: one **observation**. One observation
|
|
141
|
+
is therefore about 20 s of presence.
|
|
142
|
+
|
|
143
|
+
| Metric | Definition | Read as |
|
|
144
|
+
|---|---|---|
|
|
145
|
+
| `obs_per_h` (default) | Σ observations / Σ covered hours | vehicles seen in the segment per hour of polling; `/len × 10` for per 10 m |
|
|
146
|
+
| `obs_per_passage` | Σ observations / Σ passages | × 20 s ≈ time each vehicle spends there |
|
|
147
|
+
| `excess_per_passage` | obs per passage − the same at 20–23 h, same days | extra observations (≈ seconds / 20) lost per vehicle vs free-flowing evening |
|
|
148
|
+
| `excess_obs_per_h`, `log2_ratio` | see `metrics.py` | |
|
|
149
|
+
|
|
150
|
+
All values are ratios of sums over the selected days (optionally post-stratified by weekday). Stop zones
|
|
151
|
+
(30 m before to 60 m after each stop) can be masked to bring out signals and junctions. Colour scales are
|
|
152
|
+
fixed per metric (p98 over 6–21 h), so hours and weekdays compare directly.
|
|
153
|
+
|
|
154
|
+
## Missing data and GTFS versions
|
|
155
|
+
|
|
156
|
+
- **Hours** with < 80 % of polls, a frozen feed (> 5 min), the line absent, or abnormal vehicle counts are
|
|
157
|
+
dropped from numerator and denominator. Polling gaps are handled by the exposure (covered hours); counts
|
|
158
|
+
are never imputed.
|
|
159
|
+
- **Days** with more than 25 % of their 6–21 h dropped are excluded and listed with the reason
|
|
160
|
+
(`res.analysis.excluded_days`, coverage calendar in the page).
|
|
161
|
+
- **GTFS changes**: patterns are rebuilt per service date; `link_key` / `seg_key` are stable across
|
|
162
|
+
versions. Each segment is aggregated only over the days it belongs to the day's main pattern (the
|
|
163
|
+
reference too). The page shows the most frequent version; others are selectable. A single feed whose
|
|
164
|
+
calendar does not cover some dates lends them the patterns of the same weekday.
|
|
165
|
+
|
|
166
|
+
## Choosing the segment length
|
|
167
|
+
|
|
168
|
+
`microsegments tune` scores L ∈ {10, 15, 20, 30, 40, 50, 75, 100} m (leave-one-day-out Poisson deviance,
|
|
169
|
+
split-half reliability, hotspot localisation spread and Jaccard) and picks the smallest L with reliability
|
|
170
|
+
≥ 0.8, spread ≤ 30 m and deviance within one standard error of the minimum. 30 m is a good default for
|
|
171
|
+
20 s polling at urban speeds; go shorter only with dense GTFS-RT fixes and many days. `--sensitivity`
|
|
172
|
+
re-runs phase offsets, gap caps, stop zones, references and passage sources.
|
|
173
|
+
|
|
174
|
+
## Hotspots
|
|
175
|
+
|
|
176
|
+
Stretches where the excess over the evening has a 95 % bootstrap lower bound above 2 s per vehicle, is
|
|
177
|
+
positive on ≥ 60 % of days, survives Benjamini–Hochberg (q = 0.1) and lasts ≥ 2 hours or a peak band.
|
|
178
|
+
Adjacent bins merge, never across a stop / running border. Classes: *infrastructure* (all day),
|
|
179
|
+
*congestion* (peak only), *mixed*. Needs at least 5 included days. Export as GeoJSON with
|
|
180
|
+
`microsegments hotspots CONFIG -o hotspots.geojson`.
|
|
181
|
+
|
|
182
|
+
## License
|
|
183
|
+
|
|
184
|
+
MIT. The bundled Leaflet CSS is BSD-2-Clause.
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
# microsegments
|
|
2
|
+
|
|
3
|
+
Where do buses and trams linger? `microsegments` cuts each line into short stretches (30 m by default)
|
|
4
|
+
and counts how many times a vehicle position is observed in each one, per hour and per day of the week.
|
|
5
|
+
It works with GTFS plus any vehicle position feed: GTFS-RT VehiclePositions (lat/lon, as CSV, parquet or
|
|
6
|
+
`.pb`) or linear-referenced AVL (last stop + distance, like STIB).
|
|
7
|
+
|
|
8
|
+

|
|
9
|
+
|
|
10
|
+
## Install
|
|
11
|
+
|
|
12
|
+
```bash
|
|
13
|
+
pip install "microsegments[plot]" # plot extra = matplotlib
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
Python ≥ 3.11. Core dependencies: polars, numpy, shapely, gtfs-parquet (no pandas).
|
|
17
|
+
|
|
18
|
+
## Quick start (offline, synthetic data)
|
|
19
|
+
|
|
20
|
+
```bash
|
|
21
|
+
python examples/make_synth.py # simulated tram line S1 -> examples/data/
|
|
22
|
+
microsegments run examples/synth.toml -o out/ # result.parquet, analysis.json, report.html, matrix_dir*.png
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
## Linear-referenced feeds (STIB vehicle-distance)
|
|
26
|
+
|
|
27
|
+
```toml
|
|
28
|
+
[input]
|
|
29
|
+
paths = ["vd55/*.parquet"] # <YYYYMMDD>.parquet (ts, dir, point, dist) + <YYYYMMDD>.snaps.parquet
|
|
30
|
+
kind = "stib" # or "linear" for any "stop_id + dist_m" table (map columns below)
|
|
31
|
+
events = "punctuality/*.parquet" # optional stop events: passages from both sources, max taken
|
|
32
|
+
|
|
33
|
+
[input.linear]
|
|
34
|
+
id_normaliser = "leading_digits" # "05766" -> "5766"
|
|
35
|
+
feed_length = "auto" # rescale feed metres to GTFS link lengths
|
|
36
|
+
|
|
37
|
+
[gtfs]
|
|
38
|
+
path = "gtfs/" # zip, .txt directory or gtfs-parquet directory
|
|
39
|
+
# dated = "gtfs/{date}" # one feed per day instead
|
|
40
|
+
|
|
41
|
+
[select]
|
|
42
|
+
route = "55"
|
|
43
|
+
dates = "2025-02-17..2025-05-16"
|
|
44
|
+
weekdays = [0, 1, 2, 3, 4]
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
See `examples/stib55.toml` (runs on the two-day test fixture).
|
|
48
|
+
|
|
49
|
+
## GTFS-RT positions as CSV (lat/lon)
|
|
50
|
+
|
|
51
|
+
```toml
|
|
52
|
+
[input]
|
|
53
|
+
paths = ["positions/*.csv"]
|
|
54
|
+
kind = "latlon"
|
|
55
|
+
ts_unit = "s" # "s" | "ms" | "us" | "iso"
|
|
56
|
+
[input.columns] # canonical name = your column
|
|
57
|
+
ts = "timestamp"
|
|
58
|
+
route_id = "route_id"
|
|
59
|
+
vehicle_id = "vehicle_id"
|
|
60
|
+
lat = "latitude"
|
|
61
|
+
lon = "longitude"
|
|
62
|
+
trip_id = "trip_id" # optional, helps map matching
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
Per-vehicle fixes are thinned to one per `tick_s` (20 s) so counts stay comparable with a snapshot feed.
|
|
66
|
+
See `examples/gtfsrt_csv.toml` for every option.
|
|
67
|
+
|
|
68
|
+
## Python
|
|
69
|
+
|
|
70
|
+
```python
|
|
71
|
+
import microsegments as ms
|
|
72
|
+
|
|
73
|
+
res = ms.run(ms.Config.from_toml("ms.toml")) # obs -> coverage -> network -> place -> passages
|
|
74
|
+
# -> segments -> count -> analyse -> hotspots
|
|
75
|
+
res.result # polars: one row per segment x hour (bands as hour -1 am, -2 pm, -3 day, -4 evening)
|
|
76
|
+
res.hotspots # ranked stretches
|
|
77
|
+
ms.export(res, "report.html", lang="fr") # standalone page (fr / en)
|
|
78
|
+
ms.plot.matrix(res, direction_id=0, metric="obs_per_h", days=[0, 1, 2, 3, 4], stops=False)
|
|
79
|
+
ms.plot.map(res, 0, "pm", metric="excess_per_passage")
|
|
80
|
+
contract = res.contract() # the page JSON (also the platform's /api/analysis)
|
|
81
|
+
|
|
82
|
+
pre = ms.prepare(cfg) # reuse placement to tune the segment length
|
|
83
|
+
tr = ms.tune.segment_length(pre.placed, pre.segment_fn(), coverage=pre.coverage, passages=pre.passages,
|
|
84
|
+
pattern_days=pre.network.pattern_days)
|
|
85
|
+
ms.plot.tune_curves(tr)
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
Stages are usable on their own: `read_observations`, `build_network`, `segment`, `count`, `analyse`,
|
|
89
|
+
`hotspots`, `simulate`.
|
|
90
|
+
|
|
91
|
+
## CLI
|
|
92
|
+
|
|
93
|
+
```bash
|
|
94
|
+
microsegments run CONFIG -o OUT [--lang en] [--segment-m 20] [--dates A..B] [--source "credit"]
|
|
95
|
+
microsegments tune CONFIG [--lengths 10,20,30,50] [--sensitivity] [-o OUT]
|
|
96
|
+
microsegments hotspots CONFIG [-o hotspots.geojson|.csv|.parquet]
|
|
97
|
+
microsegments inspect CONFIG # coverage per day, GTFS versions, placement / drop counts
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
## What the numbers mean
|
|
101
|
+
|
|
102
|
+
Every poll (about every 20 s) puts each vehicle in one micro-segment: one **observation**. One observation
|
|
103
|
+
is therefore about 20 s of presence.
|
|
104
|
+
|
|
105
|
+
| Metric | Definition | Read as |
|
|
106
|
+
|---|---|---|
|
|
107
|
+
| `obs_per_h` (default) | Σ observations / Σ covered hours | vehicles seen in the segment per hour of polling; `/len × 10` for per 10 m |
|
|
108
|
+
| `obs_per_passage` | Σ observations / Σ passages | × 20 s ≈ time each vehicle spends there |
|
|
109
|
+
| `excess_per_passage` | obs per passage − the same at 20–23 h, same days | extra observations (≈ seconds / 20) lost per vehicle vs free-flowing evening |
|
|
110
|
+
| `excess_obs_per_h`, `log2_ratio` | see `metrics.py` | |
|
|
111
|
+
|
|
112
|
+
All values are ratios of sums over the selected days (optionally post-stratified by weekday). Stop zones
|
|
113
|
+
(30 m before to 60 m after each stop) can be masked to bring out signals and junctions. Colour scales are
|
|
114
|
+
fixed per metric (p98 over 6–21 h), so hours and weekdays compare directly.
|
|
115
|
+
|
|
116
|
+
## Missing data and GTFS versions
|
|
117
|
+
|
|
118
|
+
- **Hours** with < 80 % of polls, a frozen feed (> 5 min), the line absent, or abnormal vehicle counts are
|
|
119
|
+
dropped from numerator and denominator. Polling gaps are handled by the exposure (covered hours); counts
|
|
120
|
+
are never imputed.
|
|
121
|
+
- **Days** with more than 25 % of their 6–21 h dropped are excluded and listed with the reason
|
|
122
|
+
(`res.analysis.excluded_days`, coverage calendar in the page).
|
|
123
|
+
- **GTFS changes**: patterns are rebuilt per service date; `link_key` / `seg_key` are stable across
|
|
124
|
+
versions. Each segment is aggregated only over the days it belongs to the day's main pattern (the
|
|
125
|
+
reference too). The page shows the most frequent version; others are selectable. A single feed whose
|
|
126
|
+
calendar does not cover some dates lends them the patterns of the same weekday.
|
|
127
|
+
|
|
128
|
+
## Choosing the segment length
|
|
129
|
+
|
|
130
|
+
`microsegments tune` scores L ∈ {10, 15, 20, 30, 40, 50, 75, 100} m (leave-one-day-out Poisson deviance,
|
|
131
|
+
split-half reliability, hotspot localisation spread and Jaccard) and picks the smallest L with reliability
|
|
132
|
+
≥ 0.8, spread ≤ 30 m and deviance within one standard error of the minimum. 30 m is a good default for
|
|
133
|
+
20 s polling at urban speeds; go shorter only with dense GTFS-RT fixes and many days. `--sensitivity`
|
|
134
|
+
re-runs phase offsets, gap caps, stop zones, references and passage sources.
|
|
135
|
+
|
|
136
|
+
## Hotspots
|
|
137
|
+
|
|
138
|
+
Stretches where the excess over the evening has a 95 % bootstrap lower bound above 2 s per vehicle, is
|
|
139
|
+
positive on ≥ 60 % of days, survives Benjamini–Hochberg (q = 0.1) and lasts ≥ 2 hours or a peak band.
|
|
140
|
+
Adjacent bins merge, never across a stop / running border. Classes: *infrastructure* (all day),
|
|
141
|
+
*congestion* (peak only), *mixed*. Needs at least 5 included days. Export as GeoJSON with
|
|
142
|
+
`microsegments hotspots CONFIG -o hotspots.geojson`.
|
|
143
|
+
|
|
144
|
+
## License
|
|
145
|
+
|
|
146
|
+
MIT. The bundled Leaflet CSS is BSD-2-Clause.
|
|
Binary file
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
# GTFS-RT VehiclePositions flattened to CSV (one row per vehicle fix), matched on the GTFS shapes.
|
|
2
|
+
# microsegments run examples/gtfsrt_csv.toml -o out/bus
|
|
3
|
+
|
|
4
|
+
[input]
|
|
5
|
+
paths = ["positions/*.csv"] # or *.parquet; GTFS-RT .pb files also work (format = "gtfsrt_pb")
|
|
6
|
+
kind = "latlon"
|
|
7
|
+
format = "auto"
|
|
8
|
+
timezone = "Europe/Brussels"
|
|
9
|
+
service_day_start = "04:00" # 01:30 belongs to the previous service day
|
|
10
|
+
ts_unit = "s" # "s" | "ms" | "us" | "iso"
|
|
11
|
+
|
|
12
|
+
# canonical name = column in your file (only those that differ; unmapped well-known names are detected)
|
|
13
|
+
[input.columns]
|
|
14
|
+
ts = "timestamp" # fix time (vehicle.timestamp), epoch seconds here
|
|
15
|
+
route_id = "route_id" # must match the GTFS route_short_name (or route_id, see gtfs.route_key)
|
|
16
|
+
vehicle_id = "vehicle_id" # per-vehicle feed: fixes are thinned to one per tick_s
|
|
17
|
+
trip_id = "trip_id" # optional, helps map matching
|
|
18
|
+
direction_id = "direction_id" # optional
|
|
19
|
+
lat = "latitude"
|
|
20
|
+
lon = "longitude"
|
|
21
|
+
bearing = "bearing" # optional, degrees
|
|
22
|
+
|
|
23
|
+
[gtfs]
|
|
24
|
+
path = "gtfs.zip"
|
|
25
|
+
route_key = "route_short_name"
|
|
26
|
+
|
|
27
|
+
[select]
|
|
28
|
+
route = "71"
|
|
29
|
+
dates = "2025-02-17..2025-05-16"
|
|
30
|
+
weekdays = [0, 1, 2, 3, 4]
|
|
31
|
+
|
|
32
|
+
[params]
|
|
33
|
+
segment_m = 30
|
|
34
|
+
tick_s = 20 # nominal interval: counts stay comparable with a 20 s snapshot feed
|
|
35
|
+
max_lateral_m = 40 # fixes farther from the shape are dropped
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"""Generate the synthetic example data (offline) into examples/data/:
|
|
2
|
+
|
|
3
|
+
python examples/make_synth.py
|
|
4
|
+
microsegments run examples/synth.toml -o out/
|
|
5
|
+
|
|
6
|
+
A simulated tram line "S1" (two directions, 8 links), 3 weeks of STIB-like positions every ~20 s
|
|
7
|
+
with outages and a frozen feed, a GTFS change after two thirds of the period (the last two stops are cut), and three
|
|
8
|
+
injected hotspots (a signal, a peak-hour queue, an evening holding stop). See microsegments.simulate.
|
|
9
|
+
"""
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import sys
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
from microsegments.simulate import simulate
|
|
16
|
+
|
|
17
|
+
HERE = Path(__file__).resolve().parent
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def main(out: Path = HERE / "data", days: int = 21, seed: int = 0) -> Path:
|
|
21
|
+
sim = simulate(seed=seed, days=days, start="2025-03-03", mode="stib", out_dir=out / "gtfs", change_day=days * 2 // 3)
|
|
22
|
+
sim.write_stib(out / "vd")
|
|
23
|
+
sim.stop_events.write_parquet(out / "events.parquet")
|
|
24
|
+
print(f"{len(sim.observations):,} positions, {len(sim.dates)} days -> {out}")
|
|
25
|
+
return out
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
if __name__ == "__main__":
|
|
29
|
+
main(Path(sys.argv[1]) if len(sys.argv) > 1 else HERE / "data")
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# STIB tram 55, two days of vehicle-distance positions (the test fixture), linear referencing.
|
|
2
|
+
# microsegments run examples/stib55.toml -o out/stib55
|
|
3
|
+
# Real use: point `paths` at the compact daily parquet files of a longer period (MobilityTwin
|
|
4
|
+
# stib/vehicle-distance), and `gtfs.dated` at one GTFS snapshot per day.
|
|
5
|
+
|
|
6
|
+
[input]
|
|
7
|
+
paths = ["../tests/fixtures/stib55/vd55/*.parquet"] # <YYYYMMDD>.parquet (ts, dir, point, dist) + .snaps.parquet
|
|
8
|
+
kind = "stib"
|
|
9
|
+
events = "../tests/fixtures/stib55/punctuality/route55.parquet" # STIB punctuality: passages from stop events
|
|
10
|
+
|
|
11
|
+
[input.linear]
|
|
12
|
+
direction_kind = "terminus_stop" # STIB directionId = destination stop code
|
|
13
|
+
id_normaliser = "leading_digits"
|
|
14
|
+
feed_length = "auto" # rescale feed metres to the GTFS link length (p99.5 per link)
|
|
15
|
+
|
|
16
|
+
[gtfs]
|
|
17
|
+
path = "../tests/fixtures/stib55/gtfs" # gtfs-parquet directory
|
|
18
|
+
# dated = "gtfs/{date}" # or one feed per day
|
|
19
|
+
|
|
20
|
+
[select]
|
|
21
|
+
route = "55"
|
|
22
|
+
dates = "2025-03-18..2025-03-19"
|
|
23
|
+
weekdays = [0, 1, 2, 3, 4]
|
|
24
|
+
|
|
25
|
+
[params]
|
|
26
|
+
segment_m = 30
|
|
27
|
+
|
|
28
|
+
[quality]
|
|
29
|
+
min_days = 2 # only two days in the fixture
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# Synthetic tram line S1 (generate the data first: python examples/make_synth.py)
|
|
2
|
+
# microsegments run examples/synth.toml -o out/
|
|
3
|
+
|
|
4
|
+
[input]
|
|
5
|
+
paths = ["data/vd/*.parquet"] # compact STIB-like files: <YYYYMMDD>.parquet + .snaps.parquet
|
|
6
|
+
kind = "stib" # linear referencing: last stop passed + metres since
|
|
7
|
+
timezone = "Europe/Brussels"
|
|
8
|
+
events = "data/events.parquet" # stop events (arrival / departure per trip and stop), optional
|
|
9
|
+
|
|
10
|
+
[input.linear]
|
|
11
|
+
id_normaliser = "leading_digits" # "01003" -> "1003"
|
|
12
|
+
|
|
13
|
+
[gtfs]
|
|
14
|
+
path = "data/gtfs" # GTFS directory (.txt), zip or gtfs-parquet directory
|
|
15
|
+
route_key = "route_short_name"
|
|
16
|
+
|
|
17
|
+
[select]
|
|
18
|
+
route = "S1"
|
|
19
|
+
dates = "2025-03-03..2025-03-23"
|
|
20
|
+
weekdays = [0, 1, 2, 3, 4]
|
|
21
|
+
hours = [5, 24]
|
|
22
|
+
|
|
23
|
+
[params]
|
|
24
|
+
segment_m = 30
|
|
25
|
+
stop_zone = [30, 60] # metres before / after a stop masked as "stop zone"
|
|
26
|
+
reference_hours = [20, 23]
|
|
27
|
+
bootstrap = 200
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "microsegments"
|
|
3
|
+
dynamic = ["version"]
|
|
4
|
+
description = "Where do transit vehicles linger? Count vehicle position observations per micro-segment of a line, from GTFS + AVL / GTFS-RT positions."
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
license = "MIT"
|
|
7
|
+
requires-python = ">=3.11"
|
|
8
|
+
authors = [
|
|
9
|
+
{ name = "Gaspard Merten", email = "gaspard.mp.work@gmail.com" },
|
|
10
|
+
]
|
|
11
|
+
keywords = ["gtfs", "gtfs-rt", "avl", "transit", "bus", "tram", "hotspots", "polars"]
|
|
12
|
+
classifiers = [
|
|
13
|
+
"Development Status :: 3 - Alpha",
|
|
14
|
+
"Intended Audience :: Science/Research",
|
|
15
|
+
"License :: OSI Approved :: MIT License",
|
|
16
|
+
"Operating System :: OS Independent",
|
|
17
|
+
"Programming Language :: Python :: 3",
|
|
18
|
+
"Programming Language :: Python :: 3.11",
|
|
19
|
+
"Programming Language :: Python :: 3.12",
|
|
20
|
+
"Programming Language :: Python :: 3.13",
|
|
21
|
+
"Topic :: Scientific/Engineering :: GIS",
|
|
22
|
+
"Typing :: Typed",
|
|
23
|
+
]
|
|
24
|
+
dependencies = [
|
|
25
|
+
"polars>=1.0",
|
|
26
|
+
"numpy>=1.26",
|
|
27
|
+
"shapely>=2.0",
|
|
28
|
+
"gtfs-parquet>=0.7",
|
|
29
|
+
]
|
|
30
|
+
|
|
31
|
+
[project.optional-dependencies]
|
|
32
|
+
plot = ["matplotlib>=3.8"]
|
|
33
|
+
mobilitytwin = ["requests>=2.31"]
|
|
34
|
+
gtfsrt = ["gtfs-realtime-bindings>=1.0"]
|
|
35
|
+
dev = ["pytest>=8.0", "hypothesis>=6.0", "matplotlib>=3.8"]
|
|
36
|
+
|
|
37
|
+
[project.scripts]
|
|
38
|
+
microsegments = "microsegments.cli:main"
|
|
39
|
+
|
|
40
|
+
[project.urls]
|
|
41
|
+
Homepage = "https://github.com/GaspardMerten/microsegments"
|
|
42
|
+
Repository = "https://github.com/GaspardMerten/microsegments"
|
|
43
|
+
Issues = "https://github.com/GaspardMerten/microsegments/issues"
|
|
44
|
+
|
|
45
|
+
[build-system]
|
|
46
|
+
requires = ["hatchling", "hatch-vcs"]
|
|
47
|
+
build-backend = "hatchling.build"
|
|
48
|
+
|
|
49
|
+
[tool.hatch.version]
|
|
50
|
+
source = "vcs"
|
|
51
|
+
fallback-version = "0.0.0"
|
|
52
|
+
|
|
53
|
+
[tool.hatch.build.hooks.vcs]
|
|
54
|
+
version-file = "src/microsegments/_version.py"
|
|
55
|
+
|
|
56
|
+
[tool.hatch.build.targets.wheel]
|
|
57
|
+
packages = ["src/microsegments"]
|
|
58
|
+
|
|
59
|
+
[tool.pytest.ini_options]
|
|
60
|
+
testpaths = ["tests"]
|
|
61
|
+
markers = ["regression: needs the local BrusselsTransitStuck prototype data (MS_PROTOTYPE_DATA)"]
|
|
62
|
+
addopts = "-m 'not regression'"
|