mobisurvstd 1.0.0__tar.gz → 1.0.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {mobisurvstd-1.0.0/src/mobisurvstd.egg-info → mobisurvstd-1.0.2}/PKG-INFO +11 -7
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/README.md +9 -6
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/pyproject.toml +11 -3
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/cerema/deplacements.py +7 -3
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/cerema/edgt.py +26 -28
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/cerema/edvm.py +18 -21
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/cerema/emc2.py +16 -24
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/cerema/emd.py +14 -16
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/cerema/menages.py +11 -5
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/cerema/nantes_opendata.py +4 -3
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/cerema/personnes.py +18 -15
- mobisurvstd-1.0.2/src/mobisurvstd/cerema/reader.py +159 -0
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/cerema/survey.py +118 -229
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/cerema/trajets.py +3 -1
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/cerema/zones.py +8 -4
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/classes.py +14 -15
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/common/cars.py +1 -1
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/common/clean.py +11 -10
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/common/households.py +3 -3
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/common/legs.py +1 -1
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/common/motorcycles.py +1 -1
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/common/persons.py +1 -1
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/common/trips.py +3 -2
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/common/zones.py +1 -1
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/egt2010/menages.py +2 -2
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/egt2010/survey.py +4 -4
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/emp/survey.py +1 -1
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/resources/admin_express.py +22 -18
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/resources/insee_data.py +6 -5
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/schema/common.py +4 -2
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/schema/guarantees.py +9 -5
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/schema/legs.py +2 -2
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/schema/trips.py +2 -20
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/utils.py +68 -29
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2/src/mobisurvstd.egg-info}/PKG-INFO +11 -7
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd.egg-info/SOURCES.txt +1 -0
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd.egg-info/requires.txt +1 -0
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/LICENSE.md +0 -0
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/setup.cfg +0 -0
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/__init__.py +0 -0
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/__main__.py +0 -0
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/cerema/__init__.py +0 -0
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/cerema/common.py +0 -0
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/cli.py +0 -0
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/common/__init__.py +0 -0
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/egt2010/__init__.py +0 -0
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/egt2010/deplacements.py +0 -0
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/egt2010/personnes.py +0 -0
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/egt2010/trajets.py +0 -0
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/egt2010/zones.py +0 -0
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/egt2020/__init__.py +0 -0
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/egt2020/deplacements.py +0 -0
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/egt2020/menages.py +0 -0
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/egt2020/motos.py +0 -0
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/egt2020/personnes.py +0 -0
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/egt2020/survey.py +0 -0
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/egt2020/trajets.py +0 -0
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/egt2020/voitures.py +0 -0
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/emp/__init__.py +0 -0
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/emp/common.py +0 -0
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/emp/deplacements.py +0 -0
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/emp/menages.py +0 -0
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/emp/motos.py +0 -0
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/emp/personnes.py +0 -0
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/emp/voitures.py +0 -0
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/logger.py +0 -0
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/main.py +0 -0
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/resources/__init__.py +0 -0
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/resources/nuts.py +0 -0
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/schema/__init__.py +0 -0
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/schema/cars.py +0 -0
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/schema/households.py +0 -0
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/schema/motorcycles.py +0 -0
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/schema/persons.py +0 -0
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/validation.py +0 -0
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd.egg-info/dependency_links.txt +0 -0
- {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: mobisurvstd
|
|
3
|
-
Version: 1.0.
|
|
3
|
+
Version: 1.0.2
|
|
4
4
|
Summary: A tool to standardize French mobility survey datasets (EMC2, EGT, EMP, etc.) into a unified format.
|
|
5
5
|
Author-email: Lucas Javaudin <lucas@lucasjavaudin.com>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -29,6 +29,7 @@ Requires-Dist: polars>=1.33.0
|
|
|
29
29
|
Requires-Dist: py7zr>=0.20.8
|
|
30
30
|
Requires-Dist: pyarrow>=19.0.1
|
|
31
31
|
Requires-Dist: requests>=2.32.3
|
|
32
|
+
Requires-Dist: seaborn>=0.13.2
|
|
32
33
|
Requires-Dist: typer>=0.16.0
|
|
33
34
|
Dynamic: license-file
|
|
34
35
|
|
|
@@ -36,7 +37,9 @@ Dynamic: license-file
|
|
|
36
37
|
[![Forks][forks-shield]][forks-url]
|
|
37
38
|
[![Stargazers][stars-shield]][stars-url]
|
|
38
39
|
[![Issues][issues-shield]][issues-url]
|
|
40
|
+
|
|
39
41
|
[![GPL v3][license-shield]][license-url]
|
|
42
|
+
[![PyPI][pypi-shield]][pypi-url]
|
|
40
43
|
[![DOI][doi-shield]][doi-url]
|
|
41
44
|
|
|
42
45
|
# MobiSurvStd
|
|
@@ -44,7 +47,6 @@ Dynamic: license-file
|
|
|
44
47
|
*A tool to standardize French mobility survey datasets (EMC², EGT, EMP, etc.) into a unified format.*
|
|
45
48
|
|
|
46
49
|
📚 [Documentation](https://mobisurvstd.github.io/MobiSurvStd)
|
|
47
|
-
📦 [View on PyPI](https://pypi.org/project/mobisurvstd/)
|
|
48
50
|
|
|
49
51
|
<span style="color:blue">🇫🇷 [Version française](./README.fr.md)</span>
|
|
50
52
|
|
|
@@ -164,11 +166,11 @@ For more, check the [User Guide](https://mobisurvstd.github.io/MobiSurvStd/howto
|
|
|
164
166
|
|
|
165
167
|
**Citing MobiSurvStd**
|
|
166
168
|
|
|
167
|
-
If you use MobiSurvStd in a publication or project, please include the following
|
|
169
|
+
If you use MobiSurvStd in a publication or project, please include the following citation:
|
|
168
170
|
```
|
|
169
|
-
Javaudin, L
|
|
170
|
-
doi:10.5281/zenodo.17422994
|
|
171
|
+
Javaudin, L. (2025). MobiSurvStd. Zenodo. https://doi.org/10.5281/zenodo.17710203
|
|
171
172
|
```
|
|
173
|
+
A [CITATION.bib](CITATION.bib) is also available.
|
|
172
174
|
|
|
173
175
|
---
|
|
174
176
|
|
|
@@ -244,5 +246,7 @@ feel free to open an issue on GitHub or to create a Pull Request.
|
|
|
244
246
|
[issues-url]: https://github.com/MobiSurvStd/MobiSurvStd/issues
|
|
245
247
|
[license-shield]: https://img.shields.io/github/license/MobiSurvStd/MobiSurvStd.svg?style=for-the-badge
|
|
246
248
|
[license-url]: https://github.com/MobiSurvStd/MobiSurvStd/blob/master/LICENSE.txt
|
|
247
|
-
[doi-shield]: https://
|
|
248
|
-
[doi-url]: https://doi.org/10.5281/zenodo.
|
|
249
|
+
[doi-shield]: https://img.shields.io/badge/doi-10.5281/zenodo.17422994-blue?style=for-the-badge
|
|
250
|
+
[doi-url]: https://doi.org/10.5281/zenodo.17422993
|
|
251
|
+
[pypi-shield]: https://img.shields.io/pypi/v/mobisurvstd.svg?style=for-the-badge
|
|
252
|
+
[pypi-url]: https://pypi.org/project/mobisurvstd/
|
|
@@ -2,7 +2,9 @@
|
|
|
2
2
|
[![Forks][forks-shield]][forks-url]
|
|
3
3
|
[![Stargazers][stars-shield]][stars-url]
|
|
4
4
|
[![Issues][issues-shield]][issues-url]
|
|
5
|
+
|
|
5
6
|
[![GPL v3][license-shield]][license-url]
|
|
7
|
+
[![PyPI][pypi-shield]][pypi-url]
|
|
6
8
|
[![DOI][doi-shield]][doi-url]
|
|
7
9
|
|
|
8
10
|
# MobiSurvStd
|
|
@@ -10,7 +12,6 @@
|
|
|
10
12
|
*A tool to standardize French mobility survey datasets (EMC², EGT, EMP, etc.) into a unified format.*
|
|
11
13
|
|
|
12
14
|
📚 [Documentation](https://mobisurvstd.github.io/MobiSurvStd)
|
|
13
|
-
📦 [View on PyPI](https://pypi.org/project/mobisurvstd/)
|
|
14
15
|
|
|
15
16
|
<span style="color:blue">🇫🇷 [Version française](./README.fr.md)</span>
|
|
16
17
|
|
|
@@ -130,11 +131,11 @@ For more, check the [User Guide](https://mobisurvstd.github.io/MobiSurvStd/howto
|
|
|
130
131
|
|
|
131
132
|
**Citing MobiSurvStd**
|
|
132
133
|
|
|
133
|
-
If you use MobiSurvStd in a publication or project, please include the following
|
|
134
|
+
If you use MobiSurvStd in a publication or project, please include the following citation:
|
|
134
135
|
```
|
|
135
|
-
Javaudin, L
|
|
136
|
-
doi:10.5281/zenodo.17422994
|
|
136
|
+
Javaudin, L. (2025). MobiSurvStd. Zenodo. https://doi.org/10.5281/zenodo.17710203
|
|
137
137
|
```
|
|
138
|
+
A [CITATION.bib](CITATION.bib) is also available.
|
|
138
139
|
|
|
139
140
|
---
|
|
140
141
|
|
|
@@ -210,5 +211,7 @@ feel free to open an issue on GitHub or to create a Pull Request.
|
|
|
210
211
|
[issues-url]: https://github.com/MobiSurvStd/MobiSurvStd/issues
|
|
211
212
|
[license-shield]: https://img.shields.io/github/license/MobiSurvStd/MobiSurvStd.svg?style=for-the-badge
|
|
212
213
|
[license-url]: https://github.com/MobiSurvStd/MobiSurvStd/blob/master/LICENSE.txt
|
|
213
|
-
[doi-shield]: https://
|
|
214
|
-
[doi-url]: https://doi.org/10.5281/zenodo.
|
|
214
|
+
[doi-shield]: https://img.shields.io/badge/doi-10.5281/zenodo.17422994-blue?style=for-the-badge
|
|
215
|
+
[doi-url]: https://doi.org/10.5281/zenodo.17422993
|
|
216
|
+
[pypi-shield]: https://img.shields.io/pypi/v/mobisurvstd.svg?style=for-the-badge
|
|
217
|
+
[pypi-url]: https://pypi.org/project/mobisurvstd/
|
|
@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
|
|
|
6
6
|
|
|
7
7
|
[project]
|
|
8
8
|
name = "mobisurvstd"
|
|
9
|
-
version = "1.0.
|
|
9
|
+
version = "1.0.2"
|
|
10
10
|
description = "A tool to standardize French mobility survey datasets (EMC2, EGT, EMP, etc.) into a unified format."
|
|
11
11
|
readme = "README.md"
|
|
12
12
|
authors = [{ name = "Lucas Javaudin", email = "lucas@lucasjavaudin.com" }]
|
|
@@ -22,21 +22,29 @@ dependencies = [
|
|
|
22
22
|
"Cartopy >= 0.24.1",
|
|
23
23
|
"fastexcel >= 0.14.0",
|
|
24
24
|
"fsspec>=2025.9.0",
|
|
25
|
-
"geopandas
|
|
25
|
+
"geopandas>=1.0.1",
|
|
26
26
|
"humanize >= 4.13.0",
|
|
27
27
|
"loguru >= 0.7.3",
|
|
28
28
|
"matplotlib >= 3.10.1",
|
|
29
29
|
"numpy >= 2.2.4",
|
|
30
30
|
"pandas >= 2.2.3",
|
|
31
31
|
"platformdirs >= 4.3.8",
|
|
32
|
-
"polars
|
|
32
|
+
"polars>=1.33.0",
|
|
33
33
|
"py7zr >= 0.20.8",
|
|
34
34
|
"pyarrow >= 19.0.1",
|
|
35
35
|
"requests >= 2.32.3",
|
|
36
|
+
"seaborn>=0.13.2",
|
|
36
37
|
"typer >= 0.16.0",
|
|
37
38
|
]
|
|
38
39
|
requires-python = ">=3.10"
|
|
39
40
|
|
|
41
|
+
[dependency-groups]
|
|
42
|
+
dev = [
|
|
43
|
+
"ipython>=8.37.0",
|
|
44
|
+
"pytest>=9.0.2",
|
|
45
|
+
"ty>=0.0.15",
|
|
46
|
+
]
|
|
47
|
+
|
|
40
48
|
[project.urls]
|
|
41
49
|
Homepage = "https://github.com/MobiSurvStd/MobiSurvStd"
|
|
42
50
|
Documentation = "https://mobisurvstd.github.io/MobiSurvStd/index.html"
|
|
@@ -6,6 +6,8 @@ from loguru import logger
|
|
|
6
6
|
from mobisurvstd.common.trips import clean
|
|
7
7
|
from mobisurvstd.schema.common import CURRENT_YEAR
|
|
8
8
|
|
|
9
|
+
from .reader import CeremaReader
|
|
10
|
+
|
|
9
11
|
SCHEMA = {
|
|
10
12
|
"DP1": pl.UInt8, # Code fichier = 3 (déplacement)
|
|
11
13
|
"DMET": pl.UInt8, # Méthode d'enquête du ménage (EMC2 only)
|
|
@@ -114,7 +116,7 @@ def scan_trips_impl(source: str | io.BytesIO):
|
|
|
114
116
|
return pl.scan_csv(source, separator=";", schema_overrides=SCHEMA, null_values=["aa", "aaaaa"])
|
|
115
117
|
|
|
116
118
|
|
|
117
|
-
class TripsReader:
|
|
119
|
+
class TripsReader(CeremaReader):
|
|
118
120
|
def scan_trips(self):
|
|
119
121
|
lfs_iter = map(scan_trips_impl, self.trips_filenames())
|
|
120
122
|
lf = pl.concat(lfs_iter, how="vertical")
|
|
@@ -123,11 +125,13 @@ class TripsReader:
|
|
|
123
125
|
|
|
124
126
|
def standardize_trips(self):
|
|
125
127
|
lf = self.scan_trips()
|
|
126
|
-
# Add household_id, person_id, and trip
|
|
128
|
+
# Add household_id, person_id, trip date, and trip weekday.
|
|
127
129
|
lf = lf.with_columns(
|
|
128
130
|
original_person_id=pl.struct(**self.get_person_index_cols_from_trips())
|
|
129
131
|
).join(
|
|
130
|
-
self.persons.select(
|
|
132
|
+
self.persons.select(
|
|
133
|
+
"original_person_id", "person_id", "household_id", "trip_date", "trip_weekday"
|
|
134
|
+
),
|
|
131
135
|
on="original_person_id",
|
|
132
136
|
how="left",
|
|
133
137
|
coalesce=True,
|
|
@@ -5,13 +5,13 @@ from zipfile import ZipFile
|
|
|
5
5
|
|
|
6
6
|
import geopandas as gpd
|
|
7
7
|
|
|
8
|
-
from mobisurvstd.utils import MissingFileError, find_file
|
|
8
|
+
from mobisurvstd.utils import MissingFileError, find_file, find_file_path
|
|
9
9
|
|
|
10
|
-
from .survey import
|
|
10
|
+
from .survey import CeremaStandardizer
|
|
11
11
|
from .zones import find_matching_column
|
|
12
12
|
|
|
13
13
|
|
|
14
|
-
class EDGTReader(
|
|
14
|
+
class EDGTReader(CeremaStandardizer):
|
|
15
15
|
SURVEY_TYPE = "EDGT"
|
|
16
16
|
|
|
17
17
|
def households_filenames(self):
|
|
@@ -29,53 +29,49 @@ class EDGTReader(CeremaReader):
|
|
|
29
29
|
def special_locations_and_detailed_zones_filenames(self):
|
|
30
30
|
# This should match the Amiens 2010, Clermont-Ferrand 2012, and Lyon 2015 surveys.
|
|
31
31
|
subdir = os.path.join("Doc", "SIG")
|
|
32
|
-
faf_filename =
|
|
33
|
-
self.source, ".*_faf_d.?coupagefin[.](tab|shp|mif)", subdir=subdir
|
|
32
|
+
faf_filename = find_file_path(
|
|
33
|
+
self.source, ".*_faf_d.?coupagefin[.](tab|shp|mif)", subdir=subdir
|
|
34
34
|
)
|
|
35
|
-
tel_filename =
|
|
36
|
-
self.source, ".*_tel_d.?coupagefin[.](tab|shp|mif)", subdir=subdir
|
|
35
|
+
tel_filename = find_file_path(
|
|
36
|
+
self.source, ".*_tel_d.?coupagefin[.](tab|shp|mif)", subdir=subdir
|
|
37
37
|
)
|
|
38
38
|
if faf_filename and tel_filename:
|
|
39
39
|
# Amiens 2010 has two files.
|
|
40
40
|
return [faf_filename, tel_filename]
|
|
41
41
|
else:
|
|
42
42
|
return [
|
|
43
|
-
|
|
43
|
+
find_file_path(
|
|
44
44
|
self.source,
|
|
45
45
|
".*(_zf_gt|zinterne_gen_.*)[.](tab|shp|mif)",
|
|
46
46
|
subdir=subdir,
|
|
47
|
-
as_url=True,
|
|
48
47
|
)
|
|
49
48
|
]
|
|
50
49
|
|
|
51
50
|
def special_locations_filenames(self):
|
|
52
51
|
subdir = os.path.join("Doc", "SIG")
|
|
53
|
-
faf_filename =
|
|
54
|
-
self.source, ".*_faf_.*g.?n.?rateur.*[.](tab|shp|mif)", subdir=subdir
|
|
52
|
+
faf_filename = find_file_path(
|
|
53
|
+
self.source, ".*_faf_.*g.?n.?rateur.*[.](tab|shp|mif)", subdir=subdir
|
|
55
54
|
)
|
|
56
|
-
tel_filename =
|
|
57
|
-
self.source, ".*_tel_.*g.?n.?rateur.*[.](tab|shp|mif)", subdir=subdir
|
|
55
|
+
tel_filename = find_file_path(
|
|
56
|
+
self.source, ".*_tel_.*g.?n.?rateur.*[.](tab|shp|mif)", subdir=subdir
|
|
58
57
|
)
|
|
59
58
|
if faf_filename and tel_filename:
|
|
60
59
|
# Saint-Quentin-en-Yvelines has two files, for phone and face-to-face surveys.
|
|
61
60
|
return [faf_filename, tel_filename]
|
|
62
61
|
else:
|
|
63
62
|
return [
|
|
64
|
-
|
|
65
|
-
self.source,
|
|
66
|
-
".*(_gt_.*|_gt|g.?n.?rateur.*)[.](tab|shp|mif)",
|
|
67
|
-
subdir=subdir,
|
|
68
|
-
as_url=True,
|
|
63
|
+
find_file_path(
|
|
64
|
+
self.source, ".*(_gt_.*|_gt|g.?n.?rateur.*)[.](tab|shp|mif)", subdir=subdir
|
|
69
65
|
)
|
|
70
66
|
]
|
|
71
67
|
|
|
72
68
|
def detailed_zones_filenames(self):
|
|
73
69
|
subdir = os.path.join("Doc", "SIG")
|
|
74
|
-
faf_filename =
|
|
75
|
-
self.source, ".*_faf_zones_fines[.](tab|shp|mif)", subdir=subdir
|
|
70
|
+
faf_filename = find_file_path(
|
|
71
|
+
self.source, ".*_faf_zones_fines[.](tab|shp|mif)", subdir=subdir
|
|
76
72
|
)
|
|
77
|
-
tel_filename =
|
|
78
|
-
self.source, ".*_tel_zones_fines[.](tab|shp|mif)", subdir=subdir
|
|
73
|
+
tel_filename = find_file_path(
|
|
74
|
+
self.source, ".*_tel_zones_fines[.](tab|shp|mif)", subdir=subdir
|
|
79
75
|
)
|
|
80
76
|
if faf_filename and tel_filename:
|
|
81
77
|
# Nice 2009 and Saint-Quentin-en-Yvelines have two files, for phone and face-to-face
|
|
@@ -83,27 +79,29 @@ class EDGTReader(CeremaReader):
|
|
|
83
79
|
return [faf_filename, tel_filename]
|
|
84
80
|
else:
|
|
85
81
|
return [
|
|
86
|
-
|
|
82
|
+
find_file_path(
|
|
87
83
|
self.source,
|
|
88
84
|
".*(_zf|_zf_.*|zf08_.*|(?<!tel_)zones[_ ]fines)[.](tab|shp|mif)",
|
|
89
85
|
subdir=subdir,
|
|
90
|
-
as_url=True,
|
|
91
86
|
)
|
|
92
87
|
]
|
|
93
88
|
|
|
94
89
|
def draw_zones_filenames(self):
|
|
95
90
|
return [
|
|
96
|
-
|
|
91
|
+
find_file_path(
|
|
97
92
|
self.source,
|
|
98
93
|
".*(_dtir|_secteurstirage)[.](tab|shp|mif)",
|
|
99
94
|
subdir=os.path.join("Doc", "SIG"),
|
|
100
|
-
as_url=True,
|
|
101
95
|
)
|
|
102
96
|
]
|
|
103
97
|
|
|
104
98
|
def survey_name(self):
|
|
105
|
-
filename =
|
|
106
|
-
|
|
99
|
+
filename = find_file_path(self.source, ".*_std_faf_men.csv", subdir="Csv")
|
|
100
|
+
fn_match = re.match("(.*)_std_faf_men.csv", os.path.basename(filename))
|
|
101
|
+
if fn_match is not None:
|
|
102
|
+
return fn_match.group(1)
|
|
103
|
+
else:
|
|
104
|
+
return "unkown"
|
|
107
105
|
|
|
108
106
|
def gt_id_columns(self):
|
|
109
107
|
return [
|
|
@@ -4,18 +4,18 @@ import re
|
|
|
4
4
|
|
|
5
5
|
import geopandas as gpd
|
|
6
6
|
|
|
7
|
-
from mobisurvstd.cerema.survey import
|
|
8
|
-
from mobisurvstd.utils import MissingFileError, find_file
|
|
7
|
+
from mobisurvstd.cerema.survey import CeremaStandardizer
|
|
8
|
+
from mobisurvstd.utils import MissingFileError, find_file, find_file_path
|
|
9
9
|
|
|
10
10
|
from .zones import find_matching_column
|
|
11
11
|
|
|
12
12
|
|
|
13
|
-
class EDVMReader(
|
|
13
|
+
class EDVMReader(CeremaStandardizer):
|
|
14
14
|
SURVEY_TYPE = "EDVM"
|
|
15
15
|
|
|
16
16
|
def households_filenames(self) -> list[str | io.BytesIO | MissingFileError]:
|
|
17
17
|
# The filenames are usually stored in "Csv/Fichiers_Standard/[SURVEY_NAME]_std_[NAME].csv"
|
|
18
|
-
# but in some cases (e.g., Ajaccio 2017) the directory "Fichiers_Standard" is
|
|
18
|
+
# but in some cases (e.g., Ajaccio 2017) the directory "Fichiers_Standard" is named based
|
|
19
19
|
# on the survey name (e.g., Ajaccio_2017_Standard).
|
|
20
20
|
return [find_file(self.source, ".*_std_men.csv", subdir="Csv")]
|
|
21
21
|
|
|
@@ -31,47 +31,43 @@ class EDVMReader(CeremaReader):
|
|
|
31
31
|
def special_locations_and_detailed_zones_filenames(self):
|
|
32
32
|
# This should only match the Beauvais 2011 survey.
|
|
33
33
|
return [
|
|
34
|
-
|
|
35
|
-
self.source,
|
|
36
|
-
".*beauvais.*_dfin[.](tab|shp|mif)",
|
|
37
|
-
subdir=os.path.join("Doc", "SIG"),
|
|
38
|
-
as_url=True,
|
|
34
|
+
find_file_path(
|
|
35
|
+
self.source, ".*beauvais.*_dfin[.](tab|shp|mif)", subdir=os.path.join("Doc", "SIG")
|
|
39
36
|
)
|
|
40
37
|
]
|
|
41
38
|
|
|
42
39
|
def detailed_zones_filenames(self):
|
|
43
40
|
return [
|
|
44
|
-
|
|
41
|
+
find_file_path(
|
|
45
42
|
self.source,
|
|
46
43
|
".*(_zf_.*|_zf|zones?[_ ]?fines?.*|_dfin)[.](tab|shp|mif)",
|
|
47
44
|
subdir=os.path.join("Doc", "SIG"),
|
|
48
|
-
as_url=True,
|
|
49
45
|
)
|
|
50
46
|
]
|
|
51
47
|
|
|
52
48
|
def special_locations_filenames(self):
|
|
53
49
|
return [
|
|
54
|
-
|
|
50
|
+
find_file_path(
|
|
55
51
|
self.source,
|
|
56
52
|
".*(_gt_.*|_gt|_pgt|_pg|g.?n.?rateur.*)[.](tab|shp|mif)",
|
|
57
53
|
subdir=os.path.join("Doc", "SIG"),
|
|
58
|
-
as_url=True,
|
|
59
54
|
)
|
|
60
55
|
]
|
|
61
56
|
|
|
62
57
|
def draw_zones_filenames(self):
|
|
63
58
|
return [
|
|
64
|
-
|
|
65
|
-
self.source,
|
|
66
|
-
".*_DTIR[.](tab|shp|mif)",
|
|
67
|
-
subdir=os.path.join("Doc", "SIG"),
|
|
68
|
-
as_url=True,
|
|
59
|
+
find_file_path(
|
|
60
|
+
self.source, ".*_DTIR[.](tab|shp|mif)", subdir=os.path.join("Doc", "SIG")
|
|
69
61
|
)
|
|
70
62
|
]
|
|
71
63
|
|
|
72
64
|
def survey_name(self):
|
|
73
|
-
filename =
|
|
74
|
-
|
|
65
|
+
filename = find_file_path(self.source, ".*_std_men.csv", subdir="Csv")
|
|
66
|
+
fn_match = re.match("(.*)_std_men.csv", os.path.basename(filename))
|
|
67
|
+
if fn_match is not None:
|
|
68
|
+
return fn_match.group(1)
|
|
69
|
+
else:
|
|
70
|
+
return "unkown"
|
|
75
71
|
|
|
76
72
|
def gt_id_columns(self):
|
|
77
73
|
return [
|
|
@@ -277,9 +273,10 @@ class EDVMReader(CeremaReader):
|
|
|
277
273
|
super().select_zf_from_gt_column(gdf)
|
|
278
274
|
|
|
279
275
|
def postprocess_special_locations_and_detailed_zones(
|
|
280
|
-
self, zfs: gpd.GeoDataFrame, gts: gpd.GeoDataFrame
|
|
276
|
+
self, zfs: gpd.GeoDataFrame | None, gts: gpd.GeoDataFrame | None
|
|
281
277
|
):
|
|
282
278
|
# This function should only be call for Beauvais 2011.
|
|
279
|
+
assert gts is not None
|
|
283
280
|
assert "detailed_zone_id" not in gts.columns
|
|
284
281
|
# The corresponding ZF is the GT id with the last digit replaced by 0.
|
|
285
282
|
gts["detailed_zone_id"] = gts["special_location_id"].str.replace(r"\d$", "0", regex=True)
|
|
@@ -4,13 +4,13 @@ import re
|
|
|
4
4
|
|
|
5
5
|
import geopandas as gpd
|
|
6
6
|
|
|
7
|
-
from mobisurvstd.cerema.survey import
|
|
8
|
-
from mobisurvstd.utils import MissingFileError, find_file
|
|
7
|
+
from mobisurvstd.cerema.survey import CeremaStandardizer
|
|
8
|
+
from mobisurvstd.utils import MissingFileError, find_file, find_file_path
|
|
9
9
|
|
|
10
10
|
from .zones import find_matching_column
|
|
11
11
|
|
|
12
12
|
|
|
13
|
-
class EMC2Reader(
|
|
13
|
+
class EMC2Reader(CeremaStandardizer):
|
|
14
14
|
SURVEY_TYPE = "EMC2"
|
|
15
15
|
|
|
16
16
|
def households_filenames(self) -> list[str | io.BytesIO | MissingFileError]:
|
|
@@ -43,42 +43,34 @@ class EMC2Reader(CeremaReader):
|
|
|
43
43
|
|
|
44
44
|
def detailed_zones_filenames(self):
|
|
45
45
|
return [
|
|
46
|
-
|
|
47
|
-
self.source,
|
|
48
|
-
r".*_ZF(_.*)?\.(TAB|shp)",
|
|
49
|
-
subdir=os.path.join("Doc", "SIG"),
|
|
50
|
-
as_url=True,
|
|
46
|
+
find_file_path(
|
|
47
|
+
self.source, r".*_ZF(_.*)?\.(TAB|shp)", subdir=os.path.join("Doc", "SIG")
|
|
51
48
|
)
|
|
52
49
|
]
|
|
53
50
|
|
|
54
51
|
def special_locations_filenames(self):
|
|
55
52
|
return [
|
|
56
|
-
|
|
57
|
-
self.source,
|
|
58
|
-
r".*_GT(_.*)?\.(TAB|shp)",
|
|
59
|
-
subdir=os.path.join("Doc", "SIG"),
|
|
60
|
-
as_url=True,
|
|
53
|
+
find_file_path(
|
|
54
|
+
self.source, r".*_GT(_.*)?\.(TAB|shp)", subdir=os.path.join("Doc", "SIG")
|
|
61
55
|
)
|
|
62
56
|
]
|
|
63
57
|
|
|
64
58
|
def draw_zones_filenames(self):
|
|
65
59
|
return [
|
|
66
|
-
|
|
67
|
-
self.source,
|
|
68
|
-
r".*_DTIR(_.*)?\.(TAB|shp)",
|
|
69
|
-
subdir=os.path.join("Doc", "SIG"),
|
|
70
|
-
as_url=True,
|
|
60
|
+
find_file_path(
|
|
61
|
+
self.source, r".*_DTIR(_.*)?\.(TAB|shp)", subdir=os.path.join("Doc", "SIG")
|
|
71
62
|
)
|
|
72
63
|
]
|
|
73
64
|
|
|
74
65
|
def survey_name(self):
|
|
75
|
-
filename =
|
|
76
|
-
self.source,
|
|
77
|
-
".*_std_men.csv",
|
|
78
|
-
subdir=os.path.join("Csv", "Fichiers_Standard"),
|
|
79
|
-
as_url=True,
|
|
66
|
+
filename = find_file_path(
|
|
67
|
+
self.source, ".*_std_men.csv", subdir=os.path.join("Csv", "Fichiers_Standard")
|
|
80
68
|
)
|
|
81
|
-
|
|
69
|
+
fn_match = re.match("(.*)_std_men.csv", os.path.basename(filename))
|
|
70
|
+
if fn_match is not None:
|
|
71
|
+
return fn_match.group(1)
|
|
72
|
+
else:
|
|
73
|
+
return "Unknown"
|
|
82
74
|
|
|
83
75
|
def gt_id_columns(self):
|
|
84
76
|
return ["zf_160", "zf_fusion", "codegt", "zf", "num_gt"]
|
|
@@ -4,13 +4,13 @@ import re
|
|
|
4
4
|
|
|
5
5
|
import geopandas as gpd
|
|
6
6
|
|
|
7
|
-
from mobisurvstd.cerema.survey import
|
|
8
|
-
from mobisurvstd.utils import MissingFileError, find_file
|
|
7
|
+
from mobisurvstd.cerema.survey import CeremaStandardizer
|
|
8
|
+
from mobisurvstd.utils import MissingFileError, find_file, find_file_path
|
|
9
9
|
|
|
10
10
|
from .zones import find_matching_column
|
|
11
11
|
|
|
12
12
|
|
|
13
|
-
class EMDReader(
|
|
13
|
+
class EMDReader(CeremaStandardizer):
|
|
14
14
|
SURVEY_TYPE = "EMD"
|
|
15
15
|
|
|
16
16
|
def households_filenames(self) -> list[str | io.BytesIO | MissingFileError]:
|
|
@@ -31,47 +31,45 @@ class EMDReader(CeremaReader):
|
|
|
31
31
|
def special_locations_and_detailed_zones_filenames(self):
|
|
32
32
|
# This should match the Valenciennes 2011 and Grenoble 2010 surveys.
|
|
33
33
|
return [
|
|
34
|
-
|
|
34
|
+
find_file_path(
|
|
35
35
|
self.source,
|
|
36
36
|
".*(grenobloise10_zones_fines|zf_gt)[.]mif",
|
|
37
37
|
subdir=os.path.join("Doc", "SIG"),
|
|
38
|
-
as_url=True,
|
|
39
38
|
)
|
|
40
39
|
]
|
|
41
40
|
|
|
42
41
|
def detailed_zones_filenames(self):
|
|
43
42
|
return [
|
|
44
|
-
|
|
43
|
+
find_file_path(
|
|
45
44
|
self.source,
|
|
46
45
|
".*(_zf|zones?[_ ]?fines?.*)[.](tab|mif)",
|
|
47
46
|
subdir=os.path.join("Doc", "SIG"),
|
|
48
|
-
as_url=True,
|
|
49
47
|
)
|
|
50
48
|
]
|
|
51
49
|
|
|
52
50
|
def special_locations_filenames(self):
|
|
53
51
|
return [
|
|
54
|
-
|
|
55
|
-
self.source,
|
|
56
|
-
".*(_gt|g.?n.?rateur.*)[.](tab|mif)",
|
|
57
|
-
subdir=os.path.join("Doc", "SIG"),
|
|
58
|
-
as_url=True,
|
|
52
|
+
find_file_path(
|
|
53
|
+
self.source, ".*(_gt|g.?n.?rateur.*)[.](tab|mif)", subdir=os.path.join("Doc", "SIG")
|
|
59
54
|
)
|
|
60
55
|
]
|
|
61
56
|
|
|
62
57
|
def draw_zones_filenames(self):
|
|
63
58
|
return [
|
|
64
|
-
|
|
59
|
+
find_file_path(
|
|
65
60
|
self.source,
|
|
66
61
|
".*(_DTIR|secteur_.*)[.](tab|shp|mif)",
|
|
67
62
|
subdir=os.path.join("Doc", "SIG"),
|
|
68
|
-
as_url=True,
|
|
69
63
|
)
|
|
70
64
|
]
|
|
71
65
|
|
|
72
66
|
def survey_name(self):
|
|
73
|
-
filename =
|
|
74
|
-
|
|
67
|
+
filename = find_file_path(self.source, ".*_std_men.csv", subdir="Csv")
|
|
68
|
+
fn_match = re.match("(.*)_std_men.csv", os.path.basename(filename))
|
|
69
|
+
if fn_match is not None:
|
|
70
|
+
return fn_match.group(1)
|
|
71
|
+
else:
|
|
72
|
+
return "Unknown"
|
|
75
73
|
|
|
76
74
|
def gt_id_columns(self):
|
|
77
75
|
return [
|
|
@@ -7,6 +7,8 @@ from mobisurvstd.common.households import clean as clean_households
|
|
|
7
7
|
from mobisurvstd.common.motorcycles import clean as clean_motorcycles
|
|
8
8
|
from mobisurvstd.schema.common import CURRENT_YEAR
|
|
9
9
|
|
|
10
|
+
from .reader import CeremaReader
|
|
11
|
+
|
|
10
12
|
SCHEMA = {
|
|
11
13
|
"MP1": pl.UInt8, # Code fichier = 1
|
|
12
14
|
"METH": pl.UInt8, # Méthode d'enquête du ménage (for EMC2 only)
|
|
@@ -230,8 +232,8 @@ def scan_households_impl(source: str | io.BytesIO):
|
|
|
230
232
|
return pl.scan_csv(source, separator=";", schema_overrides=SCHEMA, null_values=["a"])
|
|
231
233
|
|
|
232
234
|
|
|
233
|
-
class HouseholdsReader:
|
|
234
|
-
def scan_households(self):
|
|
235
|
+
class HouseholdsReader(CeremaReader):
|
|
236
|
+
def scan_households(self) -> pl.LazyFrame:
|
|
235
237
|
lfs_iter = map(scan_households_impl, self.households_filenames())
|
|
236
238
|
lf = pl.concat(lfs_iter, how="vertical")
|
|
237
239
|
if "METH" not in lf.collect_schema().names():
|
|
@@ -244,7 +246,7 @@ class HouseholdsReader:
|
|
|
244
246
|
return lf
|
|
245
247
|
|
|
246
248
|
def main_insee(self):
|
|
247
|
-
return self.scan_households().select("IDM4").first().collect().item()
|
|
249
|
+
return self.scan_households().select("IDM4").first().collect().item() # ty: ignore[possibly-missing-attribute]
|
|
248
250
|
|
|
249
251
|
def standardize_households(self):
|
|
250
252
|
lf = self.scan_households()
|
|
@@ -299,6 +301,8 @@ class HouseholdsReader:
|
|
|
299
301
|
how="left",
|
|
300
302
|
coalesce=True,
|
|
301
303
|
)
|
|
304
|
+
# Note: the `.lazy()` after the concat is useless (this is already a LazyFrame) but it helps
|
|
305
|
+
# the syntax checker.
|
|
302
306
|
lf = pl.concat(
|
|
303
307
|
(
|
|
304
308
|
lf.select(
|
|
@@ -326,7 +330,7 @@ class HouseholdsReader:
|
|
|
326
330
|
for i, a in enumerate(("A", "B", "C", "D"))
|
|
327
331
|
),
|
|
328
332
|
how="vertical",
|
|
329
|
-
)
|
|
333
|
+
).lazy()
|
|
330
334
|
# Drop the lines with empty car characteristics (there are always 4 cars per
|
|
331
335
|
# households even when the household has less than 4 cars).
|
|
332
336
|
lf = lf.filter(
|
|
@@ -347,6 +351,8 @@ class HouseholdsReader:
|
|
|
347
351
|
how="left",
|
|
348
352
|
coalesce=True,
|
|
349
353
|
)
|
|
354
|
+
# Note: the `.lazy()` after the concat is useless (this is already a LazyFrame) but it helps
|
|
355
|
+
# the syntax checker.
|
|
350
356
|
lf = pl.concat(
|
|
351
357
|
(
|
|
352
358
|
lf.select(
|
|
@@ -388,7 +394,7 @@ class HouseholdsReader:
|
|
|
388
394
|
for i, a in enumerate(("A", "B", "C", "D"))
|
|
389
395
|
),
|
|
390
396
|
how="vertical",
|
|
391
|
-
)
|
|
397
|
+
).lazy()
|
|
392
398
|
lf = lf.filter(
|
|
393
399
|
pl.any_horizontal(
|
|
394
400
|
pl.all()
|
|
@@ -2,13 +2,14 @@ import io
|
|
|
2
2
|
|
|
3
3
|
import pandas as pd
|
|
4
4
|
import polars as pl
|
|
5
|
+
from polars.datatypes import DataTypeClass
|
|
5
6
|
|
|
6
7
|
from mobisurvstd.utils import find_file
|
|
7
8
|
|
|
8
9
|
from .deplacements import SCHEMA as TRIP_SCHEMA
|
|
9
10
|
from .menages import SCHEMA as HOUSEHOLD_SCHEMA
|
|
10
11
|
from .personnes import SCHEMA as PERSON_SCHEMA
|
|
11
|
-
from .survey import
|
|
12
|
+
from .survey import CeremaStandardizer
|
|
12
13
|
from .trajets import SCHEMA as LEG_SCHEMA
|
|
13
14
|
|
|
14
15
|
# The anonimized open-data format for Nantes uses a fixed-width text format.
|
|
@@ -140,7 +141,7 @@ LEG_FORMAT = [
|
|
|
140
141
|
]
|
|
141
142
|
|
|
142
143
|
|
|
143
|
-
class OpenDataReader(
|
|
144
|
+
class OpenDataReader(CeremaStandardizer):
|
|
144
145
|
SURVEY_TYPE = "EDGT-opendata"
|
|
145
146
|
|
|
146
147
|
def households_filenames(self):
|
|
@@ -193,7 +194,7 @@ class OpenDataReader(CeremaReader):
|
|
|
193
194
|
def scan_fwf(
|
|
194
195
|
f: str | io.BytesIO,
|
|
195
196
|
format: list[tuple[int, int, str]],
|
|
196
|
-
schema: dict[str,
|
|
197
|
+
schema: dict[str, DataTypeClass],
|
|
197
198
|
):
|
|
198
199
|
# Decrease `start` by 1 because pandas starts counting at 0 but the survey document starts
|
|
199
200
|
# counting at 1.
|