mobisurvstd 1.0.2__tar.gz → 1.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {mobisurvstd-1.0.2/src/mobisurvstd.egg-info → mobisurvstd-1.2.0}/PKG-INFO +6 -8
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/README.md +2 -1
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/pyproject.toml +22 -16
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/cerema/__init__.py +4 -2
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/cerema/deplacements.py +7 -2
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/cerema/edgt.py +1 -3
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/cerema/emc2.py +5 -23
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/cerema/menages.py +14 -33
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/cerema/nantes_opendata.py +1 -3
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/cerema/personnes.py +7 -1
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/cerema/survey.py +4 -6
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/cerema/trajets.py +4 -2
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/classes.py +19 -12
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/cli.py +4 -1
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/common/clean.py +28 -16
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/common/households.py +4 -2
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/common/trips.py +8 -5
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/common/zones.py +1 -7
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/egt2010/deplacements.py +2 -0
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/egt2010/menages.py +6 -7
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/egt2010/personnes.py +1 -5
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/egt2010/survey.py +3 -3
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/egt2020/deplacements.py +34 -27
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/egt2020/menages.py +17 -12
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/egt2020/motos.py +3 -10
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/egt2020/personnes.py +27 -29
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/egt2020/survey.py +21 -9
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/egt2020/trajets.py +23 -19
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/egt2020/voitures.py +2 -1
- mobisurvstd-1.2.0/src/mobisurvstd/emg2023/deplacements.py +251 -0
- mobisurvstd-1.2.0/src/mobisurvstd/emg2023/menages.py +24 -0
- mobisurvstd-1.2.0/src/mobisurvstd/emg2023/personnes.py +125 -0
- mobisurvstd-1.2.0/src/mobisurvstd/emg2023/survey.py +65 -0
- mobisurvstd-1.2.0/src/mobisurvstd/emp/__init__.py +1 -0
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/emp/common.py +9 -0
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/emp/deplacements.py +7 -9
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/emp/menages.py +4 -0
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/emp/motos.py +4 -26
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/emp/personnes.py +1 -4
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/emp/survey.py +1 -2
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/emp/voitures.py +5 -39
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/main.py +50 -12
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/resources/insee_data.py +155 -38
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/schema/cars.py +2 -14
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/schema/common.py +3 -6
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/schema/guarantees.py +33 -9
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/schema/households.py +10 -4
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/schema/motorcycles.py +1 -9
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/schema/persons.py +22 -75
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/schema/trips.py +20 -3
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/utils.py +18 -5
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/validation.py +49 -42
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0/src/mobisurvstd.egg-info}/PKG-INFO +6 -8
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd.egg-info/SOURCES.txt +5 -0
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd.egg-info/requires.txt +3 -6
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/LICENSE.md +0 -0
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/setup.cfg +0 -0
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/__init__.py +0 -0
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/__main__.py +0 -0
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/cerema/common.py +0 -0
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/cerema/edvm.py +0 -0
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/cerema/emd.py +0 -0
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/cerema/reader.py +0 -0
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/cerema/zones.py +0 -0
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/common/__init__.py +0 -0
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/common/cars.py +0 -0
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/common/legs.py +0 -0
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/common/motorcycles.py +0 -0
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/common/persons.py +0 -0
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/egt2010/__init__.py +0 -0
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/egt2010/trajets.py +0 -0
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/egt2010/zones.py +0 -0
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/egt2020/__init__.py +0 -0
- {mobisurvstd-1.0.2/src/mobisurvstd/emp → mobisurvstd-1.2.0/src/mobisurvstd/emg2023}/__init__.py +0 -0
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/logger.py +0 -0
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/resources/__init__.py +0 -0
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/resources/admin_express.py +0 -0
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/resources/nuts.py +0 -0
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/schema/__init__.py +0 -0
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/schema/legs.py +0 -0
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd.egg-info/dependency_links.txt +0 -0
- {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: mobisurvstd
|
|
3
|
-
Version: 1.0
|
|
3
|
+
Version: 1.2.0
|
|
4
4
|
Summary: A tool to standardize French mobility survey datasets (EMC2, EGT, EMP, etc.) into a unified format.
|
|
5
5
|
Author-email: Lucas Javaudin <lucas@lucasjavaudin.com>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -15,21 +15,18 @@ Classifier: Topic :: File Formats
|
|
|
15
15
|
Requires-Python: >=3.10
|
|
16
16
|
Description-Content-Type: text/markdown
|
|
17
17
|
License-File: LICENSE.md
|
|
18
|
-
Requires-Dist: Cartopy>=0.24.1
|
|
19
18
|
Requires-Dist: fastexcel>=0.14.0
|
|
20
|
-
Requires-Dist: fsspec>=2025.9.0
|
|
21
19
|
Requires-Dist: geopandas>=1.0.1
|
|
22
20
|
Requires-Dist: humanize>=4.13.0
|
|
23
21
|
Requires-Dist: loguru>=0.7.3
|
|
24
|
-
Requires-Dist: matplotlib>=3.10.1
|
|
25
22
|
Requires-Dist: numpy>=2.2.4
|
|
26
23
|
Requires-Dist: pandas>=2.2.3
|
|
27
24
|
Requires-Dist: platformdirs>=4.3.8
|
|
28
|
-
Requires-Dist: polars>=1.
|
|
29
|
-
Requires-Dist: py7zr>=
|
|
25
|
+
Requires-Dist: polars>=1.38.0
|
|
26
|
+
Requires-Dist: py7zr>=1.1.0
|
|
30
27
|
Requires-Dist: pyarrow>=19.0.1
|
|
31
28
|
Requires-Dist: requests>=2.32.3
|
|
32
|
-
Requires-Dist:
|
|
29
|
+
Requires-Dist: tqdm>=4.67.3
|
|
33
30
|
Requires-Dist: typer>=0.16.0
|
|
34
31
|
Dynamic: license-file
|
|
35
32
|
|
|
@@ -202,6 +199,7 @@ Currently, MobiSurvStd supports the following survey types:
|
|
|
202
199
|
|
|
203
200
|
- `emp2019`
|
|
204
201
|
- `emc2`
|
|
202
|
+
- `emg2023` (thanks to @sebhorl)
|
|
205
203
|
- `egt2020`
|
|
206
204
|
- `egt2010`
|
|
207
205
|
- `edgt` (including the [open-data version of Nantes 2015](https://www.data.gouv.fr/datasets/enquete-deplacements-2015-en-loire-atlantique))
|
|
@@ -210,7 +208,7 @@ Currently, MobiSurvStd supports the following survey types:
|
|
|
210
208
|
|
|
211
209
|
See [Survey Types](https://mobisurvstd.github.io/MobiSurvStd/surveys.html) for more.
|
|
212
210
|
|
|
213
|
-
Other survey types that are planned to be integrated are: `entd
|
|
211
|
+
Other survey types that are planned to be integrated are: `entd`.
|
|
214
212
|
If you know another survey format that could be integrated, feel free to open an issue on GitHub.
|
|
215
213
|
|
|
216
214
|
MobiSurvStd covers only French mobility survey formats.
|
|
@@ -167,6 +167,7 @@ Currently, MobiSurvStd supports the following survey types:
|
|
|
167
167
|
|
|
168
168
|
- `emp2019`
|
|
169
169
|
- `emc2`
|
|
170
|
+
- `emg2023` (thanks to @sebhorl)
|
|
170
171
|
- `egt2020`
|
|
171
172
|
- `egt2010`
|
|
172
173
|
- `edgt` (including the [open-data version of Nantes 2015](https://www.data.gouv.fr/datasets/enquete-deplacements-2015-en-loire-atlantique))
|
|
@@ -175,7 +176,7 @@ Currently, MobiSurvStd supports the following survey types:
|
|
|
175
176
|
|
|
176
177
|
See [Survey Types](https://mobisurvstd.github.io/MobiSurvStd/surveys.html) for more.
|
|
177
178
|
|
|
178
|
-
Other survey types that are planned to be integrated are: `entd
|
|
179
|
+
Other survey types that are planned to be integrated are: `entd`.
|
|
179
180
|
If you know another survey format that could be integrated, feel free to open an issue on GitHub.
|
|
180
181
|
|
|
181
182
|
MobiSurvStd covers only French mobility survey formats.
|
|
@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
|
|
|
6
6
|
|
|
7
7
|
[project]
|
|
8
8
|
name = "mobisurvstd"
|
|
9
|
-
version = "1.0
|
|
9
|
+
version = "1.2.0"
|
|
10
10
|
description = "A tool to standardize French mobility survey datasets (EMC2, EGT, EMP, etc.) into a unified format."
|
|
11
11
|
readme = "README.md"
|
|
12
12
|
authors = [{ name = "Lucas Javaudin", email = "lucas@lucasjavaudin.com" }]
|
|
@@ -19,29 +19,30 @@ classifiers = [
|
|
|
19
19
|
]
|
|
20
20
|
keywords = ["mobility survey", "polars", "data format"]
|
|
21
21
|
dependencies = [
|
|
22
|
-
"
|
|
23
|
-
"fastexcel >= 0.14.0",
|
|
24
|
-
"fsspec>=2025.9.0",
|
|
22
|
+
"fastexcel>=0.14.0",
|
|
25
23
|
"geopandas>=1.0.1",
|
|
26
|
-
"humanize
|
|
27
|
-
"loguru
|
|
28
|
-
"
|
|
29
|
-
"
|
|
30
|
-
"
|
|
31
|
-
"
|
|
32
|
-
"
|
|
33
|
-
"
|
|
34
|
-
"
|
|
35
|
-
"
|
|
36
|
-
"
|
|
37
|
-
"typer >= 0.16.0",
|
|
24
|
+
"humanize>=4.13.0",
|
|
25
|
+
"loguru>=0.7.3",
|
|
26
|
+
"numpy>=2.2.4",
|
|
27
|
+
"pandas>=2.2.3",
|
|
28
|
+
"platformdirs>=4.3.8",
|
|
29
|
+
"polars>=1.38.0",
|
|
30
|
+
"py7zr>=1.1.0",
|
|
31
|
+
"pyarrow>=19.0.1",
|
|
32
|
+
"requests>=2.32.3",
|
|
33
|
+
"tqdm>=4.67.3",
|
|
34
|
+
"typer>=0.16.0",
|
|
38
35
|
]
|
|
39
36
|
requires-python = ">=3.10"
|
|
40
37
|
|
|
41
38
|
[dependency-groups]
|
|
42
39
|
dev = [
|
|
40
|
+
"Cartopy>=0.24.1",
|
|
41
|
+
"folium>=0.20.0",
|
|
43
42
|
"ipython>=8.37.0",
|
|
43
|
+
"matplotlib>=3.10.1",
|
|
44
44
|
"pytest>=9.0.2",
|
|
45
|
+
"seaborn>=0.13.2",
|
|
45
46
|
"ty>=0.0.15",
|
|
46
47
|
]
|
|
47
48
|
|
|
@@ -50,3 +51,8 @@ Homepage = "https://github.com/MobiSurvStd/MobiSurvStd"
|
|
|
50
51
|
Documentation = "https://mobisurvstd.github.io/MobiSurvStd/index.html"
|
|
51
52
|
Repository = "https://github.com/MobiSurvStd/MobiSurvStd"
|
|
52
53
|
Issues = "https://github.com/MobiSurvStd/MobiSurvStd/issues"
|
|
54
|
+
|
|
55
|
+
[tool.ruff]
|
|
56
|
+
line-length = 100
|
|
57
|
+
[tool.ruff.format]
|
|
58
|
+
skip-magic-trailing-comma = true
|
|
@@ -7,7 +7,9 @@ from .emd import EMDReader
|
|
|
7
7
|
from .nantes_opendata import OpenDataReader
|
|
8
8
|
|
|
9
9
|
|
|
10
|
-
def standardize(
|
|
10
|
+
def standardize(
|
|
11
|
+
source: str | ZipFile, survey_type: str, skip_spatial: bool = False, skip_insee: bool = False
|
|
12
|
+
):
|
|
11
13
|
if survey_type == "EMC2":
|
|
12
14
|
reader = EMC2Reader(source)
|
|
13
15
|
elif survey_type == "EDGT":
|
|
@@ -20,4 +22,4 @@ def standardize(source: str | ZipFile, survey_type: str, skip_spatial: bool = Fa
|
|
|
20
22
|
reader = EMDReader(source)
|
|
21
23
|
else:
|
|
22
24
|
raise NotImplementedError
|
|
23
|
-
return reader.standardize(skip_spatial)
|
|
25
|
+
return reader.standardize(skip_spatial, skip_insee)
|
|
@@ -5,6 +5,7 @@ from loguru import logger
|
|
|
5
5
|
|
|
6
6
|
from mobisurvstd.common.trips import clean
|
|
7
7
|
from mobisurvstd.schema.common import CURRENT_YEAR
|
|
8
|
+
from mobisurvstd.utils import detect_csv_delimiter
|
|
8
9
|
|
|
9
10
|
from .reader import CeremaReader
|
|
10
11
|
|
|
@@ -113,7 +114,10 @@ TRIP_PERIMETER_MAP = {
|
|
|
113
114
|
|
|
114
115
|
|
|
115
116
|
def scan_trips_impl(source: str | io.BytesIO):
|
|
116
|
-
|
|
117
|
+
separator = detect_csv_delimiter(source)
|
|
118
|
+
return pl.scan_csv(
|
|
119
|
+
source, separator=separator, schema_overrides=SCHEMA, null_values=["aa", "aaaaa"]
|
|
120
|
+
)
|
|
117
121
|
|
|
118
122
|
|
|
119
123
|
class TripsReader(CeremaReader):
|
|
@@ -123,7 +127,7 @@ class TripsReader(CeremaReader):
|
|
|
123
127
|
lf = lf.sort(self.get_trip_index_cols())
|
|
124
128
|
return lf
|
|
125
129
|
|
|
126
|
-
def standardize_trips(self):
|
|
130
|
+
def standardize_trips(self, skip_insee: bool = False):
|
|
127
131
|
lf = self.scan_trips()
|
|
128
132
|
# Add household_id, person_id, trip date, and trip weekday.
|
|
129
133
|
lf = lf.with_columns(
|
|
@@ -203,6 +207,7 @@ class TripsReader(CeremaReader):
|
|
|
203
207
|
year=self.survey_year() or CURRENT_YEAR,
|
|
204
208
|
special_locations=self.special_locations_coords,
|
|
205
209
|
detailed_zones=self.detailed_zones_coords,
|
|
210
|
+
skip_insee=skip_insee,
|
|
206
211
|
)
|
|
207
212
|
# When the INSEE code ends with "000" or "999" it means "rest of the département".
|
|
208
213
|
# We drop these values because they do not add any additional information compared to `_dep`
|
|
@@ -14,32 +14,16 @@ class EMC2Reader(CeremaStandardizer):
|
|
|
14
14
|
SURVEY_TYPE = "EMC2"
|
|
15
15
|
|
|
16
16
|
def households_filenames(self) -> list[str | io.BytesIO | MissingFileError]:
|
|
17
|
-
return [
|
|
18
|
-
find_file(
|
|
19
|
-
self.source, ".*_std_men.csv", subdir=os.path.join("Csv", "Fichiers_Standard")
|
|
20
|
-
)
|
|
21
|
-
]
|
|
17
|
+
return [find_file(self.source, ".*_std_men.csv")]
|
|
22
18
|
|
|
23
19
|
def persons_filenames(self) -> list[str | io.BytesIO | MissingFileError]:
|
|
24
|
-
return [
|
|
25
|
-
find_file(
|
|
26
|
-
self.source, ".*_std_pers.csv", subdir=os.path.join("Csv", "Fichiers_Standard")
|
|
27
|
-
)
|
|
28
|
-
]
|
|
20
|
+
return [find_file(self.source, ".*_std_pers.csv")]
|
|
29
21
|
|
|
30
22
|
def trips_filenames(self) -> list[str | io.BytesIO | MissingFileError]:
|
|
31
|
-
return [
|
|
32
|
-
find_file(
|
|
33
|
-
self.source, ".*_std_depl.csv", subdir=os.path.join("Csv", "Fichiers_Standard")
|
|
34
|
-
)
|
|
35
|
-
]
|
|
23
|
+
return [find_file(self.source, ".*_std_depl.csv")]
|
|
36
24
|
|
|
37
25
|
def legs_filenames(self) -> list[str | io.BytesIO | MissingFileError]:
|
|
38
|
-
return [
|
|
39
|
-
find_file(
|
|
40
|
-
self.source, ".*_std_traj.csv", subdir=os.path.join("Csv", "Fichiers_Standard")
|
|
41
|
-
)
|
|
42
|
-
]
|
|
26
|
+
return [find_file(self.source, ".*_std_traj.csv")]
|
|
43
27
|
|
|
44
28
|
def detailed_zones_filenames(self):
|
|
45
29
|
return [
|
|
@@ -63,9 +47,7 @@ class EMC2Reader(CeremaStandardizer):
|
|
|
63
47
|
]
|
|
64
48
|
|
|
65
49
|
def survey_name(self):
|
|
66
|
-
filename = find_file_path(
|
|
67
|
-
self.source, ".*_std_men.csv", subdir=os.path.join("Csv", "Fichiers_Standard")
|
|
68
|
-
)
|
|
50
|
+
filename = find_file_path(self.source, ".*_std_men.csv")
|
|
69
51
|
fn_match = re.match("(.*)_std_men.csv", os.path.basename(filename))
|
|
70
52
|
if fn_match is not None:
|
|
71
53
|
return fn_match.group(1)
|
|
@@ -6,6 +6,7 @@ from mobisurvstd.common.cars import clean as clean_cars
|
|
|
6
6
|
from mobisurvstd.common.households import clean as clean_households
|
|
7
7
|
from mobisurvstd.common.motorcycles import clean as clean_motorcycles
|
|
8
8
|
from mobisurvstd.schema.common import CURRENT_YEAR
|
|
9
|
+
from mobisurvstd.utils import detect_csv_delimiter
|
|
9
10
|
|
|
10
11
|
from .reader import CeremaReader
|
|
11
12
|
|
|
@@ -92,10 +93,7 @@ SCHEMA = {
|
|
|
92
93
|
"COE0": pl.Float64, # COEFFICIENT DE REDRESSEMENT MENAGE
|
|
93
94
|
}
|
|
94
95
|
|
|
95
|
-
SURVEY_METHOD_MAP = {
|
|
96
|
-
1: "face_to_face",
|
|
97
|
-
2: "phone",
|
|
98
|
-
}
|
|
96
|
+
SURVEY_METHOD_MAP = {1: "face_to_face", 2: "phone"}
|
|
99
97
|
|
|
100
98
|
HOUSING_TYPE_MAP = {
|
|
101
99
|
1: "house", # Individuel isolé
|
|
@@ -165,6 +163,7 @@ PARKING_LOCATION_MAP = {
|
|
|
165
163
|
3: "parking_lot", # Dans un parc à ciel ouvert (ou place publique)
|
|
166
164
|
4: "parking_lot", # Dans un parc couvert accessible au public
|
|
167
165
|
5: "other", # Autre (egt)
|
|
166
|
+
9: "other", # Autre (egt)
|
|
168
167
|
}
|
|
169
168
|
|
|
170
169
|
PARKING_TYPE_MAP = {
|
|
@@ -173,6 +172,7 @@ PARKING_TYPE_MAP = {
|
|
|
173
172
|
3: "paid", # Payant à votre charge
|
|
174
173
|
4: "paid_by_other", # Payant à la charge de quelqu'un d'autre
|
|
175
174
|
5: "other", # Autre (egt)
|
|
175
|
+
9: "other", # Autre (egt)
|
|
176
176
|
}
|
|
177
177
|
|
|
178
178
|
MOTORCYCLE_TYPE_MAP = {
|
|
@@ -195,41 +195,20 @@ MOTORCYCLE_FUEL_TYPE_MAP = {
|
|
|
195
195
|
90: "other",
|
|
196
196
|
}
|
|
197
197
|
|
|
198
|
-
MOTORCYCLE_SIZE_LOWERBOUND_MAP = {
|
|
199
|
-
1: 0,
|
|
200
|
-
2: 50,
|
|
201
|
-
3: 125,
|
|
202
|
-
4: 250,
|
|
203
|
-
5: 750,
|
|
204
|
-
9: 125,
|
|
205
|
-
}
|
|
198
|
+
MOTORCYCLE_SIZE_LOWERBOUND_MAP = {1: 0, 2: 50, 3: 125, 4: 250, 5: 750, 9: 125}
|
|
206
199
|
|
|
207
|
-
MOTORCYCLE_SIZE_UPPERBOUND_MAP = {
|
|
208
|
-
1: 50,
|
|
209
|
-
2: 125,
|
|
210
|
-
3: 250,
|
|
211
|
-
4: 750,
|
|
212
|
-
}
|
|
200
|
+
MOTORCYCLE_SIZE_UPPERBOUND_MAP = {1: 50, 2: 125, 3: 250, 4: 750}
|
|
213
201
|
|
|
214
|
-
MOTORCYCLE_KW_LOWERBOUND_MAP = {
|
|
215
|
-
6: 0,
|
|
216
|
-
7: 4,
|
|
217
|
-
8: 11,
|
|
218
|
-
}
|
|
202
|
+
MOTORCYCLE_KW_LOWERBOUND_MAP = {6: 0, 7: 4, 8: 11}
|
|
219
203
|
|
|
220
|
-
MOTORCYCLE_KW_UPPERBOUND_MAP = {
|
|
221
|
-
6: 4,
|
|
222
|
-
7: 11,
|
|
223
|
-
}
|
|
204
|
+
MOTORCYCLE_KW_UPPERBOUND_MAP = {6: 4, 7: 11}
|
|
224
205
|
|
|
225
|
-
MOTORCYCLE_THERMIC_ENGINE_TYPE_MAP = {
|
|
226
|
-
1: "two_stroke",
|
|
227
|
-
2: "four_stroke",
|
|
228
|
-
}
|
|
206
|
+
MOTORCYCLE_THERMIC_ENGINE_TYPE_MAP = {1: "two_stroke", 2: "four_stroke"}
|
|
229
207
|
|
|
230
208
|
|
|
231
209
|
def scan_households_impl(source: str | io.BytesIO):
|
|
232
|
-
|
|
210
|
+
separator = detect_csv_delimiter(source)
|
|
211
|
+
return pl.scan_csv(source, separator=separator, schema_overrides=SCHEMA, null_values=["a"])
|
|
233
212
|
|
|
234
213
|
|
|
235
214
|
class HouseholdsReader(CeremaReader):
|
|
@@ -248,7 +227,7 @@ class HouseholdsReader(CeremaReader):
|
|
|
248
227
|
def main_insee(self):
|
|
249
228
|
return self.scan_households().select("IDM4").first().collect().item() # ty: ignore[possibly-missing-attribute]
|
|
250
229
|
|
|
251
|
-
def standardize_households(self):
|
|
230
|
+
def standardize_households(self, skip_insee: bool = False):
|
|
252
231
|
lf = self.scan_households()
|
|
253
232
|
lf = lf.rename(
|
|
254
233
|
{
|
|
@@ -262,6 +241,7 @@ class HouseholdsReader(CeremaReader):
|
|
|
262
241
|
)
|
|
263
242
|
lf = lf.with_columns(
|
|
264
243
|
original_household_id=pl.struct(self.get_household_index_cols()),
|
|
244
|
+
complete_household=True,
|
|
265
245
|
home_detailed_zone="ZFM",
|
|
266
246
|
home_draw_zone="STM",
|
|
267
247
|
# For EMD, EDVM, EDGT, the IDM2 column will be read here.
|
|
@@ -278,6 +258,7 @@ class HouseholdsReader(CeremaReader):
|
|
|
278
258
|
year=self.survey_year() or CURRENT_YEAR,
|
|
279
259
|
special_locations=self.special_locations_coords,
|
|
280
260
|
detailed_zones=self.detailed_zones_coords,
|
|
261
|
+
skip_insee=skip_insee,
|
|
281
262
|
)
|
|
282
263
|
# When the INSEE code ends with "000" or "999" it means "rest of the département".
|
|
283
264
|
# We drop these values because they do not add any additional information compared to `_dep`
|
|
@@ -192,9 +192,7 @@ class OpenDataReader(CeremaStandardizer):
|
|
|
192
192
|
|
|
193
193
|
|
|
194
194
|
def scan_fwf(
|
|
195
|
-
f: str | io.BytesIO,
|
|
196
|
-
format: list[tuple[int, int, str]],
|
|
197
|
-
schema: dict[str, DataTypeClass],
|
|
195
|
+
f: str | io.BytesIO, format: list[tuple[int, int, str]], schema: dict[str, DataTypeClass]
|
|
198
196
|
):
|
|
199
197
|
# Decrease `start` by 1 because pandas starts counting at 0 but the survey document starts
|
|
200
198
|
# counting at 1.
|
|
@@ -3,6 +3,7 @@ import io
|
|
|
3
3
|
import polars as pl
|
|
4
4
|
|
|
5
5
|
from mobisurvstd.common.persons import clean
|
|
6
|
+
from mobisurvstd.utils import detect_csv_delimiter
|
|
6
7
|
|
|
7
8
|
from .reader import CeremaReader
|
|
8
9
|
|
|
@@ -232,7 +233,10 @@ WEEKDAY_MAP = {
|
|
|
232
233
|
|
|
233
234
|
|
|
234
235
|
def scan_persons_impl(source: str | io.BytesIO):
|
|
235
|
-
|
|
236
|
+
separator = detect_csv_delimiter(source)
|
|
237
|
+
return pl.scan_csv(
|
|
238
|
+
source, separator=separator, schema_overrides=SCHEMA, null_values=["a", "aa"]
|
|
239
|
+
)
|
|
236
240
|
|
|
237
241
|
|
|
238
242
|
class PersonsReader(CeremaReader):
|
|
@@ -240,6 +244,8 @@ class PersonsReader(CeremaReader):
|
|
|
240
244
|
lfs_iter = map(scan_persons_impl, self.persons_filenames())
|
|
241
245
|
lf = pl.concat(lfs_iter, how="vertical")
|
|
242
246
|
lf = lf.sort(self.get_person_index_cols())
|
|
247
|
+
# Set empty strings and whitespace-only strings as NULL values.
|
|
248
|
+
lf = lf.with_columns(pl.col(pl.String).str.strip_chars().replace("", None))
|
|
243
249
|
columns = lf.collect_schema().names()
|
|
244
250
|
if "P13B" not in columns:
|
|
245
251
|
# In the old Cerema surveys (EMD, EDGT, EDVM), the P13B column does not exist.
|
|
@@ -26,7 +26,7 @@ LOCATION_COLUMNS = (
|
|
|
26
26
|
class CeremaStandardizer(HouseholdsReader, PersonsReader, TripsReader, LegsReader, ZonesReader):
|
|
27
27
|
SURVEY_TYPE = ""
|
|
28
28
|
|
|
29
|
-
def standardize(self, skip_spatial: bool = False):
|
|
29
|
+
def standardize(self, skip_spatial: bool = False, skip_insee: bool = False):
|
|
30
30
|
logger.info(f"Standardizing {self.SURVEY_TYPE} survey from `{self.source_name()}`")
|
|
31
31
|
if not self.validate():
|
|
32
32
|
return None
|
|
@@ -34,11 +34,11 @@ class CeremaStandardizer(HouseholdsReader, PersonsReader, TripsReader, LegsReade
|
|
|
34
34
|
if not skip_spatial:
|
|
35
35
|
self.read_spatial_data()
|
|
36
36
|
|
|
37
|
-
self.standardize_households()
|
|
37
|
+
self.standardize_households(skip_insee)
|
|
38
38
|
self.standardize_cars()
|
|
39
39
|
self.standardize_motorcycles()
|
|
40
40
|
self.standardize_persons()
|
|
41
|
-
self.standardize_trips()
|
|
41
|
+
self.standardize_trips(skip_insee)
|
|
42
42
|
self.standardize_legs()
|
|
43
43
|
|
|
44
44
|
self.finish()
|
|
@@ -115,9 +115,7 @@ class CeremaStandardizer(HouseholdsReader, PersonsReader, TripsReader, LegsReade
|
|
|
115
115
|
) # ty: ignore[invalid-assignment]
|
|
116
116
|
self.households = self.households.join(
|
|
117
117
|
household_dates.lazy(), on="household_id", how="left", coalesce=True
|
|
118
|
-
).with_columns(
|
|
119
|
-
trips_weekday=pl.when("is_valid_weekday").then(pl.col("trips_weekday")),
|
|
120
|
-
)
|
|
118
|
+
).with_columns(trips_weekday=pl.when("is_valid_weekday").then(pl.col("trips_weekday")))
|
|
121
119
|
|
|
122
120
|
def fix_main_mode(self):
|
|
123
121
|
# Special case for Douai 2012: Some trips have main mode set to "car_driver" but the legs
|
|
@@ -4,6 +4,7 @@ import polars as pl
|
|
|
4
4
|
from loguru import logger
|
|
5
5
|
|
|
6
6
|
from mobisurvstd.common.legs import clean
|
|
7
|
+
from mobisurvstd.utils import detect_csv_delimiter
|
|
7
8
|
|
|
8
9
|
from .reader import CeremaReader
|
|
9
10
|
|
|
@@ -93,8 +94,9 @@ MOTORCYCLE_MAP = {
|
|
|
93
94
|
|
|
94
95
|
|
|
95
96
|
def scan_legs_impl(source: str | io.BytesIO):
|
|
97
|
+
separator = detect_csv_delimiter(source)
|
|
96
98
|
return pl.scan_csv(
|
|
97
|
-
source, separator=
|
|
99
|
+
source, separator=separator, schema_overrides=SCHEMA, null_values=["a", "aa", "aaaaa"]
|
|
98
100
|
)
|
|
99
101
|
|
|
100
102
|
|
|
@@ -342,7 +344,7 @@ class LegsReader(CeremaReader):
|
|
|
342
344
|
.then(pl.col("nb_majors_in_vehicle") + 1)
|
|
343
345
|
.otherwise("nb_majors_in_vehicle"),
|
|
344
346
|
nb_minors_in_vehicle=pl.when(
|
|
345
|
-
pl.col("is_major").not_() & pl.col("mode").str.contains("driver")
|
|
347
|
+
pl.col("is_major").not_() & pl.col("mode").str.contains("driver")
|
|
346
348
|
)
|
|
347
349
|
.then(pl.col("nb_minors_in_vehicle") + 1)
|
|
348
350
|
.otherwise("nb_minors_in_vehicle"),
|
|
@@ -67,16 +67,19 @@ class SurveyData:
|
|
|
67
67
|
logger.warning("Output directory is not empty, some data might be erased")
|
|
68
68
|
logger.debug("Saving households")
|
|
69
69
|
self.households.write_parquet(output_directory / Path("households.parquet"))
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
self.motorcycles.
|
|
70
|
+
if not self.cars.is_empty():
|
|
71
|
+
logger.debug("Saving cars")
|
|
72
|
+
self.cars.write_parquet(output_directory / Path("cars.parquet"))
|
|
73
|
+
if not self.motorcycles.is_empty():
|
|
74
|
+
logger.debug("Saving motorcycles")
|
|
75
|
+
self.motorcycles.write_parquet(output_directory / Path("motorcycles.parquet"))
|
|
74
76
|
logger.debug("Saving persons")
|
|
75
77
|
self.persons.write_parquet(output_directory / Path("persons.parquet"))
|
|
76
78
|
logger.debug("Saving trips")
|
|
77
79
|
self.trips.write_parquet(output_directory / Path("trips.parquet"))
|
|
78
|
-
|
|
79
|
-
|
|
80
|
+
if not self.legs.is_empty():
|
|
81
|
+
logger.debug("Saving legs")
|
|
82
|
+
self.legs.write_parquet(output_directory / Path("legs.parquet"))
|
|
80
83
|
if self.special_locations is not None:
|
|
81
84
|
logger.debug("Saving special locations")
|
|
82
85
|
self.special_locations.to_parquet(
|
|
@@ -192,16 +195,22 @@ class SurveyDataReader:
|
|
|
192
195
|
def cars(self) -> pl.DataFrame:
|
|
193
196
|
if self._cars is None:
|
|
194
197
|
filename = os.path.join(self.directory, "cars.parquet")
|
|
195
|
-
|
|
196
|
-
|
|
198
|
+
if os.path.isfile(filename):
|
|
199
|
+
logger.debug(f"Reading cars from `{filename}`")
|
|
200
|
+
self._cars = pl.read_parquet(filename)
|
|
201
|
+
else:
|
|
202
|
+
self._cars = pl.DataFrame()
|
|
197
203
|
return self._cars
|
|
198
204
|
|
|
199
205
|
@property
|
|
200
206
|
def motorcycles(self) -> pl.DataFrame:
|
|
201
207
|
if self._motorcycles is None:
|
|
202
208
|
filename = os.path.join(self.directory, "motorcycles.parquet")
|
|
203
|
-
|
|
204
|
-
|
|
209
|
+
if os.path.isfile(filename):
|
|
210
|
+
logger.debug(f"Reading motorcycles from `{filename}`")
|
|
211
|
+
self._motorcycles = pl.read_parquet(filename)
|
|
212
|
+
else:
|
|
213
|
+
self._motorcycles = pl.DataFrame()
|
|
205
214
|
return self._motorcycles
|
|
206
215
|
|
|
207
216
|
@property
|
|
@@ -323,8 +332,6 @@ def is_valid_mobisurvstd_dir(directory: str) -> bool:
|
|
|
323
332
|
"persons.parquet",
|
|
324
333
|
"trips.parquet",
|
|
325
334
|
"legs.parquet",
|
|
326
|
-
"cars.parquet",
|
|
327
|
-
"motorcycles.parquet",
|
|
328
335
|
):
|
|
329
336
|
if not os.path.isfile(os.path.join(directory, name)):
|
|
330
337
|
return False
|
|
@@ -29,13 +29,14 @@ def standardizer(
|
|
|
29
29
|
survey_type: Annotated[
|
|
30
30
|
str | None,
|
|
31
31
|
typer.Option(
|
|
32
|
-
help="Format of the original survey. Possible values: `emc2`, `emp2019`, `egt2010`, `egt2020`, `edgt`, `edvm`, `emd`."
|
|
32
|
+
help="Format of the original survey. Possible values: `emc2`, `emp2019`, `egt2010`, `egt2020`, `edgt`, `edvm`, `emd`, `emg2023`."
|
|
33
33
|
),
|
|
34
34
|
] = None,
|
|
35
35
|
bulk: bool = typer.Option(
|
|
36
36
|
False, "--bulk", help="Import surveys in bulk from the given directory"
|
|
37
37
|
),
|
|
38
38
|
skip_spatial: bool = typer.Option(False, "--skip-spatial", help="Do not read spatial data"),
|
|
39
|
+
skip_insee: bool = typer.Option(False, "--skip-insee", help="Do not add INSEE data"),
|
|
39
40
|
no_validation: bool = typer.Option(
|
|
40
41
|
False,
|
|
41
42
|
"--no-validation",
|
|
@@ -57,6 +58,7 @@ def standardizer(
|
|
|
57
58
|
output_directory,
|
|
58
59
|
survey_type,
|
|
59
60
|
skip_spatial=skip_spatial,
|
|
61
|
+
skip_insee=skip_insee,
|
|
60
62
|
no_validation=no_validation,
|
|
61
63
|
)
|
|
62
64
|
else:
|
|
@@ -65,6 +67,7 @@ def standardizer(
|
|
|
65
67
|
output_directory,
|
|
66
68
|
survey_type,
|
|
67
69
|
skip_spatial=skip_spatial,
|
|
70
|
+
skip_insee=skip_insee,
|
|
68
71
|
no_validation=no_validation,
|
|
69
72
|
)
|
|
70
73
|
|
|
@@ -128,7 +128,7 @@ def create_metadata(
|
|
|
128
128
|
"nb_motorcycles": len(data["motorcycles"]),
|
|
129
129
|
"nb_persons": len(data["persons"]),
|
|
130
130
|
"nb_trips": len(data["trips"]),
|
|
131
|
-
"nb_legs": len(data["legs"]),
|
|
131
|
+
"nb_legs": len(data["legs"]) if "legs" in data else 0,
|
|
132
132
|
"nb_special_locations": nb_special_locations,
|
|
133
133
|
"nb_detailed_zones": nb_detailed_zones,
|
|
134
134
|
"nb_draw_zones": nb_draw_zones,
|
|
@@ -151,22 +151,33 @@ def count_nb_persons(households: pl.LazyFrame, persons: pl.LazyFrame):
|
|
|
151
151
|
has_age = "age" in persons.collect_schema().names()
|
|
152
152
|
has_age_class = "age_class_code" in persons.collect_schema().names()
|
|
153
153
|
person_counts = persons.group_by("household_id").agg(
|
|
154
|
+
complete_household=True,
|
|
154
155
|
nb_persons=pl.len(),
|
|
155
|
-
nb_persons_5plus=
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
156
|
+
nb_persons_5plus=(
|
|
157
|
+
pl.when(pl.col("age").is_not_null().all()).then(pl.col("age").ge(5).sum())
|
|
158
|
+
if has_age
|
|
159
|
+
else None
|
|
160
|
+
),
|
|
161
|
+
nb_majors=(
|
|
162
|
+
pl.when(pl.col("age_class_code").is_not_null().all()).then(
|
|
163
|
+
pl.col("age_class_code").gt(1).sum()
|
|
164
|
+
)
|
|
165
|
+
if has_age_class
|
|
166
|
+
else None
|
|
167
|
+
),
|
|
168
|
+
nb_minors=(
|
|
169
|
+
pl.when(pl.col("age_class_code").is_not_null().all()).then(
|
|
170
|
+
pl.col("age_class_code").eq(1).sum()
|
|
171
|
+
)
|
|
172
|
+
if has_age_class
|
|
173
|
+
else None
|
|
174
|
+
),
|
|
175
|
+
)
|
|
176
|
+
# Join on both "household_id" and "complete_household" so that counts are not added for
|
|
177
|
+
# incomplete households.
|
|
178
|
+
households = households.join(
|
|
179
|
+
person_counts, on=["household_id", "complete_household"], how="left", coalesce=True
|
|
168
180
|
)
|
|
169
|
-
households = households.join(person_counts, on="household_id", how="left", coalesce=True)
|
|
170
181
|
return households
|
|
171
182
|
|
|
172
183
|
|
|
@@ -238,6 +249,7 @@ def count_nb_trips(persons: pl.LazyFrame, trips: pl.LazyFrame):
|
|
|
238
249
|
|
|
239
250
|
def add_worked_during_surveyed_day(persons: pl.LazyFrame, trips: pl.LazyFrame):
|
|
240
251
|
persons_cols = persons.collect_schema().names()
|
|
252
|
+
trips_cols = trips.collect_schema().names()
|
|
241
253
|
# Fill null values of `traveled_during_surveyed_day` (when `is_surveyed` is True) according
|
|
242
254
|
# to the existence of trips.
|
|
243
255
|
if "traveled_during_surveyed_day" not in persons_cols:
|
|
@@ -250,7 +262,7 @@ def add_worked_during_surveyed_day(persons: pl.LazyFrame, trips: pl.LazyFrame):
|
|
|
250
262
|
.then(pl.lit("no"))
|
|
251
263
|
)
|
|
252
264
|
)
|
|
253
|
-
if "worked_during_surveyed_day" not in persons_cols:
|
|
265
|
+
if "worked_during_surveyed_day" not in persons_cols and "destination_purpose" in trips_cols:
|
|
254
266
|
persons = persons.join(
|
|
255
267
|
trips.group_by("person_id").agg(
|
|
256
268
|
has_work_activity=pl.col("destination_purpose")
|
|
@@ -14,6 +14,7 @@ def clean(
|
|
|
14
14
|
year: int | None,
|
|
15
15
|
special_locations: pl.DataFrame | None = None,
|
|
16
16
|
detailed_zones: pl.DataFrame | None = None,
|
|
17
|
+
skip_insee: bool = False,
|
|
17
18
|
):
|
|
18
19
|
existing_cols = lf.collect_schema().names()
|
|
19
20
|
columns = [variable.name for variable in HOUSEHOLD_SCHEMA if variable.name in existing_cols]
|
|
@@ -21,8 +22,9 @@ def clean(
|
|
|
21
22
|
lf = indexing(lf)
|
|
22
23
|
lf = add_bicycle_counts(lf, existing_cols)
|
|
23
24
|
lf = add_lng_lat(lf, existing_cols, special_locations, detailed_zones)
|
|
24
|
-
|
|
25
|
-
|
|
25
|
+
if not skip_insee:
|
|
26
|
+
lf = add_insee_columns(lf, existing_cols)
|
|
27
|
+
lf = add_insee_data_columns(lf, existing_cols, year)
|
|
26
28
|
if "home_dep" in existing_cols:
|
|
27
29
|
lf = add_nuts_data(lf, "home")
|
|
28
30
|
if DEBUG:
|