mobisurvstd 1.0.2__tar.gz → 1.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {mobisurvstd-1.0.2/src/mobisurvstd.egg-info → mobisurvstd-1.1.0}/PKG-INFO +5 -7
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/README.md +2 -1
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/pyproject.toml +22 -16
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/cerema/__init__.py +4 -2
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/cerema/deplacements.py +2 -1
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/cerema/edgt.py +1 -3
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/cerema/menages.py +9 -32
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/cerema/nantes_opendata.py +1 -3
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/cerema/survey.py +4 -6
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/cerema/trajets.py +1 -1
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/classes.py +19 -12
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/cli.py +4 -1
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/common/clean.py +28 -16
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/common/households.py +4 -2
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/common/trips.py +8 -5
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/common/zones.py +1 -7
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/egt2010/deplacements.py +2 -0
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/egt2010/menages.py +6 -7
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/egt2010/personnes.py +1 -5
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/egt2010/survey.py +3 -3
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/egt2020/deplacements.py +34 -27
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/egt2020/menages.py +17 -12
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/egt2020/motos.py +3 -10
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/egt2020/personnes.py +27 -29
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/egt2020/survey.py +21 -9
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/egt2020/trajets.py +23 -19
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/egt2020/voitures.py +2 -1
- mobisurvstd-1.1.0/src/mobisurvstd/emg2023/deplacements.py +251 -0
- mobisurvstd-1.1.0/src/mobisurvstd/emg2023/menages.py +24 -0
- mobisurvstd-1.1.0/src/mobisurvstd/emg2023/personnes.py +125 -0
- mobisurvstd-1.1.0/src/mobisurvstd/emg2023/survey.py +65 -0
- mobisurvstd-1.1.0/src/mobisurvstd/emp/__init__.py +1 -0
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/emp/common.py +9 -0
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/emp/deplacements.py +7 -9
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/emp/menages.py +4 -0
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/emp/motos.py +4 -26
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/emp/personnes.py +1 -4
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/emp/survey.py +1 -2
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/emp/voitures.py +5 -39
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/main.py +49 -11
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/resources/insee_data.py +155 -38
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/schema/cars.py +2 -14
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/schema/common.py +3 -6
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/schema/guarantees.py +33 -9
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/schema/households.py +10 -4
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/schema/motorcycles.py +1 -9
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/schema/persons.py +17 -74
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/schema/trips.py +20 -3
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/utils.py +16 -4
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/validation.py +49 -42
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0/src/mobisurvstd.egg-info}/PKG-INFO +5 -7
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd.egg-info/SOURCES.txt +5 -0
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd.egg-info/requires.txt +2 -5
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/LICENSE.md +0 -0
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/setup.cfg +0 -0
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/__init__.py +0 -0
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/__main__.py +0 -0
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/cerema/common.py +0 -0
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/cerema/edvm.py +0 -0
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/cerema/emc2.py +0 -0
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/cerema/emd.py +0 -0
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/cerema/personnes.py +0 -0
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/cerema/reader.py +0 -0
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/cerema/zones.py +0 -0
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/common/__init__.py +0 -0
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/common/cars.py +0 -0
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/common/legs.py +0 -0
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/common/motorcycles.py +0 -0
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/common/persons.py +0 -0
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/egt2010/__init__.py +0 -0
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/egt2010/trajets.py +0 -0
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/egt2010/zones.py +0 -0
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/egt2020/__init__.py +0 -0
- {mobisurvstd-1.0.2/src/mobisurvstd/emp → mobisurvstd-1.1.0/src/mobisurvstd/emg2023}/__init__.py +0 -0
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/logger.py +0 -0
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/resources/__init__.py +0 -0
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/resources/admin_express.py +0 -0
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/resources/nuts.py +0 -0
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/schema/__init__.py +0 -0
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/schema/legs.py +0 -0
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd.egg-info/dependency_links.txt +0 -0
- {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: mobisurvstd
|
|
3
|
-
Version: 1.0
|
|
3
|
+
Version: 1.1.0
|
|
4
4
|
Summary: A tool to standardize French mobility survey datasets (EMC2, EGT, EMP, etc.) into a unified format.
|
|
5
5
|
Author-email: Lucas Javaudin <lucas@lucasjavaudin.com>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -15,21 +15,18 @@ Classifier: Topic :: File Formats
|
|
|
15
15
|
Requires-Python: >=3.10
|
|
16
16
|
Description-Content-Type: text/markdown
|
|
17
17
|
License-File: LICENSE.md
|
|
18
|
-
Requires-Dist: Cartopy>=0.24.1
|
|
19
18
|
Requires-Dist: fastexcel>=0.14.0
|
|
20
|
-
Requires-Dist: fsspec>=2025.9.0
|
|
21
19
|
Requires-Dist: geopandas>=1.0.1
|
|
22
20
|
Requires-Dist: humanize>=4.13.0
|
|
23
21
|
Requires-Dist: loguru>=0.7.3
|
|
24
|
-
Requires-Dist: matplotlib>=3.10.1
|
|
25
22
|
Requires-Dist: numpy>=2.2.4
|
|
26
23
|
Requires-Dist: pandas>=2.2.3
|
|
27
24
|
Requires-Dist: platformdirs>=4.3.8
|
|
28
|
-
Requires-Dist: polars>=1.
|
|
25
|
+
Requires-Dist: polars>=1.38.0
|
|
29
26
|
Requires-Dist: py7zr>=0.20.8
|
|
30
27
|
Requires-Dist: pyarrow>=19.0.1
|
|
31
28
|
Requires-Dist: requests>=2.32.3
|
|
32
|
-
Requires-Dist:
|
|
29
|
+
Requires-Dist: tqdm>=4.67.3
|
|
33
30
|
Requires-Dist: typer>=0.16.0
|
|
34
31
|
Dynamic: license-file
|
|
35
32
|
|
|
@@ -202,6 +199,7 @@ Currently, MobiSurvStd supports the following survey types:
|
|
|
202
199
|
|
|
203
200
|
- `emp2019`
|
|
204
201
|
- `emc2`
|
|
202
|
+
- `emg2023` (thanks to @sebhorl)
|
|
205
203
|
- `egt2020`
|
|
206
204
|
- `egt2010`
|
|
207
205
|
- `edgt` (including the [open-data version of Nantes 2015](https://www.data.gouv.fr/datasets/enquete-deplacements-2015-en-loire-atlantique))
|
|
@@ -210,7 +208,7 @@ Currently, MobiSurvStd supports the following survey types:
|
|
|
210
208
|
|
|
211
209
|
See [Survey Types](https://mobisurvstd.github.io/MobiSurvStd/surveys.html) for more.
|
|
212
210
|
|
|
213
|
-
Other survey types that are planned to be integrated are: `entd
|
|
211
|
+
Other survey types that are planned to be integrated are: `entd`.
|
|
214
212
|
If you know another survey format that could be integrated, feel free to open an issue on GitHub.
|
|
215
213
|
|
|
216
214
|
MobiSurvStd covers only French mobility survey formats.
|
|
@@ -167,6 +167,7 @@ Currently, MobiSurvStd supports the following survey types:
|
|
|
167
167
|
|
|
168
168
|
- `emp2019`
|
|
169
169
|
- `emc2`
|
|
170
|
+
- `emg2023` (thanks to @sebhorl)
|
|
170
171
|
- `egt2020`
|
|
171
172
|
- `egt2010`
|
|
172
173
|
- `edgt` (including the [open-data version of Nantes 2015](https://www.data.gouv.fr/datasets/enquete-deplacements-2015-en-loire-atlantique))
|
|
@@ -175,7 +176,7 @@ Currently, MobiSurvStd supports the following survey types:
|
|
|
175
176
|
|
|
176
177
|
See [Survey Types](https://mobisurvstd.github.io/MobiSurvStd/surveys.html) for more.
|
|
177
178
|
|
|
178
|
-
Other survey types that are planned to be integrated are: `entd
|
|
179
|
+
Other survey types that are planned to be integrated are: `entd`.
|
|
179
180
|
If you know another survey format that could be integrated, feel free to open an issue on GitHub.
|
|
180
181
|
|
|
181
182
|
MobiSurvStd covers only French mobility survey formats.
|
|
@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
|
|
|
6
6
|
|
|
7
7
|
[project]
|
|
8
8
|
name = "mobisurvstd"
|
|
9
|
-
version = "1.0
|
|
9
|
+
version = "1.1.0"
|
|
10
10
|
description = "A tool to standardize French mobility survey datasets (EMC2, EGT, EMP, etc.) into a unified format."
|
|
11
11
|
readme = "README.md"
|
|
12
12
|
authors = [{ name = "Lucas Javaudin", email = "lucas@lucasjavaudin.com" }]
|
|
@@ -19,29 +19,30 @@ classifiers = [
|
|
|
19
19
|
]
|
|
20
20
|
keywords = ["mobility survey", "polars", "data format"]
|
|
21
21
|
dependencies = [
|
|
22
|
-
"
|
|
23
|
-
"fastexcel >= 0.14.0",
|
|
24
|
-
"fsspec>=2025.9.0",
|
|
22
|
+
"fastexcel>=0.14.0",
|
|
25
23
|
"geopandas>=1.0.1",
|
|
26
|
-
"humanize
|
|
27
|
-
"loguru
|
|
28
|
-
"
|
|
29
|
-
"
|
|
30
|
-
"
|
|
31
|
-
"
|
|
32
|
-
"
|
|
33
|
-
"
|
|
34
|
-
"
|
|
35
|
-
"
|
|
36
|
-
"
|
|
37
|
-
"typer >= 0.16.0",
|
|
24
|
+
"humanize>=4.13.0",
|
|
25
|
+
"loguru>=0.7.3",
|
|
26
|
+
"numpy>=2.2.4",
|
|
27
|
+
"pandas>=2.2.3",
|
|
28
|
+
"platformdirs>=4.3.8",
|
|
29
|
+
"polars>=1.38.0",
|
|
30
|
+
"py7zr>=0.20.8",
|
|
31
|
+
"pyarrow>=19.0.1",
|
|
32
|
+
"requests>=2.32.3",
|
|
33
|
+
"tqdm>=4.67.3",
|
|
34
|
+
"typer>=0.16.0",
|
|
38
35
|
]
|
|
39
36
|
requires-python = ">=3.10"
|
|
40
37
|
|
|
41
38
|
[dependency-groups]
|
|
42
39
|
dev = [
|
|
40
|
+
"Cartopy>=0.24.1",
|
|
41
|
+
"folium>=0.20.0",
|
|
43
42
|
"ipython>=8.37.0",
|
|
43
|
+
"matplotlib>=3.10.1",
|
|
44
44
|
"pytest>=9.0.2",
|
|
45
|
+
"seaborn>=0.13.2",
|
|
45
46
|
"ty>=0.0.15",
|
|
46
47
|
]
|
|
47
48
|
|
|
@@ -50,3 +51,8 @@ Homepage = "https://github.com/MobiSurvStd/MobiSurvStd"
|
|
|
50
51
|
Documentation = "https://mobisurvstd.github.io/MobiSurvStd/index.html"
|
|
51
52
|
Repository = "https://github.com/MobiSurvStd/MobiSurvStd"
|
|
52
53
|
Issues = "https://github.com/MobiSurvStd/MobiSurvStd/issues"
|
|
54
|
+
|
|
55
|
+
[tool.ruff]
|
|
56
|
+
line-length = 100
|
|
57
|
+
[tool.ruff.format]
|
|
58
|
+
skip-magic-trailing-comma = true
|
|
@@ -7,7 +7,9 @@ from .emd import EMDReader
|
|
|
7
7
|
from .nantes_opendata import OpenDataReader
|
|
8
8
|
|
|
9
9
|
|
|
10
|
-
def standardize(
|
|
10
|
+
def standardize(
|
|
11
|
+
source: str | ZipFile, survey_type: str, skip_spatial: bool = False, skip_insee: bool = False
|
|
12
|
+
):
|
|
11
13
|
if survey_type == "EMC2":
|
|
12
14
|
reader = EMC2Reader(source)
|
|
13
15
|
elif survey_type == "EDGT":
|
|
@@ -20,4 +22,4 @@ def standardize(source: str | ZipFile, survey_type: str, skip_spatial: bool = Fa
|
|
|
20
22
|
reader = EMDReader(source)
|
|
21
23
|
else:
|
|
22
24
|
raise NotImplementedError
|
|
23
|
-
return reader.standardize(skip_spatial)
|
|
25
|
+
return reader.standardize(skip_spatial, skip_insee)
|
|
@@ -123,7 +123,7 @@ class TripsReader(CeremaReader):
|
|
|
123
123
|
lf = lf.sort(self.get_trip_index_cols())
|
|
124
124
|
return lf
|
|
125
125
|
|
|
126
|
-
def standardize_trips(self):
|
|
126
|
+
def standardize_trips(self, skip_insee: bool = False):
|
|
127
127
|
lf = self.scan_trips()
|
|
128
128
|
# Add household_id, person_id, trip date, and trip weekday.
|
|
129
129
|
lf = lf.with_columns(
|
|
@@ -203,6 +203,7 @@ class TripsReader(CeremaReader):
|
|
|
203
203
|
year=self.survey_year() or CURRENT_YEAR,
|
|
204
204
|
special_locations=self.special_locations_coords,
|
|
205
205
|
detailed_zones=self.detailed_zones_coords,
|
|
206
|
+
skip_insee=skip_insee,
|
|
206
207
|
)
|
|
207
208
|
# When the INSEE code ends with "000" or "999" it means "rest of the département".
|
|
208
209
|
# We drop these values because they do not add any additional information compared to `_dep`
|
|
@@ -92,10 +92,7 @@ SCHEMA = {
|
|
|
92
92
|
"COE0": pl.Float64, # COEFFICIENT DE REDRESSEMENT MENAGE
|
|
93
93
|
}
|
|
94
94
|
|
|
95
|
-
SURVEY_METHOD_MAP = {
|
|
96
|
-
1: "face_to_face",
|
|
97
|
-
2: "phone",
|
|
98
|
-
}
|
|
95
|
+
SURVEY_METHOD_MAP = {1: "face_to_face", 2: "phone"}
|
|
99
96
|
|
|
100
97
|
HOUSING_TYPE_MAP = {
|
|
101
98
|
1: "house", # Individuel isolé
|
|
@@ -195,37 +192,15 @@ MOTORCYCLE_FUEL_TYPE_MAP = {
|
|
|
195
192
|
90: "other",
|
|
196
193
|
}
|
|
197
194
|
|
|
198
|
-
MOTORCYCLE_SIZE_LOWERBOUND_MAP = {
|
|
199
|
-
1: 0,
|
|
200
|
-
2: 50,
|
|
201
|
-
3: 125,
|
|
202
|
-
4: 250,
|
|
203
|
-
5: 750,
|
|
204
|
-
9: 125,
|
|
205
|
-
}
|
|
195
|
+
MOTORCYCLE_SIZE_LOWERBOUND_MAP = {1: 0, 2: 50, 3: 125, 4: 250, 5: 750, 9: 125}
|
|
206
196
|
|
|
207
|
-
MOTORCYCLE_SIZE_UPPERBOUND_MAP = {
|
|
208
|
-
1: 50,
|
|
209
|
-
2: 125,
|
|
210
|
-
3: 250,
|
|
211
|
-
4: 750,
|
|
212
|
-
}
|
|
197
|
+
MOTORCYCLE_SIZE_UPPERBOUND_MAP = {1: 50, 2: 125, 3: 250, 4: 750}
|
|
213
198
|
|
|
214
|
-
MOTORCYCLE_KW_LOWERBOUND_MAP = {
|
|
215
|
-
6: 0,
|
|
216
|
-
7: 4,
|
|
217
|
-
8: 11,
|
|
218
|
-
}
|
|
199
|
+
MOTORCYCLE_KW_LOWERBOUND_MAP = {6: 0, 7: 4, 8: 11}
|
|
219
200
|
|
|
220
|
-
MOTORCYCLE_KW_UPPERBOUND_MAP = {
|
|
221
|
-
6: 4,
|
|
222
|
-
7: 11,
|
|
223
|
-
}
|
|
201
|
+
MOTORCYCLE_KW_UPPERBOUND_MAP = {6: 4, 7: 11}
|
|
224
202
|
|
|
225
|
-
MOTORCYCLE_THERMIC_ENGINE_TYPE_MAP = {
|
|
226
|
-
1: "two_stroke",
|
|
227
|
-
2: "four_stroke",
|
|
228
|
-
}
|
|
203
|
+
MOTORCYCLE_THERMIC_ENGINE_TYPE_MAP = {1: "two_stroke", 2: "four_stroke"}
|
|
229
204
|
|
|
230
205
|
|
|
231
206
|
def scan_households_impl(source: str | io.BytesIO):
|
|
@@ -248,7 +223,7 @@ class HouseholdsReader(CeremaReader):
|
|
|
248
223
|
def main_insee(self):
|
|
249
224
|
return self.scan_households().select("IDM4").first().collect().item() # ty: ignore[possibly-missing-attribute]
|
|
250
225
|
|
|
251
|
-
def standardize_households(self):
|
|
226
|
+
def standardize_households(self, skip_insee: bool = False):
|
|
252
227
|
lf = self.scan_households()
|
|
253
228
|
lf = lf.rename(
|
|
254
229
|
{
|
|
@@ -262,6 +237,7 @@ class HouseholdsReader(CeremaReader):
|
|
|
262
237
|
)
|
|
263
238
|
lf = lf.with_columns(
|
|
264
239
|
original_household_id=pl.struct(self.get_household_index_cols()),
|
|
240
|
+
complete_household=True,
|
|
265
241
|
home_detailed_zone="ZFM",
|
|
266
242
|
home_draw_zone="STM",
|
|
267
243
|
# For EMD, EDVM, EDGT, the IDM2 column will be read here.
|
|
@@ -278,6 +254,7 @@ class HouseholdsReader(CeremaReader):
|
|
|
278
254
|
year=self.survey_year() or CURRENT_YEAR,
|
|
279
255
|
special_locations=self.special_locations_coords,
|
|
280
256
|
detailed_zones=self.detailed_zones_coords,
|
|
257
|
+
skip_insee=skip_insee,
|
|
281
258
|
)
|
|
282
259
|
# When the INSEE code ends with "000" or "999" it means "rest of the département".
|
|
283
260
|
# We drop these values because they do not add any additional information compared to `_dep`
|
|
@@ -192,9 +192,7 @@ class OpenDataReader(CeremaStandardizer):
|
|
|
192
192
|
|
|
193
193
|
|
|
194
194
|
def scan_fwf(
|
|
195
|
-
f: str | io.BytesIO,
|
|
196
|
-
format: list[tuple[int, int, str]],
|
|
197
|
-
schema: dict[str, DataTypeClass],
|
|
195
|
+
f: str | io.BytesIO, format: list[tuple[int, int, str]], schema: dict[str, DataTypeClass]
|
|
198
196
|
):
|
|
199
197
|
# Decrease `start` by 1 because pandas starts counting at 0 but the survey document starts
|
|
200
198
|
# counting at 1.
|
|
@@ -26,7 +26,7 @@ LOCATION_COLUMNS = (
|
|
|
26
26
|
class CeremaStandardizer(HouseholdsReader, PersonsReader, TripsReader, LegsReader, ZonesReader):
|
|
27
27
|
SURVEY_TYPE = ""
|
|
28
28
|
|
|
29
|
-
def standardize(self, skip_spatial: bool = False):
|
|
29
|
+
def standardize(self, skip_spatial: bool = False, skip_insee: bool = False):
|
|
30
30
|
logger.info(f"Standardizing {self.SURVEY_TYPE} survey from `{self.source_name()}`")
|
|
31
31
|
if not self.validate():
|
|
32
32
|
return None
|
|
@@ -34,11 +34,11 @@ class CeremaStandardizer(HouseholdsReader, PersonsReader, TripsReader, LegsReade
|
|
|
34
34
|
if not skip_spatial:
|
|
35
35
|
self.read_spatial_data()
|
|
36
36
|
|
|
37
|
-
self.standardize_households()
|
|
37
|
+
self.standardize_households(skip_insee)
|
|
38
38
|
self.standardize_cars()
|
|
39
39
|
self.standardize_motorcycles()
|
|
40
40
|
self.standardize_persons()
|
|
41
|
-
self.standardize_trips()
|
|
41
|
+
self.standardize_trips(skip_insee)
|
|
42
42
|
self.standardize_legs()
|
|
43
43
|
|
|
44
44
|
self.finish()
|
|
@@ -115,9 +115,7 @@ class CeremaStandardizer(HouseholdsReader, PersonsReader, TripsReader, LegsReade
|
|
|
115
115
|
) # ty: ignore[invalid-assignment]
|
|
116
116
|
self.households = self.households.join(
|
|
117
117
|
household_dates.lazy(), on="household_id", how="left", coalesce=True
|
|
118
|
-
).with_columns(
|
|
119
|
-
trips_weekday=pl.when("is_valid_weekday").then(pl.col("trips_weekday")),
|
|
120
|
-
)
|
|
118
|
+
).with_columns(trips_weekday=pl.when("is_valid_weekday").then(pl.col("trips_weekday")))
|
|
121
119
|
|
|
122
120
|
def fix_main_mode(self):
|
|
123
121
|
# Special case for Douai 2012: Some trips have main mode set to "car_driver" but the legs
|
|
@@ -342,7 +342,7 @@ class LegsReader(CeremaReader):
|
|
|
342
342
|
.then(pl.col("nb_majors_in_vehicle") + 1)
|
|
343
343
|
.otherwise("nb_majors_in_vehicle"),
|
|
344
344
|
nb_minors_in_vehicle=pl.when(
|
|
345
|
-
pl.col("is_major").not_() & pl.col("mode").str.contains("driver")
|
|
345
|
+
pl.col("is_major").not_() & pl.col("mode").str.contains("driver")
|
|
346
346
|
)
|
|
347
347
|
.then(pl.col("nb_minors_in_vehicle") + 1)
|
|
348
348
|
.otherwise("nb_minors_in_vehicle"),
|
|
@@ -67,16 +67,19 @@ class SurveyData:
|
|
|
67
67
|
logger.warning("Output directory is not empty, some data might be erased")
|
|
68
68
|
logger.debug("Saving households")
|
|
69
69
|
self.households.write_parquet(output_directory / Path("households.parquet"))
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
self.motorcycles.
|
|
70
|
+
if not self.cars.is_empty():
|
|
71
|
+
logger.debug("Saving cars")
|
|
72
|
+
self.cars.write_parquet(output_directory / Path("cars.parquet"))
|
|
73
|
+
if not self.motorcycles.is_empty():
|
|
74
|
+
logger.debug("Saving motorcycles")
|
|
75
|
+
self.motorcycles.write_parquet(output_directory / Path("motorcycles.parquet"))
|
|
74
76
|
logger.debug("Saving persons")
|
|
75
77
|
self.persons.write_parquet(output_directory / Path("persons.parquet"))
|
|
76
78
|
logger.debug("Saving trips")
|
|
77
79
|
self.trips.write_parquet(output_directory / Path("trips.parquet"))
|
|
78
|
-
|
|
79
|
-
|
|
80
|
+
if not self.legs.is_empty():
|
|
81
|
+
logger.debug("Saving legs")
|
|
82
|
+
self.legs.write_parquet(output_directory / Path("legs.parquet"))
|
|
80
83
|
if self.special_locations is not None:
|
|
81
84
|
logger.debug("Saving special locations")
|
|
82
85
|
self.special_locations.to_parquet(
|
|
@@ -192,16 +195,22 @@ class SurveyDataReader:
|
|
|
192
195
|
def cars(self) -> pl.DataFrame:
|
|
193
196
|
if self._cars is None:
|
|
194
197
|
filename = os.path.join(self.directory, "cars.parquet")
|
|
195
|
-
|
|
196
|
-
|
|
198
|
+
if os.path.isfile(filename):
|
|
199
|
+
logger.debug(f"Reading cars from `{filename}`")
|
|
200
|
+
self._cars = pl.read_parquet(filename)
|
|
201
|
+
else:
|
|
202
|
+
self._cars = pl.DataFrame()
|
|
197
203
|
return self._cars
|
|
198
204
|
|
|
199
205
|
@property
|
|
200
206
|
def motorcycles(self) -> pl.DataFrame:
|
|
201
207
|
if self._motorcycles is None:
|
|
202
208
|
filename = os.path.join(self.directory, "motorcycles.parquet")
|
|
203
|
-
|
|
204
|
-
|
|
209
|
+
if os.path.isfile(filename):
|
|
210
|
+
logger.debug(f"Reading motorcycles from `{filename}`")
|
|
211
|
+
self._motorcycles = pl.read_parquet(filename)
|
|
212
|
+
else:
|
|
213
|
+
self._motorcycles = pl.DataFrame()
|
|
205
214
|
return self._motorcycles
|
|
206
215
|
|
|
207
216
|
@property
|
|
@@ -323,8 +332,6 @@ def is_valid_mobisurvstd_dir(directory: str) -> bool:
|
|
|
323
332
|
"persons.parquet",
|
|
324
333
|
"trips.parquet",
|
|
325
334
|
"legs.parquet",
|
|
326
|
-
"cars.parquet",
|
|
327
|
-
"motorcycles.parquet",
|
|
328
335
|
):
|
|
329
336
|
if not os.path.isfile(os.path.join(directory, name)):
|
|
330
337
|
return False
|
|
@@ -29,13 +29,14 @@ def standardizer(
|
|
|
29
29
|
survey_type: Annotated[
|
|
30
30
|
str | None,
|
|
31
31
|
typer.Option(
|
|
32
|
-
help="Format of the original survey. Possible values: `emc2`, `emp2019`, `egt2010`, `egt2020`, `edgt`, `edvm`, `emd`."
|
|
32
|
+
help="Format of the original survey. Possible values: `emc2`, `emp2019`, `egt2010`, `egt2020`, `edgt`, `edvm`, `emd`, `emg2023`."
|
|
33
33
|
),
|
|
34
34
|
] = None,
|
|
35
35
|
bulk: bool = typer.Option(
|
|
36
36
|
False, "--bulk", help="Import surveys in bulk from the given directory"
|
|
37
37
|
),
|
|
38
38
|
skip_spatial: bool = typer.Option(False, "--skip-spatial", help="Do not read spatial data"),
|
|
39
|
+
skip_insee: bool = typer.Option(False, "--skip-insee", help="Do not add INSEE data"),
|
|
39
40
|
no_validation: bool = typer.Option(
|
|
40
41
|
False,
|
|
41
42
|
"--no-validation",
|
|
@@ -57,6 +58,7 @@ def standardizer(
|
|
|
57
58
|
output_directory,
|
|
58
59
|
survey_type,
|
|
59
60
|
skip_spatial=skip_spatial,
|
|
61
|
+
skip_insee=skip_insee,
|
|
60
62
|
no_validation=no_validation,
|
|
61
63
|
)
|
|
62
64
|
else:
|
|
@@ -65,6 +67,7 @@ def standardizer(
|
|
|
65
67
|
output_directory,
|
|
66
68
|
survey_type,
|
|
67
69
|
skip_spatial=skip_spatial,
|
|
70
|
+
skip_insee=skip_insee,
|
|
68
71
|
no_validation=no_validation,
|
|
69
72
|
)
|
|
70
73
|
|
|
@@ -128,7 +128,7 @@ def create_metadata(
|
|
|
128
128
|
"nb_motorcycles": len(data["motorcycles"]),
|
|
129
129
|
"nb_persons": len(data["persons"]),
|
|
130
130
|
"nb_trips": len(data["trips"]),
|
|
131
|
-
"nb_legs": len(data["legs"]),
|
|
131
|
+
"nb_legs": len(data["legs"]) if "legs" in data else 0,
|
|
132
132
|
"nb_special_locations": nb_special_locations,
|
|
133
133
|
"nb_detailed_zones": nb_detailed_zones,
|
|
134
134
|
"nb_draw_zones": nb_draw_zones,
|
|
@@ -151,22 +151,33 @@ def count_nb_persons(households: pl.LazyFrame, persons: pl.LazyFrame):
|
|
|
151
151
|
has_age = "age" in persons.collect_schema().names()
|
|
152
152
|
has_age_class = "age_class_code" in persons.collect_schema().names()
|
|
153
153
|
person_counts = persons.group_by("household_id").agg(
|
|
154
|
+
complete_household=True,
|
|
154
155
|
nb_persons=pl.len(),
|
|
155
|
-
nb_persons_5plus=
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
156
|
+
nb_persons_5plus=(
|
|
157
|
+
pl.when(pl.col("age").is_not_null().all()).then(pl.col("age").ge(5).sum())
|
|
158
|
+
if has_age
|
|
159
|
+
else None
|
|
160
|
+
),
|
|
161
|
+
nb_majors=(
|
|
162
|
+
pl.when(pl.col("age_class_code").is_not_null().all()).then(
|
|
163
|
+
pl.col("age_class_code").gt(1).sum()
|
|
164
|
+
)
|
|
165
|
+
if has_age_class
|
|
166
|
+
else None
|
|
167
|
+
),
|
|
168
|
+
nb_minors=(
|
|
169
|
+
pl.when(pl.col("age_class_code").is_not_null().all()).then(
|
|
170
|
+
pl.col("age_class_code").eq(1).sum()
|
|
171
|
+
)
|
|
172
|
+
if has_age_class
|
|
173
|
+
else None
|
|
174
|
+
),
|
|
175
|
+
)
|
|
176
|
+
# Join on both "household_id" and "complete_household" so that counts are not added for
|
|
177
|
+
# incomplete households.
|
|
178
|
+
households = households.join(
|
|
179
|
+
person_counts, on=["household_id", "complete_household"], how="left", coalesce=True
|
|
168
180
|
)
|
|
169
|
-
households = households.join(person_counts, on="household_id", how="left", coalesce=True)
|
|
170
181
|
return households
|
|
171
182
|
|
|
172
183
|
|
|
@@ -238,6 +249,7 @@ def count_nb_trips(persons: pl.LazyFrame, trips: pl.LazyFrame):
|
|
|
238
249
|
|
|
239
250
|
def add_worked_during_surveyed_day(persons: pl.LazyFrame, trips: pl.LazyFrame):
|
|
240
251
|
persons_cols = persons.collect_schema().names()
|
|
252
|
+
trips_cols = trips.collect_schema().names()
|
|
241
253
|
# Fill null values of `traveled_during_surveyed_day` (when `is_surveyed` is True) according
|
|
242
254
|
# to the existence of trips.
|
|
243
255
|
if "traveled_during_surveyed_day" not in persons_cols:
|
|
@@ -250,7 +262,7 @@ def add_worked_during_surveyed_day(persons: pl.LazyFrame, trips: pl.LazyFrame):
|
|
|
250
262
|
.then(pl.lit("no"))
|
|
251
263
|
)
|
|
252
264
|
)
|
|
253
|
-
if "worked_during_surveyed_day" not in persons_cols:
|
|
265
|
+
if "worked_during_surveyed_day" not in persons_cols and "destination_purpose" in trips_cols:
|
|
254
266
|
persons = persons.join(
|
|
255
267
|
trips.group_by("person_id").agg(
|
|
256
268
|
has_work_activity=pl.col("destination_purpose")
|
|
@@ -14,6 +14,7 @@ def clean(
|
|
|
14
14
|
year: int | None,
|
|
15
15
|
special_locations: pl.DataFrame | None = None,
|
|
16
16
|
detailed_zones: pl.DataFrame | None = None,
|
|
17
|
+
skip_insee: bool = False,
|
|
17
18
|
):
|
|
18
19
|
existing_cols = lf.collect_schema().names()
|
|
19
20
|
columns = [variable.name for variable in HOUSEHOLD_SCHEMA if variable.name in existing_cols]
|
|
@@ -21,8 +22,9 @@ def clean(
|
|
|
21
22
|
lf = indexing(lf)
|
|
22
23
|
lf = add_bicycle_counts(lf, existing_cols)
|
|
23
24
|
lf = add_lng_lat(lf, existing_cols, special_locations, detailed_zones)
|
|
24
|
-
|
|
25
|
-
|
|
25
|
+
if not skip_insee:
|
|
26
|
+
lf = add_insee_columns(lf, existing_cols)
|
|
27
|
+
lf = add_insee_data_columns(lf, existing_cols, year)
|
|
26
28
|
if "home_dep" in existing_cols:
|
|
27
29
|
lf = add_nuts_data(lf, "home")
|
|
28
30
|
if DEBUG:
|
|
@@ -16,6 +16,7 @@ def clean(
|
|
|
16
16
|
perimeter_deps: list[str] | None = None,
|
|
17
17
|
special_locations: pl.DataFrame | None = None,
|
|
18
18
|
detailed_zones: pl.DataFrame | None = None,
|
|
19
|
+
skip_insee: bool = False,
|
|
19
20
|
):
|
|
20
21
|
existing_cols = lf.collect_schema().names()
|
|
21
22
|
columns = [variable.name for variable in TRIP_SCHEMA if variable.name in existing_cols]
|
|
@@ -26,8 +27,9 @@ def clean(
|
|
|
26
27
|
lf = add_durations(lf, existing_cols)
|
|
27
28
|
lf = add_weekdays(lf, existing_cols)
|
|
28
29
|
lf = add_lng_lat(lf, existing_cols, special_locations, detailed_zones)
|
|
29
|
-
|
|
30
|
-
|
|
30
|
+
if not skip_insee:
|
|
31
|
+
lf = add_insee_columns(lf, existing_cols)
|
|
32
|
+
lf = add_insee_data_columns(lf, existing_cols, year)
|
|
31
33
|
lf = add_nuts_columns(lf, existing_cols)
|
|
32
34
|
lf = add_intra_zones(lf, existing_cols)
|
|
33
35
|
lf = add_main_mode_groups(lf, existing_cols)
|
|
@@ -111,9 +113,10 @@ def add_purpose_groups(lf: pl.LazyFrame, existing_cols: list[str]):
|
|
|
111
113
|
):
|
|
112
114
|
if col in existing_cols:
|
|
113
115
|
group_col = f"{col}_group"
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
116
|
+
if group_col not in existing_cols:
|
|
117
|
+
lf = lf.with_columns(
|
|
118
|
+
pl.col(col).cast(pl.String).str.extract(r"(\w+):?").alias(group_col)
|
|
119
|
+
)
|
|
117
120
|
return lf
|
|
118
121
|
|
|
119
122
|
|
|
@@ -4,13 +4,7 @@ import polars as pl
|
|
|
4
4
|
|
|
5
5
|
def get_coords(gdf: gpd.GeoDataFrame, name: str) -> pl.DataFrame:
|
|
6
6
|
centroids = gdf.geometry.centroid.to_crs("EPSG:4326")
|
|
7
|
-
df = pl.DataFrame(
|
|
8
|
-
{
|
|
9
|
-
f"{name}_id": gdf[f"{name}_id"],
|
|
10
|
-
"lng": centroids.x,
|
|
11
|
-
"lat": centroids.y,
|
|
12
|
-
}
|
|
13
|
-
)
|
|
7
|
+
df = pl.DataFrame({f"{name}_id": gdf[f"{name}_id"], "lng": centroids.x, "lat": centroids.y})
|
|
14
8
|
return df
|
|
15
9
|
|
|
16
10
|
|
|
@@ -134,6 +134,7 @@ def standardize_trips(
|
|
|
134
134
|
persons: pl.LazyFrame,
|
|
135
135
|
households: pl.LazyFrame,
|
|
136
136
|
detailed_zones: pl.DataFrame | None,
|
|
137
|
+
skip_insee: bool = False,
|
|
137
138
|
):
|
|
138
139
|
lf = scan_trips(filename)
|
|
139
140
|
# Add household_id and person_id.
|
|
@@ -204,6 +205,7 @@ def standardize_trips(
|
|
|
204
205
|
2010,
|
|
205
206
|
perimeter_deps=["75", "77", "78", "91", "92", "93", "94", "95"],
|
|
206
207
|
detailed_zones=detailed_zones,
|
|
208
|
+
skip_insee=skip_insee,
|
|
207
209
|
)
|
|
208
210
|
# When the INSEE code ends with "000" it means "rest of the département".
|
|
209
211
|
# We drop these values because they do not add any additional information compared to `_dep`
|
|
@@ -261,15 +261,13 @@ MOTORCYCLE_CM3_UB_MAP = {
|
|
|
261
261
|
|
|
262
262
|
def scan_households(filename: str):
|
|
263
263
|
separator = detect_csv_delimiter(filename)
|
|
264
|
-
lf = pl.scan_csv(
|
|
265
|
-
filename,
|
|
266
|
-
separator=separator,
|
|
267
|
-
schema_overrides=SCHEMA,
|
|
268
|
-
)
|
|
264
|
+
lf = pl.scan_csv(filename, separator=separator, schema_overrides=SCHEMA)
|
|
269
265
|
return lf
|
|
270
266
|
|
|
271
267
|
|
|
272
|
-
def standardize_households(
|
|
268
|
+
def standardize_households(
|
|
269
|
+
filename: str, detailed_zones: pl.DataFrame | None, skip_insee: bool = False
|
|
270
|
+
):
|
|
273
271
|
lf = scan_households(filename)
|
|
274
272
|
lf = lf.rename(
|
|
275
273
|
{
|
|
@@ -288,6 +286,7 @@ def standardize_households(filename: str, detailed_zones: pl.DataFrame | None):
|
|
|
288
286
|
lf = lf.with_columns(
|
|
289
287
|
original_household_id=pl.struct("NQUEST"),
|
|
290
288
|
survey_method=pl.lit("face_to_face"),
|
|
289
|
+
complete_household=True,
|
|
291
290
|
income_lower_bound=pl.col("REVENU").replace_strict(REVENU_LB_MAP),
|
|
292
291
|
income_upper_bound=pl.col("REVENU").replace_strict(REVENU_UB_MAP),
|
|
293
292
|
housing_type=pl.col("TYPELOG").replace_strict(HOUSING_TYPE_MAP),
|
|
@@ -314,7 +313,7 @@ def standardize_households(filename: str, detailed_zones: pl.DataFrame | None):
|
|
|
314
313
|
.otherwise("home_insee"),
|
|
315
314
|
)
|
|
316
315
|
lf = lf.sort("original_household_id")
|
|
317
|
-
lf = clean_households(lf, year=2010, detailed_zones=detailed_zones)
|
|
316
|
+
lf = clean_households(lf, year=2010, detailed_zones=detailed_zones, skip_insee=skip_insee)
|
|
318
317
|
return lf
|
|
319
318
|
|
|
320
319
|
|
|
@@ -158,11 +158,7 @@ STUDENT_CATEGORY_MAP = {
|
|
|
158
158
|
98: "supérieur:BAC+6&+", # Bac+6 ou plus
|
|
159
159
|
}
|
|
160
160
|
|
|
161
|
-
DRIVING_LICENSE_MAP = {
|
|
162
|
-
1: "yes",
|
|
163
|
-
2: "no",
|
|
164
|
-
3: "in_progress",
|
|
165
|
-
}
|
|
161
|
+
DRIVING_LICENSE_MAP = {1: "yes", 2: "no", 3: "in_progress"}
|
|
166
162
|
|
|
167
163
|
TRAVELED_MAP = {
|
|
168
164
|
1: "yes", # Oui
|
|
@@ -14,7 +14,7 @@ from .trajets import standardize_legs
|
|
|
14
14
|
from .zones import read_detailed_zones
|
|
15
15
|
|
|
16
16
|
|
|
17
|
-
def standardize(source: str | ZipFile, skip_spatial: bool = False):
|
|
17
|
+
def standardize(source: str | ZipFile, skip_spatial: bool = False, skip_insee: bool = False):
|
|
18
18
|
source_name = source.filename if isinstance(source, ZipFile) else source
|
|
19
19
|
logger.info(f"Standardizing EGT2010 survey from `{source_name}`")
|
|
20
20
|
if skip_spatial:
|
|
@@ -33,7 +33,7 @@ def standardize(source: str | ZipFile, skip_spatial: bool = False):
|
|
|
33
33
|
if not filename:
|
|
34
34
|
logger.error(f"Missing households file: {filename}")
|
|
35
35
|
return None
|
|
36
|
-
households = standardize_households(filename, detailed_zone_coords)
|
|
36
|
+
households = standardize_households(filename, detailed_zone_coords, skip_insee)
|
|
37
37
|
# Cars.
|
|
38
38
|
cars = standardize_cars(filename, households)
|
|
39
39
|
# motorcycles.
|
|
@@ -49,7 +49,7 @@ def standardize(source: str | ZipFile, skip_spatial: bool = False):
|
|
|
49
49
|
if not filename:
|
|
50
50
|
logger.error(f"Missing trips file: {filename}")
|
|
51
51
|
return None
|
|
52
|
-
trips = standardize_trips(filename, persons, households, detailed_zone_coords)
|
|
52
|
+
trips = standardize_trips(filename, persons, households, detailed_zone_coords, skip_insee)
|
|
53
53
|
# Legs.
|
|
54
54
|
filename = legs_filename(source)
|
|
55
55
|
if not filename:
|