mobisurvstd 1.4.2__tar.gz → 1.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {mobisurvstd-1.4.2/src/mobisurvstd.egg-info → mobisurvstd-1.5.0}/PKG-INFO +3 -6
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/pyproject.toml +3 -6
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/cerema/deplacements.py +3 -2
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/cerema/edgt.py +1 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/cerema/edvm.py +1 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/cerema/emc2.py +19 -3
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/cerema/menages.py +2 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/cerema/personnes.py +2 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/cerema/survey.py +8 -2
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/cerema/trajets.py +9 -5
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/cerema/zones.py +0 -3
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/common/clean.py +15 -5
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/egt2010/deplacements.py +2 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/egt2010/menages.py +2 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/egt2010/personnes.py +1 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/egt2010/trajets.py +7 -4
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/egt2020/deplacements.py +2 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/egt2020/motos.py +1 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/egt2020/personnes.py +1 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/egt2020/trajets.py +4 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/egt2020/voitures.py +1 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/emg2023/deplacements.py +3 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/emg2023/personnes.py +2 -1
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/emp/deplacements.py +9 -1
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/emp/menages.py +1 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/emp/motos.py +1 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/emp/personnes.py +53 -47
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/emp/survey.py +3 -1
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/emp/voitures.py +1 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/resources/admin_express.py +1 -1
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/resources/insee_data.py +19 -8
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/resources/nuts.py +3 -1
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/schema/legs.py +7 -2
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/utils.py +3 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/validation.py +42 -13
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0/src/mobisurvstd.egg-info}/PKG-INFO +3 -6
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd.egg-info/requires.txt +1 -4
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/LICENSE.md +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/README.md +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/setup.cfg +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/__init__.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/__main__.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/cerema/__init__.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/cerema/common.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/cerema/emd.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/cerema/nantes_opendata.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/cerema/reader.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/classes.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/cli.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/common/__init__.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/common/cars.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/common/households.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/common/legs.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/common/motorcycles.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/common/persons.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/common/trips.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/common/zones.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/egt2010/__init__.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/egt2010/survey.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/egt2010/zones.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/egt2020/__init__.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/egt2020/menages.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/egt2020/survey.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/emg2023/__init__.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/emg2023/menages.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/emg2023/survey.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/emp/__init__.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/emp/common.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/logger.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/main.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/resources/__init__.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/schema/__init__.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/schema/cars.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/schema/common.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/schema/guarantees.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/schema/households.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/schema/motorcycles.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/schema/persons.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/schema/trips.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd.egg-info/SOURCES.txt +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd.egg-info/dependency_links.txt +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd.egg-info/entry_points.txt +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: mobisurvstd
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.5.0
|
|
4
4
|
Summary: A tool to standardize French mobility survey datasets (EMC2, EGT, EMP, etc.) into a unified format.
|
|
5
5
|
Author-email: Lucas Javaudin <lucas@lucasjavaudin.com>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -12,23 +12,20 @@ Keywords: mobility survey,polars,data format
|
|
|
12
12
|
Classifier: Programming Language :: Python
|
|
13
13
|
Classifier: Programming Language :: Python :: 3
|
|
14
14
|
Classifier: Topic :: File Formats
|
|
15
|
-
Requires-Python: >=3.
|
|
15
|
+
Requires-Python: >=3.11
|
|
16
16
|
Description-Content-Type: text/markdown
|
|
17
17
|
License-File: LICENSE.md
|
|
18
|
-
Requires-Dist: fastexcel>=0.14.0
|
|
19
18
|
Requires-Dist: geopandas>=1.0.1
|
|
20
19
|
Requires-Dist: humanize>=4.13.0
|
|
21
20
|
Requires-Dist: loguru>=0.7.3
|
|
22
|
-
Requires-Dist: numpy>=2.2.4
|
|
23
21
|
Requires-Dist: pandas>=2.2.3
|
|
24
22
|
Requires-Dist: platformdirs>=4.3.8
|
|
25
|
-
Requires-Dist: polars>=
|
|
23
|
+
Requires-Dist: polars[calamine]>=2.0.0
|
|
26
24
|
Requires-Dist: py7zr>=1.1.0
|
|
27
25
|
Requires-Dist: pyarrow>=19.0.1
|
|
28
26
|
Requires-Dist: requests>=2.32.3
|
|
29
27
|
Requires-Dist: tqdm>=4.67.3
|
|
30
28
|
Requires-Dist: typer>=0.16.0
|
|
31
|
-
Requires-Dist: typing-extensions>=4.15.0
|
|
32
29
|
Dynamic: license-file
|
|
33
30
|
|
|
34
31
|
[![Contributors][contributors-shield]][contributors-url]
|
|
@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
|
|
|
6
6
|
|
|
7
7
|
[project]
|
|
8
8
|
name = "mobisurvstd"
|
|
9
|
-
version = "1.
|
|
9
|
+
version = "1.5.0"
|
|
10
10
|
description = "A tool to standardize French mobility survey datasets (EMC2, EGT, EMP, etc.) into a unified format."
|
|
11
11
|
readme = "README.md"
|
|
12
12
|
authors = [{ name = "Lucas Javaudin", email = "lucas@lucasjavaudin.com" }]
|
|
@@ -19,22 +19,19 @@ classifiers = [
|
|
|
19
19
|
]
|
|
20
20
|
keywords = ["mobility survey", "polars", "data format"]
|
|
21
21
|
dependencies = [
|
|
22
|
-
"fastexcel>=0.14.0",
|
|
23
22
|
"geopandas>=1.0.1",
|
|
24
23
|
"humanize>=4.13.0",
|
|
25
24
|
"loguru>=0.7.3",
|
|
26
|
-
"numpy>=2.2.4",
|
|
27
25
|
"pandas>=2.2.3",
|
|
28
26
|
"platformdirs>=4.3.8",
|
|
29
|
-
"polars>=
|
|
27
|
+
"polars[calamine]>=2.0.0",
|
|
30
28
|
"py7zr>=1.1.0",
|
|
31
29
|
"pyarrow>=19.0.1",
|
|
32
30
|
"requests>=2.32.3",
|
|
33
31
|
"tqdm>=4.67.3",
|
|
34
32
|
"typer>=0.16.0",
|
|
35
|
-
"typing-extensions>=4.15.0",
|
|
36
33
|
]
|
|
37
|
-
requires-python = ">=3.
|
|
34
|
+
requires-python = ">=3.11"
|
|
38
35
|
|
|
39
36
|
[project.scripts]
|
|
40
37
|
mobisurvstd = "mobisurvstd:cli.app"
|
|
@@ -142,6 +142,7 @@ class TripsReader(CeremaReader):
|
|
|
142
142
|
on="original_person_id",
|
|
143
143
|
how="left",
|
|
144
144
|
coalesce=True,
|
|
145
|
+
maintain_order="left",
|
|
145
146
|
)
|
|
146
147
|
lf = lf.rename(
|
|
147
148
|
{
|
|
@@ -158,11 +159,11 @@ class TripsReader(CeremaReader):
|
|
|
158
159
|
original_trip_id=pl.struct(self.get_trip_index_cols()),
|
|
159
160
|
origin_purpose=pl.col("D2A").replace_strict(PURPOSE_MAP),
|
|
160
161
|
origin_escort_purpose=pl.col("D2B").replace_strict(PURPOSE_MAP),
|
|
161
|
-
origin_shop_type=pl.col("D2A").
|
|
162
|
+
origin_shop_type=pl.col("D2A").replace_strict(SHOP_TYPE_MAP, default=None),
|
|
162
163
|
departure_time=60 * (pl.col("D4") // 100) + pl.col("D4") % 100,
|
|
163
164
|
destination_purpose=pl.col("D5A").replace_strict(PURPOSE_MAP),
|
|
164
165
|
destination_escort_purpose=pl.col("D5B").replace_strict(PURPOSE_MAP),
|
|
165
|
-
destination_shop_type=pl.col("D5A").
|
|
166
|
+
destination_shop_type=pl.col("D5A").replace_strict(SHOP_TYPE_MAP, default=None),
|
|
166
167
|
arrival_time=60 * (pl.col("D8") // 100) + pl.col("D8") % 100,
|
|
167
168
|
trip_euclidean_distance_km=pl.col("D11") / 1e3,
|
|
168
169
|
trip_travel_distance_km=pl.col("D12") / 1e3,
|
|
@@ -36,6 +36,7 @@ class EDVMReader(CeremaStandardizer):
|
|
|
36
36
|
def detailed_zones_filenames(self):
|
|
37
37
|
return [
|
|
38
38
|
find_file_path(self.source, ".*(_zf_.*|_zf|zones?[_ ]?fines?.*|_dfin)[.](tab|shp|mif)")
|
|
39
|
+
or find_file_path(self.source, "edvm_etb_mcg.mif") # Saint-Louis 2011 filename.
|
|
39
40
|
]
|
|
40
41
|
|
|
41
42
|
def special_locations_filenames(self):
|
|
@@ -52,7 +52,7 @@ class EMC2Reader(CeremaStandardizer):
|
|
|
52
52
|
return ["zf_160", "zf_fusion", "codegt", "zf", "num_gt"]
|
|
53
53
|
|
|
54
54
|
def zf_id_from_gt_columns(self):
|
|
55
|
-
return ["zfrat_f", "num_zf_rat", "cd_zf", "num_zf_19", "zf_rattach", "zfrat"]
|
|
55
|
+
return ["zfrat_f", "num_zf_rat", "cd_zf", "num_zf_19", "zf_rattach", "zfrat", "zf_rat"]
|
|
56
56
|
|
|
57
57
|
def gt_name_columns(self):
|
|
58
58
|
return ["nom_gt", "nom_gen", "nom_genera", "libelle", "nom", "rem"]
|
|
@@ -66,7 +66,7 @@ class EMC2Reader(CeremaStandardizer):
|
|
|
66
66
|
return ["zf_fusion", "zf_160", "zf", "num_zf"]
|
|
67
67
|
|
|
68
68
|
def zf_name_columns(self):
|
|
69
|
-
return ["zf_nom", "nom_zf", "lib_zf", "libelle", "rem"]
|
|
69
|
+
return ["zf_nom", "nom_zf", "lib_zf", "libelle", "rem", "nom"]
|
|
70
70
|
|
|
71
71
|
def dtir_id_columns(self):
|
|
72
72
|
return ["dtir_160", "codsect", "ztir", "num_dtir_f", "num_dtir", "dtir"]
|
|
@@ -75,7 +75,15 @@ class EMC2Reader(CeremaStandardizer):
|
|
|
75
75
|
return ["nomdtir", "nom_dtir"]
|
|
76
76
|
|
|
77
77
|
def insee_id_columns(self):
|
|
78
|
-
return [
|
|
78
|
+
return [
|
|
79
|
+
"insee_com",
|
|
80
|
+
"insee_commune",
|
|
81
|
+
"insee_comm",
|
|
82
|
+
"insee_gen",
|
|
83
|
+
"insee",
|
|
84
|
+
"code_com",
|
|
85
|
+
"num_com",
|
|
86
|
+
]
|
|
79
87
|
|
|
80
88
|
def select_dtir_column(self, gdf: gpd.GeoDataFrame):
|
|
81
89
|
# This is the standard case.
|
|
@@ -90,6 +98,7 @@ class EMC2Reader(CeremaStandardizer):
|
|
|
90
98
|
for dtir_col in ("zfrat", "zf"):
|
|
91
99
|
if matching_col := find_matching_column(dtir_col, gdf):
|
|
92
100
|
gdf["draw_zone_id"] = gdf[matching_col].str.slice(0, 3)
|
|
101
|
+
break
|
|
93
102
|
|
|
94
103
|
def select_zf_column(self, gdf: gpd.GeoDataFrame):
|
|
95
104
|
# For Angers 2022, three trailing zeros need to be added to the ZF id column.
|
|
@@ -123,3 +132,10 @@ class EMC2Reader(CeremaStandardizer):
|
|
|
123
132
|
gdf["detailed_zone_id"] = gdf["Pgt"].astype(int) // 10 * 10
|
|
124
133
|
# Normal case. Find the ZF id column by name.
|
|
125
134
|
super().select_zf_from_gt_column(gdf)
|
|
135
|
+
|
|
136
|
+
def preprocess_special_locations(self, gdf: gpd.GeoDataFrame):
|
|
137
|
+
# Special case for Vannes 2023: most special locations are invalid (no id or id of a
|
|
138
|
+
# detailed zone). We filter the valid special locations here.
|
|
139
|
+
if "ZF" in gdf.columns and "ZFRAT" in gdf.columns and (gdf["ZF"] != gdf["ZFRAT"]).any():
|
|
140
|
+
gdf = gdf.loc[(gdf["ZF"] != gdf["ZFRAT"]) & (~gdf["ZF"].isna())].copy()
|
|
141
|
+
return gdf
|
|
@@ -284,6 +284,7 @@ class HouseholdsReader(CeremaReader):
|
|
|
284
284
|
on="original_household_id",
|
|
285
285
|
how="left",
|
|
286
286
|
coalesce=True,
|
|
287
|
+
maintain_order="left",
|
|
287
288
|
)
|
|
288
289
|
# Note: the `.lazy()` after the concat is useless (this is already a LazyFrame) but it helps
|
|
289
290
|
# the syntax checker.
|
|
@@ -334,6 +335,7 @@ class HouseholdsReader(CeremaReader):
|
|
|
334
335
|
on="original_household_id",
|
|
335
336
|
how="left",
|
|
336
337
|
coalesce=True,
|
|
338
|
+
maintain_order="left",
|
|
337
339
|
)
|
|
338
340
|
# Note: the `.lazy()` after the concat is useless (this is already a LazyFrame) but it helps
|
|
339
341
|
# the syntax checker.
|
|
@@ -271,6 +271,7 @@ class PersonsReader(CeremaReader):
|
|
|
271
271
|
on="original_household_id",
|
|
272
272
|
how="left",
|
|
273
273
|
coalesce=True,
|
|
274
|
+
maintain_order="left",
|
|
274
275
|
)
|
|
275
276
|
lf = lf.rename({"P4": "age", "COE1": "sample_weight_all", "COEP": "sample_weight_surveyed"})
|
|
276
277
|
lf = fix_dates(lf, self.survey_name())
|
|
@@ -411,6 +412,7 @@ def fix_work_study_location(lf: pl.LazyFrame, households: pl.LazyFrame):
|
|
|
411
412
|
households.select("household_id", *(f"home_{col}" for col in zone_cols)),
|
|
412
413
|
on="household_id",
|
|
413
414
|
how="left",
|
|
415
|
+
maintain_order="left",
|
|
414
416
|
)
|
|
415
417
|
lf = lf.with_columns(
|
|
416
418
|
pl.when("work_only_at_home", pl.col(f"home_{col}") == pl.col(f"work_{col}"))
|
|
@@ -114,7 +114,11 @@ class CeremaStandardizer(HouseholdsReader, PersonsReader, TripsReader, LegsReade
|
|
|
114
114
|
.collect()
|
|
115
115
|
)
|
|
116
116
|
self.households = self.households.join(
|
|
117
|
-
household_dates.lazy(),
|
|
117
|
+
household_dates.lazy(),
|
|
118
|
+
on="household_id",
|
|
119
|
+
how="left",
|
|
120
|
+
coalesce=True,
|
|
121
|
+
maintain_order="left",
|
|
118
122
|
).with_columns(trips_weekday=pl.when("is_valid_weekday").then(pl.col("trips_weekday")))
|
|
119
123
|
|
|
120
124
|
def fix_main_mode(self):
|
|
@@ -122,7 +126,9 @@ class CeremaStandardizer(HouseholdsReader, PersonsReader, TripsReader, LegsReade
|
|
|
122
126
|
# have mode set to "motorcycle".
|
|
123
127
|
invalid_trips = (
|
|
124
128
|
self.legs.select("trip_id", "mode_group")
|
|
125
|
-
.join(
|
|
129
|
+
.join(
|
|
130
|
+
self.trips.select("trip_id", "main_mode_group"), on="trip_id", maintain_order="left"
|
|
131
|
+
)
|
|
126
132
|
.filter(pl.col("mode_group").eq(pl.col("main_mode_group")).any().over("trip_id").not_())
|
|
127
133
|
.select("trip_id")
|
|
128
134
|
.collect()
|
|
@@ -134,6 +134,7 @@ class LegsReader(CeremaReader):
|
|
|
134
134
|
on="original_trip_id",
|
|
135
135
|
how="left",
|
|
136
136
|
coalesce=True,
|
|
137
|
+
maintain_order="left",
|
|
137
138
|
)
|
|
138
139
|
|
|
139
140
|
lf = lf.with_columns(
|
|
@@ -260,7 +261,7 @@ class LegsReader(CeremaReader):
|
|
|
260
261
|
# We create a single leg for them, with the data we have.
|
|
261
262
|
lf5 = (
|
|
262
263
|
self.trips.filter(pl.col("main_mode") != "walking")
|
|
263
|
-
.join(lf, on="original_trip_id", how="anti")
|
|
264
|
+
.join(lf, on="original_trip_id", how="anti", maintain_order="left")
|
|
264
265
|
.select(
|
|
265
266
|
"household_id",
|
|
266
267
|
"person_id",
|
|
@@ -281,16 +282,16 @@ class LegsReader(CeremaReader):
|
|
|
281
282
|
# Concatenate the 5 leg types.
|
|
282
283
|
lf = pl.concat((lf1, lf2, lf3, lf4, lf5), how="diagonal")
|
|
283
284
|
# Add car and motorcycle types.
|
|
285
|
+
# Trucks ("fourgon, camionnette, camion") are reported in the same way as cars.
|
|
286
|
+
is_car_or_truck = pl.col("mode").str.contains("^(car|truck):")
|
|
284
287
|
lf = lf.with_columns(
|
|
285
|
-
car_type=pl.when(
|
|
288
|
+
car_type=pl.when(is_car_or_truck).then(
|
|
286
289
|
pl.col("T7").replace_strict(CAR_MAP, default=None)
|
|
287
290
|
),
|
|
288
291
|
motorcycle_type=pl.when(pl.col("mode").str.starts_with("motorcycle:")).then(
|
|
289
292
|
pl.col("T7").replace_strict(MOTORCYCLE_MAP, default=None)
|
|
290
293
|
),
|
|
291
|
-
original_car_index=pl.when(
|
|
292
|
-
pl.col("mode").str.starts_with("car:") & pl.col("T7").is_between(1, 4)
|
|
293
|
-
)
|
|
294
|
+
original_car_index=pl.when(is_car_or_truck & pl.col("T7").is_between(1, 4))
|
|
294
295
|
.then("T7")
|
|
295
296
|
.cast(pl.UInt8),
|
|
296
297
|
# The `original_motorcycle_index` is either 1, 2, 3, 4 or 11, 12, 13, 14 depending on
|
|
@@ -309,6 +310,7 @@ class LegsReader(CeremaReader):
|
|
|
309
310
|
on=["household_id", "original_car_index"],
|
|
310
311
|
how="left",
|
|
311
312
|
coalesce=True,
|
|
313
|
+
maintain_order="left",
|
|
312
314
|
)
|
|
313
315
|
# For Angers 2012 (and maybe others), column T7 is sometimes set to 2 (for example) while
|
|
314
316
|
# the household has no car with index 2.
|
|
@@ -324,6 +326,7 @@ class LegsReader(CeremaReader):
|
|
|
324
326
|
on=["household_id", "original_motorcycle_index"],
|
|
325
327
|
how="left",
|
|
326
328
|
coalesce=True,
|
|
329
|
+
maintain_order="left",
|
|
327
330
|
)
|
|
328
331
|
# Same fix as for cars.
|
|
329
332
|
lf = lf.with_columns(
|
|
@@ -341,6 +344,7 @@ class LegsReader(CeremaReader):
|
|
|
341
344
|
self.persons.select("person_id", is_major=pl.col("age") >= 18),
|
|
342
345
|
on="person_id",
|
|
343
346
|
how="left",
|
|
347
|
+
maintain_order="left",
|
|
344
348
|
)
|
|
345
349
|
lf = lf.with_columns(
|
|
346
350
|
nb_majors_in_vehicle=pl.when("is_major" | pl.col("mode").str.contains("passenger"))
|
|
@@ -152,9 +152,6 @@ class ZonesReader(CeremaReader):
|
|
|
152
152
|
for zf_col in self.zf_id_columns():
|
|
153
153
|
if matching_col := find_matching_column(zf_col, gdf):
|
|
154
154
|
gdf["detailed_zone_id"] = gdf[matching_col]
|
|
155
|
-
# Drop the original column so that it will not be wrongly read when finding the
|
|
156
|
-
# matching ST id column.
|
|
157
|
-
gdf.drop(columns=[matching_col], inplace=True)
|
|
158
155
|
break
|
|
159
156
|
|
|
160
157
|
def select_gt_column(self, gdf: gpd.GeoDataFrame):
|
|
@@ -178,7 +178,11 @@ def count_nb_persons(households: pl.LazyFrame, persons: pl.LazyFrame):
|
|
|
178
178
|
# Join on both "household_id" and "complete_household" so that counts are not added for
|
|
179
179
|
# incomplete households.
|
|
180
180
|
households = households.join(
|
|
181
|
-
person_counts,
|
|
181
|
+
person_counts,
|
|
182
|
+
on=["household_id", "complete_household"],
|
|
183
|
+
how="left",
|
|
184
|
+
coalesce=True,
|
|
185
|
+
maintain_order="left",
|
|
182
186
|
)
|
|
183
187
|
return households
|
|
184
188
|
|
|
@@ -211,6 +215,7 @@ def add_household_type(households: pl.LazyFrame, persons: pl.LazyFrame):
|
|
|
211
215
|
on="household_id",
|
|
212
216
|
how="left",
|
|
213
217
|
coalesce=True,
|
|
218
|
+
maintain_order="left",
|
|
214
219
|
)
|
|
215
220
|
households = households.with_columns(
|
|
216
221
|
household_type=pl.when(nb_persons=1, nb_men=1)
|
|
@@ -233,7 +238,9 @@ def add_household_type(households: pl.LazyFrame, persons: pl.LazyFrame):
|
|
|
233
238
|
|
|
234
239
|
def count_nb_trips(persons: pl.LazyFrame, trips: pl.LazyFrame):
|
|
235
240
|
trip_counts = trips.group_by("person_id").agg(nb_trips=pl.len())
|
|
236
|
-
persons = persons.join(
|
|
241
|
+
persons = persons.join(
|
|
242
|
+
trip_counts, on="person_id", how="left", coalesce=True, maintain_order="left"
|
|
243
|
+
)
|
|
237
244
|
# Set nb_trips = 0 for surveyed persons.
|
|
238
245
|
persons = persons.with_columns(
|
|
239
246
|
nb_trips=pl.when("is_surveyed").then(pl.col("nb_trips").fill_null(0)).otherwise("nb_trips")
|
|
@@ -275,6 +282,7 @@ def add_worked_during_surveyed_day(persons: pl.LazyFrame, trips: pl.LazyFrame):
|
|
|
275
282
|
on="person_id",
|
|
276
283
|
how="left",
|
|
277
284
|
coalesce=True,
|
|
285
|
+
maintain_order="left",
|
|
278
286
|
)
|
|
279
287
|
persons = persons.with_columns(
|
|
280
288
|
worked_during_surveyed_day=pl.when("has_work_activity")
|
|
@@ -314,7 +322,7 @@ def count_nb_legs(trips: pl.LazyFrame, legs: pl.LazyFrame):
|
|
|
314
322
|
]
|
|
315
323
|
)
|
|
316
324
|
leg_counts = legs.group_by("trip_id").agg(agg_cols)
|
|
317
|
-
trips = trips.join(leg_counts, on="trip_id", how="left", coalesce=True)
|
|
325
|
+
trips = trips.join(leg_counts, on="trip_id", how="left", coalesce=True, maintain_order="left")
|
|
318
326
|
if has_mode_group:
|
|
319
327
|
trips = add_intermodality_column(trips)
|
|
320
328
|
return trips
|
|
@@ -388,7 +396,7 @@ def add_main_mode(trips: pl.LazyFrame, legs: pl.LazyFrame):
|
|
|
388
396
|
)
|
|
389
397
|
)
|
|
390
398
|
# Add the `main_mode_group` and `mode_group` columns to the trips.
|
|
391
|
-
trips = trips.join(main_modes, on="trip_id", how="left", coalesce=True)
|
|
399
|
+
trips = trips.join(main_modes, on="trip_id", how="left", coalesce=True, maintain_order="left")
|
|
392
400
|
# Remove the `main_mode` column if the legs modes are unknown.
|
|
393
401
|
if not has_modes:
|
|
394
402
|
trips = trips.drop("main_mode")
|
|
@@ -418,7 +426,9 @@ def add_access_egress_modes(trips: pl.LazyFrame, legs: pl.LazyFrame):
|
|
|
418
426
|
# Egress mode is the mode of the last leg if the trip's `main_mode_group` is "public_transit"
|
|
419
427
|
# and the last leg's `mode_group` is not "public_transit", otherwise it is NULL.
|
|
420
428
|
is_pt_trip = pl.col("main_mode_group") == "public_transit"
|
|
421
|
-
trips = trips.join(
|
|
429
|
+
trips = trips.join(
|
|
430
|
+
first_last_leg_modes, on="trip_id", how="left", coalesce=True, maintain_order="left"
|
|
431
|
+
).with_columns(
|
|
422
432
|
public_transit_access_mode=pl.when(
|
|
423
433
|
is_pt_trip & pl.col("first_mode_group").ne("public_transit")
|
|
424
434
|
).then("first_mode"),
|
|
@@ -145,6 +145,7 @@ def standardize_trips(
|
|
|
145
145
|
on="original_person_id",
|
|
146
146
|
how="left",
|
|
147
147
|
coalesce=True,
|
|
148
|
+
maintain_order="left",
|
|
148
149
|
)
|
|
149
150
|
# Add interview_date.
|
|
150
151
|
lf = lf.join(
|
|
@@ -152,6 +153,7 @@ def standardize_trips(
|
|
|
152
153
|
on="household_id",
|
|
153
154
|
how="left",
|
|
154
155
|
coalesce=True,
|
|
156
|
+
maintain_order="left",
|
|
155
157
|
)
|
|
156
158
|
lf = lf.rename(
|
|
157
159
|
{
|
|
@@ -326,6 +326,7 @@ def standardize_cars(filename: Path, households: pl.LazyFrame):
|
|
|
326
326
|
on="original_household_id",
|
|
327
327
|
how="left",
|
|
328
328
|
coalesce=True,
|
|
329
|
+
maintain_order="left",
|
|
329
330
|
)
|
|
330
331
|
lf = pl.concat(
|
|
331
332
|
(
|
|
@@ -366,6 +367,7 @@ def standardize_motorcycles(filename: Path, households: pl.LazyFrame):
|
|
|
366
367
|
on="original_household_id",
|
|
367
368
|
how="left",
|
|
368
369
|
coalesce=True,
|
|
370
|
+
maintain_order="left",
|
|
369
371
|
)
|
|
370
372
|
lf = pl.concat(
|
|
371
373
|
(
|
|
@@ -62,7 +62,7 @@ MODE_MAP = {
|
|
|
62
62
|
50: "car:driver", # Conducteur voiture particulière
|
|
63
63
|
51: "car:driver", # Conducteur dans un système de covoiturage organisé
|
|
64
64
|
52: "car:driver", # Conducteur véhicule utilitaire 800 à 1 000 kg
|
|
65
|
-
53: "
|
|
65
|
+
53: "truck:driver", # Conducteur véhicule utilitaire de 1 000 kg ou plus
|
|
66
66
|
54: "motorcycle:driver:moped", # Conducteur véhicule à 2 (ou 3) roues à moteur non immatriculé
|
|
67
67
|
55: "motorcycle:driver", # Conducteur véhicule à 2 (ou 3) roues à moteur immatriculé
|
|
68
68
|
# Vélo
|
|
@@ -74,7 +74,7 @@ MODE_MAP = {
|
|
|
74
74
|
70: "car:passenger", # Passager d'une voiture particulière
|
|
75
75
|
71: "car:passenger", # Passager dans un système de covoiturage organisé
|
|
76
76
|
72: "car:passenger", # Passager d'un véhicule utilitaire 800 à 1 000 kg
|
|
77
|
-
73: "
|
|
77
|
+
73: "truck:passenger", # Passager véhicule utilitaire de 1 000 kg ou plus
|
|
78
78
|
74: "motorcycle:passenger:moped", # Passager d'un véhicule à 2 (ou 3) roues à moteur non immatriculé
|
|
79
79
|
75: "motorcycle:passenger", # Passager d'un véhicule à 2 (ou 3) roues à moteur immatriculé
|
|
80
80
|
# Autre moyen de transport
|
|
@@ -171,6 +171,7 @@ def standardize_legs(
|
|
|
171
171
|
on="original_trip_id",
|
|
172
172
|
how="left",
|
|
173
173
|
coalesce=True,
|
|
174
|
+
maintain_order="left",
|
|
174
175
|
)
|
|
175
176
|
lf = lf.rename(
|
|
176
177
|
{
|
|
@@ -193,8 +194,8 @@ def standardize_legs(
|
|
|
193
194
|
parking_type=pl.col("TSTAT").replace_strict(PARKING_TYPE_MAP),
|
|
194
195
|
)
|
|
195
196
|
lf = lf.with_columns(
|
|
196
|
-
car_type=pl.when(pl.col("mode").str.
|
|
197
|
-
car_index=pl.when(pl.col("mode").str.
|
|
197
|
+
car_type=pl.when(pl.col("mode").str.contains("^(car|truck):")).then("vehicle_type"),
|
|
198
|
+
car_index=pl.when(pl.col("mode").str.contains("^(car|truck):")).then("vehicle_index"),
|
|
198
199
|
motorcycle_type=pl.when(pl.col("mode").str.starts_with("motorcycle:")).then("vehicle_type"),
|
|
199
200
|
motorcycle_index=pl.when(pl.col("mode").str.starts_with("motorcycle:")).then(
|
|
200
201
|
"vehicle_index"
|
|
@@ -206,6 +207,7 @@ def standardize_legs(
|
|
|
206
207
|
on=["household_id", "car_index"],
|
|
207
208
|
how="left",
|
|
208
209
|
coalesce=True,
|
|
210
|
+
maintain_order="left",
|
|
209
211
|
)
|
|
210
212
|
# Add motorcycle id.
|
|
211
213
|
lf = lf.join(
|
|
@@ -213,6 +215,7 @@ def standardize_legs(
|
|
|
213
215
|
on=["household_id", "motorcycle_index"],
|
|
214
216
|
how="left",
|
|
215
217
|
coalesce=True,
|
|
218
|
+
maintain_order="left",
|
|
216
219
|
)
|
|
217
220
|
# In 3 cases, the motorcycle index TYPV is not an existing motorcycle of the household so the
|
|
218
221
|
# `motorcycle_type` is changed from "household" to "other_household".
|
|
@@ -139,6 +139,7 @@ def standardize_trips(
|
|
|
139
139
|
on="original_person_id",
|
|
140
140
|
how="left",
|
|
141
141
|
coalesce=True,
|
|
142
|
+
maintain_order="left",
|
|
142
143
|
)
|
|
143
144
|
# Add interview_date.
|
|
144
145
|
lf = lf.join(
|
|
@@ -146,6 +147,7 @@ def standardize_trips(
|
|
|
146
147
|
on="household_id",
|
|
147
148
|
how="left",
|
|
148
149
|
coalesce=True,
|
|
150
|
+
maintain_order="left",
|
|
149
151
|
)
|
|
150
152
|
lf = lf.rename(
|
|
151
153
|
{
|
|
@@ -257,6 +257,7 @@ def standardize_persons(filename: str, households: pl.LazyFrame):
|
|
|
257
257
|
on="original_household_id",
|
|
258
258
|
how="left",
|
|
259
259
|
coalesce=True,
|
|
260
|
+
maintain_order="left",
|
|
260
261
|
)
|
|
261
262
|
# For v3, POIDSTI does not exist but can be read from household's `sample_weight`.
|
|
262
263
|
lf = lf.rename(
|
|
@@ -220,6 +220,7 @@ def standardize_legs(
|
|
|
220
220
|
on="original_trip_id",
|
|
221
221
|
how="left",
|
|
222
222
|
coalesce=True,
|
|
223
|
+
maintain_order="left",
|
|
223
224
|
)
|
|
224
225
|
lf = lf.rename(
|
|
225
226
|
{
|
|
@@ -294,6 +295,7 @@ def standardize_legs(
|
|
|
294
295
|
on=["household_id", "car_index"],
|
|
295
296
|
how="left",
|
|
296
297
|
coalesce=True,
|
|
298
|
+
maintain_order="left",
|
|
297
299
|
)
|
|
298
300
|
# Add motorcycle id.
|
|
299
301
|
lf = lf.join(
|
|
@@ -301,6 +303,7 @@ def standardize_legs(
|
|
|
301
303
|
on=["household_id", "motorcycle_index"],
|
|
302
304
|
how="left",
|
|
303
305
|
coalesce=True,
|
|
306
|
+
maintain_order="left",
|
|
304
307
|
)
|
|
305
308
|
# Add `in_vehicle_person_ids`.
|
|
306
309
|
lf = (
|
|
@@ -311,6 +314,7 @@ def standardize_legs(
|
|
|
311
314
|
on="household_id",
|
|
312
315
|
how="left",
|
|
313
316
|
coalesce=True,
|
|
317
|
+
maintain_order="left",
|
|
314
318
|
)
|
|
315
319
|
# Group by leg_id to end up again with one row per leg.
|
|
316
320
|
.group_by("original_leg_id", maintain_order=True)
|
|
@@ -99,6 +99,7 @@ def standardize_trips(
|
|
|
99
99
|
on="original_person_id",
|
|
100
100
|
how="left",
|
|
101
101
|
coalesce=True,
|
|
102
|
+
maintain_order="left",
|
|
102
103
|
)
|
|
103
104
|
|
|
104
105
|
lf = lf.with_columns(
|
|
@@ -147,6 +148,7 @@ def standardize_trips(
|
|
|
147
148
|
on="original_trip_id",
|
|
148
149
|
how="left",
|
|
149
150
|
coalesce=True,
|
|
151
|
+
maintain_order="left",
|
|
150
152
|
)
|
|
151
153
|
|
|
152
154
|
lf = lf.with_columns(
|
|
@@ -225,6 +227,7 @@ def standardize_legs(filename: str, trips: pl.LazyFrame):
|
|
|
225
227
|
on="original_trip_id",
|
|
226
228
|
how="left",
|
|
227
229
|
coalesce=True,
|
|
230
|
+
maintain_order="left",
|
|
228
231
|
)
|
|
229
232
|
# Drop legs with NULL mode (there is by default 5 legs by trip).
|
|
230
233
|
lf = lf.filter(pl.col("mode").is_not_null(), pl.col("mode").ne(""))
|
|
@@ -66,7 +66,7 @@ PCS_GROUP_CODE_MAP = {
|
|
|
66
66
|
4: 5, # EMG: Employés
|
|
67
67
|
5: 6, # EMG: Ouvriers
|
|
68
68
|
6: 7, # EMG: Retraité ou pré-retraité
|
|
69
|
-
7:
|
|
69
|
+
7: 8, # EMG: Étudiant ou lycée
|
|
70
70
|
8: 8, # Au chômage ou en inactivité)
|
|
71
71
|
}
|
|
72
72
|
|
|
@@ -97,6 +97,7 @@ def standardize_persons(filename: str, households: pl.LazyFrame):
|
|
|
97
97
|
on="original_household_id",
|
|
98
98
|
how="left",
|
|
99
99
|
coalesce=True,
|
|
100
|
+
maintain_order="left",
|
|
100
101
|
)
|
|
101
102
|
|
|
102
103
|
lf = lf.with_columns(
|
|
@@ -244,6 +244,7 @@ def standardize_trips(filename: str, persons: pl.LazyFrame):
|
|
|
244
244
|
on="original_person_id",
|
|
245
245
|
how="left",
|
|
246
246
|
coalesce=True,
|
|
247
|
+
maintain_order="left",
|
|
247
248
|
)
|
|
248
249
|
lf = lf.rename(
|
|
249
250
|
{
|
|
@@ -342,7 +343,13 @@ def standardize_trips(filename: str, persons: pl.LazyFrame):
|
|
|
342
343
|
.lazy()
|
|
343
344
|
)
|
|
344
345
|
for prefix in ("origin", "destination"):
|
|
345
|
-
lf = lf.join(
|
|
346
|
+
lf = lf.join(
|
|
347
|
+
old_reg_df,
|
|
348
|
+
left_on=f"{prefix}_nuts2",
|
|
349
|
+
right_on="NUTS2",
|
|
350
|
+
how="left",
|
|
351
|
+
maintain_order="left",
|
|
352
|
+
).rename(
|
|
346
353
|
{
|
|
347
354
|
"NUTS2_name": f"{prefix}_nuts2_name",
|
|
348
355
|
"NUTS1": f"{prefix}_nuts1",
|
|
@@ -370,6 +377,7 @@ def standardize_legs(filename: str, trips: pl.LazyFrame):
|
|
|
370
377
|
on="original_trip_id",
|
|
371
378
|
how="left",
|
|
372
379
|
coalesce=True,
|
|
380
|
+
maintain_order="left",
|
|
373
381
|
)
|
|
374
382
|
lf = lf.filter(
|
|
375
383
|
# Drop legs with NULL mode (there is by default 4 legs by trip).
|