mobisurvstd 1.4.2__tar.gz → 1.6.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {mobisurvstd-1.4.2/src/mobisurvstd.egg-info → mobisurvstd-1.6.0}/PKG-INFO +3 -6
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/pyproject.toml +4 -6
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/cerema/deplacements.py +3 -2
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/cerema/edgt.py +1 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/cerema/edvm.py +1 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/cerema/emc2.py +19 -3
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/cerema/menages.py +2 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/cerema/personnes.py +2 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/cerema/survey.py +8 -2
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/cerema/trajets.py +9 -5
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/cerema/zones.py +0 -3
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/common/clean.py +55 -5
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/egt2010/deplacements.py +2 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/egt2010/menages.py +2 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/egt2010/personnes.py +1 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/egt2010/trajets.py +7 -4
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/egt2020/deplacements.py +2 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/egt2020/motos.py +1 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/egt2020/personnes.py +1 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/egt2020/trajets.py +4 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/egt2020/voitures.py +1 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/emg2023/deplacements.py +3 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/emg2023/personnes.py +46 -2
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/emg2023/survey.py +8 -8
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/emp/deplacements.py +9 -1
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/emp/menages.py +1 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/emp/motos.py +1 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/emp/personnes.py +123 -49
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/emp/survey.py +11 -7
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/emp/voitures.py +1 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/resources/admin_express.py +1 -1
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/resources/insee_data.py +19 -8
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/resources/nuts.py +3 -1
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/schema/__init__.py +1 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/schema/common.py +2 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/schema/households.py +2 -5
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/schema/legs.py +7 -2
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/schema/persons.py +11 -1
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/schema/trips.py +2 -5
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/utils.py +3 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/validation.py +74 -13
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0/src/mobisurvstd.egg-info}/PKG-INFO +3 -6
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd.egg-info/requires.txt +1 -4
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/LICENSE.md +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/README.md +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/setup.cfg +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/__init__.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/__main__.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/cerema/__init__.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/cerema/common.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/cerema/emd.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/cerema/nantes_opendata.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/cerema/reader.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/classes.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/cli.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/common/__init__.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/common/cars.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/common/households.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/common/legs.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/common/motorcycles.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/common/persons.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/common/trips.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/common/zones.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/egt2010/__init__.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/egt2010/survey.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/egt2010/zones.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/egt2020/__init__.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/egt2020/menages.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/egt2020/survey.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/emg2023/__init__.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/emg2023/menages.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/emp/__init__.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/emp/common.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/logger.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/main.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/resources/__init__.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/schema/cars.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/schema/guarantees.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/schema/motorcycles.py +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd.egg-info/SOURCES.txt +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd.egg-info/dependency_links.txt +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd.egg-info/entry_points.txt +0 -0
- {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: mobisurvstd
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.6.0
|
|
4
4
|
Summary: A tool to standardize French mobility survey datasets (EMC2, EGT, EMP, etc.) into a unified format.
|
|
5
5
|
Author-email: Lucas Javaudin <lucas@lucasjavaudin.com>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -12,23 +12,20 @@ Keywords: mobility survey,polars,data format
|
|
|
12
12
|
Classifier: Programming Language :: Python
|
|
13
13
|
Classifier: Programming Language :: Python :: 3
|
|
14
14
|
Classifier: Topic :: File Formats
|
|
15
|
-
Requires-Python: >=3.
|
|
15
|
+
Requires-Python: >=3.11
|
|
16
16
|
Description-Content-Type: text/markdown
|
|
17
17
|
License-File: LICENSE.md
|
|
18
|
-
Requires-Dist: fastexcel>=0.14.0
|
|
19
18
|
Requires-Dist: geopandas>=1.0.1
|
|
20
19
|
Requires-Dist: humanize>=4.13.0
|
|
21
20
|
Requires-Dist: loguru>=0.7.3
|
|
22
|
-
Requires-Dist: numpy>=2.2.4
|
|
23
21
|
Requires-Dist: pandas>=2.2.3
|
|
24
22
|
Requires-Dist: platformdirs>=4.3.8
|
|
25
|
-
Requires-Dist: polars>=
|
|
23
|
+
Requires-Dist: polars[calamine]>=2.0.0
|
|
26
24
|
Requires-Dist: py7zr>=1.1.0
|
|
27
25
|
Requires-Dist: pyarrow>=19.0.1
|
|
28
26
|
Requires-Dist: requests>=2.32.3
|
|
29
27
|
Requires-Dist: tqdm>=4.67.3
|
|
30
28
|
Requires-Dist: typer>=0.16.0
|
|
31
|
-
Requires-Dist: typing-extensions>=4.15.0
|
|
32
29
|
Dynamic: license-file
|
|
33
30
|
|
|
34
31
|
[![Contributors][contributors-shield]][contributors-url]
|
|
@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
|
|
|
6
6
|
|
|
7
7
|
[project]
|
|
8
8
|
name = "mobisurvstd"
|
|
9
|
-
version = "1.
|
|
9
|
+
version = "1.6.0"
|
|
10
10
|
description = "A tool to standardize French mobility survey datasets (EMC2, EGT, EMP, etc.) into a unified format."
|
|
11
11
|
readme = "README.md"
|
|
12
12
|
authors = [{ name = "Lucas Javaudin", email = "lucas@lucasjavaudin.com" }]
|
|
@@ -19,22 +19,19 @@ classifiers = [
|
|
|
19
19
|
]
|
|
20
20
|
keywords = ["mobility survey", "polars", "data format"]
|
|
21
21
|
dependencies = [
|
|
22
|
-
"fastexcel>=0.14.0",
|
|
23
22
|
"geopandas>=1.0.1",
|
|
24
23
|
"humanize>=4.13.0",
|
|
25
24
|
"loguru>=0.7.3",
|
|
26
|
-
"numpy>=2.2.4",
|
|
27
25
|
"pandas>=2.2.3",
|
|
28
26
|
"platformdirs>=4.3.8",
|
|
29
|
-
"polars>=
|
|
27
|
+
"polars[calamine]>=2.0.0",
|
|
30
28
|
"py7zr>=1.1.0",
|
|
31
29
|
"pyarrow>=19.0.1",
|
|
32
30
|
"requests>=2.32.3",
|
|
33
31
|
"tqdm>=4.67.3",
|
|
34
32
|
"typer>=0.16.0",
|
|
35
|
-
"typing-extensions>=4.15.0",
|
|
36
33
|
]
|
|
37
|
-
requires-python = ">=3.
|
|
34
|
+
requires-python = ">=3.11"
|
|
38
35
|
|
|
39
36
|
[project.scripts]
|
|
40
37
|
mobisurvstd = "mobisurvstd:cli.app"
|
|
@@ -59,6 +56,7 @@ dev = [
|
|
|
59
56
|
|
|
60
57
|
[tool.ruff]
|
|
61
58
|
line-length = 100
|
|
59
|
+
extend-exclude = ["*.md"]
|
|
62
60
|
|
|
63
61
|
[tool.ruff.format]
|
|
64
62
|
skip-magic-trailing-comma = true
|
|
@@ -142,6 +142,7 @@ class TripsReader(CeremaReader):
|
|
|
142
142
|
on="original_person_id",
|
|
143
143
|
how="left",
|
|
144
144
|
coalesce=True,
|
|
145
|
+
maintain_order="left",
|
|
145
146
|
)
|
|
146
147
|
lf = lf.rename(
|
|
147
148
|
{
|
|
@@ -158,11 +159,11 @@ class TripsReader(CeremaReader):
|
|
|
158
159
|
original_trip_id=pl.struct(self.get_trip_index_cols()),
|
|
159
160
|
origin_purpose=pl.col("D2A").replace_strict(PURPOSE_MAP),
|
|
160
161
|
origin_escort_purpose=pl.col("D2B").replace_strict(PURPOSE_MAP),
|
|
161
|
-
origin_shop_type=pl.col("D2A").
|
|
162
|
+
origin_shop_type=pl.col("D2A").replace_strict(SHOP_TYPE_MAP, default=None),
|
|
162
163
|
departure_time=60 * (pl.col("D4") // 100) + pl.col("D4") % 100,
|
|
163
164
|
destination_purpose=pl.col("D5A").replace_strict(PURPOSE_MAP),
|
|
164
165
|
destination_escort_purpose=pl.col("D5B").replace_strict(PURPOSE_MAP),
|
|
165
|
-
destination_shop_type=pl.col("D5A").
|
|
166
|
+
destination_shop_type=pl.col("D5A").replace_strict(SHOP_TYPE_MAP, default=None),
|
|
166
167
|
arrival_time=60 * (pl.col("D8") // 100) + pl.col("D8") % 100,
|
|
167
168
|
trip_euclidean_distance_km=pl.col("D11") / 1e3,
|
|
168
169
|
trip_travel_distance_km=pl.col("D12") / 1e3,
|
|
@@ -36,6 +36,7 @@ class EDVMReader(CeremaStandardizer):
|
|
|
36
36
|
def detailed_zones_filenames(self):
|
|
37
37
|
return [
|
|
38
38
|
find_file_path(self.source, ".*(_zf_.*|_zf|zones?[_ ]?fines?.*|_dfin)[.](tab|shp|mif)")
|
|
39
|
+
or find_file_path(self.source, "edvm_etb_mcg.mif") # Saint-Louis 2011 filename.
|
|
39
40
|
]
|
|
40
41
|
|
|
41
42
|
def special_locations_filenames(self):
|
|
@@ -52,7 +52,7 @@ class EMC2Reader(CeremaStandardizer):
|
|
|
52
52
|
return ["zf_160", "zf_fusion", "codegt", "zf", "num_gt"]
|
|
53
53
|
|
|
54
54
|
def zf_id_from_gt_columns(self):
|
|
55
|
-
return ["zfrat_f", "num_zf_rat", "cd_zf", "num_zf_19", "zf_rattach", "zfrat"]
|
|
55
|
+
return ["zfrat_f", "num_zf_rat", "cd_zf", "num_zf_19", "zf_rattach", "zfrat", "zf_rat"]
|
|
56
56
|
|
|
57
57
|
def gt_name_columns(self):
|
|
58
58
|
return ["nom_gt", "nom_gen", "nom_genera", "libelle", "nom", "rem"]
|
|
@@ -66,7 +66,7 @@ class EMC2Reader(CeremaStandardizer):
|
|
|
66
66
|
return ["zf_fusion", "zf_160", "zf", "num_zf"]
|
|
67
67
|
|
|
68
68
|
def zf_name_columns(self):
|
|
69
|
-
return ["zf_nom", "nom_zf", "lib_zf", "libelle", "rem"]
|
|
69
|
+
return ["zf_nom", "nom_zf", "lib_zf", "libelle", "rem", "nom"]
|
|
70
70
|
|
|
71
71
|
def dtir_id_columns(self):
|
|
72
72
|
return ["dtir_160", "codsect", "ztir", "num_dtir_f", "num_dtir", "dtir"]
|
|
@@ -75,7 +75,15 @@ class EMC2Reader(CeremaStandardizer):
|
|
|
75
75
|
return ["nomdtir", "nom_dtir"]
|
|
76
76
|
|
|
77
77
|
def insee_id_columns(self):
|
|
78
|
-
return [
|
|
78
|
+
return [
|
|
79
|
+
"insee_com",
|
|
80
|
+
"insee_commune",
|
|
81
|
+
"insee_comm",
|
|
82
|
+
"insee_gen",
|
|
83
|
+
"insee",
|
|
84
|
+
"code_com",
|
|
85
|
+
"num_com",
|
|
86
|
+
]
|
|
79
87
|
|
|
80
88
|
def select_dtir_column(self, gdf: gpd.GeoDataFrame):
|
|
81
89
|
# This is the standard case.
|
|
@@ -90,6 +98,7 @@ class EMC2Reader(CeremaStandardizer):
|
|
|
90
98
|
for dtir_col in ("zfrat", "zf"):
|
|
91
99
|
if matching_col := find_matching_column(dtir_col, gdf):
|
|
92
100
|
gdf["draw_zone_id"] = gdf[matching_col].str.slice(0, 3)
|
|
101
|
+
break
|
|
93
102
|
|
|
94
103
|
def select_zf_column(self, gdf: gpd.GeoDataFrame):
|
|
95
104
|
# For Angers 2022, three trailing zeros need to be added to the ZF id column.
|
|
@@ -123,3 +132,10 @@ class EMC2Reader(CeremaStandardizer):
|
|
|
123
132
|
gdf["detailed_zone_id"] = gdf["Pgt"].astype(int) // 10 * 10
|
|
124
133
|
# Normal case. Find the ZF id column by name.
|
|
125
134
|
super().select_zf_from_gt_column(gdf)
|
|
135
|
+
|
|
136
|
+
def preprocess_special_locations(self, gdf: gpd.GeoDataFrame):
|
|
137
|
+
# Special case for Vannes 2023: most special locations are invalid (no id or id of a
|
|
138
|
+
# detailed zone). We filter the valid special locations here.
|
|
139
|
+
if "ZF" in gdf.columns and "ZFRAT" in gdf.columns and (gdf["ZF"] != gdf["ZFRAT"]).any():
|
|
140
|
+
gdf = gdf.loc[(gdf["ZF"] != gdf["ZFRAT"]) & (~gdf["ZF"].isna())].copy()
|
|
141
|
+
return gdf
|
|
@@ -284,6 +284,7 @@ class HouseholdsReader(CeremaReader):
|
|
|
284
284
|
on="original_household_id",
|
|
285
285
|
how="left",
|
|
286
286
|
coalesce=True,
|
|
287
|
+
maintain_order="left",
|
|
287
288
|
)
|
|
288
289
|
# Note: the `.lazy()` after the concat is useless (this is already a LazyFrame) but it helps
|
|
289
290
|
# the syntax checker.
|
|
@@ -334,6 +335,7 @@ class HouseholdsReader(CeremaReader):
|
|
|
334
335
|
on="original_household_id",
|
|
335
336
|
how="left",
|
|
336
337
|
coalesce=True,
|
|
338
|
+
maintain_order="left",
|
|
337
339
|
)
|
|
338
340
|
# Note: the `.lazy()` after the concat is useless (this is already a LazyFrame) but it helps
|
|
339
341
|
# the syntax checker.
|
|
@@ -271,6 +271,7 @@ class PersonsReader(CeremaReader):
|
|
|
271
271
|
on="original_household_id",
|
|
272
272
|
how="left",
|
|
273
273
|
coalesce=True,
|
|
274
|
+
maintain_order="left",
|
|
274
275
|
)
|
|
275
276
|
lf = lf.rename({"P4": "age", "COE1": "sample_weight_all", "COEP": "sample_weight_surveyed"})
|
|
276
277
|
lf = fix_dates(lf, self.survey_name())
|
|
@@ -411,6 +412,7 @@ def fix_work_study_location(lf: pl.LazyFrame, households: pl.LazyFrame):
|
|
|
411
412
|
households.select("household_id", *(f"home_{col}" for col in zone_cols)),
|
|
412
413
|
on="household_id",
|
|
413
414
|
how="left",
|
|
415
|
+
maintain_order="left",
|
|
414
416
|
)
|
|
415
417
|
lf = lf.with_columns(
|
|
416
418
|
pl.when("work_only_at_home", pl.col(f"home_{col}") == pl.col(f"work_{col}"))
|
|
@@ -114,7 +114,11 @@ class CeremaStandardizer(HouseholdsReader, PersonsReader, TripsReader, LegsReade
|
|
|
114
114
|
.collect()
|
|
115
115
|
)
|
|
116
116
|
self.households = self.households.join(
|
|
117
|
-
household_dates.lazy(),
|
|
117
|
+
household_dates.lazy(),
|
|
118
|
+
on="household_id",
|
|
119
|
+
how="left",
|
|
120
|
+
coalesce=True,
|
|
121
|
+
maintain_order="left",
|
|
118
122
|
).with_columns(trips_weekday=pl.when("is_valid_weekday").then(pl.col("trips_weekday")))
|
|
119
123
|
|
|
120
124
|
def fix_main_mode(self):
|
|
@@ -122,7 +126,9 @@ class CeremaStandardizer(HouseholdsReader, PersonsReader, TripsReader, LegsReade
|
|
|
122
126
|
# have mode set to "motorcycle".
|
|
123
127
|
invalid_trips = (
|
|
124
128
|
self.legs.select("trip_id", "mode_group")
|
|
125
|
-
.join(
|
|
129
|
+
.join(
|
|
130
|
+
self.trips.select("trip_id", "main_mode_group"), on="trip_id", maintain_order="left"
|
|
131
|
+
)
|
|
126
132
|
.filter(pl.col("mode_group").eq(pl.col("main_mode_group")).any().over("trip_id").not_())
|
|
127
133
|
.select("trip_id")
|
|
128
134
|
.collect()
|
|
@@ -134,6 +134,7 @@ class LegsReader(CeremaReader):
|
|
|
134
134
|
on="original_trip_id",
|
|
135
135
|
how="left",
|
|
136
136
|
coalesce=True,
|
|
137
|
+
maintain_order="left",
|
|
137
138
|
)
|
|
138
139
|
|
|
139
140
|
lf = lf.with_columns(
|
|
@@ -260,7 +261,7 @@ class LegsReader(CeremaReader):
|
|
|
260
261
|
# We create a single leg for them, with the data we have.
|
|
261
262
|
lf5 = (
|
|
262
263
|
self.trips.filter(pl.col("main_mode") != "walking")
|
|
263
|
-
.join(lf, on="original_trip_id", how="anti")
|
|
264
|
+
.join(lf, on="original_trip_id", how="anti", maintain_order="left")
|
|
264
265
|
.select(
|
|
265
266
|
"household_id",
|
|
266
267
|
"person_id",
|
|
@@ -281,16 +282,16 @@ class LegsReader(CeremaReader):
|
|
|
281
282
|
# Concatenate the 5 leg types.
|
|
282
283
|
lf = pl.concat((lf1, lf2, lf3, lf4, lf5), how="diagonal")
|
|
283
284
|
# Add car and motorcycle types.
|
|
285
|
+
# Trucks ("fourgon, camionnette, camion") are reported in the same way as cars.
|
|
286
|
+
is_car_or_truck = pl.col("mode").str.contains("^(car|truck):")
|
|
284
287
|
lf = lf.with_columns(
|
|
285
|
-
car_type=pl.when(
|
|
288
|
+
car_type=pl.when(is_car_or_truck).then(
|
|
286
289
|
pl.col("T7").replace_strict(CAR_MAP, default=None)
|
|
287
290
|
),
|
|
288
291
|
motorcycle_type=pl.when(pl.col("mode").str.starts_with("motorcycle:")).then(
|
|
289
292
|
pl.col("T7").replace_strict(MOTORCYCLE_MAP, default=None)
|
|
290
293
|
),
|
|
291
|
-
original_car_index=pl.when(
|
|
292
|
-
pl.col("mode").str.starts_with("car:") & pl.col("T7").is_between(1, 4)
|
|
293
|
-
)
|
|
294
|
+
original_car_index=pl.when(is_car_or_truck & pl.col("T7").is_between(1, 4))
|
|
294
295
|
.then("T7")
|
|
295
296
|
.cast(pl.UInt8),
|
|
296
297
|
# The `original_motorcycle_index` is either 1, 2, 3, 4 or 11, 12, 13, 14 depending on
|
|
@@ -309,6 +310,7 @@ class LegsReader(CeremaReader):
|
|
|
309
310
|
on=["household_id", "original_car_index"],
|
|
310
311
|
how="left",
|
|
311
312
|
coalesce=True,
|
|
313
|
+
maintain_order="left",
|
|
312
314
|
)
|
|
313
315
|
# For Angers 2012 (and maybe others), column T7 is sometimes set to 2 (for example) while
|
|
314
316
|
# the household has no car with index 2.
|
|
@@ -324,6 +326,7 @@ class LegsReader(CeremaReader):
|
|
|
324
326
|
on=["household_id", "original_motorcycle_index"],
|
|
325
327
|
how="left",
|
|
326
328
|
coalesce=True,
|
|
329
|
+
maintain_order="left",
|
|
327
330
|
)
|
|
328
331
|
# Same fix as for cars.
|
|
329
332
|
lf = lf.with_columns(
|
|
@@ -341,6 +344,7 @@ class LegsReader(CeremaReader):
|
|
|
341
344
|
self.persons.select("person_id", is_major=pl.col("age") >= 18),
|
|
342
345
|
on="person_id",
|
|
343
346
|
how="left",
|
|
347
|
+
maintain_order="left",
|
|
344
348
|
)
|
|
345
349
|
lf = lf.with_columns(
|
|
346
350
|
nb_majors_in_vehicle=pl.when("is_major" | pl.col("mode").str.contains("passenger"))
|
|
@@ -152,9 +152,6 @@ class ZonesReader(CeremaReader):
|
|
|
152
152
|
for zf_col in self.zf_id_columns():
|
|
153
153
|
if matching_col := find_matching_column(zf_col, gdf):
|
|
154
154
|
gdf["detailed_zone_id"] = gdf[matching_col]
|
|
155
|
-
# Drop the original column so that it will not be wrongly read when finding the
|
|
156
|
-
# matching ST id column.
|
|
157
|
-
gdf.drop(columns=[matching_col], inplace=True)
|
|
158
155
|
break
|
|
159
156
|
|
|
160
157
|
def select_gt_column(self, gdf: gpd.GeoDataFrame):
|
|
@@ -15,6 +15,7 @@ from mobisurvstd.schema import (
|
|
|
15
15
|
MOTORCYCLE_SCHEMA,
|
|
16
16
|
PERSON_SCHEMA,
|
|
17
17
|
TRIP_SCHEMA,
|
|
18
|
+
WEEKDAYS,
|
|
18
19
|
)
|
|
19
20
|
|
|
20
21
|
|
|
@@ -38,6 +39,7 @@ def clean(
|
|
|
38
39
|
households = add_household_type(households, persons)
|
|
39
40
|
persons = count_nb_trips(persons, trips)
|
|
40
41
|
persons = add_worked_during_surveyed_day(persons, trips)
|
|
42
|
+
persons = add_weekday_travel(persons, households)
|
|
41
43
|
trips = count_nb_legs(trips, legs)
|
|
42
44
|
trips = add_main_mode(trips, legs)
|
|
43
45
|
trips = add_access_egress_modes(trips, legs)
|
|
@@ -178,7 +180,11 @@ def count_nb_persons(households: pl.LazyFrame, persons: pl.LazyFrame):
|
|
|
178
180
|
# Join on both "household_id" and "complete_household" so that counts are not added for
|
|
179
181
|
# incomplete households.
|
|
180
182
|
households = households.join(
|
|
181
|
-
person_counts,
|
|
183
|
+
person_counts,
|
|
184
|
+
on=["household_id", "complete_household"],
|
|
185
|
+
how="left",
|
|
186
|
+
coalesce=True,
|
|
187
|
+
maintain_order="left",
|
|
182
188
|
)
|
|
183
189
|
return households
|
|
184
190
|
|
|
@@ -211,6 +217,7 @@ def add_household_type(households: pl.LazyFrame, persons: pl.LazyFrame):
|
|
|
211
217
|
on="household_id",
|
|
212
218
|
how="left",
|
|
213
219
|
coalesce=True,
|
|
220
|
+
maintain_order="left",
|
|
214
221
|
)
|
|
215
222
|
households = households.with_columns(
|
|
216
223
|
household_type=pl.when(nb_persons=1, nb_men=1)
|
|
@@ -233,7 +240,9 @@ def add_household_type(households: pl.LazyFrame, persons: pl.LazyFrame):
|
|
|
233
240
|
|
|
234
241
|
def count_nb_trips(persons: pl.LazyFrame, trips: pl.LazyFrame):
|
|
235
242
|
trip_counts = trips.group_by("person_id").agg(nb_trips=pl.len())
|
|
236
|
-
persons = persons.join(
|
|
243
|
+
persons = persons.join(
|
|
244
|
+
trip_counts, on="person_id", how="left", coalesce=True, maintain_order="left"
|
|
245
|
+
)
|
|
237
246
|
# Set nb_trips = 0 for surveyed persons.
|
|
238
247
|
persons = persons.with_columns(
|
|
239
248
|
nb_trips=pl.when("is_surveyed").then(pl.col("nb_trips").fill_null(0)).otherwise("nb_trips")
|
|
@@ -275,6 +284,7 @@ def add_worked_during_surveyed_day(persons: pl.LazyFrame, trips: pl.LazyFrame):
|
|
|
275
284
|
on="person_id",
|
|
276
285
|
how="left",
|
|
277
286
|
coalesce=True,
|
|
287
|
+
maintain_order="left",
|
|
278
288
|
)
|
|
279
289
|
persons = persons.with_columns(
|
|
280
290
|
worked_during_surveyed_day=pl.when("has_work_activity")
|
|
@@ -299,6 +309,44 @@ def add_worked_during_surveyed_day(persons: pl.LazyFrame, trips: pl.LazyFrame):
|
|
|
299
309
|
return persons
|
|
300
310
|
|
|
301
311
|
|
|
312
|
+
def add_weekday_travel(persons: pl.LazyFrame, households: pl.LazyFrame):
|
|
313
|
+
"""Add columns `traveled_on_{weekday}` and `sample_weight_day` to the persons, if needed.
|
|
314
|
+
|
|
315
|
+
For surveys where only one day is surveyed, `traveled_on_{weekday}` is only defined for the
|
|
316
|
+
weekday of the surveyed day and `sample_weight_day` is equal to `sample_weight_surveyed`.
|
|
317
|
+
"""
|
|
318
|
+
persons_cols = persons.collect_schema().names()
|
|
319
|
+
if "traveled_on_monday" not in persons_cols:
|
|
320
|
+
if "trips_weekday" in households.collect_schema().names():
|
|
321
|
+
persons = persons.join(
|
|
322
|
+
households.select("household_id", household_trips_weekday="trips_weekday"),
|
|
323
|
+
on="household_id",
|
|
324
|
+
how="left",
|
|
325
|
+
coalesce=True,
|
|
326
|
+
maintain_order="left",
|
|
327
|
+
)
|
|
328
|
+
# The travel status is unknown for persons who were away.
|
|
329
|
+
traveled = pl.col("traveled_during_surveyed_day").replace_strict(
|
|
330
|
+
{"yes": True, "no": False, "away": None}, return_dtype=pl.Boolean
|
|
331
|
+
)
|
|
332
|
+
persons = persons.with_columns(
|
|
333
|
+
pl.when(pl.col("household_trips_weekday") == weekday)
|
|
334
|
+
.then(traveled)
|
|
335
|
+
.alias(f"traveled_on_{weekday}")
|
|
336
|
+
for weekday in WEEKDAYS
|
|
337
|
+
)
|
|
338
|
+
else:
|
|
339
|
+
persons = persons.with_columns(
|
|
340
|
+
pl.lit(None, dtype=pl.Boolean).alias(f"traveled_on_{weekday}")
|
|
341
|
+
for weekday in WEEKDAYS
|
|
342
|
+
)
|
|
343
|
+
if "sample_weight_day" not in persons_cols and "sample_weight_surveyed" in persons_cols:
|
|
344
|
+
persons = persons.with_columns(
|
|
345
|
+
sample_weight_day=pl.when("is_surveyed").then("sample_weight_surveyed")
|
|
346
|
+
)
|
|
347
|
+
return persons
|
|
348
|
+
|
|
349
|
+
|
|
302
350
|
def count_nb_legs(trips: pl.LazyFrame, legs: pl.LazyFrame):
|
|
303
351
|
"""Add column `nb_legs` to the trips.
|
|
304
352
|
|
|
@@ -314,7 +362,7 @@ def count_nb_legs(trips: pl.LazyFrame, legs: pl.LazyFrame):
|
|
|
314
362
|
]
|
|
315
363
|
)
|
|
316
364
|
leg_counts = legs.group_by("trip_id").agg(agg_cols)
|
|
317
|
-
trips = trips.join(leg_counts, on="trip_id", how="left", coalesce=True)
|
|
365
|
+
trips = trips.join(leg_counts, on="trip_id", how="left", coalesce=True, maintain_order="left")
|
|
318
366
|
if has_mode_group:
|
|
319
367
|
trips = add_intermodality_column(trips)
|
|
320
368
|
return trips
|
|
@@ -388,7 +436,7 @@ def add_main_mode(trips: pl.LazyFrame, legs: pl.LazyFrame):
|
|
|
388
436
|
)
|
|
389
437
|
)
|
|
390
438
|
# Add the `main_mode_group` and `mode_group` columns to the trips.
|
|
391
|
-
trips = trips.join(main_modes, on="trip_id", how="left", coalesce=True)
|
|
439
|
+
trips = trips.join(main_modes, on="trip_id", how="left", coalesce=True, maintain_order="left")
|
|
392
440
|
# Remove the `main_mode` column if the legs modes are unknown.
|
|
393
441
|
if not has_modes:
|
|
394
442
|
trips = trips.drop("main_mode")
|
|
@@ -418,7 +466,9 @@ def add_access_egress_modes(trips: pl.LazyFrame, legs: pl.LazyFrame):
|
|
|
418
466
|
# Egress mode is the mode of the last leg if the trip's `main_mode_group` is "public_transit"
|
|
419
467
|
# and the last leg's `mode_group` is not "public_transit", otherwise it is NULL.
|
|
420
468
|
is_pt_trip = pl.col("main_mode_group") == "public_transit"
|
|
421
|
-
trips = trips.join(
|
|
469
|
+
trips = trips.join(
|
|
470
|
+
first_last_leg_modes, on="trip_id", how="left", coalesce=True, maintain_order="left"
|
|
471
|
+
).with_columns(
|
|
422
472
|
public_transit_access_mode=pl.when(
|
|
423
473
|
is_pt_trip & pl.col("first_mode_group").ne("public_transit")
|
|
424
474
|
).then("first_mode"),
|
|
@@ -145,6 +145,7 @@ def standardize_trips(
|
|
|
145
145
|
on="original_person_id",
|
|
146
146
|
how="left",
|
|
147
147
|
coalesce=True,
|
|
148
|
+
maintain_order="left",
|
|
148
149
|
)
|
|
149
150
|
# Add interview_date.
|
|
150
151
|
lf = lf.join(
|
|
@@ -152,6 +153,7 @@ def standardize_trips(
|
|
|
152
153
|
on="household_id",
|
|
153
154
|
how="left",
|
|
154
155
|
coalesce=True,
|
|
156
|
+
maintain_order="left",
|
|
155
157
|
)
|
|
156
158
|
lf = lf.rename(
|
|
157
159
|
{
|
|
@@ -326,6 +326,7 @@ def standardize_cars(filename: Path, households: pl.LazyFrame):
|
|
|
326
326
|
on="original_household_id",
|
|
327
327
|
how="left",
|
|
328
328
|
coalesce=True,
|
|
329
|
+
maintain_order="left",
|
|
329
330
|
)
|
|
330
331
|
lf = pl.concat(
|
|
331
332
|
(
|
|
@@ -366,6 +367,7 @@ def standardize_motorcycles(filename: Path, households: pl.LazyFrame):
|
|
|
366
367
|
on="original_household_id",
|
|
367
368
|
how="left",
|
|
368
369
|
coalesce=True,
|
|
370
|
+
maintain_order="left",
|
|
369
371
|
)
|
|
370
372
|
lf = pl.concat(
|
|
371
373
|
(
|
|
@@ -62,7 +62,7 @@ MODE_MAP = {
|
|
|
62
62
|
50: "car:driver", # Conducteur voiture particulière
|
|
63
63
|
51: "car:driver", # Conducteur dans un système de covoiturage organisé
|
|
64
64
|
52: "car:driver", # Conducteur véhicule utilitaire 800 à 1 000 kg
|
|
65
|
-
53: "
|
|
65
|
+
53: "truck:driver", # Conducteur véhicule utilitaire de 1 000 kg ou plus
|
|
66
66
|
54: "motorcycle:driver:moped", # Conducteur véhicule à 2 (ou 3) roues à moteur non immatriculé
|
|
67
67
|
55: "motorcycle:driver", # Conducteur véhicule à 2 (ou 3) roues à moteur immatriculé
|
|
68
68
|
# Vélo
|
|
@@ -74,7 +74,7 @@ MODE_MAP = {
|
|
|
74
74
|
70: "car:passenger", # Passager d'une voiture particulière
|
|
75
75
|
71: "car:passenger", # Passager dans un système de covoiturage organisé
|
|
76
76
|
72: "car:passenger", # Passager d'un véhicule utilitaire 800 à 1 000 kg
|
|
77
|
-
73: "
|
|
77
|
+
73: "truck:passenger", # Passager véhicule utilitaire de 1 000 kg ou plus
|
|
78
78
|
74: "motorcycle:passenger:moped", # Passager d'un véhicule à 2 (ou 3) roues à moteur non immatriculé
|
|
79
79
|
75: "motorcycle:passenger", # Passager d'un véhicule à 2 (ou 3) roues à moteur immatriculé
|
|
80
80
|
# Autre moyen de transport
|
|
@@ -171,6 +171,7 @@ def standardize_legs(
|
|
|
171
171
|
on="original_trip_id",
|
|
172
172
|
how="left",
|
|
173
173
|
coalesce=True,
|
|
174
|
+
maintain_order="left",
|
|
174
175
|
)
|
|
175
176
|
lf = lf.rename(
|
|
176
177
|
{
|
|
@@ -193,8 +194,8 @@ def standardize_legs(
|
|
|
193
194
|
parking_type=pl.col("TSTAT").replace_strict(PARKING_TYPE_MAP),
|
|
194
195
|
)
|
|
195
196
|
lf = lf.with_columns(
|
|
196
|
-
car_type=pl.when(pl.col("mode").str.
|
|
197
|
-
car_index=pl.when(pl.col("mode").str.
|
|
197
|
+
car_type=pl.when(pl.col("mode").str.contains("^(car|truck):")).then("vehicle_type"),
|
|
198
|
+
car_index=pl.when(pl.col("mode").str.contains("^(car|truck):")).then("vehicle_index"),
|
|
198
199
|
motorcycle_type=pl.when(pl.col("mode").str.starts_with("motorcycle:")).then("vehicle_type"),
|
|
199
200
|
motorcycle_index=pl.when(pl.col("mode").str.starts_with("motorcycle:")).then(
|
|
200
201
|
"vehicle_index"
|
|
@@ -206,6 +207,7 @@ def standardize_legs(
|
|
|
206
207
|
on=["household_id", "car_index"],
|
|
207
208
|
how="left",
|
|
208
209
|
coalesce=True,
|
|
210
|
+
maintain_order="left",
|
|
209
211
|
)
|
|
210
212
|
# Add motorcycle id.
|
|
211
213
|
lf = lf.join(
|
|
@@ -213,6 +215,7 @@ def standardize_legs(
|
|
|
213
215
|
on=["household_id", "motorcycle_index"],
|
|
214
216
|
how="left",
|
|
215
217
|
coalesce=True,
|
|
218
|
+
maintain_order="left",
|
|
216
219
|
)
|
|
217
220
|
# In 3 cases, the motorcycle index TYPV is not an existing motorcycle of the household so the
|
|
218
221
|
# `motorcycle_type` is changed from "household" to "other_household".
|
|
@@ -139,6 +139,7 @@ def standardize_trips(
|
|
|
139
139
|
on="original_person_id",
|
|
140
140
|
how="left",
|
|
141
141
|
coalesce=True,
|
|
142
|
+
maintain_order="left",
|
|
142
143
|
)
|
|
143
144
|
# Add interview_date.
|
|
144
145
|
lf = lf.join(
|
|
@@ -146,6 +147,7 @@ def standardize_trips(
|
|
|
146
147
|
on="household_id",
|
|
147
148
|
how="left",
|
|
148
149
|
coalesce=True,
|
|
150
|
+
maintain_order="left",
|
|
149
151
|
)
|
|
150
152
|
lf = lf.rename(
|
|
151
153
|
{
|
|
@@ -257,6 +257,7 @@ def standardize_persons(filename: str, households: pl.LazyFrame):
|
|
|
257
257
|
on="original_household_id",
|
|
258
258
|
how="left",
|
|
259
259
|
coalesce=True,
|
|
260
|
+
maintain_order="left",
|
|
260
261
|
)
|
|
261
262
|
# For v3, POIDSTI does not exist but can be read from household's `sample_weight`.
|
|
262
263
|
lf = lf.rename(
|
|
@@ -220,6 +220,7 @@ def standardize_legs(
|
|
|
220
220
|
on="original_trip_id",
|
|
221
221
|
how="left",
|
|
222
222
|
coalesce=True,
|
|
223
|
+
maintain_order="left",
|
|
223
224
|
)
|
|
224
225
|
lf = lf.rename(
|
|
225
226
|
{
|
|
@@ -294,6 +295,7 @@ def standardize_legs(
|
|
|
294
295
|
on=["household_id", "car_index"],
|
|
295
296
|
how="left",
|
|
296
297
|
coalesce=True,
|
|
298
|
+
maintain_order="left",
|
|
297
299
|
)
|
|
298
300
|
# Add motorcycle id.
|
|
299
301
|
lf = lf.join(
|
|
@@ -301,6 +303,7 @@ def standardize_legs(
|
|
|
301
303
|
on=["household_id", "motorcycle_index"],
|
|
302
304
|
how="left",
|
|
303
305
|
coalesce=True,
|
|
306
|
+
maintain_order="left",
|
|
304
307
|
)
|
|
305
308
|
# Add `in_vehicle_person_ids`.
|
|
306
309
|
lf = (
|
|
@@ -311,6 +314,7 @@ def standardize_legs(
|
|
|
311
314
|
on="household_id",
|
|
312
315
|
how="left",
|
|
313
316
|
coalesce=True,
|
|
317
|
+
maintain_order="left",
|
|
314
318
|
)
|
|
315
319
|
# Group by leg_id to end up again with one row per leg.
|
|
316
320
|
.group_by("original_leg_id", maintain_order=True)
|
|
@@ -99,6 +99,7 @@ def standardize_trips(
|
|
|
99
99
|
on="original_person_id",
|
|
100
100
|
how="left",
|
|
101
101
|
coalesce=True,
|
|
102
|
+
maintain_order="left",
|
|
102
103
|
)
|
|
103
104
|
|
|
104
105
|
lf = lf.with_columns(
|
|
@@ -147,6 +148,7 @@ def standardize_trips(
|
|
|
147
148
|
on="original_trip_id",
|
|
148
149
|
how="left",
|
|
149
150
|
coalesce=True,
|
|
151
|
+
maintain_order="left",
|
|
150
152
|
)
|
|
151
153
|
|
|
152
154
|
lf = lf.with_columns(
|
|
@@ -225,6 +227,7 @@ def standardize_legs(filename: str, trips: pl.LazyFrame):
|
|
|
225
227
|
on="original_trip_id",
|
|
226
228
|
how="left",
|
|
227
229
|
coalesce=True,
|
|
230
|
+
maintain_order="left",
|
|
228
231
|
)
|
|
229
232
|
# Drop legs with NULL mode (there is by default 5 legs by trip).
|
|
230
233
|
lf = lf.filter(pl.col("mode").is_not_null(), pl.col("mode").ne(""))
|