mobisurvstd 1.4.2__tar.gz → 1.6.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. {mobisurvstd-1.4.2/src/mobisurvstd.egg-info → mobisurvstd-1.6.0}/PKG-INFO +3 -6
  2. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/pyproject.toml +4 -6
  3. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/cerema/deplacements.py +3 -2
  4. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/cerema/edgt.py +1 -0
  5. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/cerema/edvm.py +1 -0
  6. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/cerema/emc2.py +19 -3
  7. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/cerema/menages.py +2 -0
  8. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/cerema/personnes.py +2 -0
  9. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/cerema/survey.py +8 -2
  10. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/cerema/trajets.py +9 -5
  11. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/cerema/zones.py +0 -3
  12. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/common/clean.py +55 -5
  13. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/egt2010/deplacements.py +2 -0
  14. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/egt2010/menages.py +2 -0
  15. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/egt2010/personnes.py +1 -0
  16. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/egt2010/trajets.py +7 -4
  17. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/egt2020/deplacements.py +2 -0
  18. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/egt2020/motos.py +1 -0
  19. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/egt2020/personnes.py +1 -0
  20. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/egt2020/trajets.py +4 -0
  21. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/egt2020/voitures.py +1 -0
  22. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/emg2023/deplacements.py +3 -0
  23. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/emg2023/personnes.py +46 -2
  24. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/emg2023/survey.py +8 -8
  25. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/emp/deplacements.py +9 -1
  26. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/emp/menages.py +1 -0
  27. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/emp/motos.py +1 -0
  28. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/emp/personnes.py +123 -49
  29. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/emp/survey.py +11 -7
  30. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/emp/voitures.py +1 -0
  31. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/resources/admin_express.py +1 -1
  32. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/resources/insee_data.py +19 -8
  33. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/resources/nuts.py +3 -1
  34. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/schema/__init__.py +1 -0
  35. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/schema/common.py +2 -0
  36. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/schema/households.py +2 -5
  37. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/schema/legs.py +7 -2
  38. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/schema/persons.py +11 -1
  39. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/schema/trips.py +2 -5
  40. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/utils.py +3 -0
  41. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/validation.py +74 -13
  42. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0/src/mobisurvstd.egg-info}/PKG-INFO +3 -6
  43. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd.egg-info/requires.txt +1 -4
  44. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/LICENSE.md +0 -0
  45. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/README.md +0 -0
  46. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/setup.cfg +0 -0
  47. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/__init__.py +0 -0
  48. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/__main__.py +0 -0
  49. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/cerema/__init__.py +0 -0
  50. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/cerema/common.py +0 -0
  51. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/cerema/emd.py +0 -0
  52. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/cerema/nantes_opendata.py +0 -0
  53. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/cerema/reader.py +0 -0
  54. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/classes.py +0 -0
  55. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/cli.py +0 -0
  56. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/common/__init__.py +0 -0
  57. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/common/cars.py +0 -0
  58. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/common/households.py +0 -0
  59. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/common/legs.py +0 -0
  60. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/common/motorcycles.py +0 -0
  61. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/common/persons.py +0 -0
  62. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/common/trips.py +0 -0
  63. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/common/zones.py +0 -0
  64. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/egt2010/__init__.py +0 -0
  65. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/egt2010/survey.py +0 -0
  66. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/egt2010/zones.py +0 -0
  67. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/egt2020/__init__.py +0 -0
  68. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/egt2020/menages.py +0 -0
  69. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/egt2020/survey.py +0 -0
  70. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/emg2023/__init__.py +0 -0
  71. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/emg2023/menages.py +0 -0
  72. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/emp/__init__.py +0 -0
  73. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/emp/common.py +0 -0
  74. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/logger.py +0 -0
  75. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/main.py +0 -0
  76. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/resources/__init__.py +0 -0
  77. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/schema/cars.py +0 -0
  78. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/schema/guarantees.py +0 -0
  79. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd/schema/motorcycles.py +0 -0
  80. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd.egg-info/SOURCES.txt +0 -0
  81. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd.egg-info/dependency_links.txt +0 -0
  82. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd.egg-info/entry_points.txt +0 -0
  83. {mobisurvstd-1.4.2 → mobisurvstd-1.6.0}/src/mobisurvstd.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: mobisurvstd
3
- Version: 1.4.2
3
+ Version: 1.6.0
4
4
  Summary: A tool to standardize French mobility survey datasets (EMC2, EGT, EMP, etc.) into a unified format.
5
5
  Author-email: Lucas Javaudin <lucas@lucasjavaudin.com>
6
6
  License-Expression: MIT
@@ -12,23 +12,20 @@ Keywords: mobility survey,polars,data format
12
12
  Classifier: Programming Language :: Python
13
13
  Classifier: Programming Language :: Python :: 3
14
14
  Classifier: Topic :: File Formats
15
- Requires-Python: >=3.10
15
+ Requires-Python: >=3.11
16
16
  Description-Content-Type: text/markdown
17
17
  License-File: LICENSE.md
18
- Requires-Dist: fastexcel>=0.14.0
19
18
  Requires-Dist: geopandas>=1.0.1
20
19
  Requires-Dist: humanize>=4.13.0
21
20
  Requires-Dist: loguru>=0.7.3
22
- Requires-Dist: numpy>=2.2.4
23
21
  Requires-Dist: pandas>=2.2.3
24
22
  Requires-Dist: platformdirs>=4.3.8
25
- Requires-Dist: polars>=1.38.0
23
+ Requires-Dist: polars[calamine]>=2.0.0
26
24
  Requires-Dist: py7zr>=1.1.0
27
25
  Requires-Dist: pyarrow>=19.0.1
28
26
  Requires-Dist: requests>=2.32.3
29
27
  Requires-Dist: tqdm>=4.67.3
30
28
  Requires-Dist: typer>=0.16.0
31
- Requires-Dist: typing-extensions>=4.15.0
32
29
  Dynamic: license-file
33
30
 
34
31
  [![Contributors][contributors-shield]][contributors-url]
@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
6
6
 
7
7
  [project]
8
8
  name = "mobisurvstd"
9
- version = "1.4.2"
9
+ version = "1.6.0"
10
10
  description = "A tool to standardize French mobility survey datasets (EMC2, EGT, EMP, etc.) into a unified format."
11
11
  readme = "README.md"
12
12
  authors = [{ name = "Lucas Javaudin", email = "lucas@lucasjavaudin.com" }]
@@ -19,22 +19,19 @@ classifiers = [
19
19
  ]
20
20
  keywords = ["mobility survey", "polars", "data format"]
21
21
  dependencies = [
22
- "fastexcel>=0.14.0",
23
22
  "geopandas>=1.0.1",
24
23
  "humanize>=4.13.0",
25
24
  "loguru>=0.7.3",
26
- "numpy>=2.2.4",
27
25
  "pandas>=2.2.3",
28
26
  "platformdirs>=4.3.8",
29
- "polars>=1.38.0",
27
+ "polars[calamine]>=2.0.0",
30
28
  "py7zr>=1.1.0",
31
29
  "pyarrow>=19.0.1",
32
30
  "requests>=2.32.3",
33
31
  "tqdm>=4.67.3",
34
32
  "typer>=0.16.0",
35
- "typing-extensions>=4.15.0",
36
33
  ]
37
- requires-python = ">=3.10"
34
+ requires-python = ">=3.11"
38
35
 
39
36
  [project.scripts]
40
37
  mobisurvstd = "mobisurvstd:cli.app"
@@ -59,6 +56,7 @@ dev = [
59
56
 
60
57
  [tool.ruff]
61
58
  line-length = 100
59
+ extend-exclude = ["*.md"]
62
60
 
63
61
  [tool.ruff.format]
64
62
  skip-magic-trailing-comma = true
@@ -142,6 +142,7 @@ class TripsReader(CeremaReader):
142
142
  on="original_person_id",
143
143
  how="left",
144
144
  coalesce=True,
145
+ maintain_order="left",
145
146
  )
146
147
  lf = lf.rename(
147
148
  {
@@ -158,11 +159,11 @@ class TripsReader(CeremaReader):
158
159
  original_trip_id=pl.struct(self.get_trip_index_cols()),
159
160
  origin_purpose=pl.col("D2A").replace_strict(PURPOSE_MAP),
160
161
  origin_escort_purpose=pl.col("D2B").replace_strict(PURPOSE_MAP),
161
- origin_shop_type=pl.col("D2A").replace(SHOP_TYPE_MAP, default=None),
162
+ origin_shop_type=pl.col("D2A").replace_strict(SHOP_TYPE_MAP, default=None),
162
163
  departure_time=60 * (pl.col("D4") // 100) + pl.col("D4") % 100,
163
164
  destination_purpose=pl.col("D5A").replace_strict(PURPOSE_MAP),
164
165
  destination_escort_purpose=pl.col("D5B").replace_strict(PURPOSE_MAP),
165
- destination_shop_type=pl.col("D5A").replace(SHOP_TYPE_MAP, default=None),
166
+ destination_shop_type=pl.col("D5A").replace_strict(SHOP_TYPE_MAP, default=None),
166
167
  arrival_time=60 * (pl.col("D8") // 100) + pl.col("D8") % 100,
167
168
  trip_euclidean_distance_km=pl.col("D11") / 1e3,
168
169
  trip_travel_distance_km=pl.col("D12") / 1e3,
@@ -148,6 +148,7 @@ class EDGTReader(CeremaStandardizer):
148
148
  "num_tirage",
149
149
  "num_secteur",
150
150
  "num_secteurs",
151
+ "secteur",
151
152
  "dtir",
152
153
  "code",
153
154
  ]
@@ -36,6 +36,7 @@ class EDVMReader(CeremaStandardizer):
36
36
  def detailed_zones_filenames(self):
37
37
  return [
38
38
  find_file_path(self.source, ".*(_zf_.*|_zf|zones?[_ ]?fines?.*|_dfin)[.](tab|shp|mif)")
39
+ or find_file_path(self.source, "edvm_etb_mcg.mif") # Saint-Louis 2011 filename.
39
40
  ]
40
41
 
41
42
  def special_locations_filenames(self):
@@ -52,7 +52,7 @@ class EMC2Reader(CeremaStandardizer):
52
52
  return ["zf_160", "zf_fusion", "codegt", "zf", "num_gt"]
53
53
 
54
54
  def zf_id_from_gt_columns(self):
55
- return ["zfrat_f", "num_zf_rat", "cd_zf", "num_zf_19", "zf_rattach", "zfrat"]
55
+ return ["zfrat_f", "num_zf_rat", "cd_zf", "num_zf_19", "zf_rattach", "zfrat", "zf_rat"]
56
56
 
57
57
  def gt_name_columns(self):
58
58
  return ["nom_gt", "nom_gen", "nom_genera", "libelle", "nom", "rem"]
@@ -66,7 +66,7 @@ class EMC2Reader(CeremaStandardizer):
66
66
  return ["zf_fusion", "zf_160", "zf", "num_zf"]
67
67
 
68
68
  def zf_name_columns(self):
69
- return ["zf_nom", "nom_zf", "lib_zf", "libelle", "rem"]
69
+ return ["zf_nom", "nom_zf", "lib_zf", "libelle", "rem", "nom"]
70
70
 
71
71
  def dtir_id_columns(self):
72
72
  return ["dtir_160", "codsect", "ztir", "num_dtir_f", "num_dtir", "dtir"]
@@ -75,7 +75,15 @@ class EMC2Reader(CeremaStandardizer):
75
75
  return ["nomdtir", "nom_dtir"]
76
76
 
77
77
  def insee_id_columns(self):
78
- return ["insee_com", "insee_commune", "insee_gen", "insee", "code_com", "num_com"]
78
+ return [
79
+ "insee_com",
80
+ "insee_commune",
81
+ "insee_comm",
82
+ "insee_gen",
83
+ "insee",
84
+ "code_com",
85
+ "num_com",
86
+ ]
79
87
 
80
88
  def select_dtir_column(self, gdf: gpd.GeoDataFrame):
81
89
  # This is the standard case.
@@ -90,6 +98,7 @@ class EMC2Reader(CeremaStandardizer):
90
98
  for dtir_col in ("zfrat", "zf"):
91
99
  if matching_col := find_matching_column(dtir_col, gdf):
92
100
  gdf["draw_zone_id"] = gdf[matching_col].str.slice(0, 3)
101
+ break
93
102
 
94
103
  def select_zf_column(self, gdf: gpd.GeoDataFrame):
95
104
  # For Angers 2022, three trailing zeros need to be added to the ZF id column.
@@ -123,3 +132,10 @@ class EMC2Reader(CeremaStandardizer):
123
132
  gdf["detailed_zone_id"] = gdf["Pgt"].astype(int) // 10 * 10
124
133
  # Normal case. Find the ZF id column by name.
125
134
  super().select_zf_from_gt_column(gdf)
135
+
136
+ def preprocess_special_locations(self, gdf: gpd.GeoDataFrame):
137
+ # Special case for Vannes 2023: most special locations are invalid (no id or id of a
138
+ # detailed zone). We filter the valid special locations here.
139
+ if "ZF" in gdf.columns and "ZFRAT" in gdf.columns and (gdf["ZF"] != gdf["ZFRAT"]).any():
140
+ gdf = gdf.loc[(gdf["ZF"] != gdf["ZFRAT"]) & (~gdf["ZF"].isna())].copy()
141
+ return gdf
@@ -284,6 +284,7 @@ class HouseholdsReader(CeremaReader):
284
284
  on="original_household_id",
285
285
  how="left",
286
286
  coalesce=True,
287
+ maintain_order="left",
287
288
  )
288
289
  # Note: the `.lazy()` after the concat is useless (this is already a LazyFrame) but it helps
289
290
  # the syntax checker.
@@ -334,6 +335,7 @@ class HouseholdsReader(CeremaReader):
334
335
  on="original_household_id",
335
336
  how="left",
336
337
  coalesce=True,
338
+ maintain_order="left",
337
339
  )
338
340
  # Note: the `.lazy()` after the concat is useless (this is already a LazyFrame) but it helps
339
341
  # the syntax checker.
@@ -271,6 +271,7 @@ class PersonsReader(CeremaReader):
271
271
  on="original_household_id",
272
272
  how="left",
273
273
  coalesce=True,
274
+ maintain_order="left",
274
275
  )
275
276
  lf = lf.rename({"P4": "age", "COE1": "sample_weight_all", "COEP": "sample_weight_surveyed"})
276
277
  lf = fix_dates(lf, self.survey_name())
@@ -411,6 +412,7 @@ def fix_work_study_location(lf: pl.LazyFrame, households: pl.LazyFrame):
411
412
  households.select("household_id", *(f"home_{col}" for col in zone_cols)),
412
413
  on="household_id",
413
414
  how="left",
415
+ maintain_order="left",
414
416
  )
415
417
  lf = lf.with_columns(
416
418
  pl.when("work_only_at_home", pl.col(f"home_{col}") == pl.col(f"work_{col}"))
@@ -114,7 +114,11 @@ class CeremaStandardizer(HouseholdsReader, PersonsReader, TripsReader, LegsReade
114
114
  .collect()
115
115
  )
116
116
  self.households = self.households.join(
117
- household_dates.lazy(), on="household_id", how="left", coalesce=True
117
+ household_dates.lazy(),
118
+ on="household_id",
119
+ how="left",
120
+ coalesce=True,
121
+ maintain_order="left",
118
122
  ).with_columns(trips_weekday=pl.when("is_valid_weekday").then(pl.col("trips_weekday")))
119
123
 
120
124
  def fix_main_mode(self):
@@ -122,7 +126,9 @@ class CeremaStandardizer(HouseholdsReader, PersonsReader, TripsReader, LegsReade
122
126
  # have mode set to "motorcycle".
123
127
  invalid_trips = (
124
128
  self.legs.select("trip_id", "mode_group")
125
- .join(self.trips.select("trip_id", "main_mode_group"), on="trip_id")
129
+ .join(
130
+ self.trips.select("trip_id", "main_mode_group"), on="trip_id", maintain_order="left"
131
+ )
126
132
  .filter(pl.col("mode_group").eq(pl.col("main_mode_group")).any().over("trip_id").not_())
127
133
  .select("trip_id")
128
134
  .collect()
@@ -134,6 +134,7 @@ class LegsReader(CeremaReader):
134
134
  on="original_trip_id",
135
135
  how="left",
136
136
  coalesce=True,
137
+ maintain_order="left",
137
138
  )
138
139
 
139
140
  lf = lf.with_columns(
@@ -260,7 +261,7 @@ class LegsReader(CeremaReader):
260
261
  # We create a single leg for them, with the data we have.
261
262
  lf5 = (
262
263
  self.trips.filter(pl.col("main_mode") != "walking")
263
- .join(lf, on="original_trip_id", how="anti")
264
+ .join(lf, on="original_trip_id", how="anti", maintain_order="left")
264
265
  .select(
265
266
  "household_id",
266
267
  "person_id",
@@ -281,16 +282,16 @@ class LegsReader(CeremaReader):
281
282
  # Concatenate the 5 leg types.
282
283
  lf = pl.concat((lf1, lf2, lf3, lf4, lf5), how="diagonal")
283
284
  # Add car and motorcycle types.
285
+ # Trucks ("fourgon, camionnette, camion") are reported in the same way as cars.
286
+ is_car_or_truck = pl.col("mode").str.contains("^(car|truck):")
284
287
  lf = lf.with_columns(
285
- car_type=pl.when(pl.col("mode").str.starts_with("car:")).then(
288
+ car_type=pl.when(is_car_or_truck).then(
286
289
  pl.col("T7").replace_strict(CAR_MAP, default=None)
287
290
  ),
288
291
  motorcycle_type=pl.when(pl.col("mode").str.starts_with("motorcycle:")).then(
289
292
  pl.col("T7").replace_strict(MOTORCYCLE_MAP, default=None)
290
293
  ),
291
- original_car_index=pl.when(
292
- pl.col("mode").str.starts_with("car:") & pl.col("T7").is_between(1, 4)
293
- )
294
+ original_car_index=pl.when(is_car_or_truck & pl.col("T7").is_between(1, 4))
294
295
  .then("T7")
295
296
  .cast(pl.UInt8),
296
297
  # The `original_motorcycle_index` is either 1, 2, 3, 4 or 11, 12, 13, 14 depending on
@@ -309,6 +310,7 @@ class LegsReader(CeremaReader):
309
310
  on=["household_id", "original_car_index"],
310
311
  how="left",
311
312
  coalesce=True,
313
+ maintain_order="left",
312
314
  )
313
315
  # For Angers 2012 (and maybe others), column T7 is sometimes set to 2 (for example) while
314
316
  # the household has no car with index 2.
@@ -324,6 +326,7 @@ class LegsReader(CeremaReader):
324
326
  on=["household_id", "original_motorcycle_index"],
325
327
  how="left",
326
328
  coalesce=True,
329
+ maintain_order="left",
327
330
  )
328
331
  # Same fix as for cars.
329
332
  lf = lf.with_columns(
@@ -341,6 +344,7 @@ class LegsReader(CeremaReader):
341
344
  self.persons.select("person_id", is_major=pl.col("age") >= 18),
342
345
  on="person_id",
343
346
  how="left",
347
+ maintain_order="left",
344
348
  )
345
349
  lf = lf.with_columns(
346
350
  nb_majors_in_vehicle=pl.when("is_major" | pl.col("mode").str.contains("passenger"))
@@ -152,9 +152,6 @@ class ZonesReader(CeremaReader):
152
152
  for zf_col in self.zf_id_columns():
153
153
  if matching_col := find_matching_column(zf_col, gdf):
154
154
  gdf["detailed_zone_id"] = gdf[matching_col]
155
- # Drop the original column so that it will not be wrongly read when finding the
156
- # matching ST id column.
157
- gdf.drop(columns=[matching_col], inplace=True)
158
155
  break
159
156
 
160
157
  def select_gt_column(self, gdf: gpd.GeoDataFrame):
@@ -15,6 +15,7 @@ from mobisurvstd.schema import (
15
15
  MOTORCYCLE_SCHEMA,
16
16
  PERSON_SCHEMA,
17
17
  TRIP_SCHEMA,
18
+ WEEKDAYS,
18
19
  )
19
20
 
20
21
 
@@ -38,6 +39,7 @@ def clean(
38
39
  households = add_household_type(households, persons)
39
40
  persons = count_nb_trips(persons, trips)
40
41
  persons = add_worked_during_surveyed_day(persons, trips)
42
+ persons = add_weekday_travel(persons, households)
41
43
  trips = count_nb_legs(trips, legs)
42
44
  trips = add_main_mode(trips, legs)
43
45
  trips = add_access_egress_modes(trips, legs)
@@ -178,7 +180,11 @@ def count_nb_persons(households: pl.LazyFrame, persons: pl.LazyFrame):
178
180
  # Join on both "household_id" and "complete_household" so that counts are not added for
179
181
  # incomplete households.
180
182
  households = households.join(
181
- person_counts, on=["household_id", "complete_household"], how="left", coalesce=True
183
+ person_counts,
184
+ on=["household_id", "complete_household"],
185
+ how="left",
186
+ coalesce=True,
187
+ maintain_order="left",
182
188
  )
183
189
  return households
184
190
 
@@ -211,6 +217,7 @@ def add_household_type(households: pl.LazyFrame, persons: pl.LazyFrame):
211
217
  on="household_id",
212
218
  how="left",
213
219
  coalesce=True,
220
+ maintain_order="left",
214
221
  )
215
222
  households = households.with_columns(
216
223
  household_type=pl.when(nb_persons=1, nb_men=1)
@@ -233,7 +240,9 @@ def add_household_type(households: pl.LazyFrame, persons: pl.LazyFrame):
233
240
 
234
241
  def count_nb_trips(persons: pl.LazyFrame, trips: pl.LazyFrame):
235
242
  trip_counts = trips.group_by("person_id").agg(nb_trips=pl.len())
236
- persons = persons.join(trip_counts, on="person_id", how="left", coalesce=True)
243
+ persons = persons.join(
244
+ trip_counts, on="person_id", how="left", coalesce=True, maintain_order="left"
245
+ )
237
246
  # Set nb_trips = 0 for surveyed persons.
238
247
  persons = persons.with_columns(
239
248
  nb_trips=pl.when("is_surveyed").then(pl.col("nb_trips").fill_null(0)).otherwise("nb_trips")
@@ -275,6 +284,7 @@ def add_worked_during_surveyed_day(persons: pl.LazyFrame, trips: pl.LazyFrame):
275
284
  on="person_id",
276
285
  how="left",
277
286
  coalesce=True,
287
+ maintain_order="left",
278
288
  )
279
289
  persons = persons.with_columns(
280
290
  worked_during_surveyed_day=pl.when("has_work_activity")
@@ -299,6 +309,44 @@ def add_worked_during_surveyed_day(persons: pl.LazyFrame, trips: pl.LazyFrame):
299
309
  return persons
300
310
 
301
311
 
312
+ def add_weekday_travel(persons: pl.LazyFrame, households: pl.LazyFrame):
313
+ """Add columns `traveled_on_{weekday}` and `sample_weight_day` to the persons, if needed.
314
+
315
+ For surveys where only one day is surveyed, `traveled_on_{weekday}` is only defined for the
316
+ weekday of the surveyed day and `sample_weight_day` is equal to `sample_weight_surveyed`.
317
+ """
318
+ persons_cols = persons.collect_schema().names()
319
+ if "traveled_on_monday" not in persons_cols:
320
+ if "trips_weekday" in households.collect_schema().names():
321
+ persons = persons.join(
322
+ households.select("household_id", household_trips_weekday="trips_weekday"),
323
+ on="household_id",
324
+ how="left",
325
+ coalesce=True,
326
+ maintain_order="left",
327
+ )
328
+ # The travel status is unknown for persons who were away.
329
+ traveled = pl.col("traveled_during_surveyed_day").replace_strict(
330
+ {"yes": True, "no": False, "away": None}, return_dtype=pl.Boolean
331
+ )
332
+ persons = persons.with_columns(
333
+ pl.when(pl.col("household_trips_weekday") == weekday)
334
+ .then(traveled)
335
+ .alias(f"traveled_on_{weekday}")
336
+ for weekday in WEEKDAYS
337
+ )
338
+ else:
339
+ persons = persons.with_columns(
340
+ pl.lit(None, dtype=pl.Boolean).alias(f"traveled_on_{weekday}")
341
+ for weekday in WEEKDAYS
342
+ )
343
+ if "sample_weight_day" not in persons_cols and "sample_weight_surveyed" in persons_cols:
344
+ persons = persons.with_columns(
345
+ sample_weight_day=pl.when("is_surveyed").then("sample_weight_surveyed")
346
+ )
347
+ return persons
348
+
349
+
302
350
  def count_nb_legs(trips: pl.LazyFrame, legs: pl.LazyFrame):
303
351
  """Add column `nb_legs` to the trips.
304
352
 
@@ -314,7 +362,7 @@ def count_nb_legs(trips: pl.LazyFrame, legs: pl.LazyFrame):
314
362
  ]
315
363
  )
316
364
  leg_counts = legs.group_by("trip_id").agg(agg_cols)
317
- trips = trips.join(leg_counts, on="trip_id", how="left", coalesce=True)
365
+ trips = trips.join(leg_counts, on="trip_id", how="left", coalesce=True, maintain_order="left")
318
366
  if has_mode_group:
319
367
  trips = add_intermodality_column(trips)
320
368
  return trips
@@ -388,7 +436,7 @@ def add_main_mode(trips: pl.LazyFrame, legs: pl.LazyFrame):
388
436
  )
389
437
  )
390
438
  # Add the `main_mode_group` and `mode_group` columns to the trips.
391
- trips = trips.join(main_modes, on="trip_id", how="left", coalesce=True)
439
+ trips = trips.join(main_modes, on="trip_id", how="left", coalesce=True, maintain_order="left")
392
440
  # Remove the `main_mode` column if the legs modes are unknown.
393
441
  if not has_modes:
394
442
  trips = trips.drop("main_mode")
@@ -418,7 +466,9 @@ def add_access_egress_modes(trips: pl.LazyFrame, legs: pl.LazyFrame):
418
466
  # Egress mode is the mode of the last leg if the trip's `main_mode_group` is "public_transit"
419
467
  # and the last leg's `mode_group` is not "public_transit", otherwise it is NULL.
420
468
  is_pt_trip = pl.col("main_mode_group") == "public_transit"
421
- trips = trips.join(first_last_leg_modes, on="trip_id", how="left", coalesce=True).with_columns(
469
+ trips = trips.join(
470
+ first_last_leg_modes, on="trip_id", how="left", coalesce=True, maintain_order="left"
471
+ ).with_columns(
422
472
  public_transit_access_mode=pl.when(
423
473
  is_pt_trip & pl.col("first_mode_group").ne("public_transit")
424
474
  ).then("first_mode"),
@@ -145,6 +145,7 @@ def standardize_trips(
145
145
  on="original_person_id",
146
146
  how="left",
147
147
  coalesce=True,
148
+ maintain_order="left",
148
149
  )
149
150
  # Add interview_date.
150
151
  lf = lf.join(
@@ -152,6 +153,7 @@ def standardize_trips(
152
153
  on="household_id",
153
154
  how="left",
154
155
  coalesce=True,
156
+ maintain_order="left",
155
157
  )
156
158
  lf = lf.rename(
157
159
  {
@@ -326,6 +326,7 @@ def standardize_cars(filename: Path, households: pl.LazyFrame):
326
326
  on="original_household_id",
327
327
  how="left",
328
328
  coalesce=True,
329
+ maintain_order="left",
329
330
  )
330
331
  lf = pl.concat(
331
332
  (
@@ -366,6 +367,7 @@ def standardize_motorcycles(filename: Path, households: pl.LazyFrame):
366
367
  on="original_household_id",
367
368
  how="left",
368
369
  coalesce=True,
370
+ maintain_order="left",
369
371
  )
370
372
  lf = pl.concat(
371
373
  (
@@ -210,6 +210,7 @@ def standardize_persons(
210
210
  on="original_household_id",
211
211
  how="left",
212
212
  coalesce=True,
213
+ maintain_order="left",
213
214
  )
214
215
  is_worker = pl.col("OCCP").is_in((1, 2))
215
216
  is_student = pl.col("OCCP").is_in((3, 4, 5))
@@ -62,7 +62,7 @@ MODE_MAP = {
62
62
  50: "car:driver", # Conducteur voiture particulière
63
63
  51: "car:driver", # Conducteur dans un système de covoiturage organisé
64
64
  52: "car:driver", # Conducteur véhicule utilitaire 800 à 1 000 kg
65
- 53: "car:driver", # Conducteur véhicule utilitaire de 1 000 kg ou plus
65
+ 53: "truck:driver", # Conducteur véhicule utilitaire de 1 000 kg ou plus
66
66
  54: "motorcycle:driver:moped", # Conducteur véhicule à 2 (ou 3) roues à moteur non immatriculé
67
67
  55: "motorcycle:driver", # Conducteur véhicule à 2 (ou 3) roues à moteur immatriculé
68
68
  # Vélo
@@ -74,7 +74,7 @@ MODE_MAP = {
74
74
  70: "car:passenger", # Passager d'une voiture particulière
75
75
  71: "car:passenger", # Passager dans un système de covoiturage organisé
76
76
  72: "car:passenger", # Passager d'un véhicule utilitaire 800 à 1 000 kg
77
- 73: "car:passenger", # Passager véhicule utilitaire de 1 000 kg ou plus
77
+ 73: "truck:passenger", # Passager véhicule utilitaire de 1 000 kg ou plus
78
78
  74: "motorcycle:passenger:moped", # Passager d'un véhicule à 2 (ou 3) roues à moteur non immatriculé
79
79
  75: "motorcycle:passenger", # Passager d'un véhicule à 2 (ou 3) roues à moteur immatriculé
80
80
  # Autre moyen de transport
@@ -171,6 +171,7 @@ def standardize_legs(
171
171
  on="original_trip_id",
172
172
  how="left",
173
173
  coalesce=True,
174
+ maintain_order="left",
174
175
  )
175
176
  lf = lf.rename(
176
177
  {
@@ -193,8 +194,8 @@ def standardize_legs(
193
194
  parking_type=pl.col("TSTAT").replace_strict(PARKING_TYPE_MAP),
194
195
  )
195
196
  lf = lf.with_columns(
196
- car_type=pl.when(pl.col("mode").str.starts_with("car:")).then("vehicle_type"),
197
- car_index=pl.when(pl.col("mode").str.starts_with("car:")).then("vehicle_index"),
197
+ car_type=pl.when(pl.col("mode").str.contains("^(car|truck):")).then("vehicle_type"),
198
+ car_index=pl.when(pl.col("mode").str.contains("^(car|truck):")).then("vehicle_index"),
198
199
  motorcycle_type=pl.when(pl.col("mode").str.starts_with("motorcycle:")).then("vehicle_type"),
199
200
  motorcycle_index=pl.when(pl.col("mode").str.starts_with("motorcycle:")).then(
200
201
  "vehicle_index"
@@ -206,6 +207,7 @@ def standardize_legs(
206
207
  on=["household_id", "car_index"],
207
208
  how="left",
208
209
  coalesce=True,
210
+ maintain_order="left",
209
211
  )
210
212
  # Add motorcycle id.
211
213
  lf = lf.join(
@@ -213,6 +215,7 @@ def standardize_legs(
213
215
  on=["household_id", "motorcycle_index"],
214
216
  how="left",
215
217
  coalesce=True,
218
+ maintain_order="left",
216
219
  )
217
220
  # In 3 cases, the motorcycle index TYPV is not an existing motorcycle of the household so the
218
221
  # `motorcycle_type` is changed from "household" to "other_household".
@@ -139,6 +139,7 @@ def standardize_trips(
139
139
  on="original_person_id",
140
140
  how="left",
141
141
  coalesce=True,
142
+ maintain_order="left",
142
143
  )
143
144
  # Add interview_date.
144
145
  lf = lf.join(
@@ -146,6 +147,7 @@ def standardize_trips(
146
147
  on="household_id",
147
148
  how="left",
148
149
  coalesce=True,
150
+ maintain_order="left",
149
151
  )
150
152
  lf = lf.rename(
151
153
  {
@@ -77,6 +77,7 @@ def standardize_motorcycles(filename: str, households: pl.LazyFrame):
77
77
  on="original_household_id",
78
78
  how="left",
79
79
  coalesce=True,
80
+ maintain_order="left",
80
81
  )
81
82
  lf = lf.with_columns(
82
83
  original_motorcycle_id=pl.struct("IDCEREMA", "NRM"),
@@ -257,6 +257,7 @@ def standardize_persons(filename: str, households: pl.LazyFrame):
257
257
  on="original_household_id",
258
258
  how="left",
259
259
  coalesce=True,
260
+ maintain_order="left",
260
261
  )
261
262
  # For v3, POIDSTI does not exist but can be read from household's `sample_weight`.
262
263
  lf = lf.rename(
@@ -220,6 +220,7 @@ def standardize_legs(
220
220
  on="original_trip_id",
221
221
  how="left",
222
222
  coalesce=True,
223
+ maintain_order="left",
223
224
  )
224
225
  lf = lf.rename(
225
226
  {
@@ -294,6 +295,7 @@ def standardize_legs(
294
295
  on=["household_id", "car_index"],
295
296
  how="left",
296
297
  coalesce=True,
298
+ maintain_order="left",
297
299
  )
298
300
  # Add motorcycle id.
299
301
  lf = lf.join(
@@ -301,6 +303,7 @@ def standardize_legs(
301
303
  on=["household_id", "motorcycle_index"],
302
304
  how="left",
303
305
  coalesce=True,
306
+ maintain_order="left",
304
307
  )
305
308
  # Add `in_vehicle_person_ids`.
306
309
  lf = (
@@ -311,6 +314,7 @@ def standardize_legs(
311
314
  on="household_id",
312
315
  how="left",
313
316
  coalesce=True,
317
+ maintain_order="left",
314
318
  )
315
319
  # Group by leg_id to end up again with one row per leg.
316
320
  .group_by("original_leg_id", maintain_order=True)
@@ -97,6 +97,7 @@ def standardize_cars(filename: str, households: pl.LazyFrame):
97
97
  on="original_household_id",
98
98
  how="left",
99
99
  coalesce=True,
100
+ maintain_order="left",
100
101
  )
101
102
  lf = lf.rename({"PUISS": "tax_horsepower"})
102
103
  lf = lf.with_columns(
@@ -99,6 +99,7 @@ def standardize_trips(
99
99
  on="original_person_id",
100
100
  how="left",
101
101
  coalesce=True,
102
+ maintain_order="left",
102
103
  )
103
104
 
104
105
  lf = lf.with_columns(
@@ -147,6 +148,7 @@ def standardize_trips(
147
148
  on="original_trip_id",
148
149
  how="left",
149
150
  coalesce=True,
151
+ maintain_order="left",
150
152
  )
151
153
 
152
154
  lf = lf.with_columns(
@@ -225,6 +227,7 @@ def standardize_legs(filename: str, trips: pl.LazyFrame):
225
227
  on="original_trip_id",
226
228
  how="left",
227
229
  coalesce=True,
230
+ maintain_order="left",
228
231
  )
229
232
  # Drop legs with NULL mode (there is by default 5 legs by trip).
230
233
  lf = lf.filter(pl.col("mode").is_not_null(), pl.col("mode").ne(""))