mobisurvstd 1.4.2__tar.gz → 1.5.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. {mobisurvstd-1.4.2/src/mobisurvstd.egg-info → mobisurvstd-1.5.0}/PKG-INFO +3 -6
  2. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/pyproject.toml +3 -6
  3. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/cerema/deplacements.py +3 -2
  4. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/cerema/edgt.py +1 -0
  5. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/cerema/edvm.py +1 -0
  6. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/cerema/emc2.py +19 -3
  7. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/cerema/menages.py +2 -0
  8. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/cerema/personnes.py +2 -0
  9. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/cerema/survey.py +8 -2
  10. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/cerema/trajets.py +9 -5
  11. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/cerema/zones.py +0 -3
  12. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/common/clean.py +15 -5
  13. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/egt2010/deplacements.py +2 -0
  14. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/egt2010/menages.py +2 -0
  15. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/egt2010/personnes.py +1 -0
  16. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/egt2010/trajets.py +7 -4
  17. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/egt2020/deplacements.py +2 -0
  18. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/egt2020/motos.py +1 -0
  19. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/egt2020/personnes.py +1 -0
  20. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/egt2020/trajets.py +4 -0
  21. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/egt2020/voitures.py +1 -0
  22. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/emg2023/deplacements.py +3 -0
  23. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/emg2023/personnes.py +2 -1
  24. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/emp/deplacements.py +9 -1
  25. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/emp/menages.py +1 -0
  26. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/emp/motos.py +1 -0
  27. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/emp/personnes.py +53 -47
  28. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/emp/survey.py +3 -1
  29. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/emp/voitures.py +1 -0
  30. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/resources/admin_express.py +1 -1
  31. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/resources/insee_data.py +19 -8
  32. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/resources/nuts.py +3 -1
  33. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/schema/legs.py +7 -2
  34. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/utils.py +3 -0
  35. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/validation.py +42 -13
  36. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0/src/mobisurvstd.egg-info}/PKG-INFO +3 -6
  37. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd.egg-info/requires.txt +1 -4
  38. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/LICENSE.md +0 -0
  39. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/README.md +0 -0
  40. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/setup.cfg +0 -0
  41. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/__init__.py +0 -0
  42. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/__main__.py +0 -0
  43. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/cerema/__init__.py +0 -0
  44. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/cerema/common.py +0 -0
  45. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/cerema/emd.py +0 -0
  46. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/cerema/nantes_opendata.py +0 -0
  47. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/cerema/reader.py +0 -0
  48. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/classes.py +0 -0
  49. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/cli.py +0 -0
  50. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/common/__init__.py +0 -0
  51. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/common/cars.py +0 -0
  52. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/common/households.py +0 -0
  53. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/common/legs.py +0 -0
  54. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/common/motorcycles.py +0 -0
  55. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/common/persons.py +0 -0
  56. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/common/trips.py +0 -0
  57. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/common/zones.py +0 -0
  58. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/egt2010/__init__.py +0 -0
  59. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/egt2010/survey.py +0 -0
  60. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/egt2010/zones.py +0 -0
  61. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/egt2020/__init__.py +0 -0
  62. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/egt2020/menages.py +0 -0
  63. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/egt2020/survey.py +0 -0
  64. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/emg2023/__init__.py +0 -0
  65. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/emg2023/menages.py +0 -0
  66. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/emg2023/survey.py +0 -0
  67. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/emp/__init__.py +0 -0
  68. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/emp/common.py +0 -0
  69. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/logger.py +0 -0
  70. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/main.py +0 -0
  71. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/resources/__init__.py +0 -0
  72. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/schema/__init__.py +0 -0
  73. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/schema/cars.py +0 -0
  74. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/schema/common.py +0 -0
  75. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/schema/guarantees.py +0 -0
  76. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/schema/households.py +0 -0
  77. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/schema/motorcycles.py +0 -0
  78. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/schema/persons.py +0 -0
  79. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd/schema/trips.py +0 -0
  80. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd.egg-info/SOURCES.txt +0 -0
  81. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd.egg-info/dependency_links.txt +0 -0
  82. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd.egg-info/entry_points.txt +0 -0
  83. {mobisurvstd-1.4.2 → mobisurvstd-1.5.0}/src/mobisurvstd.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: mobisurvstd
3
- Version: 1.4.2
3
+ Version: 1.5.0
4
4
  Summary: A tool to standardize French mobility survey datasets (EMC2, EGT, EMP, etc.) into a unified format.
5
5
  Author-email: Lucas Javaudin <lucas@lucasjavaudin.com>
6
6
  License-Expression: MIT
@@ -12,23 +12,20 @@ Keywords: mobility survey,polars,data format
12
12
  Classifier: Programming Language :: Python
13
13
  Classifier: Programming Language :: Python :: 3
14
14
  Classifier: Topic :: File Formats
15
- Requires-Python: >=3.10
15
+ Requires-Python: >=3.11
16
16
  Description-Content-Type: text/markdown
17
17
  License-File: LICENSE.md
18
- Requires-Dist: fastexcel>=0.14.0
19
18
  Requires-Dist: geopandas>=1.0.1
20
19
  Requires-Dist: humanize>=4.13.0
21
20
  Requires-Dist: loguru>=0.7.3
22
- Requires-Dist: numpy>=2.2.4
23
21
  Requires-Dist: pandas>=2.2.3
24
22
  Requires-Dist: platformdirs>=4.3.8
25
- Requires-Dist: polars>=1.38.0
23
+ Requires-Dist: polars[calamine]>=2.0.0
26
24
  Requires-Dist: py7zr>=1.1.0
27
25
  Requires-Dist: pyarrow>=19.0.1
28
26
  Requires-Dist: requests>=2.32.3
29
27
  Requires-Dist: tqdm>=4.67.3
30
28
  Requires-Dist: typer>=0.16.0
31
- Requires-Dist: typing-extensions>=4.15.0
32
29
  Dynamic: license-file
33
30
 
34
31
  [![Contributors][contributors-shield]][contributors-url]
@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
6
6
 
7
7
  [project]
8
8
  name = "mobisurvstd"
9
- version = "1.4.2"
9
+ version = "1.5.0"
10
10
  description = "A tool to standardize French mobility survey datasets (EMC2, EGT, EMP, etc.) into a unified format."
11
11
  readme = "README.md"
12
12
  authors = [{ name = "Lucas Javaudin", email = "lucas@lucasjavaudin.com" }]
@@ -19,22 +19,19 @@ classifiers = [
19
19
  ]
20
20
  keywords = ["mobility survey", "polars", "data format"]
21
21
  dependencies = [
22
- "fastexcel>=0.14.0",
23
22
  "geopandas>=1.0.1",
24
23
  "humanize>=4.13.0",
25
24
  "loguru>=0.7.3",
26
- "numpy>=2.2.4",
27
25
  "pandas>=2.2.3",
28
26
  "platformdirs>=4.3.8",
29
- "polars>=1.38.0",
27
+ "polars[calamine]>=2.0.0",
30
28
  "py7zr>=1.1.0",
31
29
  "pyarrow>=19.0.1",
32
30
  "requests>=2.32.3",
33
31
  "tqdm>=4.67.3",
34
32
  "typer>=0.16.0",
35
- "typing-extensions>=4.15.0",
36
33
  ]
37
- requires-python = ">=3.10"
34
+ requires-python = ">=3.11"
38
35
 
39
36
  [project.scripts]
40
37
  mobisurvstd = "mobisurvstd:cli.app"
@@ -142,6 +142,7 @@ class TripsReader(CeremaReader):
142
142
  on="original_person_id",
143
143
  how="left",
144
144
  coalesce=True,
145
+ maintain_order="left",
145
146
  )
146
147
  lf = lf.rename(
147
148
  {
@@ -158,11 +159,11 @@ class TripsReader(CeremaReader):
158
159
  original_trip_id=pl.struct(self.get_trip_index_cols()),
159
160
  origin_purpose=pl.col("D2A").replace_strict(PURPOSE_MAP),
160
161
  origin_escort_purpose=pl.col("D2B").replace_strict(PURPOSE_MAP),
161
- origin_shop_type=pl.col("D2A").replace(SHOP_TYPE_MAP, default=None),
162
+ origin_shop_type=pl.col("D2A").replace_strict(SHOP_TYPE_MAP, default=None),
162
163
  departure_time=60 * (pl.col("D4") // 100) + pl.col("D4") % 100,
163
164
  destination_purpose=pl.col("D5A").replace_strict(PURPOSE_MAP),
164
165
  destination_escort_purpose=pl.col("D5B").replace_strict(PURPOSE_MAP),
165
- destination_shop_type=pl.col("D5A").replace(SHOP_TYPE_MAP, default=None),
166
+ destination_shop_type=pl.col("D5A").replace_strict(SHOP_TYPE_MAP, default=None),
166
167
  arrival_time=60 * (pl.col("D8") // 100) + pl.col("D8") % 100,
167
168
  trip_euclidean_distance_km=pl.col("D11") / 1e3,
168
169
  trip_travel_distance_km=pl.col("D12") / 1e3,
@@ -148,6 +148,7 @@ class EDGTReader(CeremaStandardizer):
148
148
  "num_tirage",
149
149
  "num_secteur",
150
150
  "num_secteurs",
151
+ "secteur",
151
152
  "dtir",
152
153
  "code",
153
154
  ]
@@ -36,6 +36,7 @@ class EDVMReader(CeremaStandardizer):
36
36
  def detailed_zones_filenames(self):
37
37
  return [
38
38
  find_file_path(self.source, ".*(_zf_.*|_zf|zones?[_ ]?fines?.*|_dfin)[.](tab|shp|mif)")
39
+ or find_file_path(self.source, "edvm_etb_mcg.mif") # Saint-Louis 2011 filename.
39
40
  ]
40
41
 
41
42
  def special_locations_filenames(self):
@@ -52,7 +52,7 @@ class EMC2Reader(CeremaStandardizer):
52
52
  return ["zf_160", "zf_fusion", "codegt", "zf", "num_gt"]
53
53
 
54
54
  def zf_id_from_gt_columns(self):
55
- return ["zfrat_f", "num_zf_rat", "cd_zf", "num_zf_19", "zf_rattach", "zfrat"]
55
+ return ["zfrat_f", "num_zf_rat", "cd_zf", "num_zf_19", "zf_rattach", "zfrat", "zf_rat"]
56
56
 
57
57
  def gt_name_columns(self):
58
58
  return ["nom_gt", "nom_gen", "nom_genera", "libelle", "nom", "rem"]
@@ -66,7 +66,7 @@ class EMC2Reader(CeremaStandardizer):
66
66
  return ["zf_fusion", "zf_160", "zf", "num_zf"]
67
67
 
68
68
  def zf_name_columns(self):
69
- return ["zf_nom", "nom_zf", "lib_zf", "libelle", "rem"]
69
+ return ["zf_nom", "nom_zf", "lib_zf", "libelle", "rem", "nom"]
70
70
 
71
71
  def dtir_id_columns(self):
72
72
  return ["dtir_160", "codsect", "ztir", "num_dtir_f", "num_dtir", "dtir"]
@@ -75,7 +75,15 @@ class EMC2Reader(CeremaStandardizer):
75
75
  return ["nomdtir", "nom_dtir"]
76
76
 
77
77
  def insee_id_columns(self):
78
- return ["insee_com", "insee_commune", "insee_gen", "insee", "code_com", "num_com"]
78
+ return [
79
+ "insee_com",
80
+ "insee_commune",
81
+ "insee_comm",
82
+ "insee_gen",
83
+ "insee",
84
+ "code_com",
85
+ "num_com",
86
+ ]
79
87
 
80
88
  def select_dtir_column(self, gdf: gpd.GeoDataFrame):
81
89
  # This is the standard case.
@@ -90,6 +98,7 @@ class EMC2Reader(CeremaStandardizer):
90
98
  for dtir_col in ("zfrat", "zf"):
91
99
  if matching_col := find_matching_column(dtir_col, gdf):
92
100
  gdf["draw_zone_id"] = gdf[matching_col].str.slice(0, 3)
101
+ break
93
102
 
94
103
  def select_zf_column(self, gdf: gpd.GeoDataFrame):
95
104
  # For Angers 2022, three trailing zeros need to be added to the ZF id column.
@@ -123,3 +132,10 @@ class EMC2Reader(CeremaStandardizer):
123
132
  gdf["detailed_zone_id"] = gdf["Pgt"].astype(int) // 10 * 10
124
133
  # Normal case. Find the ZF id column by name.
125
134
  super().select_zf_from_gt_column(gdf)
135
+
136
+ def preprocess_special_locations(self, gdf: gpd.GeoDataFrame):
137
+ # Special case for Vannes 2023: most special locations are invalid (no id or id of a
138
+ # detailed zone). We filter the valid special locations here.
139
+ if "ZF" in gdf.columns and "ZFRAT" in gdf.columns and (gdf["ZF"] != gdf["ZFRAT"]).any():
140
+ gdf = gdf.loc[(gdf["ZF"] != gdf["ZFRAT"]) & (~gdf["ZF"].isna())].copy()
141
+ return gdf
@@ -284,6 +284,7 @@ class HouseholdsReader(CeremaReader):
284
284
  on="original_household_id",
285
285
  how="left",
286
286
  coalesce=True,
287
+ maintain_order="left",
287
288
  )
288
289
  # Note: the `.lazy()` after the concat is useless (this is already a LazyFrame) but it helps
289
290
  # the syntax checker.
@@ -334,6 +335,7 @@ class HouseholdsReader(CeremaReader):
334
335
  on="original_household_id",
335
336
  how="left",
336
337
  coalesce=True,
338
+ maintain_order="left",
337
339
  )
338
340
  # Note: the `.lazy()` after the concat is useless (this is already a LazyFrame) but it helps
339
341
  # the syntax checker.
@@ -271,6 +271,7 @@ class PersonsReader(CeremaReader):
271
271
  on="original_household_id",
272
272
  how="left",
273
273
  coalesce=True,
274
+ maintain_order="left",
274
275
  )
275
276
  lf = lf.rename({"P4": "age", "COE1": "sample_weight_all", "COEP": "sample_weight_surveyed"})
276
277
  lf = fix_dates(lf, self.survey_name())
@@ -411,6 +412,7 @@ def fix_work_study_location(lf: pl.LazyFrame, households: pl.LazyFrame):
411
412
  households.select("household_id", *(f"home_{col}" for col in zone_cols)),
412
413
  on="household_id",
413
414
  how="left",
415
+ maintain_order="left",
414
416
  )
415
417
  lf = lf.with_columns(
416
418
  pl.when("work_only_at_home", pl.col(f"home_{col}") == pl.col(f"work_{col}"))
@@ -114,7 +114,11 @@ class CeremaStandardizer(HouseholdsReader, PersonsReader, TripsReader, LegsReade
114
114
  .collect()
115
115
  )
116
116
  self.households = self.households.join(
117
- household_dates.lazy(), on="household_id", how="left", coalesce=True
117
+ household_dates.lazy(),
118
+ on="household_id",
119
+ how="left",
120
+ coalesce=True,
121
+ maintain_order="left",
118
122
  ).with_columns(trips_weekday=pl.when("is_valid_weekday").then(pl.col("trips_weekday")))
119
123
 
120
124
  def fix_main_mode(self):
@@ -122,7 +126,9 @@ class CeremaStandardizer(HouseholdsReader, PersonsReader, TripsReader, LegsReade
122
126
  # have mode set to "motorcycle".
123
127
  invalid_trips = (
124
128
  self.legs.select("trip_id", "mode_group")
125
- .join(self.trips.select("trip_id", "main_mode_group"), on="trip_id")
129
+ .join(
130
+ self.trips.select("trip_id", "main_mode_group"), on="trip_id", maintain_order="left"
131
+ )
126
132
  .filter(pl.col("mode_group").eq(pl.col("main_mode_group")).any().over("trip_id").not_())
127
133
  .select("trip_id")
128
134
  .collect()
@@ -134,6 +134,7 @@ class LegsReader(CeremaReader):
134
134
  on="original_trip_id",
135
135
  how="left",
136
136
  coalesce=True,
137
+ maintain_order="left",
137
138
  )
138
139
 
139
140
  lf = lf.with_columns(
@@ -260,7 +261,7 @@ class LegsReader(CeremaReader):
260
261
  # We create a single leg for them, with the data we have.
261
262
  lf5 = (
262
263
  self.trips.filter(pl.col("main_mode") != "walking")
263
- .join(lf, on="original_trip_id", how="anti")
264
+ .join(lf, on="original_trip_id", how="anti", maintain_order="left")
264
265
  .select(
265
266
  "household_id",
266
267
  "person_id",
@@ -281,16 +282,16 @@ class LegsReader(CeremaReader):
281
282
  # Concatenate the 5 leg types.
282
283
  lf = pl.concat((lf1, lf2, lf3, lf4, lf5), how="diagonal")
283
284
  # Add car and motorcycle types.
285
+ # Trucks ("fourgon, camionnette, camion") are reported in the same way as cars.
286
+ is_car_or_truck = pl.col("mode").str.contains("^(car|truck):")
284
287
  lf = lf.with_columns(
285
- car_type=pl.when(pl.col("mode").str.starts_with("car:")).then(
288
+ car_type=pl.when(is_car_or_truck).then(
286
289
  pl.col("T7").replace_strict(CAR_MAP, default=None)
287
290
  ),
288
291
  motorcycle_type=pl.when(pl.col("mode").str.starts_with("motorcycle:")).then(
289
292
  pl.col("T7").replace_strict(MOTORCYCLE_MAP, default=None)
290
293
  ),
291
- original_car_index=pl.when(
292
- pl.col("mode").str.starts_with("car:") & pl.col("T7").is_between(1, 4)
293
- )
294
+ original_car_index=pl.when(is_car_or_truck & pl.col("T7").is_between(1, 4))
294
295
  .then("T7")
295
296
  .cast(pl.UInt8),
296
297
  # The `original_motorcycle_index` is either 1, 2, 3, 4 or 11, 12, 13, 14 depending on
@@ -309,6 +310,7 @@ class LegsReader(CeremaReader):
309
310
  on=["household_id", "original_car_index"],
310
311
  how="left",
311
312
  coalesce=True,
313
+ maintain_order="left",
312
314
  )
313
315
  # For Angers 2012 (and maybe others), column T7 is sometimes set to 2 (for example) while
314
316
  # the household has no car with index 2.
@@ -324,6 +326,7 @@ class LegsReader(CeremaReader):
324
326
  on=["household_id", "original_motorcycle_index"],
325
327
  how="left",
326
328
  coalesce=True,
329
+ maintain_order="left",
327
330
  )
328
331
  # Same fix as for cars.
329
332
  lf = lf.with_columns(
@@ -341,6 +344,7 @@ class LegsReader(CeremaReader):
341
344
  self.persons.select("person_id", is_major=pl.col("age") >= 18),
342
345
  on="person_id",
343
346
  how="left",
347
+ maintain_order="left",
344
348
  )
345
349
  lf = lf.with_columns(
346
350
  nb_majors_in_vehicle=pl.when("is_major" | pl.col("mode").str.contains("passenger"))
@@ -152,9 +152,6 @@ class ZonesReader(CeremaReader):
152
152
  for zf_col in self.zf_id_columns():
153
153
  if matching_col := find_matching_column(zf_col, gdf):
154
154
  gdf["detailed_zone_id"] = gdf[matching_col]
155
- # Drop the original column so that it will not be wrongly read when finding the
156
- # matching ST id column.
157
- gdf.drop(columns=[matching_col], inplace=True)
158
155
  break
159
156
 
160
157
  def select_gt_column(self, gdf: gpd.GeoDataFrame):
@@ -178,7 +178,11 @@ def count_nb_persons(households: pl.LazyFrame, persons: pl.LazyFrame):
178
178
  # Join on both "household_id" and "complete_household" so that counts are not added for
179
179
  # incomplete households.
180
180
  households = households.join(
181
- person_counts, on=["household_id", "complete_household"], how="left", coalesce=True
181
+ person_counts,
182
+ on=["household_id", "complete_household"],
183
+ how="left",
184
+ coalesce=True,
185
+ maintain_order="left",
182
186
  )
183
187
  return households
184
188
 
@@ -211,6 +215,7 @@ def add_household_type(households: pl.LazyFrame, persons: pl.LazyFrame):
211
215
  on="household_id",
212
216
  how="left",
213
217
  coalesce=True,
218
+ maintain_order="left",
214
219
  )
215
220
  households = households.with_columns(
216
221
  household_type=pl.when(nb_persons=1, nb_men=1)
@@ -233,7 +238,9 @@ def add_household_type(households: pl.LazyFrame, persons: pl.LazyFrame):
233
238
 
234
239
  def count_nb_trips(persons: pl.LazyFrame, trips: pl.LazyFrame):
235
240
  trip_counts = trips.group_by("person_id").agg(nb_trips=pl.len())
236
- persons = persons.join(trip_counts, on="person_id", how="left", coalesce=True)
241
+ persons = persons.join(
242
+ trip_counts, on="person_id", how="left", coalesce=True, maintain_order="left"
243
+ )
237
244
  # Set nb_trips = 0 for surveyed persons.
238
245
  persons = persons.with_columns(
239
246
  nb_trips=pl.when("is_surveyed").then(pl.col("nb_trips").fill_null(0)).otherwise("nb_trips")
@@ -275,6 +282,7 @@ def add_worked_during_surveyed_day(persons: pl.LazyFrame, trips: pl.LazyFrame):
275
282
  on="person_id",
276
283
  how="left",
277
284
  coalesce=True,
285
+ maintain_order="left",
278
286
  )
279
287
  persons = persons.with_columns(
280
288
  worked_during_surveyed_day=pl.when("has_work_activity")
@@ -314,7 +322,7 @@ def count_nb_legs(trips: pl.LazyFrame, legs: pl.LazyFrame):
314
322
  ]
315
323
  )
316
324
  leg_counts = legs.group_by("trip_id").agg(agg_cols)
317
- trips = trips.join(leg_counts, on="trip_id", how="left", coalesce=True)
325
+ trips = trips.join(leg_counts, on="trip_id", how="left", coalesce=True, maintain_order="left")
318
326
  if has_mode_group:
319
327
  trips = add_intermodality_column(trips)
320
328
  return trips
@@ -388,7 +396,7 @@ def add_main_mode(trips: pl.LazyFrame, legs: pl.LazyFrame):
388
396
  )
389
397
  )
390
398
  # Add the `main_mode_group` and `mode_group` columns to the trips.
391
- trips = trips.join(main_modes, on="trip_id", how="left", coalesce=True)
399
+ trips = trips.join(main_modes, on="trip_id", how="left", coalesce=True, maintain_order="left")
392
400
  # Remove the `main_mode` column if the legs modes are unknown.
393
401
  if not has_modes:
394
402
  trips = trips.drop("main_mode")
@@ -418,7 +426,9 @@ def add_access_egress_modes(trips: pl.LazyFrame, legs: pl.LazyFrame):
418
426
  # Egress mode is the mode of the last leg if the trip's `main_mode_group` is "public_transit"
419
427
  # and the last leg's `mode_group` is not "public_transit", otherwise it is NULL.
420
428
  is_pt_trip = pl.col("main_mode_group") == "public_transit"
421
- trips = trips.join(first_last_leg_modes, on="trip_id", how="left", coalesce=True).with_columns(
429
+ trips = trips.join(
430
+ first_last_leg_modes, on="trip_id", how="left", coalesce=True, maintain_order="left"
431
+ ).with_columns(
422
432
  public_transit_access_mode=pl.when(
423
433
  is_pt_trip & pl.col("first_mode_group").ne("public_transit")
424
434
  ).then("first_mode"),
@@ -145,6 +145,7 @@ def standardize_trips(
145
145
  on="original_person_id",
146
146
  how="left",
147
147
  coalesce=True,
148
+ maintain_order="left",
148
149
  )
149
150
  # Add interview_date.
150
151
  lf = lf.join(
@@ -152,6 +153,7 @@ def standardize_trips(
152
153
  on="household_id",
153
154
  how="left",
154
155
  coalesce=True,
156
+ maintain_order="left",
155
157
  )
156
158
  lf = lf.rename(
157
159
  {
@@ -326,6 +326,7 @@ def standardize_cars(filename: Path, households: pl.LazyFrame):
326
326
  on="original_household_id",
327
327
  how="left",
328
328
  coalesce=True,
329
+ maintain_order="left",
329
330
  )
330
331
  lf = pl.concat(
331
332
  (
@@ -366,6 +367,7 @@ def standardize_motorcycles(filename: Path, households: pl.LazyFrame):
366
367
  on="original_household_id",
367
368
  how="left",
368
369
  coalesce=True,
370
+ maintain_order="left",
369
371
  )
370
372
  lf = pl.concat(
371
373
  (
@@ -210,6 +210,7 @@ def standardize_persons(
210
210
  on="original_household_id",
211
211
  how="left",
212
212
  coalesce=True,
213
+ maintain_order="left",
213
214
  )
214
215
  is_worker = pl.col("OCCP").is_in((1, 2))
215
216
  is_student = pl.col("OCCP").is_in((3, 4, 5))
@@ -62,7 +62,7 @@ MODE_MAP = {
62
62
  50: "car:driver", # Conducteur voiture particulière
63
63
  51: "car:driver", # Conducteur dans un système de covoiturage organisé
64
64
  52: "car:driver", # Conducteur véhicule utilitaire 800 à 1 000 kg
65
- 53: "car:driver", # Conducteur véhicule utilitaire de 1 000 kg ou plus
65
+ 53: "truck:driver", # Conducteur véhicule utilitaire de 1 000 kg ou plus
66
66
  54: "motorcycle:driver:moped", # Conducteur véhicule à 2 (ou 3) roues à moteur non immatriculé
67
67
  55: "motorcycle:driver", # Conducteur véhicule à 2 (ou 3) roues à moteur immatriculé
68
68
  # Vélo
@@ -74,7 +74,7 @@ MODE_MAP = {
74
74
  70: "car:passenger", # Passager d'une voiture particulière
75
75
  71: "car:passenger", # Passager dans un système de covoiturage organisé
76
76
  72: "car:passenger", # Passager d'un véhicule utilitaire 800 à 1 000 kg
77
- 73: "car:passenger", # Passager véhicule utilitaire de 1 000 kg ou plus
77
+ 73: "truck:passenger", # Passager véhicule utilitaire de 1 000 kg ou plus
78
78
  74: "motorcycle:passenger:moped", # Passager d'un véhicule à 2 (ou 3) roues à moteur non immatriculé
79
79
  75: "motorcycle:passenger", # Passager d'un véhicule à 2 (ou 3) roues à moteur immatriculé
80
80
  # Autre moyen de transport
@@ -171,6 +171,7 @@ def standardize_legs(
171
171
  on="original_trip_id",
172
172
  how="left",
173
173
  coalesce=True,
174
+ maintain_order="left",
174
175
  )
175
176
  lf = lf.rename(
176
177
  {
@@ -193,8 +194,8 @@ def standardize_legs(
193
194
  parking_type=pl.col("TSTAT").replace_strict(PARKING_TYPE_MAP),
194
195
  )
195
196
  lf = lf.with_columns(
196
- car_type=pl.when(pl.col("mode").str.starts_with("car:")).then("vehicle_type"),
197
- car_index=pl.when(pl.col("mode").str.starts_with("car:")).then("vehicle_index"),
197
+ car_type=pl.when(pl.col("mode").str.contains("^(car|truck):")).then("vehicle_type"),
198
+ car_index=pl.when(pl.col("mode").str.contains("^(car|truck):")).then("vehicle_index"),
198
199
  motorcycle_type=pl.when(pl.col("mode").str.starts_with("motorcycle:")).then("vehicle_type"),
199
200
  motorcycle_index=pl.when(pl.col("mode").str.starts_with("motorcycle:")).then(
200
201
  "vehicle_index"
@@ -206,6 +207,7 @@ def standardize_legs(
206
207
  on=["household_id", "car_index"],
207
208
  how="left",
208
209
  coalesce=True,
210
+ maintain_order="left",
209
211
  )
210
212
  # Add motorcycle id.
211
213
  lf = lf.join(
@@ -213,6 +215,7 @@ def standardize_legs(
213
215
  on=["household_id", "motorcycle_index"],
214
216
  how="left",
215
217
  coalesce=True,
218
+ maintain_order="left",
216
219
  )
217
220
  # In 3 cases, the motorcycle index TYPV is not an existing motorcycle of the household so the
218
221
  # `motorcycle_type` is changed from "household" to "other_household".
@@ -139,6 +139,7 @@ def standardize_trips(
139
139
  on="original_person_id",
140
140
  how="left",
141
141
  coalesce=True,
142
+ maintain_order="left",
142
143
  )
143
144
  # Add interview_date.
144
145
  lf = lf.join(
@@ -146,6 +147,7 @@ def standardize_trips(
146
147
  on="household_id",
147
148
  how="left",
148
149
  coalesce=True,
150
+ maintain_order="left",
149
151
  )
150
152
  lf = lf.rename(
151
153
  {
@@ -77,6 +77,7 @@ def standardize_motorcycles(filename: str, households: pl.LazyFrame):
77
77
  on="original_household_id",
78
78
  how="left",
79
79
  coalesce=True,
80
+ maintain_order="left",
80
81
  )
81
82
  lf = lf.with_columns(
82
83
  original_motorcycle_id=pl.struct("IDCEREMA", "NRM"),
@@ -257,6 +257,7 @@ def standardize_persons(filename: str, households: pl.LazyFrame):
257
257
  on="original_household_id",
258
258
  how="left",
259
259
  coalesce=True,
260
+ maintain_order="left",
260
261
  )
261
262
  # For v3, POIDSTI does not exist but can be read from household's `sample_weight`.
262
263
  lf = lf.rename(
@@ -220,6 +220,7 @@ def standardize_legs(
220
220
  on="original_trip_id",
221
221
  how="left",
222
222
  coalesce=True,
223
+ maintain_order="left",
223
224
  )
224
225
  lf = lf.rename(
225
226
  {
@@ -294,6 +295,7 @@ def standardize_legs(
294
295
  on=["household_id", "car_index"],
295
296
  how="left",
296
297
  coalesce=True,
298
+ maintain_order="left",
297
299
  )
298
300
  # Add motorcycle id.
299
301
  lf = lf.join(
@@ -301,6 +303,7 @@ def standardize_legs(
301
303
  on=["household_id", "motorcycle_index"],
302
304
  how="left",
303
305
  coalesce=True,
306
+ maintain_order="left",
304
307
  )
305
308
  # Add `in_vehicle_person_ids`.
306
309
  lf = (
@@ -311,6 +314,7 @@ def standardize_legs(
311
314
  on="household_id",
312
315
  how="left",
313
316
  coalesce=True,
317
+ maintain_order="left",
314
318
  )
315
319
  # Group by leg_id to end up again with one row per leg.
316
320
  .group_by("original_leg_id", maintain_order=True)
@@ -97,6 +97,7 @@ def standardize_cars(filename: str, households: pl.LazyFrame):
97
97
  on="original_household_id",
98
98
  how="left",
99
99
  coalesce=True,
100
+ maintain_order="left",
100
101
  )
101
102
  lf = lf.rename({"PUISS": "tax_horsepower"})
102
103
  lf = lf.with_columns(
@@ -99,6 +99,7 @@ def standardize_trips(
99
99
  on="original_person_id",
100
100
  how="left",
101
101
  coalesce=True,
102
+ maintain_order="left",
102
103
  )
103
104
 
104
105
  lf = lf.with_columns(
@@ -147,6 +148,7 @@ def standardize_trips(
147
148
  on="original_trip_id",
148
149
  how="left",
149
150
  coalesce=True,
151
+ maintain_order="left",
150
152
  )
151
153
 
152
154
  lf = lf.with_columns(
@@ -225,6 +227,7 @@ def standardize_legs(filename: str, trips: pl.LazyFrame):
225
227
  on="original_trip_id",
226
228
  how="left",
227
229
  coalesce=True,
230
+ maintain_order="left",
228
231
  )
229
232
  # Drop legs with NULL mode (there is by default 5 legs by trip).
230
233
  lf = lf.filter(pl.col("mode").is_not_null(), pl.col("mode").ne(""))
@@ -66,7 +66,7 @@ PCS_GROUP_CODE_MAP = {
66
66
  4: 5, # EMG: Employés
67
67
  5: 6, # EMG: Ouvriers
68
68
  6: 7, # EMG: Retraité ou pré-retraité
69
- 7: None, # EMG: Étudiant ou lycée
69
+ 7: 8, # EMG: Étudiant ou lycée
70
70
  8: 8, # Au chômage ou en inactivité)
71
71
  }
72
72
 
@@ -97,6 +97,7 @@ def standardize_persons(filename: str, households: pl.LazyFrame):
97
97
  on="original_household_id",
98
98
  how="left",
99
99
  coalesce=True,
100
+ maintain_order="left",
100
101
  )
101
102
 
102
103
  lf = lf.with_columns(
@@ -244,6 +244,7 @@ def standardize_trips(filename: str, persons: pl.LazyFrame):
244
244
  on="original_person_id",
245
245
  how="left",
246
246
  coalesce=True,
247
+ maintain_order="left",
247
248
  )
248
249
  lf = lf.rename(
249
250
  {
@@ -342,7 +343,13 @@ def standardize_trips(filename: str, persons: pl.LazyFrame):
342
343
  .lazy()
343
344
  )
344
345
  for prefix in ("origin", "destination"):
345
- lf = lf.join(old_reg_df, left_on=f"{prefix}_nuts2", right_on="NUTS2", how="left").rename(
346
+ lf = lf.join(
347
+ old_reg_df,
348
+ left_on=f"{prefix}_nuts2",
349
+ right_on="NUTS2",
350
+ how="left",
351
+ maintain_order="left",
352
+ ).rename(
346
353
  {
347
354
  "NUTS2_name": f"{prefix}_nuts2_name",
348
355
  "NUTS1": f"{prefix}_nuts1",
@@ -370,6 +377,7 @@ def standardize_legs(filename: str, trips: pl.LazyFrame):
370
377
  on="original_trip_id",
371
378
  how="left",
372
379
  coalesce=True,
380
+ maintain_order="left",
373
381
  )
374
382
  lf = lf.filter(
375
383
  # Drop legs with NULL mode (there is by default 4 legs by trip).
@@ -156,6 +156,7 @@ def scan_households(filename1: str, filename2: str):
156
156
  left_on="ident_men",
157
157
  right_on="IDENT_MEN",
158
158
  coalesce=False,
159
+ maintain_order="left",
159
160
  )
160
161
  return lf
161
162
 
@@ -83,6 +83,7 @@ def standardize_motorcycles(filename: str, households: pl.LazyFrame):
83
83
  on="original_household_id",
84
84
  how="left",
85
85
  coalesce=True,
86
+ maintain_order="left",
86
87
  )
87
88
  lf = lf.rename({"ANNEE_1mec": "year"})
88
89
  lf = lf.with_columns(