mobisurvstd 1.0.2__tar.gz → 1.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. {mobisurvstd-1.0.2/src/mobisurvstd.egg-info → mobisurvstd-1.1.0}/PKG-INFO +5 -7
  2. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/README.md +2 -1
  3. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/pyproject.toml +22 -16
  4. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/cerema/__init__.py +4 -2
  5. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/cerema/deplacements.py +2 -1
  6. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/cerema/edgt.py +1 -3
  7. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/cerema/menages.py +9 -32
  8. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/cerema/nantes_opendata.py +1 -3
  9. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/cerema/survey.py +4 -6
  10. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/cerema/trajets.py +1 -1
  11. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/classes.py +19 -12
  12. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/cli.py +4 -1
  13. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/common/clean.py +28 -16
  14. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/common/households.py +4 -2
  15. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/common/trips.py +8 -5
  16. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/common/zones.py +1 -7
  17. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/egt2010/deplacements.py +2 -0
  18. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/egt2010/menages.py +6 -7
  19. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/egt2010/personnes.py +1 -5
  20. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/egt2010/survey.py +3 -3
  21. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/egt2020/deplacements.py +34 -27
  22. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/egt2020/menages.py +17 -12
  23. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/egt2020/motos.py +3 -10
  24. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/egt2020/personnes.py +27 -29
  25. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/egt2020/survey.py +21 -9
  26. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/egt2020/trajets.py +23 -19
  27. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/egt2020/voitures.py +2 -1
  28. mobisurvstd-1.1.0/src/mobisurvstd/emg2023/deplacements.py +251 -0
  29. mobisurvstd-1.1.0/src/mobisurvstd/emg2023/menages.py +24 -0
  30. mobisurvstd-1.1.0/src/mobisurvstd/emg2023/personnes.py +125 -0
  31. mobisurvstd-1.1.0/src/mobisurvstd/emg2023/survey.py +65 -0
  32. mobisurvstd-1.1.0/src/mobisurvstd/emp/__init__.py +1 -0
  33. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/emp/common.py +9 -0
  34. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/emp/deplacements.py +7 -9
  35. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/emp/menages.py +4 -0
  36. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/emp/motos.py +4 -26
  37. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/emp/personnes.py +1 -4
  38. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/emp/survey.py +1 -2
  39. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/emp/voitures.py +5 -39
  40. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/main.py +49 -11
  41. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/resources/insee_data.py +155 -38
  42. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/schema/cars.py +2 -14
  43. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/schema/common.py +3 -6
  44. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/schema/guarantees.py +33 -9
  45. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/schema/households.py +10 -4
  46. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/schema/motorcycles.py +1 -9
  47. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/schema/persons.py +17 -74
  48. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/schema/trips.py +20 -3
  49. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/utils.py +16 -4
  50. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/validation.py +49 -42
  51. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0/src/mobisurvstd.egg-info}/PKG-INFO +5 -7
  52. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd.egg-info/SOURCES.txt +5 -0
  53. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd.egg-info/requires.txt +2 -5
  54. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/LICENSE.md +0 -0
  55. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/setup.cfg +0 -0
  56. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/__init__.py +0 -0
  57. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/__main__.py +0 -0
  58. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/cerema/common.py +0 -0
  59. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/cerema/edvm.py +0 -0
  60. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/cerema/emc2.py +0 -0
  61. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/cerema/emd.py +0 -0
  62. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/cerema/personnes.py +0 -0
  63. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/cerema/reader.py +0 -0
  64. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/cerema/zones.py +0 -0
  65. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/common/__init__.py +0 -0
  66. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/common/cars.py +0 -0
  67. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/common/legs.py +0 -0
  68. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/common/motorcycles.py +0 -0
  69. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/common/persons.py +0 -0
  70. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/egt2010/__init__.py +0 -0
  71. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/egt2010/trajets.py +0 -0
  72. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/egt2010/zones.py +0 -0
  73. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/egt2020/__init__.py +0 -0
  74. {mobisurvstd-1.0.2/src/mobisurvstd/emp → mobisurvstd-1.1.0/src/mobisurvstd/emg2023}/__init__.py +0 -0
  75. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/logger.py +0 -0
  76. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/resources/__init__.py +0 -0
  77. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/resources/admin_express.py +0 -0
  78. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/resources/nuts.py +0 -0
  79. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/schema/__init__.py +0 -0
  80. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd/schema/legs.py +0 -0
  81. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd.egg-info/dependency_links.txt +0 -0
  82. {mobisurvstd-1.0.2 → mobisurvstd-1.1.0}/src/mobisurvstd.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: mobisurvstd
3
- Version: 1.0.2
3
+ Version: 1.1.0
4
4
  Summary: A tool to standardize French mobility survey datasets (EMC2, EGT, EMP, etc.) into a unified format.
5
5
  Author-email: Lucas Javaudin <lucas@lucasjavaudin.com>
6
6
  License-Expression: MIT
@@ -15,21 +15,18 @@ Classifier: Topic :: File Formats
15
15
  Requires-Python: >=3.10
16
16
  Description-Content-Type: text/markdown
17
17
  License-File: LICENSE.md
18
- Requires-Dist: Cartopy>=0.24.1
19
18
  Requires-Dist: fastexcel>=0.14.0
20
- Requires-Dist: fsspec>=2025.9.0
21
19
  Requires-Dist: geopandas>=1.0.1
22
20
  Requires-Dist: humanize>=4.13.0
23
21
  Requires-Dist: loguru>=0.7.3
24
- Requires-Dist: matplotlib>=3.10.1
25
22
  Requires-Dist: numpy>=2.2.4
26
23
  Requires-Dist: pandas>=2.2.3
27
24
  Requires-Dist: platformdirs>=4.3.8
28
- Requires-Dist: polars>=1.33.0
25
+ Requires-Dist: polars>=1.38.0
29
26
  Requires-Dist: py7zr>=0.20.8
30
27
  Requires-Dist: pyarrow>=19.0.1
31
28
  Requires-Dist: requests>=2.32.3
32
- Requires-Dist: seaborn>=0.13.2
29
+ Requires-Dist: tqdm>=4.67.3
33
30
  Requires-Dist: typer>=0.16.0
34
31
  Dynamic: license-file
35
32
 
@@ -202,6 +199,7 @@ Currently, MobiSurvStd supports the following survey types:
202
199
 
203
200
  - `emp2019`
204
201
  - `emc2`
202
+ - `emg2023` (thanks to @sebhorl)
205
203
  - `egt2020`
206
204
  - `egt2010`
207
205
  - `edgt` (including the [open-data version of Nantes 2015](https://www.data.gouv.fr/datasets/enquete-deplacements-2015-en-loire-atlantique))
@@ -210,7 +208,7 @@ Currently, MobiSurvStd supports the following survey types:
210
208
 
211
209
  See [Survey Types](https://mobisurvstd.github.io/MobiSurvStd/surveys.html) for more.
212
210
 
213
- Other survey types that are planned to be integrated are: `entd`, `emg`.
211
+ Other survey types that are planned to be integrated are: `entd`.
214
212
  If you know another survey format that could be integrated, feel free to open an issue on GitHub.
215
213
 
216
214
  MobiSurvStd covers only French mobility survey formats.
@@ -167,6 +167,7 @@ Currently, MobiSurvStd supports the following survey types:
167
167
 
168
168
  - `emp2019`
169
169
  - `emc2`
170
+ - `emg2023` (thanks to @sebhorl)
170
171
  - `egt2020`
171
172
  - `egt2010`
172
173
  - `edgt` (including the [open-data version of Nantes 2015](https://www.data.gouv.fr/datasets/enquete-deplacements-2015-en-loire-atlantique))
@@ -175,7 +176,7 @@ Currently, MobiSurvStd supports the following survey types:
175
176
 
176
177
  See [Survey Types](https://mobisurvstd.github.io/MobiSurvStd/surveys.html) for more.
177
178
 
178
- Other survey types that are planned to be integrated are: `entd`, `emg`.
179
+ Other survey types that are planned to be integrated are: `entd`.
179
180
  If you know another survey format that could be integrated, feel free to open an issue on GitHub.
180
181
 
181
182
  MobiSurvStd covers only French mobility survey formats.
@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
6
6
 
7
7
  [project]
8
8
  name = "mobisurvstd"
9
- version = "1.0.2"
9
+ version = "1.1.0"
10
10
  description = "A tool to standardize French mobility survey datasets (EMC2, EGT, EMP, etc.) into a unified format."
11
11
  readme = "README.md"
12
12
  authors = [{ name = "Lucas Javaudin", email = "lucas@lucasjavaudin.com" }]
@@ -19,29 +19,30 @@ classifiers = [
19
19
  ]
20
20
  keywords = ["mobility survey", "polars", "data format"]
21
21
  dependencies = [
22
- "Cartopy >= 0.24.1",
23
- "fastexcel >= 0.14.0",
24
- "fsspec>=2025.9.0",
22
+ "fastexcel>=0.14.0",
25
23
  "geopandas>=1.0.1",
26
- "humanize >= 4.13.0",
27
- "loguru >= 0.7.3",
28
- "matplotlib >= 3.10.1",
29
- "numpy >= 2.2.4",
30
- "pandas >= 2.2.3",
31
- "platformdirs >= 4.3.8",
32
- "polars>=1.33.0",
33
- "py7zr >= 0.20.8",
34
- "pyarrow >= 19.0.1",
35
- "requests >= 2.32.3",
36
- "seaborn>=0.13.2",
37
- "typer >= 0.16.0",
24
+ "humanize>=4.13.0",
25
+ "loguru>=0.7.3",
26
+ "numpy>=2.2.4",
27
+ "pandas>=2.2.3",
28
+ "platformdirs>=4.3.8",
29
+ "polars>=1.38.0",
30
+ "py7zr>=0.20.8",
31
+ "pyarrow>=19.0.1",
32
+ "requests>=2.32.3",
33
+ "tqdm>=4.67.3",
34
+ "typer>=0.16.0",
38
35
  ]
39
36
  requires-python = ">=3.10"
40
37
 
41
38
  [dependency-groups]
42
39
  dev = [
40
+ "Cartopy>=0.24.1",
41
+ "folium>=0.20.0",
43
42
  "ipython>=8.37.0",
43
+ "matplotlib>=3.10.1",
44
44
  "pytest>=9.0.2",
45
+ "seaborn>=0.13.2",
45
46
  "ty>=0.0.15",
46
47
  ]
47
48
 
@@ -50,3 +51,8 @@ Homepage = "https://github.com/MobiSurvStd/MobiSurvStd"
50
51
  Documentation = "https://mobisurvstd.github.io/MobiSurvStd/index.html"
51
52
  Repository = "https://github.com/MobiSurvStd/MobiSurvStd"
52
53
  Issues = "https://github.com/MobiSurvStd/MobiSurvStd/issues"
54
+
55
+ [tool.ruff]
56
+ line-length = 100
57
+ [tool.ruff.format]
58
+ skip-magic-trailing-comma = true
@@ -7,7 +7,9 @@ from .emd import EMDReader
7
7
  from .nantes_opendata import OpenDataReader
8
8
 
9
9
 
10
- def standardize(source: str | ZipFile, survey_type: str, skip_spatial: bool = False):
10
+ def standardize(
11
+ source: str | ZipFile, survey_type: str, skip_spatial: bool = False, skip_insee: bool = False
12
+ ):
11
13
  if survey_type == "EMC2":
12
14
  reader = EMC2Reader(source)
13
15
  elif survey_type == "EDGT":
@@ -20,4 +22,4 @@ def standardize(source: str | ZipFile, survey_type: str, skip_spatial: bool = Fa
20
22
  reader = EMDReader(source)
21
23
  else:
22
24
  raise NotImplementedError
23
- return reader.standardize(skip_spatial)
25
+ return reader.standardize(skip_spatial, skip_insee)
@@ -123,7 +123,7 @@ class TripsReader(CeremaReader):
123
123
  lf = lf.sort(self.get_trip_index_cols())
124
124
  return lf
125
125
 
126
- def standardize_trips(self):
126
+ def standardize_trips(self, skip_insee: bool = False):
127
127
  lf = self.scan_trips()
128
128
  # Add household_id, person_id, trip date, and trip weekday.
129
129
  lf = lf.with_columns(
@@ -203,6 +203,7 @@ class TripsReader(CeremaReader):
203
203
  year=self.survey_year() or CURRENT_YEAR,
204
204
  special_locations=self.special_locations_coords,
205
205
  detailed_zones=self.detailed_zones_coords,
206
+ skip_insee=skip_insee,
206
207
  )
207
208
  # When the INSEE code ends with "000" or "999" it means "rest of the département".
208
209
  # We drop these values because they do not add any additional information compared to `_dep`
@@ -41,9 +41,7 @@ class EDGTReader(CeremaStandardizer):
41
41
  else:
42
42
  return [
43
43
  find_file_path(
44
- self.source,
45
- ".*(_zf_gt|zinterne_gen_.*)[.](tab|shp|mif)",
46
- subdir=subdir,
44
+ self.source, ".*(_zf_gt|zinterne_gen_.*)[.](tab|shp|mif)", subdir=subdir
47
45
  )
48
46
  ]
49
47
 
@@ -92,10 +92,7 @@ SCHEMA = {
92
92
  "COE0": pl.Float64, # COEFFICIENT DE REDRESSEMENT MENAGE
93
93
  }
94
94
 
95
- SURVEY_METHOD_MAP = {
96
- 1: "face_to_face",
97
- 2: "phone",
98
- }
95
+ SURVEY_METHOD_MAP = {1: "face_to_face", 2: "phone"}
99
96
 
100
97
  HOUSING_TYPE_MAP = {
101
98
  1: "house", # Individuel isolé
@@ -195,37 +192,15 @@ MOTORCYCLE_FUEL_TYPE_MAP = {
195
192
  90: "other",
196
193
  }
197
194
 
198
- MOTORCYCLE_SIZE_LOWERBOUND_MAP = {
199
- 1: 0,
200
- 2: 50,
201
- 3: 125,
202
- 4: 250,
203
- 5: 750,
204
- 9: 125,
205
- }
195
+ MOTORCYCLE_SIZE_LOWERBOUND_MAP = {1: 0, 2: 50, 3: 125, 4: 250, 5: 750, 9: 125}
206
196
 
207
- MOTORCYCLE_SIZE_UPPERBOUND_MAP = {
208
- 1: 50,
209
- 2: 125,
210
- 3: 250,
211
- 4: 750,
212
- }
197
+ MOTORCYCLE_SIZE_UPPERBOUND_MAP = {1: 50, 2: 125, 3: 250, 4: 750}
213
198
 
214
- MOTORCYCLE_KW_LOWERBOUND_MAP = {
215
- 6: 0,
216
- 7: 4,
217
- 8: 11,
218
- }
199
+ MOTORCYCLE_KW_LOWERBOUND_MAP = {6: 0, 7: 4, 8: 11}
219
200
 
220
- MOTORCYCLE_KW_UPPERBOUND_MAP = {
221
- 6: 4,
222
- 7: 11,
223
- }
201
+ MOTORCYCLE_KW_UPPERBOUND_MAP = {6: 4, 7: 11}
224
202
 
225
- MOTORCYCLE_THERMIC_ENGINE_TYPE_MAP = {
226
- 1: "two_stroke",
227
- 2: "four_stroke",
228
- }
203
+ MOTORCYCLE_THERMIC_ENGINE_TYPE_MAP = {1: "two_stroke", 2: "four_stroke"}
229
204
 
230
205
 
231
206
  def scan_households_impl(source: str | io.BytesIO):
@@ -248,7 +223,7 @@ class HouseholdsReader(CeremaReader):
248
223
  def main_insee(self):
249
224
  return self.scan_households().select("IDM4").first().collect().item() # ty: ignore[possibly-missing-attribute]
250
225
 
251
- def standardize_households(self):
226
+ def standardize_households(self, skip_insee: bool = False):
252
227
  lf = self.scan_households()
253
228
  lf = lf.rename(
254
229
  {
@@ -262,6 +237,7 @@ class HouseholdsReader(CeremaReader):
262
237
  )
263
238
  lf = lf.with_columns(
264
239
  original_household_id=pl.struct(self.get_household_index_cols()),
240
+ complete_household=True,
265
241
  home_detailed_zone="ZFM",
266
242
  home_draw_zone="STM",
267
243
  # For EMD, EDVM, EDGT, the IDM2 column will be read here.
@@ -278,6 +254,7 @@ class HouseholdsReader(CeremaReader):
278
254
  year=self.survey_year() or CURRENT_YEAR,
279
255
  special_locations=self.special_locations_coords,
280
256
  detailed_zones=self.detailed_zones_coords,
257
+ skip_insee=skip_insee,
281
258
  )
282
259
  # When the INSEE code ends with "000" or "999" it means "rest of the département".
283
260
  # We drop these values because they do not add any additional information compared to `_dep`
@@ -192,9 +192,7 @@ class OpenDataReader(CeremaStandardizer):
192
192
 
193
193
 
194
194
  def scan_fwf(
195
- f: str | io.BytesIO,
196
- format: list[tuple[int, int, str]],
197
- schema: dict[str, DataTypeClass],
195
+ f: str | io.BytesIO, format: list[tuple[int, int, str]], schema: dict[str, DataTypeClass]
198
196
  ):
199
197
  # Decrease `start` by 1 because pandas starts counting at 0 but the survey document starts
200
198
  # counting at 1.
@@ -26,7 +26,7 @@ LOCATION_COLUMNS = (
26
26
  class CeremaStandardizer(HouseholdsReader, PersonsReader, TripsReader, LegsReader, ZonesReader):
27
27
  SURVEY_TYPE = ""
28
28
 
29
- def standardize(self, skip_spatial: bool = False):
29
+ def standardize(self, skip_spatial: bool = False, skip_insee: bool = False):
30
30
  logger.info(f"Standardizing {self.SURVEY_TYPE} survey from `{self.source_name()}`")
31
31
  if not self.validate():
32
32
  return None
@@ -34,11 +34,11 @@ class CeremaStandardizer(HouseholdsReader, PersonsReader, TripsReader, LegsReade
34
34
  if not skip_spatial:
35
35
  self.read_spatial_data()
36
36
 
37
- self.standardize_households()
37
+ self.standardize_households(skip_insee)
38
38
  self.standardize_cars()
39
39
  self.standardize_motorcycles()
40
40
  self.standardize_persons()
41
- self.standardize_trips()
41
+ self.standardize_trips(skip_insee)
42
42
  self.standardize_legs()
43
43
 
44
44
  self.finish()
@@ -115,9 +115,7 @@ class CeremaStandardizer(HouseholdsReader, PersonsReader, TripsReader, LegsReade
115
115
  ) # ty: ignore[invalid-assignment]
116
116
  self.households = self.households.join(
117
117
  household_dates.lazy(), on="household_id", how="left", coalesce=True
118
- ).with_columns(
119
- trips_weekday=pl.when("is_valid_weekday").then(pl.col("trips_weekday")),
120
- )
118
+ ).with_columns(trips_weekday=pl.when("is_valid_weekday").then(pl.col("trips_weekday")))
121
119
 
122
120
  def fix_main_mode(self):
123
121
  # Special case for Douai 2012: Some trips have main mode set to "car_driver" but the legs
@@ -342,7 +342,7 @@ class LegsReader(CeremaReader):
342
342
  .then(pl.col("nb_majors_in_vehicle") + 1)
343
343
  .otherwise("nb_majors_in_vehicle"),
344
344
  nb_minors_in_vehicle=pl.when(
345
- pl.col("is_major").not_() & pl.col("mode").str.contains("driver"),
345
+ pl.col("is_major").not_() & pl.col("mode").str.contains("driver")
346
346
  )
347
347
  .then(pl.col("nb_minors_in_vehicle") + 1)
348
348
  .otherwise("nb_minors_in_vehicle"),
@@ -67,16 +67,19 @@ class SurveyData:
67
67
  logger.warning("Output directory is not empty, some data might be erased")
68
68
  logger.debug("Saving households")
69
69
  self.households.write_parquet(output_directory / Path("households.parquet"))
70
- logger.debug("Saving cars")
71
- self.cars.write_parquet(output_directory / Path("cars.parquet"))
72
- logger.debug("Saving motorcycles")
73
- self.motorcycles.write_parquet(output_directory / Path("motorcycles.parquet"))
70
+ if not self.cars.is_empty():
71
+ logger.debug("Saving cars")
72
+ self.cars.write_parquet(output_directory / Path("cars.parquet"))
73
+ if not self.motorcycles.is_empty():
74
+ logger.debug("Saving motorcycles")
75
+ self.motorcycles.write_parquet(output_directory / Path("motorcycles.parquet"))
74
76
  logger.debug("Saving persons")
75
77
  self.persons.write_parquet(output_directory / Path("persons.parquet"))
76
78
  logger.debug("Saving trips")
77
79
  self.trips.write_parquet(output_directory / Path("trips.parquet"))
78
- logger.debug("Saving legs")
79
- self.legs.write_parquet(output_directory / Path("legs.parquet"))
80
+ if not self.legs.is_empty():
81
+ logger.debug("Saving legs")
82
+ self.legs.write_parquet(output_directory / Path("legs.parquet"))
80
83
  if self.special_locations is not None:
81
84
  logger.debug("Saving special locations")
82
85
  self.special_locations.to_parquet(
@@ -192,16 +195,22 @@ class SurveyDataReader:
192
195
  def cars(self) -> pl.DataFrame:
193
196
  if self._cars is None:
194
197
  filename = os.path.join(self.directory, "cars.parquet")
195
- logger.debug(f"Reading cars from `{filename}`")
196
- self._cars = pl.read_parquet(filename)
198
+ if os.path.isfile(filename):
199
+ logger.debug(f"Reading cars from `{filename}`")
200
+ self._cars = pl.read_parquet(filename)
201
+ else:
202
+ self._cars = pl.DataFrame()
197
203
  return self._cars
198
204
 
199
205
  @property
200
206
  def motorcycles(self) -> pl.DataFrame:
201
207
  if self._motorcycles is None:
202
208
  filename = os.path.join(self.directory, "motorcycles.parquet")
203
- logger.debug(f"Reading motorcycles from `{filename}`")
204
- self._motorcycles = pl.read_parquet(filename)
209
+ if os.path.isfile(filename):
210
+ logger.debug(f"Reading motorcycles from `{filename}`")
211
+ self._motorcycles = pl.read_parquet(filename)
212
+ else:
213
+ self._motorcycles = pl.DataFrame()
205
214
  return self._motorcycles
206
215
 
207
216
  @property
@@ -323,8 +332,6 @@ def is_valid_mobisurvstd_dir(directory: str) -> bool:
323
332
  "persons.parquet",
324
333
  "trips.parquet",
325
334
  "legs.parquet",
326
- "cars.parquet",
327
- "motorcycles.parquet",
328
335
  ):
329
336
  if not os.path.isfile(os.path.join(directory, name)):
330
337
  return False
@@ -29,13 +29,14 @@ def standardizer(
29
29
  survey_type: Annotated[
30
30
  str | None,
31
31
  typer.Option(
32
- help="Format of the original survey. Possible values: `emc2`, `emp2019`, `egt2010`, `egt2020`, `edgt`, `edvm`, `emd`."
32
+ help="Format of the original survey. Possible values: `emc2`, `emp2019`, `egt2010`, `egt2020`, `edgt`, `edvm`, `emd`, `emg2023`."
33
33
  ),
34
34
  ] = None,
35
35
  bulk: bool = typer.Option(
36
36
  False, "--bulk", help="Import surveys in bulk from the given directory"
37
37
  ),
38
38
  skip_spatial: bool = typer.Option(False, "--skip-spatial", help="Do not read spatial data"),
39
+ skip_insee: bool = typer.Option(False, "--skip-insee", help="Do not add INSEE data"),
39
40
  no_validation: bool = typer.Option(
40
41
  False,
41
42
  "--no-validation",
@@ -57,6 +58,7 @@ def standardizer(
57
58
  output_directory,
58
59
  survey_type,
59
60
  skip_spatial=skip_spatial,
61
+ skip_insee=skip_insee,
60
62
  no_validation=no_validation,
61
63
  )
62
64
  else:
@@ -65,6 +67,7 @@ def standardizer(
65
67
  output_directory,
66
68
  survey_type,
67
69
  skip_spatial=skip_spatial,
70
+ skip_insee=skip_insee,
68
71
  no_validation=no_validation,
69
72
  )
70
73
 
@@ -128,7 +128,7 @@ def create_metadata(
128
128
  "nb_motorcycles": len(data["motorcycles"]),
129
129
  "nb_persons": len(data["persons"]),
130
130
  "nb_trips": len(data["trips"]),
131
- "nb_legs": len(data["legs"]),
131
+ "nb_legs": len(data["legs"]) if "legs" in data else 0,
132
132
  "nb_special_locations": nb_special_locations,
133
133
  "nb_detailed_zones": nb_detailed_zones,
134
134
  "nb_draw_zones": nb_draw_zones,
@@ -151,22 +151,33 @@ def count_nb_persons(households: pl.LazyFrame, persons: pl.LazyFrame):
151
151
  has_age = "age" in persons.collect_schema().names()
152
152
  has_age_class = "age_class_code" in persons.collect_schema().names()
153
153
  person_counts = persons.group_by("household_id").agg(
154
+ complete_household=True,
154
155
  nb_persons=pl.len(),
155
- nb_persons_5plus=pl.when(pl.col("age").is_not_null().all()).then(pl.col("age").ge(5).sum())
156
- if has_age
157
- else None,
158
- nb_majors=pl.when(pl.col("age_class_code").is_not_null().all()).then(
159
- pl.col("age_class_code").gt(1).sum()
160
- )
161
- if has_age_class
162
- else None,
163
- nb_minors=pl.when(pl.col("age_class_code").is_not_null().all()).then(
164
- pl.col("age_class_code").eq(1).sum()
165
- )
166
- if has_age_class
167
- else None,
156
+ nb_persons_5plus=(
157
+ pl.when(pl.col("age").is_not_null().all()).then(pl.col("age").ge(5).sum())
158
+ if has_age
159
+ else None
160
+ ),
161
+ nb_majors=(
162
+ pl.when(pl.col("age_class_code").is_not_null().all()).then(
163
+ pl.col("age_class_code").gt(1).sum()
164
+ )
165
+ if has_age_class
166
+ else None
167
+ ),
168
+ nb_minors=(
169
+ pl.when(pl.col("age_class_code").is_not_null().all()).then(
170
+ pl.col("age_class_code").eq(1).sum()
171
+ )
172
+ if has_age_class
173
+ else None
174
+ ),
175
+ )
176
+ # Join on both "household_id" and "complete_household" so that counts are not added for
177
+ # incomplete households.
178
+ households = households.join(
179
+ person_counts, on=["household_id", "complete_household"], how="left", coalesce=True
168
180
  )
169
- households = households.join(person_counts, on="household_id", how="left", coalesce=True)
170
181
  return households
171
182
 
172
183
 
@@ -238,6 +249,7 @@ def count_nb_trips(persons: pl.LazyFrame, trips: pl.LazyFrame):
238
249
 
239
250
  def add_worked_during_surveyed_day(persons: pl.LazyFrame, trips: pl.LazyFrame):
240
251
  persons_cols = persons.collect_schema().names()
252
+ trips_cols = trips.collect_schema().names()
241
253
  # Fill null values of `traveled_during_surveyed_day` (when `is_surveyed` is True) according
242
254
  # to the existence of trips.
243
255
  if "traveled_during_surveyed_day" not in persons_cols:
@@ -250,7 +262,7 @@ def add_worked_during_surveyed_day(persons: pl.LazyFrame, trips: pl.LazyFrame):
250
262
  .then(pl.lit("no"))
251
263
  )
252
264
  )
253
- if "worked_during_surveyed_day" not in persons_cols:
265
+ if "worked_during_surveyed_day" not in persons_cols and "destination_purpose" in trips_cols:
254
266
  persons = persons.join(
255
267
  trips.group_by("person_id").agg(
256
268
  has_work_activity=pl.col("destination_purpose")
@@ -14,6 +14,7 @@ def clean(
14
14
  year: int | None,
15
15
  special_locations: pl.DataFrame | None = None,
16
16
  detailed_zones: pl.DataFrame | None = None,
17
+ skip_insee: bool = False,
17
18
  ):
18
19
  existing_cols = lf.collect_schema().names()
19
20
  columns = [variable.name for variable in HOUSEHOLD_SCHEMA if variable.name in existing_cols]
@@ -21,8 +22,9 @@ def clean(
21
22
  lf = indexing(lf)
22
23
  lf = add_bicycle_counts(lf, existing_cols)
23
24
  lf = add_lng_lat(lf, existing_cols, special_locations, detailed_zones)
24
- lf = add_insee_columns(lf, existing_cols)
25
- lf = add_insee_data_columns(lf, existing_cols, year)
25
+ if not skip_insee:
26
+ lf = add_insee_columns(lf, existing_cols)
27
+ lf = add_insee_data_columns(lf, existing_cols, year)
26
28
  if "home_dep" in existing_cols:
27
29
  lf = add_nuts_data(lf, "home")
28
30
  if DEBUG:
@@ -16,6 +16,7 @@ def clean(
16
16
  perimeter_deps: list[str] | None = None,
17
17
  special_locations: pl.DataFrame | None = None,
18
18
  detailed_zones: pl.DataFrame | None = None,
19
+ skip_insee: bool = False,
19
20
  ):
20
21
  existing_cols = lf.collect_schema().names()
21
22
  columns = [variable.name for variable in TRIP_SCHEMA if variable.name in existing_cols]
@@ -26,8 +27,9 @@ def clean(
26
27
  lf = add_durations(lf, existing_cols)
27
28
  lf = add_weekdays(lf, existing_cols)
28
29
  lf = add_lng_lat(lf, existing_cols, special_locations, detailed_zones)
29
- lf = add_insee_columns(lf, existing_cols)
30
- lf = add_insee_data_columns(lf, existing_cols, year)
30
+ if not skip_insee:
31
+ lf = add_insee_columns(lf, existing_cols)
32
+ lf = add_insee_data_columns(lf, existing_cols, year)
31
33
  lf = add_nuts_columns(lf, existing_cols)
32
34
  lf = add_intra_zones(lf, existing_cols)
33
35
  lf = add_main_mode_groups(lf, existing_cols)
@@ -111,9 +113,10 @@ def add_purpose_groups(lf: pl.LazyFrame, existing_cols: list[str]):
111
113
  ):
112
114
  if col in existing_cols:
113
115
  group_col = f"{col}_group"
114
- lf = lf.with_columns(
115
- pl.col(col).cast(pl.String).str.extract(r"(\w+):?").alias(group_col)
116
- )
116
+ if group_col not in existing_cols:
117
+ lf = lf.with_columns(
118
+ pl.col(col).cast(pl.String).str.extract(r"(\w+):?").alias(group_col)
119
+ )
117
120
  return lf
118
121
 
119
122
 
@@ -4,13 +4,7 @@ import polars as pl
4
4
 
5
5
  def get_coords(gdf: gpd.GeoDataFrame, name: str) -> pl.DataFrame:
6
6
  centroids = gdf.geometry.centroid.to_crs("EPSG:4326")
7
- df = pl.DataFrame(
8
- {
9
- f"{name}_id": gdf[f"{name}_id"],
10
- "lng": centroids.x,
11
- "lat": centroids.y,
12
- }
13
- )
7
+ df = pl.DataFrame({f"{name}_id": gdf[f"{name}_id"], "lng": centroids.x, "lat": centroids.y})
14
8
  return df
15
9
 
16
10
 
@@ -134,6 +134,7 @@ def standardize_trips(
134
134
  persons: pl.LazyFrame,
135
135
  households: pl.LazyFrame,
136
136
  detailed_zones: pl.DataFrame | None,
137
+ skip_insee: bool = False,
137
138
  ):
138
139
  lf = scan_trips(filename)
139
140
  # Add household_id and person_id.
@@ -204,6 +205,7 @@ def standardize_trips(
204
205
  2010,
205
206
  perimeter_deps=["75", "77", "78", "91", "92", "93", "94", "95"],
206
207
  detailed_zones=detailed_zones,
208
+ skip_insee=skip_insee,
207
209
  )
208
210
  # When the INSEE code ends with "000" it means "rest of the département".
209
211
  # We drop these values because they do not add any additional information compared to `_dep`
@@ -261,15 +261,13 @@ MOTORCYCLE_CM3_UB_MAP = {
261
261
 
262
262
  def scan_households(filename: str):
263
263
  separator = detect_csv_delimiter(filename)
264
- lf = pl.scan_csv(
265
- filename,
266
- separator=separator,
267
- schema_overrides=SCHEMA,
268
- )
264
+ lf = pl.scan_csv(filename, separator=separator, schema_overrides=SCHEMA)
269
265
  return lf
270
266
 
271
267
 
272
- def standardize_households(filename: str, detailed_zones: pl.DataFrame | None):
268
+ def standardize_households(
269
+ filename: str, detailed_zones: pl.DataFrame | None, skip_insee: bool = False
270
+ ):
273
271
  lf = scan_households(filename)
274
272
  lf = lf.rename(
275
273
  {
@@ -288,6 +286,7 @@ def standardize_households(filename: str, detailed_zones: pl.DataFrame | None):
288
286
  lf = lf.with_columns(
289
287
  original_household_id=pl.struct("NQUEST"),
290
288
  survey_method=pl.lit("face_to_face"),
289
+ complete_household=True,
291
290
  income_lower_bound=pl.col("REVENU").replace_strict(REVENU_LB_MAP),
292
291
  income_upper_bound=pl.col("REVENU").replace_strict(REVENU_UB_MAP),
293
292
  housing_type=pl.col("TYPELOG").replace_strict(HOUSING_TYPE_MAP),
@@ -314,7 +313,7 @@ def standardize_households(filename: str, detailed_zones: pl.DataFrame | None):
314
313
  .otherwise("home_insee"),
315
314
  )
316
315
  lf = lf.sort("original_household_id")
317
- lf = clean_households(lf, year=2010, detailed_zones=detailed_zones)
316
+ lf = clean_households(lf, year=2010, detailed_zones=detailed_zones, skip_insee=skip_insee)
318
317
  return lf
319
318
 
320
319
 
@@ -158,11 +158,7 @@ STUDENT_CATEGORY_MAP = {
158
158
  98: "supérieur:BAC+6&+", # Bac+6 ou plus
159
159
  }
160
160
 
161
- DRIVING_LICENSE_MAP = {
162
- 1: "yes",
163
- 2: "no",
164
- 3: "in_progress",
165
- }
161
+ DRIVING_LICENSE_MAP = {1: "yes", 2: "no", 3: "in_progress"}
166
162
 
167
163
  TRAVELED_MAP = {
168
164
  1: "yes", # Oui
@@ -14,7 +14,7 @@ from .trajets import standardize_legs
14
14
  from .zones import read_detailed_zones
15
15
 
16
16
 
17
- def standardize(source: str | ZipFile, skip_spatial: bool = False):
17
+ def standardize(source: str | ZipFile, skip_spatial: bool = False, skip_insee: bool = False):
18
18
  source_name = source.filename if isinstance(source, ZipFile) else source
19
19
  logger.info(f"Standardizing EGT2010 survey from `{source_name}`")
20
20
  if skip_spatial:
@@ -33,7 +33,7 @@ def standardize(source: str | ZipFile, skip_spatial: bool = False):
33
33
  if not filename:
34
34
  logger.error(f"Missing households file: {filename}")
35
35
  return None
36
- households = standardize_households(filename, detailed_zone_coords)
36
+ households = standardize_households(filename, detailed_zone_coords, skip_insee)
37
37
  # Cars.
38
38
  cars = standardize_cars(filename, households)
39
39
  # motorcycles.
@@ -49,7 +49,7 @@ def standardize(source: str | ZipFile, skip_spatial: bool = False):
49
49
  if not filename:
50
50
  logger.error(f"Missing trips file: {filename}")
51
51
  return None
52
- trips = standardize_trips(filename, persons, households, detailed_zone_coords)
52
+ trips = standardize_trips(filename, persons, households, detailed_zone_coords, skip_insee)
53
53
  # Legs.
54
54
  filename = legs_filename(source)
55
55
  if not filename: