mobisurvstd 1.0.2__tar.gz → 1.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. {mobisurvstd-1.0.2/src/mobisurvstd.egg-info → mobisurvstd-1.2.0}/PKG-INFO +6 -8
  2. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/README.md +2 -1
  3. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/pyproject.toml +22 -16
  4. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/cerema/__init__.py +4 -2
  5. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/cerema/deplacements.py +7 -2
  6. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/cerema/edgt.py +1 -3
  7. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/cerema/emc2.py +5 -23
  8. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/cerema/menages.py +14 -33
  9. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/cerema/nantes_opendata.py +1 -3
  10. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/cerema/personnes.py +7 -1
  11. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/cerema/survey.py +4 -6
  12. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/cerema/trajets.py +4 -2
  13. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/classes.py +19 -12
  14. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/cli.py +4 -1
  15. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/common/clean.py +28 -16
  16. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/common/households.py +4 -2
  17. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/common/trips.py +8 -5
  18. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/common/zones.py +1 -7
  19. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/egt2010/deplacements.py +2 -0
  20. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/egt2010/menages.py +6 -7
  21. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/egt2010/personnes.py +1 -5
  22. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/egt2010/survey.py +3 -3
  23. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/egt2020/deplacements.py +34 -27
  24. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/egt2020/menages.py +17 -12
  25. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/egt2020/motos.py +3 -10
  26. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/egt2020/personnes.py +27 -29
  27. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/egt2020/survey.py +21 -9
  28. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/egt2020/trajets.py +23 -19
  29. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/egt2020/voitures.py +2 -1
  30. mobisurvstd-1.2.0/src/mobisurvstd/emg2023/deplacements.py +251 -0
  31. mobisurvstd-1.2.0/src/mobisurvstd/emg2023/menages.py +24 -0
  32. mobisurvstd-1.2.0/src/mobisurvstd/emg2023/personnes.py +125 -0
  33. mobisurvstd-1.2.0/src/mobisurvstd/emg2023/survey.py +65 -0
  34. mobisurvstd-1.2.0/src/mobisurvstd/emp/__init__.py +1 -0
  35. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/emp/common.py +9 -0
  36. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/emp/deplacements.py +7 -9
  37. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/emp/menages.py +4 -0
  38. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/emp/motos.py +4 -26
  39. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/emp/personnes.py +1 -4
  40. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/emp/survey.py +1 -2
  41. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/emp/voitures.py +5 -39
  42. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/main.py +50 -12
  43. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/resources/insee_data.py +155 -38
  44. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/schema/cars.py +2 -14
  45. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/schema/common.py +3 -6
  46. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/schema/guarantees.py +33 -9
  47. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/schema/households.py +10 -4
  48. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/schema/motorcycles.py +1 -9
  49. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/schema/persons.py +22 -75
  50. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/schema/trips.py +20 -3
  51. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/utils.py +18 -5
  52. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/validation.py +49 -42
  53. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0/src/mobisurvstd.egg-info}/PKG-INFO +6 -8
  54. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd.egg-info/SOURCES.txt +5 -0
  55. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd.egg-info/requires.txt +3 -6
  56. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/LICENSE.md +0 -0
  57. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/setup.cfg +0 -0
  58. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/__init__.py +0 -0
  59. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/__main__.py +0 -0
  60. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/cerema/common.py +0 -0
  61. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/cerema/edvm.py +0 -0
  62. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/cerema/emd.py +0 -0
  63. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/cerema/reader.py +0 -0
  64. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/cerema/zones.py +0 -0
  65. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/common/__init__.py +0 -0
  66. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/common/cars.py +0 -0
  67. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/common/legs.py +0 -0
  68. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/common/motorcycles.py +0 -0
  69. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/common/persons.py +0 -0
  70. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/egt2010/__init__.py +0 -0
  71. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/egt2010/trajets.py +0 -0
  72. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/egt2010/zones.py +0 -0
  73. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/egt2020/__init__.py +0 -0
  74. {mobisurvstd-1.0.2/src/mobisurvstd/emp → mobisurvstd-1.2.0/src/mobisurvstd/emg2023}/__init__.py +0 -0
  75. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/logger.py +0 -0
  76. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/resources/__init__.py +0 -0
  77. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/resources/admin_express.py +0 -0
  78. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/resources/nuts.py +0 -0
  79. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/schema/__init__.py +0 -0
  80. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd/schema/legs.py +0 -0
  81. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd.egg-info/dependency_links.txt +0 -0
  82. {mobisurvstd-1.0.2 → mobisurvstd-1.2.0}/src/mobisurvstd.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: mobisurvstd
3
- Version: 1.0.2
3
+ Version: 1.2.0
4
4
  Summary: A tool to standardize French mobility survey datasets (EMC2, EGT, EMP, etc.) into a unified format.
5
5
  Author-email: Lucas Javaudin <lucas@lucasjavaudin.com>
6
6
  License-Expression: MIT
@@ -15,21 +15,18 @@ Classifier: Topic :: File Formats
15
15
  Requires-Python: >=3.10
16
16
  Description-Content-Type: text/markdown
17
17
  License-File: LICENSE.md
18
- Requires-Dist: Cartopy>=0.24.1
19
18
  Requires-Dist: fastexcel>=0.14.0
20
- Requires-Dist: fsspec>=2025.9.0
21
19
  Requires-Dist: geopandas>=1.0.1
22
20
  Requires-Dist: humanize>=4.13.0
23
21
  Requires-Dist: loguru>=0.7.3
24
- Requires-Dist: matplotlib>=3.10.1
25
22
  Requires-Dist: numpy>=2.2.4
26
23
  Requires-Dist: pandas>=2.2.3
27
24
  Requires-Dist: platformdirs>=4.3.8
28
- Requires-Dist: polars>=1.33.0
29
- Requires-Dist: py7zr>=0.20.8
25
+ Requires-Dist: polars>=1.38.0
26
+ Requires-Dist: py7zr>=1.1.0
30
27
  Requires-Dist: pyarrow>=19.0.1
31
28
  Requires-Dist: requests>=2.32.3
32
- Requires-Dist: seaborn>=0.13.2
29
+ Requires-Dist: tqdm>=4.67.3
33
30
  Requires-Dist: typer>=0.16.0
34
31
  Dynamic: license-file
35
32
 
@@ -202,6 +199,7 @@ Currently, MobiSurvStd supports the following survey types:
202
199
 
203
200
  - `emp2019`
204
201
  - `emc2`
202
+ - `emg2023` (thanks to @sebhorl)
205
203
  - `egt2020`
206
204
  - `egt2010`
207
205
  - `edgt` (including the [open-data version of Nantes 2015](https://www.data.gouv.fr/datasets/enquete-deplacements-2015-en-loire-atlantique))
@@ -210,7 +208,7 @@ Currently, MobiSurvStd supports the following survey types:
210
208
 
211
209
  See [Survey Types](https://mobisurvstd.github.io/MobiSurvStd/surveys.html) for more.
212
210
 
213
- Other survey types that are planned to be integrated are: `entd`, `emg`.
211
+ Other survey types that are planned to be integrated are: `entd`.
214
212
  If you know another survey format that could be integrated, feel free to open an issue on GitHub.
215
213
 
216
214
  MobiSurvStd covers only French mobility survey formats.
@@ -167,6 +167,7 @@ Currently, MobiSurvStd supports the following survey types:
167
167
 
168
168
  - `emp2019`
169
169
  - `emc2`
170
+ - `emg2023` (thanks to @sebhorl)
170
171
  - `egt2020`
171
172
  - `egt2010`
172
173
  - `edgt` (including the [open-data version of Nantes 2015](https://www.data.gouv.fr/datasets/enquete-deplacements-2015-en-loire-atlantique))
@@ -175,7 +176,7 @@ Currently, MobiSurvStd supports the following survey types:
175
176
 
176
177
  See [Survey Types](https://mobisurvstd.github.io/MobiSurvStd/surveys.html) for more.
177
178
 
178
- Other survey types that are planned to be integrated are: `entd`, `emg`.
179
+ Other survey types that are planned to be integrated are: `entd`.
179
180
  If you know another survey format that could be integrated, feel free to open an issue on GitHub.
180
181
 
181
182
  MobiSurvStd covers only French mobility survey formats.
@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
6
6
 
7
7
  [project]
8
8
  name = "mobisurvstd"
9
- version = "1.0.2"
9
+ version = "1.2.0"
10
10
  description = "A tool to standardize French mobility survey datasets (EMC2, EGT, EMP, etc.) into a unified format."
11
11
  readme = "README.md"
12
12
  authors = [{ name = "Lucas Javaudin", email = "lucas@lucasjavaudin.com" }]
@@ -19,29 +19,30 @@ classifiers = [
19
19
  ]
20
20
  keywords = ["mobility survey", "polars", "data format"]
21
21
  dependencies = [
22
- "Cartopy >= 0.24.1",
23
- "fastexcel >= 0.14.0",
24
- "fsspec>=2025.9.0",
22
+ "fastexcel>=0.14.0",
25
23
  "geopandas>=1.0.1",
26
- "humanize >= 4.13.0",
27
- "loguru >= 0.7.3",
28
- "matplotlib >= 3.10.1",
29
- "numpy >= 2.2.4",
30
- "pandas >= 2.2.3",
31
- "platformdirs >= 4.3.8",
32
- "polars>=1.33.0",
33
- "py7zr >= 0.20.8",
34
- "pyarrow >= 19.0.1",
35
- "requests >= 2.32.3",
36
- "seaborn>=0.13.2",
37
- "typer >= 0.16.0",
24
+ "humanize>=4.13.0",
25
+ "loguru>=0.7.3",
26
+ "numpy>=2.2.4",
27
+ "pandas>=2.2.3",
28
+ "platformdirs>=4.3.8",
29
+ "polars>=1.38.0",
30
+ "py7zr>=1.1.0",
31
+ "pyarrow>=19.0.1",
32
+ "requests>=2.32.3",
33
+ "tqdm>=4.67.3",
34
+ "typer>=0.16.0",
38
35
  ]
39
36
  requires-python = ">=3.10"
40
37
 
41
38
  [dependency-groups]
42
39
  dev = [
40
+ "Cartopy>=0.24.1",
41
+ "folium>=0.20.0",
43
42
  "ipython>=8.37.0",
43
+ "matplotlib>=3.10.1",
44
44
  "pytest>=9.0.2",
45
+ "seaborn>=0.13.2",
45
46
  "ty>=0.0.15",
46
47
  ]
47
48
 
@@ -50,3 +51,8 @@ Homepage = "https://github.com/MobiSurvStd/MobiSurvStd"
50
51
  Documentation = "https://mobisurvstd.github.io/MobiSurvStd/index.html"
51
52
  Repository = "https://github.com/MobiSurvStd/MobiSurvStd"
52
53
  Issues = "https://github.com/MobiSurvStd/MobiSurvStd/issues"
54
+
55
+ [tool.ruff]
56
+ line-length = 100
57
+ [tool.ruff.format]
58
+ skip-magic-trailing-comma = true
@@ -7,7 +7,9 @@ from .emd import EMDReader
7
7
  from .nantes_opendata import OpenDataReader
8
8
 
9
9
 
10
- def standardize(source: str | ZipFile, survey_type: str, skip_spatial: bool = False):
10
+ def standardize(
11
+ source: str | ZipFile, survey_type: str, skip_spatial: bool = False, skip_insee: bool = False
12
+ ):
11
13
  if survey_type == "EMC2":
12
14
  reader = EMC2Reader(source)
13
15
  elif survey_type == "EDGT":
@@ -20,4 +22,4 @@ def standardize(source: str | ZipFile, survey_type: str, skip_spatial: bool = Fa
20
22
  reader = EMDReader(source)
21
23
  else:
22
24
  raise NotImplementedError
23
- return reader.standardize(skip_spatial)
25
+ return reader.standardize(skip_spatial, skip_insee)
@@ -5,6 +5,7 @@ from loguru import logger
5
5
 
6
6
  from mobisurvstd.common.trips import clean
7
7
  from mobisurvstd.schema.common import CURRENT_YEAR
8
+ from mobisurvstd.utils import detect_csv_delimiter
8
9
 
9
10
  from .reader import CeremaReader
10
11
 
@@ -113,7 +114,10 @@ TRIP_PERIMETER_MAP = {
113
114
 
114
115
 
115
116
  def scan_trips_impl(source: str | io.BytesIO):
116
- return pl.scan_csv(source, separator=";", schema_overrides=SCHEMA, null_values=["aa", "aaaaa"])
117
+ separator = detect_csv_delimiter(source)
118
+ return pl.scan_csv(
119
+ source, separator=separator, schema_overrides=SCHEMA, null_values=["aa", "aaaaa"]
120
+ )
117
121
 
118
122
 
119
123
  class TripsReader(CeremaReader):
@@ -123,7 +127,7 @@ class TripsReader(CeremaReader):
123
127
  lf = lf.sort(self.get_trip_index_cols())
124
128
  return lf
125
129
 
126
- def standardize_trips(self):
130
+ def standardize_trips(self, skip_insee: bool = False):
127
131
  lf = self.scan_trips()
128
132
  # Add household_id, person_id, trip date, and trip weekday.
129
133
  lf = lf.with_columns(
@@ -203,6 +207,7 @@ class TripsReader(CeremaReader):
203
207
  year=self.survey_year() or CURRENT_YEAR,
204
208
  special_locations=self.special_locations_coords,
205
209
  detailed_zones=self.detailed_zones_coords,
210
+ skip_insee=skip_insee,
206
211
  )
207
212
  # When the INSEE code ends with "000" or "999" it means "rest of the département".
208
213
  # We drop these values because they do not add any additional information compared to `_dep`
@@ -41,9 +41,7 @@ class EDGTReader(CeremaStandardizer):
41
41
  else:
42
42
  return [
43
43
  find_file_path(
44
- self.source,
45
- ".*(_zf_gt|zinterne_gen_.*)[.](tab|shp|mif)",
46
- subdir=subdir,
44
+ self.source, ".*(_zf_gt|zinterne_gen_.*)[.](tab|shp|mif)", subdir=subdir
47
45
  )
48
46
  ]
49
47
 
@@ -14,32 +14,16 @@ class EMC2Reader(CeremaStandardizer):
14
14
  SURVEY_TYPE = "EMC2"
15
15
 
16
16
  def households_filenames(self) -> list[str | io.BytesIO | MissingFileError]:
17
- return [
18
- find_file(
19
- self.source, ".*_std_men.csv", subdir=os.path.join("Csv", "Fichiers_Standard")
20
- )
21
- ]
17
+ return [find_file(self.source, ".*_std_men.csv")]
22
18
 
23
19
  def persons_filenames(self) -> list[str | io.BytesIO | MissingFileError]:
24
- return [
25
- find_file(
26
- self.source, ".*_std_pers.csv", subdir=os.path.join("Csv", "Fichiers_Standard")
27
- )
28
- ]
20
+ return [find_file(self.source, ".*_std_pers.csv")]
29
21
 
30
22
  def trips_filenames(self) -> list[str | io.BytesIO | MissingFileError]:
31
- return [
32
- find_file(
33
- self.source, ".*_std_depl.csv", subdir=os.path.join("Csv", "Fichiers_Standard")
34
- )
35
- ]
23
+ return [find_file(self.source, ".*_std_depl.csv")]
36
24
 
37
25
  def legs_filenames(self) -> list[str | io.BytesIO | MissingFileError]:
38
- return [
39
- find_file(
40
- self.source, ".*_std_traj.csv", subdir=os.path.join("Csv", "Fichiers_Standard")
41
- )
42
- ]
26
+ return [find_file(self.source, ".*_std_traj.csv")]
43
27
 
44
28
  def detailed_zones_filenames(self):
45
29
  return [
@@ -63,9 +47,7 @@ class EMC2Reader(CeremaStandardizer):
63
47
  ]
64
48
 
65
49
  def survey_name(self):
66
- filename = find_file_path(
67
- self.source, ".*_std_men.csv", subdir=os.path.join("Csv", "Fichiers_Standard")
68
- )
50
+ filename = find_file_path(self.source, ".*_std_men.csv")
69
51
  fn_match = re.match("(.*)_std_men.csv", os.path.basename(filename))
70
52
  if fn_match is not None:
71
53
  return fn_match.group(1)
@@ -6,6 +6,7 @@ from mobisurvstd.common.cars import clean as clean_cars
6
6
  from mobisurvstd.common.households import clean as clean_households
7
7
  from mobisurvstd.common.motorcycles import clean as clean_motorcycles
8
8
  from mobisurvstd.schema.common import CURRENT_YEAR
9
+ from mobisurvstd.utils import detect_csv_delimiter
9
10
 
10
11
  from .reader import CeremaReader
11
12
 
@@ -92,10 +93,7 @@ SCHEMA = {
92
93
  "COE0": pl.Float64, # COEFFICIENT DE REDRESSEMENT MENAGE
93
94
  }
94
95
 
95
- SURVEY_METHOD_MAP = {
96
- 1: "face_to_face",
97
- 2: "phone",
98
- }
96
+ SURVEY_METHOD_MAP = {1: "face_to_face", 2: "phone"}
99
97
 
100
98
  HOUSING_TYPE_MAP = {
101
99
  1: "house", # Individuel isolé
@@ -165,6 +163,7 @@ PARKING_LOCATION_MAP = {
165
163
  3: "parking_lot", # Dans un parc à ciel ouvert (ou place publique)
166
164
  4: "parking_lot", # Dans un parc couvert accessible au public
167
165
  5: "other", # Autre (egt)
166
+ 9: "other", # Autre (egt)
168
167
  }
169
168
 
170
169
  PARKING_TYPE_MAP = {
@@ -173,6 +172,7 @@ PARKING_TYPE_MAP = {
173
172
  3: "paid", # Payant à votre charge
174
173
  4: "paid_by_other", # Payant à la charge de quelqu'un d'autre
175
174
  5: "other", # Autre (egt)
175
+ 9: "other", # Autre (egt)
176
176
  }
177
177
 
178
178
  MOTORCYCLE_TYPE_MAP = {
@@ -195,41 +195,20 @@ MOTORCYCLE_FUEL_TYPE_MAP = {
195
195
  90: "other",
196
196
  }
197
197
 
198
- MOTORCYCLE_SIZE_LOWERBOUND_MAP = {
199
- 1: 0,
200
- 2: 50,
201
- 3: 125,
202
- 4: 250,
203
- 5: 750,
204
- 9: 125,
205
- }
198
+ MOTORCYCLE_SIZE_LOWERBOUND_MAP = {1: 0, 2: 50, 3: 125, 4: 250, 5: 750, 9: 125}
206
199
 
207
- MOTORCYCLE_SIZE_UPPERBOUND_MAP = {
208
- 1: 50,
209
- 2: 125,
210
- 3: 250,
211
- 4: 750,
212
- }
200
+ MOTORCYCLE_SIZE_UPPERBOUND_MAP = {1: 50, 2: 125, 3: 250, 4: 750}
213
201
 
214
- MOTORCYCLE_KW_LOWERBOUND_MAP = {
215
- 6: 0,
216
- 7: 4,
217
- 8: 11,
218
- }
202
+ MOTORCYCLE_KW_LOWERBOUND_MAP = {6: 0, 7: 4, 8: 11}
219
203
 
220
- MOTORCYCLE_KW_UPPERBOUND_MAP = {
221
- 6: 4,
222
- 7: 11,
223
- }
204
+ MOTORCYCLE_KW_UPPERBOUND_MAP = {6: 4, 7: 11}
224
205
 
225
- MOTORCYCLE_THERMIC_ENGINE_TYPE_MAP = {
226
- 1: "two_stroke",
227
- 2: "four_stroke",
228
- }
206
+ MOTORCYCLE_THERMIC_ENGINE_TYPE_MAP = {1: "two_stroke", 2: "four_stroke"}
229
207
 
230
208
 
231
209
  def scan_households_impl(source: str | io.BytesIO):
232
- return pl.scan_csv(source, separator=";", schema_overrides=SCHEMA, null_values=["a"])
210
+ separator = detect_csv_delimiter(source)
211
+ return pl.scan_csv(source, separator=separator, schema_overrides=SCHEMA, null_values=["a"])
233
212
 
234
213
 
235
214
  class HouseholdsReader(CeremaReader):
@@ -248,7 +227,7 @@ class HouseholdsReader(CeremaReader):
248
227
  def main_insee(self):
249
228
  return self.scan_households().select("IDM4").first().collect().item() # ty: ignore[possibly-missing-attribute]
250
229
 
251
- def standardize_households(self):
230
+ def standardize_households(self, skip_insee: bool = False):
252
231
  lf = self.scan_households()
253
232
  lf = lf.rename(
254
233
  {
@@ -262,6 +241,7 @@ class HouseholdsReader(CeremaReader):
262
241
  )
263
242
  lf = lf.with_columns(
264
243
  original_household_id=pl.struct(self.get_household_index_cols()),
244
+ complete_household=True,
265
245
  home_detailed_zone="ZFM",
266
246
  home_draw_zone="STM",
267
247
  # For EMD, EDVM, EDGT, the IDM2 column will be read here.
@@ -278,6 +258,7 @@ class HouseholdsReader(CeremaReader):
278
258
  year=self.survey_year() or CURRENT_YEAR,
279
259
  special_locations=self.special_locations_coords,
280
260
  detailed_zones=self.detailed_zones_coords,
261
+ skip_insee=skip_insee,
281
262
  )
282
263
  # When the INSEE code ends with "000" or "999" it means "rest of the département".
283
264
  # We drop these values because they do not add any additional information compared to `_dep`
@@ -192,9 +192,7 @@ class OpenDataReader(CeremaStandardizer):
192
192
 
193
193
 
194
194
  def scan_fwf(
195
- f: str | io.BytesIO,
196
- format: list[tuple[int, int, str]],
197
- schema: dict[str, DataTypeClass],
195
+ f: str | io.BytesIO, format: list[tuple[int, int, str]], schema: dict[str, DataTypeClass]
198
196
  ):
199
197
  # Decrease `start` by 1 because pandas starts counting at 0 but the survey document starts
200
198
  # counting at 1.
@@ -3,6 +3,7 @@ import io
3
3
  import polars as pl
4
4
 
5
5
  from mobisurvstd.common.persons import clean
6
+ from mobisurvstd.utils import detect_csv_delimiter
6
7
 
7
8
  from .reader import CeremaReader
8
9
 
@@ -232,7 +233,10 @@ WEEKDAY_MAP = {
232
233
 
233
234
 
234
235
  def scan_persons_impl(source: str | io.BytesIO):
235
- return pl.scan_csv(source, separator=";", schema_overrides=SCHEMA, null_values=["a", "aa"])
236
+ separator = detect_csv_delimiter(source)
237
+ return pl.scan_csv(
238
+ source, separator=separator, schema_overrides=SCHEMA, null_values=["a", "aa"]
239
+ )
236
240
 
237
241
 
238
242
  class PersonsReader(CeremaReader):
@@ -240,6 +244,8 @@ class PersonsReader(CeremaReader):
240
244
  lfs_iter = map(scan_persons_impl, self.persons_filenames())
241
245
  lf = pl.concat(lfs_iter, how="vertical")
242
246
  lf = lf.sort(self.get_person_index_cols())
247
+ # Set empty strings and whitespace-only strings as NULL values.
248
+ lf = lf.with_columns(pl.col(pl.String).str.strip_chars().replace("", None))
243
249
  columns = lf.collect_schema().names()
244
250
  if "P13B" not in columns:
245
251
  # In the old Cerema surveys (EMD, EDGT, EDVM), the P13B column does not exist.
@@ -26,7 +26,7 @@ LOCATION_COLUMNS = (
26
26
  class CeremaStandardizer(HouseholdsReader, PersonsReader, TripsReader, LegsReader, ZonesReader):
27
27
  SURVEY_TYPE = ""
28
28
 
29
- def standardize(self, skip_spatial: bool = False):
29
+ def standardize(self, skip_spatial: bool = False, skip_insee: bool = False):
30
30
  logger.info(f"Standardizing {self.SURVEY_TYPE} survey from `{self.source_name()}`")
31
31
  if not self.validate():
32
32
  return None
@@ -34,11 +34,11 @@ class CeremaStandardizer(HouseholdsReader, PersonsReader, TripsReader, LegsReade
34
34
  if not skip_spatial:
35
35
  self.read_spatial_data()
36
36
 
37
- self.standardize_households()
37
+ self.standardize_households(skip_insee)
38
38
  self.standardize_cars()
39
39
  self.standardize_motorcycles()
40
40
  self.standardize_persons()
41
- self.standardize_trips()
41
+ self.standardize_trips(skip_insee)
42
42
  self.standardize_legs()
43
43
 
44
44
  self.finish()
@@ -115,9 +115,7 @@ class CeremaStandardizer(HouseholdsReader, PersonsReader, TripsReader, LegsReade
115
115
  ) # ty: ignore[invalid-assignment]
116
116
  self.households = self.households.join(
117
117
  household_dates.lazy(), on="household_id", how="left", coalesce=True
118
- ).with_columns(
119
- trips_weekday=pl.when("is_valid_weekday").then(pl.col("trips_weekday")),
120
- )
118
+ ).with_columns(trips_weekday=pl.when("is_valid_weekday").then(pl.col("trips_weekday")))
121
119
 
122
120
  def fix_main_mode(self):
123
121
  # Special case for Douai 2012: Some trips have main mode set to "car_driver" but the legs
@@ -4,6 +4,7 @@ import polars as pl
4
4
  from loguru import logger
5
5
 
6
6
  from mobisurvstd.common.legs import clean
7
+ from mobisurvstd.utils import detect_csv_delimiter
7
8
 
8
9
  from .reader import CeremaReader
9
10
 
@@ -93,8 +94,9 @@ MOTORCYCLE_MAP = {
93
94
 
94
95
 
95
96
  def scan_legs_impl(source: str | io.BytesIO):
97
+ separator = detect_csv_delimiter(source)
96
98
  return pl.scan_csv(
97
- source, separator=";", schema_overrides=SCHEMA, null_values=["a", "aa", "aaaaa"]
99
+ source, separator=separator, schema_overrides=SCHEMA, null_values=["a", "aa", "aaaaa"]
98
100
  )
99
101
 
100
102
 
@@ -342,7 +344,7 @@ class LegsReader(CeremaReader):
342
344
  .then(pl.col("nb_majors_in_vehicle") + 1)
343
345
  .otherwise("nb_majors_in_vehicle"),
344
346
  nb_minors_in_vehicle=pl.when(
345
- pl.col("is_major").not_() & pl.col("mode").str.contains("driver"),
347
+ pl.col("is_major").not_() & pl.col("mode").str.contains("driver")
346
348
  )
347
349
  .then(pl.col("nb_minors_in_vehicle") + 1)
348
350
  .otherwise("nb_minors_in_vehicle"),
@@ -67,16 +67,19 @@ class SurveyData:
67
67
  logger.warning("Output directory is not empty, some data might be erased")
68
68
  logger.debug("Saving households")
69
69
  self.households.write_parquet(output_directory / Path("households.parquet"))
70
- logger.debug("Saving cars")
71
- self.cars.write_parquet(output_directory / Path("cars.parquet"))
72
- logger.debug("Saving motorcycles")
73
- self.motorcycles.write_parquet(output_directory / Path("motorcycles.parquet"))
70
+ if not self.cars.is_empty():
71
+ logger.debug("Saving cars")
72
+ self.cars.write_parquet(output_directory / Path("cars.parquet"))
73
+ if not self.motorcycles.is_empty():
74
+ logger.debug("Saving motorcycles")
75
+ self.motorcycles.write_parquet(output_directory / Path("motorcycles.parquet"))
74
76
  logger.debug("Saving persons")
75
77
  self.persons.write_parquet(output_directory / Path("persons.parquet"))
76
78
  logger.debug("Saving trips")
77
79
  self.trips.write_parquet(output_directory / Path("trips.parquet"))
78
- logger.debug("Saving legs")
79
- self.legs.write_parquet(output_directory / Path("legs.parquet"))
80
+ if not self.legs.is_empty():
81
+ logger.debug("Saving legs")
82
+ self.legs.write_parquet(output_directory / Path("legs.parquet"))
80
83
  if self.special_locations is not None:
81
84
  logger.debug("Saving special locations")
82
85
  self.special_locations.to_parquet(
@@ -192,16 +195,22 @@ class SurveyDataReader:
192
195
  def cars(self) -> pl.DataFrame:
193
196
  if self._cars is None:
194
197
  filename = os.path.join(self.directory, "cars.parquet")
195
- logger.debug(f"Reading cars from `{filename}`")
196
- self._cars = pl.read_parquet(filename)
198
+ if os.path.isfile(filename):
199
+ logger.debug(f"Reading cars from `{filename}`")
200
+ self._cars = pl.read_parquet(filename)
201
+ else:
202
+ self._cars = pl.DataFrame()
197
203
  return self._cars
198
204
 
199
205
  @property
200
206
  def motorcycles(self) -> pl.DataFrame:
201
207
  if self._motorcycles is None:
202
208
  filename = os.path.join(self.directory, "motorcycles.parquet")
203
- logger.debug(f"Reading motorcycles from `{filename}`")
204
- self._motorcycles = pl.read_parquet(filename)
209
+ if os.path.isfile(filename):
210
+ logger.debug(f"Reading motorcycles from `{filename}`")
211
+ self._motorcycles = pl.read_parquet(filename)
212
+ else:
213
+ self._motorcycles = pl.DataFrame()
205
214
  return self._motorcycles
206
215
 
207
216
  @property
@@ -323,8 +332,6 @@ def is_valid_mobisurvstd_dir(directory: str) -> bool:
323
332
  "persons.parquet",
324
333
  "trips.parquet",
325
334
  "legs.parquet",
326
- "cars.parquet",
327
- "motorcycles.parquet",
328
335
  ):
329
336
  if not os.path.isfile(os.path.join(directory, name)):
330
337
  return False
@@ -29,13 +29,14 @@ def standardizer(
29
29
  survey_type: Annotated[
30
30
  str | None,
31
31
  typer.Option(
32
- help="Format of the original survey. Possible values: `emc2`, `emp2019`, `egt2010`, `egt2020`, `edgt`, `edvm`, `emd`."
32
+ help="Format of the original survey. Possible values: `emc2`, `emp2019`, `egt2010`, `egt2020`, `edgt`, `edvm`, `emd`, `emg2023`."
33
33
  ),
34
34
  ] = None,
35
35
  bulk: bool = typer.Option(
36
36
  False, "--bulk", help="Import surveys in bulk from the given directory"
37
37
  ),
38
38
  skip_spatial: bool = typer.Option(False, "--skip-spatial", help="Do not read spatial data"),
39
+ skip_insee: bool = typer.Option(False, "--skip-insee", help="Do not add INSEE data"),
39
40
  no_validation: bool = typer.Option(
40
41
  False,
41
42
  "--no-validation",
@@ -57,6 +58,7 @@ def standardizer(
57
58
  output_directory,
58
59
  survey_type,
59
60
  skip_spatial=skip_spatial,
61
+ skip_insee=skip_insee,
60
62
  no_validation=no_validation,
61
63
  )
62
64
  else:
@@ -65,6 +67,7 @@ def standardizer(
65
67
  output_directory,
66
68
  survey_type,
67
69
  skip_spatial=skip_spatial,
70
+ skip_insee=skip_insee,
68
71
  no_validation=no_validation,
69
72
  )
70
73
 
@@ -128,7 +128,7 @@ def create_metadata(
128
128
  "nb_motorcycles": len(data["motorcycles"]),
129
129
  "nb_persons": len(data["persons"]),
130
130
  "nb_trips": len(data["trips"]),
131
- "nb_legs": len(data["legs"]),
131
+ "nb_legs": len(data["legs"]) if "legs" in data else 0,
132
132
  "nb_special_locations": nb_special_locations,
133
133
  "nb_detailed_zones": nb_detailed_zones,
134
134
  "nb_draw_zones": nb_draw_zones,
@@ -151,22 +151,33 @@ def count_nb_persons(households: pl.LazyFrame, persons: pl.LazyFrame):
151
151
  has_age = "age" in persons.collect_schema().names()
152
152
  has_age_class = "age_class_code" in persons.collect_schema().names()
153
153
  person_counts = persons.group_by("household_id").agg(
154
+ complete_household=True,
154
155
  nb_persons=pl.len(),
155
- nb_persons_5plus=pl.when(pl.col("age").is_not_null().all()).then(pl.col("age").ge(5).sum())
156
- if has_age
157
- else None,
158
- nb_majors=pl.when(pl.col("age_class_code").is_not_null().all()).then(
159
- pl.col("age_class_code").gt(1).sum()
160
- )
161
- if has_age_class
162
- else None,
163
- nb_minors=pl.when(pl.col("age_class_code").is_not_null().all()).then(
164
- pl.col("age_class_code").eq(1).sum()
165
- )
166
- if has_age_class
167
- else None,
156
+ nb_persons_5plus=(
157
+ pl.when(pl.col("age").is_not_null().all()).then(pl.col("age").ge(5).sum())
158
+ if has_age
159
+ else None
160
+ ),
161
+ nb_majors=(
162
+ pl.when(pl.col("age_class_code").is_not_null().all()).then(
163
+ pl.col("age_class_code").gt(1).sum()
164
+ )
165
+ if has_age_class
166
+ else None
167
+ ),
168
+ nb_minors=(
169
+ pl.when(pl.col("age_class_code").is_not_null().all()).then(
170
+ pl.col("age_class_code").eq(1).sum()
171
+ )
172
+ if has_age_class
173
+ else None
174
+ ),
175
+ )
176
+ # Join on both "household_id" and "complete_household" so that counts are not added for
177
+ # incomplete households.
178
+ households = households.join(
179
+ person_counts, on=["household_id", "complete_household"], how="left", coalesce=True
168
180
  )
169
- households = households.join(person_counts, on="household_id", how="left", coalesce=True)
170
181
  return households
171
182
 
172
183
 
@@ -238,6 +249,7 @@ def count_nb_trips(persons: pl.LazyFrame, trips: pl.LazyFrame):
238
249
 
239
250
  def add_worked_during_surveyed_day(persons: pl.LazyFrame, trips: pl.LazyFrame):
240
251
  persons_cols = persons.collect_schema().names()
252
+ trips_cols = trips.collect_schema().names()
241
253
  # Fill null values of `traveled_during_surveyed_day` (when `is_surveyed` is True) according
242
254
  # to the existence of trips.
243
255
  if "traveled_during_surveyed_day" not in persons_cols:
@@ -250,7 +262,7 @@ def add_worked_during_surveyed_day(persons: pl.LazyFrame, trips: pl.LazyFrame):
250
262
  .then(pl.lit("no"))
251
263
  )
252
264
  )
253
- if "worked_during_surveyed_day" not in persons_cols:
265
+ if "worked_during_surveyed_day" not in persons_cols and "destination_purpose" in trips_cols:
254
266
  persons = persons.join(
255
267
  trips.group_by("person_id").agg(
256
268
  has_work_activity=pl.col("destination_purpose")
@@ -14,6 +14,7 @@ def clean(
14
14
  year: int | None,
15
15
  special_locations: pl.DataFrame | None = None,
16
16
  detailed_zones: pl.DataFrame | None = None,
17
+ skip_insee: bool = False,
17
18
  ):
18
19
  existing_cols = lf.collect_schema().names()
19
20
  columns = [variable.name for variable in HOUSEHOLD_SCHEMA if variable.name in existing_cols]
@@ -21,8 +22,9 @@ def clean(
21
22
  lf = indexing(lf)
22
23
  lf = add_bicycle_counts(lf, existing_cols)
23
24
  lf = add_lng_lat(lf, existing_cols, special_locations, detailed_zones)
24
- lf = add_insee_columns(lf, existing_cols)
25
- lf = add_insee_data_columns(lf, existing_cols, year)
25
+ if not skip_insee:
26
+ lf = add_insee_columns(lf, existing_cols)
27
+ lf = add_insee_data_columns(lf, existing_cols, year)
26
28
  if "home_dep" in existing_cols:
27
29
  lf = add_nuts_data(lf, "home")
28
30
  if DEBUG: