mobisurvstd 1.1.0__tar.gz → 1.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. {mobisurvstd-1.1.0/src/mobisurvstd.egg-info → mobisurvstd-1.2.0}/PKG-INFO +2 -2
  2. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/pyproject.toml +2 -2
  3. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/cerema/deplacements.py +5 -1
  4. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/cerema/emc2.py +5 -23
  5. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/cerema/menages.py +5 -1
  6. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/cerema/personnes.py +7 -1
  7. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/cerema/trajets.py +3 -1
  8. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/main.py +1 -1
  9. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/schema/persons.py +5 -1
  10. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/utils.py +2 -1
  11. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0/src/mobisurvstd.egg-info}/PKG-INFO +2 -2
  12. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd.egg-info/requires.txt +1 -1
  13. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/LICENSE.md +0 -0
  14. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/README.md +0 -0
  15. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/setup.cfg +0 -0
  16. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/__init__.py +0 -0
  17. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/__main__.py +0 -0
  18. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/cerema/__init__.py +0 -0
  19. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/cerema/common.py +0 -0
  20. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/cerema/edgt.py +0 -0
  21. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/cerema/edvm.py +0 -0
  22. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/cerema/emd.py +0 -0
  23. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/cerema/nantes_opendata.py +0 -0
  24. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/cerema/reader.py +0 -0
  25. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/cerema/survey.py +0 -0
  26. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/cerema/zones.py +0 -0
  27. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/classes.py +0 -0
  28. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/cli.py +0 -0
  29. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/common/__init__.py +0 -0
  30. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/common/cars.py +0 -0
  31. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/common/clean.py +0 -0
  32. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/common/households.py +0 -0
  33. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/common/legs.py +0 -0
  34. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/common/motorcycles.py +0 -0
  35. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/common/persons.py +0 -0
  36. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/common/trips.py +0 -0
  37. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/common/zones.py +0 -0
  38. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/egt2010/__init__.py +0 -0
  39. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/egt2010/deplacements.py +0 -0
  40. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/egt2010/menages.py +0 -0
  41. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/egt2010/personnes.py +0 -0
  42. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/egt2010/survey.py +0 -0
  43. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/egt2010/trajets.py +0 -0
  44. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/egt2010/zones.py +0 -0
  45. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/egt2020/__init__.py +0 -0
  46. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/egt2020/deplacements.py +0 -0
  47. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/egt2020/menages.py +0 -0
  48. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/egt2020/motos.py +0 -0
  49. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/egt2020/personnes.py +0 -0
  50. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/egt2020/survey.py +0 -0
  51. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/egt2020/trajets.py +0 -0
  52. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/egt2020/voitures.py +0 -0
  53. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/emg2023/__init__.py +0 -0
  54. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/emg2023/deplacements.py +0 -0
  55. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/emg2023/menages.py +0 -0
  56. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/emg2023/personnes.py +0 -0
  57. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/emg2023/survey.py +0 -0
  58. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/emp/__init__.py +0 -0
  59. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/emp/common.py +0 -0
  60. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/emp/deplacements.py +0 -0
  61. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/emp/menages.py +0 -0
  62. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/emp/motos.py +0 -0
  63. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/emp/personnes.py +0 -0
  64. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/emp/survey.py +0 -0
  65. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/emp/voitures.py +0 -0
  66. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/logger.py +0 -0
  67. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/resources/__init__.py +0 -0
  68. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/resources/admin_express.py +0 -0
  69. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/resources/insee_data.py +0 -0
  70. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/resources/nuts.py +0 -0
  71. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/schema/__init__.py +0 -0
  72. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/schema/cars.py +0 -0
  73. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/schema/common.py +0 -0
  74. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/schema/guarantees.py +0 -0
  75. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/schema/households.py +0 -0
  76. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/schema/legs.py +0 -0
  77. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/schema/motorcycles.py +0 -0
  78. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/schema/trips.py +0 -0
  79. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd/validation.py +0 -0
  80. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd.egg-info/SOURCES.txt +0 -0
  81. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd.egg-info/dependency_links.txt +0 -0
  82. {mobisurvstd-1.1.0 → mobisurvstd-1.2.0}/src/mobisurvstd.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: mobisurvstd
3
- Version: 1.1.0
3
+ Version: 1.2.0
4
4
  Summary: A tool to standardize French mobility survey datasets (EMC2, EGT, EMP, etc.) into a unified format.
5
5
  Author-email: Lucas Javaudin <lucas@lucasjavaudin.com>
6
6
  License-Expression: MIT
@@ -23,7 +23,7 @@ Requires-Dist: numpy>=2.2.4
23
23
  Requires-Dist: pandas>=2.2.3
24
24
  Requires-Dist: platformdirs>=4.3.8
25
25
  Requires-Dist: polars>=1.38.0
26
- Requires-Dist: py7zr>=0.20.8
26
+ Requires-Dist: py7zr>=1.1.0
27
27
  Requires-Dist: pyarrow>=19.0.1
28
28
  Requires-Dist: requests>=2.32.3
29
29
  Requires-Dist: tqdm>=4.67.3
@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
6
6
 
7
7
  [project]
8
8
  name = "mobisurvstd"
9
- version = "1.1.0"
9
+ version = "1.2.0"
10
10
  description = "A tool to standardize French mobility survey datasets (EMC2, EGT, EMP, etc.) into a unified format."
11
11
  readme = "README.md"
12
12
  authors = [{ name = "Lucas Javaudin", email = "lucas@lucasjavaudin.com" }]
@@ -27,7 +27,7 @@ dependencies = [
27
27
  "pandas>=2.2.3",
28
28
  "platformdirs>=4.3.8",
29
29
  "polars>=1.38.0",
30
- "py7zr>=0.20.8",
30
+ "py7zr>=1.1.0",
31
31
  "pyarrow>=19.0.1",
32
32
  "requests>=2.32.3",
33
33
  "tqdm>=4.67.3",
@@ -5,6 +5,7 @@ from loguru import logger
5
5
 
6
6
  from mobisurvstd.common.trips import clean
7
7
  from mobisurvstd.schema.common import CURRENT_YEAR
8
+ from mobisurvstd.utils import detect_csv_delimiter
8
9
 
9
10
  from .reader import CeremaReader
10
11
 
@@ -113,7 +114,10 @@ TRIP_PERIMETER_MAP = {
113
114
 
114
115
 
115
116
  def scan_trips_impl(source: str | io.BytesIO):
116
- return pl.scan_csv(source, separator=";", schema_overrides=SCHEMA, null_values=["aa", "aaaaa"])
117
+ separator = detect_csv_delimiter(source)
118
+ return pl.scan_csv(
119
+ source, separator=separator, schema_overrides=SCHEMA, null_values=["aa", "aaaaa"]
120
+ )
117
121
 
118
122
 
119
123
  class TripsReader(CeremaReader):
@@ -14,32 +14,16 @@ class EMC2Reader(CeremaStandardizer):
14
14
  SURVEY_TYPE = "EMC2"
15
15
 
16
16
  def households_filenames(self) -> list[str | io.BytesIO | MissingFileError]:
17
- return [
18
- find_file(
19
- self.source, ".*_std_men.csv", subdir=os.path.join("Csv", "Fichiers_Standard")
20
- )
21
- ]
17
+ return [find_file(self.source, ".*_std_men.csv")]
22
18
 
23
19
  def persons_filenames(self) -> list[str | io.BytesIO | MissingFileError]:
24
- return [
25
- find_file(
26
- self.source, ".*_std_pers.csv", subdir=os.path.join("Csv", "Fichiers_Standard")
27
- )
28
- ]
20
+ return [find_file(self.source, ".*_std_pers.csv")]
29
21
 
30
22
  def trips_filenames(self) -> list[str | io.BytesIO | MissingFileError]:
31
- return [
32
- find_file(
33
- self.source, ".*_std_depl.csv", subdir=os.path.join("Csv", "Fichiers_Standard")
34
- )
35
- ]
23
+ return [find_file(self.source, ".*_std_depl.csv")]
36
24
 
37
25
  def legs_filenames(self) -> list[str | io.BytesIO | MissingFileError]:
38
- return [
39
- find_file(
40
- self.source, ".*_std_traj.csv", subdir=os.path.join("Csv", "Fichiers_Standard")
41
- )
42
- ]
26
+ return [find_file(self.source, ".*_std_traj.csv")]
43
27
 
44
28
  def detailed_zones_filenames(self):
45
29
  return [
@@ -63,9 +47,7 @@ class EMC2Reader(CeremaStandardizer):
63
47
  ]
64
48
 
65
49
  def survey_name(self):
66
- filename = find_file_path(
67
- self.source, ".*_std_men.csv", subdir=os.path.join("Csv", "Fichiers_Standard")
68
- )
50
+ filename = find_file_path(self.source, ".*_std_men.csv")
69
51
  fn_match = re.match("(.*)_std_men.csv", os.path.basename(filename))
70
52
  if fn_match is not None:
71
53
  return fn_match.group(1)
@@ -6,6 +6,7 @@ from mobisurvstd.common.cars import clean as clean_cars
6
6
  from mobisurvstd.common.households import clean as clean_households
7
7
  from mobisurvstd.common.motorcycles import clean as clean_motorcycles
8
8
  from mobisurvstd.schema.common import CURRENT_YEAR
9
+ from mobisurvstd.utils import detect_csv_delimiter
9
10
 
10
11
  from .reader import CeremaReader
11
12
 
@@ -162,6 +163,7 @@ PARKING_LOCATION_MAP = {
162
163
  3: "parking_lot", # Dans un parc à ciel ouvert (ou place publique)
163
164
  4: "parking_lot", # Dans un parc couvert accessible au public
164
165
  5: "other", # Autre (egt)
166
+ 9: "other", # Autre (egt)
165
167
  }
166
168
 
167
169
  PARKING_TYPE_MAP = {
@@ -170,6 +172,7 @@ PARKING_TYPE_MAP = {
170
172
  3: "paid", # Payant à votre charge
171
173
  4: "paid_by_other", # Payant à la charge de quelqu'un d'autre
172
174
  5: "other", # Autre (egt)
175
+ 9: "other", # Autre (egt)
173
176
  }
174
177
 
175
178
  MOTORCYCLE_TYPE_MAP = {
@@ -204,7 +207,8 @@ MOTORCYCLE_THERMIC_ENGINE_TYPE_MAP = {1: "two_stroke", 2: "four_stroke"}
204
207
 
205
208
 
206
209
  def scan_households_impl(source: str | io.BytesIO):
207
- return pl.scan_csv(source, separator=";", schema_overrides=SCHEMA, null_values=["a"])
210
+ separator = detect_csv_delimiter(source)
211
+ return pl.scan_csv(source, separator=separator, schema_overrides=SCHEMA, null_values=["a"])
208
212
 
209
213
 
210
214
  class HouseholdsReader(CeremaReader):
@@ -3,6 +3,7 @@ import io
3
3
  import polars as pl
4
4
 
5
5
  from mobisurvstd.common.persons import clean
6
+ from mobisurvstd.utils import detect_csv_delimiter
6
7
 
7
8
  from .reader import CeremaReader
8
9
 
@@ -232,7 +233,10 @@ WEEKDAY_MAP = {
232
233
 
233
234
 
234
235
  def scan_persons_impl(source: str | io.BytesIO):
235
- return pl.scan_csv(source, separator=";", schema_overrides=SCHEMA, null_values=["a", "aa"])
236
+ separator = detect_csv_delimiter(source)
237
+ return pl.scan_csv(
238
+ source, separator=separator, schema_overrides=SCHEMA, null_values=["a", "aa"]
239
+ )
236
240
 
237
241
 
238
242
  class PersonsReader(CeremaReader):
@@ -240,6 +244,8 @@ class PersonsReader(CeremaReader):
240
244
  lfs_iter = map(scan_persons_impl, self.persons_filenames())
241
245
  lf = pl.concat(lfs_iter, how="vertical")
242
246
  lf = lf.sort(self.get_person_index_cols())
247
+ # Set empty strings and whitespace-only strings as NULL values.
248
+ lf = lf.with_columns(pl.col(pl.String).str.strip_chars().replace("", None))
243
249
  columns = lf.collect_schema().names()
244
250
  if "P13B" not in columns:
245
251
  # In the old Cerema surveys (EMD, EDGT, EDVM), the P13B column does not exist.
@@ -4,6 +4,7 @@ import polars as pl
4
4
  from loguru import logger
5
5
 
6
6
  from mobisurvstd.common.legs import clean
7
+ from mobisurvstd.utils import detect_csv_delimiter
7
8
 
8
9
  from .reader import CeremaReader
9
10
 
@@ -93,8 +94,9 @@ MOTORCYCLE_MAP = {
93
94
 
94
95
 
95
96
  def scan_legs_impl(source: str | io.BytesIO):
97
+ separator = detect_csv_delimiter(source)
96
98
  return pl.scan_csv(
97
- source, separator=";", schema_overrides=SCHEMA, null_values=["a", "aa", "aaaaa"]
99
+ source, separator=separator, schema_overrides=SCHEMA, null_values=["a", "aa", "aaaaa"]
98
100
  )
99
101
 
100
102
 
@@ -209,7 +209,7 @@ def bulk_standardize_impl(
209
209
  for inner in os.listdir(directory):
210
210
  source = os.path.join(directory, inner)
211
211
  if os.path.isdir(source):
212
- maybe_type = guess_survey_type(directory)
212
+ maybe_type = guess_survey_type(source)
213
213
  if maybe_type is None:
214
214
  # The directory does not seem to be a valid survey.
215
215
  # We try to iteratively read that directory.
@@ -855,7 +855,11 @@ PERSON_SCHEMA = [
855
855
  ),
856
856
  [
857
857
  Null(when=pl.col("is_surveyed").eq(False)),
858
- ValueIsNot("yes:home:usual", when=pl.col("work_only_at_home").eq(False)),
858
+ ValueIsNot(
859
+ "yes:home:usual",
860
+ when=pl.col("work_only_at_home").eq(False),
861
+ fix_value="yes:home:other",
862
+ ),
859
863
  ],
860
864
  ),
861
865
  # Number of trips that this person performed.
@@ -225,8 +225,9 @@ def guess_survey_type(source: str | ZipFile) -> str | None:
225
225
  # Nantes 2015 open data format.
226
226
  return "edgt-opendata"
227
227
  if bytes := find_file(source, ".*_std_men.csv", subdir="Csv"):
228
+ separator = detect_csv_delimiter(bytes)
228
229
  survey_type = (
229
- pl.scan_csv(bytes, separator=";", schema_overrides={"IDM1": pl.UInt8})
230
+ pl.scan_csv(bytes, separator=separator, schema_overrides={"IDM1": pl.UInt8})
230
231
  .select(pl.col("IDM1").first())
231
232
  .collect()
232
233
  .item() # ty: ignore[possibly-missing-attribute]
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: mobisurvstd
3
- Version: 1.1.0
3
+ Version: 1.2.0
4
4
  Summary: A tool to standardize French mobility survey datasets (EMC2, EGT, EMP, etc.) into a unified format.
5
5
  Author-email: Lucas Javaudin <lucas@lucasjavaudin.com>
6
6
  License-Expression: MIT
@@ -23,7 +23,7 @@ Requires-Dist: numpy>=2.2.4
23
23
  Requires-Dist: pandas>=2.2.3
24
24
  Requires-Dist: platformdirs>=4.3.8
25
25
  Requires-Dist: polars>=1.38.0
26
- Requires-Dist: py7zr>=0.20.8
26
+ Requires-Dist: py7zr>=1.1.0
27
27
  Requires-Dist: pyarrow>=19.0.1
28
28
  Requires-Dist: requests>=2.32.3
29
29
  Requires-Dist: tqdm>=4.67.3
@@ -6,7 +6,7 @@ numpy>=2.2.4
6
6
  pandas>=2.2.3
7
7
  platformdirs>=4.3.8
8
8
  polars>=1.38.0
9
- py7zr>=0.20.8
9
+ py7zr>=1.1.0
10
10
  pyarrow>=19.0.1
11
11
  requests>=2.32.3
12
12
  tqdm>=4.67.3
File without changes
File without changes
File without changes