mobisurvstd 1.3.0__tar.gz → 1.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. {mobisurvstd-1.3.0/src/mobisurvstd.egg-info → mobisurvstd-1.4.0}/PKG-INFO +2 -2
  2. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/README.md +1 -1
  3. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/pyproject.toml +1 -1
  4. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/cerema/__init__.py +2 -1
  5. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/cerema/deplacements.py +4 -2
  6. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/cerema/edgt.py +16 -46
  7. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/cerema/edvm.py +14 -29
  8. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/cerema/emc2.py +9 -8
  9. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/cerema/emd.py +14 -35
  10. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/cerema/menages.py +5 -3
  11. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/cerema/personnes.py +4 -2
  12. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/cerema/reader.py +4 -3
  13. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/cerema/survey.py +15 -15
  14. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/cerema/trajets.py +4 -2
  15. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/cli.py +17 -0
  16. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/common/cars.py +1 -1
  17. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/common/clean.py +7 -7
  18. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/common/households.py +1 -1
  19. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/common/legs.py +1 -1
  20. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/common/motorcycles.py +1 -1
  21. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/common/persons.py +1 -1
  22. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/common/trips.py +1 -1
  23. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/egt2010/deplacements.py +3 -2
  24. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/egt2010/menages.py +5 -4
  25. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/egt2010/personnes.py +4 -2
  26. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/egt2010/survey.py +14 -22
  27. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/egt2010/trajets.py +4 -2
  28. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/egt2020/survey.py +14 -21
  29. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/emg2023/deplacements.py +2 -2
  30. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/emg2023/personnes.py +1 -1
  31. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/emg2023/survey.py +5 -4
  32. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/emp/survey.py +8 -7
  33. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/main.py +32 -26
  34. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/resources/insee_data.py +3 -3
  35. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/utils.py +78 -66
  36. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0/src/mobisurvstd.egg-info}/PKG-INFO +2 -2
  37. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/LICENSE.md +0 -0
  38. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/setup.cfg +0 -0
  39. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/__init__.py +0 -0
  40. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/__main__.py +0 -0
  41. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/cerema/common.py +0 -0
  42. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/cerema/nantes_opendata.py +0 -0
  43. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/cerema/zones.py +0 -0
  44. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/classes.py +0 -0
  45. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/common/__init__.py +0 -0
  46. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/common/zones.py +0 -0
  47. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/egt2010/__init__.py +0 -0
  48. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/egt2010/zones.py +0 -0
  49. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/egt2020/__init__.py +0 -0
  50. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/egt2020/deplacements.py +0 -0
  51. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/egt2020/menages.py +0 -0
  52. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/egt2020/motos.py +0 -0
  53. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/egt2020/personnes.py +0 -0
  54. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/egt2020/trajets.py +0 -0
  55. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/egt2020/voitures.py +0 -0
  56. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/emg2023/__init__.py +0 -0
  57. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/emg2023/menages.py +0 -0
  58. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/emp/__init__.py +0 -0
  59. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/emp/common.py +0 -0
  60. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/emp/deplacements.py +0 -0
  61. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/emp/menages.py +0 -0
  62. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/emp/motos.py +0 -0
  63. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/emp/personnes.py +0 -0
  64. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/emp/voitures.py +0 -0
  65. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/logger.py +0 -0
  66. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/resources/__init__.py +0 -0
  67. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/resources/admin_express.py +0 -0
  68. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/resources/nuts.py +0 -0
  69. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/schema/__init__.py +0 -0
  70. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/schema/cars.py +0 -0
  71. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/schema/common.py +0 -0
  72. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/schema/guarantees.py +0 -0
  73. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/schema/households.py +0 -0
  74. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/schema/legs.py +0 -0
  75. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/schema/motorcycles.py +0 -0
  76. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/schema/persons.py +0 -0
  77. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/schema/trips.py +0 -0
  78. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/validation.py +0 -0
  79. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd.egg-info/SOURCES.txt +0 -0
  80. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd.egg-info/dependency_links.txt +0 -0
  81. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd.egg-info/entry_points.txt +0 -0
  82. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd.egg-info/requires.txt +0 -0
  83. {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: mobisurvstd
3
- Version: 1.3.0
3
+ Version: 1.4.0
4
4
  Summary: A tool to standardize French mobility survey datasets (EMC2, EGT, EMP, etc.) into a unified format.
5
5
  Author-email: Lucas Javaudin <lucas@lucasjavaudin.com>
6
6
  License-Expression: MIT
@@ -135,7 +135,7 @@ Download the
135
135
  _(Données individuelles anonymisées (fichiers au format CSV))_, then run the following command:
136
136
 
137
137
  ```bash
138
- python -m mobisurvstd emp_2019_donnees_individuelles_anonymisees_novembre2024.zip standardized_emp2019 --survey-type emp2019
138
+ mobisurvstd emp_2019_donnees_individuelles_anonymisees_novembre2024.zip standardized_emp2019 --survey-type emp2019
139
139
  ```
140
140
 
141
141
  It will standardize the EMP 2019 survey and save the resulting Parquet files in the
@@ -102,7 +102,7 @@ Download the
102
102
  _(Données individuelles anonymisées (fichiers au format CSV))_, then run the following command:
103
103
 
104
104
  ```bash
105
- python -m mobisurvstd emp_2019_donnees_individuelles_anonymisees_novembre2024.zip standardized_emp2019 --survey-type emp2019
105
+ mobisurvstd emp_2019_donnees_individuelles_anonymisees_novembre2024.zip standardized_emp2019 --survey-type emp2019
106
106
  ```
107
107
 
108
108
  It will standardize the EMP 2019 survey and save the resulting Parquet files in the
@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
6
6
 
7
7
  [project]
8
8
  name = "mobisurvstd"
9
- version = "1.3.0"
9
+ version = "1.4.0"
10
10
  description = "A tool to standardize French mobility survey datasets (EMC2, EGT, EMP, etc.) into a unified format."
11
11
  readme = "README.md"
12
12
  authors = [{ name = "Lucas Javaudin", email = "lucas@lucasjavaudin.com" }]
@@ -1,3 +1,4 @@
1
+ from pathlib import Path
1
2
  from zipfile import ZipFile
2
3
 
3
4
  from .edgt import EDGTReader
@@ -8,7 +9,7 @@ from .nantes_opendata import OpenDataReader
8
9
 
9
10
 
10
11
  def standardize(
11
- source: str | ZipFile, survey_type: str, skip_spatial: bool = False, skip_insee: bool = False
12
+ source: Path | ZipFile, survey_type: str, skip_spatial: bool = False, skip_insee: bool = False
12
13
  ):
13
14
  if survey_type == "EMC2":
14
15
  reader = EMC2Reader(source)
@@ -1,7 +1,9 @@
1
1
  import io
2
+ from pathlib import Path
2
3
 
3
4
  import polars as pl
4
5
  from loguru import logger
6
+ from polars.datatypes import DataTypeClass
5
7
 
6
8
  from mobisurvstd.common.trips import clean
7
9
  from mobisurvstd.schema.common import CURRENT_YEAR
@@ -9,7 +11,7 @@ from mobisurvstd.utils import detect_csv_delimiter
9
11
 
10
12
  from .reader import CeremaReader
11
13
 
12
- SCHEMA = {
14
+ SCHEMA: dict[str, DataTypeClass] = {
13
15
  "DP1": pl.UInt8, # Code fichier = 3 (déplacement)
14
16
  "DMET": pl.UInt8, # Méthode d'enquête du ménage (EMC2 only)
15
17
  "IDD3": pl.UInt16, # Année de fin d'enquête
@@ -113,7 +115,7 @@ TRIP_PERIMETER_MAP = {
113
115
  }
114
116
 
115
117
 
116
- def scan_trips_impl(source: str | io.BytesIO):
118
+ def scan_trips_impl(source: Path | io.BytesIO):
117
119
  separator = detect_csv_delimiter(source)
118
120
  return pl.scan_csv(
119
121
  source, separator=separator, schema_overrides=SCHEMA, null_values=["aa", "aaaaa"]
@@ -1,6 +1,7 @@
1
1
  import io
2
2
  import os
3
3
  import re
4
+ from pathlib import Path
4
5
  from zipfile import ZipFile
5
6
 
6
7
  import geopandas as gpd
@@ -28,49 +29,26 @@ class EDGTReader(CeremaStandardizer):
28
29
 
29
30
  def special_locations_and_detailed_zones_filenames(self):
30
31
  # This should match the Amiens 2010, Clermont-Ferrand 2012, and Lyon 2015 surveys.
31
- subdir = os.path.join("Doc", "SIG")
32
- faf_filename = find_file_path(
33
- self.source, ".*_faf_d.?coupagefin[.](tab|shp|mif)", subdir=subdir
34
- )
35
- tel_filename = find_file_path(
36
- self.source, ".*_tel_d.?coupagefin[.](tab|shp|mif)", subdir=subdir
37
- )
32
+ faf_filename = find_file_path(self.source, ".*_faf_d.?coupagefin[.](tab|shp|mif)")
33
+ tel_filename = find_file_path(self.source, ".*_tel_d.?coupagefin[.](tab|shp|mif)")
38
34
  if faf_filename and tel_filename:
39
35
  # Amiens 2010 has two files.
40
36
  return [faf_filename, tel_filename]
41
37
  else:
42
- return [
43
- find_file_path(
44
- self.source, ".*(_zf_gt|zinterne_gen_.*)[.](tab|shp|mif)", subdir=subdir
45
- )
46
- ]
38
+ return [find_file_path(self.source, ".*(_zf_gt|zinterne_gen_.*)[.](tab|shp|mif)")]
47
39
 
48
40
  def special_locations_filenames(self):
49
- subdir = os.path.join("Doc", "SIG")
50
- faf_filename = find_file_path(
51
- self.source, ".*_faf_.*g.?n.?rateur.*[.](tab|shp|mif)", subdir=subdir
52
- )
53
- tel_filename = find_file_path(
54
- self.source, ".*_tel_.*g.?n.?rateur.*[.](tab|shp|mif)", subdir=subdir
55
- )
41
+ faf_filename = find_file_path(self.source, ".*_faf_.*g.?n.?rateur.*[.](tab|shp|mif)")
42
+ tel_filename = find_file_path(self.source, ".*_tel_.*g.?n.?rateur.*[.](tab|shp|mif)")
56
43
  if faf_filename and tel_filename:
57
44
  # Saint-Quentin-en-Yvelines has two files, for phone and face-to-face surveys.
58
45
  return [faf_filename, tel_filename]
59
46
  else:
60
- return [
61
- find_file_path(
62
- self.source, ".*(_gt_.*|_gt|g.?n.?rateur.*)[.](tab|shp|mif)", subdir=subdir
63
- )
64
- ]
47
+ return [find_file_path(self.source, ".*(_gt_.*|_gt|g.?n.?rateur.*)[.](tab|shp|mif)")]
65
48
 
66
49
  def detailed_zones_filenames(self):
67
- subdir = os.path.join("Doc", "SIG")
68
- faf_filename = find_file_path(
69
- self.source, ".*_faf_zones_fines[.](tab|shp|mif)", subdir=subdir
70
- )
71
- tel_filename = find_file_path(
72
- self.source, ".*_tel_zones_fines[.](tab|shp|mif)", subdir=subdir
73
- )
50
+ faf_filename = find_file_path(self.source, ".*_faf_zones_fines[.](tab|shp|mif)")
51
+ tel_filename = find_file_path(self.source, ".*_tel_zones_fines[.](tab|shp|mif)")
74
52
  if faf_filename and tel_filename:
75
53
  # Nice 2009 and Saint-Quentin-en-Yvelines have two files, for phone and face-to-face
76
54
  # surveys.
@@ -78,28 +56,20 @@ class EDGTReader(CeremaStandardizer):
78
56
  else:
79
57
  return [
80
58
  find_file_path(
81
- self.source,
82
- ".*(_zf|_zf_.*|zf08_.*|(?<!tel_)zones[_ ]fines)[.](tab|shp|mif)",
83
- subdir=subdir,
59
+ self.source, ".*(_zf|_zf_.*|zf08_.*|(?<!tel_)zones[_ ]fines)[.](tab|shp|mif)"
84
60
  )
85
61
  ]
86
62
 
87
63
  def draw_zones_filenames(self):
88
- return [
89
- find_file_path(
90
- self.source,
91
- ".*(_dtir|_secteurstirage)[.](tab|shp|mif)",
92
- subdir=os.path.join("Doc", "SIG"),
93
- )
94
- ]
64
+ return [find_file_path(self.source, ".*(_dtir|_secteurstirage)[.](tab|shp|mif)")]
95
65
 
96
66
  def survey_name(self):
97
- filename = find_file_path(self.source, ".*_std_faf_men.csv", subdir="Csv")
67
+ filename = find_file_path(self.source, ".*_std_faf_men.csv")
98
68
  fn_match = re.match("(.*)_std_faf_men.csv", os.path.basename(filename))
99
69
  if fn_match is not None:
100
70
  return fn_match.group(1)
101
71
  else:
102
- return "unkown"
72
+ return "unknown"
103
73
 
104
74
  def gt_id_columns(self):
105
75
  return [
@@ -270,7 +240,7 @@ class EDGTReader(CeremaStandardizer):
270
240
  return zfs, gts
271
241
 
272
242
 
273
- def get_files(source: str | ZipFile, name: str) -> list[str | io.BytesIO | MissingFileError]:
243
+ def get_files(source: Path | ZipFile, name: str) -> list[Path | io.BytesIO | MissingFileError]:
274
244
  # In the EDGT surveys, there are two directories with "standardized" data in the Csv directory,
275
245
  # usually "Fichiers_Standard_Face_a_face" and "Fichiers_Standard_Telephone" but it can be
276
246
  # something else.
@@ -278,6 +248,6 @@ def get_files(source: str | ZipFile, name: str) -> list[str | io.BytesIO | Missi
278
248
  # "[SURVEY_NAME]_std_faf_[NAME].csv" and "[SURVEY_NAME]_std_tel_[NAME].csv" where NAME is either
279
249
  # "men", "pers", "depl", or "traj".
280
250
  return [
281
- find_file(source, f".*_std_faf_{name}.csv", subdir="Csv"),
282
- find_file(source, f".*_std_tel_{name}.csv", subdir="Csv"),
251
+ find_file(source, f".*_std_faf_{name}.csv"),
252
+ find_file(source, f".*_std_tel_{name}.csv"),
283
253
  ]
@@ -1,6 +1,7 @@
1
1
  import io
2
2
  import os
3
3
  import re
4
+ from pathlib import Path
4
5
 
5
6
  import geopandas as gpd
6
7
 
@@ -13,56 +14,40 @@ from .zones import find_matching_column
13
14
  class EDVMReader(CeremaStandardizer):
14
15
  SURVEY_TYPE = "EDVM"
15
16
 
16
- def households_filenames(self) -> list[str | io.BytesIO | MissingFileError]:
17
+ def households_filenames(self) -> list[Path | io.BytesIO | MissingFileError]:
17
18
  # The filenames are usually stored in "Csv/Fichiers_Standard/[SURVEY_NAME]_std_[NAME].csv"
18
19
  # but in some cases (e.g., Ajaccio 2017) the directory "Fichiers_Standard" is named based
19
20
  # on the survey name (e.g., Ajaccio_2017_Standard).
20
- return [find_file(self.source, ".*_std_men.csv", subdir="Csv")]
21
+ return [find_file(self.source, ".*_std_men.csv")]
21
22
 
22
- def persons_filenames(self) -> list[str | io.BytesIO | MissingFileError]:
23
- return [find_file(self.source, ".*_std_pers.csv", subdir="Csv")]
23
+ def persons_filenames(self) -> list[Path | io.BytesIO | MissingFileError]:
24
+ return [find_file(self.source, ".*_std_pers.csv")]
24
25
 
25
- def trips_filenames(self) -> list[str | io.BytesIO | MissingFileError]:
26
- return [find_file(self.source, ".*_std_depl.csv", subdir="Csv")]
26
+ def trips_filenames(self) -> list[Path | io.BytesIO | MissingFileError]:
27
+ return [find_file(self.source, ".*_std_depl.csv")]
27
28
 
28
- def legs_filenames(self) -> list[str | io.BytesIO | MissingFileError]:
29
- return [find_file(self.source, ".*_std_traj.csv", subdir="Csv")]
29
+ def legs_filenames(self) -> list[Path | io.BytesIO | MissingFileError]:
30
+ return [find_file(self.source, ".*_std_traj.csv")]
30
31
 
31
32
  def special_locations_and_detailed_zones_filenames(self):
32
33
  # This should only match the Beauvais 2011 survey.
33
- return [
34
- find_file_path(
35
- self.source, ".*beauvais.*_dfin[.](tab|shp|mif)", subdir=os.path.join("Doc", "SIG")
36
- )
37
- ]
34
+ return [find_file_path(self.source, ".*beauvais.*_dfin[.](tab|shp|mif)")]
38
35
 
39
36
  def detailed_zones_filenames(self):
40
37
  return [
41
- find_file_path(
42
- self.source,
43
- ".*(_zf_.*|_zf|zones?[_ ]?fines?.*|_dfin)[.](tab|shp|mif)",
44
- subdir=os.path.join("Doc", "SIG"),
45
- )
38
+ find_file_path(self.source, ".*(_zf_.*|_zf|zones?[_ ]?fines?.*|_dfin)[.](tab|shp|mif)")
46
39
  ]
47
40
 
48
41
  def special_locations_filenames(self):
49
42
  return [
50
- find_file_path(
51
- self.source,
52
- ".*(_gt_.*|_gt|_pgt|_pg|g.?n.?rateur.*)[.](tab|shp|mif)",
53
- subdir=os.path.join("Doc", "SIG"),
54
- )
43
+ find_file_path(self.source, ".*(_gt_.*|_gt|_pgt|_pg|g.?n.?rateur.*)[.](tab|shp|mif)")
55
44
  ]
56
45
 
57
46
  def draw_zones_filenames(self):
58
- return [
59
- find_file_path(
60
- self.source, ".*_DTIR[.](tab|shp|mif)", subdir=os.path.join("Doc", "SIG")
61
- )
62
- ]
47
+ return [find_file_path(self.source, ".*_DTIR[.](tab|shp|mif)")]
63
48
 
64
49
  def survey_name(self):
65
- filename = find_file_path(self.source, ".*_std_men.csv", subdir="Csv")
50
+ filename = find_file_path(self.source, ".*_std_men.csv")
66
51
  fn_match = re.match("(.*)_std_men.csv", os.path.basename(filename))
67
52
  if fn_match is not None:
68
53
  return fn_match.group(1)
@@ -1,6 +1,7 @@
1
1
  import io
2
2
  import os
3
3
  import re
4
+ from pathlib import Path
4
5
 
5
6
  import geopandas as gpd
6
7
 
@@ -13,31 +14,31 @@ from .zones import find_matching_column
13
14
  class EMC2Reader(CeremaStandardizer):
14
15
  SURVEY_TYPE = "EMC2"
15
16
 
16
- def households_filenames(self) -> list[str | io.BytesIO | MissingFileError]:
17
+ def households_filenames(self) -> list[Path | io.BytesIO | MissingFileError]:
17
18
  return [find_file(self.source, ".*_std_men.csv")]
18
19
 
19
- def persons_filenames(self) -> list[str | io.BytesIO | MissingFileError]:
20
+ def persons_filenames(self) -> list[Path | io.BytesIO | MissingFileError]:
20
21
  return [find_file(self.source, ".*_std_pers.csv")]
21
22
 
22
- def trips_filenames(self) -> list[str | io.BytesIO | MissingFileError]:
23
+ def trips_filenames(self) -> list[Path | io.BytesIO | MissingFileError]:
23
24
  return [find_file(self.source, ".*_std_depl.csv")]
24
25
 
25
- def legs_filenames(self) -> list[str | io.BytesIO | MissingFileError]:
26
+ def legs_filenames(self) -> list[Path | io.BytesIO | MissingFileError]:
26
27
  return [find_file(self.source, ".*_std_traj.csv")]
27
28
 
28
29
  def detailed_zones_filenames(self):
29
- return [find_file_path(self.source, r".*_ZF(_.*)?\.(TAB|shp)", subdir="SIG")]
30
+ return [find_file_path(self.source, r".*_ZF(_.*)?\.(TAB|shp)")]
30
31
 
31
32
  def special_locations_filenames(self):
32
- base = find_file_path(self.source, r".*_GT(?!_ext)(_.*)?\.(TAB|shp)", subdir="SIG")
33
- ext = find_file_path(self.source, r".*_GT_externes(_.*)?\.(TAB|shp)", subdir="SIG")
33
+ base = find_file_path(self.source, r".*_GT(?!_ext)(_.*)?\.(TAB|shp)")
34
+ ext = find_file_path(self.source, r".*_GT_externes(_.*)?\.(TAB|shp)")
34
35
  if base and ext:
35
36
  return [base, ext]
36
37
  else:
37
38
  return [base]
38
39
 
39
40
  def draw_zones_filenames(self):
40
- return [find_file_path(self.source, r".*_DTIR(_.*)?\.(TAB|shp)", subdir="SIG")]
41
+ return [find_file_path(self.source, r".*_DTIR(_.*)?\.(TAB|shp)")]
41
42
 
42
43
  def survey_name(self):
43
44
  filename = find_file_path(self.source, ".*_std_men.csv")
@@ -1,6 +1,7 @@
1
1
  import io
2
2
  import os
3
3
  import re
4
+ from pathlib import Path
4
5
 
5
6
  import geopandas as gpd
6
7
 
@@ -13,58 +14,36 @@ from .zones import find_matching_column
13
14
  class EMDReader(CeremaStandardizer):
14
15
  SURVEY_TYPE = "EMD"
15
16
 
16
- def households_filenames(self) -> list[str | io.BytesIO | MissingFileError]:
17
+ def households_filenames(self) -> list[Path | io.BytesIO | MissingFileError]:
17
18
  # The filenames are usually stored in "Csv/Fichiers_Standard/[SURVEY_NAME]_std_[NAME].csv"
18
19
  # but in some cases (e.g., Strasbourg 2009, Rouen 2017) the directory "Fichiers_Standard" is
19
20
  # named based on the survey name (e.g., Rouen_2017_Standard).
20
- return [find_file(self.source, ".*_std_men.csv", subdir="Csv")]
21
+ return [find_file(self.source, ".*_std_men.csv")]
21
22
 
22
- def persons_filenames(self) -> list[str | io.BytesIO | MissingFileError]:
23
- return [find_file(self.source, ".*_std_pers.csv", subdir="Csv")]
23
+ def persons_filenames(self) -> list[Path | io.BytesIO | MissingFileError]:
24
+ return [find_file(self.source, ".*_std_pers.csv")]
24
25
 
25
- def trips_filenames(self) -> list[str | io.BytesIO | MissingFileError]:
26
- return [find_file(self.source, ".*_std_depl.csv", subdir="Csv")]
26
+ def trips_filenames(self) -> list[Path | io.BytesIO | MissingFileError]:
27
+ return [find_file(self.source, ".*_std_depl.csv")]
27
28
 
28
- def legs_filenames(self) -> list[str | io.BytesIO | MissingFileError]:
29
- return [find_file(self.source, ".*_std_traj.csv", subdir="Csv")]
29
+ def legs_filenames(self) -> list[Path | io.BytesIO | MissingFileError]:
30
+ return [find_file(self.source, ".*_std_traj.csv")]
30
31
 
31
32
  def special_locations_and_detailed_zones_filenames(self):
32
33
  # This should match the Valenciennes 2011 and Grenoble 2010 surveys.
33
- return [
34
- find_file_path(
35
- self.source,
36
- ".*(grenobloise10_zones_fines|zf_gt)[.]mif",
37
- subdir=os.path.join("Doc", "SIG"),
38
- )
39
- ]
34
+ return [find_file_path(self.source, ".*(grenobloise10_zones_fines|zf_gt)[.]mif")]
40
35
 
41
36
  def detailed_zones_filenames(self):
42
- return [
43
- find_file_path(
44
- self.source,
45
- ".*(_zf|zones?[_ ]?fines?.*)[.](tab|mif)",
46
- subdir=os.path.join("Doc", "SIG"),
47
- )
48
- ]
37
+ return [find_file_path(self.source, ".*(_zf|zones?[_ ]?fines?.*)[.](tab|mif)")]
49
38
 
50
39
  def special_locations_filenames(self):
51
- return [
52
- find_file_path(
53
- self.source, ".*(_gt|g.?n.?rateur.*)[.](tab|mif)", subdir=os.path.join("Doc", "SIG")
54
- )
55
- ]
40
+ return [find_file_path(self.source, ".*(_gt|g.?n.?rateur.*)[.](tab|mif)")]
56
41
 
57
42
  def draw_zones_filenames(self):
58
- return [
59
- find_file_path(
60
- self.source,
61
- ".*(_DTIR|secteur_.*)[.](tab|shp|mif)",
62
- subdir=os.path.join("Doc", "SIG"),
63
- )
64
- ]
43
+ return [find_file_path(self.source, ".*(_DTIR|secteur_.*)[.](tab|shp|mif)")]
65
44
 
66
45
  def survey_name(self):
67
- filename = find_file_path(self.source, ".*_std_men.csv", subdir="Csv")
46
+ filename = find_file_path(self.source, ".*_std_men.csv")
68
47
  fn_match = re.match("(.*)_std_men.csv", os.path.basename(filename))
69
48
  if fn_match is not None:
70
49
  return fn_match.group(1)
@@ -1,6 +1,8 @@
1
1
  import io
2
+ from pathlib import Path
2
3
 
3
4
  import polars as pl
5
+ from polars.datatypes import DataTypeClass
4
6
 
5
7
  from mobisurvstd.common.cars import clean as clean_cars
6
8
  from mobisurvstd.common.households import clean as clean_households
@@ -10,7 +12,7 @@ from mobisurvstd.utils import detect_csv_delimiter
10
12
 
11
13
  from .reader import CeremaReader
12
14
 
13
- SCHEMA = {
15
+ SCHEMA: dict[str, DataTypeClass] = {
14
16
  "MP1": pl.UInt8, # Code fichier = 1
15
17
  "METH": pl.UInt8, # Méthode d'enquête du ménage (for EMC2 only)
16
18
  "IDM1": pl.UInt8, # Type Enquête
@@ -206,7 +208,7 @@ MOTORCYCLE_KW_UPPERBOUND_MAP = {6: 4, 7: 11}
206
208
  MOTORCYCLE_THERMIC_ENGINE_TYPE_MAP = {1: "two_stroke", 2: "four_stroke"}
207
209
 
208
210
 
209
- def scan_households_impl(source: str | io.BytesIO):
211
+ def scan_households_impl(source: Path | io.BytesIO):
210
212
  separator = detect_csv_delimiter(source)
211
213
  return pl.scan_csv(source, separator=separator, schema_overrides=SCHEMA, null_values=["a"])
212
214
 
@@ -225,7 +227,7 @@ class HouseholdsReader(CeremaReader):
225
227
  return lf
226
228
 
227
229
  def main_insee(self):
228
- return self.scan_households().select("IDM4").first().collect().item() # ty: ignore[possibly-missing-attribute]
230
+ return self.scan_households().select("IDM4").first().collect().item() # ty: ignore[unresolved-attribute]
229
231
 
230
232
  def standardize_households(self, skip_insee: bool = False):
231
233
  lf = self.scan_households()
@@ -1,13 +1,15 @@
1
1
  import io
2
+ from pathlib import Path
2
3
 
3
4
  import polars as pl
5
+ from polars.datatypes import DataTypeClass
4
6
 
5
7
  from mobisurvstd.common.persons import clean
6
8
  from mobisurvstd.utils import detect_csv_delimiter
7
9
 
8
10
  from .reader import CeremaReader
9
11
 
10
- SCHEMA = {
12
+ SCHEMA: dict[str, DataTypeClass] = {
11
13
  "PP1": pl.UInt8, # Code fichier = 2 (personne)
12
14
  "PMET": pl.UInt8, # Méthode d'enquête du ménage (EMC2 only)
13
15
  "IDP3": pl.UInt16, # Année de fin d'enquête
@@ -232,7 +234,7 @@ WEEKDAY_MAP = {
232
234
  }
233
235
 
234
236
 
235
- def scan_persons_impl(source: str | io.BytesIO):
237
+ def scan_persons_impl(source: Path | io.BytesIO):
236
238
  separator = detect_csv_delimiter(source)
237
239
  return pl.scan_csv(
238
240
  source, separator=separator, schema_overrides=SCHEMA, null_values=["a", "aa"]
@@ -1,4 +1,5 @@
1
1
  import re
2
+ from pathlib import Path
2
3
  from zipfile import ZipFile
3
4
 
4
5
  import geopandas as gpd
@@ -10,7 +11,7 @@ from .common import EMC2_MODE_MAP, MODE_MAP, NANTES_MODE_MAP
10
11
 
11
12
  class CeremaReader:
12
13
  SURVEY_TYPE = ""
13
- source: str | ZipFile
14
+ source: Path | ZipFile
14
15
  households: pl.LazyFrame
15
16
  persons: pl.LazyFrame
16
17
  trips: pl.LazyFrame
@@ -23,7 +24,7 @@ class CeremaReader:
23
24
  special_locations_coords: pl.DataFrame | None
24
25
  detailed_zones_coords: pl.DataFrame | None
25
26
 
26
- def __init__(self, source: str | ZipFile):
27
+ def __init__(self, source: Path | ZipFile):
27
28
  self.source = source
28
29
  self.special_locations = None
29
30
  self.detailed_zones = None
@@ -35,7 +36,7 @@ class CeremaReader:
35
36
  if isinstance(self.source, ZipFile):
36
37
  return self.source.filename or "unknown zipfile"
37
38
  else:
38
- return self.source
39
+ return str(self.source)
39
40
 
40
41
  def households_filenames(self):
41
42
  raise NotImplementedError
@@ -85,12 +85,12 @@ class CeremaStandardizer(HouseholdsReader, PersonsReader, TripsReader, LegsReade
85
85
 
86
86
  def finish(self):
87
87
  # Collecting all LazyFrames at this point can speed up next computations.
88
- self.households = self.households.collect().lazy() # ty: ignore[possibly-missing-attribute]
89
- self.persons = self.persons.collect().lazy() # ty: ignore[possibly-missing-attribute]
90
- self.trips = self.trips.collect().lazy() # ty: ignore[possibly-missing-attribute]
91
- self.legs = self.legs.collect().lazy() # ty: ignore[possibly-missing-attribute]
92
- self.cars = self.cars.collect().lazy() # ty: ignore[possibly-missing-attribute]
93
- self.motorcycles = self.motorcycles.collect().lazy() # ty: ignore[possibly-missing-attribute]
88
+ self.households = self.households.collect().lazy() # ty: ignore[unresolved-attribute]
89
+ self.persons = self.persons.collect().lazy() # ty: ignore[unresolved-attribute]
90
+ self.trips = self.trips.collect().lazy() # ty: ignore[unresolved-attribute]
91
+ self.legs = self.legs.collect().lazy() # ty: ignore[unresolved-attribute]
92
+ self.cars = self.cars.collect().lazy() # ty: ignore[unresolved-attribute]
93
+ self.motorcycles = self.motorcycles.collect().lazy() # ty: ignore[unresolved-attribute]
94
94
  self.add_survey_dates()
95
95
  self.fix_main_mode()
96
96
  self.fix_detailed_zones()
@@ -126,7 +126,7 @@ class CeremaStandardizer(HouseholdsReader, PersonsReader, TripsReader, LegsReade
126
126
  .filter(pl.col("mode_group").eq(pl.col("main_mode_group")).any().over("trip_id").not_())
127
127
  .select("trip_id")
128
128
  .collect()
129
- .to_series() # ty: ignore[possibly-missing-attribute]
129
+ .to_series() # ty: ignore[unresolved-attribute]
130
130
  .unique()
131
131
  )
132
132
  n = len(invalid_trips)
@@ -178,21 +178,21 @@ class CeremaStandardizer(HouseholdsReader, PersonsReader, TripsReader, LegsReade
178
178
  self.detailed_zones["detailed_zone_id"].str.len().max(),
179
179
  self.households.select(pl.col("home_detailed_zone").str.len_chars().max())
180
180
  .collect()
181
- .item(), # ty: ignore[possibly-missing-attribute]
181
+ .item(), # ty: ignore[unresolved-attribute]
182
182
  self.persons.select(pl.col("work_detailed_zone").str.len_chars().max())
183
183
  .collect()
184
- .item(), # ty: ignore[possibly-missing-attribute]
184
+ .item(), # ty: ignore[unresolved-attribute]
185
185
  self.persons.select(pl.col("study_detailed_zone").str.len_chars().max())
186
186
  .collect()
187
- .item(), # ty: ignore[possibly-missing-attribute]
187
+ .item(), # ty: ignore[unresolved-attribute]
188
188
  self.trips.select(pl.col("origin_detailed_zone").str.len_chars().max())
189
189
  .collect()
190
- .item(), # ty: ignore[possibly-missing-attribute]
190
+ .item(), # ty: ignore[unresolved-attribute]
191
191
  self.trips.select(pl.col("destination_detailed_zone").str.len_chars().max())
192
192
  .collect()
193
- .item(), # ty: ignore[possibly-missing-attribute]
194
- self.legs.select(pl.col("start_detailed_zone").str.len_chars().max()).collect().item(), # ty: ignore[possibly-missing-attribute]
195
- self.legs.select(pl.col("end_detailed_zone").str.len_chars().max()).collect().item(), # ty: ignore[possibly-missing-attribute]
193
+ .item(), # ty: ignore[unresolved-attribute]
194
+ self.legs.select(pl.col("start_detailed_zone").str.len_chars().max()).collect().item(), # ty: ignore[unresolved-attribute]
195
+ self.legs.select(pl.col("end_detailed_zone").str.len_chars().max()).collect().item(), # ty: ignore[unresolved-attribute]
196
196
  )
197
197
  self.detailed_zones["detailed_zone_id"] = self.detailed_zones["detailed_zone_id"].str.pad(
198
198
  width=max_len, fillchar="0"
@@ -268,7 +268,7 @@ class CeremaStandardizer(HouseholdsReader, PersonsReader, TripsReader, LegsReade
268
268
  lambda df, prefix: remove_external_zones(df, prefix, zf_ids, gt_ids)
269
269
  )
270
270
 
271
- def apply_function_to_location_columns(self, func: Callable[[pl.DataFrame, str], pl.Expr]):
271
+ def apply_function_to_location_columns(self, func: Callable[[pl.DataFrame, str], pl.DataFrame]):
272
272
  """Applies the function `func` to all the location columns (households' `home_*`, persons'
273
273
  `work_*`, etc.
274
274
 
@@ -1,14 +1,16 @@
1
1
  import io
2
+ from pathlib import Path
2
3
 
3
4
  import polars as pl
4
5
  from loguru import logger
6
+ from polars.datatypes import DataTypeClass
5
7
 
6
8
  from mobisurvstd.common.legs import clean
7
9
  from mobisurvstd.utils import detect_csv_delimiter
8
10
 
9
11
  from .reader import CeremaReader
10
12
 
11
- SCHEMA = {
13
+ SCHEMA: dict[str, DataTypeClass] = {
12
14
  "TP1": pl.UInt8, # Code fichier = 4 (trajet)
13
15
  "TMET": pl.UInt8, # Méthode d'enquête du ménage (EMC2 only)
14
16
  "IDT3": pl.UInt16, # Année de fin d'enquête
@@ -93,7 +95,7 @@ MOTORCYCLE_MAP = {
93
95
  }
94
96
 
95
97
 
96
- def scan_legs_impl(source: str | io.BytesIO):
98
+ def scan_legs_impl(source: Path | io.BytesIO):
97
99
  separator = detect_csv_delimiter(source)
98
100
  return pl.scan_csv(
99
101
  source, separator=separator, schema_overrides=SCHEMA, null_values=["a", "aa", "aaaaa"]
@@ -3,6 +3,8 @@ from typing import Optional
3
3
  import typer
4
4
  from typing_extensions import Annotated
5
5
 
6
+ import mobisurvstd
7
+
6
8
  from .main import bulk_standardize, standardize
7
9
  from .resources import clear_cache
8
10
 
@@ -13,6 +15,12 @@ def clear_cache_callback(value: bool):
13
15
  raise typer.Exit()
14
16
 
15
17
 
18
+ def version_callback(value: bool):
19
+ if value:
20
+ print(mobisurvstd.__version__)
21
+ raise typer.Exit()
22
+
23
+
16
24
  def standardizer(
17
25
  source: Annotated[
18
26
  str,
@@ -48,6 +56,15 @@ def standardizer(
48
56
  "--clear-cache", callback=clear_cache_callback, help="Clear the cache data and exit"
49
57
  ),
50
58
  ] = None,
59
+ version: Annotated[
60
+ bool | None,
61
+ typer.Option(
62
+ "--version",
63
+ help="Show MobiSurvStd version and exit",
64
+ callback=version_callback,
65
+ is_eager=True,
66
+ ),
67
+ ] = None,
51
68
  ):
52
69
  """Mobility Survey Standardizer: a Python command line tool to convert mobility surveys to a
53
70
  clean standardized format.
@@ -10,7 +10,7 @@ def clean(lf: pl.LazyFrame, extra_cols: list[str] | None = None):
10
10
  columns = [variable.name for variable in CAR_SCHEMA if variable.name in existing_cols]
11
11
  if extra_cols is not None:
12
12
  columns.extend(extra_cols)
13
- lf = lf.select(columns).sort("original_car_id").collect().lazy() # ty: ignore[possibly-missing-attribute]
13
+ lf = lf.select(columns).sort("original_car_id").collect().lazy() # ty: ignore[unresolved-attribute]
14
14
  lf = indexing(lf, existing_cols)
15
15
  lf = add_fuel_type_group(lf, existing_cols)
16
16
  lf = add_mileage_bounds(lf, existing_cols)