mobisurvstd 1.0.1__tar.gz → 1.0.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. {mobisurvstd-1.0.1/src/mobisurvstd.egg-info → mobisurvstd-1.0.2}/PKG-INFO +2 -1
  2. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/pyproject.toml +6 -3
  3. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/cerema/deplacements.py +3 -1
  4. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/cerema/edgt.py +26 -28
  5. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/cerema/edvm.py +18 -21
  6. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/cerema/emc2.py +16 -24
  7. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/cerema/emd.py +14 -16
  8. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/cerema/menages.py +11 -5
  9. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/cerema/nantes_opendata.py +4 -3
  10. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/cerema/personnes.py +3 -4
  11. mobisurvstd-1.0.2/src/mobisurvstd/cerema/reader.py +159 -0
  12. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/cerema/survey.py +113 -223
  13. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/cerema/trajets.py +3 -1
  14. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/cerema/zones.py +8 -4
  15. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/classes.py +14 -15
  16. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/common/cars.py +1 -1
  17. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/common/clean.py +11 -10
  18. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/common/households.py +3 -3
  19. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/common/legs.py +1 -1
  20. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/common/motorcycles.py +1 -1
  21. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/common/persons.py +1 -1
  22. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/common/trips.py +2 -1
  23. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/common/zones.py +1 -1
  24. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/egt2010/menages.py +2 -2
  25. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/egt2010/survey.py +4 -4
  26. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/emp/survey.py +1 -1
  27. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/resources/admin_express.py +22 -18
  28. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/resources/insee_data.py +6 -5
  29. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/schema/common.py +4 -2
  30. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/schema/guarantees.py +8 -5
  31. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/schema/legs.py +2 -2
  32. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/schema/trips.py +2 -2
  33. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/utils.py +68 -29
  34. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2/src/mobisurvstd.egg-info}/PKG-INFO +2 -1
  35. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd.egg-info/SOURCES.txt +1 -0
  36. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd.egg-info/requires.txt +1 -0
  37. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/LICENSE.md +0 -0
  38. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/README.md +0 -0
  39. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/setup.cfg +0 -0
  40. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/__init__.py +0 -0
  41. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/__main__.py +0 -0
  42. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/cerema/__init__.py +0 -0
  43. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/cerema/common.py +0 -0
  44. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/cli.py +0 -0
  45. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/common/__init__.py +0 -0
  46. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/egt2010/__init__.py +0 -0
  47. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/egt2010/deplacements.py +0 -0
  48. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/egt2010/personnes.py +0 -0
  49. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/egt2010/trajets.py +0 -0
  50. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/egt2010/zones.py +0 -0
  51. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/egt2020/__init__.py +0 -0
  52. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/egt2020/deplacements.py +0 -0
  53. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/egt2020/menages.py +0 -0
  54. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/egt2020/motos.py +0 -0
  55. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/egt2020/personnes.py +0 -0
  56. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/egt2020/survey.py +0 -0
  57. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/egt2020/trajets.py +0 -0
  58. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/egt2020/voitures.py +0 -0
  59. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/emp/__init__.py +0 -0
  60. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/emp/common.py +0 -0
  61. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/emp/deplacements.py +0 -0
  62. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/emp/menages.py +0 -0
  63. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/emp/motos.py +0 -0
  64. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/emp/personnes.py +0 -0
  65. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/emp/voitures.py +0 -0
  66. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/logger.py +0 -0
  67. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/main.py +0 -0
  68. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/resources/__init__.py +0 -0
  69. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/resources/nuts.py +0 -0
  70. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/schema/__init__.py +0 -0
  71. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/schema/cars.py +0 -0
  72. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/schema/households.py +0 -0
  73. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/schema/motorcycles.py +0 -0
  74. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/schema/persons.py +0 -0
  75. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/validation.py +0 -0
  76. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd.egg-info/dependency_links.txt +0 -0
  77. {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: mobisurvstd
3
- Version: 1.0.1
3
+ Version: 1.0.2
4
4
  Summary: A tool to standardize French mobility survey datasets (EMC2, EGT, EMP, etc.) into a unified format.
5
5
  Author-email: Lucas Javaudin <lucas@lucasjavaudin.com>
6
6
  License-Expression: MIT
@@ -29,6 +29,7 @@ Requires-Dist: polars>=1.33.0
29
29
  Requires-Dist: py7zr>=0.20.8
30
30
  Requires-Dist: pyarrow>=19.0.1
31
31
  Requires-Dist: requests>=2.32.3
32
+ Requires-Dist: seaborn>=0.13.2
32
33
  Requires-Dist: typer>=0.16.0
33
34
  Dynamic: license-file
34
35
 
@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
6
6
 
7
7
  [project]
8
8
  name = "mobisurvstd"
9
- version = "1.0.1"
9
+ version = "1.0.2"
10
10
  description = "A tool to standardize French mobility survey datasets (EMC2, EGT, EMP, etc.) into a unified format."
11
11
  readme = "README.md"
12
12
  authors = [{ name = "Lucas Javaudin", email = "lucas@lucasjavaudin.com" }]
@@ -22,17 +22,18 @@ dependencies = [
22
22
  "Cartopy >= 0.24.1",
23
23
  "fastexcel >= 0.14.0",
24
24
  "fsspec>=2025.9.0",
25
- "geopandas >= 1.0.1",
25
+ "geopandas>=1.0.1",
26
26
  "humanize >= 4.13.0",
27
27
  "loguru >= 0.7.3",
28
28
  "matplotlib >= 3.10.1",
29
29
  "numpy >= 2.2.4",
30
30
  "pandas >= 2.2.3",
31
31
  "platformdirs >= 4.3.8",
32
- "polars >= 1.33.0",
32
+ "polars>=1.33.0",
33
33
  "py7zr >= 0.20.8",
34
34
  "pyarrow >= 19.0.1",
35
35
  "requests >= 2.32.3",
36
+ "seaborn>=0.13.2",
36
37
  "typer >= 0.16.0",
37
38
  ]
38
39
  requires-python = ">=3.10"
@@ -40,6 +41,8 @@ requires-python = ">=3.10"
40
41
  [dependency-groups]
41
42
  dev = [
42
43
  "ipython>=8.37.0",
44
+ "pytest>=9.0.2",
45
+ "ty>=0.0.15",
43
46
  ]
44
47
 
45
48
  [project.urls]
@@ -6,6 +6,8 @@ from loguru import logger
6
6
  from mobisurvstd.common.trips import clean
7
7
  from mobisurvstd.schema.common import CURRENT_YEAR
8
8
 
9
+ from .reader import CeremaReader
10
+
9
11
  SCHEMA = {
10
12
  "DP1": pl.UInt8, # Code fichier = 3 (déplacement)
11
13
  "DMET": pl.UInt8, # Méthode d'enquête du ménage (EMC2 only)
@@ -114,7 +116,7 @@ def scan_trips_impl(source: str | io.BytesIO):
114
116
  return pl.scan_csv(source, separator=";", schema_overrides=SCHEMA, null_values=["aa", "aaaaa"])
115
117
 
116
118
 
117
- class TripsReader:
119
+ class TripsReader(CeremaReader):
118
120
  def scan_trips(self):
119
121
  lfs_iter = map(scan_trips_impl, self.trips_filenames())
120
122
  lf = pl.concat(lfs_iter, how="vertical")
@@ -5,13 +5,13 @@ from zipfile import ZipFile
5
5
 
6
6
  import geopandas as gpd
7
7
 
8
- from mobisurvstd.utils import MissingFileError, find_file
8
+ from mobisurvstd.utils import MissingFileError, find_file, find_file_path
9
9
 
10
- from .survey import CeremaReader
10
+ from .survey import CeremaStandardizer
11
11
  from .zones import find_matching_column
12
12
 
13
13
 
14
- class EDGTReader(CeremaReader):
14
+ class EDGTReader(CeremaStandardizer):
15
15
  SURVEY_TYPE = "EDGT"
16
16
 
17
17
  def households_filenames(self):
@@ -29,53 +29,49 @@ class EDGTReader(CeremaReader):
29
29
  def special_locations_and_detailed_zones_filenames(self):
30
30
  # This should match the Amiens 2010, Clermont-Ferrand 2012, and Lyon 2015 surveys.
31
31
  subdir = os.path.join("Doc", "SIG")
32
- faf_filename = find_file(
33
- self.source, ".*_faf_d.?coupagefin[.](tab|shp|mif)", subdir=subdir, as_url=True
32
+ faf_filename = find_file_path(
33
+ self.source, ".*_faf_d.?coupagefin[.](tab|shp|mif)", subdir=subdir
34
34
  )
35
- tel_filename = find_file(
36
- self.source, ".*_tel_d.?coupagefin[.](tab|shp|mif)", subdir=subdir, as_url=True
35
+ tel_filename = find_file_path(
36
+ self.source, ".*_tel_d.?coupagefin[.](tab|shp|mif)", subdir=subdir
37
37
  )
38
38
  if faf_filename and tel_filename:
39
39
  # Amiens 2010 has two files.
40
40
  return [faf_filename, tel_filename]
41
41
  else:
42
42
  return [
43
- find_file(
43
+ find_file_path(
44
44
  self.source,
45
45
  ".*(_zf_gt|zinterne_gen_.*)[.](tab|shp|mif)",
46
46
  subdir=subdir,
47
- as_url=True,
48
47
  )
49
48
  ]
50
49
 
51
50
  def special_locations_filenames(self):
52
51
  subdir = os.path.join("Doc", "SIG")
53
- faf_filename = find_file(
54
- self.source, ".*_faf_.*g.?n.?rateur.*[.](tab|shp|mif)", subdir=subdir, as_url=True
52
+ faf_filename = find_file_path(
53
+ self.source, ".*_faf_.*g.?n.?rateur.*[.](tab|shp|mif)", subdir=subdir
55
54
  )
56
- tel_filename = find_file(
57
- self.source, ".*_tel_.*g.?n.?rateur.*[.](tab|shp|mif)", subdir=subdir, as_url=True
55
+ tel_filename = find_file_path(
56
+ self.source, ".*_tel_.*g.?n.?rateur.*[.](tab|shp|mif)", subdir=subdir
58
57
  )
59
58
  if faf_filename and tel_filename:
60
59
  # Saint-Quentin-en-Yvelines has two files, for phone and face-to-face surveys.
61
60
  return [faf_filename, tel_filename]
62
61
  else:
63
62
  return [
64
- find_file(
65
- self.source,
66
- ".*(_gt_.*|_gt|g.?n.?rateur.*)[.](tab|shp|mif)",
67
- subdir=subdir,
68
- as_url=True,
63
+ find_file_path(
64
+ self.source, ".*(_gt_.*|_gt|g.?n.?rateur.*)[.](tab|shp|mif)", subdir=subdir
69
65
  )
70
66
  ]
71
67
 
72
68
  def detailed_zones_filenames(self):
73
69
  subdir = os.path.join("Doc", "SIG")
74
- faf_filename = find_file(
75
- self.source, ".*_faf_zones_fines[.](tab|shp|mif)", subdir=subdir, as_url=True
70
+ faf_filename = find_file_path(
71
+ self.source, ".*_faf_zones_fines[.](tab|shp|mif)", subdir=subdir
76
72
  )
77
- tel_filename = find_file(
78
- self.source, ".*_tel_zones_fines[.](tab|shp|mif)", subdir=subdir, as_url=True
73
+ tel_filename = find_file_path(
74
+ self.source, ".*_tel_zones_fines[.](tab|shp|mif)", subdir=subdir
79
75
  )
80
76
  if faf_filename and tel_filename:
81
77
  # Nice 2009 and Saint-Quentin-en-Yvelines have two files, for phone and face-to-face
@@ -83,27 +79,29 @@ class EDGTReader(CeremaReader):
83
79
  return [faf_filename, tel_filename]
84
80
  else:
85
81
  return [
86
- find_file(
82
+ find_file_path(
87
83
  self.source,
88
84
  ".*(_zf|_zf_.*|zf08_.*|(?<!tel_)zones[_ ]fines)[.](tab|shp|mif)",
89
85
  subdir=subdir,
90
- as_url=True,
91
86
  )
92
87
  ]
93
88
 
94
89
  def draw_zones_filenames(self):
95
90
  return [
96
- find_file(
91
+ find_file_path(
97
92
  self.source,
98
93
  ".*(_dtir|_secteurstirage)[.](tab|shp|mif)",
99
94
  subdir=os.path.join("Doc", "SIG"),
100
- as_url=True,
101
95
  )
102
96
  ]
103
97
 
104
98
  def survey_name(self):
105
- filename = find_file(self.source, ".*_std_faf_men.csv", subdir="Csv", as_url=True)
106
- return re.match("(.*)_std_faf_men.csv", os.path.basename(filename)).group(1)
99
+ filename = find_file_path(self.source, ".*_std_faf_men.csv", subdir="Csv")
100
+ fn_match = re.match("(.*)_std_faf_men.csv", os.path.basename(filename))
101
+ if fn_match is not None:
102
+ return fn_match.group(1)
103
+ else:
104
+ return "unkown"
107
105
 
108
106
  def gt_id_columns(self):
109
107
  return [
@@ -4,18 +4,18 @@ import re
4
4
 
5
5
  import geopandas as gpd
6
6
 
7
- from mobisurvstd.cerema.survey import CeremaReader
8
- from mobisurvstd.utils import MissingFileError, find_file
7
+ from mobisurvstd.cerema.survey import CeremaStandardizer
8
+ from mobisurvstd.utils import MissingFileError, find_file, find_file_path
9
9
 
10
10
  from .zones import find_matching_column
11
11
 
12
12
 
13
- class EDVMReader(CeremaReader):
13
+ class EDVMReader(CeremaStandardizer):
14
14
  SURVEY_TYPE = "EDVM"
15
15
 
16
16
  def households_filenames(self) -> list[str | io.BytesIO | MissingFileError]:
17
17
  # The filenames are usually stored in "Csv/Fichiers_Standard/[SURVEY_NAME]_std_[NAME].csv"
18
- # but in some cases (e.g., Ajaccio 2017) the directory "Fichiers_Standard" is # named based
18
+ # but in some cases (e.g., Ajaccio 2017) the directory "Fichiers_Standard" is named based
19
19
  # on the survey name (e.g., Ajaccio_2017_Standard).
20
20
  return [find_file(self.source, ".*_std_men.csv", subdir="Csv")]
21
21
 
@@ -31,47 +31,43 @@ class EDVMReader(CeremaReader):
31
31
  def special_locations_and_detailed_zones_filenames(self):
32
32
  # This should only match the Beauvais 2011 survey.
33
33
  return [
34
- find_file(
35
- self.source,
36
- ".*beauvais.*_dfin[.](tab|shp|mif)",
37
- subdir=os.path.join("Doc", "SIG"),
38
- as_url=True,
34
+ find_file_path(
35
+ self.source, ".*beauvais.*_dfin[.](tab|shp|mif)", subdir=os.path.join("Doc", "SIG")
39
36
  )
40
37
  ]
41
38
 
42
39
  def detailed_zones_filenames(self):
43
40
  return [
44
- find_file(
41
+ find_file_path(
45
42
  self.source,
46
43
  ".*(_zf_.*|_zf|zones?[_ ]?fines?.*|_dfin)[.](tab|shp|mif)",
47
44
  subdir=os.path.join("Doc", "SIG"),
48
- as_url=True,
49
45
  )
50
46
  ]
51
47
 
52
48
  def special_locations_filenames(self):
53
49
  return [
54
- find_file(
50
+ find_file_path(
55
51
  self.source,
56
52
  ".*(_gt_.*|_gt|_pgt|_pg|g.?n.?rateur.*)[.](tab|shp|mif)",
57
53
  subdir=os.path.join("Doc", "SIG"),
58
- as_url=True,
59
54
  )
60
55
  ]
61
56
 
62
57
  def draw_zones_filenames(self):
63
58
  return [
64
- find_file(
65
- self.source,
66
- ".*_DTIR[.](tab|shp|mif)",
67
- subdir=os.path.join("Doc", "SIG"),
68
- as_url=True,
59
+ find_file_path(
60
+ self.source, ".*_DTIR[.](tab|shp|mif)", subdir=os.path.join("Doc", "SIG")
69
61
  )
70
62
  ]
71
63
 
72
64
  def survey_name(self):
73
- filename = find_file(self.source, ".*_std_men.csv", subdir="Csv", as_url=True)
74
- return re.match("(.*)_std_men.csv", os.path.basename(filename)).group(1)
65
+ filename = find_file_path(self.source, ".*_std_men.csv", subdir="Csv")
66
+ fn_match = re.match("(.*)_std_men.csv", os.path.basename(filename))
67
+ if fn_match is not None:
68
+ return fn_match.group(1)
69
+ else:
70
+ return "unkown"
75
71
 
76
72
  def gt_id_columns(self):
77
73
  return [
@@ -277,9 +273,10 @@ class EDVMReader(CeremaReader):
277
273
  super().select_zf_from_gt_column(gdf)
278
274
 
279
275
  def postprocess_special_locations_and_detailed_zones(
280
- self, zfs: gpd.GeoDataFrame, gts: gpd.GeoDataFrame
276
+ self, zfs: gpd.GeoDataFrame | None, gts: gpd.GeoDataFrame | None
281
277
  ):
282
278
  # This function should only be call for Beauvais 2011.
279
+ assert gts is not None
283
280
  assert "detailed_zone_id" not in gts.columns
284
281
  # The corresponding ZF is the GT id with the last digit replaced by 0.
285
282
  gts["detailed_zone_id"] = gts["special_location_id"].str.replace(r"\d$", "0", regex=True)
@@ -4,13 +4,13 @@ import re
4
4
 
5
5
  import geopandas as gpd
6
6
 
7
- from mobisurvstd.cerema.survey import CeremaReader
8
- from mobisurvstd.utils import MissingFileError, find_file
7
+ from mobisurvstd.cerema.survey import CeremaStandardizer
8
+ from mobisurvstd.utils import MissingFileError, find_file, find_file_path
9
9
 
10
10
  from .zones import find_matching_column
11
11
 
12
12
 
13
- class EMC2Reader(CeremaReader):
13
+ class EMC2Reader(CeremaStandardizer):
14
14
  SURVEY_TYPE = "EMC2"
15
15
 
16
16
  def households_filenames(self) -> list[str | io.BytesIO | MissingFileError]:
@@ -43,42 +43,34 @@ class EMC2Reader(CeremaReader):
43
43
 
44
44
  def detailed_zones_filenames(self):
45
45
  return [
46
- find_file(
47
- self.source,
48
- r".*_ZF(_.*)?\.(TAB|shp)",
49
- subdir=os.path.join("Doc", "SIG"),
50
- as_url=True,
46
+ find_file_path(
47
+ self.source, r".*_ZF(_.*)?\.(TAB|shp)", subdir=os.path.join("Doc", "SIG")
51
48
  )
52
49
  ]
53
50
 
54
51
  def special_locations_filenames(self):
55
52
  return [
56
- find_file(
57
- self.source,
58
- r".*_GT(_.*)?\.(TAB|shp)",
59
- subdir=os.path.join("Doc", "SIG"),
60
- as_url=True,
53
+ find_file_path(
54
+ self.source, r".*_GT(_.*)?\.(TAB|shp)", subdir=os.path.join("Doc", "SIG")
61
55
  )
62
56
  ]
63
57
 
64
58
  def draw_zones_filenames(self):
65
59
  return [
66
- find_file(
67
- self.source,
68
- r".*_DTIR(_.*)?\.(TAB|shp)",
69
- subdir=os.path.join("Doc", "SIG"),
70
- as_url=True,
60
+ find_file_path(
61
+ self.source, r".*_DTIR(_.*)?\.(TAB|shp)", subdir=os.path.join("Doc", "SIG")
71
62
  )
72
63
  ]
73
64
 
74
65
  def survey_name(self):
75
- filename = find_file(
76
- self.source,
77
- ".*_std_men.csv",
78
- subdir=os.path.join("Csv", "Fichiers_Standard"),
79
- as_url=True,
66
+ filename = find_file_path(
67
+ self.source, ".*_std_men.csv", subdir=os.path.join("Csv", "Fichiers_Standard")
80
68
  )
81
- return re.match("(.*)_std_men.csv", os.path.basename(filename)).group(1)
69
+ fn_match = re.match("(.*)_std_men.csv", os.path.basename(filename))
70
+ if fn_match is not None:
71
+ return fn_match.group(1)
72
+ else:
73
+ return "Unknown"
82
74
 
83
75
  def gt_id_columns(self):
84
76
  return ["zf_160", "zf_fusion", "codegt", "zf", "num_gt"]
@@ -4,13 +4,13 @@ import re
4
4
 
5
5
  import geopandas as gpd
6
6
 
7
- from mobisurvstd.cerema.survey import CeremaReader
8
- from mobisurvstd.utils import MissingFileError, find_file
7
+ from mobisurvstd.cerema.survey import CeremaStandardizer
8
+ from mobisurvstd.utils import MissingFileError, find_file, find_file_path
9
9
 
10
10
  from .zones import find_matching_column
11
11
 
12
12
 
13
- class EMDReader(CeremaReader):
13
+ class EMDReader(CeremaStandardizer):
14
14
  SURVEY_TYPE = "EMD"
15
15
 
16
16
  def households_filenames(self) -> list[str | io.BytesIO | MissingFileError]:
@@ -31,47 +31,45 @@ class EMDReader(CeremaReader):
31
31
  def special_locations_and_detailed_zones_filenames(self):
32
32
  # This should match the Valenciennes 2011 and Grenoble 2010 surveys.
33
33
  return [
34
- find_file(
34
+ find_file_path(
35
35
  self.source,
36
36
  ".*(grenobloise10_zones_fines|zf_gt)[.]mif",
37
37
  subdir=os.path.join("Doc", "SIG"),
38
- as_url=True,
39
38
  )
40
39
  ]
41
40
 
42
41
  def detailed_zones_filenames(self):
43
42
  return [
44
- find_file(
43
+ find_file_path(
45
44
  self.source,
46
45
  ".*(_zf|zones?[_ ]?fines?.*)[.](tab|mif)",
47
46
  subdir=os.path.join("Doc", "SIG"),
48
- as_url=True,
49
47
  )
50
48
  ]
51
49
 
52
50
  def special_locations_filenames(self):
53
51
  return [
54
- find_file(
55
- self.source,
56
- ".*(_gt|g.?n.?rateur.*)[.](tab|mif)",
57
- subdir=os.path.join("Doc", "SIG"),
58
- as_url=True,
52
+ find_file_path(
53
+ self.source, ".*(_gt|g.?n.?rateur.*)[.](tab|mif)", subdir=os.path.join("Doc", "SIG")
59
54
  )
60
55
  ]
61
56
 
62
57
  def draw_zones_filenames(self):
63
58
  return [
64
- find_file(
59
+ find_file_path(
65
60
  self.source,
66
61
  ".*(_DTIR|secteur_.*)[.](tab|shp|mif)",
67
62
  subdir=os.path.join("Doc", "SIG"),
68
- as_url=True,
69
63
  )
70
64
  ]
71
65
 
72
66
  def survey_name(self):
73
- filename = find_file(self.source, ".*_std_men.csv", subdir="Csv", as_url=True)
74
- return re.match("(.*)_std_men.csv", os.path.basename(filename)).group(1)
67
+ filename = find_file_path(self.source, ".*_std_men.csv", subdir="Csv")
68
+ fn_match = re.match("(.*)_std_men.csv", os.path.basename(filename))
69
+ if fn_match is not None:
70
+ return fn_match.group(1)
71
+ else:
72
+ return "Unknown"
75
73
 
76
74
  def gt_id_columns(self):
77
75
  return [
@@ -7,6 +7,8 @@ from mobisurvstd.common.households import clean as clean_households
7
7
  from mobisurvstd.common.motorcycles import clean as clean_motorcycles
8
8
  from mobisurvstd.schema.common import CURRENT_YEAR
9
9
 
10
+ from .reader import CeremaReader
11
+
10
12
  SCHEMA = {
11
13
  "MP1": pl.UInt8, # Code fichier = 1
12
14
  "METH": pl.UInt8, # Méthode d'enquête du ménage (for EMC2 only)
@@ -230,8 +232,8 @@ def scan_households_impl(source: str | io.BytesIO):
230
232
  return pl.scan_csv(source, separator=";", schema_overrides=SCHEMA, null_values=["a"])
231
233
 
232
234
 
233
- class HouseholdsReader:
234
- def scan_households(self):
235
+ class HouseholdsReader(CeremaReader):
236
+ def scan_households(self) -> pl.LazyFrame:
235
237
  lfs_iter = map(scan_households_impl, self.households_filenames())
236
238
  lf = pl.concat(lfs_iter, how="vertical")
237
239
  if "METH" not in lf.collect_schema().names():
@@ -244,7 +246,7 @@ class HouseholdsReader:
244
246
  return lf
245
247
 
246
248
  def main_insee(self):
247
- return self.scan_households().select("IDM4").first().collect().item()
249
+ return self.scan_households().select("IDM4").first().collect().item() # ty: ignore[possibly-missing-attribute]
248
250
 
249
251
  def standardize_households(self):
250
252
  lf = self.scan_households()
@@ -299,6 +301,8 @@ class HouseholdsReader:
299
301
  how="left",
300
302
  coalesce=True,
301
303
  )
304
+ # Note: the `.lazy()` after the concat is useless (this is already a LazyFrame) but it helps
305
+ # the syntax checker.
302
306
  lf = pl.concat(
303
307
  (
304
308
  lf.select(
@@ -326,7 +330,7 @@ class HouseholdsReader:
326
330
  for i, a in enumerate(("A", "B", "C", "D"))
327
331
  ),
328
332
  how="vertical",
329
- )
333
+ ).lazy()
330
334
  # Drop the lines with empty car characteristics (there are always 4 cars per
331
335
  # households even when the household has less than 4 cars).
332
336
  lf = lf.filter(
@@ -347,6 +351,8 @@ class HouseholdsReader:
347
351
  how="left",
348
352
  coalesce=True,
349
353
  )
354
+ # Note: the `.lazy()` after the concat is useless (this is already a LazyFrame) but it helps
355
+ # the syntax checker.
350
356
  lf = pl.concat(
351
357
  (
352
358
  lf.select(
@@ -388,7 +394,7 @@ class HouseholdsReader:
388
394
  for i, a in enumerate(("A", "B", "C", "D"))
389
395
  ),
390
396
  how="vertical",
391
- )
397
+ ).lazy()
392
398
  lf = lf.filter(
393
399
  pl.any_horizontal(
394
400
  pl.all()
@@ -2,13 +2,14 @@ import io
2
2
 
3
3
  import pandas as pd
4
4
  import polars as pl
5
+ from polars.datatypes import DataTypeClass
5
6
 
6
7
  from mobisurvstd.utils import find_file
7
8
 
8
9
  from .deplacements import SCHEMA as TRIP_SCHEMA
9
10
  from .menages import SCHEMA as HOUSEHOLD_SCHEMA
10
11
  from .personnes import SCHEMA as PERSON_SCHEMA
11
- from .survey import CeremaReader
12
+ from .survey import CeremaStandardizer
12
13
  from .trajets import SCHEMA as LEG_SCHEMA
13
14
 
14
15
  # The anonimized open-data format for Nantes uses a fixed-width text format.
@@ -140,7 +141,7 @@ LEG_FORMAT = [
140
141
  ]
141
142
 
142
143
 
143
- class OpenDataReader(CeremaReader):
144
+ class OpenDataReader(CeremaStandardizer):
144
145
  SURVEY_TYPE = "EDGT-opendata"
145
146
 
146
147
  def households_filenames(self):
@@ -193,7 +194,7 @@ class OpenDataReader(CeremaReader):
193
194
  def scan_fwf(
194
195
  f: str | io.BytesIO,
195
196
  format: list[tuple[int, int, str]],
196
- schema: dict[str, pl.DataType],
197
+ schema: dict[str, DataTypeClass],
197
198
  ):
198
199
  # Decrease `start` by 1 because pandas starts counting at 0 but the survey document starts
199
200
  # counting at 1.
@@ -4,6 +4,8 @@ import polars as pl
4
4
 
5
5
  from mobisurvstd.common.persons import clean
6
6
 
7
+ from .reader import CeremaReader
8
+
7
9
  SCHEMA = {
8
10
  "PP1": pl.UInt8, # Code fichier = 2 (personne)
9
11
  "PMET": pl.UInt8, # Méthode d'enquête du ménage (EMC2 only)
@@ -64,7 +66,6 @@ REFERENCE_PERSON_LINK_MAP = {
64
66
  7: "other:non_relative", # Autre Non précisé
65
67
  }
66
68
 
67
- 200 / 2600 / 2490 / 2809
68
69
  DRIVING_LICENSE_MAP = {
69
70
  1: "yes",
70
71
  2: "no",
@@ -234,7 +235,7 @@ def scan_persons_impl(source: str | io.BytesIO):
234
235
  return pl.scan_csv(source, separator=";", schema_overrides=SCHEMA, null_values=["a", "aa"])
235
236
 
236
237
 
237
- class PersonsReader:
238
+ class PersonsReader(CeremaReader):
238
239
  def scan_persons(self):
239
240
  lfs_iter = map(scan_persons_impl, self.persons_filenames())
240
241
  lf = pl.concat(lfs_iter, how="vertical")
@@ -371,8 +372,6 @@ class PersonsReader:
371
372
 
372
373
  # For Arras 2014, the "MOIS" column is always 14 (for 2014) and the "DATE" column
373
374
  # corresponds to the month (January or February). The exact date is unknown. We set to 1.
374
-
375
-
376
375
  def fix_dates(lf: pl.LazyFrame, survey_name: str):
377
376
  # Arras 2014 case.
378
377
  lf = lf.with_columns(