mobisurvstd 1.0.0__tar.gz → 1.0.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. {mobisurvstd-1.0.0/src/mobisurvstd.egg-info → mobisurvstd-1.0.2}/PKG-INFO +11 -7
  2. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/README.md +9 -6
  3. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/pyproject.toml +11 -3
  4. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/cerema/deplacements.py +7 -3
  5. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/cerema/edgt.py +26 -28
  6. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/cerema/edvm.py +18 -21
  7. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/cerema/emc2.py +16 -24
  8. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/cerema/emd.py +14 -16
  9. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/cerema/menages.py +11 -5
  10. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/cerema/nantes_opendata.py +4 -3
  11. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/cerema/personnes.py +18 -15
  12. mobisurvstd-1.0.2/src/mobisurvstd/cerema/reader.py +159 -0
  13. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/cerema/survey.py +118 -229
  14. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/cerema/trajets.py +3 -1
  15. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/cerema/zones.py +8 -4
  16. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/classes.py +14 -15
  17. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/common/cars.py +1 -1
  18. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/common/clean.py +11 -10
  19. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/common/households.py +3 -3
  20. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/common/legs.py +1 -1
  21. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/common/motorcycles.py +1 -1
  22. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/common/persons.py +1 -1
  23. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/common/trips.py +3 -2
  24. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/common/zones.py +1 -1
  25. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/egt2010/menages.py +2 -2
  26. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/egt2010/survey.py +4 -4
  27. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/emp/survey.py +1 -1
  28. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/resources/admin_express.py +22 -18
  29. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/resources/insee_data.py +6 -5
  30. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/schema/common.py +4 -2
  31. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/schema/guarantees.py +9 -5
  32. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/schema/legs.py +2 -2
  33. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/schema/trips.py +2 -20
  34. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/utils.py +68 -29
  35. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2/src/mobisurvstd.egg-info}/PKG-INFO +11 -7
  36. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd.egg-info/SOURCES.txt +1 -0
  37. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd.egg-info/requires.txt +1 -0
  38. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/LICENSE.md +0 -0
  39. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/setup.cfg +0 -0
  40. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/__init__.py +0 -0
  41. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/__main__.py +0 -0
  42. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/cerema/__init__.py +0 -0
  43. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/cerema/common.py +0 -0
  44. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/cli.py +0 -0
  45. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/common/__init__.py +0 -0
  46. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/egt2010/__init__.py +0 -0
  47. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/egt2010/deplacements.py +0 -0
  48. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/egt2010/personnes.py +0 -0
  49. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/egt2010/trajets.py +0 -0
  50. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/egt2010/zones.py +0 -0
  51. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/egt2020/__init__.py +0 -0
  52. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/egt2020/deplacements.py +0 -0
  53. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/egt2020/menages.py +0 -0
  54. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/egt2020/motos.py +0 -0
  55. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/egt2020/personnes.py +0 -0
  56. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/egt2020/survey.py +0 -0
  57. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/egt2020/trajets.py +0 -0
  58. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/egt2020/voitures.py +0 -0
  59. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/emp/__init__.py +0 -0
  60. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/emp/common.py +0 -0
  61. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/emp/deplacements.py +0 -0
  62. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/emp/menages.py +0 -0
  63. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/emp/motos.py +0 -0
  64. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/emp/personnes.py +0 -0
  65. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/emp/voitures.py +0 -0
  66. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/logger.py +0 -0
  67. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/main.py +0 -0
  68. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/resources/__init__.py +0 -0
  69. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/resources/nuts.py +0 -0
  70. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/schema/__init__.py +0 -0
  71. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/schema/cars.py +0 -0
  72. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/schema/households.py +0 -0
  73. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/schema/motorcycles.py +0 -0
  74. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/schema/persons.py +0 -0
  75. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd/validation.py +0 -0
  76. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd.egg-info/dependency_links.txt +0 -0
  77. {mobisurvstd-1.0.0 → mobisurvstd-1.0.2}/src/mobisurvstd.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: mobisurvstd
3
- Version: 1.0.0
3
+ Version: 1.0.2
4
4
  Summary: A tool to standardize French mobility survey datasets (EMC2, EGT, EMP, etc.) into a unified format.
5
5
  Author-email: Lucas Javaudin <lucas@lucasjavaudin.com>
6
6
  License-Expression: MIT
@@ -29,6 +29,7 @@ Requires-Dist: polars>=1.33.0
29
29
  Requires-Dist: py7zr>=0.20.8
30
30
  Requires-Dist: pyarrow>=19.0.1
31
31
  Requires-Dist: requests>=2.32.3
32
+ Requires-Dist: seaborn>=0.13.2
32
33
  Requires-Dist: typer>=0.16.0
33
34
  Dynamic: license-file
34
35
 
@@ -36,7 +37,9 @@ Dynamic: license-file
36
37
  [![Forks][forks-shield]][forks-url]
37
38
  [![Stargazers][stars-shield]][stars-url]
38
39
  [![Issues][issues-shield]][issues-url]
40
+
39
41
  [![GPL v3][license-shield]][license-url]
42
+ [![PyPI][pypi-shield]][pypi-url]
40
43
  [![DOI][doi-shield]][doi-url]
41
44
 
42
45
  # MobiSurvStd
@@ -44,7 +47,6 @@ Dynamic: license-file
44
47
  *A tool to standardize French mobility survey datasets (EMC², EGT, EMP, etc.) into a unified format.*
45
48
 
46
49
  📚 [Documentation](https://mobisurvstd.github.io/MobiSurvStd)
47
- 📦 [View on PyPI](https://pypi.org/project/mobisurvstd/)
48
50
 
49
51
  <span style="color:blue">🇫🇷 [Version française](./README.fr.md)</span>
50
52
 
@@ -164,11 +166,11 @@ For more, check the [User Guide](https://mobisurvstd.github.io/MobiSurvStd/howto
164
166
 
165
167
  **Citing MobiSurvStd**
166
168
 
167
- If you use MobiSurvStd in a publication or project, please include the following attribution:
169
+ If you use MobiSurvStd in a publication or project, please include the following citation:
168
170
  ```
169
- Javaudin, L., 2025. MobiSurvStd. URL: https://doi.org/10.5281/zenodo.17422994,
170
- doi:10.5281/zenodo.17422994
171
+ Javaudin, L. (2025). MobiSurvStd. Zenodo. https://doi.org/10.5281/zenodo.17710203
171
172
  ```
173
+ A [CITATION.bib](CITATION.bib) is also available.
172
174
 
173
175
  ---
174
176
 
@@ -244,5 +246,7 @@ feel free to open an issue on GitHub or to create a Pull Request.
244
246
  [issues-url]: https://github.com/MobiSurvStd/MobiSurvStd/issues
245
247
  [license-shield]: https://img.shields.io/github/license/MobiSurvStd/MobiSurvStd.svg?style=for-the-badge
246
248
  [license-url]: https://github.com/MobiSurvStd/MobiSurvStd/blob/master/LICENSE.txt
247
- [doi-shield]: https://zenodo.org/badge/DOI/10.5281/zenodo.17422994.svg
248
- [doi-url]: https://doi.org/10.5281/zenodo.17422994
249
+ [doi-shield]: https://img.shields.io/badge/doi-10.5281/zenodo.17422994-blue?style=for-the-badge
250
+ [doi-url]: https://doi.org/10.5281/zenodo.17422993
251
+ [pypi-shield]: https://img.shields.io/pypi/v/mobisurvstd.svg?style=for-the-badge
252
+ [pypi-url]: https://pypi.org/project/mobisurvstd/
@@ -2,7 +2,9 @@
2
2
  [![Forks][forks-shield]][forks-url]
3
3
  [![Stargazers][stars-shield]][stars-url]
4
4
  [![Issues][issues-shield]][issues-url]
5
+
5
6
  [![GPL v3][license-shield]][license-url]
7
+ [![PyPI][pypi-shield]][pypi-url]
6
8
  [![DOI][doi-shield]][doi-url]
7
9
 
8
10
  # MobiSurvStd
@@ -10,7 +12,6 @@
10
12
  *A tool to standardize French mobility survey datasets (EMC², EGT, EMP, etc.) into a unified format.*
11
13
 
12
14
  📚 [Documentation](https://mobisurvstd.github.io/MobiSurvStd)
13
- 📦 [View on PyPI](https://pypi.org/project/mobisurvstd/)
14
15
 
15
16
  <span style="color:blue">🇫🇷 [Version française](./README.fr.md)</span>
16
17
 
@@ -130,11 +131,11 @@ For more, check the [User Guide](https://mobisurvstd.github.io/MobiSurvStd/howto
130
131
 
131
132
  **Citing MobiSurvStd**
132
133
 
133
- If you use MobiSurvStd in a publication or project, please include the following attribution:
134
+ If you use MobiSurvStd in a publication or project, please include the following citation:
134
135
  ```
135
- Javaudin, L., 2025. MobiSurvStd. URL: https://doi.org/10.5281/zenodo.17422994,
136
- doi:10.5281/zenodo.17422994
136
+ Javaudin, L. (2025). MobiSurvStd. Zenodo. https://doi.org/10.5281/zenodo.17710203
137
137
  ```
138
+ A [CITATION.bib](CITATION.bib) is also available.
138
139
 
139
140
  ---
140
141
 
@@ -210,5 +211,7 @@ feel free to open an issue on GitHub or to create a Pull Request.
210
211
  [issues-url]: https://github.com/MobiSurvStd/MobiSurvStd/issues
211
212
  [license-shield]: https://img.shields.io/github/license/MobiSurvStd/MobiSurvStd.svg?style=for-the-badge
212
213
  [license-url]: https://github.com/MobiSurvStd/MobiSurvStd/blob/master/LICENSE.txt
213
- [doi-shield]: https://zenodo.org/badge/DOI/10.5281/zenodo.17422994.svg
214
- [doi-url]: https://doi.org/10.5281/zenodo.17422994
214
+ [doi-shield]: https://img.shields.io/badge/doi-10.5281/zenodo.17422994-blue?style=for-the-badge
215
+ [doi-url]: https://doi.org/10.5281/zenodo.17422993
216
+ [pypi-shield]: https://img.shields.io/pypi/v/mobisurvstd.svg?style=for-the-badge
217
+ [pypi-url]: https://pypi.org/project/mobisurvstd/
@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
6
6
 
7
7
  [project]
8
8
  name = "mobisurvstd"
9
- version = "1.0.0"
9
+ version = "1.0.2"
10
10
  description = "A tool to standardize French mobility survey datasets (EMC2, EGT, EMP, etc.) into a unified format."
11
11
  readme = "README.md"
12
12
  authors = [{ name = "Lucas Javaudin", email = "lucas@lucasjavaudin.com" }]
@@ -22,21 +22,29 @@ dependencies = [
22
22
  "Cartopy >= 0.24.1",
23
23
  "fastexcel >= 0.14.0",
24
24
  "fsspec>=2025.9.0",
25
- "geopandas >= 1.0.1",
25
+ "geopandas>=1.0.1",
26
26
  "humanize >= 4.13.0",
27
27
  "loguru >= 0.7.3",
28
28
  "matplotlib >= 3.10.1",
29
29
  "numpy >= 2.2.4",
30
30
  "pandas >= 2.2.3",
31
31
  "platformdirs >= 4.3.8",
32
- "polars >= 1.33.0",
32
+ "polars>=1.33.0",
33
33
  "py7zr >= 0.20.8",
34
34
  "pyarrow >= 19.0.1",
35
35
  "requests >= 2.32.3",
36
+ "seaborn>=0.13.2",
36
37
  "typer >= 0.16.0",
37
38
  ]
38
39
  requires-python = ">=3.10"
39
40
 
41
+ [dependency-groups]
42
+ dev = [
43
+ "ipython>=8.37.0",
44
+ "pytest>=9.0.2",
45
+ "ty>=0.0.15",
46
+ ]
47
+
40
48
  [project.urls]
41
49
  Homepage = "https://github.com/MobiSurvStd/MobiSurvStd"
42
50
  Documentation = "https://mobisurvstd.github.io/MobiSurvStd/index.html"
@@ -6,6 +6,8 @@ from loguru import logger
6
6
  from mobisurvstd.common.trips import clean
7
7
  from mobisurvstd.schema.common import CURRENT_YEAR
8
8
 
9
+ from .reader import CeremaReader
10
+
9
11
  SCHEMA = {
10
12
  "DP1": pl.UInt8, # Code fichier = 3 (déplacement)
11
13
  "DMET": pl.UInt8, # Méthode d'enquête du ménage (EMC2 only)
@@ -114,7 +116,7 @@ def scan_trips_impl(source: str | io.BytesIO):
114
116
  return pl.scan_csv(source, separator=";", schema_overrides=SCHEMA, null_values=["aa", "aaaaa"])
115
117
 
116
118
 
117
- class TripsReader:
119
+ class TripsReader(CeremaReader):
118
120
  def scan_trips(self):
119
121
  lfs_iter = map(scan_trips_impl, self.trips_filenames())
120
122
  lf = pl.concat(lfs_iter, how="vertical")
@@ -123,11 +125,13 @@ class TripsReader:
123
125
 
124
126
  def standardize_trips(self):
125
127
  lf = self.scan_trips()
126
- # Add household_id, person_id, and trip date.
128
+ # Add household_id, person_id, trip date, and trip weekday.
127
129
  lf = lf.with_columns(
128
130
  original_person_id=pl.struct(**self.get_person_index_cols_from_trips())
129
131
  ).join(
130
- self.persons.select("original_person_id", "person_id", "household_id", "trip_date"),
132
+ self.persons.select(
133
+ "original_person_id", "person_id", "household_id", "trip_date", "trip_weekday"
134
+ ),
131
135
  on="original_person_id",
132
136
  how="left",
133
137
  coalesce=True,
@@ -5,13 +5,13 @@ from zipfile import ZipFile
5
5
 
6
6
  import geopandas as gpd
7
7
 
8
- from mobisurvstd.utils import MissingFileError, find_file
8
+ from mobisurvstd.utils import MissingFileError, find_file, find_file_path
9
9
 
10
- from .survey import CeremaReader
10
+ from .survey import CeremaStandardizer
11
11
  from .zones import find_matching_column
12
12
 
13
13
 
14
- class EDGTReader(CeremaReader):
14
+ class EDGTReader(CeremaStandardizer):
15
15
  SURVEY_TYPE = "EDGT"
16
16
 
17
17
  def households_filenames(self):
@@ -29,53 +29,49 @@ class EDGTReader(CeremaReader):
29
29
  def special_locations_and_detailed_zones_filenames(self):
30
30
  # This should match the Amiens 2010, Clermont-Ferrand 2012, and Lyon 2015 surveys.
31
31
  subdir = os.path.join("Doc", "SIG")
32
- faf_filename = find_file(
33
- self.source, ".*_faf_d.?coupagefin[.](tab|shp|mif)", subdir=subdir, as_url=True
32
+ faf_filename = find_file_path(
33
+ self.source, ".*_faf_d.?coupagefin[.](tab|shp|mif)", subdir=subdir
34
34
  )
35
- tel_filename = find_file(
36
- self.source, ".*_tel_d.?coupagefin[.](tab|shp|mif)", subdir=subdir, as_url=True
35
+ tel_filename = find_file_path(
36
+ self.source, ".*_tel_d.?coupagefin[.](tab|shp|mif)", subdir=subdir
37
37
  )
38
38
  if faf_filename and tel_filename:
39
39
  # Amiens 2010 has two files.
40
40
  return [faf_filename, tel_filename]
41
41
  else:
42
42
  return [
43
- find_file(
43
+ find_file_path(
44
44
  self.source,
45
45
  ".*(_zf_gt|zinterne_gen_.*)[.](tab|shp|mif)",
46
46
  subdir=subdir,
47
- as_url=True,
48
47
  )
49
48
  ]
50
49
 
51
50
  def special_locations_filenames(self):
52
51
  subdir = os.path.join("Doc", "SIG")
53
- faf_filename = find_file(
54
- self.source, ".*_faf_.*g.?n.?rateur.*[.](tab|shp|mif)", subdir=subdir, as_url=True
52
+ faf_filename = find_file_path(
53
+ self.source, ".*_faf_.*g.?n.?rateur.*[.](tab|shp|mif)", subdir=subdir
55
54
  )
56
- tel_filename = find_file(
57
- self.source, ".*_tel_.*g.?n.?rateur.*[.](tab|shp|mif)", subdir=subdir, as_url=True
55
+ tel_filename = find_file_path(
56
+ self.source, ".*_tel_.*g.?n.?rateur.*[.](tab|shp|mif)", subdir=subdir
58
57
  )
59
58
  if faf_filename and tel_filename:
60
59
  # Saint-Quentin-en-Yvelines has two files, for phone and face-to-face surveys.
61
60
  return [faf_filename, tel_filename]
62
61
  else:
63
62
  return [
64
- find_file(
65
- self.source,
66
- ".*(_gt_.*|_gt|g.?n.?rateur.*)[.](tab|shp|mif)",
67
- subdir=subdir,
68
- as_url=True,
63
+ find_file_path(
64
+ self.source, ".*(_gt_.*|_gt|g.?n.?rateur.*)[.](tab|shp|mif)", subdir=subdir
69
65
  )
70
66
  ]
71
67
 
72
68
  def detailed_zones_filenames(self):
73
69
  subdir = os.path.join("Doc", "SIG")
74
- faf_filename = find_file(
75
- self.source, ".*_faf_zones_fines[.](tab|shp|mif)", subdir=subdir, as_url=True
70
+ faf_filename = find_file_path(
71
+ self.source, ".*_faf_zones_fines[.](tab|shp|mif)", subdir=subdir
76
72
  )
77
- tel_filename = find_file(
78
- self.source, ".*_tel_zones_fines[.](tab|shp|mif)", subdir=subdir, as_url=True
73
+ tel_filename = find_file_path(
74
+ self.source, ".*_tel_zones_fines[.](tab|shp|mif)", subdir=subdir
79
75
  )
80
76
  if faf_filename and tel_filename:
81
77
  # Nice 2009 and Saint-Quentin-en-Yvelines have two files, for phone and face-to-face
@@ -83,27 +79,29 @@ class EDGTReader(CeremaReader):
83
79
  return [faf_filename, tel_filename]
84
80
  else:
85
81
  return [
86
- find_file(
82
+ find_file_path(
87
83
  self.source,
88
84
  ".*(_zf|_zf_.*|zf08_.*|(?<!tel_)zones[_ ]fines)[.](tab|shp|mif)",
89
85
  subdir=subdir,
90
- as_url=True,
91
86
  )
92
87
  ]
93
88
 
94
89
  def draw_zones_filenames(self):
95
90
  return [
96
- find_file(
91
+ find_file_path(
97
92
  self.source,
98
93
  ".*(_dtir|_secteurstirage)[.](tab|shp|mif)",
99
94
  subdir=os.path.join("Doc", "SIG"),
100
- as_url=True,
101
95
  )
102
96
  ]
103
97
 
104
98
  def survey_name(self):
105
- filename = find_file(self.source, ".*_std_faf_men.csv", subdir="Csv", as_url=True)
106
- return re.match("(.*)_std_faf_men.csv", os.path.basename(filename)).group(1)
99
+ filename = find_file_path(self.source, ".*_std_faf_men.csv", subdir="Csv")
100
+ fn_match = re.match("(.*)_std_faf_men.csv", os.path.basename(filename))
101
+ if fn_match is not None:
102
+ return fn_match.group(1)
103
+ else:
104
+ return "unkown"
107
105
 
108
106
  def gt_id_columns(self):
109
107
  return [
@@ -4,18 +4,18 @@ import re
4
4
 
5
5
  import geopandas as gpd
6
6
 
7
- from mobisurvstd.cerema.survey import CeremaReader
8
- from mobisurvstd.utils import MissingFileError, find_file
7
+ from mobisurvstd.cerema.survey import CeremaStandardizer
8
+ from mobisurvstd.utils import MissingFileError, find_file, find_file_path
9
9
 
10
10
  from .zones import find_matching_column
11
11
 
12
12
 
13
- class EDVMReader(CeremaReader):
13
+ class EDVMReader(CeremaStandardizer):
14
14
  SURVEY_TYPE = "EDVM"
15
15
 
16
16
  def households_filenames(self) -> list[str | io.BytesIO | MissingFileError]:
17
17
  # The filenames are usually stored in "Csv/Fichiers_Standard/[SURVEY_NAME]_std_[NAME].csv"
18
- # but in some cases (e.g., Ajaccio 2017) the directory "Fichiers_Standard" is # named based
18
+ # but in some cases (e.g., Ajaccio 2017) the directory "Fichiers_Standard" is named based
19
19
  # on the survey name (e.g., Ajaccio_2017_Standard).
20
20
  return [find_file(self.source, ".*_std_men.csv", subdir="Csv")]
21
21
 
@@ -31,47 +31,43 @@ class EDVMReader(CeremaReader):
31
31
  def special_locations_and_detailed_zones_filenames(self):
32
32
  # This should only match the Beauvais 2011 survey.
33
33
  return [
34
- find_file(
35
- self.source,
36
- ".*beauvais.*_dfin[.](tab|shp|mif)",
37
- subdir=os.path.join("Doc", "SIG"),
38
- as_url=True,
34
+ find_file_path(
35
+ self.source, ".*beauvais.*_dfin[.](tab|shp|mif)", subdir=os.path.join("Doc", "SIG")
39
36
  )
40
37
  ]
41
38
 
42
39
  def detailed_zones_filenames(self):
43
40
  return [
44
- find_file(
41
+ find_file_path(
45
42
  self.source,
46
43
  ".*(_zf_.*|_zf|zones?[_ ]?fines?.*|_dfin)[.](tab|shp|mif)",
47
44
  subdir=os.path.join("Doc", "SIG"),
48
- as_url=True,
49
45
  )
50
46
  ]
51
47
 
52
48
  def special_locations_filenames(self):
53
49
  return [
54
- find_file(
50
+ find_file_path(
55
51
  self.source,
56
52
  ".*(_gt_.*|_gt|_pgt|_pg|g.?n.?rateur.*)[.](tab|shp|mif)",
57
53
  subdir=os.path.join("Doc", "SIG"),
58
- as_url=True,
59
54
  )
60
55
  ]
61
56
 
62
57
  def draw_zones_filenames(self):
63
58
  return [
64
- find_file(
65
- self.source,
66
- ".*_DTIR[.](tab|shp|mif)",
67
- subdir=os.path.join("Doc", "SIG"),
68
- as_url=True,
59
+ find_file_path(
60
+ self.source, ".*_DTIR[.](tab|shp|mif)", subdir=os.path.join("Doc", "SIG")
69
61
  )
70
62
  ]
71
63
 
72
64
  def survey_name(self):
73
- filename = find_file(self.source, ".*_std_men.csv", subdir="Csv", as_url=True)
74
- return re.match("(.*)_std_men.csv", os.path.basename(filename)).group(1)
65
+ filename = find_file_path(self.source, ".*_std_men.csv", subdir="Csv")
66
+ fn_match = re.match("(.*)_std_men.csv", os.path.basename(filename))
67
+ if fn_match is not None:
68
+ return fn_match.group(1)
69
+ else:
70
+ return "unkown"
75
71
 
76
72
  def gt_id_columns(self):
77
73
  return [
@@ -277,9 +273,10 @@ class EDVMReader(CeremaReader):
277
273
  super().select_zf_from_gt_column(gdf)
278
274
 
279
275
  def postprocess_special_locations_and_detailed_zones(
280
- self, zfs: gpd.GeoDataFrame, gts: gpd.GeoDataFrame
276
+ self, zfs: gpd.GeoDataFrame | None, gts: gpd.GeoDataFrame | None
281
277
  ):
282
278
  # This function should only be call for Beauvais 2011.
279
+ assert gts is not None
283
280
  assert "detailed_zone_id" not in gts.columns
284
281
  # The corresponding ZF is the GT id with the last digit replaced by 0.
285
282
  gts["detailed_zone_id"] = gts["special_location_id"].str.replace(r"\d$", "0", regex=True)
@@ -4,13 +4,13 @@ import re
4
4
 
5
5
  import geopandas as gpd
6
6
 
7
- from mobisurvstd.cerema.survey import CeremaReader
8
- from mobisurvstd.utils import MissingFileError, find_file
7
+ from mobisurvstd.cerema.survey import CeremaStandardizer
8
+ from mobisurvstd.utils import MissingFileError, find_file, find_file_path
9
9
 
10
10
  from .zones import find_matching_column
11
11
 
12
12
 
13
- class EMC2Reader(CeremaReader):
13
+ class EMC2Reader(CeremaStandardizer):
14
14
  SURVEY_TYPE = "EMC2"
15
15
 
16
16
  def households_filenames(self) -> list[str | io.BytesIO | MissingFileError]:
@@ -43,42 +43,34 @@ class EMC2Reader(CeremaReader):
43
43
 
44
44
  def detailed_zones_filenames(self):
45
45
  return [
46
- find_file(
47
- self.source,
48
- r".*_ZF(_.*)?\.(TAB|shp)",
49
- subdir=os.path.join("Doc", "SIG"),
50
- as_url=True,
46
+ find_file_path(
47
+ self.source, r".*_ZF(_.*)?\.(TAB|shp)", subdir=os.path.join("Doc", "SIG")
51
48
  )
52
49
  ]
53
50
 
54
51
  def special_locations_filenames(self):
55
52
  return [
56
- find_file(
57
- self.source,
58
- r".*_GT(_.*)?\.(TAB|shp)",
59
- subdir=os.path.join("Doc", "SIG"),
60
- as_url=True,
53
+ find_file_path(
54
+ self.source, r".*_GT(_.*)?\.(TAB|shp)", subdir=os.path.join("Doc", "SIG")
61
55
  )
62
56
  ]
63
57
 
64
58
  def draw_zones_filenames(self):
65
59
  return [
66
- find_file(
67
- self.source,
68
- r".*_DTIR(_.*)?\.(TAB|shp)",
69
- subdir=os.path.join("Doc", "SIG"),
70
- as_url=True,
60
+ find_file_path(
61
+ self.source, r".*_DTIR(_.*)?\.(TAB|shp)", subdir=os.path.join("Doc", "SIG")
71
62
  )
72
63
  ]
73
64
 
74
65
  def survey_name(self):
75
- filename = find_file(
76
- self.source,
77
- ".*_std_men.csv",
78
- subdir=os.path.join("Csv", "Fichiers_Standard"),
79
- as_url=True,
66
+ filename = find_file_path(
67
+ self.source, ".*_std_men.csv", subdir=os.path.join("Csv", "Fichiers_Standard")
80
68
  )
81
- return re.match("(.*)_std_men.csv", os.path.basename(filename)).group(1)
69
+ fn_match = re.match("(.*)_std_men.csv", os.path.basename(filename))
70
+ if fn_match is not None:
71
+ return fn_match.group(1)
72
+ else:
73
+ return "Unknown"
82
74
 
83
75
  def gt_id_columns(self):
84
76
  return ["zf_160", "zf_fusion", "codegt", "zf", "num_gt"]
@@ -4,13 +4,13 @@ import re
4
4
 
5
5
  import geopandas as gpd
6
6
 
7
- from mobisurvstd.cerema.survey import CeremaReader
8
- from mobisurvstd.utils import MissingFileError, find_file
7
+ from mobisurvstd.cerema.survey import CeremaStandardizer
8
+ from mobisurvstd.utils import MissingFileError, find_file, find_file_path
9
9
 
10
10
  from .zones import find_matching_column
11
11
 
12
12
 
13
- class EMDReader(CeremaReader):
13
+ class EMDReader(CeremaStandardizer):
14
14
  SURVEY_TYPE = "EMD"
15
15
 
16
16
  def households_filenames(self) -> list[str | io.BytesIO | MissingFileError]:
@@ -31,47 +31,45 @@ class EMDReader(CeremaReader):
31
31
  def special_locations_and_detailed_zones_filenames(self):
32
32
  # This should match the Valenciennes 2011 and Grenoble 2010 surveys.
33
33
  return [
34
- find_file(
34
+ find_file_path(
35
35
  self.source,
36
36
  ".*(grenobloise10_zones_fines|zf_gt)[.]mif",
37
37
  subdir=os.path.join("Doc", "SIG"),
38
- as_url=True,
39
38
  )
40
39
  ]
41
40
 
42
41
  def detailed_zones_filenames(self):
43
42
  return [
44
- find_file(
43
+ find_file_path(
45
44
  self.source,
46
45
  ".*(_zf|zones?[_ ]?fines?.*)[.](tab|mif)",
47
46
  subdir=os.path.join("Doc", "SIG"),
48
- as_url=True,
49
47
  )
50
48
  ]
51
49
 
52
50
  def special_locations_filenames(self):
53
51
  return [
54
- find_file(
55
- self.source,
56
- ".*(_gt|g.?n.?rateur.*)[.](tab|mif)",
57
- subdir=os.path.join("Doc", "SIG"),
58
- as_url=True,
52
+ find_file_path(
53
+ self.source, ".*(_gt|g.?n.?rateur.*)[.](tab|mif)", subdir=os.path.join("Doc", "SIG")
59
54
  )
60
55
  ]
61
56
 
62
57
  def draw_zones_filenames(self):
63
58
  return [
64
- find_file(
59
+ find_file_path(
65
60
  self.source,
66
61
  ".*(_DTIR|secteur_.*)[.](tab|shp|mif)",
67
62
  subdir=os.path.join("Doc", "SIG"),
68
- as_url=True,
69
63
  )
70
64
  ]
71
65
 
72
66
  def survey_name(self):
73
- filename = find_file(self.source, ".*_std_men.csv", subdir="Csv", as_url=True)
74
- return re.match("(.*)_std_men.csv", os.path.basename(filename)).group(1)
67
+ filename = find_file_path(self.source, ".*_std_men.csv", subdir="Csv")
68
+ fn_match = re.match("(.*)_std_men.csv", os.path.basename(filename))
69
+ if fn_match is not None:
70
+ return fn_match.group(1)
71
+ else:
72
+ return "Unknown"
75
73
 
76
74
  def gt_id_columns(self):
77
75
  return [
@@ -7,6 +7,8 @@ from mobisurvstd.common.households import clean as clean_households
7
7
  from mobisurvstd.common.motorcycles import clean as clean_motorcycles
8
8
  from mobisurvstd.schema.common import CURRENT_YEAR
9
9
 
10
+ from .reader import CeremaReader
11
+
10
12
  SCHEMA = {
11
13
  "MP1": pl.UInt8, # Code fichier = 1
12
14
  "METH": pl.UInt8, # Méthode d'enquête du ménage (for EMC2 only)
@@ -230,8 +232,8 @@ def scan_households_impl(source: str | io.BytesIO):
230
232
  return pl.scan_csv(source, separator=";", schema_overrides=SCHEMA, null_values=["a"])
231
233
 
232
234
 
233
- class HouseholdsReader:
234
- def scan_households(self):
235
+ class HouseholdsReader(CeremaReader):
236
+ def scan_households(self) -> pl.LazyFrame:
235
237
  lfs_iter = map(scan_households_impl, self.households_filenames())
236
238
  lf = pl.concat(lfs_iter, how="vertical")
237
239
  if "METH" not in lf.collect_schema().names():
@@ -244,7 +246,7 @@ class HouseholdsReader:
244
246
  return lf
245
247
 
246
248
  def main_insee(self):
247
- return self.scan_households().select("IDM4").first().collect().item()
249
+ return self.scan_households().select("IDM4").first().collect().item() # ty: ignore[possibly-missing-attribute]
248
250
 
249
251
  def standardize_households(self):
250
252
  lf = self.scan_households()
@@ -299,6 +301,8 @@ class HouseholdsReader:
299
301
  how="left",
300
302
  coalesce=True,
301
303
  )
304
+ # Note: the `.lazy()` after the concat is useless (this is already a LazyFrame) but it helps
305
+ # the syntax checker.
302
306
  lf = pl.concat(
303
307
  (
304
308
  lf.select(
@@ -326,7 +330,7 @@ class HouseholdsReader:
326
330
  for i, a in enumerate(("A", "B", "C", "D"))
327
331
  ),
328
332
  how="vertical",
329
- )
333
+ ).lazy()
330
334
  # Drop the lines with empty car characteristics (there are always 4 cars per
331
335
  # households even when the household has less than 4 cars).
332
336
  lf = lf.filter(
@@ -347,6 +351,8 @@ class HouseholdsReader:
347
351
  how="left",
348
352
  coalesce=True,
349
353
  )
354
+ # Note: the `.lazy()` after the concat is useless (this is already a LazyFrame) but it helps
355
+ # the syntax checker.
350
356
  lf = pl.concat(
351
357
  (
352
358
  lf.select(
@@ -388,7 +394,7 @@ class HouseholdsReader:
388
394
  for i, a in enumerate(("A", "B", "C", "D"))
389
395
  ),
390
396
  how="vertical",
391
- )
397
+ ).lazy()
392
398
  lf = lf.filter(
393
399
  pl.any_horizontal(
394
400
  pl.all()
@@ -2,13 +2,14 @@ import io
2
2
 
3
3
  import pandas as pd
4
4
  import polars as pl
5
+ from polars.datatypes import DataTypeClass
5
6
 
6
7
  from mobisurvstd.utils import find_file
7
8
 
8
9
  from .deplacements import SCHEMA as TRIP_SCHEMA
9
10
  from .menages import SCHEMA as HOUSEHOLD_SCHEMA
10
11
  from .personnes import SCHEMA as PERSON_SCHEMA
11
- from .survey import CeremaReader
12
+ from .survey import CeremaStandardizer
12
13
  from .trajets import SCHEMA as LEG_SCHEMA
13
14
 
14
15
  # The anonimized open-data format for Nantes uses a fixed-width text format.
@@ -140,7 +141,7 @@ LEG_FORMAT = [
140
141
  ]
141
142
 
142
143
 
143
- class OpenDataReader(CeremaReader):
144
+ class OpenDataReader(CeremaStandardizer):
144
145
  SURVEY_TYPE = "EDGT-opendata"
145
146
 
146
147
  def households_filenames(self):
@@ -193,7 +194,7 @@ class OpenDataReader(CeremaReader):
193
194
  def scan_fwf(
194
195
  f: str | io.BytesIO,
195
196
  format: list[tuple[int, int, str]],
196
- schema: dict[str, pl.DataType],
197
+ schema: dict[str, DataTypeClass],
197
198
  ):
198
199
  # Decrease `start` by 1 because pandas starts counting at 0 but the survey document starts
199
200
  # counting at 1.