mobisurvstd 0.4.0__tar.gz → 0.6.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. {mobisurvstd-0.4.0/src/mobisurvstd.egg-info → mobisurvstd-0.6.0}/PKG-INFO +2 -2
  2. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/pyproject.toml +2 -2
  3. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/classes.py +1 -13
  4. mobisurvstd-0.6.0/src/mobisurvstd/clean.py +50 -0
  5. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/edgt/survey.py +62 -55
  6. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/edvm/survey.py +46 -39
  7. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/egt2010/survey.py +10 -7
  8. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/egt2010/trajets.py +1 -1
  9. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/egt2020/survey.py +1 -1
  10. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/emc2/survey.py +57 -48
  11. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/emd/survey.py +61 -53
  12. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/emp/deplacements.py +2 -2
  13. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/emp/survey.py +1 -1
  14. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/main.py +43 -16
  15. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/utils.py +9 -3
  16. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0/src/mobisurvstd.egg-info}/PKG-INFO +2 -2
  17. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd.egg-info/SOURCES.txt +1 -0
  18. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd.egg-info/requires.txt +1 -1
  19. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/LICENSE.md +0 -0
  20. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/README.md +0 -0
  21. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/setup.cfg +0 -0
  22. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/__init__.py +0 -0
  23. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/__main__.py +0 -0
  24. /mobisurvstd-0.4.0/src/mobisurvstd/clean.py → /mobisurvstd-0.6.0/src/mobisurvstd/clean-old.py +0 -0
  25. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/cli.py +0 -0
  26. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/common/__init__.py +0 -0
  27. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/common/cars.py +0 -0
  28. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/common/clean.py +0 -0
  29. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/common/households.py +0 -0
  30. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/common/legs.py +0 -0
  31. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/common/modes.py +0 -0
  32. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/common/motorcycles.py +0 -0
  33. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/common/persons.py +0 -0
  34. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/common/trips.py +0 -0
  35. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/common/zones.py +0 -0
  36. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/edgt/__init__.py +0 -0
  37. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/edgt/common.py +0 -0
  38. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/edgt/deplacements.py +0 -0
  39. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/edgt/menages.py +0 -0
  40. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/edgt/personnes.py +0 -0
  41. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/edgt/trajets.py +0 -0
  42. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/edgt/zones.py +0 -0
  43. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/edvm/__init__.py +0 -0
  44. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/edvm/common.py +0 -0
  45. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/edvm/deplacements.py +0 -0
  46. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/edvm/menages.py +0 -0
  47. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/edvm/personnes.py +0 -0
  48. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/edvm/trajets.py +0 -0
  49. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/edvm/zones.py +0 -0
  50. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/egt2010/__init__.py +0 -0
  51. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/egt2010/deplacements.py +0 -0
  52. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/egt2010/menages.py +0 -0
  53. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/egt2010/personnes.py +0 -0
  54. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/egt2010/zones.py +0 -0
  55. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/egt2020/__init__.py +0 -0
  56. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/egt2020/deplacements.py +0 -0
  57. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/egt2020/menages.py +0 -0
  58. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/egt2020/motos.py +0 -0
  59. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/egt2020/personnes.py +0 -0
  60. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/egt2020/trajets.py +0 -0
  61. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/egt2020/voitures.py +0 -0
  62. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/emc2/__init__.py +0 -0
  63. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/emc2/common.py +0 -0
  64. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/emc2/deplacements.py +0 -0
  65. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/emc2/menages.py +0 -0
  66. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/emc2/personnes.py +0 -0
  67. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/emc2/trajets.py +0 -0
  68. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/emc2/zones.py +0 -0
  69. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/emd/__init__.py +0 -0
  70. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/emd/common.py +0 -0
  71. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/emd/deplacements.py +0 -0
  72. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/emd/menages.py +0 -0
  73. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/emd/personnes.py +0 -0
  74. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/emd/trajets.py +0 -0
  75. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/emd/zones.py +0 -0
  76. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/emp/__init__.py +0 -0
  77. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/emp/menages.py +0 -0
  78. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/emp/motos.py +0 -0
  79. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/emp/personnes.py +0 -0
  80. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/emp/voitures.py +0 -0
  81. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/logger.py +0 -0
  82. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/resources/__init__.py +0 -0
  83. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/resources/admin_express.py +0 -0
  84. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/resources/insee_data.py +0 -0
  85. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/resources/nuts.py +0 -0
  86. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd/schema.py +0 -0
  87. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd.egg-info/dependency_links.txt +0 -0
  88. {mobisurvstd-0.4.0 → mobisurvstd-0.6.0}/src/mobisurvstd.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: mobisurvstd
3
- Version: 0.4.0
3
+ Version: 0.6.0
4
4
  Summary: A tool to standardize French mobility survey datasets (EMC2, EGT, EMP, etc.) into a unified format.
5
5
  Author-email: Lucas Javaudin <lucas@lucasjavaudin.com>
6
6
  License-Expression: MIT
@@ -24,7 +24,7 @@ Requires-Dist: numpy>=2.2.4
24
24
  Requires-Dist: pandas>=2.2.3
25
25
  Requires-Dist: platformdirs>=4.3.8
26
26
  Requires-Dist: polars>=1.31.0
27
- Requires-Dist: py7zr>=0.22.0
27
+ Requires-Dist: py7zr>=0.20.8
28
28
  Requires-Dist: pyarrow>=19.0.1
29
29
  Requires-Dist: requests>=2.32.3
30
30
  Requires-Dist: typer>=0.16.0
@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
6
6
 
7
7
  [project]
8
8
  name = "mobisurvstd"
9
- version = "0.4.0"
9
+ version = "0.6.0"
10
10
  description = "A tool to standardize French mobility survey datasets (EMC2, EGT, EMP, etc.) into a unified format."
11
11
  readme = "README.md"
12
12
  authors = [{ name = "Lucas Javaudin", email = "lucas@lucasjavaudin.com" }]
@@ -28,7 +28,7 @@ dependencies = [
28
28
  "pandas >= 2.2.3",
29
29
  "platformdirs >= 4.3.8",
30
30
  "polars >= 1.31.0",
31
- "py7zr >= 0.22.0",
31
+ "py7zr >= 0.20.8",
32
32
  "pyarrow >= 19.0.1",
33
33
  "requests >= 2.32.3",
34
34
  "typer >= 0.16.0",
@@ -54,19 +54,7 @@ class SurveyData:
54
54
  )
55
55
 
56
56
  def clean(self):
57
- data = clean(
58
- households=self.households,
59
- cars=self.cars,
60
- motorcycles=self.motorcycles,
61
- persons=self.persons,
62
- trips=self.trips,
63
- legs=self.legs,
64
- special_locations=self.special_locations,
65
- detailed_zones=self.detailed_zones,
66
- draw_zones=self.draw_zones,
67
- )
68
- data["metadata"] = self.metadata
69
- return self.__class__.from_dict(data)
57
+ clean(self)
70
58
 
71
59
  def save(self, output_directory: str):
72
60
  """Saves the MobiSurvStd survey to the given directory."""
@@ -0,0 +1,50 @@
1
+ import polars as pl
2
+ from loguru import logger
3
+
4
+
5
+ def clean(data):
6
+ # === Trips ===
7
+ invalid_persons = set(
8
+ data.trips.filter(pl.col("arrival_time") < pl.col("departure_time"))["person_id"]
9
+ )
10
+ if invalid_persons:
11
+ n = len(invalid_persons)
12
+ logger.warning(
13
+ f"{n} persons have at least one trip with `arrival_time` smaller than `departure_time`."
14
+ " The `departure_time`, `arrival_time`, and `travel_time` values for these persons "
15
+ "are automatically set to null."
16
+ )
17
+ data.trips = data.trips.with_columns(
18
+ pl.when(pl.col("person_id").is_in(invalid_persons))
19
+ .then(pl.lit(None))
20
+ .otherwise(col)
21
+ .alias(col)
22
+ for col in ("departure_time", "arrival_time", "travel_time")
23
+ )
24
+ invalid_persons = set(
25
+ data.trips.filter(
26
+ pl.col("arrival_time") > pl.col("departure_time").shift(-1).over("person_id")
27
+ )["person_id"]
28
+ )
29
+ if invalid_persons:
30
+ n = len(invalid_persons)
31
+ logger.warning(
32
+ f"{n} persons have at least one trip that starts before the previous trip ended. "
33
+ "The `departure_time`, `arrival_time`, and `travel_time` values for these persons "
34
+ "are automatically set to null."
35
+ )
36
+ data.trips = data.trips.with_columns(
37
+ pl.when(pl.col("person_id").is_in(invalid_persons))
38
+ .then(pl.lit(None))
39
+ .otherwise(col)
40
+ .alias(col)
41
+ for col in ("departure_time", "arrival_time", "travel_time")
42
+ )
43
+ assert (
44
+ data.trips["travel_time"]
45
+ .cast(pl.Int32)
46
+ .eq_missing(
47
+ data.trips["arrival_time"].cast(pl.Int32) - data.trips["departure_time"].cast(pl.Int32)
48
+ )
49
+ .all()
50
+ ), "`travel_time` is not equal to `arrival_time - departure_time`"
@@ -23,59 +23,15 @@ from .trajets import standardize_legs
23
23
  from .zones import read_detailed_zones, read_draw_zones, read_special_locations
24
24
 
25
25
 
26
- def standardize(source: str | ZipFile):
26
+ def standardize(source: str | ZipFile, skip_spatial: bool = False):
27
27
  source_name = source.filename if isinstance(source, ZipFile) else source
28
28
  logger.info(f"Standardizing EDGT survey from `{source_name}`")
29
- # Special locations.
30
- filename = special_locations_filename(source)
31
- if filename is None:
32
- special_locations = None
33
- elif "EDGT_SCOTAM_2017" in source_name or "Angers12" in source_name or "EDGTFVG2016":
34
- # Note. Special case for Metz 2017 and Angers 2012: I cannot figure out the ids in the GT
35
- # and ZF files.
36
- # For Annemasse 2016, the GT id does not seem to be specified.
37
- logger.warning("Cannot read special locations: ids are invalid")
29
+ if skip_spatial:
38
30
  special_locations = None
39
- else:
40
- logger.debug(f"Reading special locations from `{filename}`")
41
- special_locations = read_special_locations(filename)
42
- # Detailed zones.
43
- filename = detailed_zones_filename(source)
44
- if filename is None:
45
- logger.debug(f"No file with detailed zones in `{source_name}`")
46
- detailed_zones = None
47
- elif "EDGT_SCOTAM_2017" in source_name or "Angers12" in source_name or "FaF_Zones_fines":
48
- # Note. Special case for Metz 2017 and Angers 2012: I cannot figure out the ids in the GT
49
- # and ZF files.
50
- # For Bayonne 2010, the ids are missing.
51
- logger.warning("Cannot read detailed zones: ids are invalid")
52
31
  detailed_zones = None
53
- else:
54
- logger.debug(f"Reading detailed zones from `{filename}`")
55
- detailed_zones = read_detailed_zones(filename)
56
- # Draw zones.
57
- filename = draw_zones_filename(source)
58
- if filename is None:
59
- logger.debug(f"No file with draw zones in `{source_name}`")
60
32
  draw_zones = None
61
33
  else:
62
- logger.debug(f"Reading draw zones from `{filename}`")
63
- draw_zones = read_draw_zones(filename)
64
-
65
- if special_locations is not None:
66
- # For the Angers 2022 survey, the ids in the detailed zones shapefile have 3 zeros removed
67
- # compared to the ids in the special locations shapefile.
68
- assert detailed_zones is not None, (
69
- "Special locations are defined but there is no data on detailed zones"
70
- )
71
- if "detailed_zone_id" not in special_locations.columns:
72
- special_locations = identify_detailed_zone_id(special_locations, detailed_zones)
73
- dz_gcd = np.gcd.reduce(detailed_zones["detailed_zone_id"].astype(int))
74
- sl_gcd = np.gcd.reduce(special_locations["detailed_zone_id"].astype(int))
75
- if dz_gcd < sl_gcd and np.log10(sl_gcd).is_integer():
76
- detailed_zones["detailed_zone_id"] = (
77
- detailed_zones["detailed_zone_id"].astype(int) * sl_gcd
78
- ).astype(str)
34
+ special_locations, detailed_zones, draw_zones = read_spatial_data(source, source_name)
79
35
 
80
36
  if special_locations is not None:
81
37
  special_locations_coords = get_coords(special_locations, "special_location")
@@ -180,6 +136,61 @@ def standardize(source: str | ZipFile):
180
136
  )
181
137
 
182
138
 
139
+ def read_spatial_data(source: str | ZipFile, source_name: str):
140
+ # Special locations.
141
+ filename = special_locations_filename(source)
142
+ if filename is None:
143
+ special_locations = None
144
+ elif "EDGT_SCOTAM_2017" in source_name or "Angers12" in source_name or "EDGTFVG2016":
145
+ # Note. Special case for Metz 2017 and Angers 2012: I cannot figure out the ids in the GT
146
+ # and ZF files.
147
+ # For Annemasse 2016, the GT id does not seem to be specified.
148
+ logger.warning("Cannot read special locations: ids are invalid")
149
+ special_locations = None
150
+ else:
151
+ logger.debug(f"Reading special locations from `{filename}`")
152
+ special_locations = read_special_locations(filename)
153
+ # Detailed zones.
154
+ filename = detailed_zones_filename(source)
155
+ if filename is None:
156
+ logger.debug(f"No file with detailed zones in `{source_name}`")
157
+ detailed_zones = None
158
+ elif "EDGT_SCOTAM_2017" in source_name or "Angers12" in source_name or "FaF_Zones_fines":
159
+ # Note. Special case for Metz 2017 and Angers 2012: I cannot figure out the ids in the GT
160
+ # and ZF files.
161
+ # For Bayonne 2010, the ids are missing.
162
+ logger.warning("Cannot read detailed zones: ids are invalid")
163
+ detailed_zones = None
164
+ else:
165
+ logger.debug(f"Reading detailed zones from `{filename}`")
166
+ detailed_zones = read_detailed_zones(filename)
167
+ # Draw zones.
168
+ filename = draw_zones_filename(source)
169
+ if filename is None:
170
+ logger.debug(f"No file with draw zones in `{source_name}`")
171
+ draw_zones = None
172
+ else:
173
+ logger.debug(f"Reading draw zones from `{filename}`")
174
+ draw_zones = read_draw_zones(filename)
175
+
176
+ if special_locations is not None:
177
+ # For the Angers 2022 survey, the ids in the detailed zones shapefile have 3 zeros removed
178
+ # compared to the ids in the special locations shapefile.
179
+ assert (
180
+ detailed_zones is not None
181
+ ), "Special locations are defined but there is no data on detailed zones"
182
+ if "detailed_zone_id" not in special_locations.columns:
183
+ special_locations = identify_detailed_zone_id(special_locations, detailed_zones)
184
+ dz_gcd = np.gcd.reduce(detailed_zones["detailed_zone_id"].astype(int))
185
+ sl_gcd = np.gcd.reduce(special_locations["detailed_zone_id"].astype(int))
186
+ if dz_gcd < sl_gcd and np.log10(sl_gcd).is_integer():
187
+ detailed_zones["detailed_zone_id"] = (
188
+ detailed_zones["detailed_zone_id"].astype(int) * sl_gcd
189
+ ).astype(str)
190
+
191
+ return special_locations, detailed_zones, draw_zones
192
+
193
+
183
194
  def survey_main_insee(source: str | bytes):
184
195
  return pl.scan_csv(source, separator=";").select("IDM4").first().collect().item()
185
196
 
@@ -267,15 +278,13 @@ def fix_special_locations_with_gcd(
267
278
  pl.from_pandas(special_locations["detailed_zone_id"].astype(int)),
268
279
  default=None,
269
280
  )
270
- )
271
- .alias(f"{col}_detailed_zone"),
281
+ ).alias(f"{col}_detailed_zone"),
272
282
  # When the ZF is an actual ZF
273
283
  pl.when(mask)
274
284
  # Then the GT is null
275
285
  .then(pl.lit(None))
276
286
  # Otherwise use that GT as GT.
277
- .otherwise(f"{col}_detailed_zone")
278
- .alias(f"{col}_special_location"),
287
+ .otherwise(f"{col}_detailed_zone").alias(f"{col}_special_location"),
279
288
  )
280
289
  return df
281
290
 
@@ -298,15 +307,13 @@ def fix_special_locations(df: pl.LazyFrame, col: str, special_locations: gpd.Geo
298
307
  pl.from_pandas(special_locations["detailed_zone_id"].astype(int)),
299
308
  default=None,
300
309
  )
301
- )
302
- .alias(f"{col}_detailed_zone"),
310
+ ).alias(f"{col}_detailed_zone"),
303
311
  # When the ZF is an actual ZF
304
312
  pl.when(mask)
305
313
  # Then the GT is null
306
314
  .then(pl.lit(None))
307
315
  # Otherwise use that GT as GT.
308
- .otherwise(f"{col}_detailed_zone")
309
- .alias(f"{col}_special_location"),
316
+ .otherwise(f"{col}_detailed_zone").alias(f"{col}_special_location"),
310
317
  )
311
318
  return df
312
319
 
@@ -28,40 +28,15 @@ from .zones import (
28
28
  )
29
29
 
30
30
 
31
- def standardize(source: str | ZipFile):
31
+ def standardize(source: str | ZipFile, skip_spatial: bool = False):
32
32
  source_name = source.filename if isinstance(source, ZipFile) else source
33
33
  logger.info(f"Standardizing EDVM survey from `{source_name}`")
34
- if filename := special_locations_and_detailed_zones_filename(source):
35
- # Special case for Beauvais 2011.
36
- detailed_zones, special_locations = read_special_locations_and_detailed_zones(filename)
37
- else:
38
- # Special locations.
39
- if filename := special_locations_filename(source):
40
- logger.debug(f"Reading special locations from `{filename}`")
41
- special_locations = read_special_locations(filename)
42
- else:
43
- special_locations = None
44
- # Detailed zones.
45
- if filename := detailed_zones_filename(source):
46
- logger.debug(f"Reading detailed zones from `{filename}`")
47
- detailed_zones = read_detailed_zones(filename)
48
- else:
49
- logger.debug(f"No file with detailed zones in `{source_name}`")
50
- detailed_zones = None
51
- # Draw zones.
52
- if filename := draw_zones_filename(source):
53
- logger.debug(f"Reading draw zones from `{filename}`")
54
- draw_zones = read_draw_zones(filename)
55
- else:
56
- logger.debug(f"No file with draw zones in `{source_name}`")
34
+ if skip_spatial:
35
+ special_locations = None
36
+ detailed_zones = None
57
37
  draw_zones = None
58
-
59
- if special_locations is not None:
60
- assert detailed_zones is not None, (
61
- "Special locations are defined but there is no data on detailed zones"
62
- )
63
- if "detailed_zone_id" not in special_locations.columns:
64
- special_locations = identify_detailed_zone_id(special_locations, detailed_zones)
38
+ else:
39
+ special_locations, detailed_zones, draw_zones = read_spatial_data(source, source_name)
65
40
 
66
41
  if special_locations is not None:
67
42
  special_locations_coords = get_coords(special_locations, "special_location")
@@ -133,6 +108,42 @@ def standardize(source: str | ZipFile):
133
108
  )
134
109
 
135
110
 
111
+ def read_spatial_data(source: str | ZipFile, source_name: str):
112
+ if filename := special_locations_and_detailed_zones_filename(source):
113
+ # Special case for Beauvais 2011.
114
+ detailed_zones, special_locations = read_special_locations_and_detailed_zones(filename)
115
+ else:
116
+ # Special locations.
117
+ if filename := special_locations_filename(source):
118
+ logger.debug(f"Reading special locations from `{filename}`")
119
+ special_locations = read_special_locations(filename)
120
+ else:
121
+ special_locations = None
122
+ # Detailed zones.
123
+ if filename := detailed_zones_filename(source):
124
+ logger.debug(f"Reading detailed zones from `{filename}`")
125
+ detailed_zones = read_detailed_zones(filename)
126
+ else:
127
+ logger.debug(f"No file with detailed zones in `{source_name}`")
128
+ detailed_zones = None
129
+ # Draw zones.
130
+ if filename := draw_zones_filename(source):
131
+ logger.debug(f"Reading draw zones from `{filename}`")
132
+ draw_zones = read_draw_zones(filename)
133
+ else:
134
+ logger.debug(f"No file with draw zones in `{source_name}`")
135
+ draw_zones = None
136
+
137
+ if special_locations is not None:
138
+ assert (
139
+ detailed_zones is not None
140
+ ), "Special locations are defined but there is no data on detailed zones"
141
+ if "detailed_zone_id" not in special_locations.columns:
142
+ special_locations = identify_detailed_zone_id(special_locations, detailed_zones)
143
+
144
+ return special_locations, detailed_zones, draw_zones
145
+
146
+
136
147
  def survey_name(source: str | ZipFile):
137
148
  filename = find_file(source, ".*_std_men.csv", subdir="Csv", as_url=True)
138
149
  return re.match("(.*)_std_men.csv", os.path.basename(filename)).group(1)
@@ -212,15 +223,13 @@ def fix_special_locations_with_gcd(
212
223
  pl.from_pandas(special_locations["detailed_zone_id"].astype(int)),
213
224
  default=None,
214
225
  )
215
- )
216
- .alias(f"{col}_detailed_zone"),
226
+ ).alias(f"{col}_detailed_zone"),
217
227
  # When the ZF is an actual ZF
218
228
  pl.when(mask)
219
229
  # Then the GT is null
220
230
  .then(pl.lit(None))
221
231
  # Otherwise use that GT as GT.
222
- .otherwise(f"{col}_detailed_zone")
223
- .alias(f"{col}_special_location"),
232
+ .otherwise(f"{col}_detailed_zone").alias(f"{col}_special_location"),
224
233
  )
225
234
  return df
226
235
 
@@ -243,15 +252,13 @@ def fix_special_locations(df: pl.LazyFrame, col: str, special_locations: gpd.Geo
243
252
  pl.from_pandas(special_locations["detailed_zone_id"].astype(int)),
244
253
  default=None,
245
254
  )
246
- )
247
- .alias(f"{col}_detailed_zone"),
255
+ ).alias(f"{col}_detailed_zone"),
248
256
  # When the ZF is an actual ZF
249
257
  pl.when(mask)
250
258
  # Then the GT is null
251
259
  .then(pl.lit(None))
252
260
  # Otherwise use that GT as GT.
253
- .otherwise(f"{col}_detailed_zone")
254
- .alias(f"{col}_special_location"),
261
+ .otherwise(f"{col}_detailed_zone").alias(f"{col}_special_location"),
255
262
  )
256
263
  return df
257
264
 
@@ -13,15 +13,18 @@ from .trajets import standardize_legs
13
13
  from .zones import read_detailed_zones
14
14
 
15
15
 
16
- def standardize(source: str | ZipFile):
16
+ def standardize(source: str | ZipFile, skip_spatial: bool = False):
17
17
  source_name = source.filename if isinstance(source, ZipFile) else source
18
18
  logger.info(f"Standardizing EGT2010 survey from `{source_name}`")
19
- # Detailed zones.
20
- filename = detailed_zones_filename(source)
21
- if filename is None:
22
- logger.error("Missing detailed zones file")
23
- return None
24
- detailed_zones = read_detailed_zones(filename)
19
+ if skip_spatial:
20
+ detailed_zones = None
21
+ else:
22
+ # Detailed zones.
23
+ filename = detailed_zones_filename(source)
24
+ if filename is None:
25
+ logger.error("Missing detailed zones file")
26
+ return None
27
+ detailed_zones = read_detailed_zones(filename)
25
28
  # Households.
26
29
  filename = households_filename(source)
27
30
  if filename is None:
@@ -155,7 +155,7 @@ def standardize_legs(
155
155
  trips: pl.LazyFrame,
156
156
  cars: pl.LazyFrame,
157
157
  motorcycles: pl.LazyFrame,
158
- detailed_zones: gpd.GeoDataFrame,
158
+ detailed_zones: gpd.GeoDataFrame | None,
159
159
  ):
160
160
  lf = scan_legs(filename)
161
161
  # Add household_id, person_id, and trip_id.
@@ -13,7 +13,7 @@ from .trajets import standardize_legs
13
13
  from .voitures import standardize_cars
14
14
 
15
15
 
16
- def standardize(source: str | ZipFile):
16
+ def standardize(source: str | ZipFile, skip_spatial: bool = False):
17
17
  source_name = source.filename if isinstance(source, ZipFile) else source
18
18
  logger.info(f"Standardizing EGT2020 survey from `{source_name}`")
19
19
  # Households.
@@ -24,57 +24,24 @@ from .trajets import standardize_legs
24
24
  from .zones import read_detailed_zones, read_draw_zones, read_special_locations
25
25
 
26
26
 
27
- def standardize(source: str | ZipFile):
27
+ def standardize(source: str | ZipFile, skip_spatial: bool = False):
28
28
  source_name = source.filename if isinstance(source, ZipFile) else source
29
29
  logger.info(f"Standardizing EMC2 survey from `{source_name}`")
30
- # Special locations.
31
- filename = special_locations_filename(source)
32
- if filename is None:
30
+ if skip_spatial:
33
31
  special_locations = None
34
- special_locations_coords = None
35
- else:
36
- logger.debug(f"Reading special locations from `{filename}`")
37
- special_locations = read_special_locations(filename)
38
- special_locations_coords = get_coords(special_locations, "special_location")
39
- # Detailed zones.
40
- filename = detailed_zones_filename(source)
41
- if filename is None:
42
- logger.debug(f"No file with detailed zones in `{source_name}`")
43
32
  detailed_zones = None
44
- detailed_zones_coords = None
45
- else:
46
- logger.debug(f"Reading detailed zones from `{filename}`")
47
- detailed_zones = read_detailed_zones(filename)
48
- detailed_zones_coords = get_coords(detailed_zones, "detailed_zone")
49
- # Draw zones.
50
- filename = draw_zones_filename(source)
51
- if filename is None:
52
- logger.debug(f"No file with draw zones in `{source_name}`")
53
33
  draw_zones = None
54
34
  else:
55
- logger.debug(f"Reading draw zones from `{filename}`")
56
- draw_zones = read_draw_zones(filename)
57
-
58
- if special_locations is not None:
59
- # For the Angers 2022 survey, the ids in the detailed zones shapefile have 3 zeros removed
60
- # compared to the # ids in the special locations shapefile.
61
- assert detailed_zones is not None, (
62
- "Special locations are defined but there is no data on detailed zones"
63
- )
64
- assert "detailed_zone_id" in special_locations.columns, (
65
- "Special locations are defined and used but the corresponding detailed zone are unknown"
66
- )
67
- dz_gcd = np.gcd.reduce(detailed_zones["detailed_zone_id"].astype(int))
68
- sl_gcd = np.gcd.reduce(special_locations["detailed_zone_id"].astype(int))
69
- if dz_gcd < sl_gcd and np.log10(sl_gcd).is_integer():
70
- detailed_zones["detailed_zone_id"] = (
71
- detailed_zones["detailed_zone_id"].astype(int) * sl_gcd
72
- ).astype(str)
35
+ special_locations, detailed_zones, draw_zones = read_spatial_data(source, source_name)
73
36
 
74
37
  if special_locations is not None:
75
38
  special_locations_coords = get_coords(special_locations, "special_location")
39
+ else:
40
+ special_locations_coords = None
76
41
  if detailed_zones is not None:
77
42
  detailed_zones_coords = get_coords(detailed_zones, "detailed_zone")
43
+ else:
44
+ detailed_zones_coords = None
78
45
 
79
46
  # Households.
80
47
  filename = households_filename(source)
@@ -112,6 +79,7 @@ def standardize(source: str | ZipFile):
112
79
 
113
80
  # Fix the special locations which are being used as detailed zones.
114
81
  if special_locations is not None:
82
+ assert detailed_zones is not None
115
83
  assert (
116
84
  len(
117
85
  set(special_locations["special_location_id"]).intersection(
@@ -126,6 +94,7 @@ def standardize(source: str | ZipFile):
126
94
  # In some cases, not all GTs are defined.
127
95
  # For example, `origin_detailed_zone` can be defined as 70103 but there is no special
128
96
  # location with that id. In this case, we can guess that the ZF id is 70100.
97
+ sl_gcd = np.gcd.reduce(special_locations["detailed_zone_id"].astype(int))
129
98
  if np.log10(sl_gcd).is_integer() and sl_gcd > 1:
130
99
  # All ZF ids are ending with zeros. In principle, all GT ids do _not_ end with zeros.
131
100
  # We raise an error if at least 2 GT ids end with zeros.
@@ -170,6 +139,50 @@ def standardize(source: str | ZipFile):
170
139
  )
171
140
 
172
141
 
142
+ def read_spatial_data(source: str | ZipFile, source_name: str):
143
+ # Special locations.
144
+ filename = special_locations_filename(source)
145
+ if filename is None:
146
+ special_locations = None
147
+ else:
148
+ logger.debug(f"Reading special locations from `{filename}`")
149
+ special_locations = read_special_locations(filename)
150
+ # Detailed zones.
151
+ filename = detailed_zones_filename(source)
152
+ if filename is None:
153
+ logger.debug(f"No file with detailed zones in `{source_name}`")
154
+ detailed_zones = None
155
+ else:
156
+ logger.debug(f"Reading detailed zones from `{filename}`")
157
+ detailed_zones = read_detailed_zones(filename)
158
+ # Draw zones.
159
+ filename = draw_zones_filename(source)
160
+ if filename is None:
161
+ logger.debug(f"No file with draw zones in `{source_name}`")
162
+ draw_zones = None
163
+ else:
164
+ logger.debug(f"Reading draw zones from `{filename}`")
165
+ draw_zones = read_draw_zones(filename)
166
+
167
+ if special_locations is not None:
168
+ # For the Angers 2022 survey, the ids in the detailed zones shapefile have 3 zeros removed
169
+ # compared to the # ids in the special locations shapefile.
170
+ assert (
171
+ detailed_zones is not None
172
+ ), "Special locations are defined but there is no data on detailed zones"
173
+ assert (
174
+ "detailed_zone_id" in special_locations.columns
175
+ ), "Special locations are defined and used but the corresponding detailed zone are unknown"
176
+ dz_gcd = np.gcd.reduce(detailed_zones["detailed_zone_id"].astype(int))
177
+ sl_gcd = np.gcd.reduce(special_locations["detailed_zone_id"].astype(int))
178
+ if dz_gcd < sl_gcd and np.log10(sl_gcd).is_integer():
179
+ detailed_zones["detailed_zone_id"] = (
180
+ detailed_zones["detailed_zone_id"].astype(int) * sl_gcd
181
+ ).astype(str)
182
+
183
+ return special_locations, detailed_zones, draw_zones
184
+
185
+
173
186
  def survey_name(source: str | ZipFile):
174
187
  filename = find_file(
175
188
  source, ".*_std_men.csv", subdir=os.path.join("Csv", "Fichiers_Standard"), as_url=True
@@ -238,15 +251,13 @@ def fix_special_locations_with_gcd(
238
251
  pl.from_pandas(special_locations["detailed_zone_id"].astype(int)),
239
252
  default=None,
240
253
  )
241
- )
242
- .alias(f"{col}_detailed_zone"),
254
+ ).alias(f"{col}_detailed_zone"),
243
255
  # When the ZF is an actual ZF
244
256
  pl.when(mask)
245
257
  # Then the GT is null
246
258
  .then(pl.lit(None))
247
259
  # Otherwise use that GT as GT.
248
- .otherwise(f"{col}_detailed_zone")
249
- .alias(f"{col}_special_location"),
260
+ .otherwise(f"{col}_detailed_zone").alias(f"{col}_special_location"),
250
261
  )
251
262
  return df
252
263
 
@@ -269,14 +280,12 @@ def fix_special_locations(df: pl.LazyFrame, col: str, special_locations: gpd.Geo
269
280
  pl.from_pandas(special_locations["detailed_zone_id"].astype(int)),
270
281
  default=None,
271
282
  )
272
- )
273
- .alias(f"{col}_detailed_zone"),
283
+ ).alias(f"{col}_detailed_zone"),
274
284
  # When the ZF is an actual ZF
275
285
  pl.when(mask)
276
286
  # Then the GT is null
277
287
  .then(pl.lit(None))
278
288
  # Otherwise use that GT as GT.
279
- .otherwise(f"{col}_detailed_zone")
280
- .alias(f"{col}_special_location"),
289
+ .otherwise(f"{col}_detailed_zone").alias(f"{col}_special_location"),
281
290
  )
282
291
  return df