mobisurvstd 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. mobisurvstd/__init__.py +8 -0
  2. mobisurvstd/__main__.py +4 -0
  3. mobisurvstd/classes.py +344 -0
  4. mobisurvstd/clean.py +1661 -0
  5. mobisurvstd/cli.py +44 -0
  6. mobisurvstd/common/__init__.py +1 -0
  7. mobisurvstd/common/cars.py +120 -0
  8. mobisurvstd/common/clean.py +470 -0
  9. mobisurvstd/common/households.py +97 -0
  10. mobisurvstd/common/legs.py +136 -0
  11. mobisurvstd/common/modes.py +52 -0
  12. mobisurvstd/common/motorcycles.py +44 -0
  13. mobisurvstd/common/persons.py +228 -0
  14. mobisurvstd/common/trips.py +253 -0
  15. mobisurvstd/common/zones.py +43 -0
  16. mobisurvstd/egt2010/__init__.py +1 -0
  17. mobisurvstd/egt2010/deplacements.py +170 -0
  18. mobisurvstd/egt2010/menages.py +378 -0
  19. mobisurvstd/egt2010/personnes.py +307 -0
  20. mobisurvstd/egt2010/survey.py +84 -0
  21. mobisurvstd/egt2010/trajets.py +211 -0
  22. mobisurvstd/egt2010/zones.py +11 -0
  23. mobisurvstd/egt2020/__init__.py +1 -0
  24. mobisurvstd/egt2020/deplacements.py +173 -0
  25. mobisurvstd/egt2020/menages.py +162 -0
  26. mobisurvstd/egt2020/motos.py +103 -0
  27. mobisurvstd/egt2020/personnes.py +391 -0
  28. mobisurvstd/egt2020/survey.py +89 -0
  29. mobisurvstd/egt2020/trajets.py +317 -0
  30. mobisurvstd/egt2020/voitures.py +128 -0
  31. mobisurvstd/emc2/__init__.py +1 -0
  32. mobisurvstd/emc2/common.py +42 -0
  33. mobisurvstd/emc2/deplacements.py +153 -0
  34. mobisurvstd/emc2/menages.py +358 -0
  35. mobisurvstd/emc2/personnes.py +305 -0
  36. mobisurvstd/emc2/survey.py +309 -0
  37. mobisurvstd/emc2/trajets.py +274 -0
  38. mobisurvstd/emc2/zones.py +107 -0
  39. mobisurvstd/emp/__init__.py +1 -0
  40. mobisurvstd/emp/deplacements.py +407 -0
  41. mobisurvstd/emp/menages.py +182 -0
  42. mobisurvstd/emp/motos.py +126 -0
  43. mobisurvstd/emp/personnes.py +543 -0
  44. mobisurvstd/emp/survey.py +93 -0
  45. mobisurvstd/emp/voitures.py +218 -0
  46. mobisurvstd/logger.py +15 -0
  47. mobisurvstd/main.py +158 -0
  48. mobisurvstd/resources/__init__.py +0 -0
  49. mobisurvstd/resources/admin_express.py +88 -0
  50. mobisurvstd/resources/insee_data.py +307 -0
  51. mobisurvstd/resources/nuts.py +129 -0
  52. mobisurvstd/schema.py +902 -0
  53. mobisurvstd/utils.py +125 -0
  54. mobisurvstd-0.1.0.dist-info/METADATA +210 -0
  55. mobisurvstd-0.1.0.dist-info/RECORD +58 -0
  56. mobisurvstd-0.1.0.dist-info/WHEEL +5 -0
  57. mobisurvstd-0.1.0.dist-info/licenses/LICENSE.md +7 -0
  58. mobisurvstd-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,8 @@
1
+ from .classes import SurveyDataReader as SurveyDataReader
2
+ from .classes import read_many as read_many
3
+ from .logger import setup as setup
4
+ from .main import bulk_standardize as bulk_standardize
5
+ from .main import standardize as standardize
6
+
7
+ # Initialize logging.
8
+ setup()
@@ -0,0 +1,4 @@
1
+ if __name__ == "__main__":
2
+ from .cli import app
3
+
4
+ app()
mobisurvstd/classes.py ADDED
@@ -0,0 +1,344 @@
1
+ import functools
2
+ import json
3
+ import os
4
+ from collections.abc import Callable
5
+ from datetime import date
6
+
7
+ import geopandas as gpd
8
+ import polars as pl
9
+ from loguru import logger
10
+
11
+ from .clean import clean
12
+
13
+
14
+ class SurveyData:
15
+ """Data structure to hold the standardized survey data."""
16
+
17
+ def __init__(
18
+ self,
19
+ households: pl.DataFrame,
20
+ cars: pl.DataFrame,
21
+ motorcycles: pl.DataFrame,
22
+ persons: pl.DataFrame,
23
+ trips: pl.DataFrame,
24
+ legs: pl.DataFrame,
25
+ special_locations: gpd.GeoDataFrame | None,
26
+ detailed_zones: gpd.GeoDataFrame | None,
27
+ draw_zones: gpd.GeoDataFrame | None,
28
+ metadata: dict,
29
+ ):
30
+ self.households = households
31
+ self.cars = cars
32
+ self.motorcycles = motorcycles
33
+ self.persons = persons
34
+ self.trips = trips
35
+ self.legs = legs
36
+ self.special_locations = special_locations
37
+ self.detailed_zones = detailed_zones
38
+ self.draw_zones = draw_zones
39
+ self.metadata = metadata
40
+
41
+ @classmethod
42
+ def from_dict(cls, data: dict):
43
+ return cls(
44
+ data["households"],
45
+ data["cars"],
46
+ data["motorcycles"],
47
+ data["persons"],
48
+ data["trips"],
49
+ data["legs"],
50
+ data.get("special_locations"),
51
+ data.get("detailed_zones"),
52
+ data.get("draw_zones"),
53
+ data["metadata"],
54
+ )
55
+
56
+ def clean(self):
57
+ data = clean(
58
+ households=self.households,
59
+ cars=self.cars,
60
+ motorcycles=self.motorcycles,
61
+ persons=self.persons,
62
+ trips=self.trips,
63
+ legs=self.legs,
64
+ special_locations=self.special_locations,
65
+ detailed_zones=self.detailed_zones,
66
+ draw_zones=self.draw_zones,
67
+ )
68
+ data["metadata"] = self.metadata
69
+ return self.__class__.from_dict(data)
70
+
71
+ def save(self, output_directory: str):
72
+ """Saves the MobiSurvStd survey to the given directory."""
73
+ logger.info(f"Saving standardized survey data to `{output_directory}`")
74
+ if not os.path.isdir(output_directory):
75
+ logger.debug(f"Creating missing directory: `{output_directory}`")
76
+ os.makedirs(output_directory)
77
+ elif os.listdir(output_directory):
78
+ # Directory is not empty.
79
+ logger.warning("Output directory is not empty, some data might be erased")
80
+ logger.debug("Saving households")
81
+ self.households.write_parquet(os.path.join(output_directory, "households.parquet"))
82
+ logger.debug("Saving cars")
83
+ self.cars.write_parquet(os.path.join(output_directory, "cars.parquet"))
84
+ logger.debug("Saving motorcycles")
85
+ self.motorcycles.write_parquet(os.path.join(output_directory, "motorcycles.parquet"))
86
+ logger.debug("Saving persons")
87
+ self.persons.write_parquet(os.path.join(output_directory, "persons.parquet"))
88
+ logger.debug("Saving trips")
89
+ self.trips.write_parquet(os.path.join(output_directory, "trips.parquet"))
90
+ logger.debug("Saving legs")
91
+ self.legs.write_parquet(os.path.join(output_directory, "legs.parquet"))
92
+ if self.special_locations is not None:
93
+ logger.debug("Saving special locations")
94
+ self.special_locations.to_parquet(
95
+ os.path.join(output_directory, "special_locations.geo.parquet")
96
+ )
97
+ if self.detailed_zones is not None:
98
+ logger.debug("Saving detailed zones")
99
+ self.detailed_zones.to_parquet(
100
+ os.path.join(output_directory, "detailed_zones.geo.parquet")
101
+ )
102
+ if self.draw_zones is not None:
103
+ logger.debug("Saving draw zones")
104
+ self.draw_zones.to_parquet(os.path.join(output_directory, "draw_zones.geo.parquet"))
105
+ with open(os.path.join(output_directory, "metadata.json"), "w") as f:
106
+ logger.debug("Saving metadata")
107
+ json.dump(self.metadata, f, indent=2)
108
+
109
+
110
+ class SurveyDataReader:
111
+ """Data structure representing a MobiSurvStd survey.
112
+
113
+ Parameters
114
+ ----------
115
+ directory: str
116
+ Path to the directory where the MobiSurvStd survey to read is stored.
117
+
118
+ Examples
119
+ --------
120
+ Create a SurveyDataReader from a directory.
121
+
122
+ >>> data = SurveyDataReader("output/emp2019/")
123
+
124
+ Access the survey's metadata as a dictionary:
125
+
126
+ >>> data.metadata
127
+ {'type': 'EMP2019',
128
+ 'survey_method': 'face_to_face',
129
+ 'nb_households': 13825,
130
+ 'nb_cars': 18817,
131
+ 'nb_motorcycles': 1264,
132
+ 'nb_persons': 31694,
133
+ 'nb_trips': 45169,
134
+ 'nb_legs': 46507,
135
+ 'nb_special_locations': 0,
136
+ 'nb_detailed_zones': 0,
137
+ 'nb_draw_zones': 0,
138
+ 'start_date': '2018-05-01',
139
+ 'end_date': '2019-04-30',
140
+ 'insee': None}
141
+
142
+ Access the survey's households as a polars.DataFrame:
143
+
144
+ >>> data.households
145
+ ┌─────────────┬─────────────┬────────────┬────────────┬───┬────────────┬────────────┬───────────┬───────────┐
146
+ │ household_i ┆ original_ho ┆ survey_met ┆ interview_ ┆ … ┆ nb_persons ┆ nb_persons ┆ nb_majors ┆ nb_minors │
147
+ │ d ┆ usehold_id ┆ hod ┆ date ┆ ┆ --- ┆ _5plus ┆ --- ┆ --- │
148
+ │ --- ┆ --- ┆ --- ┆ --- ┆ ┆ u8 ┆ --- ┆ u8 ┆ u8 │
149
+ │ u32 ┆ struct[1] ┆ enum ┆ date ┆ ┆ ┆ u8 ┆ ┆ │
150
+ ╞═════════════╪═════════════╪════════════╪════════════╪═══╪════════════╪════════════╪═══════════╪═══════════╡
151
+ │ 1 ┆ {"110000011 ┆ face_to_fa ┆ null ┆ … ┆ 1 ┆ 1 ┆ 1 ┆ 0 │
152
+ │ ┆ 4000"} ┆ ce ┆ ┆ ┆ ┆ ┆ ┆ │
153
+ │ 2 ┆ {"110000011 ┆ face_to_fa ┆ null ┆ … ┆ 4 ┆ null ┆ 3 ┆ 1 │
154
+ │ ┆ 5000"} ┆ ce ┆ ┆ ┆ ┆ ┆ ┆ │
155
+ │ 3 ┆ {"110000011 ┆ face_to_fa ┆ null ┆ … ┆ 2 ┆ null ┆ 2 ┆ 0 │
156
+ │ ┆ 6000"} ┆ ce ┆ ┆ ┆ ┆ ┆ ┆ │
157
+ │ 4 ┆ {"110000012 ┆ face_to_fa ┆ null ┆ … ┆ 2 ┆ null ┆ 2 ┆ 0 │
158
+ │ ┆ 4000"} ┆ ce ┆ ┆ ┆ ┆ ┆ ┆ │
159
+ │ 5 ┆ {"110000012 ┆ face_to_fa ┆ null ┆ … ┆ 2 ┆ null ┆ 2 ┆ 0 │
160
+ │ ┆ 5000"} ┆ ce ┆ ┆ ┆ ┆ ┆ ┆ │
161
+ │ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … │
162
+ │ 13821 ┆ {"940000036 ┆ face_to_fa ┆ null ┆ … ┆ 1 ┆ 1 ┆ 1 ┆ 0 │
163
+ │ ┆ 1000"} ┆ ce ┆ ┆ ┆ ┆ ┆ ┆ │
164
+ │ 13822 ┆ {"940000036 ┆ face_to_fa ┆ null ┆ … ┆ 1 ┆ 1 ┆ 1 ┆ 0 │
165
+ │ ┆ 4000"} ┆ ce ┆ ┆ ┆ ┆ ┆ ┆ │
166
+ │ 13823 ┆ {"940000041 ┆ face_to_fa ┆ null ┆ … ┆ 2 ┆ null ┆ 2 ┆ 0 │
167
+ │ ┆ 5000"} ┆ ce ┆ ┆ ┆ ┆ ┆ ┆ │
168
+ │ 13824 ┆ {"940000044 ┆ face_to_fa ┆ null ┆ … ┆ 1 ┆ 1 ┆ 1 ┆ 0 │
169
+ │ ┆ 1000"} ┆ ce ┆ ┆ ┆ ┆ ┆ ┆ │
170
+ │ 13825 ┆ {"940000052 ┆ face_to_fa ┆ null ┆ … ┆ 1 ┆ 1 ┆ 1 ┆ 0 │
171
+ │ ┆ 1000"} ┆ ce ┆ ┆ ┆ ┆ ┆ ┆ │
172
+ └─────────────┴─────────────┴────────────┴────────────┴───┴────────────┴────────────┴───────────┴───────────┘
173
+ """
174
+
175
+ def __init__(self, directory: str):
176
+ self.directory = directory
177
+ self._households = None
178
+ self._cars = None
179
+ self._motorcycles = None
180
+ self._persons = None
181
+ self._trips = None
182
+ self._legs = None
183
+ self._special_locations = None
184
+ self._detailed_zones = None
185
+ self._draw_zones = None
186
+ self._metadata = None
187
+
188
+ @property
189
+ def metadata(self) -> dict:
190
+ if self._metadata is None:
191
+ filename = os.path.join(self.directory, "metadata.json")
192
+ logger.debug(f"Reading metadata from `{filename}`")
193
+ self._metadata = json.load(open(filename))
194
+ return self._metadata
195
+
196
+ @property
197
+ def households(self) -> pl.DataFrame:
198
+ if self._households is None:
199
+ filename = os.path.join(self.directory, "households.parquet")
200
+ logger.debug(f"Reading households from `{filename}`")
201
+ self._households = pl.read_parquet(filename)
202
+ return self._households
203
+
204
+ @property
205
+ def cars(self) -> pl.DataFrame:
206
+ if self._cars is None:
207
+ filename = os.path.join(self.directory, "cars.parquet")
208
+ logger.debug(f"Reading cars from `{filename}`")
209
+ self._cars = pl.read_parquet(filename)
210
+ return self._cars
211
+
212
+ @property
213
+ def motorcycles(self) -> pl.DataFrame:
214
+ if self._motorcycles is None:
215
+ filename = os.path.join(self.directory, "motorcycles.parquet")
216
+ logger.debug(f"Reading motorcycles from `{filename}`")
217
+ self._motorcycles = pl.read_parquet(filename)
218
+ return self._motorcycles
219
+
220
+ @property
221
+ def persons(self) -> pl.DataFrame:
222
+ if self._persons is None:
223
+ filename = os.path.join(self.directory, "persons.parquet")
224
+ logger.debug(f"Reading persons from `{filename}`")
225
+ self._persons = pl.read_parquet(filename)
226
+ return self._persons
227
+
228
+ @property
229
+ def trips(self) -> pl.DataFrame:
230
+ if self._trips is None:
231
+ filename = os.path.join(self.directory, "trips.parquet")
232
+ logger.debug(f"Reading trips from `{filename}`")
233
+ self._trips = pl.read_parquet(filename)
234
+ return self._trips
235
+
236
+ @property
237
+ def legs(self) -> pl.DataFrame:
238
+ if self._legs is None:
239
+ filename = os.path.join(self.directory, "legs.parquet")
240
+ logger.debug(f"Reading legs from `{filename}`")
241
+ self._legs = pl.read_parquet(filename)
242
+ return self._legs
243
+
244
+ @property
245
+ def special_locations(self) -> gpd.GeoDataFrame:
246
+ if self._special_locations is None:
247
+ filename = os.path.join(self.directory, "special_locations.geo.parquet")
248
+ if os.path.isfile(filename):
249
+ logger.debug(f"Reading special locations from `{filename}`")
250
+ self._special_locations = pl.read_parquet(filename)
251
+ return self._special_locations
252
+
253
+ @property
254
+ def detailed_zones(self) -> gpd.GeoDataFrame:
255
+ if self._detailed_zones is None:
256
+ filename = os.path.join(self.directory, "detailed_zones.geo.parquet")
257
+ if os.path.isfile(filename):
258
+ logger.debug(f"Reading detailed zones from `{filename}`")
259
+ self._detailed_zones = pl.read_parquet(filename)
260
+ return self._detailed_zones
261
+
262
+ @property
263
+ def draw_zones(self) -> gpd.GeoDataFrame:
264
+ if self._draw_zones is None:
265
+ filename = os.path.join(self.directory, "draw_zones.geo.parquet")
266
+ if os.path.isfile(filename):
267
+ logger.debug(f"Reading draw zones from `{filename}`")
268
+ self._draw_zones = pl.read_parquet(filename)
269
+ return self._draw_zones
270
+
271
+ def mean_date(self) -> date:
272
+ """Returns the mean date at which the survey was conducted (the middle date between the
273
+ survey's start and end date).
274
+ """
275
+ start_date = date.fromisoformat(self.metadata["start_date"])
276
+ end_date = date.fromisoformat(self.metadata["end_date"])
277
+ return start_date + (end_date - start_date) / 2
278
+
279
+
280
+ def iter_many(directory: str):
281
+ """Iterates over the MobiSurvStd data located within a directory (potentially recursively)."""
282
+ if not os.path.isdir(directory):
283
+ raise Exception(f"Not a valid directory: `{directory}`")
284
+ for name in os.listdir(directory):
285
+ item = os.path.join(directory, name)
286
+ if not os.path.isdir(item):
287
+ logger.debug(f"Skipping non-directory: `{item}`")
288
+ continue
289
+ if not is_valid_mobisurvstd_dir(item):
290
+ # The current item is a directory but does not contain valid MobiSurvStd data.
291
+ # We go recursively to read MobiSurvStd directories within that directory (if any).
292
+ logger.debug(f"Recursively reading data from `{item}`")
293
+ yield from iter_many(item)
294
+ continue
295
+ logger.debug(f"Found a valid MobiSurvStd directory at `{item}`")
296
+ data = SurveyDataReader(item)
297
+ yield data
298
+
299
+
300
+ def read_many(directory: str, read_fn: Callable, acc_fn: Callable):
301
+ """Runs a function on all MobiSurvStd surveys found in a directory and aggregates the results.
302
+
303
+ Parameters
304
+ ----------
305
+ directory
306
+ Path to the directory where the MobiSurvStd surveys to read are stored.
307
+ The directory will be read recursively so the surveys can be stored in subdirectories.
308
+ read_fn
309
+ Function to be run on each survey.
310
+ It takes a single argument whose type is `SurveyDataReader`.
311
+ acc_fn
312
+ Function to aggregate the results from two surveys.
313
+ This must be a function of two arguments, whose type is the same as the return type of
314
+ `read_fn`.
315
+
316
+ Examples
317
+ --------
318
+ Read the total number of households from all surveys in the "output" directory:
319
+
320
+ >>> read_many("output/", lambda d: len(d.households), lambda x, y: x + y)
321
+
322
+ Concatenate all trips in a single DataFrame:
323
+
324
+ >>> read_many("output/", lambda d: d.trips, lambda x, y: pl.concat((x, y)))
325
+ """
326
+ return functools.reduce(
327
+ acc_fn, filter(lambda r: r is not None, map(read_fn, iter_many(directory)))
328
+ )
329
+
330
+
331
+ def is_valid_mobisurvstd_dir(directory: str) -> bool:
332
+ """Checks if a directory contains all the mandatory MobiSurvStd files."""
333
+ for name in (
334
+ "metadata.json",
335
+ "households.parquet",
336
+ "persons.parquet",
337
+ "trips.parquet",
338
+ "legs.parquet",
339
+ "cars.parquet",
340
+ "motorcycles.parquet",
341
+ ):
342
+ if not os.path.isfile(os.path.join(directory, name)):
343
+ return False
344
+ return True