mobisurvstd 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. mobisurvstd/__init__.py +8 -0
  2. mobisurvstd/__main__.py +4 -0
  3. mobisurvstd/classes.py +344 -0
  4. mobisurvstd/clean.py +1661 -0
  5. mobisurvstd/cli.py +44 -0
  6. mobisurvstd/common/__init__.py +1 -0
  7. mobisurvstd/common/cars.py +120 -0
  8. mobisurvstd/common/clean.py +470 -0
  9. mobisurvstd/common/households.py +97 -0
  10. mobisurvstd/common/legs.py +136 -0
  11. mobisurvstd/common/modes.py +52 -0
  12. mobisurvstd/common/motorcycles.py +44 -0
  13. mobisurvstd/common/persons.py +228 -0
  14. mobisurvstd/common/trips.py +253 -0
  15. mobisurvstd/common/zones.py +43 -0
  16. mobisurvstd/egt2010/__init__.py +1 -0
  17. mobisurvstd/egt2010/deplacements.py +170 -0
  18. mobisurvstd/egt2010/menages.py +378 -0
  19. mobisurvstd/egt2010/personnes.py +307 -0
  20. mobisurvstd/egt2010/survey.py +84 -0
  21. mobisurvstd/egt2010/trajets.py +211 -0
  22. mobisurvstd/egt2010/zones.py +11 -0
  23. mobisurvstd/egt2020/__init__.py +1 -0
  24. mobisurvstd/egt2020/deplacements.py +173 -0
  25. mobisurvstd/egt2020/menages.py +162 -0
  26. mobisurvstd/egt2020/motos.py +103 -0
  27. mobisurvstd/egt2020/personnes.py +391 -0
  28. mobisurvstd/egt2020/survey.py +89 -0
  29. mobisurvstd/egt2020/trajets.py +317 -0
  30. mobisurvstd/egt2020/voitures.py +128 -0
  31. mobisurvstd/emc2/__init__.py +1 -0
  32. mobisurvstd/emc2/common.py +42 -0
  33. mobisurvstd/emc2/deplacements.py +153 -0
  34. mobisurvstd/emc2/menages.py +358 -0
  35. mobisurvstd/emc2/personnes.py +305 -0
  36. mobisurvstd/emc2/survey.py +309 -0
  37. mobisurvstd/emc2/trajets.py +274 -0
  38. mobisurvstd/emc2/zones.py +107 -0
  39. mobisurvstd/emp/__init__.py +1 -0
  40. mobisurvstd/emp/deplacements.py +407 -0
  41. mobisurvstd/emp/menages.py +182 -0
  42. mobisurvstd/emp/motos.py +126 -0
  43. mobisurvstd/emp/personnes.py +543 -0
  44. mobisurvstd/emp/survey.py +93 -0
  45. mobisurvstd/emp/voitures.py +218 -0
  46. mobisurvstd/logger.py +15 -0
  47. mobisurvstd/main.py +158 -0
  48. mobisurvstd/resources/__init__.py +0 -0
  49. mobisurvstd/resources/admin_express.py +88 -0
  50. mobisurvstd/resources/insee_data.py +307 -0
  51. mobisurvstd/resources/nuts.py +129 -0
  52. mobisurvstd/schema.py +902 -0
  53. mobisurvstd/utils.py +125 -0
  54. mobisurvstd-0.1.0.dist-info/METADATA +210 -0
  55. mobisurvstd-0.1.0.dist-info/RECORD +58 -0
  56. mobisurvstd-0.1.0.dist-info/WHEEL +5 -0
  57. mobisurvstd-0.1.0.dist-info/licenses/LICENSE.md +7 -0
  58. mobisurvstd-0.1.0.dist-info/top_level.txt +1 -0
mobisurvstd/clean.py ADDED
@@ -0,0 +1,1661 @@
1
+ import itertools
2
+
3
+ import geopandas as gpd
4
+ import polars as pl
5
+
6
+ from .common.modes import MODE_TO_GROUP
7
+ from .common.persons import PCS_CODES
8
+
9
+ # Regular expression to match INSEE municipality codes.
10
+ INSEE_REGEX = "^[0-9][0-9AB][0-9][0-9][0-9]$"
11
+ # Minimum authorized age between a parent and their child.
12
+ CHILD_AGE_THRS = 14
13
+ # Check if the origin (purpose, location) matches the destination of the previous trip: "yes",
14
+ # "warn", or "no".
15
+ CHECK_ORIGIN_MATCHES_PREV_DEST = "warn"
16
+ # Check if the origin special location matches the destination special location of the previous leg:
17
+ # "yes", "warn", or "no".
18
+ CHECK_START_MATCHED_PREV_END = "warn"
19
+ PCS2020_CODES = (
20
+ 10,
21
+ 21,
22
+ 22,
23
+ 23,
24
+ 31,
25
+ 33,
26
+ 34,
27
+ 35,
28
+ 37,
29
+ 38,
30
+ 42,
31
+ 43,
32
+ 44,
33
+ 45,
34
+ 46,
35
+ 47,
36
+ 47,
37
+ 52,
38
+ 53,
39
+ 54,
40
+ 55,
41
+ 56,
42
+ 62,
43
+ 63,
44
+ 64,
45
+ 65,
46
+ 67,
47
+ 68,
48
+ 69,
49
+ )
50
+ PCS2003_CODES = (
51
+ 10,
52
+ 21,
53
+ 22,
54
+ 23,
55
+ 31,
56
+ 32,
57
+ 36,
58
+ 41,
59
+ 46,
60
+ 47,
61
+ 48,
62
+ 51,
63
+ 54,
64
+ 55,
65
+ 56,
66
+ 61,
67
+ 66,
68
+ 69,
69
+ 71,
70
+ 72,
71
+ 73,
72
+ 76,
73
+ 81,
74
+ 82,
75
+ )
76
+
77
+
78
+ def all_null_or_all_defined(s: pl.Series, msg: str):
79
+ assert s.is_null().all() or s.is_not_null().all(), msg
80
+
81
+
82
+ def all_defined(df: pl.DataFrame, col: str):
83
+ assert df[col].is_not_null().all(), f"Column `{col}` is not always non-null"
84
+
85
+
86
+ def all_null_for(df: pl.DataFrame, col: str, cond: pl.Expr, cond_str: str, force=False):
87
+ mask = cond & pl.col(col).is_not_null()
88
+ n = df.filter(mask).select(pl.len()).item()
89
+ if n == 0:
90
+ return
91
+ if force:
92
+ print(f"Warning. For {n} {cond_str}, value of `{col}` is not null.")
93
+ print(f"The `{col}` value is automatically changed to null.")
94
+ df = df.with_columns(pl.when(mask).then(pl.lit(None)).otherwise(pl.col(col)).alias(col))
95
+ else:
96
+ raise AssertionError(f"There are some {cond_str} with a non-null value for `{col}`")
97
+
98
+
99
+ def all_defined_for(df: pl.DataFrame, col: str, cond: pl.Expr, cond_str: str):
100
+ assert df.select((cond & pl.col(col).is_null()).any().not_()).item(), (
101
+ f"There are some {cond_str} with a null value for `{col}`"
102
+ )
103
+
104
+
105
+ def is_sorted(df: pl.DataFrame, col: str):
106
+ assert df[col].is_sorted(), f"Values of `{col}` are not sorted in increasing order"
107
+
108
+
109
+ def valid_insee_codes(df: pl.DataFrame, col: str):
110
+ assert df[col].str.contains(INSEE_REGEX).all(), (
111
+ f"Column `{col}` contains invalid INSEE municipalities"
112
+ )
113
+
114
+
115
+ def same_as_next_in_group(
116
+ df: pl.DataFrame, col_prev: str, col_next: str, group_col: str, warn_only=False
117
+ ):
118
+ # We use `ne_missing` to handle cases where prev is null or next is defined (or the reverse).
119
+ # The second condition means that we don't consider the last row in the group (where next is by
120
+ # definition null).
121
+ nb_invalids = df.select(
122
+ (
123
+ pl.col(col_prev).ne_missing(pl.col(col_next).shift(-1).over(group_col))
124
+ & (pl.int_range(1, pl.len() + 1) != pl.len()).over(group_col)
125
+ ).sum()
126
+ ).item()
127
+ if nb_invalids == 0:
128
+ return
129
+ if warn_only:
130
+ print(
131
+ f"Warning. `{col_prev}` does not match the next `{col_next}` over `{group_col}` for "
132
+ f"{nb_invalids} values"
133
+ )
134
+ else:
135
+ raise AssertionError(f"`{col_prev}` does not match the next `{col_next}`")
136
+
137
+
138
+ def perimeter_check(trips: pl.DataFrame, ids: set, zone: str):
139
+ n = (
140
+ trips.filter(
141
+ pl.col("trip_perimeter") == "internal",
142
+ (
143
+ pl.col(f"origin_{zone}_zone").is_in(ids)
144
+ & pl.col(f"destination_{zone}_zone").is_in(ids)
145
+ ).not_(),
146
+ )
147
+ .select(pl.len())
148
+ .item()
149
+ )
150
+ if n > 0:
151
+ print(
152
+ f"Warning. For {n} trips, origin or destination is not in the survey's {zone} zone ids "
153
+ 'with `trip_perimeter` equal to "internal"'
154
+ )
155
+ n = (
156
+ trips.filter(
157
+ pl.col("trip_perimeter") == "crossing",
158
+ (
159
+ pl.col(f"origin_{zone}_zone").is_in(ids)
160
+ | pl.col(f"destination_{zone}_zone").is_in(ids)
161
+ ).not_(),
162
+ )
163
+ .select(pl.len())
164
+ .item()
165
+ )
166
+ if n > 0:
167
+ print(
168
+ f"Warning. For {n} trips, origin and destination are both not in the survey's {zone} "
169
+ 'zone ids with `trip_perimeter` equal to "crossing"'
170
+ )
171
+ n = (
172
+ trips.filter(
173
+ pl.col("trip_perimeter") == "crossing",
174
+ pl.col(f"origin_{zone}_zone").is_in(ids),
175
+ pl.col(f"destination_{zone}_zone").is_in(ids),
176
+ )
177
+ .select(pl.len())
178
+ .item()
179
+ )
180
+ if n > 0:
181
+ print(
182
+ f"Warning. For {n} trips, origin and destination are both in the survey's {zone} zone "
183
+ 'ids with `trip_perimeter` equal to "crossing"'
184
+ )
185
+ n = (
186
+ trips.filter(
187
+ pl.col("trip_perimeter") == "external",
188
+ pl.col(f"origin_{zone}_zone").is_in(ids)
189
+ | pl.col(f"destination_{zone}_zone").is_in(ids),
190
+ )
191
+ .select(pl.len())
192
+ .item()
193
+ )
194
+ if n > 0:
195
+ print(
196
+ f"Warning. For {n} trips, origin or destination is in the survey's {zone} zone ids "
197
+ 'with `trip_perimeter` equal to "external"'
198
+ )
199
+
200
+
201
+ def intersect_check(
202
+ df: pl.DataFrame,
203
+ intersect_df: pl.DataFrame,
204
+ col_prefix: str,
205
+ zone1: str,
206
+ zone2: str,
207
+ id_col: str,
208
+ ):
209
+ invalids = (
210
+ df.select(id_col, f"{col_prefix}_{zone1}", f"{col_prefix}_{zone2}")
211
+ # Only run the check for the rows were both columns correspond to zones whose intersection
212
+ # is observed.
213
+ .filter(
214
+ pl.col(f"{col_prefix}_{zone1}").is_in(intersect_df[f"{zone1}_id"]),
215
+ pl.col(f"{col_prefix}_{zone2}").is_in(intersect_df[f"{zone2}_id"]),
216
+ )
217
+ .join(
218
+ intersect_df,
219
+ left_on=[f"{col_prefix}_{zone1}", f"{col_prefix}_{zone2}"],
220
+ right_on=[f"{zone1}_id", f"{zone2}_id"],
221
+ how="anti",
222
+ )
223
+ )
224
+ if not invalids.is_empty():
225
+ n = len(invalids)
226
+ print(
227
+ f"Warning. {n} values of `{col_prefix}_{zone1}` correspond to zones which do not "
228
+ f"intersect with the zone from `{col_prefix}_{zone2}`"
229
+ )
230
+ matching_zones = intersect_df.filter(
231
+ pl.col(f"{zone1}_id").is_in(invalids[f"{col_prefix}_{zone1}"])
232
+ )
233
+ assert (
234
+ matching_zones[f"{zone1}_id"].n_unique() == invalids[f"{col_prefix}_{zone1}"].n_unique()
235
+ ), "Cannot reconcile bad zone intersections"
236
+ # Note. The `default=None` in the `replace_strict` is only here to provide some value for
237
+ # the rows not matched in the `pl.when`. For the matched value, the default should not be
238
+ # used.
239
+ df = df.with_columns(
240
+ pl.when(pl.col(id_col).is_in(invalids[id_col]))
241
+ .then(
242
+ pl.col(f"{col_prefix}_{zone1}").replace_strict(
243
+ matching_zones[f"{zone1}_id"], matching_zones[f"{zone2}_id"], default=None
244
+ )
245
+ )
246
+ .otherwise(f"{col_prefix}_{zone2}")
247
+ .alias(f"{col_prefix}_{zone2}")
248
+ )
249
+ print("The values have been automatically fixed")
250
+ return df
251
+
252
+
253
+ def clean(
254
+ households: pl.DataFrame,
255
+ cars: pl.DataFrame,
256
+ motorcycles: pl.DataFrame,
257
+ persons: pl.DataFrame,
258
+ trips: pl.DataFrame,
259
+ legs: pl.DataFrame,
260
+ special_locations: gpd.GeoDataFrame | None,
261
+ detailed_zones: gpd.GeoDataFrame | None,
262
+ draw_zones: gpd.GeoDataFrame | None,
263
+ insee_zones: gpd.GeoDataFrame | None,
264
+ ):
265
+ trips_with_persons = trips.lazy().join(persons.lazy(), on="person_id", how="left")
266
+ legs_with_trips = legs.lazy().join(trips.lazy(), on="trip_id", how="left")
267
+ legs_with_persons = legs.lazy().join(persons.lazy(), on="person_id", how="left")
268
+
269
+ # === Households ===
270
+ # Some columns have values that are either all null or all defined.
271
+ for col in (
272
+ "survey_method",
273
+ "sample_weight",
274
+ "home_detailed_zone",
275
+ "home_draw_zone",
276
+ "home_insee",
277
+ ):
278
+ all_null_or_all_defined(households[col], "Column `col` is null for some values")
279
+ all_defined(households, "household_id")
280
+ assert households["household_id"].min() == 1, "Column `household_id` does not start at 1"
281
+ assert households["household_id"].max() == len(households), (
282
+ "Column `household_id` does not end at the number of households"
283
+ )
284
+ is_sorted(households, "household_id")
285
+ assert households["sample_weight"].ge(0).all(), (
286
+ "Column `sample_weight` is negative for some values"
287
+ )
288
+ if detailed_zones is not None:
289
+ assert households["home_detailed_zone"].is_in(detailed_zones["detailed_zone_id"]).all(), (
290
+ "Some values in `home_detailed_zone` are not valid detailed zone ids"
291
+ )
292
+ if draw_zones is not None:
293
+ assert households["home_draw_zone"].is_in(draw_zones["draw_zone_id"]).all(), (
294
+ "Some values in `home_draw_zone` are not valid draw zone ids"
295
+ )
296
+ valid_insee_codes(households, "home_insee")
297
+ if insee_zones is not None:
298
+ n = households["home_insee"].is_in(insee_zones["insee_id"]).not_().sum()
299
+ if n > 0:
300
+ print(f"Warning. `home_insee` is not in the defined INSEE zones for {n} households")
301
+ car_counts = cars["household_id"].value_counts()
302
+ assert (
303
+ households.lazy()
304
+ .join(car_counts.lazy(), on="household_id")
305
+ .select((pl.col("nb_cars") >= pl.col("count")).all())
306
+ .collect()
307
+ .item()
308
+ ), "Some households have more cars defined than the value of `nb_cars`"
309
+ motorcycle_counts = motorcycles["household_id"].value_counts()
310
+ assert (
311
+ households.lazy()
312
+ .join(motorcycle_counts.lazy(), on="household_id")
313
+ .select((pl.col("nb_motorcycles") >= pl.col("count")).all())
314
+ .collect()
315
+ .item()
316
+ ), "Some households have more motorcycles defined than the value of `nb_motorcycles`"
317
+ assert households.select(
318
+ (
319
+ pl.col("nb_standard_bicycles").is_null()
320
+ | pl.col("nb_electric_bicycles").is_null()
321
+ | (
322
+ (
323
+ pl.col("nb_standard_bicycles") + pl.col("nb_electric_bicycles")
324
+ == pl.col("nb_bicycles")
325
+ )
326
+ & pl.col("nb_bicycles").is_not_null()
327
+ )
328
+ ).all()
329
+ ).item(), (
330
+ "The property `nb_bicycles = nb_standard_bicycles + nb_electric_bicycles` is not always "
331
+ "verified"
332
+ )
333
+ # The test below ensures that (i) `nb_persons` is not null, (ii) `nb_persons` is positive,
334
+ # (iii) `nb_persons` matches the number of persons for this household.
335
+ person_counts = persons.group_by("household_id").agg(nb_persons=pl.len())
336
+ assert households.join(
337
+ person_counts, on=["household_id", "nb_persons"], how="anti"
338
+ ).is_empty(), (
339
+ "Column `nb_persons` does not match the actual number of persons for some households"
340
+ )
341
+
342
+ # === Persons ===
343
+ all_defined(persons, "person_id")
344
+ assert persons["person_id"].min() == 1, "Column `person_id` does not start at 1"
345
+ assert persons["person_id"].max() == len(persons), (
346
+ "Column `person_id` does not end at the number of persons"
347
+ )
348
+ is_sorted(persons, "person_id")
349
+ all_defined(persons, "household_id")
350
+ assert persons["household_id"].is_in(households["household_id"]).all(), (
351
+ "Column `household_id` does not match a valid household id"
352
+ )
353
+ is_sorted(persons, "household_id")
354
+ all_defined(persons, "person_index")
355
+ # The test below ensures that `person_index` values are sorted within a household.
356
+ assert persons.select(
357
+ (pl.col("person_index") == pl.int_range(1, pl.len() + 1).over("household_id")).all()
358
+ ).item(), (
359
+ "Column `person_index` does not range from 1 to the number of persons within a household"
360
+ )
361
+ # Check: at least one reference person per household.
362
+ invalid_household_ids = (
363
+ persons.group_by("household_id")
364
+ .agg(
365
+ is_valid=pl.col("reference_person_link").is_null().all()
366
+ | pl.col("reference_person_link").eq("reference_person").any()
367
+ )
368
+ .filter(pl.col("is_valid").not_())["household_id"]
369
+ )
370
+ if not invalid_household_ids.is_empty():
371
+ n = len(invalid_household_ids)
372
+ print(f"Warning. There is no reference person defined for {n} households.")
373
+ print("The reference person is automatically set to the first person.")
374
+ persons = persons.with_columns(
375
+ reference_person_link=pl.when(
376
+ pl.col("household_id").is_in(invalid_household_ids) & pl.col("person_index") == 1
377
+ )
378
+ .then(pl.lit("reference_person"))
379
+ .otherwise(pl.col("reference_person_link"))
380
+ )
381
+ # Check: exactly one reference person per household.
382
+ invalid_household_ids = (
383
+ persons.group_by("household_id")
384
+ .agg(
385
+ is_valid=pl.col("reference_person_link").is_null().all()
386
+ | pl.col("reference_person_link").eq("reference_person").sum().eq(1)
387
+ )
388
+ .filter(pl.col("is_valid").not_())["household_id"]
389
+ )
390
+ if not invalid_household_ids.is_empty():
391
+ n = len(invalid_household_ids)
392
+ print(f"Warning. There is more than one reference person defined for {n} households.")
393
+ print(
394
+ "The `reference_person_link` value is automatically set to null for the extra "
395
+ "reference persons."
396
+ )
397
+ persons = persons.with_columns(
398
+ reference_person_link=pl.when(
399
+ pl.col("household_id").is_in(invalid_household_ids)
400
+ & (pl.col("reference_person_link") == "reference_person")
401
+ & (
402
+ (pl.col("reference_person_link") == "reference_person")
403
+ .cum_sum()
404
+ .over("household_id")
405
+ > 1
406
+ )
407
+ )
408
+ .then(pl.lit(None))
409
+ .otherwise(pl.col("reference_person_link"))
410
+ )
411
+ # Maximum age is 125.
412
+ if persons["age"].max() > 125:
413
+ n = (persons["age"] > 125).sum()
414
+ print(f"Warning. There are {n} persons with a age larger than 125.")
415
+ print("The age is automatically capped to 125.")
416
+ persons = persons.with_columns(pl.min_horizontal(pl.col("age"), 125))
417
+ # Check: Age difference between parents and children.
418
+ invalid_person_ids = persons.with_columns(
419
+ parent_max_age=pl.col("age")
420
+ .filter(pl.col("reference_person_link").is_in(("reference_person", "spouse")))
421
+ .max()
422
+ .over("household_id"),
423
+ ).filter(
424
+ pl.col("reference_person_link").eq("child")
425
+ & (pl.col("parent_max_age") - pl.col("age") < CHILD_AGE_THRS)
426
+ )["person_id"]
427
+ if not invalid_person_ids.is_empty():
428
+ n = len(invalid_person_ids)
429
+ print(
430
+ "Warning. The age difference to the oldest parent is less than "
431
+ f"{CHILD_AGE_THRS} years for {n} children."
432
+ )
433
+ # If there is no spouse and there is only one child with invalid age, then the child's link
434
+ # is changed to spouse.
435
+ # Example: reference_person (37 y.o.), child (37 y.o.), child (3 y.o.)
436
+ new_spouses = persons.filter(
437
+ # The person has an age problem.
438
+ pl.col("person_id").is_in(invalid_person_ids)
439
+ # There is no spouse in the household.
440
+ & pl.col("reference_person_link").ne("spouse").all().over("household_id")
441
+ # There is only one person in the household with an age problem.
442
+ & pl.col("person_id").is_in(invalid_person_ids).sum().over("household_id").eq(1)
443
+ )["person_id"]
444
+ n1 = len(new_spouses)
445
+ if n1 > 0:
446
+ print(f"For {n1} persons the status is automatically changed from child to spouse.")
447
+ persons = persons.with_columns(
448
+ reference_person_link=pl.when(pl.col("person_id").is_in(new_spouses))
449
+ .then(pl.lit("spouse"))
450
+ .otherwise(pl.col("reference_person_link"))
451
+ )
452
+ # If (i) there are two spouses (= 3 parents), (ii) the youngest parent is a spouse,
453
+ # (iii) the age difference between the youngest parent and the second-youngest parent is not
454
+ # less than # CHILD_AGE_THRS years, and (iv) the age difference between the second-youngest
455
+ # parent and # the oldest is less than CHILD_AGE_THRS years, then the youngest parent can be
456
+ # changed to a # child while satisfying all age constraints.
457
+ # Example: reference_person (56 y.o.), spouse (55 y.o.), spouse (23 y.o.), child (22 y.o.).
458
+ new_children = persons.with_columns(
459
+ parent_min_age=pl.col("age")
460
+ .filter(pl.col("reference_person_link").is_in(("reference_person", "spouse")))
461
+ .min()
462
+ .over("household_id"),
463
+ parent_median_age=pl.col("age")
464
+ .filter(pl.col("reference_person_link").is_in(("reference_person", "spouse")))
465
+ .median()
466
+ .over("household_id"),
467
+ parent_max_age=pl.col("age")
468
+ .filter(pl.col("reference_person_link").is_in(("reference_person", "spouse")))
469
+ .max()
470
+ .over("household_id"),
471
+ ).filter(
472
+ # There is exactly one person with an age problem in the household.
473
+ pl.col("person_id").is_in(invalid_person_ids).sum().over("household_id").eq(1)
474
+ # There are two spouses in the household.
475
+ & pl.col("reference_person_link").eq("spouse").sum().over("household_id").eq(2)
476
+ # The youngest parent is younger by more than CHILD_AGE_THRS years to the other parents.
477
+ & (pl.col("parent_median_age") - pl.col("parent_min_age") >= CHILD_AGE_THRS)
478
+ # The two oldest parents have less than CHILD_AGE_THRS age difference.
479
+ & (pl.col("parent_max_age") - pl.col("parent_median_age") < CHILD_AGE_THRS)
480
+ # The current person is the youngest parent.
481
+ & pl.col("age").eq(pl.col("parent_min_age"))
482
+ # The current person is a spouse.
483
+ & pl.col("reference_person_link").eq("spouse")
484
+ )["person_id"]
485
+ n2 = len(new_children)
486
+ if n2 > 0:
487
+ print(f"For {n2} persons the status is automatically changed from spouse to child.")
488
+ persons = persons.with_columns(
489
+ reference_person_link=pl.when(pl.col("person_id").is_in(new_children))
490
+ .then(pl.lit("child"))
491
+ .otherwise(pl.col("reference_person_link"))
492
+ )
493
+ if n > n1 + n2:
494
+ m = n - n1 - n2
495
+ print(
496
+ f"For {m} persons, the `reference_person_link` value is automatically set to null "
497
+ 'instead of "child".'
498
+ )
499
+ persons = persons.with_columns(
500
+ reference_person_link=pl.when(
501
+ pl.col("person_id").is_in(invalid_person_ids)
502
+ & pl.col("person_id").is_in(new_spouses).not_()
503
+ & pl.col("person_id").is_in(new_children).not_()
504
+ )
505
+ .then(pl.lit(None))
506
+ .otherwise(pl.col("reference_person_link"))
507
+ )
508
+ invalid_persons = persons.filter(
509
+ pl.col("education_level").is_not_null() | pl.col("detailed_education_level").is_not_null(),
510
+ pl.col("professional_occupation").eq("student"),
511
+ )["person_id"]
512
+ if not invalid_persons.is_empty():
513
+ n = len(invalid_persons)
514
+ print(
515
+ f"Warning. {n} students have a non-null `education_level` or `detailed_education_level"
516
+ )
517
+ print("The values are automatically set to null")
518
+ persons = persons.with_columns(
519
+ education_level=pl.when(pl.col("person_id").is_in(invalid_persons))
520
+ .then(pl.lit(None))
521
+ .otherwise(pl.col("education_level")),
522
+ detailed_education_level=pl.when(pl.col("person_id").is_in(invalid_persons))
523
+ .then(pl.lit(None))
524
+ .otherwise(pl.col("detailed_education_level")),
525
+ )
526
+ assert persons.select(
527
+ (pl.col("education_level").is_not_null() & pl.col("professional_occupation").eq("student"))
528
+ .not_()
529
+ .all()
530
+ ).item(), (
531
+ "The value of `education_level` is incompatible with the value of "
532
+ "`professional_occupation` for some persons"
533
+ )
534
+ assert persons.select(
535
+ (
536
+ pl.col("professional_occupation").ne("worker")
537
+ | pl.col("detailed_professional_occupation").is_in(
538
+ ("worker:full_time", "worker:part_time", "worker:unspecified")
539
+ )
540
+ ).all()
541
+ ).item(), "Some workers have a `detailed_professional_occupation` that is not work-related"
542
+ assert persons.select(
543
+ (
544
+ pl.col("professional_occupation").ne("student")
545
+ | pl.col("detailed_professional_occupation").is_in(
546
+ ("student:primary_or_secondary", "student:higher", "student:apprenticeship")
547
+ )
548
+ ).all()
549
+ ).item(), "Some students have a `detailed_professional_occupation` that is not student-related"
550
+ assert persons.select(
551
+ (
552
+ pl.col("professional_occupation").ne("other")
553
+ | pl.col("detailed_professional_occupation").is_in(
554
+ ("other:unemployed", "other:retired", "other:homemaker", "other:unspecified")
555
+ )
556
+ ).all()
557
+ ).item(), (
558
+ 'Some persons with `professional_occupation` "other" have a '
559
+ "`detailed_professional_occupation` that is work or study related"
560
+ )
561
+ assert persons.select(
562
+ (
563
+ (pl.col("professional_occupation") == "worker")
564
+ & (pl.col("secondary_professional_occupation") == "work")
565
+ )
566
+ .any()
567
+ .not_()
568
+ ).item(), 'Some workers have a `secondary_professional_occupation` equal to "work"'
569
+ assert persons.select(
570
+ (
571
+ (pl.col("professional_occupation") == "student")
572
+ & (pl.col("secondary_professional_occupation") == "education")
573
+ )
574
+ .any()
575
+ .not_()
576
+ ).item(), 'Some students have a `secondary_professional_occupation` equal to "education"'
577
+ is_nonworking_student = pl.col("professional_occupation").eq("student") & pl.col(
578
+ "secondary_professional_occupation"
579
+ ).ne_missing("work")
580
+ invalid_persons = (
581
+ persons.filter(is_nonworking_student & pl.col("pcs_group").is_not_null())
582
+ .select("person_id")
583
+ .to_series()
584
+ )
585
+ if not invalid_persons.is_empty():
586
+ n = len(invalid_persons)
587
+ print(f"Warning. {n} students have a non-null `pcs_group`")
588
+ print('The `secondary_professional_occupation` value is automatically set to "work"')
589
+ persons = persons.with_columns(
590
+ secondary_professional_occupation=pl.when(pl.col("person_id").is_in(invalid_persons))
591
+ .then(pl.lit("work"))
592
+ .otherwise(pl.col("secondary_professional_occupation"))
593
+ )
594
+ # The test below check the 3 guarantees for `pcs_group_code`.
595
+ assert persons.select(
596
+ pl.col("pcs_group_code")
597
+ .replace_strict(PCS_CODES, return_dtype=pl.String)
598
+ .eq_missing(pl.col("pcs_group"))
599
+ .all()
600
+ ).item(), "Columns `pcs_group` and `pcs_group_code` have inconsistencies"
601
+ invalid_persons = persons.filter(pl.col("pcs_category_code2020").is_in(PCS2020_CODES).not_())[
602
+ "person_id"
603
+ ]
604
+ if not invalid_persons.is_empty():
605
+ n = len(invalid_persons)
606
+ print(f"Warning. {n} persons have an invalid `pcs_category_code2020`")
607
+ print("The value is automatically set to null")
608
+ persons = persons.with_columns(
609
+ pcs_category_code2020=pl.when(pl.col("person_id").is_in(invalid_persons))
610
+ .then(pl.lit(None))
611
+ .otherwise(pl.col("pcs_category_code2020"))
612
+ )
613
+ all_defined_for(
614
+ persons,
615
+ "pcs_group_code",
616
+ pl.col("pcs_category_code2020").is_not_null(),
617
+ "non-null `pcs_category_code2020`",
618
+ )
619
+ assert (persons["pcs_category_code2020"] // 10 == persons["pcs_group_code"]).all(), (
620
+ "Columns `pcs_category_code2020` and `pcs_group_code have inconsistencies"
621
+ )
622
+ invalid_persons = persons.filter(pl.col("pcs_category_code2003").is_in(PCS2003_CODES).not_())[
623
+ "person_id"
624
+ ]
625
+ if not invalid_persons.is_empty():
626
+ n = len(invalid_persons)
627
+ print(f"Warning. {n} persons have an invalid `pcs_category_code2003`")
628
+ print("The value is automatically set to null")
629
+ persons = persons.with_columns(
630
+ pcs_category_code2003=pl.when(pl.col("person_id").is_in(invalid_persons))
631
+ .then(pl.lit(None))
632
+ .otherwise(pl.col("pcs_category_code2003"))
633
+ )
634
+ all_defined_for(
635
+ persons,
636
+ "pcs_group_code",
637
+ pl.col("pcs_category_code2003").is_not_null(),
638
+ "non-null `pcs_category_code2003`",
639
+ )
640
+ assert (persons["pcs_category_code2003"] // 10 == persons["pcs_group_code"]).all(), (
641
+ "Columns `pcs_category_code2003` and `pcs_group_code have inconsistencies"
642
+ )
643
+ valid_insee_codes(persons, "work_insee")
644
+ assert persons["work_commute_euclidean_distance_km"].ge(0).all(), (
645
+ "Column `work_commute_euclidean_distance_km` is negative for some values"
646
+ )
647
+ is_not_worker = pl.col("professional_occupation").ne("worker") & pl.col(
648
+ "secondary_professional_occupation"
649
+ ).ne("work")
650
+ mask = pl.col("work_commute_euclidean_distance_km").is_not_null() & is_not_worker
651
+ if persons.select(mask.any()).item():
652
+ if not (persons.filter(mask)["work_commute_euclidean_distance_km"] == 0.0).all():
653
+ # When all distances are 0, we can silently set them to null.
654
+ n = persons.select(mask.sum()).item()
655
+ print(
656
+ f"Warning. The `work_commute_euclidean_distance_km` variable is set for {n} "
657
+ "non-worker persons."
658
+ )
659
+ print("The value is automatically set to null.")
660
+ persons = persons.with_columns(
661
+ work_commute_euclidean_distance_km=pl.when(is_not_worker)
662
+ .then(pl.lit(None))
663
+ .otherwise(pl.col("work_commute_euclidean_distance_km"))
664
+ )
665
+ for col in (
666
+ "work_only_at_home",
667
+ "work_special_location",
668
+ "work_detailed_zone",
669
+ "work_draw_zone",
670
+ "work_insee",
671
+ "has_car_for_work_commute",
672
+ "telework",
673
+ "work_car_parking",
674
+ "work_bicycle_parking",
675
+ ):
676
+ all_null_for(persons, col, is_not_worker, "non-workers")
677
+ if col not in ("work_only_at_home", "worked_during_surveyed_day"):
678
+ all_null_for(persons, col, pl.col("work_only_at_home"), "workers working only at home")
679
+ invalid_persons = (
680
+ persons.filter(
681
+ is_not_worker
682
+ & pl.col("secondary_professional_occupation").ne_missing("work")
683
+ & pl.col("worked_during_surveyed_day").is_in(
684
+ (
685
+ "yes:outside",
686
+ "yes:home:usual",
687
+ "yes:home:telework",
688
+ "yes:home:other",
689
+ "yes:unspecified",
690
+ )
691
+ )
692
+ )
693
+ .select("person_id")
694
+ .to_series()
695
+ )
696
+ if not invalid_persons.is_empty():
697
+ n = len(invalid_persons)
698
+ print(f"Warning. {n} non-workers have indicating having worked during the surveyed day")
699
+ print('Their `secondary_professional_occupation` value is automatically set to "work"')
700
+ persons = persons.with_columns(
701
+ secondary_professional_occupation=pl.when(pl.col("person_id").is_in(invalid_persons))
702
+ .then(pl.lit("work"))
703
+ .otherwise(pl.col("secondary_professional_occupation"))
704
+ )
705
+ invalid_persons = (
706
+ persons.filter(
707
+ is_not_worker
708
+ & pl.col("secondary_professional_occupation").ne_missing("work")
709
+ & pl.col("worked_during_surveyed_day").is_in(
710
+ ("no:weekday", "no:reason", "no:unspecified")
711
+ )
712
+ )
713
+ .select("person_id")
714
+ .to_series()
715
+ )
716
+ if not invalid_persons.is_empty():
717
+ n = len(invalid_persons)
718
+ print(f"Warning. {n} non-workers have indicating having not worked during the surveyed day")
719
+ print("The `worked_during_surveyed_day` value is automatically set to null")
720
+ persons = persons.with_columns(
721
+ worked_during_surveyed_day=pl.when(pl.col("person_id").is_in(invalid_persons))
722
+ .then(pl.lit(None))
723
+ .otherwise(pl.col("worked_during_surveyed_day"))
724
+ )
725
+ all_null_for(
726
+ persons,
727
+ "worked_during_surveyed_day",
728
+ is_not_worker & pl.col("secondary_professional_occupation").ne_missing("work"),
729
+ "non-wokers",
730
+ )
731
+ assert persons.select(
732
+ (pl.col("work_only_at_home") & (pl.col("work_commute_euclidean_distance_km") > 0.0))
733
+ .any()
734
+ .not_()
735
+ ).item(), (
736
+ "Column `work_commute_euclidean_distance_km` is positive for some persons working only at "
737
+ "home"
738
+ )
739
+ is_not_student = pl.col("professional_occupation").ne("student")
740
+ for col in (
741
+ "student_group",
742
+ "student_category",
743
+ "study_only_at_home",
744
+ "study_special_location",
745
+ "study_detailed_zone",
746
+ "study_draw_zone",
747
+ "study_insee",
748
+ "has_car_for_study_commute",
749
+ "study_car_parking",
750
+ "study_bicycle_parking",
751
+ ):
752
+ all_null_for(persons, col, is_not_student, "non-students")
753
+ if col not in ("student_group", "student_category", "study_only_at_home"):
754
+ all_null_for(
755
+ persons, col, pl.col("study_only_at_home"), "persons studying only at home"
756
+ )
757
+ student_constraints = (
758
+ ("student:primary_or_secondary", ("primaire", "collège", "lycée")),
759
+ ("student:apprenticeship", ("lycée", "supérieur")),
760
+ ("student:higher", ("supérieur",)),
761
+ )
762
+ for occ, groups in student_constraints:
763
+ assert persons.select(
764
+ (
765
+ pl.col("detailed_professional_occupation").eq(occ)
766
+ & pl.col("student_group").is_in(groups).not_()
767
+ )
768
+ .any()
769
+ .not_()
770
+ ).item(), (
771
+ "Columns `student_group` and `detailed_professional_occupation` have inconsistencies"
772
+ )
773
+ student_constraints = (
774
+ ("primaire", ("maternelle", "primaire")),
775
+ ("collège", ("collège:6e", "collège:5e", "collège:4e", "collège:3e", "collège:SEGPA")),
776
+ ("lycée", ("lycée:seconde", "lycée:première", "lycée:terminale", "lycée:CAP")),
777
+ (
778
+ "supérieur",
779
+ (
780
+ "supérieur:technique",
781
+ "supérieur:prépa1",
782
+ "supérieur:prépa2",
783
+ "supérieur:BAC+1",
784
+ "supérieur:BAC+2",
785
+ "supérieur:BAC+3",
786
+ "supérieur:BAC+4",
787
+ "supérieur:BAC+5",
788
+ "supérieur:BAC+6&+",
789
+ ),
790
+ ),
791
+ )
792
+ for group, cats in student_constraints:
793
+ assert persons.select(
794
+ (pl.col("student_group").eq(group) & pl.col("student_category").is_in(cats).not_())
795
+ .any()
796
+ .not_()
797
+ ).item(), "Columns `student_group` and `student_category` have inconsistencies"
798
+ valid_insee_codes(persons, "study_insee")
799
+ assert persons["study_commute_euclidean_distance_km"].ge(0).all(), (
800
+ "Column `study_commute_euclidean_distance_km` is negative for some values"
801
+ )
802
+ mask = pl.col("study_commute_euclidean_distance_km").is_not_null() & is_not_student
803
+ if persons.select(mask.any()).item():
804
+ if not (persons.filter(mask)["study_commute_euclidean_distance_km"] == 0.0).all():
805
+ # When all distances are 0, we can silently set them to null.
806
+ n = persons.select(mask.sum()).item()
807
+ print(
808
+ f"Warning. The `study_commute_euclidean_distance_km` variable is set for {n} "
809
+ "non-student persons."
810
+ )
811
+ print("The value is automatically set to null.")
812
+ persons = persons.with_columns(
813
+ study_commute_euclidean_distance_km=pl.when(is_not_student)
814
+ .then(pl.lit(None))
815
+ .otherwise(pl.col("study_commute_euclidean_distance_km"))
816
+ )
817
+ assert persons.select(
818
+ (pl.col("study_only_at_home") & (pl.col("study_commute_euclidean_distance_km") > 0.0))
819
+ .any()
820
+ .not_()
821
+ ).item(), (
822
+ "Column `study_commute_euclidean_distance_km` is positive for some persons studying only "
823
+ "at home"
824
+ )
825
+ invalid_persons = persons.filter(
826
+ pl.col("has_driving_license").eq("yes") & pl.col("age").is_between(15, 17)
827
+ )["person_id"]
828
+ if not invalid_persons.is_empty():
829
+ n = len(invalid_persons)
830
+ print(
831
+ f"Warning. {n} persons indicated having a driving license but are between 15 and 17 "
832
+ "year old"
833
+ )
834
+ print('Their `has_driving_license` value is automatically set to "in_progress"')
835
+ persons = persons.with_columns(
836
+ has_driving_license=pl.when(pl.col("person_id").is_in(invalid_persons))
837
+ .then(pl.lit("in_progress"))
838
+ .otherwise("has_driving_license")
839
+ )
840
+ assert persons.select(
841
+ (pl.col("has_driving_license").eq("yes") & pl.col("age").lt(17)).any().not_()
842
+ ).item(), "Some persons have a driving license and are younger than 17"
843
+ assert persons.select(
844
+ (pl.col("has_driving_license").eq("in_progress") & pl.col("age").lt(15)).any().not_()
845
+ ).item(), "Some persons are taking driving lessons and are younger than 15"
846
+ assert persons.select(
847
+ (pl.col("has_public_transit_subscription") & pl.col("public_transit_subscription").eq("no"))
848
+ .any()
849
+ .not_()
850
+ ).item(), (
851
+ "Columns `has_public_transit_subscription` and public_transit_subscription` have some inconsistencies"
852
+ )
853
+ all_defined(persons, "is_surveyed")
854
+ assert trips_with_persons.select(pl.col("is_surveyed").all()).collect().item(), (
855
+ "Column `is_surveyed` is false for some persons with reported trips"
856
+ )
857
+ all_null_for(
858
+ persons,
859
+ "traveled_during_surveyed_day",
860
+ pl.col("is_surveyed").not_(),
861
+ "persons not surveyed for trips",
862
+ )
863
+ all_defined_for(
864
+ persons, "traveled_during_surveyed_day", pl.col("is_surveyed"), "persons surveyed for trips"
865
+ )
866
+ assert persons.select(
867
+ (pl.col("traveled_during_surveyed_day").eq("yes") & pl.col("nb_trips").eq(0)).any().not_()
868
+ ).item(), (
869
+ 'Some persons with `traveled_during_surveyed_day` equal to "yes" have no reported trip'
870
+ )
871
+ assert persons.select(
872
+ (pl.col("traveled_during_surveyed_day").is_in(("no", "away")) & pl.col("nb_trips").ne(0))
873
+ .any()
874
+ .not_()
875
+ ).item(), (
876
+ 'Some persons with `traveled_during_surveyed_day` equal to "no" or "away" have at least '
877
+ "one reported trip"
878
+ )
879
+ invalid_persons = persons.filter(
880
+ pl.col("worked_during_surveyed_day").eq("yes:home:usual")
881
+ & pl.col("work_only_at_home").not_()
882
+ )["person_id"]
883
+ if not invalid_persons.is_empty():
884
+ n = len(invalid_persons)
885
+ print(
886
+ f"Warning. {n} persons indicated that they worked at home as usual during the surveyed "
887
+ "day but the value `work_only_at_home` is not true"
888
+ )
889
+ print('The `worked_during_surveyed_day` value is automatically set to "yes:home:telework"')
890
+ persons = persons.with_columns(
891
+ worked_during_surveyed_day=pl.when(pl.col("person_id").is_in(invalid_persons))
892
+ .then(pl.lit("yes:home:telework"))
893
+ .otherwise(pl.col("worked_during_surveyed_day"))
894
+ )
895
+ assert (
896
+ trips_with_persons.select(
897
+ (
898
+ pl.col("worked_during_surveyed_day").eq("no:weekday")
899
+ & pl.col("destination_purpose_group").eq("work")
900
+ )
901
+ .any()
902
+ .not_()
903
+ )
904
+ .collect()
905
+ .item()
906
+ ), (
907
+ 'Some persons have `worked_during_surveyed_day` equal to "no:weekday" but did at least one '
908
+ "trip to work"
909
+ )
910
+ invalid_persons = (
911
+ trips_with_persons.filter(pl.col("worked_during_surveyed_day") == "yes:outside")
912
+ .group_by("person_id")
913
+ .agg(
914
+ is_valid=pl.col("origin_purpose_group")
915
+ .eq("work")
916
+ .any()
917
+ .or_(pl.col("destination_purpose_group").eq("work").any())
918
+ )
919
+ .filter(pl.col("is_valid").not_())
920
+ .select("person_id")
921
+ .collect()
922
+ .to_series()
923
+ )
924
+ if not invalid_persons.is_empty():
925
+ n = len(invalid_persons)
926
+ print(
927
+ f"Warning. {n} persons reported having work outside during the surveyed day but did "
928
+ "not actually report any trip to work"
929
+ )
930
+ print(
931
+ 'Their `worked_during_surveyed_day` value is automatically changed to "no:unspecified"'
932
+ )
933
+ persons = persons.with_columns(
934
+ worked_during_surveyed_day=pl.when(pl.col("person_id").is_in(invalid_persons))
935
+ .then(pl.lit("no:unspecified"))
936
+ .otherwise(pl.col("worked_during_surveyed_day"))
937
+ )
938
+ all_null_for(persons, "nb_trips", pl.col("is_surveyed").not_(), "non-surveyed persons")
939
+ all_defined_for(persons, "nb_trips", pl.col("is_surveyed"), "surveyed persons")
940
+ trip_counts = trips.group_by("person_id").agg(count=pl.len())
941
+ assert (
942
+ persons.lazy()
943
+ .join(trip_counts.lazy(), on="person_id", how="full")
944
+ .filter(pl.col("nb_trips") != pl.col("count"))
945
+ .collect()
946
+ .is_empty()
947
+ ), "column `nb_trips` does not match the actual number of trips for some persons"
948
+ assert persons["sample_weight_all"].ge(0).all(), (
949
+ "Column `sample_weight_all` is negative for some values"
950
+ )
951
+ assert persons["sample_weight_surveyed"].ge(0).all(), (
952
+ "Column `sample_weight_surveyed` is negative for some values"
953
+ )
954
+ all_null_for(
955
+ persons,
956
+ "sample_weight_surveyed",
957
+ pl.col("is_surveyed").not_(),
958
+ "non-surveyed persons",
959
+ force=True,
960
+ )
961
+
962
+ # === Trips ===
963
+ all_defined(trips, "trip_id")
964
+ assert trips["trip_id"].min() == 1, "Column `trip_id` does not start at 1"
965
+ assert trips["trip_id"].max() == len(trips), (
966
+ "Column `trip_id` does not end at the number of trips"
967
+ )
968
+ is_sorted(trips, "trip_id")
969
+ all_defined(trips, "person_id")
970
+ assert trips["person_id"].is_in(persons["person_id"]).all(), (
971
+ "Column `person_id` does not match a valid person id"
972
+ )
973
+ is_sorted(trips, "person_id")
974
+ all_defined(trips, "household_id")
975
+ assert (
976
+ trips.lazy()
977
+ .join(
978
+ persons.select("person_id", "household_id").lazy(),
979
+ on=["person_id", "household_id"],
980
+ how="anti",
981
+ )
982
+ .collect()
983
+ .is_empty()
984
+ ), "Incompatible combination for `person_id` and `household_id` in trips"
985
+ all_defined(trips, "trip_index")
986
+ # The test below ensures that `trip_index` values are sorted within a person.
987
+ assert trips.select(
988
+ (pl.col("trip_index") == pl.int_range(1, pl.len() + 1).over("person_id")).all()
989
+ ).item(), "Column `trip_index` does not range from 1 to the number of trips within a person"
990
+ all_defined(trips, "first_trip")
991
+ all_defined(trips, "last_trip")
992
+ assert trips.select(
993
+ (pl.col("trip_index").eq(1) & pl.col("first_trip").not_()).any().not_()
994
+ ).item(), "`trip_index` is 1 but `first_trip` is false"
995
+ assert trips.select((pl.col("trip_index").ne(1) & pl.col("first_trip")).any().not_()).item(), (
996
+ "`first_trip` is true but `trip_index` is not 1"
997
+ )
998
+ assert (
999
+ trips_with_persons.select(
1000
+ (pl.col("origin_purpose").eq("work:declared") & is_not_worker).any().not_()
1001
+ )
1002
+ .collect()
1003
+ .item()
1004
+ ), '`origin_purpose` is "work:declared" for a non-worker person'
1005
+ education_purposes = (
1006
+ "education:primary",
1007
+ "education:middle_school",
1008
+ "education:high_school",
1009
+ "education:higher",
1010
+ )
1011
+ assert (
1012
+ trips_with_persons.select(
1013
+ (pl.col("origin_purpose").is_in(education_purposes) & is_not_student).any().not_()
1014
+ )
1015
+ .collect()
1016
+ .item()
1017
+ ), "`origin_purpose` is education-related for a non-student person"
1018
+ assert trips.select(
1019
+ (
1020
+ pl.col("origin_purpose").cast(pl.String).str.extract("(\w+):?")
1021
+ == pl.col("origin_purpose_group")
1022
+ ).all()
1023
+ ).item(), "Columns `origin_purpose` and `origin_purpose_group` have some inconsistencies"
1024
+ all_defined_for(
1025
+ trips,
1026
+ "origin_purpose_group",
1027
+ pl.col("origin_purpose").is_not_null(),
1028
+ "trips with non-null `origin_purpose`",
1029
+ )
1030
+ assert (
1031
+ trips_with_persons.select(
1032
+ (pl.col("destination_purpose").eq("work:declared") & is_not_worker).any().not_()
1033
+ )
1034
+ .collect()
1035
+ .item()
1036
+ ), '`destination_purpose` is "work:declared" for a non-worker person'
1037
+ assert (
1038
+ trips_with_persons.select(
1039
+ (pl.col("destination_purpose").is_in(education_purposes) & is_not_student).any().not_()
1040
+ )
1041
+ .collect()
1042
+ .item()
1043
+ ), "`destination_purpose` is education-related for a non-student person"
1044
+ assert trips.select(
1045
+ (
1046
+ pl.col("destination_purpose").cast(pl.String).str.extract("(\w+):?")
1047
+ == pl.col("destination_purpose_group")
1048
+ ).all()
1049
+ ).item(), (
1050
+ "Columns `destination_purpose` and `destination_purpose_group` have some inconsistencies"
1051
+ )
1052
+ all_defined_for(
1053
+ trips,
1054
+ "destination_purpose_group",
1055
+ pl.col("destination_purpose").is_not_null(),
1056
+ "trips with non-null `destination_purpose`",
1057
+ )
1058
+ if CHECK_ORIGIN_MATCHES_PREV_DEST != "no":
1059
+ same_as_next_in_group(
1060
+ trips,
1061
+ "destination_purpose",
1062
+ "origin_purpose",
1063
+ "person_id",
1064
+ CHECK_ORIGIN_MATCHES_PREV_DEST == "warn",
1065
+ )
1066
+ same_as_next_in_group(
1067
+ trips,
1068
+ "destination_purpose_group",
1069
+ "origin_purpose_group",
1070
+ "person_id",
1071
+ CHECK_ORIGIN_MATCHES_PREV_DEST == "warn",
1072
+ )
1073
+ all_null_for(
1074
+ trips,
1075
+ "origin_escort_purpose",
1076
+ pl.col("origin_purpose_group").ne_missing("escort"),
1077
+ "non-escort trips",
1078
+ )
1079
+ assert trips.select(
1080
+ (
1081
+ pl.col("origin_escort_purpose").cast(pl.String).str.extract("(\w+):?")
1082
+ == pl.col("origin_escort_purpose_group")
1083
+ ).all()
1084
+ ).item(), (
1085
+ "Columns `origin_escort_purpose` and `origin_escort_purpose_group` have some inconsistencies"
1086
+ )
1087
+ all_defined_for(
1088
+ trips,
1089
+ "origin_escort_purpose_group",
1090
+ pl.col("origin_escort_purpose").is_not_null(),
1091
+ "trips with non-null `origin_escort_purpose`",
1092
+ )
1093
+ assert (trips["origin_escort_purpose_group"] != "escort").all(), (
1094
+ 'Some trips have "escort" as `origin_escort_purpose_group`'
1095
+ )
1096
+ all_null_for(
1097
+ trips,
1098
+ "destination_escort_purpose",
1099
+ pl.col("destination_purpose_group").ne_missing("escort"),
1100
+ "non-escort trips",
1101
+ )
1102
+ assert trips.select(
1103
+ (
1104
+ pl.col("destination_escort_purpose").cast(pl.String).str.extract("(\w+):?")
1105
+ == pl.col("destination_escort_purpose_group")
1106
+ ).all()
1107
+ ).item(), (
1108
+ "Columns `destination_escort_purpose` and `destination_escort_purpose_group` have some inconsistencies"
1109
+ )
1110
+ all_defined_for(
1111
+ trips,
1112
+ "destination_escort_purpose_group",
1113
+ pl.col("destination_escort_purpose").is_not_null(),
1114
+ "trips with non-null `destination_escort_purpose`",
1115
+ )
1116
+ assert (trips["destination_escort_purpose_group"] != "escort").all(), (
1117
+ 'Some trips have "escort" as `destination_escort_purpose_group`'
1118
+ )
1119
+ if CHECK_ORIGIN_MATCHES_PREV_DEST != "no":
1120
+ same_as_next_in_group(
1121
+ trips,
1122
+ "destination_escort_purpose",
1123
+ "origin_escort_purpose",
1124
+ "person_id",
1125
+ CHECK_ORIGIN_MATCHES_PREV_DEST == "warn",
1126
+ )
1127
+ same_as_next_in_group(
1128
+ trips,
1129
+ "destination_escort_purpose_group",
1130
+ "origin_escort_purpose_group",
1131
+ "person_id",
1132
+ CHECK_ORIGIN_MATCHES_PREV_DEST == "warn",
1133
+ )
1134
+ valid_insee_codes(trips, "origin_insee")
1135
+ valid_insee_codes(trips, "destination_insee")
1136
+ if CHECK_ORIGIN_MATCHES_PREV_DEST != "no":
1137
+ for zone in ("special_location", "detailed_zone", "draw_zone", "insee"):
1138
+ same_as_next_in_group(
1139
+ trips,
1140
+ f"destination_{zone}",
1141
+ f"origin_{zone}",
1142
+ "person_id",
1143
+ CHECK_ORIGIN_MATCHES_PREV_DEST == "warn",
1144
+ )
1145
+ assert trips.select(pl.col("departure_time").diff().over("person_id").min()).item() > 0, (
1146
+ "Column `departure_time` is not increasing over persons"
1147
+ )
1148
+ invalid_persons = trips.filter(pl.col("arrival_time") < pl.col("departure_time"))[
1149
+ "person_id"
1150
+ ].unique()
1151
+ if not invalid_persons.is_empty():
1152
+ n = len(invalid_persons)
1153
+ print(
1154
+ f"Warning. {n} persons have at least one trip with `arrival_time` smaller than "
1155
+ "`departure_time`"
1156
+ )
1157
+ print(
1158
+ "The `departure_time`, `arrival_time`, and `travel_time` values for these persons are "
1159
+ "automatically set to null."
1160
+ )
1161
+ trips = trips.with_columns(
1162
+ pl.when(pl.col("person_id").is_in(invalid_person_ids))
1163
+ .then(pl.lit(None))
1164
+ .otherwise(col)
1165
+ .alias(col)
1166
+ for col in ("departure_time", "arrival_time", "travel_time")
1167
+ )
1168
+ invalid_persons = trips.filter(
1169
+ pl.col("arrival_time") > pl.col("departure_time").shift(-1).over("person_id")
1170
+ )["person_id"].unique()
1171
+ if not invalid_persons.is_empty():
1172
+ n = len(invalid_persons)
1173
+ print(
1174
+ f"Warning. {n} persons have at least one trip that starts before the previous trip "
1175
+ "ended"
1176
+ )
1177
+ print(
1178
+ "The `departure_time`, `arrival_time`, and `travel_time` values for these persons are "
1179
+ "automatically set to null."
1180
+ )
1181
+ trips = trips.with_columns(
1182
+ pl.when(pl.col("person_id").is_in(invalid_person_ids))
1183
+ .then(pl.lit(None))
1184
+ .otherwise(col)
1185
+ .alias(col)
1186
+ for col in ("departure_time", "arrival_time", "travel_time")
1187
+ )
1188
+ assert trips["travel_time"].eq_missing(trips["arrival_time"] - trips["departure_time"]).all(), (
1189
+ "`travel_time` is not equal to `arrival_time - departure_time`"
1190
+ )
1191
+ assert (
1192
+ legs_with_trips.select(
1193
+ ((pl.col("main_mode") == pl.col("mode")).any() | pl.col("main_mode").is_null().all())
1194
+ .over("trip_id")
1195
+ .all()
1196
+ )
1197
+ .collect()
1198
+ .item()
1199
+ ), "Some trips have no leg whose `mode` matches the trip's `main_mode`"
1200
+ assert trips.select(
1201
+ (pl.col("main_mode").replace_strict(MODE_TO_GROUP) == pl.col("main_mode_group")).all()
1202
+ ).item(), "Columns `main_mode` and `main_mode_group` have some inconsistencies"
1203
+ all_defined_for(
1204
+ trips,
1205
+ "main_mode_group",
1206
+ pl.col("main_mode").is_not_null(),
1207
+ "trips with non-null `main_mode`",
1208
+ )
1209
+ assert trips["trip_euclidean_distance_km"].ge(0).all(), (
1210
+ "Column `trip_euclidean_distance_km` is negative for some values"
1211
+ )
1212
+ assert trips["trip_travel_distance_km"].ge(0).all(), (
1213
+ "Column `trip_travel_distance_km` is negative for some values"
1214
+ )
1215
+ assert (trips["trip_travel_distance_km"] >= trips["trip_euclidean_distance_km"]).all(), (
1216
+ "`trip_travel_distance_km` is smaller than `trip_euclidean_distance_km` for some values"
1217
+ )
1218
+ if detailed_zones is not None:
1219
+ zone_ids = set(detailed_zones["detailed_zone_id"])
1220
+ perimeter_check(trips, zone_ids, "detailed")
1221
+ if draw_zones is not None:
1222
+ zone_ids = set(draw_zones["draw_zone_id"])
1223
+ perimeter_check(trips, zone_ids, "draw")
1224
+ if insee_zones is not None:
1225
+ zone_ids = set(insee_zones["insee_id"])
1226
+ perimeter_check(trips, zone_ids, "insee")
1227
+ tour_purposes = ("work:professional_tour", "shopping:tour_no_purchase")
1228
+ all_null_for(
1229
+ trips,
1230
+ "nb_tour_stops",
1231
+ (
1232
+ pl.col("origin_purpose").is_in(tour_purposes)
1233
+ | pl.col("destination_purpose").is_in(tour_purposes)
1234
+ ).not_(),
1235
+ "non-tour purposes",
1236
+ )
1237
+ # The test below ensures that (i) `nb_legs` is not null, (ii) `nb_legs` is positive,
1238
+ # (iii) `nb_legs` matches the number of legs for this trip.
1239
+ leg_counts = legs.group_by("trip_id").agg(nb_legs=pl.len())
1240
+ assert trips.join(leg_counts, on=["trip_id", "nb_legs"], how="anti").is_empty(), (
1241
+ "Column `nb_legs` does not match the actual number of legs for some trips"
1242
+ )
1243
+
1244
+ # === Legs ===
1245
+ all_defined(legs, "leg_id")
1246
+ assert legs["leg_id"].min() == 1, "Column `leg_id` does not start at 1"
1247
+ assert legs["leg_id"].max() == len(legs), "Column `leg_id` does not end at the number of legs"
1248
+ is_sorted(legs, "leg_id")
1249
+ all_defined(legs, "trip_id")
1250
+ assert legs["trip_id"].is_in(trips["trip_id"]).all(), (
1251
+ "Column `trip_id` does not match a valid trip id"
1252
+ )
1253
+ is_sorted(legs, "trip_id")
1254
+ all_defined(legs, "person_id")
1255
+ assert (
1256
+ legs.lazy()
1257
+ .join(
1258
+ trips.lazy().select("trip_id", "person_id"),
1259
+ on=["trip_id", "person_id"],
1260
+ how="anti",
1261
+ )
1262
+ .collect()
1263
+ .is_empty()
1264
+ ), "Incompatible combination for `trip_id` and `person_id` in legs"
1265
+ all_defined(legs, "household_id")
1266
+ assert (
1267
+ legs.lazy()
1268
+ .join(
1269
+ persons.lazy().select("person_id", "household_id"),
1270
+ on=["person_id", "household_id"],
1271
+ how="anti",
1272
+ )
1273
+ .collect()
1274
+ .is_empty()
1275
+ ), "Incompatible combination for `person_id` and `household_id` in legs"
1276
+ all_defined(legs, "leg_index")
1277
+ # The test below ensures that `leg_index` values are sorted within a trip.
1278
+ assert legs.select(
1279
+ (pl.col("leg_index") == pl.int_range(1, pl.len() + 1).over("trip_id")).all()
1280
+ ).item(), "Column `leg_index` does not range from 1 to the number of legs within a trip"
1281
+ all_defined(legs, "first_leg")
1282
+ all_defined(legs, "last_leg")
1283
+ assert legs.select(
1284
+ (pl.col("leg_index").eq(1) & pl.col("first_leg").not_()).any().not_()
1285
+ ).item(), "`leg_index` is 1 but `first_leg` is false"
1286
+ assert legs.select((pl.col("leg_index").ne(1) & pl.col("first_leg")).any().not_()).item(), (
1287
+ "`first_leg` is true but `leg_index` is not 1"
1288
+ )
1289
+ assert legs.select(
1290
+ (pl.col("mode").replace_strict(MODE_TO_GROUP) == pl.col("mode_group")).all()
1291
+ ).item(), "Columns `mode` and `mode_group` have some inconsistencies"
1292
+ valid_insee_codes(legs, "start_insee")
1293
+ valid_insee_codes(legs, "end_insee")
1294
+ if CHECK_START_MATCHED_PREV_END != "no":
1295
+ for zone in ("special_location", "detailed_zone", "draw_zone", "insee"):
1296
+ same_as_next_in_group(
1297
+ legs,
1298
+ f"end_{zone}",
1299
+ f"start_{zone}",
1300
+ "trip_id",
1301
+ CHECK_START_MATCHED_PREV_END == "warn",
1302
+ )
1303
+ # When we observe this:
1304
+ # start1: a ; end1: a
1305
+ # start2: b ; end2: c
1306
+ # we switch `end1` to `start2` to satisfy the constraint
1307
+ legs_to_fix = (
1308
+ legs.filter(
1309
+ pl.col("end_detailed_zone").ne_missing(
1310
+ pl.col("start_detailed_zone").shift(-1).over("trip_id")
1311
+ )
1312
+ & (pl.int_range(1, pl.len() + 1) != pl.len()).over("trip_id")
1313
+ & pl.col("start_detailed_zone").eq_missing(pl.col("end_detailed_zone"))
1314
+ )
1315
+ .select("leg_id")
1316
+ .to_series()
1317
+ )
1318
+ if not legs_to_fix.is_empty():
1319
+ n = len(legs_to_fix)
1320
+ print(
1321
+ f"Warning. {n} legs have `end_detailed_zone` that does not match "
1322
+ "`start_detailed_zone` of the next leg but it matches `start_detailed_zone` of the "
1323
+ "current leg"
1324
+ )
1325
+ print(
1326
+ "Their `end_detailed_zone` value is automatically changed to `start_detailed_zone` "
1327
+ "of the current leg"
1328
+ )
1329
+ legs = legs.with_columns(
1330
+ end_detailed_zone=pl.when(pl.col("leg_id").is_in(legs_to_fix))
1331
+ .then(pl.col("start_detailed_zone").shift(-1).over("trip_id"))
1332
+ .otherwise(pl.col("end_detailed_zone"))
1333
+ )
1334
+ n = (
1335
+ legs_with_trips.group_by("trip_id")
1336
+ .agg(
1337
+ leg_tt=pl.col("leg_travel_time").fill_null(0).sum(),
1338
+ trip_tt=pl.col("travel_time").first(),
1339
+ )
1340
+ .filter(pl.col("leg_tt") > pl.col("trip_tt"))
1341
+ .select(pl.len())
1342
+ .collect()
1343
+ .item()
1344
+ )
1345
+ if n > 0:
1346
+ print(
1347
+ f"Warning. The sum of the legs' `leg_travel_time is larger than the trip's "
1348
+ f"`travel_time` for {n} trips."
1349
+ )
1350
+ assert legs["leg_euclidean_distance_km"].ge(0).all(), (
1351
+ "Column `leg_euclidean_distance_km` is negative for some values"
1352
+ )
1353
+ assert legs["leg_travel_distance_km"].ge(0).all(), (
1354
+ "Column `leg_travel_distance_km` is negative for some values"
1355
+ )
1356
+ assert (legs["leg_travel_distance_km"] >= legs["leg_euclidean_distance_km"]).all(), (
1357
+ "`leg_travel_distance_km` is smaller than `leg_euclidean_distance_km` for some values"
1358
+ )
1359
+ is_non_car_leg = pl.col("mode").is_in(("car:driver", "car:passenger")).not_()
1360
+ all_null_for(
1361
+ legs,
1362
+ "car_type",
1363
+ is_non_car_leg,
1364
+ "non-car legs",
1365
+ )
1366
+ all_null_for(
1367
+ legs, "car_id", pl.col("car_type").ne("household"), "legs with a non-household car"
1368
+ )
1369
+ invalid_legs = legs.filter(pl.col("car_type").eq("household") & pl.col("car_id").is_null())[
1370
+ "leg_id"
1371
+ ]
1372
+ if not invalid_legs.is_empty():
1373
+ n = len(invalid_legs)
1374
+ print(f'For {n} legs `car_type` is "household" but `car_id` is null')
1375
+ print('The value of `car_type` is automatically changed to "other_household"')
1376
+ legs = legs.with_columns(
1377
+ car_type=pl.when(pl.col("leg_id").is_in(invalid_legs))
1378
+ .then(pl.lit("other_household"))
1379
+ .otherwise(pl.col("car_type"))
1380
+ )
1381
+ all_defined_for(legs, "car_id", pl.col("car_type").eq("household"), "legs with a household car")
1382
+ extra_cars_households = (
1383
+ households.lazy()
1384
+ .join(car_counts.lazy(), on="household_id")
1385
+ .filter(pl.col("nb_cars") > pl.col("count"))
1386
+ .select("household_id")
1387
+ .collect()
1388
+ .to_series()
1389
+ )
1390
+ invalid_legs = legs.filter(
1391
+ pl.col("car_type").eq("other_household")
1392
+ & pl.col("household_id").is_in(extra_cars_households).not_()
1393
+ )["leg_id"]
1394
+ if not invalid_legs.is_empty():
1395
+ n = len(invalid_legs)
1396
+ print(
1397
+ f'Warning. {n} legs have `car_type` equal to "other_household" but there is no extra '
1398
+ "car in the household"
1399
+ )
1400
+ print('The `car_type` value is automatically changed to "other"')
1401
+ legs = legs.with_columns(
1402
+ car_type=pl.when(pl.col("leg_id").is_in(invalid_legs))
1403
+ .then(pl.lit("other"))
1404
+ .otherwise(pl.col("car_type"))
1405
+ )
1406
+ # The test below ensure that `car_id` is a valid id and that it matches a car from the
1407
+ # household.
1408
+ assert (
1409
+ legs.lazy()
1410
+ .filter(pl.col("car_id").is_not_null())
1411
+ .select("car_id", leg_household_id="household_id")
1412
+ .join(cars.lazy(), on="car_id", how="left")
1413
+ .select("leg_household_id", car_household_id="household_id")
1414
+ .select(pl.col("car_household_id").eq_missing(pl.col("leg_household_id")).all())
1415
+ .collect()
1416
+ .item()
1417
+ ), "`car_id` does not match a valid car from the household"
1418
+ all_null_for(legs, "nolicense_car", pl.col("mode").ne("car:driver"), "non car-driver legs")
1419
+ is_vehicle_mode = pl.col("mode_group").is_in(
1420
+ ("car_driver", "motorcycle", "bicycle", "car_passenger")
1421
+ ) | pl.col("mode").is_in(
1422
+ (
1423
+ "truck:driver",
1424
+ "truck:passenger",
1425
+ "personal_transporter:non_motorized",
1426
+ "personal_transporter:motorized",
1427
+ "personal_transporter:unspecified",
1428
+ )
1429
+ )
1430
+ for col in (
1431
+ "nb_persons_in_vehicle",
1432
+ "nb_majors_in_vehicle",
1433
+ "nb_minors_in_vehicle",
1434
+ "parking_location",
1435
+ "parking_search_time",
1436
+ ):
1437
+ all_null_for(legs, col, is_vehicle_mode.not_(), "non-vehicle mode")
1438
+ assert legs["nb_persons_in_vehicle"].ge(1).all(), (
1439
+ "Column `nb_persons_in_vehicle` is non-positive for some values"
1440
+ )
1441
+ assert legs.select(
1442
+ (
1443
+ (
1444
+ pl.col("mode_group").eq("car_passenger")
1445
+ | pl.col("mode").is_in(
1446
+ (
1447
+ "motorcycle:passenger",
1448
+ "motorcycle:passenger:moped",
1449
+ "motorcycle:passenger:moto",
1450
+ "bicycle:passenger",
1451
+ "truck:passenger",
1452
+ )
1453
+ )
1454
+ )
1455
+ & pl.col("nb_persons_in_vehicle").eq(1)
1456
+ )
1457
+ .any()
1458
+ .not_()
1459
+ ).item(), "`nb_persons_in_vehicle` is 1 for passenger legs"
1460
+ assert (legs["nb_persons_in_vehicle"] >= legs["nb_majors_in_vehicle"]).all(), (
1461
+ "`nb_majors_in_vehicle` is larger than `nb_persons_in_vehicle` for some legs"
1462
+ )
1463
+ assert (legs["nb_persons_in_vehicle"] >= legs["nb_minors_in_vehicle"]).all(), (
1464
+ "`nb_minors_in_vehicle` is larger than `nb_persons_in_vehicle` for some legs"
1465
+ )
1466
+ is_major = pl.col("age").ge(18)
1467
+ assert (
1468
+ legs_with_persons.select((pl.col("nb_majors_in_vehicle").eq(0) & is_major).any().not_())
1469
+ .collect()
1470
+ .item()
1471
+ ), "`nb_majors_in_vehicle` is 0 but the person is major"
1472
+ assert (
1473
+ legs_with_persons.select(
1474
+ (pl.col("nb_minors_in_vehicle").eq(0) & is_major.not_()).any().not_()
1475
+ )
1476
+ .collect()
1477
+ .item()
1478
+ ), "`nb_minors_in_vehicle` is 0 but the person is minor"
1479
+ all_defined_for(
1480
+ legs,
1481
+ "nb_persons_in_vehicle",
1482
+ pl.col("nb_majors_in_vehicle").is_not_null() & pl.col("nb_minors_in_vehicle").is_not_null(),
1483
+ "non-null `nb_majors_in_vehicle` and `nb_minors_in_vehicle`",
1484
+ )
1485
+ assert (
1486
+ legs["nb_majors_in_vehicle"] + legs["nb_minors_in_vehicle"] == legs["nb_persons_in_vehicle"]
1487
+ ).all(), "`nb_persons_in_vehicle` is not equal to `nb_majors_in_vehicle + nb_minors_in_vehicle`"
1488
+ is_non_motorcycle_leg = pl.col("mode_group").ne("motorcycle")
1489
+ all_null_for(
1490
+ legs,
1491
+ "motorcycle_type",
1492
+ is_non_motorcycle_leg,
1493
+ "non-motorcycle legs",
1494
+ )
1495
+ all_null_for(
1496
+ legs,
1497
+ "motorcycle_id",
1498
+ pl.col("motorcycle_type").ne("household"),
1499
+ "legs with a non-household motorcycle",
1500
+ )
1501
+ all_defined_for(
1502
+ legs,
1503
+ "motorcycle_id",
1504
+ pl.col("motorcycle_type").eq("household"),
1505
+ "legs with a household motorcycle",
1506
+ )
1507
+ extra_motorcycles_households = (
1508
+ households.lazy()
1509
+ .join(motorcycle_counts.lazy(), on="household_id")
1510
+ .filter(pl.col("nb_motorcycles") > pl.col("count"))
1511
+ .select("household_id")
1512
+ .collect()
1513
+ .to_series()
1514
+ )
1515
+ invalid_legs = legs.filter(
1516
+ pl.col("motorcycle_type").eq("other_household")
1517
+ & pl.col("household_id").is_in(extra_motorcycles_households).not_()
1518
+ )["leg_id"]
1519
+ if not invalid_legs.is_empty():
1520
+ n = len(invalid_legs)
1521
+ print(
1522
+ 'Warning. {n} legs have `motorcycle_type` equal to "other_household" but there is no '
1523
+ "extra motorcycle in the household"
1524
+ )
1525
+ print('The `motorcycle_type` value is automatically changed to "other"')
1526
+ legs = legs.with_columns(
1527
+ motorcycle_type=pl.when(pl.col("leg_id").is_in(invalid_legs))
1528
+ .then(pl.lit("other"))
1529
+ .otherwise(pl.col("motorcycle_type"))
1530
+ )
1531
+ # The test below ensure that `motorcycle_id` is a valid id and that it matches a motorcycle from
1532
+ # the household.
1533
+ assert (
1534
+ legs.lazy()
1535
+ .filter(pl.col("motorcycle_id").is_not_null())
1536
+ .select("motorcycle_id", leg_household_id="household_id")
1537
+ .join(motorcycles.lazy(), on="motorcycle_id", how="left")
1538
+ .select("leg_household_id", motorcycle_household_id="household_id")
1539
+ .select(pl.col("motorcycle_household_id").eq_missing(pl.col("leg_household_id")).all())
1540
+ .collect()
1541
+ .item()
1542
+ ), "`motorcycle_id` does not match a valid motorcycle from the household"
1543
+ all_null_for(
1544
+ legs,
1545
+ "parking_type",
1546
+ pl.col("parking_location").is_null()
1547
+ | pl.col("parking_location").is_in(("stop_only", "none")),
1548
+ "legs without valid parking location",
1549
+ force=True,
1550
+ )
1551
+
1552
+ # === Cars ===
1553
+ all_defined(cars, "car_id")
1554
+ assert cars["car_id"].min() == 1, "Column `car_id` does not start at 1"
1555
+ assert cars["car_id"].max() == len(cars), "Column `car_id` does not end at the number of cars"
1556
+ is_sorted(cars, "car_id")
1557
+ all_defined(cars, "household_id")
1558
+ assert cars["household_id"].is_in(households["household_id"]).all(), (
1559
+ "Column `household_id` does not match a valid household id"
1560
+ )
1561
+ is_sorted(cars, "household_id")
1562
+ all_defined(cars, "car_index")
1563
+ # The test below ensures that `car_index` values are sorted within a household.
1564
+ assert cars.select(
1565
+ (pl.col("car_index") == pl.int_range(1, pl.len() + 1).over("household_id")).all()
1566
+ ).item(), "Column `car_index` does not range from 1 to the number of cars within a household"
1567
+ assert cars["year"].min() >= 1900, "Some cars are older than 1900"
1568
+ assert cars["year"].max() <= persons["interview_date"].max().year, (
1569
+ "Some cars have a initial year in the future"
1570
+ )
1571
+ all_null_for(
1572
+ cars,
1573
+ "parking_type",
1574
+ pl.col("parking_location").is_null(),
1575
+ "cars with null parking location",
1576
+ )
1577
+
1578
+ # === Motorcycles ===
1579
+ all_defined(motorcycles, "motorcycle_id")
1580
+ assert motorcycles["motorcycle_id"].min() == 1, "Column `motorcycle_id` does not start at 1"
1581
+ assert motorcycles["motorcycle_id"].max() == len(motorcycles), (
1582
+ "Column `motorcycle_id` does not end at the number of motorcycles"
1583
+ )
1584
+ is_sorted(motorcycles, "motorcycle_id")
1585
+ all_defined(motorcycles, "household_id")
1586
+ assert motorcycles["household_id"].is_in(households["household_id"]).all(), (
1587
+ "Column `household_id` does not match a valid household id"
1588
+ )
1589
+ is_sorted(motorcycles, "household_id")
1590
+ all_defined(motorcycles, "motorcycle_index")
1591
+ # The test below ensures that `motorcycle_index` values are sorted within a household.
1592
+ assert motorcycles.select(
1593
+ (pl.col("motorcycle_index") == pl.int_range(1, pl.len() + 1).over("household_id")).all()
1594
+ ).item(), (
1595
+ "Column `motorcycle_index` does not range from 1 to the number of motorcycles within a household"
1596
+ )
1597
+ all_null_for(
1598
+ motorcycles,
1599
+ "parking_type",
1600
+ pl.col("parking_location").is_null(),
1601
+ "motorcycles with null parking location",
1602
+ )
1603
+ for col in ("thermic_engine_type", "cm3_lower_bound", "cm3_upper_bound"):
1604
+ all_null_for(
1605
+ motorcycles,
1606
+ col,
1607
+ pl.col("fuel_type").ne("thermic"),
1608
+ "non-thermic motorcycles",
1609
+ force=True,
1610
+ )
1611
+ assert (motorcycles["cm3_upper_bound"] >= motorcycles["cm3_lower_bound"]).all(), (
1612
+ "Some motorocycles have `cm3_upper_bound` smaller than `cm3_lower_bound`"
1613
+ )
1614
+ for col in ("kw_lower_bound", "kw_upper_bound"):
1615
+ all_null_for(
1616
+ motorcycles,
1617
+ col,
1618
+ pl.col("fuel_type").ne("electric"),
1619
+ "non-electric motorcycles",
1620
+ )
1621
+ assert (motorcycles["kw_upper_bound"] >= motorcycles["kw_lower_bound"]).all(), (
1622
+ "Some motorocycles have `kw_upper_bound` smaller than `kw_lower_bound`"
1623
+ )
1624
+
1625
+ # === Zone intersects ===
1626
+ zones = (
1627
+ (special_locations, "special_location"),
1628
+ (detailed_zones, "detailed_zone"),
1629
+ (draw_zones, "draw_zone"),
1630
+ (insee_zones, "insee"),
1631
+ )
1632
+ for (gdf1, z1), (gdf2, z2) in itertools.combinations(zones, 2):
1633
+ if gdf1 is None or gdf2 is None:
1634
+ continue
1635
+ intersects = pl.from_pandas(
1636
+ gpd.sjoin(
1637
+ gdf1.loc[:, ["geometry", f"{z1}_id"]],
1638
+ gdf2.loc[:, ["geometry", f"{z2}_id"]].to_crs(gdf1.crs),
1639
+ predicate="intersects",
1640
+ ).loc[:, [f"{z1}_id", f"{z2}_id"]]
1641
+ )
1642
+ households = intersect_check(households, intersects, "home", z1, z2, "household_id")
1643
+ persons = intersect_check(persons, intersects, "work", z1, z2, "person_id")
1644
+ persons = intersect_check(persons, intersects, "study", z1, z2, "person_id")
1645
+ trips = intersect_check(trips, intersects, "origin", z1, z2, "trip_id")
1646
+ trips = intersect_check(trips, intersects, "destination", z1, z2, "trip_id")
1647
+ legs = intersect_check(legs, intersects, "start", z1, z2, "leg_id")
1648
+ legs = intersect_check(legs, intersects, "end", z1, z2, "leg_id")
1649
+
1650
+ return dict(
1651
+ households=households,
1652
+ cars=cars,
1653
+ motorcycles=motorcycles,
1654
+ persons=persons,
1655
+ trips=trips,
1656
+ legs=legs,
1657
+ special_locations=special_locations,
1658
+ detailed_zones=detailed_zones,
1659
+ draw_zones=draw_zones,
1660
+ insee_zones=insee_zones,
1661
+ )