mobisurvstd 0.7.0__tar.gz → 0.8.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. {mobisurvstd-0.7.0/src/mobisurvstd.egg-info → mobisurvstd-0.8.0}/PKG-INFO +1 -1
  2. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/pyproject.toml +1 -1
  3. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/cerema/survey.py +5 -1
  4. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/egt2010/menages.py +12 -0
  5. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/egt2020/menages.py +11 -0
  6. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/emp/personnes.py +32 -4
  7. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/emp/survey.py +13 -0
  8. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/schema/households.py +5 -0
  9. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/validation.py +23 -0
  10. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0/src/mobisurvstd.egg-info}/PKG-INFO +1 -1
  11. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/LICENSE.md +0 -0
  12. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/README.md +0 -0
  13. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/setup.cfg +0 -0
  14. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/__init__.py +0 -0
  15. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/__main__.py +0 -0
  16. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/cerema/__init__.py +0 -0
  17. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/cerema/common.py +0 -0
  18. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/cerema/deplacements.py +0 -0
  19. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/cerema/edgt.py +0 -0
  20. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/cerema/edvm.py +0 -0
  21. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/cerema/emc2.py +0 -0
  22. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/cerema/emd.py +0 -0
  23. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/cerema/menages.py +0 -0
  24. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/cerema/personnes.py +0 -0
  25. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/cerema/trajets.py +0 -0
  26. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/cerema/zones.py +0 -0
  27. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/classes.py +0 -0
  28. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/cli.py +0 -0
  29. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/common/__init__.py +0 -0
  30. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/common/cars.py +0 -0
  31. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/common/clean.py +0 -0
  32. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/common/households.py +0 -0
  33. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/common/legs.py +0 -0
  34. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/common/motorcycles.py +0 -0
  35. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/common/persons.py +0 -0
  36. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/common/trips.py +0 -0
  37. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/common/zones.py +0 -0
  38. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/egt2010/__init__.py +0 -0
  39. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/egt2010/deplacements.py +0 -0
  40. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/egt2010/personnes.py +0 -0
  41. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/egt2010/survey.py +0 -0
  42. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/egt2010/trajets.py +0 -0
  43. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/egt2010/zones.py +0 -0
  44. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/egt2020/__init__.py +0 -0
  45. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/egt2020/deplacements.py +0 -0
  46. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/egt2020/motos.py +0 -0
  47. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/egt2020/personnes.py +0 -0
  48. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/egt2020/survey.py +0 -0
  49. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/egt2020/trajets.py +0 -0
  50. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/egt2020/voitures.py +0 -0
  51. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/emp/__init__.py +0 -0
  52. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/emp/deplacements.py +0 -0
  53. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/emp/menages.py +0 -0
  54. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/emp/motos.py +0 -0
  55. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/emp/voitures.py +0 -0
  56. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/logger.py +0 -0
  57. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/main.py +0 -0
  58. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/resources/__init__.py +0 -0
  59. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/resources/admin_express.py +0 -0
  60. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/resources/insee_data.py +0 -0
  61. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/resources/nuts.py +0 -0
  62. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/schema/__init__.py +0 -0
  63. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/schema/cars.py +0 -0
  64. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/schema/common.py +0 -0
  65. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/schema/guarantees.py +0 -0
  66. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/schema/legs.py +0 -0
  67. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/schema/motorcycles.py +0 -0
  68. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/schema/persons.py +0 -0
  69. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/schema/trips.py +0 -0
  70. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd/utils.py +0 -0
  71. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd.egg-info/SOURCES.txt +0 -0
  72. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd.egg-info/dependency_links.txt +0 -0
  73. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd.egg-info/requires.txt +0 -0
  74. {mobisurvstd-0.7.0 → mobisurvstd-0.8.0}/src/mobisurvstd.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: mobisurvstd
3
- Version: 0.7.0
3
+ Version: 0.8.0
4
4
  Summary: A tool to standardize French mobility survey datasets (EMC2, EGT, EMP, etc.) into a unified format.
5
5
  Author-email: Lucas Javaudin <lucas@lucasjavaudin.com>
6
6
  License-Expression: MIT
@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
6
6
 
7
7
  [project]
8
8
  name = "mobisurvstd"
9
- version = "0.7.0"
9
+ version = "0.8.0"
10
10
  description = "A tool to standardize French mobility survey datasets (EMC2, EGT, EMP, etc.) into a unified format."
11
11
  readme = "README.md"
12
12
  authors = [{ name = "Lucas Javaudin", email = "lucas@lucasjavaudin.com" }]
@@ -7,6 +7,7 @@ import polars as pl
7
7
  from loguru import logger
8
8
 
9
9
  from mobisurvstd.common.clean import clean
10
+ from mobisurvstd.common.trips import WEEKDAY_MAP
10
11
  from mobisurvstd.common.zones import get_coords
11
12
 
12
13
  from .common import EMC2_MODE_MAP, MODE_MAP
@@ -225,7 +226,10 @@ class CeremaReader(HouseholdsReader, PersonsReader, TripsReader, LegsReader, Zon
225
226
  )
226
227
  self.households = self.households.join(
227
228
  household_dates, on="household_id", how="left", coalesce=True
228
- ).with_columns(interview_date=pl.col("trip_date") + timedelta(days=1))
229
+ ).with_columns(
230
+ interview_date=pl.col("trip_date") + timedelta(days=1),
231
+ trips_weekday=pl.col("trip_date").dt.weekday().replace_strict(WEEKDAY_MAP),
232
+ )
229
233
 
230
234
  def fix_main_mode(self):
231
235
  # Special case for Douai 2012: Some trips have main mode set to "car_driver" but the legs
@@ -132,6 +132,17 @@ HOUSING_STATUS_MAP = {
132
132
  9: "other", # Autres
133
133
  }
134
134
 
135
+ WEEKDAY_MAP = {
136
+ 1: "monday",
137
+ 2: "tuesday",
138
+ 3: "wednesday",
139
+ 4: "thursday",
140
+ 5: "friday",
141
+ 6: "saturday",
142
+ 7: "sunday",
143
+ }
144
+
145
+
135
146
  REVENU_LB_MAP = {
136
147
  1: 0, # Moins de 800€
137
148
  2: 800, # De 800 à 1200€
@@ -292,6 +303,7 @@ def standardize_households(filename: str, detailed_zones: pl.DataFrame | None):
292
303
  # The interview date is 1 day after that date.
293
304
  interview_date=pl.concat_str("year", "week_number", "JDEP").str.to_date("%G%V%w")
294
305
  + timedelta(days=1),
306
+ trips_weekday=pl.col("JDEP").replace_strict(WEEKDAY_MAP),
295
307
  # It seems that the survey uses "990xx" codes to represent foreign countries but I did not
296
308
  # find the documentation for these codes so we set them all to the special code "99200"
297
309
  # (i.e., any foreign country).
@@ -91,6 +91,16 @@ REVENU_UB_MAP = {
91
91
  10: None, # 5500€ et plus
92
92
  }
93
93
 
94
+ WEEKDAY_MAP = {
95
+ "lundi": "monday",
96
+ "mardi": "tuesday",
97
+ "mercredi": "wednesday",
98
+ "jeudi": "thursday",
99
+ "vendredi": "friday",
100
+ "samedi": "saturday",
101
+ "dimanche": "sunday",
102
+ }
103
+
94
104
 
95
105
  def scan_households(filename: str):
96
106
  # We use the inefficient `read_csv().lazy()` because we need to use `encoding="latin1"`, which
@@ -128,6 +138,7 @@ def standardize_households(filename: str):
128
138
  original_household_id=pl.struct("IDCEREMA"),
129
139
  survey_method=pl.col("TYPE_QUEST").replace_strict(SURVEY_METHOD_MAP),
130
140
  interview_date=pl.date(year="ANNEE", month="MOIS", day="JOUR") + timedelta(days=1),
141
+ trips_weekday=pl.col("JOURSEM").replace_strict(WEEKDAY_MAP),
131
142
  income_lower_bound=pl.col("REVENU").replace_strict(REVENU_LB_MAP),
132
143
  income_upper_bound=pl.col("REVENU").replace_strict(REVENU_UB_MAP),
133
144
  housing_type=pl.col("TYPELOG").replace_strict(HOUSING_TYPE_MAP),
@@ -458,6 +458,16 @@ RESIDENT_TYPE_MAP = {
458
458
  5: None,
459
459
  }
460
460
 
461
+ WEEKDAY_MAP = {
462
+ "lundi": "monday",
463
+ "mardi": "tuesday",
464
+ "mercredi": "wednesday",
465
+ "jeudi": "thursday",
466
+ "vendredi": "friday",
467
+ "samedi": "saturday",
468
+ "dimanche": "sunday",
469
+ }
470
+
461
471
 
462
472
  def scan_persons(filename1: str, filename2: str, filename3: str):
463
473
  lf = (
@@ -527,14 +537,32 @@ def standardize_persons(filename1: str, filename2: str, filename3: str, househol
527
537
  has_bike_sharing_subscription=pl.col("BABONNVLS") == 1,
528
538
  has_travel_inconvenience=pl.col("GAGENE").is_in((1, 2, 3)),
529
539
  is_surveyed=pl.col("NOIK").is_not_null(),
540
+ trips_weekday=pl.col("MDATE_jour").replace_strict(WEEKDAY_MAP),
541
+ )
542
+ lf = lf.with_columns(
543
+ # Many persons have ETUDIE = 1 (they study) but SITUA is null.
544
+ # We can assign them "student:unspecified" as detailed_professional_occupation.
545
+ detailed_professional_occupation=pl.when(
546
+ pl.col("ETUDIE") == 1, pl.col("detailed_professional_occupation").is_null()
547
+ )
548
+ .then(pl.lit("student:unspecified"))
549
+ .otherwise("detailed_professional_occupation")
550
+ )
551
+ lf = lf.with_columns(
552
+ # Secondary professional occupation is work when the person is working (TRAVAILLE = 1) but
553
+ # it is not a worker (SITUA != 1). These are mostly retirees or students with a job.
554
+ # Secondary professional occupation is education when the person is studying (ETUDIE = 1)
555
+ # but it is not a student (professional_occupation != student).
530
556
  secondary_professional_occupation=pl.when(
531
557
  pl.col("TRAVAILLE").eq(1).and_(pl.col("SITUA").ne(1))
532
558
  )
533
559
  .then(pl.lit("work"))
534
- .when(pl.col("ETUDIE").eq(1).and_(pl.col("SITUA").is_in((2, 3)).not_()))
560
+ .when(
561
+ pl.col("ETUDIE")
562
+ .eq(1)
563
+ .and_(pl.col("detailed_professional_occupation").str.starts_with("student").not_())
564
+ )
535
565
  .then(pl.lit("education")),
536
- )
537
- lf = lf.with_columns(
538
566
  # Only specify the education-level for non-students.
539
567
  education_level=pl.when(
540
568
  pl.col("detailed_professional_occupation").str.starts_with("student").not_()
@@ -548,5 +576,5 @@ def standardize_persons(filename1: str, filename2: str, filename3: str, househol
548
576
  .then(pl.lit("reference_person"))
549
577
  .otherwise("reference_person_link"),
550
578
  )
551
- lf = clean(lf)
579
+ lf = clean(lf, extra_cols=["trips_weekday"])
552
580
  return lf
@@ -1,6 +1,7 @@
1
1
  from datetime import date
2
2
  from zipfile import ZipFile
3
3
 
4
+ import polars as pl
4
5
  from loguru import logger
5
6
 
6
7
  from mobisurvstd.common.clean import clean
@@ -43,6 +44,18 @@ def standardize(source: str | ZipFile, skip_spatial: bool = False):
43
44
  trips = standardize_trips(filename, persons)
44
45
  # Legs.
45
46
  legs = standardize_legs(filename, trips)
47
+ # Survey weekday is specified at the person-level, we create here the household-level
48
+ # `trips_weekday`.
49
+ household_weekdays = (
50
+ persons.filter(pl.col("trips_weekday").is_not_null())
51
+ .group_by("household_id")
52
+ .agg(pl.col("trips_weekday").first())
53
+ )
54
+ households = (
55
+ households.join(household_weekdays, on="household_id", how="left", coalesce=True)
56
+ .collect()
57
+ .lazy()
58
+ )
46
59
  return clean(
47
60
  households=households,
48
61
  persons=persons,
@@ -24,6 +24,11 @@ HOUSEHOLD_SCHEMA = [
24
24
  Variable("survey_method", pl.Enum(["face_to_face", "phone"]), [AllDefinedOrAllNull()]),
25
25
  # Date at which the interview took place.
26
26
  Variable("interview_date", pl.Date),
27
+ # Day of the week when the reported trips took place.
28
+ Variable(
29
+ "trips_weekday",
30
+ pl.Enum(["monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday"]),
31
+ ),
27
32
  # Sample weight of the household.
28
33
  Variable("sample_weight", pl.Float64, [AllDefinedOrAllNull(), NonNegative()]),
29
34
  # Longitude of home coordinates.
@@ -132,6 +132,29 @@ def validate_households(data):
132
132
  # Note that the NUTS correspondence guarantees and the `household_type` guarantees are not
133
133
  # validated because these variables are computed automatically by MobiSurvStd so the guarantees
134
134
  # are always satisfied by definition.
135
+ # Guarantee that `trip_weekday` is equal to the household's `trips_weekday` (when defined).
136
+ invalid_households = set(
137
+ (
138
+ data.trips.select("trip_id", "household_id", "trip_weekday")
139
+ .join(data.households.select("household_id", "trips_weekday"), on="household_id")
140
+ .filter(
141
+ pl.col("trips_weekday").is_not_null(),
142
+ pl.col("trip_weekday").is_not_null(),
143
+ pl.col("trip_weekday") != pl.col("trips_weekday"),
144
+ )
145
+ )["household_id"]
146
+ )
147
+ if invalid_households:
148
+ n = len(invalid_households)
149
+ logger.warning(
150
+ f"{n} households have `trips_weekday` != `trip_weekday` for at least one trip. "
151
+ "The `trips_weekday` value is set to null."
152
+ )
153
+ data.households = data.households.with_columns(
154
+ trips_weekday=pl.when(pl.col("household_id").is_in(invalid_households))
155
+ .then(None)
156
+ .otherwise("trips_weekday")
157
+ )
135
158
  # Guarantee that `nb_cars` is not smaller than the number of cars in `cars.parquet`.
136
159
  nb_cars = data.cars.group_by("household_id").len()
137
160
  invalid_households = set(
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: mobisurvstd
3
- Version: 0.7.0
3
+ Version: 0.8.0
4
4
  Summary: A tool to standardize French mobility survey datasets (EMC2, EGT, EMP, etc.) into a unified format.
5
5
  Author-email: Lucas Javaudin <lucas@lucasjavaudin.com>
6
6
  License-Expression: MIT
File without changes
File without changes
File without changes