mobisurvstd 0.8.2__tar.gz → 0.9.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {mobisurvstd-0.8.2/src/mobisurvstd.egg-info → mobisurvstd-0.9.0}/PKG-INFO +1 -1
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/pyproject.toml +1 -1
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/cerema/deplacements.py +2 -2
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/cerema/menages.py +2 -0
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/cerema/survey.py +6 -0
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/common/households.py +5 -4
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/egt2010/menages.py +1 -1
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/egt2010/personnes.py +5 -0
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/egt2020/menages.py +1 -1
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/egt2020/personnes.py +4 -2
- mobisurvstd-0.9.0/src/mobisurvstd/emp/common.py +22 -0
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/emp/deplacements.py +2 -21
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/emp/menages.py +11 -2
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/emp/personnes.py +0 -1
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/schema/households.py +12 -0
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/validation.py +2 -2
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0/src/mobisurvstd.egg-info}/PKG-INFO +1 -1
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd.egg-info/SOURCES.txt +1 -0
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/LICENSE.md +0 -0
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/README.md +0 -0
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/setup.cfg +0 -0
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/__init__.py +0 -0
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/__main__.py +0 -0
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/cerema/__init__.py +0 -0
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/cerema/common.py +0 -0
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/cerema/edgt.py +0 -0
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/cerema/edvm.py +0 -0
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/cerema/emc2.py +0 -0
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/cerema/emd.py +0 -0
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/cerema/personnes.py +0 -0
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/cerema/trajets.py +0 -0
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/cerema/zones.py +0 -0
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/classes.py +0 -0
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/cli.py +0 -0
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/common/__init__.py +0 -0
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/common/cars.py +0 -0
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/common/clean.py +0 -0
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/common/legs.py +0 -0
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/common/motorcycles.py +0 -0
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/common/persons.py +0 -0
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/common/trips.py +0 -0
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/common/zones.py +0 -0
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/egt2010/__init__.py +0 -0
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/egt2010/deplacements.py +0 -0
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/egt2010/survey.py +0 -0
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/egt2010/trajets.py +0 -0
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/egt2010/zones.py +0 -0
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/egt2020/__init__.py +0 -0
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/egt2020/deplacements.py +0 -0
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/egt2020/motos.py +0 -0
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/egt2020/survey.py +0 -0
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/egt2020/trajets.py +0 -0
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/egt2020/voitures.py +0 -0
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/emp/__init__.py +0 -0
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/emp/motos.py +0 -0
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/emp/survey.py +0 -0
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/emp/voitures.py +0 -0
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/logger.py +0 -0
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/main.py +0 -0
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/resources/__init__.py +0 -0
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/resources/admin_express.py +0 -0
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/resources/insee_data.py +0 -0
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/resources/nuts.py +0 -0
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/schema/__init__.py +0 -0
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/schema/cars.py +0 -0
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/schema/common.py +0 -0
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/schema/guarantees.py +0 -0
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/schema/legs.py +0 -0
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/schema/motorcycles.py +0 -0
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/schema/persons.py +0 -0
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/schema/trips.py +0 -0
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd/utils.py +0 -0
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd.egg-info/dependency_links.txt +0 -0
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd.egg-info/requires.txt +0 -0
- {mobisurvstd-0.8.2 → mobisurvstd-0.9.0}/src/mobisurvstd.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: mobisurvstd
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.9.0
|
|
4
4
|
Summary: A tool to standardize French mobility survey datasets (EMC2, EGT, EMP, etc.) into a unified format.
|
|
5
5
|
Author-email: Lucas Javaudin <lucas@lucasjavaudin.com>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
|
|
|
6
6
|
|
|
7
7
|
[project]
|
|
8
8
|
name = "mobisurvstd"
|
|
9
|
-
version = "0.
|
|
9
|
+
version = "0.9.0"
|
|
10
10
|
description = "A tool to standardize French mobility survey datasets (EMC2, EGT, EMP, etc.) into a unified format."
|
|
11
11
|
readme = "README.md"
|
|
12
12
|
authors = [{ name = "Lucas Javaudin", email = "lucas@lucasjavaudin.com" }]
|
|
@@ -2,6 +2,7 @@ import polars as pl
|
|
|
2
2
|
from loguru import logger
|
|
3
3
|
|
|
4
4
|
from mobisurvstd.common.trips import clean
|
|
5
|
+
from mobisurvstd.schema.common import CURRENT_YEAR
|
|
5
6
|
|
|
6
7
|
SCHEMA = {
|
|
7
8
|
"DP1": pl.UInt8, # Code fichier = 3 (déplacement)
|
|
@@ -183,7 +184,6 @@ class TripsReader:
|
|
|
183
184
|
lf = lf.sort("original_trip_id")
|
|
184
185
|
lf = fix_origin_destination_detailed_zones(lf)
|
|
185
186
|
df = lf.collect()
|
|
186
|
-
year = round(df["trip_date"].dt.year().mean())
|
|
187
187
|
# For Besançon 2018, there is one missing person.
|
|
188
188
|
n = df["person_id"].null_count()
|
|
189
189
|
if n > 0:
|
|
@@ -191,7 +191,7 @@ class TripsReader:
|
|
|
191
191
|
df = df.filter(pl.col("person_id").is_not_null())
|
|
192
192
|
self.trips = clean(
|
|
193
193
|
df.lazy(),
|
|
194
|
-
year=
|
|
194
|
+
year=self.survey_year() or CURRENT_YEAR,
|
|
195
195
|
special_locations=self.special_locations_coords,
|
|
196
196
|
detailed_zones=self.detailed_zones_coords,
|
|
197
197
|
)
|
|
@@ -3,6 +3,7 @@ import polars as pl
|
|
|
3
3
|
from mobisurvstd.common.cars import clean as clean_cars
|
|
4
4
|
from mobisurvstd.common.households import clean as clean_households
|
|
5
5
|
from mobisurvstd.common.motorcycles import clean as clean_motorcycles
|
|
6
|
+
from mobisurvstd.schema.common import CURRENT_YEAR
|
|
6
7
|
|
|
7
8
|
SCHEMA = {
|
|
8
9
|
"MP1": pl.UInt8, # Code fichier = 1
|
|
@@ -269,6 +270,7 @@ class HouseholdsReader:
|
|
|
269
270
|
)
|
|
270
271
|
self.households = clean_households(
|
|
271
272
|
lf,
|
|
273
|
+
year=self.survey_year() or CURRENT_YEAR,
|
|
272
274
|
special_locations=self.special_locations_coords,
|
|
273
275
|
detailed_zones=self.detailed_zones_coords,
|
|
274
276
|
)
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import re
|
|
1
2
|
from datetime import timedelta
|
|
2
3
|
from zipfile import ZipFile
|
|
3
4
|
|
|
@@ -66,6 +67,11 @@ class CeremaReader(HouseholdsReader, PersonsReader, TripsReader, LegsReader, Zon
|
|
|
66
67
|
def survey_name(self):
|
|
67
68
|
raise NotImplementedError
|
|
68
69
|
|
|
70
|
+
def survey_year(self):
|
|
71
|
+
name = self.survey_name()
|
|
72
|
+
matches = re.findall(r"_(\d{4})$", name)
|
|
73
|
+
return int(matches[0]) if matches else None
|
|
74
|
+
|
|
69
75
|
def standardize(self, skip_spatial: bool = False):
|
|
70
76
|
logger.info(f"Standardizing {self.SURVEY_TYPE} survey from `{self.source_name()}`")
|
|
71
77
|
if not self.validate():
|
|
@@ -11,6 +11,7 @@ from . import DEBUG
|
|
|
11
11
|
|
|
12
12
|
def clean(
|
|
13
13
|
lf: pl.LazyFrame,
|
|
14
|
+
year: int,
|
|
14
15
|
special_locations: pl.DataFrame | None = None,
|
|
15
16
|
detailed_zones: pl.DataFrame | None = None,
|
|
16
17
|
):
|
|
@@ -21,7 +22,7 @@ def clean(
|
|
|
21
22
|
lf = add_bicycle_counts(lf, existing_cols)
|
|
22
23
|
lf = add_lng_lat(lf, existing_cols, special_locations, detailed_zones)
|
|
23
24
|
lf = add_insee_columns(lf, existing_cols)
|
|
24
|
-
lf = add_insee_data_columns(lf, existing_cols)
|
|
25
|
+
lf = add_insee_data_columns(lf, existing_cols, year)
|
|
25
26
|
if "home_dep" in existing_cols:
|
|
26
27
|
lf = add_nuts_data(lf, "home")
|
|
27
28
|
if DEBUG:
|
|
@@ -87,10 +88,10 @@ def add_insee_columns(lf: pl.LazyFrame, existing_cols: list[str]):
|
|
|
87
88
|
return lf
|
|
88
89
|
|
|
89
90
|
|
|
90
|
-
def add_insee_data_columns(lf: pl.LazyFrame, existing_cols: list[str]):
|
|
91
|
-
"""Add insee name
|
|
91
|
+
def add_insee_data_columns(lf: pl.LazyFrame, existing_cols: list[str], year: int):
|
|
92
|
+
"""Add insee name, département code and all AAV / density columns for the home municipality."""
|
|
92
93
|
if "home_insee" in existing_cols:
|
|
93
94
|
# If the `home_dep` column already exists, then it is not added again.
|
|
94
|
-
lf = add_insee_data(lf, "home", year=
|
|
95
|
+
lf = add_insee_data(lf, "home", year=year, skip_dep="home_dep" in existing_cols)
|
|
95
96
|
existing_cols.append("home_dep")
|
|
96
97
|
return lf
|
|
@@ -311,7 +311,7 @@ def standardize_households(filename: str, detailed_zones: pl.DataFrame | None):
|
|
|
311
311
|
.then(pl.lit("99200"))
|
|
312
312
|
.otherwise("home_insee"),
|
|
313
313
|
)
|
|
314
|
-
lf = clean_households(lf, detailed_zones=detailed_zones)
|
|
314
|
+
lf = clean_households(lf, year=2010, detailed_zones=detailed_zones)
|
|
315
315
|
return lf
|
|
316
316
|
|
|
317
317
|
|
|
@@ -259,6 +259,7 @@ def standardize_persons(
|
|
|
259
259
|
has_driving_license=pl.when(pl.col("age") <= 15)
|
|
260
260
|
.then(pl.lit("no"))
|
|
261
261
|
.otherwise(pl.col("PERMVP").replace_strict(DRIVING_LICENSE_MAP)),
|
|
262
|
+
# PERM2RM value is NULL for all persons below 15.
|
|
262
263
|
has_motorcycle_driving_license=pl.when(pl.col("age") <= 15)
|
|
263
264
|
.then(pl.lit("no"))
|
|
264
265
|
.otherwise(pl.col("PERM2RM").replace_strict(DRIVING_LICENSE_MAP)),
|
|
@@ -273,6 +274,10 @@ def standardize_persons(
|
|
|
273
274
|
.otherwise(pl.lit("yes:unspecified"))
|
|
274
275
|
)
|
|
275
276
|
.when(pl.col("ABONTC").eq(1))
|
|
277
|
+
.then(pl.lit("no"))
|
|
278
|
+
# All persons below 4 have NULL value but can be assumed to have no public transit
|
|
279
|
+
# subscription.
|
|
280
|
+
.when(pl.col("age") <= 4)
|
|
276
281
|
.then(pl.lit("no")),
|
|
277
282
|
has_car_sharing_subscription=pl.col("ABONVP").eq(1),
|
|
278
283
|
has_bike_sharing_subscription=pl.col("ABONVLS").is_in((1, 2, 3)),
|
|
@@ -318,7 +318,6 @@ def standardize_persons(filename: str, households: pl.LazyFrame):
|
|
|
318
318
|
has_motorcycle_driving_license=pl.when(pl.col("age") <= 14)
|
|
319
319
|
.then(pl.lit("no"))
|
|
320
320
|
.otherwise(pl.col("PERM2RM").replace_strict(DRIVING_LICENSE_MAP)),
|
|
321
|
-
has_public_transit_subscription=pl.col("ABONTC").eq(1),
|
|
322
321
|
public_transit_subscription=pl.when(
|
|
323
322
|
# Forfait Navigo Gratuité, Améthyste et Gratuité Jeunes en insertion
|
|
324
323
|
pl.col("ABONTC_20").eq(1) | pl.col("ABONTC_30").eq(1) | pl.col("ABONTC_60").eq(1)
|
|
@@ -339,7 +338,10 @@ def standardize_persons(filename: str, households: pl.LazyFrame):
|
|
|
339
338
|
.then(pl.lit("yes:paid"))
|
|
340
339
|
.when(pl.col("ABONTC").eq(0))
|
|
341
340
|
.then(pl.lit("no"))
|
|
342
|
-
|
|
341
|
+
# All persons below 4 have NULL value but can be assumed to have no public transit
|
|
342
|
+
# subscription.
|
|
343
|
+
.when(pl.col("age") <= 4)
|
|
344
|
+
.then(pl.lit("no")),
|
|
343
345
|
has_car_sharing_subscription=pl.col("ABONVP").eq(1),
|
|
344
346
|
car_sharing_subscription=pl.when(pl.col("ABONVP").eq(0))
|
|
345
347
|
.then(pl.lit("no"))
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import polars as pl
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
def insee_density_col(col: str):
|
|
5
|
+
# The encoding of the file is incorrect so we need to resort to the following function to
|
|
6
|
+
# convert INSEE density names to codes.
|
|
7
|
+
return (
|
|
8
|
+
pl.when(pl.col(col).eq("Grands centres urbains"))
|
|
9
|
+
.then(1)
|
|
10
|
+
.when(pl.col(col).str.starts_with("Centres urbanis interm"))
|
|
11
|
+
.then(2)
|
|
12
|
+
.when(pl.col(col).eq("Ceintures urbaines"))
|
|
13
|
+
.then(3)
|
|
14
|
+
.when(pl.col(col).eq("Petites villes"))
|
|
15
|
+
.then(4)
|
|
16
|
+
.when(pl.col(col).eq("Bourgs ruraux"))
|
|
17
|
+
.then(5)
|
|
18
|
+
.when(pl.col(col).str.contains("habitat dispers"))
|
|
19
|
+
.then(6)
|
|
20
|
+
.when(pl.col(col).str.contains("habitat tr"))
|
|
21
|
+
.then(7)
|
|
22
|
+
)
|
|
@@ -4,6 +4,8 @@ from mobisurvstd.common.legs import clean as clean_legs
|
|
|
4
4
|
from mobisurvstd.common.trips import clean as clean_trips
|
|
5
5
|
from mobisurvstd.resources.nuts import NUTS_DF
|
|
6
6
|
|
|
7
|
+
from .common import insee_density_col
|
|
8
|
+
|
|
7
9
|
SCHEMA = {
|
|
8
10
|
"IDENT_DEP": pl.String, # Identifiant déplacement
|
|
9
11
|
"IDENT_MEN": pl.String, # Identifiant ménage
|
|
@@ -392,24 +394,3 @@ def time_col_to_seconds(col: str):
|
|
|
392
394
|
)
|
|
393
395
|
.struct.field("seconds")
|
|
394
396
|
)
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
def insee_density_col(col: str):
|
|
398
|
-
# The encoding of the file is incorrect so we need to resort to the following function to
|
|
399
|
-
# convert INSEE density names to codes.
|
|
400
|
-
return (
|
|
401
|
-
pl.when(pl.col(col).eq("Grands centres urbains"))
|
|
402
|
-
.then(1)
|
|
403
|
-
.when(pl.col(col).str.starts_with("Centres urbanis interm"))
|
|
404
|
-
.then(2)
|
|
405
|
-
.when(pl.col(col).eq("Ceintures urbaines"))
|
|
406
|
-
.then(3)
|
|
407
|
-
.when(pl.col(col).eq("Petites villes"))
|
|
408
|
-
.then(4)
|
|
409
|
-
.when(pl.col(col).eq("Bourgs ruraux"))
|
|
410
|
-
.then(5)
|
|
411
|
-
.when(pl.col(col).str.contains("habitat dispers"))
|
|
412
|
-
.then(6)
|
|
413
|
-
.when(pl.col(col).str.contains("habitat tr"))
|
|
414
|
-
.then(7)
|
|
415
|
-
)
|
|
@@ -159,17 +159,26 @@ def scan_households(filename1: str, filename2: str):
|
|
|
159
159
|
|
|
160
160
|
def standardize_households(filename1: str, filename2: str):
|
|
161
161
|
lf = scan_households(filename1, filename2)
|
|
162
|
-
lf = lf.rename(
|
|
162
|
+
lf = lf.rename(
|
|
163
|
+
{
|
|
164
|
+
"pond_menC": "sample_weight",
|
|
165
|
+
"DEP_RES": "home_dep",
|
|
166
|
+
"JNBVEH": "nb_cars",
|
|
167
|
+
"DENSITECOM_RES": "home_insee_density",
|
|
168
|
+
}
|
|
169
|
+
)
|
|
163
170
|
lf = lf.with_columns(
|
|
164
171
|
original_household_id=pl.struct("IDENT_MEN"),
|
|
165
172
|
survey_method=pl.lit("face_to_face"),
|
|
166
173
|
housing_status=(10 * pl.col("STOC") + pl.col("PROPRI").fill_null(0)).replace_strict(
|
|
167
174
|
HOUSING_STATUS_MAP
|
|
168
175
|
),
|
|
176
|
+
home_aav_category=pl.col("TAA2017_RES").replace({0: None, 8: None, 9: None}),
|
|
177
|
+
home_insee_aav_type=pl.col("CATCOM_AA_RES").replace({98: None, 99: None}),
|
|
169
178
|
# There should be no null values so we can safely sum the two columns without facing null
|
|
170
179
|
# propagation.
|
|
171
180
|
nb_motorcycles=pl.col("JNBMOTO") + pl.col("JNBCYCLO"),
|
|
172
181
|
nb_bicycles=pl.col("JNBVELOAD") + pl.col("JNBVELOENF"),
|
|
173
182
|
)
|
|
174
|
-
lf = clean(lf)
|
|
183
|
+
lf = clean(lf, year=None)
|
|
175
184
|
return lf
|
|
@@ -544,7 +544,6 @@ def standardize_persons(filename1: str, filename2: str, filename3: str, househol
|
|
|
544
544
|
is_surveyed=pl.col("NOIK").is_not_null(),
|
|
545
545
|
trips_weekday=pl.col("MDATE_jour").replace_strict(WEEKDAY_MAP),
|
|
546
546
|
)
|
|
547
|
-
breakpoint()
|
|
548
547
|
lf = lf.with_columns(
|
|
549
548
|
# Many persons have ETUDIE = 1 (they study) but SITUA is null.
|
|
550
549
|
# We can assign them "student:unspecified" as detailed_professional_occupation.
|
|
@@ -3,6 +3,7 @@ import polars as pl
|
|
|
3
3
|
from .common import Variable
|
|
4
4
|
from .guarantees import (
|
|
5
5
|
AllDefinedOrAllNull,
|
|
6
|
+
Bounded,
|
|
6
7
|
Defined,
|
|
7
8
|
EqualTo,
|
|
8
9
|
Indexed,
|
|
@@ -13,6 +14,7 @@ from .guarantees import (
|
|
|
13
14
|
Unique,
|
|
14
15
|
ValidDepCode,
|
|
15
16
|
ValidInsee,
|
|
17
|
+
ValueInSet,
|
|
16
18
|
)
|
|
17
19
|
|
|
18
20
|
HOUSEHOLD_SCHEMA = [
|
|
@@ -45,6 +47,16 @@ HOUSEHOLD_SCHEMA = [
|
|
|
45
47
|
Variable("home_insee", pl.String, [ValidInsee(), InseeConsistentWithDep("home_dep")]),
|
|
46
48
|
# Name of the municipality where the household is located.
|
|
47
49
|
Variable("home_insee_name", pl.String),
|
|
50
|
+
# Density category of the home municipality.
|
|
51
|
+
Variable("home_insee_density", pl.UInt8, [Bounded(1, 7)]),
|
|
52
|
+
# Category of the home municipality within the AAV.
|
|
53
|
+
Variable("home_insee_aav_type", pl.UInt8, [ValueInSet({11, 12, 13, 20, 30})]),
|
|
54
|
+
# Code of the AAV of the household home.
|
|
55
|
+
Variable("home_aav", pl.String),
|
|
56
|
+
# Name of the AAV of the household home.
|
|
57
|
+
Variable("home_aav_name", pl.String),
|
|
58
|
+
# Category of the AAV of the household home.
|
|
59
|
+
Variable("home_aav_category", pl.UInt8, [Bounded(1, 5)]),
|
|
48
60
|
# Département code of the household home.
|
|
49
61
|
Variable("home_dep", pl.String, [ValidDepCode()]),
|
|
50
62
|
# Département name of the household home.
|
|
@@ -175,7 +175,7 @@ def validate_households(data):
|
|
|
175
175
|
pl.col("household_id").replace_strict(
|
|
176
176
|
nb_cars["household_id"], nb_cars["len"], default=0
|
|
177
177
|
),
|
|
178
|
-
)
|
|
178
|
+
).cast(data.households["nb_cars"].dtype)
|
|
179
179
|
)
|
|
180
180
|
# Guarantee that `nb_motorcycles` is not smaller than the number of motorcycles in
|
|
181
181
|
# `motorcycles.parquet`.
|
|
@@ -201,7 +201,7 @@ def validate_households(data):
|
|
|
201
201
|
pl.col("household_id").replace_strict(
|
|
202
202
|
nb_motorcycles["household_id"], nb_motorcycles["len"], default=0
|
|
203
203
|
),
|
|
204
|
-
)
|
|
204
|
+
).cast(data.households["nb_cars"].dtype)
|
|
205
205
|
)
|
|
206
206
|
# Guarantee that `nb_persons` matches the actual number of persons.
|
|
207
207
|
nb_persons = data.persons.group_by("household_id").len()
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: mobisurvstd
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.9.0
|
|
4
4
|
Summary: A tool to standardize French mobility survey datasets (EMC2, EGT, EMP, etc.) into a unified format.
|
|
5
5
|
Author-email: Lucas Javaudin <lucas@lucasjavaudin.com>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -51,6 +51,7 @@ src/mobisurvstd/egt2020/survey.py
|
|
|
51
51
|
src/mobisurvstd/egt2020/trajets.py
|
|
52
52
|
src/mobisurvstd/egt2020/voitures.py
|
|
53
53
|
src/mobisurvstd/emp/__init__.py
|
|
54
|
+
src/mobisurvstd/emp/common.py
|
|
54
55
|
src/mobisurvstd/emp/deplacements.py
|
|
55
56
|
src/mobisurvstd/emp/menages.py
|
|
56
57
|
src/mobisurvstd/emp/motos.py
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|