mobisurvstd 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mobisurvstd/__init__.py +8 -0
- mobisurvstd/__main__.py +4 -0
- mobisurvstd/classes.py +344 -0
- mobisurvstd/clean.py +1661 -0
- mobisurvstd/cli.py +44 -0
- mobisurvstd/common/__init__.py +1 -0
- mobisurvstd/common/cars.py +120 -0
- mobisurvstd/common/clean.py +470 -0
- mobisurvstd/common/households.py +97 -0
- mobisurvstd/common/legs.py +136 -0
- mobisurvstd/common/modes.py +52 -0
- mobisurvstd/common/motorcycles.py +44 -0
- mobisurvstd/common/persons.py +228 -0
- mobisurvstd/common/trips.py +253 -0
- mobisurvstd/common/zones.py +43 -0
- mobisurvstd/egt2010/__init__.py +1 -0
- mobisurvstd/egt2010/deplacements.py +170 -0
- mobisurvstd/egt2010/menages.py +378 -0
- mobisurvstd/egt2010/personnes.py +307 -0
- mobisurvstd/egt2010/survey.py +84 -0
- mobisurvstd/egt2010/trajets.py +211 -0
- mobisurvstd/egt2010/zones.py +11 -0
- mobisurvstd/egt2020/__init__.py +1 -0
- mobisurvstd/egt2020/deplacements.py +173 -0
- mobisurvstd/egt2020/menages.py +162 -0
- mobisurvstd/egt2020/motos.py +103 -0
- mobisurvstd/egt2020/personnes.py +391 -0
- mobisurvstd/egt2020/survey.py +89 -0
- mobisurvstd/egt2020/trajets.py +317 -0
- mobisurvstd/egt2020/voitures.py +128 -0
- mobisurvstd/emc2/__init__.py +1 -0
- mobisurvstd/emc2/common.py +42 -0
- mobisurvstd/emc2/deplacements.py +153 -0
- mobisurvstd/emc2/menages.py +358 -0
- mobisurvstd/emc2/personnes.py +305 -0
- mobisurvstd/emc2/survey.py +309 -0
- mobisurvstd/emc2/trajets.py +274 -0
- mobisurvstd/emc2/zones.py +107 -0
- mobisurvstd/emp/__init__.py +1 -0
- mobisurvstd/emp/deplacements.py +407 -0
- mobisurvstd/emp/menages.py +182 -0
- mobisurvstd/emp/motos.py +126 -0
- mobisurvstd/emp/personnes.py +543 -0
- mobisurvstd/emp/survey.py +93 -0
- mobisurvstd/emp/voitures.py +218 -0
- mobisurvstd/logger.py +15 -0
- mobisurvstd/main.py +158 -0
- mobisurvstd/resources/__init__.py +0 -0
- mobisurvstd/resources/admin_express.py +88 -0
- mobisurvstd/resources/insee_data.py +307 -0
- mobisurvstd/resources/nuts.py +129 -0
- mobisurvstd/schema.py +902 -0
- mobisurvstd/utils.py +125 -0
- mobisurvstd-0.1.0.dist-info/METADATA +210 -0
- mobisurvstd-0.1.0.dist-info/RECORD +58 -0
- mobisurvstd-0.1.0.dist-info/WHEEL +5 -0
- mobisurvstd-0.1.0.dist-info/licenses/LICENSE.md +7 -0
- mobisurvstd-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
import polars as pl
|
|
2
|
+
|
|
3
|
+
from mobisurvstd.common.zones import add_lng_lat_columns
|
|
4
|
+
from mobisurvstd.resources.admin_express import find_insee
|
|
5
|
+
from mobisurvstd.resources.insee_data import add_insee_data
|
|
6
|
+
from mobisurvstd.resources.nuts import add_nuts_data
|
|
7
|
+
from mobisurvstd.schema import HOUSEHOLD_SCHEMA
|
|
8
|
+
|
|
9
|
+
from . import DEBUG
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def clean(
|
|
13
|
+
lf: pl.LazyFrame,
|
|
14
|
+
special_locations: pl.DataFrame | None = None,
|
|
15
|
+
detailed_zones: pl.DataFrame | None = None,
|
|
16
|
+
):
|
|
17
|
+
existing_cols = lf.collect_schema().names()
|
|
18
|
+
lf = lf.sort("original_household_id")
|
|
19
|
+
lf = indexing(lf)
|
|
20
|
+
lf = add_bicycle_counts(lf, existing_cols)
|
|
21
|
+
lf = add_lng_lat(lf, existing_cols, special_locations, detailed_zones)
|
|
22
|
+
lf = add_insee_columns(lf, existing_cols)
|
|
23
|
+
lf = add_insee_data_columns(lf, existing_cols)
|
|
24
|
+
if "home_dep" in existing_cols:
|
|
25
|
+
lf = add_nuts_data(lf, "home")
|
|
26
|
+
if DEBUG:
|
|
27
|
+
# Try to collect the schema to check if it is valid.
|
|
28
|
+
lf.collect_schema()
|
|
29
|
+
lf.collect()
|
|
30
|
+
return lf
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def indexing(lf: pl.LazyFrame):
|
|
34
|
+
lf = lf.with_columns(
|
|
35
|
+
household_id=pl.int_range(1, pl.len() + 1, dtype=HOUSEHOLD_SCHEMA["household_id"])
|
|
36
|
+
)
|
|
37
|
+
return lf
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def add_bicycle_counts(lf: pl.LazyFrame, existing_cols: list[str]):
|
|
41
|
+
has_nb = "nb_bicycles" in existing_cols
|
|
42
|
+
has_nb_std = "nb_standard_bicycles" in existing_cols
|
|
43
|
+
has_nb_elec = "nb_electric_bicycles" in existing_cols
|
|
44
|
+
if has_nb + has_nb_std + has_nb_elec == 2:
|
|
45
|
+
# The third column (missing) can be deduced from the first two.
|
|
46
|
+
if not has_nb:
|
|
47
|
+
lf = lf.with_columns(
|
|
48
|
+
nb_bicycles=pl.col("nb_standard_bicycles") + pl.col("nb_electric_bicycles")
|
|
49
|
+
)
|
|
50
|
+
if not has_nb_std:
|
|
51
|
+
lf = lf.with_columns(
|
|
52
|
+
nb_standard_bicycles=pl.col("nb_bicycles") - pl.col("nb_electric_bicycles")
|
|
53
|
+
)
|
|
54
|
+
if not has_nb_elec:
|
|
55
|
+
lf = lf.with_columns(
|
|
56
|
+
nb_electric_bicycles=pl.col("nb_bicycles") - pl.col("nb_standard_bicycles")
|
|
57
|
+
)
|
|
58
|
+
return lf
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def add_lng_lat(
|
|
62
|
+
lf: pl.LazyFrame,
|
|
63
|
+
existing_cols: list[str],
|
|
64
|
+
special_locations: pl.DataFrame | None,
|
|
65
|
+
detailed_zones: pl.DataFrame | None,
|
|
66
|
+
):
|
|
67
|
+
for coords, name in (
|
|
68
|
+
(special_locations, "special_location"),
|
|
69
|
+
(detailed_zones, "detailed_zone"),
|
|
70
|
+
):
|
|
71
|
+
if coords is not None and f"home_{name}" in existing_cols:
|
|
72
|
+
lf = add_lng_lat_columns(lf, existing_cols, coords, prefix="home", name=name)
|
|
73
|
+
return lf
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def add_insee_columns(lf: pl.LazyFrame, existing_cols: list[str]):
|
|
77
|
+
"""Add `home_insee`column (if it does not exist already), by reading the home longitudes and
|
|
78
|
+
latitudes.
|
|
79
|
+
"""
|
|
80
|
+
if (
|
|
81
|
+
"home_insee" in existing_cols
|
|
82
|
+
or "home_lng" not in existing_cols
|
|
83
|
+
or "home_lat" not in existing_cols
|
|
84
|
+
):
|
|
85
|
+
return lf
|
|
86
|
+
lf = find_insee(lf, "home", "household_id")
|
|
87
|
+
existing_cols.append("home_insee")
|
|
88
|
+
return lf
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def add_insee_data_columns(lf: pl.LazyFrame, existing_cols: list[str]):
|
|
92
|
+
"""Add insee name and département code for the home municipality."""
|
|
93
|
+
if "home_insee" in existing_cols:
|
|
94
|
+
# If the `home_dep` column already exists, then it is not added again.
|
|
95
|
+
lf = add_insee_data(lf, "home", year=None, skip_dep="home_dep" in existing_cols)
|
|
96
|
+
existing_cols.append("home_dep")
|
|
97
|
+
return lf
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
import polars as pl
|
|
2
|
+
|
|
3
|
+
from mobisurvstd.common.modes import MODE_TO_GROUP
|
|
4
|
+
from mobisurvstd.common.zones import add_lng_lat_columns
|
|
5
|
+
from mobisurvstd.resources.admin_express import find_insee
|
|
6
|
+
from mobisurvstd.resources.insee_data import add_insee_data
|
|
7
|
+
from mobisurvstd.resources.nuts import add_nuts_data
|
|
8
|
+
from mobisurvstd.schema import LEG_SCHEMA
|
|
9
|
+
|
|
10
|
+
from . import DEBUG
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def clean(
|
|
14
|
+
lf: pl.LazyFrame,
|
|
15
|
+
special_locations: pl.DataFrame | None = None,
|
|
16
|
+
detailed_zones: pl.DataFrame | None = None,
|
|
17
|
+
):
|
|
18
|
+
existing_cols = lf.collect_schema().names()
|
|
19
|
+
lf = lf.sort("original_leg_id")
|
|
20
|
+
lf = add_indexing(lf, existing_cols)
|
|
21
|
+
lf = add_nb_persons_in_vehicle(lf, existing_cols)
|
|
22
|
+
lf = add_mode_groups(lf, existing_cols)
|
|
23
|
+
lf = add_lng_lat(lf, existing_cols, special_locations, detailed_zones)
|
|
24
|
+
lf = add_insee_columns(lf, existing_cols)
|
|
25
|
+
lf = add_insee_data_columns(lf, existing_cols)
|
|
26
|
+
lf = add_nuts_columns(lf, existing_cols)
|
|
27
|
+
if DEBUG:
|
|
28
|
+
# Try to collect the schema to check if it is valid.
|
|
29
|
+
lf.collect_schema()
|
|
30
|
+
lf.collect()
|
|
31
|
+
return lf
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def add_indexing(lf: pl.LazyFrame, existing_cols: list[str]):
|
|
35
|
+
"""Add columns `leg_index`, `first_leg`, `last_leg` to a leg LazyFrame."""
|
|
36
|
+
if "leg_id" not in existing_cols:
|
|
37
|
+
lf = lf.with_columns(leg_id=pl.int_range(1, pl.len() + 1, dtype=LEG_SCHEMA["trip_id"]))
|
|
38
|
+
existing_cols.append("leg_id")
|
|
39
|
+
if "leg_index" not in existing_cols:
|
|
40
|
+
lf = lf.with_columns(
|
|
41
|
+
leg_index=pl.int_range(1, pl.len() + 1, dtype=LEG_SCHEMA["leg_index"]).over("trip_id")
|
|
42
|
+
)
|
|
43
|
+
existing_cols.append("leg_index")
|
|
44
|
+
if "first_leg" not in existing_cols:
|
|
45
|
+
lf = lf.with_columns(first_leg=pl.col("leg_index").eq(1))
|
|
46
|
+
existing_cols.append("first_leg")
|
|
47
|
+
if "last_leg" not in existing_cols:
|
|
48
|
+
lf = lf.with_columns(
|
|
49
|
+
last_leg=pl.col("leg_index").eq(pl.col("leg_index").last().over("trip_id"))
|
|
50
|
+
)
|
|
51
|
+
existing_cols.append("last_leg")
|
|
52
|
+
return lf
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def add_nb_persons_in_vehicle(lf: pl.LazyFrame, existing_cols: list[str]):
|
|
56
|
+
if (
|
|
57
|
+
"nb_household_members_in_vehicle" in existing_cols
|
|
58
|
+
and "nb_non_household_members_in_vehicle" in existing_cols
|
|
59
|
+
and "nb_persons_in_vehicle" not in existing_cols
|
|
60
|
+
):
|
|
61
|
+
lf = lf.with_columns(
|
|
62
|
+
nb_persons_in_vehicle=pl.col("nb_household_members_in_vehicle")
|
|
63
|
+
+ pl.col("nb_non_household_members_in_vehicle")
|
|
64
|
+
)
|
|
65
|
+
existing_cols.append("nb_persons_in_vehicle")
|
|
66
|
+
if (
|
|
67
|
+
"nb_majors_in_vehicle" in existing_cols
|
|
68
|
+
and "nb_minors_in_vehicle" in existing_cols
|
|
69
|
+
and "nb_persons_in_vehicle" not in existing_cols
|
|
70
|
+
):
|
|
71
|
+
lf = lf.with_columns(
|
|
72
|
+
nb_persons_in_vehicle=pl.col("nb_majors_in_vehicle") + pl.col("nb_minors_in_vehicle")
|
|
73
|
+
)
|
|
74
|
+
existing_cols.append("nb_persons_in_vehicle")
|
|
75
|
+
return lf
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def add_mode_groups(lf: pl.LazyFrame, existing_cols: list[str]):
|
|
79
|
+
"""Add `mode_group` columns (if the `mode` column exists)."""
|
|
80
|
+
if "mode" in existing_cols:
|
|
81
|
+
lf = lf.with_columns(mode_group=pl.col("mode").replace_strict(MODE_TO_GROUP))
|
|
82
|
+
existing_cols.append("mode_group")
|
|
83
|
+
return lf
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def add_lng_lat(
|
|
87
|
+
lf: pl.LazyFrame,
|
|
88
|
+
existing_cols: list[str],
|
|
89
|
+
special_locations: pl.DataFrame | None,
|
|
90
|
+
detailed_zones: pl.DataFrame | None,
|
|
91
|
+
):
|
|
92
|
+
for coords, name in (
|
|
93
|
+
(special_locations, "special_location"),
|
|
94
|
+
(detailed_zones, "detailed_zone"),
|
|
95
|
+
):
|
|
96
|
+
if coords is not None and (
|
|
97
|
+
f"start_{name}" in existing_cols or f"end_{name}" in existing_cols
|
|
98
|
+
):
|
|
99
|
+
lf = add_lng_lat_columns(lf, existing_cols, coords, prefix="start", name=name)
|
|
100
|
+
lf = add_lng_lat_columns(lf, existing_cols, coords, prefix="end", name=name)
|
|
101
|
+
return lf
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def add_insee_columns(lf: pl.LazyFrame, existing_cols: list[str]):
|
|
105
|
+
"""Add `start_insee` and `end_insee` columns (if they do not exist already), by reading the
|
|
106
|
+
start and end longitudes and latitudes.
|
|
107
|
+
"""
|
|
108
|
+
for prefix in ("start", "end"):
|
|
109
|
+
if (
|
|
110
|
+
f"{prefix}_insee" in existing_cols
|
|
111
|
+
or f"{prefix}_lng" not in existing_cols
|
|
112
|
+
or f"{prefix}_lat" not in existing_cols
|
|
113
|
+
):
|
|
114
|
+
continue
|
|
115
|
+
lf = find_insee(lf, prefix, "leg_id")
|
|
116
|
+
existing_cols.append(f"{prefix}_insee")
|
|
117
|
+
return lf
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def add_insee_data_columns(lf: pl.LazyFrame, existing_cols: list[str]):
|
|
121
|
+
"""Add all insee data for the origin and destination municipalities."""
|
|
122
|
+
for prefix in ("start", "end"):
|
|
123
|
+
if f"{prefix}_insee" in existing_cols:
|
|
124
|
+
# If the corresponding `*_dep` column already exists, then it is not added again.
|
|
125
|
+
lf = add_insee_data(lf, prefix, skip_dep=f"{prefix}_dep" in existing_cols)
|
|
126
|
+
existing_cols.append(f"{prefix}_insee_name")
|
|
127
|
+
existing_cols.append(f"{prefix}_dep")
|
|
128
|
+
return lf
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def add_nuts_columns(lf: pl.LazyFrame, existing_cols: list[str]):
|
|
132
|
+
"""Add all département and NUTS-related columns."""
|
|
133
|
+
for prefix in ("start", "end"):
|
|
134
|
+
if f"{prefix}_dep" in existing_cols:
|
|
135
|
+
lf = add_nuts_data(lf, prefix)
|
|
136
|
+
return lf
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
MODE_TO_GROUP = {
|
|
2
|
+
# Walking
|
|
3
|
+
"walking": "walking",
|
|
4
|
+
# Bicycle
|
|
5
|
+
"bicycle:driver": "bicycle",
|
|
6
|
+
"bicycle:driver:shared": "bicycle",
|
|
7
|
+
"bicycle:driver:traditional": "bicycle",
|
|
8
|
+
"bicycle:driver:traditional:shared": "bicycle",
|
|
9
|
+
"bicycle:driver:electric": "bicycle",
|
|
10
|
+
"bicycle:driver:electric:shared": "bicycle",
|
|
11
|
+
"bicycle:passenger": "bicycle",
|
|
12
|
+
# Motorcycle
|
|
13
|
+
"motorcycle:driver": "motorcycle",
|
|
14
|
+
"motorcycle:passenger": "motorcycle",
|
|
15
|
+
"motorcycle:driver:moped": "motorcycle",
|
|
16
|
+
"motorcycle:passenger:moped": "motorcycle",
|
|
17
|
+
"motorcycle:driver:moto": "motorcycle",
|
|
18
|
+
"motorcycle:passenger:moto": "motorcycle",
|
|
19
|
+
# Car
|
|
20
|
+
"car:driver": "car_driver",
|
|
21
|
+
"car:passenger": "car_passenger",
|
|
22
|
+
"taxi": "car_passenger",
|
|
23
|
+
"VTC": "car_passenger",
|
|
24
|
+
"taxi_or_VTC": "car_passenger",
|
|
25
|
+
# Public transit
|
|
26
|
+
"public_transit:urban": "public_transit",
|
|
27
|
+
"public_transit:urban:bus": "public_transit",
|
|
28
|
+
"public_transit:urban:coach": "public_transit",
|
|
29
|
+
"public_transit:urban:tram": "public_transit",
|
|
30
|
+
"public_transit:urban:metro": "public_transit",
|
|
31
|
+
"public_transit:urban:funicular": "public_transit",
|
|
32
|
+
"public_transit:urban:rail": "public_transit",
|
|
33
|
+
"public_transit:urban:TER": "public_transit",
|
|
34
|
+
"public_transit:urban:demand_responsive": "public_transit",
|
|
35
|
+
"public_transit:interurban:coach": "public_transit",
|
|
36
|
+
"public_transit:interurban:TGV": "public_transit",
|
|
37
|
+
"public_transit:interurban:intercités": "public_transit",
|
|
38
|
+
"public_transit:interurban:other_train": "public_transit",
|
|
39
|
+
"public_transit:school": "public_transit",
|
|
40
|
+
# Other
|
|
41
|
+
"reduced_mobility_transport": "other",
|
|
42
|
+
"employer_transport": "other",
|
|
43
|
+
"truck:driver": "other",
|
|
44
|
+
"truck:passenger": "other",
|
|
45
|
+
"water_transport": "other",
|
|
46
|
+
"airplane": "other",
|
|
47
|
+
"wheelchair": "other",
|
|
48
|
+
"personal_transporter:non_motorized": "other",
|
|
49
|
+
"personal_transporter:motorized": "other",
|
|
50
|
+
"personal_transporter:unspecified": "other",
|
|
51
|
+
"other": "other",
|
|
52
|
+
}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import polars as pl
|
|
2
|
+
|
|
3
|
+
from mobisurvstd.schema import MOTORCYCLE_SCHEMA
|
|
4
|
+
|
|
5
|
+
from . import DEBUG
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def clean(lf: pl.LazyFrame):
|
|
9
|
+
existing_cols = lf.collect_schema().names()
|
|
10
|
+
lf = lf.sort("original_motorcycle_id")
|
|
11
|
+
lf = indexing(lf, existing_cols)
|
|
12
|
+
lf = add_mileage_bounds(lf, existing_cols)
|
|
13
|
+
if DEBUG:
|
|
14
|
+
# Try to collect the schema to check if it is valid.
|
|
15
|
+
lf.collect_schema()
|
|
16
|
+
lf.collect()
|
|
17
|
+
return lf
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def indexing(lf: pl.LazyFrame, existing_cols: list[str]):
|
|
21
|
+
if "motorcycle_id" not in existing_cols:
|
|
22
|
+
lf = lf.with_columns(
|
|
23
|
+
motorcycle_id=pl.int_range(1, pl.len() + 1, dtype=MOTORCYCLE_SCHEMA["motorcycle_id"])
|
|
24
|
+
)
|
|
25
|
+
existing_cols.append("motorcycle_id")
|
|
26
|
+
if "motorcycle_index" not in existing_cols:
|
|
27
|
+
lf = lf.with_columns(
|
|
28
|
+
motorcycle_index=pl.int_range(
|
|
29
|
+
1, pl.len() + 1, dtype=MOTORCYCLE_SCHEMA["motorcycle_index"]
|
|
30
|
+
).over("household_id")
|
|
31
|
+
)
|
|
32
|
+
existing_cols.append("motorcycle_index")
|
|
33
|
+
return lf
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def add_mileage_bounds(lf: pl.LazyFrame, existing_cols: list[str]):
|
|
37
|
+
col = "annual_mileage"
|
|
38
|
+
lb_col = f"{col}_lower_bound"
|
|
39
|
+
ub_col = f"{col}_upper_bound"
|
|
40
|
+
if col in existing_cols and lb_col not in existing_cols and ub_col not in existing_cols:
|
|
41
|
+
lf = lf.with_columns(pl.col(col).alias(lb_col), pl.col(col).alias(ub_col))
|
|
42
|
+
existing_cols.append(lb_col)
|
|
43
|
+
existing_cols.append(ub_col)
|
|
44
|
+
return lf
|
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
import polars as pl
|
|
2
|
+
|
|
3
|
+
from mobisurvstd.common.zones import add_lng_lat_columns
|
|
4
|
+
from mobisurvstd.resources.admin_express import find_insee
|
|
5
|
+
from mobisurvstd.resources.insee_data import add_insee_data
|
|
6
|
+
from mobisurvstd.resources.nuts import add_nuts_data
|
|
7
|
+
from mobisurvstd.schema import PERSON_SCHEMA
|
|
8
|
+
|
|
9
|
+
from . import DEBUG
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def clean(
|
|
13
|
+
lf: pl.LazyFrame,
|
|
14
|
+
special_locations: pl.DataFrame | None = None,
|
|
15
|
+
detailed_zones: pl.DataFrame | None = None,
|
|
16
|
+
):
|
|
17
|
+
existing_cols = lf.collect_schema().names()
|
|
18
|
+
lf = lf.sort("original_person_id")
|
|
19
|
+
lf = indexing(lf, existing_cols)
|
|
20
|
+
lf = add_age_class(lf, existing_cols)
|
|
21
|
+
lf = add_pcs_group(lf, existing_cols)
|
|
22
|
+
lf = add_professional_occupation(lf, existing_cols)
|
|
23
|
+
lf = add_student_group(lf, existing_cols)
|
|
24
|
+
lf = add_has_pt_subscription(lf, existing_cols)
|
|
25
|
+
lf = add_work_only_at_home(lf, existing_cols)
|
|
26
|
+
lf = add_lng_lat(lf, existing_cols, special_locations, detailed_zones)
|
|
27
|
+
lf = add_insee_columns(lf, existing_cols)
|
|
28
|
+
lf = add_insee_data_columns(lf, existing_cols)
|
|
29
|
+
lf = add_nuts_columns(lf, existing_cols)
|
|
30
|
+
if DEBUG:
|
|
31
|
+
# Try to collect the schema to check if it is valid.
|
|
32
|
+
lf.collect_schema()
|
|
33
|
+
lf.collect()
|
|
34
|
+
return lf
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def indexing(lf: pl.LazyFrame, existing_cols: list[str]):
|
|
38
|
+
if "person_id" not in existing_cols:
|
|
39
|
+
lf = lf.with_columns(
|
|
40
|
+
person_id=pl.int_range(1, pl.len() + 1, dtype=PERSON_SCHEMA["person_id"])
|
|
41
|
+
)
|
|
42
|
+
existing_cols.append("person_id")
|
|
43
|
+
if "person_index" not in existing_cols:
|
|
44
|
+
lf = lf.with_columns(
|
|
45
|
+
person_index=pl.int_range(1, pl.len() + 1, dtype=PERSON_SCHEMA["person_index"]).over(
|
|
46
|
+
"household_id"
|
|
47
|
+
)
|
|
48
|
+
)
|
|
49
|
+
existing_cols.append("person_index")
|
|
50
|
+
return lf
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def age_to_age_class(expr: pl.Expr):
|
|
54
|
+
return (
|
|
55
|
+
pl.when(expr < 18)
|
|
56
|
+
.then(pl.lit("17-"))
|
|
57
|
+
.when(expr < 25)
|
|
58
|
+
.then(pl.lit("18-24"))
|
|
59
|
+
.when(expr < 35)
|
|
60
|
+
.then(pl.lit("25-34"))
|
|
61
|
+
.when(expr < 50)
|
|
62
|
+
.then(pl.lit("35-49"))
|
|
63
|
+
.when(expr < 65)
|
|
64
|
+
.then(pl.lit("50-64"))
|
|
65
|
+
.when(expr < 75)
|
|
66
|
+
.then(pl.lit("65-74"))
|
|
67
|
+
.otherwise(pl.lit("75+"))
|
|
68
|
+
.cast(PERSON_SCHEMA["age_class"])
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
AGE_CLASS_TO_CODE = {
|
|
73
|
+
"17-": 1,
|
|
74
|
+
"18-24": 2,
|
|
75
|
+
"25-34": 3,
|
|
76
|
+
"35-49": 4,
|
|
77
|
+
"50-64": 5,
|
|
78
|
+
"65-74": 6,
|
|
79
|
+
"75+": 7,
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def add_age_class(lf: pl.LazyFrame, existing_cols: list[str]):
|
|
84
|
+
if "age" in existing_cols and "age_class" not in existing_cols:
|
|
85
|
+
lf = lf.with_columns(age_class=age_to_age_class(pl.col("age")))
|
|
86
|
+
existing_cols.append("age_class")
|
|
87
|
+
if "age_class" in existing_cols:
|
|
88
|
+
assert "age_class_code" not in existing_cols
|
|
89
|
+
lf = lf.with_columns(age_class_code=pl.col("age_class").replace_strict(AGE_CLASS_TO_CODE))
|
|
90
|
+
existing_cols.append("age_class_code")
|
|
91
|
+
return lf
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def add_professional_occupation(lf: pl.LazyFrame, existing_cols: list[str]):
|
|
95
|
+
if (
|
|
96
|
+
"detailed_professional_occupation" in existing_cols
|
|
97
|
+
and "professional_occupation" not in existing_cols
|
|
98
|
+
):
|
|
99
|
+
lf = lf.with_columns(
|
|
100
|
+
professional_occupation=pl.col("detailed_professional_occupation").str.extract(
|
|
101
|
+
"(\w+):?"
|
|
102
|
+
)
|
|
103
|
+
)
|
|
104
|
+
existing_cols.append("professional_occupation")
|
|
105
|
+
return lf
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
PCS_CODES = {
|
|
109
|
+
1: "agriculteurs_exploitants",
|
|
110
|
+
2: "artisans_commerçants_chefs_d'entreprise",
|
|
111
|
+
3: "cadres_et_professions_intellectuelles_supérieures",
|
|
112
|
+
4: "professions_intermédiaires",
|
|
113
|
+
5: "employés",
|
|
114
|
+
6: "ouvriers",
|
|
115
|
+
7: "retraités",
|
|
116
|
+
8: "autres_personnes_sans_activité_professionnelle",
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def add_pcs_group(lf: pl.LazyFrame, existing_cols: list[str]):
|
|
121
|
+
if "pcs_group_code" in existing_cols and "pcs_group" not in existing_cols:
|
|
122
|
+
lf = lf.with_columns(pcs_group=pl.col("pcs_group_code").replace_strict(PCS_CODES))
|
|
123
|
+
existing_cols.append("pcs_group")
|
|
124
|
+
return lf
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
STUDENT_GROUP_MAP = {
|
|
128
|
+
"maternelle": "primaire",
|
|
129
|
+
"primaire": "primaire",
|
|
130
|
+
"collège:6e": "collège",
|
|
131
|
+
"collège:5e": "collège",
|
|
132
|
+
"collège:4e": "collège",
|
|
133
|
+
"collège:3e": "collège",
|
|
134
|
+
"collège:SEGPA": "collège",
|
|
135
|
+
"lycée:seconde": "lycée",
|
|
136
|
+
"lycée:première": "lycée",
|
|
137
|
+
"lycée:terminale": "lycée",
|
|
138
|
+
"lycée:CAP": "lycée",
|
|
139
|
+
"supérieur:technique": "supérieur",
|
|
140
|
+
"supérieur:prépa1": "supérieur",
|
|
141
|
+
"supérieur:prépa2": "supérieur",
|
|
142
|
+
"supérieur:BAC+1": "supérieur",
|
|
143
|
+
"supérieur:BAC+2": "supérieur",
|
|
144
|
+
"supérieur:BAC+3": "supérieur",
|
|
145
|
+
"supérieur:BAC+4": "supérieur",
|
|
146
|
+
"supérieur:BAC+5": "supérieur",
|
|
147
|
+
"supérieur:BAC+6&+": "supérieur",
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def add_student_group(lf: pl.LazyFrame, existing_cols: list[str]):
|
|
152
|
+
if "student_category" in existing_cols and "student_group" not in existing_cols:
|
|
153
|
+
lf = lf.with_columns(
|
|
154
|
+
student_group=pl.col("student_category").replace_strict(STUDENT_GROUP_MAP)
|
|
155
|
+
)
|
|
156
|
+
return lf
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def add_has_pt_subscription(lf: pl.LazyFrame, existing_cols: list[str]):
|
|
160
|
+
if (
|
|
161
|
+
"public_transit_subscription" in existing_cols
|
|
162
|
+
and "has_public_transit_subscription" not in existing_cols
|
|
163
|
+
):
|
|
164
|
+
lf = lf.with_columns(
|
|
165
|
+
has_public_transit_subscription=pl.col("public_transit_subscription").str.starts_with(
|
|
166
|
+
"yes:"
|
|
167
|
+
)
|
|
168
|
+
)
|
|
169
|
+
existing_cols.append("has_public_transit_subscription")
|
|
170
|
+
return lf
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def add_work_only_at_home(lf: pl.LazyFrame, existing_cols: list[str]):
|
|
174
|
+
if "workplace_singularity" in existing_cols and "work_only_at_home" not in existing_cols:
|
|
175
|
+
lf = lf.with_columns(work_only_at_home=pl.col("workplace_singularity").eq("unique:home"))
|
|
176
|
+
return lf
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def add_lng_lat(
|
|
180
|
+
lf: pl.LazyFrame,
|
|
181
|
+
existing_cols: list[str],
|
|
182
|
+
special_locations: pl.DataFrame | None,
|
|
183
|
+
detailed_zones: pl.DataFrame | None,
|
|
184
|
+
):
|
|
185
|
+
for coords, name in (
|
|
186
|
+
(special_locations, "special_location"),
|
|
187
|
+
(detailed_zones, "detailed_zone"),
|
|
188
|
+
):
|
|
189
|
+
if coords is not None and (
|
|
190
|
+
f"work_{name}" in existing_cols or f"study_{name}" in existing_cols
|
|
191
|
+
):
|
|
192
|
+
lf = add_lng_lat_columns(lf, existing_cols, coords, prefix="work", name=name)
|
|
193
|
+
lf = add_lng_lat_columns(lf, existing_cols, coords, prefix="study", name=name)
|
|
194
|
+
return lf
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def add_insee_columns(lf: pl.LazyFrame, existing_cols: list[str]):
|
|
198
|
+
"""Add `work_insee` and `study_insee` columns (if they do not exist already), by reading
|
|
199
|
+
the work and study longitudes and latitudes.
|
|
200
|
+
"""
|
|
201
|
+
for prefix in ("work", "study"):
|
|
202
|
+
if (
|
|
203
|
+
f"{prefix}_insee" in existing_cols
|
|
204
|
+
or f"{prefix}_lng" not in existing_cols
|
|
205
|
+
or f"{prefix}_lat" not in existing_cols
|
|
206
|
+
):
|
|
207
|
+
continue
|
|
208
|
+
lf = find_insee(lf, prefix, "person_id")
|
|
209
|
+
existing_cols.append(f"{prefix}_insee")
|
|
210
|
+
return lf
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def add_insee_data_columns(lf: pl.LazyFrame, existing_cols: list[str]):
|
|
214
|
+
for prefix in ("work", "study"):
|
|
215
|
+
insee_col = f"{prefix}_insee"
|
|
216
|
+
dep_col = f"{prefix}_dep"
|
|
217
|
+
if insee_col in existing_cols:
|
|
218
|
+
lf = add_insee_data(lf, prefix, year=None, skip_dep=dep_col in existing_cols)
|
|
219
|
+
existing_cols.append(dep_col)
|
|
220
|
+
return lf
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def add_nuts_columns(lf: pl.LazyFrame, existing_cols: list[str]):
|
|
224
|
+
"""Add all département and NUTS-related columns."""
|
|
225
|
+
for prefix in ("work", "study"):
|
|
226
|
+
if f"{prefix}_dep" in existing_cols:
|
|
227
|
+
lf = add_nuts_data(lf, prefix)
|
|
228
|
+
return lf
|