mobisurvstd 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mobisurvstd/__init__.py +8 -0
- mobisurvstd/__main__.py +4 -0
- mobisurvstd/classes.py +344 -0
- mobisurvstd/clean.py +1661 -0
- mobisurvstd/cli.py +44 -0
- mobisurvstd/common/__init__.py +1 -0
- mobisurvstd/common/cars.py +120 -0
- mobisurvstd/common/clean.py +470 -0
- mobisurvstd/common/households.py +97 -0
- mobisurvstd/common/legs.py +136 -0
- mobisurvstd/common/modes.py +52 -0
- mobisurvstd/common/motorcycles.py +44 -0
- mobisurvstd/common/persons.py +228 -0
- mobisurvstd/common/trips.py +253 -0
- mobisurvstd/common/zones.py +43 -0
- mobisurvstd/egt2010/__init__.py +1 -0
- mobisurvstd/egt2010/deplacements.py +170 -0
- mobisurvstd/egt2010/menages.py +378 -0
- mobisurvstd/egt2010/personnes.py +307 -0
- mobisurvstd/egt2010/survey.py +84 -0
- mobisurvstd/egt2010/trajets.py +211 -0
- mobisurvstd/egt2010/zones.py +11 -0
- mobisurvstd/egt2020/__init__.py +1 -0
- mobisurvstd/egt2020/deplacements.py +173 -0
- mobisurvstd/egt2020/menages.py +162 -0
- mobisurvstd/egt2020/motos.py +103 -0
- mobisurvstd/egt2020/personnes.py +391 -0
- mobisurvstd/egt2020/survey.py +89 -0
- mobisurvstd/egt2020/trajets.py +317 -0
- mobisurvstd/egt2020/voitures.py +128 -0
- mobisurvstd/emc2/__init__.py +1 -0
- mobisurvstd/emc2/common.py +42 -0
- mobisurvstd/emc2/deplacements.py +153 -0
- mobisurvstd/emc2/menages.py +358 -0
- mobisurvstd/emc2/personnes.py +305 -0
- mobisurvstd/emc2/survey.py +309 -0
- mobisurvstd/emc2/trajets.py +274 -0
- mobisurvstd/emc2/zones.py +107 -0
- mobisurvstd/emp/__init__.py +1 -0
- mobisurvstd/emp/deplacements.py +407 -0
- mobisurvstd/emp/menages.py +182 -0
- mobisurvstd/emp/motos.py +126 -0
- mobisurvstd/emp/personnes.py +543 -0
- mobisurvstd/emp/survey.py +93 -0
- mobisurvstd/emp/voitures.py +218 -0
- mobisurvstd/logger.py +15 -0
- mobisurvstd/main.py +158 -0
- mobisurvstd/resources/__init__.py +0 -0
- mobisurvstd/resources/admin_express.py +88 -0
- mobisurvstd/resources/insee_data.py +307 -0
- mobisurvstd/resources/nuts.py +129 -0
- mobisurvstd/schema.py +902 -0
- mobisurvstd/utils.py +125 -0
- mobisurvstd-0.1.0.dist-info/METADATA +210 -0
- mobisurvstd-0.1.0.dist-info/RECORD +58 -0
- mobisurvstd-0.1.0.dist-info/WHEEL +5 -0
- mobisurvstd-0.1.0.dist-info/licenses/LICENSE.md +7 -0
- mobisurvstd-0.1.0.dist-info/top_level.txt +1 -0
mobisurvstd/cli.py
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import typer
|
|
2
|
+
from typing_extensions import Annotated
|
|
3
|
+
|
|
4
|
+
from .main import bulk_standardize, standardize
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def standardizer(
|
|
8
|
+
source: Annotated[
|
|
9
|
+
str,
|
|
10
|
+
typer.Argument(
|
|
11
|
+
help="Path to the directory or the zipfile where the survey data is located."
|
|
12
|
+
),
|
|
13
|
+
],
|
|
14
|
+
output_directory: Annotated[
|
|
15
|
+
str,
|
|
16
|
+
typer.Argument(
|
|
17
|
+
help="Path to the directory where the standardized survey should be stored."
|
|
18
|
+
),
|
|
19
|
+
],
|
|
20
|
+
survey_type: Annotated[
|
|
21
|
+
str | None,
|
|
22
|
+
typer.Option(
|
|
23
|
+
help="Format of the original survey. Possible values: `emc2`, `emp2019`, `egt2010`, `egt2020`."
|
|
24
|
+
),
|
|
25
|
+
] = None,
|
|
26
|
+
bulk: Annotated[
|
|
27
|
+
bool, typer.Option(help="Import surveys in bulk from the given directory")
|
|
28
|
+
] = False,
|
|
29
|
+
):
|
|
30
|
+
"""Mobility Survey Standardizer: a Python command line tool to convert mobility surveys to a
|
|
31
|
+
clean standardized format.
|
|
32
|
+
"""
|
|
33
|
+
if bulk:
|
|
34
|
+
bulk_standardize(source, output_directory, survey_type)
|
|
35
|
+
else:
|
|
36
|
+
standardize(source, output_directory, survey_type)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
app = typer.Typer()
|
|
40
|
+
app.command()(standardizer)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
if __name__ == "__main__":
|
|
44
|
+
app()
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
DEBUG = False
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
import polars as pl
|
|
2
|
+
|
|
3
|
+
from mobisurvstd.schema import CAR_SCHEMA
|
|
4
|
+
|
|
5
|
+
from . import DEBUG
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def clean(lf: pl.LazyFrame):
|
|
9
|
+
existing_cols = lf.collect_schema().names()
|
|
10
|
+
lf = lf.sort("original_car_id")
|
|
11
|
+
lf = indexing(lf, existing_cols)
|
|
12
|
+
lf = add_fuel_type_group(lf, existing_cols)
|
|
13
|
+
lf = add_mileage_bounds(lf, existing_cols)
|
|
14
|
+
lf = add_critair_column(lf, existing_cols)
|
|
15
|
+
if DEBUG:
|
|
16
|
+
# Try to collect the schema to check if it is valid.
|
|
17
|
+
lf.collect_schema()
|
|
18
|
+
lf.collect()
|
|
19
|
+
return lf
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def indexing(lf: pl.LazyFrame, existing_cols: list[str]):
|
|
23
|
+
if "car_id" not in existing_cols:
|
|
24
|
+
lf = lf.with_columns(car_id=pl.int_range(1, pl.len() + 1, dtype=CAR_SCHEMA["car_id"]))
|
|
25
|
+
existing_cols.append("car_id")
|
|
26
|
+
if "car_index" not in existing_cols:
|
|
27
|
+
lf = lf.with_columns(
|
|
28
|
+
car_index=pl.int_range(1, pl.len() + 1, dtype=CAR_SCHEMA["car_index"]).over(
|
|
29
|
+
"household_id"
|
|
30
|
+
)
|
|
31
|
+
)
|
|
32
|
+
existing_cols.append("car_index")
|
|
33
|
+
return lf
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def add_fuel_type_group(lf: pl.LazyFrame, existing_cols: list[str]):
|
|
37
|
+
if "fuel_type" in existing_cols:
|
|
38
|
+
lf = lf.with_columns(
|
|
39
|
+
fuel_type_group=pl.col("fuel_type").cast(pl.String).str.extract("(\w+):?")
|
|
40
|
+
)
|
|
41
|
+
existing_cols.append("fuel_type_group")
|
|
42
|
+
return lf
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def add_mileage_bounds(lf: pl.LazyFrame, existing_cols: list[str]):
|
|
46
|
+
for col in ("total_mileage", "annual_mileage"):
|
|
47
|
+
lb_col = f"{col}_lower_bound"
|
|
48
|
+
ub_col = f"{col}_upper_bound"
|
|
49
|
+
if col in existing_cols and lb_col not in existing_cols and ub_col not in existing_cols:
|
|
50
|
+
lf = lf.with_columns(pl.col(col).alias(lb_col), pl.col(col).alias(ub_col))
|
|
51
|
+
existing_cols.append(lb_col)
|
|
52
|
+
existing_cols.append(ub_col)
|
|
53
|
+
return lf
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def is_essence():
|
|
57
|
+
return pl.col("fuel_type").is_in(("thermic:petrol", "hybrid:regular:petrol"))
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def is_diesel():
|
|
61
|
+
return pl.col("fuel_type").is_in(("thermic:diesel", "hybrid:regular:diesel"))
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def is_critair_e():
|
|
65
|
+
return pl.col("fuel_type").eq("electric")
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def is_critair_1():
|
|
69
|
+
return (
|
|
70
|
+
pl.col("fuel_type").eq("thermic:gas")
|
|
71
|
+
| pl.col("fuel_type").eq("hybrid:plug-in")
|
|
72
|
+
| is_essence().and_(pl.col("year") >= 2011)
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def is_critair_2():
|
|
77
|
+
return is_essence().and_(pl.col("year").is_between(2006, 2010)) | is_diesel().and_(
|
|
78
|
+
pl.col("year") >= 2011
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def is_critair_3():
|
|
83
|
+
return is_essence().and_(pl.col("year").is_between(1997, 2005)) | is_diesel().and_(
|
|
84
|
+
pl.col("year").is_between(2006, 2010)
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def is_critair_4():
|
|
89
|
+
return is_diesel().and_(pl.col("year").is_between(2001, 2005))
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def is_critair_5():
|
|
93
|
+
return is_diesel().and_(pl.col("year").is_between(1997, 2000))
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def is_critair_unclassified():
|
|
97
|
+
return pl.col("year") <= 1996
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def add_critair_column(lf: pl.LazyFrame, existing_cols: list[str]):
|
|
101
|
+
if "critair" not in existing_cols and "year" in existing_cols and "fuel_type" in existing_cols:
|
|
102
|
+
lf = lf.with_columns(
|
|
103
|
+
critair=pl.when(is_critair_e())
|
|
104
|
+
.then(pl.lit("E"))
|
|
105
|
+
.when(is_critair_1())
|
|
106
|
+
.then(pl.lit("1"))
|
|
107
|
+
.when(is_critair_2())
|
|
108
|
+
.then(pl.lit("2"))
|
|
109
|
+
.when(is_critair_3())
|
|
110
|
+
.then(pl.lit("3"))
|
|
111
|
+
.when(is_critair_4())
|
|
112
|
+
.then(pl.lit("4"))
|
|
113
|
+
.when(is_critair_5())
|
|
114
|
+
.then(pl.lit("5"))
|
|
115
|
+
.when(is_critair_unclassified())
|
|
116
|
+
.then(pl.lit("N"))
|
|
117
|
+
.otherwise(pl.lit(None))
|
|
118
|
+
)
|
|
119
|
+
existing_cols.append("critair")
|
|
120
|
+
return lf
|
|
@@ -0,0 +1,470 @@
|
|
|
1
|
+
from datetime import date
|
|
2
|
+
|
|
3
|
+
import geopandas as gpd
|
|
4
|
+
import polars as pl
|
|
5
|
+
from loguru import logger
|
|
6
|
+
|
|
7
|
+
from mobisurvstd.classes import SurveyData
|
|
8
|
+
from mobisurvstd.common.trips import add_intermodality_column
|
|
9
|
+
from mobisurvstd.schema import (
|
|
10
|
+
CAR_SCHEMA,
|
|
11
|
+
HOUSEHOLD_SCHEMA,
|
|
12
|
+
LEG_SCHEMA,
|
|
13
|
+
MOTORCYCLE_SCHEMA,
|
|
14
|
+
PERSON_SCHEMA,
|
|
15
|
+
TRIP_SCHEMA,
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def clean(
|
|
20
|
+
households: pl.LazyFrame,
|
|
21
|
+
persons: pl.LazyFrame,
|
|
22
|
+
trips: pl.LazyFrame,
|
|
23
|
+
legs: pl.LazyFrame,
|
|
24
|
+
cars: pl.LazyFrame,
|
|
25
|
+
motorcycles: pl.LazyFrame,
|
|
26
|
+
survey_type: str,
|
|
27
|
+
survey_name: str,
|
|
28
|
+
main_insee: str | None = None,
|
|
29
|
+
start_date: date | None = None,
|
|
30
|
+
end_date: date | None = None,
|
|
31
|
+
special_locations: gpd.GeoDataFrame | None = None,
|
|
32
|
+
detailed_zones: gpd.GeoDataFrame | None = None,
|
|
33
|
+
draw_zones: gpd.GeoDataFrame | None = None,
|
|
34
|
+
):
|
|
35
|
+
households = count_nb_persons(households, persons)
|
|
36
|
+
households = add_household_type(households, persons)
|
|
37
|
+
persons = count_nb_trips(persons, trips)
|
|
38
|
+
persons = add_worked_during_surveyed_day(persons, trips)
|
|
39
|
+
trips = count_nb_legs(trips, legs)
|
|
40
|
+
trips = add_main_mode(trips, legs)
|
|
41
|
+
trips = add_access_egress_modes(trips, legs)
|
|
42
|
+
# Select only the column in the schema and add the missing columns with null values.
|
|
43
|
+
data = dict()
|
|
44
|
+
for name, lf, schema in (
|
|
45
|
+
("households", households, HOUSEHOLD_SCHEMA),
|
|
46
|
+
("persons", persons, PERSON_SCHEMA),
|
|
47
|
+
("trips", trips, TRIP_SCHEMA),
|
|
48
|
+
("legs", legs, LEG_SCHEMA),
|
|
49
|
+
("cars", cars, CAR_SCHEMA),
|
|
50
|
+
("motorcycles", motorcycles, MOTORCYCLE_SCHEMA),
|
|
51
|
+
):
|
|
52
|
+
logger.debug(f"Collecting {name}")
|
|
53
|
+
# `short_name` is the name without the "s"
|
|
54
|
+
short_name = name[:-1]
|
|
55
|
+
existing_columns = lf.collect_schema().names()
|
|
56
|
+
columns = [
|
|
57
|
+
cast_column(col, dtype)
|
|
58
|
+
if col in existing_columns
|
|
59
|
+
else pl.lit(None, dtype=dtype).alias(col)
|
|
60
|
+
for col, dtype in schema.items()
|
|
61
|
+
]
|
|
62
|
+
data[name] = lf.select(columns).sort(f"{short_name}_id").collect()
|
|
63
|
+
if special_locations is not None:
|
|
64
|
+
data["special_locations"] = special_locations
|
|
65
|
+
if detailed_zones is not None:
|
|
66
|
+
data["detailed_zones"] = detailed_zones
|
|
67
|
+
if draw_zones is not None:
|
|
68
|
+
data["draw_zones"] = draw_zones
|
|
69
|
+
data["metadata"] = create_metadata(
|
|
70
|
+
data, survey_type, survey_name, main_insee, start_date, end_date
|
|
71
|
+
)
|
|
72
|
+
return SurveyData.from_dict(data)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def create_metadata(
|
|
76
|
+
data: dict,
|
|
77
|
+
survey_type: str,
|
|
78
|
+
survey_name: str,
|
|
79
|
+
main_insee: str | None,
|
|
80
|
+
start_date: date | None = None,
|
|
81
|
+
end_date: date | None = None,
|
|
82
|
+
):
|
|
83
|
+
# Find survey method.
|
|
84
|
+
survey_methods = data["households"]["survey_method"].unique()
|
|
85
|
+
if len(survey_methods) == 1:
|
|
86
|
+
survey_method = survey_methods[0]
|
|
87
|
+
else:
|
|
88
|
+
assert len(survey_methods) == 2
|
|
89
|
+
survey_method = "mixed"
|
|
90
|
+
# Count number of zones and add zone counts.
|
|
91
|
+
if "special_locations" in data:
|
|
92
|
+
nb_special_locations = len(data["special_locations"])
|
|
93
|
+
data["special_locations"] = add_zone_counts(
|
|
94
|
+
data["special_locations"], "special_location", data
|
|
95
|
+
)
|
|
96
|
+
else:
|
|
97
|
+
nb_special_locations = 0
|
|
98
|
+
if "detailed_zones" in data:
|
|
99
|
+
nb_detailed_zones = len(data["detailed_zones"])
|
|
100
|
+
data["detailed_zones"] = add_zone_counts(data["detailed_zones"], "detailed_zone", data)
|
|
101
|
+
else:
|
|
102
|
+
nb_detailed_zones = 0
|
|
103
|
+
if "draw_zones" in data:
|
|
104
|
+
nb_draw_zones = len(data["draw_zones"])
|
|
105
|
+
data["draw_zones"] = add_zone_counts(data["draw_zones"], "draw_zone", data)
|
|
106
|
+
else:
|
|
107
|
+
nb_draw_zones = 0
|
|
108
|
+
# Find start and end date of survey.
|
|
109
|
+
if start_date is None:
|
|
110
|
+
start_date: date = data["households"]["interview_date"].min() # type: ignore
|
|
111
|
+
if end_date is None:
|
|
112
|
+
end_date: date = data["households"]["interview_date"].max() # type: ignore
|
|
113
|
+
metadata = {
|
|
114
|
+
"name": survey_name,
|
|
115
|
+
"type": survey_type,
|
|
116
|
+
"survey_method": survey_method,
|
|
117
|
+
"nb_households": len(data["households"]),
|
|
118
|
+
"nb_cars": len(data["cars"]),
|
|
119
|
+
"nb_motorcycles": len(data["motorcycles"]),
|
|
120
|
+
"nb_persons": len(data["persons"]),
|
|
121
|
+
"nb_trips": len(data["trips"]),
|
|
122
|
+
"nb_legs": len(data["legs"]),
|
|
123
|
+
"nb_special_locations": nb_special_locations,
|
|
124
|
+
"nb_detailed_zones": nb_detailed_zones,
|
|
125
|
+
"nb_draw_zones": nb_draw_zones,
|
|
126
|
+
"start_date": start_date.isoformat(),
|
|
127
|
+
"end_date": end_date.isoformat(),
|
|
128
|
+
"insee": main_insee,
|
|
129
|
+
}
|
|
130
|
+
return metadata
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def cast_column(col: str, dtype: pl.DataType):
|
|
134
|
+
if dtype == pl.Struct:
|
|
135
|
+
# The struct fields are unspecified so the column is not cast.
|
|
136
|
+
return pl.col(col).alias(col)
|
|
137
|
+
else:
|
|
138
|
+
return pl.col(col).cast(dtype).alias(col)
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def count_nb_persons(households: pl.LazyFrame, persons: pl.LazyFrame):
|
|
142
|
+
has_age = "age" in persons.collect_schema().names()
|
|
143
|
+
has_age_class = "age_class_code" in persons.collect_schema().names()
|
|
144
|
+
person_counts = persons.group_by("household_id").agg(
|
|
145
|
+
nb_persons=pl.len(),
|
|
146
|
+
nb_persons_5plus=pl.when(pl.col("age").is_not_null().all()).then(pl.col("age").gt(5).sum())
|
|
147
|
+
if has_age
|
|
148
|
+
else None,
|
|
149
|
+
nb_majors=pl.when(pl.col("age_class_code").is_not_null().all()).then(
|
|
150
|
+
pl.col("age_class_code").gt(1).sum()
|
|
151
|
+
)
|
|
152
|
+
if has_age_class
|
|
153
|
+
else None,
|
|
154
|
+
nb_minors=pl.when(pl.col("age_class_code").is_not_null().all()).then(
|
|
155
|
+
pl.col("age_class_code").eq(1).sum()
|
|
156
|
+
)
|
|
157
|
+
if has_age_class
|
|
158
|
+
else None,
|
|
159
|
+
)
|
|
160
|
+
households = households.join(person_counts, on="household_id", how="left", coalesce=True)
|
|
161
|
+
return households
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def add_household_type(households: pl.LazyFrame, persons: pl.LazyFrame):
|
|
165
|
+
household_columns = households.collect_schema().names()
|
|
166
|
+
person_columns = persons.collect_schema().names()
|
|
167
|
+
if (
|
|
168
|
+
"household_type" in household_columns
|
|
169
|
+
or "reference_person_link" not in person_columns
|
|
170
|
+
or "woman" not in person_columns
|
|
171
|
+
):
|
|
172
|
+
return households
|
|
173
|
+
households = households.join(
|
|
174
|
+
persons.group_by("household_id").agg(
|
|
175
|
+
nb_men=pl.col("woman").not_().sum(),
|
|
176
|
+
nb_women=pl.col("woman").sum(),
|
|
177
|
+
nb_spouses=pl.col("reference_person_link").eq("spouse").sum(),
|
|
178
|
+
nb_children=pl.col("reference_person_link").eq("child").sum(),
|
|
179
|
+
nb_refs=pl.col("reference_person_link").eq("reference_person").sum(),
|
|
180
|
+
only_family=pl.col("reference_person_link")
|
|
181
|
+
.is_in(("reference_person", "spouse", "child"))
|
|
182
|
+
.all(),
|
|
183
|
+
ref_is_man=pl.col("reference_person_link")
|
|
184
|
+
.eq("reference_person")
|
|
185
|
+
.and_(pl.col("woman").not_())
|
|
186
|
+
.any(),
|
|
187
|
+
ref_is_woman=pl.col("reference_person_link").eq("reference_person").and_("woman").any(),
|
|
188
|
+
),
|
|
189
|
+
on="household_id",
|
|
190
|
+
how="left",
|
|
191
|
+
coalesce=True,
|
|
192
|
+
)
|
|
193
|
+
households = households.with_columns(
|
|
194
|
+
household_type=pl.when(nb_persons=1, nb_men=1)
|
|
195
|
+
.then(pl.lit("single:man"))
|
|
196
|
+
.when(nb_persons=1, nb_women=1)
|
|
197
|
+
.then(pl.lit("single:woman"))
|
|
198
|
+
.when(nb_persons=2, nb_spouses=1, nb_refs=1)
|
|
199
|
+
.then(pl.lit("couple:no_child"))
|
|
200
|
+
.when("only_family", pl.col("nb_children") > 0, nb_refs=1, nb_spouses=1)
|
|
201
|
+
.then(pl.lit("couple:children"))
|
|
202
|
+
.when("only_family", "ref_is_man", pl.col("nb_children") > 0, nb_refs=1, nb_spouses=0)
|
|
203
|
+
.then(pl.lit("singleparent:father"))
|
|
204
|
+
.when("only_family", "ref_is_woman", pl.col("nb_children") > 0, nb_refs=1, nb_spouses=0)
|
|
205
|
+
.then(pl.lit("singleparent:mother"))
|
|
206
|
+
.when(pl.col("only_family").not_())
|
|
207
|
+
.then(pl.lit("other"))
|
|
208
|
+
)
|
|
209
|
+
return households
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
def count_nb_trips(persons: pl.LazyFrame, trips: pl.LazyFrame):
|
|
213
|
+
trip_counts = trips.group_by("person_id").agg(nb_trips=pl.len())
|
|
214
|
+
persons = persons.join(trip_counts, on="person_id", how="left", coalesce=True)
|
|
215
|
+
# Set nb_trips = 0 for surveyed persons.
|
|
216
|
+
persons = persons.with_columns(
|
|
217
|
+
nb_trips=pl.when("is_surveyed").then(pl.col("nb_trips").fill_null(0)).otherwise("nb_trips")
|
|
218
|
+
)
|
|
219
|
+
# Add `traveled_during_surveyed_day` if needed.
|
|
220
|
+
if "traveled_during_surveyed_day" not in persons.collect_schema().names():
|
|
221
|
+
persons = persons.with_columns(
|
|
222
|
+
traveled_during_surveyed_day=pl.when(pl.col("nb_trips") > 0)
|
|
223
|
+
.then(pl.lit("yes"))
|
|
224
|
+
.otherwise(pl.lit("no"))
|
|
225
|
+
)
|
|
226
|
+
return persons
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def add_worked_during_surveyed_day(persons: pl.LazyFrame, trips: pl.LazyFrame):
|
|
230
|
+
persons_cols = persons.collect_schema().names()
|
|
231
|
+
persons = persons.join(
|
|
232
|
+
trips.group_by("person_id").agg(
|
|
233
|
+
has_work_activity=pl.col("destination_purpose")
|
|
234
|
+
.is_in(("work:declared", "work:secondary", "work:other", "work:professional_tour"))
|
|
235
|
+
.any(),
|
|
236
|
+
has_telework_activity=pl.col("destination_purpose").eq("work:telework").any(),
|
|
237
|
+
),
|
|
238
|
+
on="person_id",
|
|
239
|
+
how="left",
|
|
240
|
+
coalesce=True,
|
|
241
|
+
)
|
|
242
|
+
persons = persons.with_columns(
|
|
243
|
+
worked_during_surveyed_day=pl.when("has_work_activity")
|
|
244
|
+
.then(pl.lit("yes:outside"))
|
|
245
|
+
.when(
|
|
246
|
+
"work_only_at_home" if "work_only_at_home" in persons_cols else False,
|
|
247
|
+
"has_telework_activity",
|
|
248
|
+
)
|
|
249
|
+
.then(pl.lit("yes:home:usual"))
|
|
250
|
+
.when(
|
|
251
|
+
pl.col("work_only_at_home").not_() if "work_only_at_home" in persons_cols else False,
|
|
252
|
+
"has_telework_activity",
|
|
253
|
+
)
|
|
254
|
+
.then(pl.lit("yes:home:telework"))
|
|
255
|
+
.when("work_only_at_home" not in persons_cols, "has_telework_activity")
|
|
256
|
+
.then(pl.lit("yes:home:other"))
|
|
257
|
+
.when("is_surveyed", pl.col("professional_occupation").eq("worker"))
|
|
258
|
+
.then(pl.lit("no:unspecified"))
|
|
259
|
+
)
|
|
260
|
+
return persons
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def count_nb_legs(trips: pl.LazyFrame, legs: pl.LazyFrame):
|
|
264
|
+
"""Add column `nb_legs` to the trips.
|
|
265
|
+
|
|
266
|
+
Also add columns `nb_legs_{mode_group}` and `intermodality` if leg's `mode_group` are known.
|
|
267
|
+
"""
|
|
268
|
+
agg_cols = [pl.len().alias("nb_legs")]
|
|
269
|
+
has_mode_group = "mode_group" in legs.collect_schema().names()
|
|
270
|
+
if has_mode_group:
|
|
271
|
+
agg_cols.extend(
|
|
272
|
+
[
|
|
273
|
+
pl.col("mode_group").eq(mode_group).sum().alias(f"nb_legs_{mode_group}")
|
|
274
|
+
for mode_group in LEG_SCHEMA["mode_group"].categories
|
|
275
|
+
]
|
|
276
|
+
)
|
|
277
|
+
leg_counts = legs.group_by("trip_id").agg(agg_cols)
|
|
278
|
+
trips = trips.join(leg_counts, on="trip_id", how="left", coalesce=True)
|
|
279
|
+
if has_mode_group:
|
|
280
|
+
trips = add_intermodality_column(trips)
|
|
281
|
+
return trips
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
def add_main_mode(trips: pl.LazyFrame, legs: pl.LazyFrame):
|
|
285
|
+
"""Identify the main mode and main mode group of each trip, given the modes and mode groups of
|
|
286
|
+
the trips' legs.
|
|
287
|
+
|
|
288
|
+
If the column `main_mode_group` already exists in `trips`, the function does nothing (even if
|
|
289
|
+
the column `main_mode` does not exist).
|
|
290
|
+
|
|
291
|
+
If the column `mode_group` does not exist in `legs`, the function does nothing.
|
|
292
|
+
|
|
293
|
+
If the column `mode_group` exists in `legs` but the column `mode` does not, then the main mode
|
|
294
|
+
groups are identified but not the groups.
|
|
295
|
+
|
|
296
|
+
The most used modes of a trip are identified using the legs' travel time (if column
|
|
297
|
+
`leg_travel_time` exists), or using the legs' euclidean distance (if column
|
|
298
|
+
`leg_euclidean_distance_km` exists). If neither column is available, the function does nothing.
|
|
299
|
+
"""
|
|
300
|
+
trip_columns = trips.collect_schema().names()
|
|
301
|
+
if "main_mode_group" in trip_columns:
|
|
302
|
+
# Note. It is possible that the `main_mode` column does not exist and that we would like to
|
|
303
|
+
# add it but that would require a different function.
|
|
304
|
+
return trips
|
|
305
|
+
leg_columns = legs.collect_schema().names()
|
|
306
|
+
if "mode_group" not in leg_columns:
|
|
307
|
+
assert "mode" not in leg_columns, "Legs have `mode` but no `mode_group`"
|
|
308
|
+
return trips
|
|
309
|
+
has_modes = "mode" in leg_columns
|
|
310
|
+
# === Step A ===
|
|
311
|
+
# Find the column used to identify the most used mode over legs.
|
|
312
|
+
agg_column = None
|
|
313
|
+
if "leg_travel_time" in leg_columns:
|
|
314
|
+
agg_column = "leg_travel_time"
|
|
315
|
+
elif "leg_euclidean_distance_km" in leg_columns:
|
|
316
|
+
agg_column = "leg_euclidean_distance_km"
|
|
317
|
+
if agg_column is None:
|
|
318
|
+
# There is no way to find the main mode.
|
|
319
|
+
return trips
|
|
320
|
+
# === Step B ===
|
|
321
|
+
# Find the `main_mode_group`: the mode group that is the most used over the legs.
|
|
322
|
+
main_modes = (
|
|
323
|
+
# Exclude walking legs (unless it is the only mode_group used in the leg).
|
|
324
|
+
legs.filter(
|
|
325
|
+
pl.col("mode_group").ne("walking")
|
|
326
|
+
| pl.col("mode_group").eq("walking").all().over("trip_id")
|
|
327
|
+
)
|
|
328
|
+
# Exclude trips where the agg_column is NULL for at least one non-walking leg (unless
|
|
329
|
+
# the trip as at most 1 mode / mode group).
|
|
330
|
+
.filter(
|
|
331
|
+
pl.col(agg_column).is_not_null().all().over("trip_id")
|
|
332
|
+
| pl.col("mode" if has_modes else "mode_group").n_unique().over("trip_id").eq(1)
|
|
333
|
+
)
|
|
334
|
+
# Compute the agg_column sum by `mode_group` and `mode`, for each trip.
|
|
335
|
+
.with_columns(
|
|
336
|
+
mode_group_total=pl.col(agg_column).sum().over("trip_id", "mode_group"),
|
|
337
|
+
# The `mode_total` column is None if the `mode` column does not exist.
|
|
338
|
+
mode_total=pl.col(agg_column).sum().over("trip_id", "mode") if has_modes else None,
|
|
339
|
+
)
|
|
340
|
+
.group_by("trip_id")
|
|
341
|
+
.agg(
|
|
342
|
+
# Find the `mode_group` with the largest total for each trip.
|
|
343
|
+
main_mode_group=pl.col("mode_group").sort_by("mode_group_total").last(),
|
|
344
|
+
# Find the `mode` with the largest total, among the modes in the `main_mode_group`,
|
|
345
|
+
# for each trip.
|
|
346
|
+
main_mode=pl.col("mode").sort_by("mode_group_total", "mode_total").last()
|
|
347
|
+
if has_modes
|
|
348
|
+
else None,
|
|
349
|
+
)
|
|
350
|
+
)
|
|
351
|
+
# Add the `main_mode_group` and `mode_group` columns to the trips.
|
|
352
|
+
trips = trips.join(main_modes, on="trip_id", how="left", coalesce=True)
|
|
353
|
+
# Remove the `main_mode` column if the legs modes are unknown.
|
|
354
|
+
if not has_modes:
|
|
355
|
+
trips = trips.drop("main_mode")
|
|
356
|
+
return trips
|
|
357
|
+
|
|
358
|
+
|
|
359
|
+
def add_access_egress_modes(trips: pl.LazyFrame, legs: pl.LazyFrame):
|
|
360
|
+
trip_columns = trips.collect_schema().names()
|
|
361
|
+
leg_columns = legs.collect_schema().names()
|
|
362
|
+
if "main_mode_group" not in trip_columns or "mode_group" not in leg_columns:
|
|
363
|
+
# Access and egress modes cannot be identified.
|
|
364
|
+
return trips
|
|
365
|
+
has_modes = "mode" in leg_columns
|
|
366
|
+
# Find first and last mode / mode group of each trip's legs.
|
|
367
|
+
first_last_leg_modes = (
|
|
368
|
+
legs.sort("trip_id", "leg_index")
|
|
369
|
+
.group_by("trip_id")
|
|
370
|
+
.agg(
|
|
371
|
+
first_mode_group=pl.col("mode_group").first(),
|
|
372
|
+
first_mode=pl.col("mode").first() if has_modes else None,
|
|
373
|
+
last_mode_group=pl.col("mode_group").last(),
|
|
374
|
+
last_mode=pl.col("mode").last() if has_modes else None,
|
|
375
|
+
)
|
|
376
|
+
)
|
|
377
|
+
# Access mode is the mode of the first leg if the trip's `main_mode_group` is "public_transit"
|
|
378
|
+
# and the first leg's `mode_group` is not "public_transit", otherwise it is NULL.
|
|
379
|
+
# Egress mode is the mode of the last leg if the trip's `main_mode_group` is "public_transit"
|
|
380
|
+
# and the last leg's `mode_group` is not "public_transit", otherwise it is NULL.
|
|
381
|
+
is_pt_trip = pl.col("main_mode_group") == "public_transit"
|
|
382
|
+
trips = trips.join(first_last_leg_modes, on="trip_id", how="left", coalesce=True).with_columns(
|
|
383
|
+
public_transit_access_mode=pl.when(
|
|
384
|
+
is_pt_trip & pl.col("first_mode_group").ne("public_transit")
|
|
385
|
+
).then("first_mode"),
|
|
386
|
+
public_transit_access_mode_group=pl.when(
|
|
387
|
+
is_pt_trip & pl.col("first_mode_group").ne("public_transit")
|
|
388
|
+
).then("first_mode_group"),
|
|
389
|
+
public_transit_egress_mode=pl.when(
|
|
390
|
+
is_pt_trip & pl.col("last_mode_group").ne("public_transit")
|
|
391
|
+
).then("last_mode"),
|
|
392
|
+
public_transit_egress_mode_group=pl.when(
|
|
393
|
+
is_pt_trip & pl.col("last_mode_group").ne("public_transit")
|
|
394
|
+
).then("last_mode_group"),
|
|
395
|
+
)
|
|
396
|
+
return trips
|
|
397
|
+
|
|
398
|
+
|
|
399
|
+
def add_zone_counts(gdf: gpd.GeoDataFrame, prefix: str, data: dict):
|
|
400
|
+
# Add nb_homes.
|
|
401
|
+
gdf = gdf.merge(
|
|
402
|
+
data["households"]
|
|
403
|
+
.group_by(pl.col(f"home_{prefix}").alias(f"{prefix}_id"))
|
|
404
|
+
.agg(nb_homes=pl.len())
|
|
405
|
+
.to_pandas(),
|
|
406
|
+
on=f"{prefix}_id",
|
|
407
|
+
how="left",
|
|
408
|
+
)
|
|
409
|
+
gdf["nb_homes"] = gdf["nb_homes"].fillna(0).astype("UInt32")
|
|
410
|
+
# Add nb_work_locations.
|
|
411
|
+
gdf = gdf.merge(
|
|
412
|
+
data["persons"]
|
|
413
|
+
.group_by(pl.col(f"work_{prefix}").alias(f"{prefix}_id"))
|
|
414
|
+
.agg(nb_work_locations=pl.len())
|
|
415
|
+
.to_pandas(),
|
|
416
|
+
on=f"{prefix}_id",
|
|
417
|
+
how="left",
|
|
418
|
+
)
|
|
419
|
+
gdf["nb_work_locations"] = gdf["nb_work_locations"].fillna(0).astype("UInt32")
|
|
420
|
+
# Add nb_study_locations.
|
|
421
|
+
gdf = gdf.merge(
|
|
422
|
+
data["persons"]
|
|
423
|
+
.group_by(pl.col(f"study_{prefix}").alias(f"{prefix}_id"))
|
|
424
|
+
.agg(nb_study_locations=pl.len())
|
|
425
|
+
.to_pandas(),
|
|
426
|
+
on=f"{prefix}_id",
|
|
427
|
+
how="left",
|
|
428
|
+
)
|
|
429
|
+
gdf["nb_study_locations"] = gdf["nb_study_locations"].fillna(0).astype("UInt32")
|
|
430
|
+
# Add nb_trip_origins.
|
|
431
|
+
gdf = gdf.merge(
|
|
432
|
+
data["trips"]
|
|
433
|
+
.group_by(pl.col(f"origin_{prefix}").alias(f"{prefix}_id"))
|
|
434
|
+
.agg(nb_trip_origins=pl.len())
|
|
435
|
+
.to_pandas(),
|
|
436
|
+
on=f"{prefix}_id",
|
|
437
|
+
how="left",
|
|
438
|
+
)
|
|
439
|
+
gdf["nb_trip_origins"] = gdf["nb_trip_origins"].fillna(0).astype("UInt32")
|
|
440
|
+
# Add nb_trip_destinations.
|
|
441
|
+
gdf = gdf.merge(
|
|
442
|
+
data["trips"]
|
|
443
|
+
.group_by(pl.col(f"destination_{prefix}").alias(f"{prefix}_id"))
|
|
444
|
+
.agg(nb_trip_destinations=pl.len())
|
|
445
|
+
.to_pandas(),
|
|
446
|
+
on=f"{prefix}_id",
|
|
447
|
+
how="left",
|
|
448
|
+
)
|
|
449
|
+
gdf["nb_trip_destinations"] = gdf["nb_trip_destinations"].fillna(0).astype("UInt32")
|
|
450
|
+
# Add nb_leg_starts.
|
|
451
|
+
gdf = gdf.merge(
|
|
452
|
+
data["legs"]
|
|
453
|
+
.group_by(pl.col(f"start_{prefix}").alias(f"{prefix}_id"))
|
|
454
|
+
.agg(nb_leg_starts=pl.len())
|
|
455
|
+
.to_pandas(),
|
|
456
|
+
on=f"{prefix}_id",
|
|
457
|
+
how="left",
|
|
458
|
+
)
|
|
459
|
+
gdf["nb_leg_starts"] = gdf["nb_leg_starts"].fillna(0).astype("UInt32")
|
|
460
|
+
# Add nb_leg_stops.
|
|
461
|
+
gdf = gdf.merge(
|
|
462
|
+
data["legs"]
|
|
463
|
+
.group_by(pl.col(f"end_{prefix}").alias(f"{prefix}_id"))
|
|
464
|
+
.agg(nb_leg_ends=pl.len())
|
|
465
|
+
.to_pandas(),
|
|
466
|
+
on=f"{prefix}_id",
|
|
467
|
+
how="left",
|
|
468
|
+
)
|
|
469
|
+
gdf["nb_leg_ends"] = gdf["nb_leg_ends"].fillna(0).astype("UInt32")
|
|
470
|
+
return gdf
|