mobisurvstd 0.3.0__tar.gz → 0.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {mobisurvstd-0.3.0/src/mobisurvstd.egg-info → mobisurvstd-0.5.0}/PKG-INFO +11 -2
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/README.md +10 -1
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/pyproject.toml +1 -1
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/clean.py +4 -4
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/cli.py +1 -1
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/common/cars.py +1 -1
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/common/persons.py +1 -1
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/common/trips.py +1 -1
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/edgt/survey.py +65 -58
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/edvm/survey.py +46 -39
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/edvm/trajets.py +1 -1
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/edvm/zones.py +2 -2
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/egt2010/survey.py +10 -7
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/egt2010/trajets.py +1 -1
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/egt2020/survey.py +1 -1
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/egt2020/trajets.py +5 -5
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/emc2/survey.py +60 -51
- mobisurvstd-0.5.0/src/mobisurvstd/emd/common.py +42 -0
- mobisurvstd-0.5.0/src/mobisurvstd/emd/deplacements.py +160 -0
- mobisurvstd-0.5.0/src/mobisurvstd/emd/menages.py +355 -0
- mobisurvstd-0.5.0/src/mobisurvstd/emd/personnes.py +312 -0
- mobisurvstd-0.5.0/src/mobisurvstd/emd/survey.py +283 -0
- mobisurvstd-0.5.0/src/mobisurvstd/emd/trajets.py +271 -0
- mobisurvstd-0.5.0/src/mobisurvstd/emd/zones.py +222 -0
- mobisurvstd-0.5.0/src/mobisurvstd/emp/__init__.py +1 -0
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/emp/survey.py +1 -1
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/main.py +46 -17
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/resources/admin_express.py +1 -1
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/utils.py +1 -4
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0/src/mobisurvstd.egg-info}/PKG-INFO +11 -2
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd.egg-info/SOURCES.txt +8 -0
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/LICENSE.md +0 -0
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/setup.cfg +0 -0
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/__init__.py +0 -0
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/__main__.py +0 -0
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/classes.py +0 -0
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/common/__init__.py +0 -0
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/common/clean.py +0 -0
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/common/households.py +0 -0
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/common/legs.py +0 -0
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/common/modes.py +0 -0
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/common/motorcycles.py +0 -0
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/common/zones.py +0 -0
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/edgt/__init__.py +0 -0
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/edgt/common.py +0 -0
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/edgt/deplacements.py +0 -0
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/edgt/menages.py +0 -0
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/edgt/personnes.py +0 -0
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/edgt/trajets.py +0 -0
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/edgt/zones.py +0 -0
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/edvm/__init__.py +0 -0
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/edvm/common.py +0 -0
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/edvm/deplacements.py +0 -0
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/edvm/menages.py +0 -0
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/edvm/personnes.py +0 -0
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/egt2010/__init__.py +0 -0
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/egt2010/deplacements.py +0 -0
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/egt2010/menages.py +0 -0
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/egt2010/personnes.py +0 -0
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/egt2010/zones.py +0 -0
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/egt2020/__init__.py +0 -0
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/egt2020/deplacements.py +0 -0
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/egt2020/menages.py +0 -0
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/egt2020/motos.py +0 -0
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/egt2020/personnes.py +0 -0
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/egt2020/voitures.py +0 -0
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/emc2/__init__.py +0 -0
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/emc2/common.py +0 -0
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/emc2/deplacements.py +0 -0
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/emc2/menages.py +0 -0
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/emc2/personnes.py +0 -0
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/emc2/trajets.py +0 -0
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/emc2/zones.py +0 -0
- {mobisurvstd-0.3.0/src/mobisurvstd/emp → mobisurvstd-0.5.0/src/mobisurvstd/emd}/__init__.py +0 -0
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/emp/deplacements.py +0 -0
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/emp/menages.py +0 -0
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/emp/motos.py +0 -0
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/emp/personnes.py +0 -0
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/emp/voitures.py +0 -0
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/logger.py +0 -0
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/resources/__init__.py +0 -0
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/resources/insee_data.py +0 -0
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/resources/nuts.py +0 -0
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd/schema.py +0 -0
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd.egg-info/dependency_links.txt +0 -0
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd.egg-info/requires.txt +0 -0
- {mobisurvstd-0.3.0 → mobisurvstd-0.5.0}/src/mobisurvstd.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: mobisurvstd
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.5.0
|
|
4
4
|
Summary: A tool to standardize French mobility survey datasets (EMC2, EGT, EMP, etc.) into a unified format.
|
|
5
5
|
Author-email: Lucas Javaudin <lucas@lucasjavaudin.com>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -78,6 +78,7 @@ Also surveys based on previous formats are still in use today:
|
|
|
78
78
|
(until 2018);
|
|
79
79
|
- **EDGT** (Enquêtes Déplacements Grands Territoires, CEREMA): mobility surveys for periphery areas
|
|
80
80
|
(until 2018);
|
|
81
|
+
- **EMD** (Enquêtes Ménages Déplacements, CEREMA): other mobility surveys from CEREMA (until 2018);
|
|
81
82
|
- [**EGT 2010**](https://omnil.fr/egt-2010) (Enquête Globale Transport, Île-de-France Mobilités):
|
|
82
83
|
previous version of the Île-de-France mobility survey;
|
|
83
84
|
- [**ENTD**](https://www.statistiques.developpement-durable.gouv.fr/enquete-nationale-transports-et-deplacements-entd-2008)
|
|
@@ -149,11 +150,18 @@ mobisurvstd.standardize(
|
|
|
149
150
|
|
|
150
151
|
For more, check the [User Guide](https://mobisurvstd.github.io/MobiSurvStd/howto.html).
|
|
151
152
|
|
|
153
|
+
**Citing MobiSurvStd**
|
|
154
|
+
|
|
155
|
+
If you use MobiSurvStd in a publication or project, please include the following attribution:
|
|
156
|
+
"This work used the MobiSurvStd library developed by Lucas Javaudin, available at
|
|
157
|
+
[https://github.com/MobiSurvStd/MobiSurvStd]()".
|
|
158
|
+
|
|
152
159
|
---
|
|
153
160
|
|
|
154
161
|
## Case Study: Bicycle Use
|
|
155
162
|
|
|
156
|
-
The following graph represents the share of bicycle trips for EMC², EDGT, EDVM, and EGT
|
|
163
|
+
The following graph represents the share of bicycle trips for EMC², EDGT, EDVM, EMD, and EGT
|
|
164
|
+
surveys.
|
|
157
165
|
The circle colors represent the average number of bicycles in the surveyed households.
|
|
158
166
|
The circle sizes represent the expected number of trips in the surveyed area.
|
|
159
167
|
|
|
@@ -182,6 +190,7 @@ Currently, MobiSurvStd supports the following survey types:
|
|
|
182
190
|
- `egt2010`
|
|
183
191
|
- `edgt`
|
|
184
192
|
- `edvm`
|
|
193
|
+
- `emd`
|
|
185
194
|
|
|
186
195
|
See [Survey Types](https://mobisurvstd.github.io/MobiSurvStd/surveys.html) for more.
|
|
187
196
|
|
|
@@ -46,6 +46,7 @@ Also surveys based on previous formats are still in use today:
|
|
|
46
46
|
(until 2018);
|
|
47
47
|
- **EDGT** (Enquêtes Déplacements Grands Territoires, CEREMA): mobility surveys for periphery areas
|
|
48
48
|
(until 2018);
|
|
49
|
+
- **EMD** (Enquêtes Ménages Déplacements, CEREMA): other mobility surveys from CEREMA (until 2018);
|
|
49
50
|
- [**EGT 2010**](https://omnil.fr/egt-2010) (Enquête Globale Transport, Île-de-France Mobilités):
|
|
50
51
|
previous version of the Île-de-France mobility survey;
|
|
51
52
|
- [**ENTD**](https://www.statistiques.developpement-durable.gouv.fr/enquete-nationale-transports-et-deplacements-entd-2008)
|
|
@@ -117,11 +118,18 @@ mobisurvstd.standardize(
|
|
|
117
118
|
|
|
118
119
|
For more, check the [User Guide](https://mobisurvstd.github.io/MobiSurvStd/howto.html).
|
|
119
120
|
|
|
121
|
+
**Citing MobiSurvStd**
|
|
122
|
+
|
|
123
|
+
If you use MobiSurvStd in a publication or project, please include the following attribution:
|
|
124
|
+
"This work used the MobiSurvStd library developed by Lucas Javaudin, available at
|
|
125
|
+
[https://github.com/MobiSurvStd/MobiSurvStd]()".
|
|
126
|
+
|
|
120
127
|
---
|
|
121
128
|
|
|
122
129
|
## Case Study: Bicycle Use
|
|
123
130
|
|
|
124
|
-
The following graph represents the share of bicycle trips for EMC², EDGT, EDVM, and EGT
|
|
131
|
+
The following graph represents the share of bicycle trips for EMC², EDGT, EDVM, EMD, and EGT
|
|
132
|
+
surveys.
|
|
125
133
|
The circle colors represent the average number of bicycles in the surveyed households.
|
|
126
134
|
The circle sizes represent the expected number of trips in the surveyed area.
|
|
127
135
|
|
|
@@ -150,6 +158,7 @@ Currently, MobiSurvStd supports the following survey types:
|
|
|
150
158
|
- `egt2010`
|
|
151
159
|
- `edgt`
|
|
152
160
|
- `edvm`
|
|
161
|
+
- `emd`
|
|
153
162
|
|
|
154
163
|
See [Survey Types](https://mobisurvstd.github.io/MobiSurvStd/surveys.html) for more.
|
|
155
164
|
|
|
@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
|
|
|
6
6
|
|
|
7
7
|
[project]
|
|
8
8
|
name = "mobisurvstd"
|
|
9
|
-
version = "0.
|
|
9
|
+
version = "0.5.0"
|
|
10
10
|
description = "A tool to standardize French mobility survey datasets (EMC2, EGT, EMP, etc.) into a unified format."
|
|
11
11
|
readme = "README.md"
|
|
12
12
|
authors = [{ name = "Lucas Javaudin", email = "lucas@lucasjavaudin.com" }]
|
|
@@ -1017,7 +1017,7 @@ def clean(
|
|
|
1017
1017
|
), "`origin_purpose` is education-related for a non-student person"
|
|
1018
1018
|
assert trips.select(
|
|
1019
1019
|
(
|
|
1020
|
-
pl.col("origin_purpose").cast(pl.String).str.extract("(\w+):?")
|
|
1020
|
+
pl.col("origin_purpose").cast(pl.String).str.extract(r"(\w+):?")
|
|
1021
1021
|
== pl.col("origin_purpose_group")
|
|
1022
1022
|
).all()
|
|
1023
1023
|
).item(), "Columns `origin_purpose` and `origin_purpose_group` have some inconsistencies"
|
|
@@ -1043,7 +1043,7 @@ def clean(
|
|
|
1043
1043
|
), "`destination_purpose` is education-related for a non-student person"
|
|
1044
1044
|
assert trips.select(
|
|
1045
1045
|
(
|
|
1046
|
-
pl.col("destination_purpose").cast(pl.String).str.extract("(\w+):?")
|
|
1046
|
+
pl.col("destination_purpose").cast(pl.String).str.extract(r"(\w+):?")
|
|
1047
1047
|
== pl.col("destination_purpose_group")
|
|
1048
1048
|
).all()
|
|
1049
1049
|
).item(), (
|
|
@@ -1078,7 +1078,7 @@ def clean(
|
|
|
1078
1078
|
)
|
|
1079
1079
|
assert trips.select(
|
|
1080
1080
|
(
|
|
1081
|
-
pl.col("origin_escort_purpose").cast(pl.String).str.extract("(\w+):?")
|
|
1081
|
+
pl.col("origin_escort_purpose").cast(pl.String).str.extract(r"(\w+):?")
|
|
1082
1082
|
== pl.col("origin_escort_purpose_group")
|
|
1083
1083
|
).all()
|
|
1084
1084
|
).item(), (
|
|
@@ -1101,7 +1101,7 @@ def clean(
|
|
|
1101
1101
|
)
|
|
1102
1102
|
assert trips.select(
|
|
1103
1103
|
(
|
|
1104
|
-
pl.col("destination_escort_purpose").cast(pl.String).str.extract("(\w+):?")
|
|
1104
|
+
pl.col("destination_escort_purpose").cast(pl.String).str.extract(r"(\w+):?")
|
|
1105
1105
|
== pl.col("destination_escort_purpose_group")
|
|
1106
1106
|
).all()
|
|
1107
1107
|
).item(), (
|
|
@@ -20,7 +20,7 @@ def standardizer(
|
|
|
20
20
|
survey_type: Annotated[
|
|
21
21
|
str | None,
|
|
22
22
|
typer.Option(
|
|
23
|
-
help="Format of the original survey. Possible values: `emc2`, `emp2019`, `egt2010`, `egt2020`, `edgt`, `edvm`."
|
|
23
|
+
help="Format of the original survey. Possible values: `emc2`, `emp2019`, `egt2010`, `egt2020`, `edgt`, `edvm`, `emd`."
|
|
24
24
|
),
|
|
25
25
|
] = None,
|
|
26
26
|
bulk: Annotated[
|
|
@@ -36,7 +36,7 @@ def indexing(lf: pl.LazyFrame, existing_cols: list[str]):
|
|
|
36
36
|
def add_fuel_type_group(lf: pl.LazyFrame, existing_cols: list[str]):
|
|
37
37
|
if "fuel_type" in existing_cols:
|
|
38
38
|
lf = lf.with_columns(
|
|
39
|
-
fuel_type_group=pl.col("fuel_type").cast(pl.String).str.extract("(\w+):?")
|
|
39
|
+
fuel_type_group=pl.col("fuel_type").cast(pl.String).str.extract(r"(\w+):?")
|
|
40
40
|
)
|
|
41
41
|
existing_cols.append("fuel_type_group")
|
|
42
42
|
return lf
|
|
@@ -98,7 +98,7 @@ def add_professional_occupation(lf: pl.LazyFrame, existing_cols: list[str]):
|
|
|
98
98
|
):
|
|
99
99
|
lf = lf.with_columns(
|
|
100
100
|
professional_occupation=pl.col("detailed_professional_occupation").str.extract(
|
|
101
|
-
"(\w+):?"
|
|
101
|
+
r"(\w+):?"
|
|
102
102
|
)
|
|
103
103
|
)
|
|
104
104
|
existing_cols.append("professional_occupation")
|
|
@@ -108,7 +108,7 @@ def add_purpose_groups(lf: pl.LazyFrame, existing_cols: list[str]):
|
|
|
108
108
|
if col in existing_cols:
|
|
109
109
|
group_col = f"{col}_group"
|
|
110
110
|
lf = lf.with_columns(
|
|
111
|
-
pl.col(col).cast(pl.String).str.extract("(\w+):?").alias(group_col)
|
|
111
|
+
pl.col(col).cast(pl.String).str.extract(r"(\w+):?").alias(group_col)
|
|
112
112
|
)
|
|
113
113
|
return lf
|
|
114
114
|
|
|
@@ -23,59 +23,15 @@ from .trajets import standardize_legs
|
|
|
23
23
|
from .zones import read_detailed_zones, read_draw_zones, read_special_locations
|
|
24
24
|
|
|
25
25
|
|
|
26
|
-
def standardize(source: str | ZipFile):
|
|
26
|
+
def standardize(source: str | ZipFile, skip_spatial: bool = False):
|
|
27
27
|
source_name = source.filename if isinstance(source, ZipFile) else source
|
|
28
28
|
logger.info(f"Standardizing EDGT survey from `{source_name}`")
|
|
29
|
-
|
|
30
|
-
filename = special_locations_filename(source)
|
|
31
|
-
if filename is None:
|
|
32
|
-
special_locations = None
|
|
33
|
-
elif "EDGT_SCOTAM_2017" in source_name or "Angers12" in source_name or "EDGTFVG2016":
|
|
34
|
-
# Note. Special case for Metz 2017 and Angers 2012: I cannot figure out the ids in the GT
|
|
35
|
-
# and ZF files.
|
|
36
|
-
# For Annemasse 2016, the GT id does not seem to be specified.
|
|
37
|
-
logger.warning("Cannot read special locations: ids are invalid")
|
|
29
|
+
if skip_spatial:
|
|
38
30
|
special_locations = None
|
|
39
|
-
else:
|
|
40
|
-
logger.debug(f"Reading special locations from `{filename}`")
|
|
41
|
-
special_locations = read_special_locations(filename)
|
|
42
|
-
# Detailed zones.
|
|
43
|
-
filename = detailed_zones_filename(source)
|
|
44
|
-
if filename is None:
|
|
45
|
-
logger.debug(f"No file with detailed zones in `{source_name}`")
|
|
46
|
-
detailed_zones = None
|
|
47
|
-
elif "EDGT_SCOTAM_2017" in source_name or "Angers12" in source_name or "FaF_Zones_fines":
|
|
48
|
-
# Note. Special case for Metz 2017 and Angers 2012: I cannot figure out the ids in the GT
|
|
49
|
-
# and ZF files.
|
|
50
|
-
# For Bayonne 2010, the ids are missing.
|
|
51
|
-
logger.warning("Cannot read detailed zones: ids are invalid")
|
|
52
31
|
detailed_zones = None
|
|
53
|
-
else:
|
|
54
|
-
logger.debug(f"Reading detailed zones from `{filename}`")
|
|
55
|
-
detailed_zones = read_detailed_zones(filename)
|
|
56
|
-
# Draw zones.
|
|
57
|
-
filename = draw_zones_filename(source)
|
|
58
|
-
if filename is None:
|
|
59
|
-
logger.debug(f"No file with draw zones in `{source_name}`")
|
|
60
32
|
draw_zones = None
|
|
61
33
|
else:
|
|
62
|
-
|
|
63
|
-
draw_zones = read_draw_zones(filename)
|
|
64
|
-
|
|
65
|
-
if special_locations is not None:
|
|
66
|
-
# For the Angers 2022 survey, the ids in the detailed zones shapefile have 3 zeros removed
|
|
67
|
-
# compared to the ids in the special locations shapefile.
|
|
68
|
-
assert detailed_zones is not None, (
|
|
69
|
-
"Special locations are defined but there is no data on detailed zones"
|
|
70
|
-
)
|
|
71
|
-
if "detailed_zone_id" not in special_locations.columns:
|
|
72
|
-
special_locations = identify_detailed_zone_id(special_locations, detailed_zones)
|
|
73
|
-
dz_gcd = np.gcd.reduce(detailed_zones["detailed_zone_id"].astype(int))
|
|
74
|
-
sl_gcd = np.gcd.reduce(special_locations["detailed_zone_id"].astype(int))
|
|
75
|
-
if dz_gcd < sl_gcd and np.log10(sl_gcd).is_integer():
|
|
76
|
-
detailed_zones["detailed_zone_id"] = (
|
|
77
|
-
detailed_zones["detailed_zone_id"].astype(int) * sl_gcd
|
|
78
|
-
).astype(str)
|
|
34
|
+
special_locations, detailed_zones, draw_zones = read_spatial_data(source, source_name)
|
|
79
35
|
|
|
80
36
|
if special_locations is not None:
|
|
81
37
|
special_locations_coords = get_coords(special_locations, "special_location")
|
|
@@ -180,6 +136,61 @@ def standardize(source: str | ZipFile):
|
|
|
180
136
|
)
|
|
181
137
|
|
|
182
138
|
|
|
139
|
+
def read_spatial_data(source: str | ZipFile, source_name: str):
|
|
140
|
+
# Special locations.
|
|
141
|
+
filename = special_locations_filename(source)
|
|
142
|
+
if filename is None:
|
|
143
|
+
special_locations = None
|
|
144
|
+
elif "EDGT_SCOTAM_2017" in source_name or "Angers12" in source_name or "EDGTFVG2016":
|
|
145
|
+
# Note. Special case for Metz 2017 and Angers 2012: I cannot figure out the ids in the GT
|
|
146
|
+
# and ZF files.
|
|
147
|
+
# For Annemasse 2016, the GT id does not seem to be specified.
|
|
148
|
+
logger.warning("Cannot read special locations: ids are invalid")
|
|
149
|
+
special_locations = None
|
|
150
|
+
else:
|
|
151
|
+
logger.debug(f"Reading special locations from `{filename}`")
|
|
152
|
+
special_locations = read_special_locations(filename)
|
|
153
|
+
# Detailed zones.
|
|
154
|
+
filename = detailed_zones_filename(source)
|
|
155
|
+
if filename is None:
|
|
156
|
+
logger.debug(f"No file with detailed zones in `{source_name}`")
|
|
157
|
+
detailed_zones = None
|
|
158
|
+
elif "EDGT_SCOTAM_2017" in source_name or "Angers12" in source_name or "FaF_Zones_fines":
|
|
159
|
+
# Note. Special case for Metz 2017 and Angers 2012: I cannot figure out the ids in the GT
|
|
160
|
+
# and ZF files.
|
|
161
|
+
# For Bayonne 2010, the ids are missing.
|
|
162
|
+
logger.warning("Cannot read detailed zones: ids are invalid")
|
|
163
|
+
detailed_zones = None
|
|
164
|
+
else:
|
|
165
|
+
logger.debug(f"Reading detailed zones from `{filename}`")
|
|
166
|
+
detailed_zones = read_detailed_zones(filename)
|
|
167
|
+
# Draw zones.
|
|
168
|
+
filename = draw_zones_filename(source)
|
|
169
|
+
if filename is None:
|
|
170
|
+
logger.debug(f"No file with draw zones in `{source_name}`")
|
|
171
|
+
draw_zones = None
|
|
172
|
+
else:
|
|
173
|
+
logger.debug(f"Reading draw zones from `{filename}`")
|
|
174
|
+
draw_zones = read_draw_zones(filename)
|
|
175
|
+
|
|
176
|
+
if special_locations is not None:
|
|
177
|
+
# For the Angers 2022 survey, the ids in the detailed zones shapefile have 3 zeros removed
|
|
178
|
+
# compared to the ids in the special locations shapefile.
|
|
179
|
+
assert (
|
|
180
|
+
detailed_zones is not None
|
|
181
|
+
), "Special locations are defined but there is no data on detailed zones"
|
|
182
|
+
if "detailed_zone_id" not in special_locations.columns:
|
|
183
|
+
special_locations = identify_detailed_zone_id(special_locations, detailed_zones)
|
|
184
|
+
dz_gcd = np.gcd.reduce(detailed_zones["detailed_zone_id"].astype(int))
|
|
185
|
+
sl_gcd = np.gcd.reduce(special_locations["detailed_zone_id"].astype(int))
|
|
186
|
+
if dz_gcd < sl_gcd and np.log10(sl_gcd).is_integer():
|
|
187
|
+
detailed_zones["detailed_zone_id"] = (
|
|
188
|
+
detailed_zones["detailed_zone_id"].astype(int) * sl_gcd
|
|
189
|
+
).astype(str)
|
|
190
|
+
|
|
191
|
+
return special_locations, detailed_zones, draw_zones
|
|
192
|
+
|
|
193
|
+
|
|
183
194
|
def survey_main_insee(source: str | bytes):
|
|
184
195
|
return pl.scan_csv(source, separator=";").select("IDM4").first().collect().item()
|
|
185
196
|
|
|
@@ -215,7 +226,7 @@ def legs_filename(source: str | ZipFile):
|
|
|
215
226
|
def detailed_zones_filename(source: str | ZipFile):
|
|
216
227
|
return find_file(
|
|
217
228
|
source,
|
|
218
|
-
".*_(ZF|Zones_Fines|Zones fines|ZInterne)(.*)?\.(TAB|shp|MIF)",
|
|
229
|
+
r".*_(ZF|Zones_Fines|Zones fines|ZInterne)(.*)?\.(TAB|shp|MIF)",
|
|
219
230
|
subdir=os.path.join("Doc", "SIG"),
|
|
220
231
|
as_url=True,
|
|
221
232
|
)
|
|
@@ -228,7 +239,7 @@ def special_locations_filename(source: str | ZipFile):
|
|
|
228
239
|
# locations for Lyon.)
|
|
229
240
|
return find_file(
|
|
230
241
|
source,
|
|
231
|
-
"(?!.*ZF_GT).*(_GT|Gener)(.*)?\.(TAB|shp|MIF)",
|
|
242
|
+
r"(?!.*ZF_GT).*(_GT|Gener)(.*)?\.(TAB|shp|MIF)",
|
|
232
243
|
subdir=os.path.join("Doc", "SIG"),
|
|
233
244
|
as_url=True,
|
|
234
245
|
)
|
|
@@ -236,7 +247,7 @@ def special_locations_filename(source: str | ZipFile):
|
|
|
236
247
|
|
|
237
248
|
def draw_zones_filename(source: str | ZipFile):
|
|
238
249
|
return find_file(
|
|
239
|
-
source, ".*_DTIR(_.*)?\.(TAB|shp|MIF)", subdir=os.path.join("Doc", "SIG"), as_url=True
|
|
250
|
+
source, r".*_DTIR(_.*)?\.(TAB|shp|MIF)", subdir=os.path.join("Doc", "SIG"), as_url=True
|
|
240
251
|
)
|
|
241
252
|
|
|
242
253
|
|
|
@@ -267,15 +278,13 @@ def fix_special_locations_with_gcd(
|
|
|
267
278
|
pl.from_pandas(special_locations["detailed_zone_id"].astype(int)),
|
|
268
279
|
default=None,
|
|
269
280
|
)
|
|
270
|
-
)
|
|
271
|
-
.alias(f"{col}_detailed_zone"),
|
|
281
|
+
).alias(f"{col}_detailed_zone"),
|
|
272
282
|
# When the ZF is an actual ZF
|
|
273
283
|
pl.when(mask)
|
|
274
284
|
# Then the GT is null
|
|
275
285
|
.then(pl.lit(None))
|
|
276
286
|
# Otherwise use that GT as GT.
|
|
277
|
-
.otherwise(f"{col}_detailed_zone")
|
|
278
|
-
.alias(f"{col}_special_location"),
|
|
287
|
+
.otherwise(f"{col}_detailed_zone").alias(f"{col}_special_location"),
|
|
279
288
|
)
|
|
280
289
|
return df
|
|
281
290
|
|
|
@@ -298,15 +307,13 @@ def fix_special_locations(df: pl.LazyFrame, col: str, special_locations: gpd.Geo
|
|
|
298
307
|
pl.from_pandas(special_locations["detailed_zone_id"].astype(int)),
|
|
299
308
|
default=None,
|
|
300
309
|
)
|
|
301
|
-
)
|
|
302
|
-
.alias(f"{col}_detailed_zone"),
|
|
310
|
+
).alias(f"{col}_detailed_zone"),
|
|
303
311
|
# When the ZF is an actual ZF
|
|
304
312
|
pl.when(mask)
|
|
305
313
|
# Then the GT is null
|
|
306
314
|
.then(pl.lit(None))
|
|
307
315
|
# Otherwise use that GT as GT.
|
|
308
|
-
.otherwise(f"{col}_detailed_zone")
|
|
309
|
-
.alias(f"{col}_special_location"),
|
|
316
|
+
.otherwise(f"{col}_detailed_zone").alias(f"{col}_special_location"),
|
|
310
317
|
)
|
|
311
318
|
return df
|
|
312
319
|
|
|
@@ -28,40 +28,15 @@ from .zones import (
|
|
|
28
28
|
)
|
|
29
29
|
|
|
30
30
|
|
|
31
|
-
def standardize(source: str | ZipFile):
|
|
31
|
+
def standardize(source: str | ZipFile, skip_spatial: bool = False):
|
|
32
32
|
source_name = source.filename if isinstance(source, ZipFile) else source
|
|
33
33
|
logger.info(f"Standardizing EDVM survey from `{source_name}`")
|
|
34
|
-
if
|
|
35
|
-
|
|
36
|
-
detailed_zones
|
|
37
|
-
else:
|
|
38
|
-
# Special locations.
|
|
39
|
-
if filename := special_locations_filename(source):
|
|
40
|
-
logger.debug(f"Reading special locations from `{filename}`")
|
|
41
|
-
special_locations = read_special_locations(filename)
|
|
42
|
-
else:
|
|
43
|
-
special_locations = None
|
|
44
|
-
# Detailed zones.
|
|
45
|
-
if filename := detailed_zones_filename(source):
|
|
46
|
-
logger.debug(f"Reading detailed zones from `{filename}`")
|
|
47
|
-
detailed_zones = read_detailed_zones(filename)
|
|
48
|
-
else:
|
|
49
|
-
logger.debug(f"No file with detailed zones in `{source_name}`")
|
|
50
|
-
detailed_zones = None
|
|
51
|
-
# Draw zones.
|
|
52
|
-
if filename := draw_zones_filename(source):
|
|
53
|
-
logger.debug(f"Reading draw zones from `{filename}`")
|
|
54
|
-
draw_zones = read_draw_zones(filename)
|
|
55
|
-
else:
|
|
56
|
-
logger.debug(f"No file with draw zones in `{source_name}`")
|
|
34
|
+
if skip_spatial:
|
|
35
|
+
special_locations = None
|
|
36
|
+
detailed_zones = None
|
|
57
37
|
draw_zones = None
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
assert detailed_zones is not None, (
|
|
61
|
-
"Special locations are defined but there is no data on detailed zones"
|
|
62
|
-
)
|
|
63
|
-
if "detailed_zone_id" not in special_locations.columns:
|
|
64
|
-
special_locations = identify_detailed_zone_id(special_locations, detailed_zones)
|
|
38
|
+
else:
|
|
39
|
+
special_locations, detailed_zones, draw_zones = read_spatial_data(source, source_name)
|
|
65
40
|
|
|
66
41
|
if special_locations is not None:
|
|
67
42
|
special_locations_coords = get_coords(special_locations, "special_location")
|
|
@@ -133,6 +108,42 @@ def standardize(source: str | ZipFile):
|
|
|
133
108
|
)
|
|
134
109
|
|
|
135
110
|
|
|
111
|
+
def read_spatial_data(source: str | ZipFile, source_name: str):
|
|
112
|
+
if filename := special_locations_and_detailed_zones_filename(source):
|
|
113
|
+
# Special case for Beauvais 2011.
|
|
114
|
+
detailed_zones, special_locations = read_special_locations_and_detailed_zones(filename)
|
|
115
|
+
else:
|
|
116
|
+
# Special locations.
|
|
117
|
+
if filename := special_locations_filename(source):
|
|
118
|
+
logger.debug(f"Reading special locations from `{filename}`")
|
|
119
|
+
special_locations = read_special_locations(filename)
|
|
120
|
+
else:
|
|
121
|
+
special_locations = None
|
|
122
|
+
# Detailed zones.
|
|
123
|
+
if filename := detailed_zones_filename(source):
|
|
124
|
+
logger.debug(f"Reading detailed zones from `{filename}`")
|
|
125
|
+
detailed_zones = read_detailed_zones(filename)
|
|
126
|
+
else:
|
|
127
|
+
logger.debug(f"No file with detailed zones in `{source_name}`")
|
|
128
|
+
detailed_zones = None
|
|
129
|
+
# Draw zones.
|
|
130
|
+
if filename := draw_zones_filename(source):
|
|
131
|
+
logger.debug(f"Reading draw zones from `{filename}`")
|
|
132
|
+
draw_zones = read_draw_zones(filename)
|
|
133
|
+
else:
|
|
134
|
+
logger.debug(f"No file with draw zones in `{source_name}`")
|
|
135
|
+
draw_zones = None
|
|
136
|
+
|
|
137
|
+
if special_locations is not None:
|
|
138
|
+
assert (
|
|
139
|
+
detailed_zones is not None
|
|
140
|
+
), "Special locations are defined but there is no data on detailed zones"
|
|
141
|
+
if "detailed_zone_id" not in special_locations.columns:
|
|
142
|
+
special_locations = identify_detailed_zone_id(special_locations, detailed_zones)
|
|
143
|
+
|
|
144
|
+
return special_locations, detailed_zones, draw_zones
|
|
145
|
+
|
|
146
|
+
|
|
136
147
|
def survey_name(source: str | ZipFile):
|
|
137
148
|
filename = find_file(source, ".*_std_men.csv", subdir="Csv", as_url=True)
|
|
138
149
|
return re.match("(.*)_std_men.csv", os.path.basename(filename)).group(1)
|
|
@@ -212,15 +223,13 @@ def fix_special_locations_with_gcd(
|
|
|
212
223
|
pl.from_pandas(special_locations["detailed_zone_id"].astype(int)),
|
|
213
224
|
default=None,
|
|
214
225
|
)
|
|
215
|
-
)
|
|
216
|
-
.alias(f"{col}_detailed_zone"),
|
|
226
|
+
).alias(f"{col}_detailed_zone"),
|
|
217
227
|
# When the ZF is an actual ZF
|
|
218
228
|
pl.when(mask)
|
|
219
229
|
# Then the GT is null
|
|
220
230
|
.then(pl.lit(None))
|
|
221
231
|
# Otherwise use that GT as GT.
|
|
222
|
-
.otherwise(f"{col}_detailed_zone")
|
|
223
|
-
.alias(f"{col}_special_location"),
|
|
232
|
+
.otherwise(f"{col}_detailed_zone").alias(f"{col}_special_location"),
|
|
224
233
|
)
|
|
225
234
|
return df
|
|
226
235
|
|
|
@@ -243,15 +252,13 @@ def fix_special_locations(df: pl.LazyFrame, col: str, special_locations: gpd.Geo
|
|
|
243
252
|
pl.from_pandas(special_locations["detailed_zone_id"].astype(int)),
|
|
244
253
|
default=None,
|
|
245
254
|
)
|
|
246
|
-
)
|
|
247
|
-
.alias(f"{col}_detailed_zone"),
|
|
255
|
+
).alias(f"{col}_detailed_zone"),
|
|
248
256
|
# When the ZF is an actual ZF
|
|
249
257
|
pl.when(mask)
|
|
250
258
|
# Then the GT is null
|
|
251
259
|
.then(pl.lit(None))
|
|
252
260
|
# Otherwise use that GT as GT.
|
|
253
|
-
.otherwise(f"{col}_detailed_zone")
|
|
254
|
-
.alias(f"{col}_special_location"),
|
|
261
|
+
.otherwise(f"{col}_detailed_zone").alias(f"{col}_special_location"),
|
|
255
262
|
)
|
|
256
263
|
return df
|
|
257
264
|
|
|
@@ -124,7 +124,7 @@ def standardize_legs(
|
|
|
124
124
|
)
|
|
125
125
|
idx = ["ECH", "STT", "PER", "NDEP"]
|
|
126
126
|
|
|
127
|
-
# For
|
|
127
|
+
# For EDVM, the walking legs are not recorded explicitly, instead, the walking time before and
|
|
128
128
|
# after the legs are defined.
|
|
129
129
|
# We create actual walking legs from these walking times.
|
|
130
130
|
|
|
@@ -122,14 +122,14 @@ def cast_id(s: gpd.GeoSeries):
|
|
|
122
122
|
# "012345" -> "12345"
|
|
123
123
|
# "123 456" -> "123456"
|
|
124
124
|
# "123.456" -> "123456"
|
|
125
|
-
#
|
|
125
|
+
# Although it would be cleaner to keep the leading zero, it is safer to remove them because in
|
|
126
126
|
# some cases there is not the same number of leading zeros in the zone files compared to the
|
|
127
127
|
# survey's CSV files.
|
|
128
128
|
return (
|
|
129
129
|
s.astype(str)
|
|
130
130
|
.str.replace(" ", "")
|
|
131
131
|
.str.replace("[.]0$", "", n=1, regex=True)
|
|
132
|
-
.str.replace("(?<=\d)[.](?=\d)", "", n=1, regex=True)
|
|
132
|
+
.str.replace(r"(?<=\d)[.](?=\d)", "", n=1, regex=True)
|
|
133
133
|
.astype(int, errors="ignore")
|
|
134
134
|
.astype(str)
|
|
135
135
|
)
|
|
@@ -13,15 +13,18 @@ from .trajets import standardize_legs
|
|
|
13
13
|
from .zones import read_detailed_zones
|
|
14
14
|
|
|
15
15
|
|
|
16
|
-
def standardize(source: str | ZipFile):
|
|
16
|
+
def standardize(source: str | ZipFile, skip_spatial: bool = False):
|
|
17
17
|
source_name = source.filename if isinstance(source, ZipFile) else source
|
|
18
18
|
logger.info(f"Standardizing EGT2010 survey from `{source_name}`")
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
19
|
+
if skip_spatial:
|
|
20
|
+
detailed_zones = None
|
|
21
|
+
else:
|
|
22
|
+
# Detailed zones.
|
|
23
|
+
filename = detailed_zones_filename(source)
|
|
24
|
+
if filename is None:
|
|
25
|
+
logger.error("Missing detailed zones file")
|
|
26
|
+
return None
|
|
27
|
+
detailed_zones = read_detailed_zones(filename)
|
|
25
28
|
# Households.
|
|
26
29
|
filename = households_filename(source)
|
|
27
30
|
if filename is None:
|
|
@@ -155,7 +155,7 @@ def standardize_legs(
|
|
|
155
155
|
trips: pl.LazyFrame,
|
|
156
156
|
cars: pl.LazyFrame,
|
|
157
157
|
motorcycles: pl.LazyFrame,
|
|
158
|
-
detailed_zones: gpd.GeoDataFrame,
|
|
158
|
+
detailed_zones: gpd.GeoDataFrame | None,
|
|
159
159
|
):
|
|
160
160
|
lf = scan_legs(filename)
|
|
161
161
|
# Add household_id, person_id, and trip_id.
|
|
@@ -13,7 +13,7 @@ from .trajets import standardize_legs
|
|
|
13
13
|
from .voitures import standardize_cars
|
|
14
14
|
|
|
15
15
|
|
|
16
|
-
def standardize(source: str | ZipFile):
|
|
16
|
+
def standardize(source: str | ZipFile, skip_spatial: bool = False):
|
|
17
17
|
source_name = source.filename if isinstance(source, ZipFile) else source
|
|
18
18
|
logger.info(f"Standardizing EGT2020 survey from `{source_name}`")
|
|
19
19
|
# Households.
|
|
@@ -236,10 +236,10 @@ def standardize_legs(
|
|
|
236
236
|
lf = lf.with_columns(
|
|
237
237
|
original_leg_id=pl.struct("IDCEREMA", "NP", "ND", "NT"),
|
|
238
238
|
# The extract call is required because some modes are reported as "B616,skate", for example.
|
|
239
|
-
mode=pl.col("MOYEN").str.extract("(\w+),?").replace_strict(MODE_MAP),
|
|
240
|
-
car_type=pl.col("UVP").str.extract("(\w+),?").replace_strict(UVP_MAP),
|
|
239
|
+
mode=pl.col("MOYEN").str.extract(r"(\w+),?").replace_strict(MODE_MAP),
|
|
240
|
+
car_type=pl.col("UVP").str.extract(r"(\w+),?").replace_strict(UVP_MAP),
|
|
241
241
|
car_index=pl.col("UVP")
|
|
242
|
-
.str.extract("(\w+),?")
|
|
242
|
+
.str.extract(r"(\w+),?")
|
|
243
243
|
.replace_strict(VEHICLE_INDEX_MAP, default=None),
|
|
244
244
|
motorcycle_type=pl.col("U2RM").replace_strict(U2RM_MAP),
|
|
245
245
|
motorcycle_index=pl.col("U2RM").replace_strict(
|
|
@@ -257,9 +257,9 @@ def standardize_legs(
|
|
|
257
257
|
.cast(pl.List(pl.UInt8))
|
|
258
258
|
.list.set_difference(pl.lit([99])),
|
|
259
259
|
parking_location=pl.col("TSTAT_VP")
|
|
260
|
-
.str.extract("(\w+),?")
|
|
260
|
+
.str.extract(r"(\w+),?")
|
|
261
261
|
.replace_strict(PARKING_LOCATION_MAP),
|
|
262
|
-
parking_type=pl.col("TSTAT_VP").str.extract("(\w+),?").replace_strict(PARKING_TYPE_MAP),
|
|
262
|
+
parking_type=pl.col("TSTAT_VP").str.extract(r"(\w+),?").replace_strict(PARKING_TYPE_MAP),
|
|
263
263
|
)
|
|
264
264
|
lf = lf.with_columns(
|
|
265
265
|
# In some cases, the parking location is a paid location but the person reported a
|