mobisurvstd 0.6.0__tar.gz → 0.7.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {mobisurvstd-0.6.0/src/mobisurvstd.egg-info → mobisurvstd-0.7.0}/PKG-INFO +6 -2
- {mobisurvstd-0.6.0 → mobisurvstd-0.7.0}/README.md +4 -1
- {mobisurvstd-0.6.0 → mobisurvstd-0.7.0}/pyproject.toml +2 -1
- mobisurvstd-0.7.0/src/mobisurvstd/cerema/__init__.py +20 -0
- {mobisurvstd-0.6.0/src/mobisurvstd/edgt → mobisurvstd-0.7.0/src/mobisurvstd/cerema}/common.py +14 -0
- mobisurvstd-0.7.0/src/mobisurvstd/cerema/deplacements.py +236 -0
- mobisurvstd-0.7.0/src/mobisurvstd/cerema/edgt.py +286 -0
- mobisurvstd-0.7.0/src/mobisurvstd/cerema/edvm.py +285 -0
- mobisurvstd-0.7.0/src/mobisurvstd/cerema/emc2.py +156 -0
- mobisurvstd-0.7.0/src/mobisurvstd/cerema/emd.py +144 -0
- mobisurvstd-0.7.0/src/mobisurvstd/cerema/menages.py +394 -0
- mobisurvstd-0.7.0/src/mobisurvstd/cerema/personnes.py +512 -0
- mobisurvstd-0.7.0/src/mobisurvstd/cerema/survey.py +531 -0
- mobisurvstd-0.7.0/src/mobisurvstd/cerema/trajets.py +409 -0
- mobisurvstd-0.7.0/src/mobisurvstd/cerema/zones.py +405 -0
- {mobisurvstd-0.6.0 → mobisurvstd-0.7.0}/src/mobisurvstd/classes.py +7 -7
- mobisurvstd-0.7.0/src/mobisurvstd/cli.py +77 -0
- {mobisurvstd-0.6.0 → mobisurvstd-0.7.0}/src/mobisurvstd/common/cars.py +8 -9
- {mobisurvstd-0.6.0 → mobisurvstd-0.7.0}/src/mobisurvstd/common/clean.py +50 -33
- {mobisurvstd-0.6.0 → mobisurvstd-0.7.0}/src/mobisurvstd/common/households.py +4 -5
- {mobisurvstd-0.6.0 → mobisurvstd-0.7.0}/src/mobisurvstd/common/legs.py +6 -8
- {mobisurvstd-0.6.0 → mobisurvstd-0.7.0}/src/mobisurvstd/common/motorcycles.py +8 -11
- {mobisurvstd-0.6.0 → mobisurvstd-0.7.0}/src/mobisurvstd/common/persons.py +9 -59
- {mobisurvstd-0.6.0 → mobisurvstd-0.7.0}/src/mobisurvstd/common/trips.py +20 -16
- {mobisurvstd-0.6.0 → mobisurvstd-0.7.0}/src/mobisurvstd/egt2010/deplacements.py +39 -8
- {mobisurvstd-0.6.0 → mobisurvstd-0.7.0}/src/mobisurvstd/egt2010/menages.py +8 -2
- {mobisurvstd-0.6.0 → mobisurvstd-0.7.0}/src/mobisurvstd/egt2010/personnes.py +63 -33
- {mobisurvstd-0.6.0 → mobisurvstd-0.7.0}/src/mobisurvstd/egt2010/survey.py +7 -4
- {mobisurvstd-0.6.0 → mobisurvstd-0.7.0}/src/mobisurvstd/egt2010/trajets.py +10 -2
- {mobisurvstd-0.6.0 → mobisurvstd-0.7.0}/src/mobisurvstd/egt2020/deplacements.py +13 -9
- {mobisurvstd-0.6.0 → mobisurvstd-0.7.0}/src/mobisurvstd/egt2020/menages.py +11 -18
- {mobisurvstd-0.6.0 → mobisurvstd-0.7.0}/src/mobisurvstd/egt2020/personnes.py +18 -9
- {mobisurvstd-0.6.0 → mobisurvstd-0.7.0}/src/mobisurvstd/egt2020/trajets.py +45 -21
- {mobisurvstd-0.6.0 → mobisurvstd-0.7.0}/src/mobisurvstd/emp/deplacements.py +3 -2
- {mobisurvstd-0.6.0 → mobisurvstd-0.7.0}/src/mobisurvstd/emp/menages.py +1 -8
- {mobisurvstd-0.6.0 → mobisurvstd-0.7.0}/src/mobisurvstd/emp/motos.py +3 -3
- {mobisurvstd-0.6.0 → mobisurvstd-0.7.0}/src/mobisurvstd/emp/personnes.py +18 -9
- {mobisurvstd-0.6.0 → mobisurvstd-0.7.0}/src/mobisurvstd/emp/voitures.py +15 -17
- {mobisurvstd-0.6.0 → mobisurvstd-0.7.0}/src/mobisurvstd/main.py +26 -8
- mobisurvstd-0.7.0/src/mobisurvstd/resources/__init__.py +21 -0
- {mobisurvstd-0.6.0 → mobisurvstd-0.7.0}/src/mobisurvstd/resources/insee_data.py +9 -1
- mobisurvstd-0.7.0/src/mobisurvstd/schema/__init__.py +11 -0
- mobisurvstd-0.7.0/src/mobisurvstd/schema/cars.py +107 -0
- mobisurvstd-0.7.0/src/mobisurvstd/schema/common.py +162 -0
- mobisurvstd-0.7.0/src/mobisurvstd/schema/guarantees.py +698 -0
- mobisurvstd-0.7.0/src/mobisurvstd/schema/households.py +163 -0
- mobisurvstd-0.7.0/src/mobisurvstd/schema/legs.py +272 -0
- mobisurvstd-0.7.0/src/mobisurvstd/schema/motorcycles.py +78 -0
- mobisurvstd-0.7.0/src/mobisurvstd/schema/persons.py +940 -0
- mobisurvstd-0.7.0/src/mobisurvstd/schema/trips.py +485 -0
- mobisurvstd-0.7.0/src/mobisurvstd/validation.py +706 -0
- {mobisurvstd-0.6.0 → mobisurvstd-0.7.0/src/mobisurvstd.egg-info}/PKG-INFO +6 -2
- {mobisurvstd-0.6.0 → mobisurvstd-0.7.0}/src/mobisurvstd.egg-info/SOURCES.txt +23 -37
- {mobisurvstd-0.6.0 → mobisurvstd-0.7.0}/src/mobisurvstd.egg-info/requires.txt +1 -0
- mobisurvstd-0.6.0/src/mobisurvstd/clean-old.py +0 -1661
- mobisurvstd-0.6.0/src/mobisurvstd/clean.py +0 -50
- mobisurvstd-0.6.0/src/mobisurvstd/cli.py +0 -44
- mobisurvstd-0.6.0/src/mobisurvstd/common/modes.py +0 -52
- mobisurvstd-0.6.0/src/mobisurvstd/edgt/deplacements.py +0 -167
- mobisurvstd-0.6.0/src/mobisurvstd/edgt/menages.py +0 -353
- mobisurvstd-0.6.0/src/mobisurvstd/edgt/personnes.py +0 -307
- mobisurvstd-0.6.0/src/mobisurvstd/edgt/survey.py +0 -335
- mobisurvstd-0.6.0/src/mobisurvstd/edgt/trajets.py +0 -278
- mobisurvstd-0.6.0/src/mobisurvstd/edgt/zones.py +0 -193
- mobisurvstd-0.6.0/src/mobisurvstd/edvm/common.py +0 -43
- mobisurvstd-0.6.0/src/mobisurvstd/edvm/deplacements.py +0 -160
- mobisurvstd-0.6.0/src/mobisurvstd/edvm/menages.py +0 -353
- mobisurvstd-0.6.0/src/mobisurvstd/edvm/personnes.py +0 -308
- mobisurvstd-0.6.0/src/mobisurvstd/edvm/survey.py +0 -280
- mobisurvstd-0.6.0/src/mobisurvstd/edvm/trajets.py +0 -271
- mobisurvstd-0.6.0/src/mobisurvstd/edvm/zones.py +0 -374
- mobisurvstd-0.6.0/src/mobisurvstd/egt2020/__init__.py +0 -1
- mobisurvstd-0.6.0/src/mobisurvstd/emc2/__init__.py +0 -1
- mobisurvstd-0.6.0/src/mobisurvstd/emc2/common.py +0 -42
- mobisurvstd-0.6.0/src/mobisurvstd/emc2/deplacements.py +0 -163
- mobisurvstd-0.6.0/src/mobisurvstd/emc2/menages.py +0 -358
- mobisurvstd-0.6.0/src/mobisurvstd/emc2/personnes.py +0 -305
- mobisurvstd-0.6.0/src/mobisurvstd/emc2/survey.py +0 -291
- mobisurvstd-0.6.0/src/mobisurvstd/emc2/trajets.py +0 -274
- mobisurvstd-0.6.0/src/mobisurvstd/emc2/zones.py +0 -107
- mobisurvstd-0.6.0/src/mobisurvstd/emd/__init__.py +0 -1
- mobisurvstd-0.6.0/src/mobisurvstd/emd/common.py +0 -42
- mobisurvstd-0.6.0/src/mobisurvstd/emd/deplacements.py +0 -160
- mobisurvstd-0.6.0/src/mobisurvstd/emd/menages.py +0 -355
- mobisurvstd-0.6.0/src/mobisurvstd/emd/personnes.py +0 -312
- mobisurvstd-0.6.0/src/mobisurvstd/emd/survey.py +0 -283
- mobisurvstd-0.6.0/src/mobisurvstd/emd/trajets.py +0 -271
- mobisurvstd-0.6.0/src/mobisurvstd/emd/zones.py +0 -222
- mobisurvstd-0.6.0/src/mobisurvstd/emp/__init__.py +0 -1
- mobisurvstd-0.6.0/src/mobisurvstd/resources/__init__.py +0 -3
- mobisurvstd-0.6.0/src/mobisurvstd/schema.py +0 -917
- {mobisurvstd-0.6.0 → mobisurvstd-0.7.0}/LICENSE.md +0 -0
- {mobisurvstd-0.6.0 → mobisurvstd-0.7.0}/setup.cfg +0 -0
- {mobisurvstd-0.6.0 → mobisurvstd-0.7.0}/src/mobisurvstd/__init__.py +0 -0
- {mobisurvstd-0.6.0 → mobisurvstd-0.7.0}/src/mobisurvstd/__main__.py +0 -0
- {mobisurvstd-0.6.0 → mobisurvstd-0.7.0}/src/mobisurvstd/common/__init__.py +0 -0
- {mobisurvstd-0.6.0 → mobisurvstd-0.7.0}/src/mobisurvstd/common/zones.py +0 -0
- {mobisurvstd-0.6.0/src/mobisurvstd/edgt → mobisurvstd-0.7.0/src/mobisurvstd/egt2010}/__init__.py +0 -0
- {mobisurvstd-0.6.0 → mobisurvstd-0.7.0}/src/mobisurvstd/egt2010/zones.py +0 -0
- {mobisurvstd-0.6.0/src/mobisurvstd/edvm → mobisurvstd-0.7.0/src/mobisurvstd/egt2020}/__init__.py +0 -0
- {mobisurvstd-0.6.0 → mobisurvstd-0.7.0}/src/mobisurvstd/egt2020/motos.py +0 -0
- {mobisurvstd-0.6.0 → mobisurvstd-0.7.0}/src/mobisurvstd/egt2020/survey.py +0 -0
- {mobisurvstd-0.6.0 → mobisurvstd-0.7.0}/src/mobisurvstd/egt2020/voitures.py +0 -0
- {mobisurvstd-0.6.0/src/mobisurvstd/egt2010 → mobisurvstd-0.7.0/src/mobisurvstd/emp}/__init__.py +0 -0
- {mobisurvstd-0.6.0 → mobisurvstd-0.7.0}/src/mobisurvstd/emp/survey.py +0 -0
- {mobisurvstd-0.6.0 → mobisurvstd-0.7.0}/src/mobisurvstd/logger.py +0 -0
- {mobisurvstd-0.6.0 → mobisurvstd-0.7.0}/src/mobisurvstd/resources/admin_express.py +0 -0
- {mobisurvstd-0.6.0 → mobisurvstd-0.7.0}/src/mobisurvstd/resources/nuts.py +0 -0
- {mobisurvstd-0.6.0 → mobisurvstd-0.7.0}/src/mobisurvstd/utils.py +0 -0
- {mobisurvstd-0.6.0 → mobisurvstd-0.7.0}/src/mobisurvstd.egg-info/dependency_links.txt +0 -0
- {mobisurvstd-0.6.0 → mobisurvstd-0.7.0}/src/mobisurvstd.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: mobisurvstd
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.7.0
|
|
4
4
|
Summary: A tool to standardize French mobility survey datasets (EMC2, EGT, EMP, etc.) into a unified format.
|
|
5
5
|
Author-email: Lucas Javaudin <lucas@lucasjavaudin.com>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -18,6 +18,7 @@ License-File: LICENSE.md
|
|
|
18
18
|
Requires-Dist: Cartopy>=0.24.1
|
|
19
19
|
Requires-Dist: fastexcel>=0.14.0
|
|
20
20
|
Requires-Dist: geopandas>=1.0.1
|
|
21
|
+
Requires-Dist: humanize>=4.13.0
|
|
21
22
|
Requires-Dist: loguru>=0.7.3
|
|
22
23
|
Requires-Dist: matplotlib>=3.10.1
|
|
23
24
|
Requires-Dist: numpy>=2.2.4
|
|
@@ -134,6 +135,10 @@ It will standardize the EMP 2019 survey and save the resulting Parquet files in
|
|
|
134
135
|
These Parquet files can then be analyzed using e.g., [polars](https://pola.rs/) or
|
|
135
136
|
[pandas](https://pandas.pydata.org) in Python or [arrow](https://arrow.apache.org/docs/r/) in R.
|
|
136
137
|
|
|
138
|
+
**Note:** During standardization, MobiSurvStd may issue warnings when certain variable guarantees
|
|
139
|
+
are not initially satisfied but are automatically fixed.
|
|
140
|
+
These warnings can usually be safely ignored, unless they pertain to a large number of observations.
|
|
141
|
+
|
|
137
142
|
A detailed definition of the Parquet format used by MobiSurvStd is available
|
|
138
143
|
[here](https://mobisurvstd.github.io/MobiSurvStd/format/index.html).
|
|
139
144
|
|
|
@@ -144,7 +149,6 @@ import mobisurvstd
|
|
|
144
149
|
mobisurvstd.standardize(
|
|
145
150
|
"emp_2019_donnees_individuelles_anonymisees_novembre2024.zip",
|
|
146
151
|
"standardized_emp2019",
|
|
147
|
-
survey_type="emp2019",
|
|
148
152
|
)
|
|
149
153
|
```
|
|
150
154
|
|
|
@@ -102,6 +102,10 @@ It will standardize the EMP 2019 survey and save the resulting Parquet files in
|
|
|
102
102
|
These Parquet files can then be analyzed using e.g., [polars](https://pola.rs/) or
|
|
103
103
|
[pandas](https://pandas.pydata.org) in Python or [arrow](https://arrow.apache.org/docs/r/) in R.
|
|
104
104
|
|
|
105
|
+
**Note:** During standardization, MobiSurvStd may issue warnings when certain variable guarantees
|
|
106
|
+
are not initially satisfied but are automatically fixed.
|
|
107
|
+
These warnings can usually be safely ignored, unless they pertain to a large number of observations.
|
|
108
|
+
|
|
105
109
|
A detailed definition of the Parquet format used by MobiSurvStd is available
|
|
106
110
|
[here](https://mobisurvstd.github.io/MobiSurvStd/format/index.html).
|
|
107
111
|
|
|
@@ -112,7 +116,6 @@ import mobisurvstd
|
|
|
112
116
|
mobisurvstd.standardize(
|
|
113
117
|
"emp_2019_donnees_individuelles_anonymisees_novembre2024.zip",
|
|
114
118
|
"standardized_emp2019",
|
|
115
|
-
survey_type="emp2019",
|
|
116
119
|
)
|
|
117
120
|
```
|
|
118
121
|
|
|
@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
|
|
|
6
6
|
|
|
7
7
|
[project]
|
|
8
8
|
name = "mobisurvstd"
|
|
9
|
-
version = "0.
|
|
9
|
+
version = "0.7.0"
|
|
10
10
|
description = "A tool to standardize French mobility survey datasets (EMC2, EGT, EMP, etc.) into a unified format."
|
|
11
11
|
readme = "README.md"
|
|
12
12
|
authors = [{ name = "Lucas Javaudin", email = "lucas@lucasjavaudin.com" }]
|
|
@@ -22,6 +22,7 @@ dependencies = [
|
|
|
22
22
|
"Cartopy >= 0.24.1",
|
|
23
23
|
"fastexcel >= 0.14.0",
|
|
24
24
|
"geopandas >= 1.0.1",
|
|
25
|
+
"humanize >= 4.13.0",
|
|
25
26
|
"loguru >= 0.7.3",
|
|
26
27
|
"matplotlib >= 3.10.1",
|
|
27
28
|
"numpy >= 2.2.4",
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
from zipfile import ZipFile
|
|
2
|
+
|
|
3
|
+
from .edgt import EDGTReader
|
|
4
|
+
from .edvm import EDVMReader
|
|
5
|
+
from .emc2 import EMC2Reader
|
|
6
|
+
from .emd import EMDReader
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def standardize(source: str | ZipFile, survey_type: str, skip_spatial: bool = False):
|
|
10
|
+
if survey_type == "EMC2":
|
|
11
|
+
reader = EMC2Reader(source)
|
|
12
|
+
elif survey_type == "EDGT":
|
|
13
|
+
reader = EDGTReader(source)
|
|
14
|
+
elif survey_type == "EDVM":
|
|
15
|
+
reader = EDVMReader(source)
|
|
16
|
+
elif survey_type == "EMD":
|
|
17
|
+
reader = EMDReader(source)
|
|
18
|
+
else:
|
|
19
|
+
raise NotImplementedError
|
|
20
|
+
return reader.standardize(skip_spatial)
|
{mobisurvstd-0.6.0/src/mobisurvstd/edgt → mobisurvstd-0.7.0/src/mobisurvstd/cerema}/common.py
RENAMED
|
@@ -28,6 +28,7 @@ MODE_MAP = {
|
|
|
28
28
|
61: "taxi", # Passager taxi.
|
|
29
29
|
62: "VTC", # Passager VTC
|
|
30
30
|
71: "employer_transport", # Transport employeur (exclusivement)
|
|
31
|
+
72: "public_transit:school", # For Thionville 2012
|
|
31
32
|
81: "truck:driver", # Conducteur de fourgon, camionnette, camion (pour tournées professionnelles ou déplacements privés).
|
|
32
33
|
82: "truck:passenger", # Passager de fourgon, camionnette, camion (pour tournées professionnelles ou déplacements privés)..
|
|
33
34
|
91: "water_transport", # Transport Fluvial ou maritime.
|
|
@@ -38,3 +39,16 @@ MODE_MAP = {
|
|
|
38
39
|
96: "personal_transporter:motorized", # Petits engins électriques (trottinette, segway, solowheel, etc)
|
|
39
40
|
97: "personal_transporter:unspecified", # Roller, skate, trottinette électrique ou non (ancien code)
|
|
40
41
|
}
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
# For the EMC2 surveys, categories 17 and 18 have moved to 19 and 20, while 17 and 18 are now
|
|
45
|
+
# representing electric bicycle.
|
|
46
|
+
EMC2_MODE_MAP = MODE_MAP.copy()
|
|
47
|
+
EMC2_MODE_MAP.update(
|
|
48
|
+
{
|
|
49
|
+
17: "bicycle:driver:electric", # Conducteur de vélo Assistance Electrique
|
|
50
|
+
18: "bicycle:driver:electric:shared", # Conducteur de vélo Assistance Electrique en Libre Service
|
|
51
|
+
19: "motorcycle:driver", # Conducteur de deux ou trois roues motorisés (si pas de détail sur la cylindrée).
|
|
52
|
+
20: "motorcycle:passenger", # Passager de deux ou trois roues motorisés (si pas de détail sur la cylindrée)..
|
|
53
|
+
}
|
|
54
|
+
)
|
|
@@ -0,0 +1,236 @@
|
|
|
1
|
+
import polars as pl
|
|
2
|
+
from loguru import logger
|
|
3
|
+
|
|
4
|
+
from mobisurvstd.common.trips import clean
|
|
5
|
+
|
|
6
|
+
SCHEMA = {
|
|
7
|
+
"DP1": pl.UInt8, # Code fichier = 3 (déplacement)
|
|
8
|
+
"DMET": pl.UInt8, # Méthode d'enquête du ménage (EMC2 only)
|
|
9
|
+
"IDD3": pl.UInt16, # Année de fin d'enquête
|
|
10
|
+
"IDD4": pl.String, # Code Insee ville centre
|
|
11
|
+
"ZFD": pl.String, # Zone fine de résidence
|
|
12
|
+
"ECH": pl.UInt32, # Numéro d’échantillon
|
|
13
|
+
"PER": pl.UInt8, # Numéro de personne
|
|
14
|
+
"NDEP": pl.UInt8, # Numéro de déplacement
|
|
15
|
+
"GD1": pl.String, # Insee Zone fine du lieu de résidence de la personne concernée par le déplacement
|
|
16
|
+
"STD": pl.String, # Secteur de tirage dans l’enquête d’origine (résidence)
|
|
17
|
+
"D2A": pl.UInt8, # Motif Origine du déplacement
|
|
18
|
+
"D2B": pl.UInt8, # Motif Origine de la personne accompagnée
|
|
19
|
+
"D3": pl.String, # Zone fine Origine du déplacement
|
|
20
|
+
"GDO1": pl.String, # Insee Zone fine Origine du déplacement
|
|
21
|
+
"STDO": pl.String, # Secteur de tirage dans l’enquête d’origine (origine du déplacement)
|
|
22
|
+
"D4": pl.Int32, # Heure de départ du déplacement
|
|
23
|
+
"D5A": pl.UInt8, # Motif Destination du déplacement
|
|
24
|
+
"D5B": pl.UInt8, # Motif Destination de la personne accompagnée
|
|
25
|
+
"D6": pl.UInt8, # Nombre d’arrêts sur la tournée
|
|
26
|
+
"D7": pl.String, # Zone fine Destination du déplacement
|
|
27
|
+
"GDD1": pl.String, # Insee Zone fine Destination du déplacement
|
|
28
|
+
"STDD": pl.String, # Secteur de tirage dans l’enquête d’origine (destination du déplacement)
|
|
29
|
+
"D8": pl.Int32, # Heure d'arrivée du déplacement
|
|
30
|
+
"D9": pl.Int32, # Durée du déplacement
|
|
31
|
+
"D10": pl.UInt8, # Nombre de trajets (en modes mécanisés)
|
|
32
|
+
"D11": pl.Float64, # Longueur à vol d'oiseau
|
|
33
|
+
"D12": pl.Float64, # Distance parcourue
|
|
34
|
+
"MODP": pl.UInt8, # Mode principal
|
|
35
|
+
"TYPD": pl.UInt8, # Type de déplacement
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
PURPOSE_MAP = {
|
|
39
|
+
1: "home:main", # Domicile (partir de, se rendre à).
|
|
40
|
+
2: "home:secondary", # Résidence secondaire, logement occasionnel, hôtel, autre domicile (partir de, se rendre à).
|
|
41
|
+
11: "work:usual", # Travailler sur le lieu d’emploi déclaré.
|
|
42
|
+
12: "work:telework", # Travailler sur un autre lieu – télétravail.
|
|
43
|
+
13: "work:secondary", # Travailler sur un autre lieu hors télétravail
|
|
44
|
+
14: "work:other", # Travailler sur un autre lieu sans distinction
|
|
45
|
+
# NOTE. We discard the information on the exact study location type (middle-school, high-school,
|
|
46
|
+
# etc.) because the information can read from the person characteristics (`student_group`).
|
|
47
|
+
21: "education:childcare", # Être gardé (Nourrice, crèche...).
|
|
48
|
+
22: "education:usual", # Étudier sur le lieu d'études déclaré (école maternelle et primaire).
|
|
49
|
+
23: "education:usual", # Étudier sur le lieu d'études déclaré (collège).
|
|
50
|
+
24: "education:usual", # Étudier sur le lieu d'études déclaré (lycée).
|
|
51
|
+
25: "education:usual", # Étudier sur le lieu d'études déclaré (universités et grandes écoles).
|
|
52
|
+
26: "education:other", # Étudier sur un autre lieu (école maternelle et primaire).
|
|
53
|
+
27: "education:other", # Étudier sur un autre lieu (collège).
|
|
54
|
+
28: "education:other", # Étudier sur un autre lieu (lycée).
|
|
55
|
+
29: "education:other", # Étudier sur un autre lieu (universités et grandes écoles).
|
|
56
|
+
30: "shopping:no_purchase", # Visite d’un magasin, d’un centre commercial ou d’un marché de plein vent sans effectuer d’achat
|
|
57
|
+
31: "shopping:unspecified", # Réaliser plusieurs motifs en centre commercial.
|
|
58
|
+
32: "shopping:unspecified", # Faire des achats en grand magasin, supermarché, hypermarché et leurs galeries marchandes.
|
|
59
|
+
33: "shopping:unspecified", # Faire des achats en petit et moyen commerce et drive in
|
|
60
|
+
34: "shopping:unspecified", # Faire des achats en marché couvert et de plein vent.
|
|
61
|
+
35: "shopping:pickup", # Récupérer des achats faits à distance (Drive, points relais)
|
|
62
|
+
41: "task:healthcare", # Recevoir des soins (santé).
|
|
63
|
+
42: "task:procedure", # Faire une démarche autre que rechercher un emploi.
|
|
64
|
+
43: "task:job_search", # Rechercher un emploi.
|
|
65
|
+
51: "leisure:sport_or_culture", # Participer à des loisirs, des activités sportives, culturelles ou associatives.
|
|
66
|
+
52: "leisure:walk_or_driving_lesson", # Faire une promenade, du « lèche-vitrines », prendre une leçon de conduite.
|
|
67
|
+
53: "leisure:restaurant", # Se restaurer hors du domicile.
|
|
68
|
+
54: "leisure:visiting", # Visiter des parents ou des amis.
|
|
69
|
+
# NOTE. We discard the information on whether the escorted person is present during pick_up /
|
|
70
|
+
# drop_off activity as it can be deduced (when I go pick-up someone, he's not there; when I come
|
|
71
|
+
# back from picking-up someone, he's there). Also, there are some errors in the data (like
|
|
72
|
+
# having the escorted person both before and after picking him up).
|
|
73
|
+
61: "escort:activity:drop_off", # Accompagner quelqu’un (personne présente).
|
|
74
|
+
62: "escort:activity:pick_up", # Aller chercher quelqu’un (personne présente).
|
|
75
|
+
63: "escort:activity:drop_off", # Accompagner quelqu’un (personne absente).
|
|
76
|
+
64: "escort:activity:pick_up", # Aller chercher quelqu’un (personne absente).
|
|
77
|
+
68: "escort:unspecified:drop_off", # Accompagner quelqu’un ou déposer quelqu’un à un mode de transport (sans info présence personne)
|
|
78
|
+
69: "escort:unspecified:pick_up", # Aller chercher quelqu’un ou reprendre quelqu’un à un mode de transport (sans info présence personne)
|
|
79
|
+
71: "escort:transport:drop_off", # Déposer une personne à un mode de transport (personne présente).
|
|
80
|
+
72: "escort:transport:pick_up", # Reprendre une personne à un mode de transport (personne présente).
|
|
81
|
+
73: "escort:transport:drop_off", # Déposer d’une personne à un mode de transport (personne absente).
|
|
82
|
+
74: "escort:transport:pick_up", # Reprendre une personne à un mode de transport( personne absente).
|
|
83
|
+
81: "work:professional_tour", # Réaliser une tournée professionnelle.
|
|
84
|
+
82: "shopping:tour_no_purchase", # Tournée de magasin sans achat
|
|
85
|
+
91: "other", # Autres motifs
|
|
86
|
+
# 96: "escort:middle_high_school", # Étudier sur le lieu d'études déclaré (collège ou lycée). Cas egt personne accompagnée
|
|
87
|
+
# 97: "escort:middle_high_school:other", # Étudier sur un autre lieu (collège ou lycée). Cas egt personne accompagnée
|
|
88
|
+
# 98: "escort:shopping", # Faire des achats sans précision (egt, motif personne accompagnée)
|
|
89
|
+
# Note: Values 96, 97, and 98 are documented but never appear in a survey.
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
SHOP_TYPE_MAP = {
|
|
93
|
+
31: "mall", # Réaliser plusieurs motifs en centre commercial.
|
|
94
|
+
32: "supermarket_or_hypermarket", # Faire des achats en grand magasin, supermarché, hypermarché et leurs galeries marchandes.
|
|
95
|
+
33: "small_shop", # Faire des achats en petit et moyen commerce et drive in
|
|
96
|
+
34: "market", # Faire des achats en marché couvert et de plein vent.
|
|
97
|
+
35: "drive_in", # Récupérer des achats faits à distance (Drive, points relais)
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
TRIP_PERIMETER_MAP = {
|
|
101
|
+
1: "internal", # interne au périmètre d'enquête
|
|
102
|
+
2: "crossing", # en échange
|
|
103
|
+
3: "external", # externe au périmètre d'enquête
|
|
104
|
+
9: None, # inconnu (cas où origine ou destination inconnue)
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def scan_trips_impl(source: str | bytes):
|
|
109
|
+
return pl.scan_csv(source, separator=";", schema_overrides=SCHEMA, null_values=["aa", "aaaaa"])
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
class TripsReader:
|
|
113
|
+
def scan_trips(self):
|
|
114
|
+
lfs_iter = map(scan_trips_impl, self.trips_filenames())
|
|
115
|
+
lf = pl.concat(lfs_iter, how="vertical")
|
|
116
|
+
return lf
|
|
117
|
+
|
|
118
|
+
def standardize_trips(self):
|
|
119
|
+
lf = self.scan_trips()
|
|
120
|
+
# Add household_id, person_id, and trip date.
|
|
121
|
+
lf = lf.with_columns(
|
|
122
|
+
original_person_id=pl.struct(**self.get_person_index_cols_from_trips())
|
|
123
|
+
).join(
|
|
124
|
+
self.persons.select("original_person_id", "person_id", "household_id", "trip_date"),
|
|
125
|
+
on="original_person_id",
|
|
126
|
+
how="left",
|
|
127
|
+
coalesce=True,
|
|
128
|
+
)
|
|
129
|
+
lf = lf.rename(
|
|
130
|
+
{
|
|
131
|
+
"GDO1": "origin_insee",
|
|
132
|
+
"D3": "origin_detailed_zone",
|
|
133
|
+
"STDO": "origin_draw_zone",
|
|
134
|
+
"GDD1": "destination_insee",
|
|
135
|
+
"D7": "destination_detailed_zone",
|
|
136
|
+
"STDD": "destination_draw_zone",
|
|
137
|
+
"D6": "nb_tour_stops",
|
|
138
|
+
}
|
|
139
|
+
)
|
|
140
|
+
lf = lf.with_columns(
|
|
141
|
+
original_trip_id=pl.struct(self.get_trip_index_cols()),
|
|
142
|
+
origin_purpose=pl.col("D2A").replace_strict(PURPOSE_MAP),
|
|
143
|
+
origin_escort_purpose=pl.col("D2B").replace_strict(PURPOSE_MAP),
|
|
144
|
+
origin_shop_type=pl.col("D2A").replace(SHOP_TYPE_MAP, default=None),
|
|
145
|
+
departure_time=60 * (pl.col("D4") // 100) + pl.col("D4") % 100,
|
|
146
|
+
destination_purpose=pl.col("D5A").replace_strict(PURPOSE_MAP),
|
|
147
|
+
destination_escort_purpose=pl.col("D5B").replace_strict(PURPOSE_MAP),
|
|
148
|
+
destination_shop_type=pl.col("D5A").replace(SHOP_TYPE_MAP, default=None),
|
|
149
|
+
arrival_time=60 * (pl.col("D8") // 100) + pl.col("D8") % 100,
|
|
150
|
+
trip_euclidean_distance_km=pl.col("D11") / 1e3,
|
|
151
|
+
trip_travel_distance_km=pl.col("D12") / 1e3,
|
|
152
|
+
main_mode=pl.col("MODP").replace_strict(self.get_mode_map()),
|
|
153
|
+
trip_perimeter=pl.col("TYPD").replace_strict(TRIP_PERIMETER_MAP),
|
|
154
|
+
)
|
|
155
|
+
lf = lf.with_columns(
|
|
156
|
+
# In some cases, the departure and arrival times are switched.
|
|
157
|
+
departure_time=pl.when(
|
|
158
|
+
pl.col("D9") == pl.col("departure_time") - pl.col("arrival_time")
|
|
159
|
+
)
|
|
160
|
+
.then("arrival_time")
|
|
161
|
+
.otherwise("departure_time"),
|
|
162
|
+
arrival_time=pl.when(pl.col("D9") == pl.col("departure_time") - pl.col("arrival_time"))
|
|
163
|
+
.then("departure_time")
|
|
164
|
+
.otherwise("arrival_time"),
|
|
165
|
+
# When destination purpose is 52 (« Faire une promenade, du « lèche-vitrines », prendre
|
|
166
|
+
# une leçon de conduite. ») and `nb_tour_stops` is defined, then we know that purpose is
|
|
167
|
+
# « lèche-vitrines » so we can set purpose to "shopping:tour_no_purchase".
|
|
168
|
+
destination_purpose=pl.when(pl.col("D5A").eq(52), pl.col("nb_tour_stops").is_not_null())
|
|
169
|
+
.then(pl.lit("shopping:tour_no_purchase"))
|
|
170
|
+
.otherwise("destination_purpose"),
|
|
171
|
+
# Special case for Saint-Brieuc 2012: there is no NULL value in the D6 column
|
|
172
|
+
# (nb_tour_stops). It seems that the column represents either the trip travel time or
|
|
173
|
+
# the actual number of tour stops. To be safe, we just throw away all the values.
|
|
174
|
+
nb_tour_stops=pl.when(pl.col("nb_tour_stops").is_not_null().all())
|
|
175
|
+
.then(pl.lit(None))
|
|
176
|
+
.otherwise("nb_tour_stops"),
|
|
177
|
+
# Values 99000, 99999, 99095, 99300 do not represent any known INSEE / country.
|
|
178
|
+
origin_insee=pl.col("origin_insee").replace(["99000", "99999", "99095", "99300"], None),
|
|
179
|
+
destination_insee=pl.col("destination_insee").replace(
|
|
180
|
+
["99000", "99999", "99095", "99300"], None
|
|
181
|
+
),
|
|
182
|
+
)
|
|
183
|
+
lf = lf.sort("original_trip_id")
|
|
184
|
+
lf = fix_origin_destination_detailed_zones(lf)
|
|
185
|
+
df = lf.collect()
|
|
186
|
+
year = round(df["trip_date"].dt.year().mean())
|
|
187
|
+
# For Besançon 2018, there is one missing person.
|
|
188
|
+
n = df["person_id"].null_count()
|
|
189
|
+
if n > 0:
|
|
190
|
+
logger.warning(f"{n} trips are assigned to an unknown person. They are dropped.")
|
|
191
|
+
df = df.filter(pl.col("person_id").is_not_null())
|
|
192
|
+
self.trips = clean(
|
|
193
|
+
df.lazy(),
|
|
194
|
+
year=year,
|
|
195
|
+
special_locations=self.special_locations_coords,
|
|
196
|
+
detailed_zones=self.detailed_zones_coords,
|
|
197
|
+
)
|
|
198
|
+
# When the INSEE code ends with "000" or "999" it means "rest of the département".
|
|
199
|
+
# We drop these values because they do not add any additional information compared to `_dep`
|
|
200
|
+
# columns.
|
|
201
|
+
# This is done after the automatic cleaning so that the département is correctly read.
|
|
202
|
+
self.trips = self.trips.with_columns(
|
|
203
|
+
origin_insee=pl.when(
|
|
204
|
+
pl.col("origin_insee").str.ends_with("000")
|
|
205
|
+
| pl.col("origin_insee").str.ends_with("999")
|
|
206
|
+
)
|
|
207
|
+
.then(None)
|
|
208
|
+
.otherwise("origin_insee"),
|
|
209
|
+
destination_insee=pl.when(
|
|
210
|
+
pl.col("destination_insee").str.ends_with("000")
|
|
211
|
+
| pl.col("destination_insee").str.ends_with("999")
|
|
212
|
+
)
|
|
213
|
+
.then(None)
|
|
214
|
+
.otherwise("destination_insee"),
|
|
215
|
+
)
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def fix_origin_destination_detailed_zones(lf: pl.LazyFrame):
|
|
219
|
+
# For external origin / destination location, the detailed zone id is sometimes set to
|
|
220
|
+
# "8" + INSEE or "9" + INSEE. In this case, keeping the detailed zone id does not add any
|
|
221
|
+
# information so we set it to NULL.
|
|
222
|
+
lf = lf.with_columns(
|
|
223
|
+
origin_detailed_zone=pl.when("8" + pl.col("origin_insee") == pl.col("origin_detailed_zone"))
|
|
224
|
+
.then(None)
|
|
225
|
+
.when("9" + pl.col("origin_insee") == pl.col("origin_detailed_zone"))
|
|
226
|
+
.then(None)
|
|
227
|
+
.otherwise("origin_detailed_zone"),
|
|
228
|
+
destination_detailed_zone=pl.when(
|
|
229
|
+
"8" + pl.col("destination_insee") == pl.col("destination_detailed_zone")
|
|
230
|
+
)
|
|
231
|
+
.then(None)
|
|
232
|
+
.when("9" + pl.col("destination_insee") == pl.col("destination_detailed_zone"))
|
|
233
|
+
.then(None)
|
|
234
|
+
.otherwise("destination_detailed_zone"),
|
|
235
|
+
)
|
|
236
|
+
return lf
|
|
@@ -0,0 +1,286 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import re
|
|
3
|
+
from zipfile import ZipFile
|
|
4
|
+
|
|
5
|
+
import geopandas as gpd
|
|
6
|
+
|
|
7
|
+
from mobisurvstd.utils import find_file
|
|
8
|
+
|
|
9
|
+
from .survey import CeremaReader
|
|
10
|
+
from .zones import find_matching_column
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class EDGTReader(CeremaReader):
|
|
14
|
+
SURVEY_TYPE = "EDGT"
|
|
15
|
+
|
|
16
|
+
def households_filenames(self):
|
|
17
|
+
return get_files(self.source, "men")
|
|
18
|
+
|
|
19
|
+
def persons_filenames(self):
|
|
20
|
+
return get_files(self.source, "pers")
|
|
21
|
+
|
|
22
|
+
def trips_filenames(self):
|
|
23
|
+
return get_files(self.source, "depl")
|
|
24
|
+
|
|
25
|
+
def legs_filenames(self):
|
|
26
|
+
return get_files(self.source, "traj")
|
|
27
|
+
|
|
28
|
+
def special_locations_and_detailed_zones_filenames(self):
|
|
29
|
+
# This should match the Amiens 2010, Clermont-Ferrand 2012, and Lyon 2015 surveys.
|
|
30
|
+
subdir = os.path.join("Doc", "SIG")
|
|
31
|
+
faf_filename = find_file(
|
|
32
|
+
self.source, ".*_faf_d.?coupagefin[.](tab|shp|mif)", subdir=subdir, as_url=True
|
|
33
|
+
)
|
|
34
|
+
tel_filename = find_file(
|
|
35
|
+
self.source, ".*_tel_d.?coupagefin[.](tab|shp|mif)", subdir=subdir, as_url=True
|
|
36
|
+
)
|
|
37
|
+
if faf_filename is not None and tel_filename is not None:
|
|
38
|
+
# Amiens 2010 has two files.
|
|
39
|
+
return [faf_filename, tel_filename]
|
|
40
|
+
else:
|
|
41
|
+
return [
|
|
42
|
+
find_file(
|
|
43
|
+
self.source,
|
|
44
|
+
".*(_zf_gt|zinterne_gen_.*)[.](tab|shp|mif)",
|
|
45
|
+
subdir=subdir,
|
|
46
|
+
as_url=True,
|
|
47
|
+
)
|
|
48
|
+
]
|
|
49
|
+
|
|
50
|
+
def special_locations_filenames(self):
|
|
51
|
+
subdir = os.path.join("Doc", "SIG")
|
|
52
|
+
faf_filename = find_file(
|
|
53
|
+
self.source, ".*_faf_.*g.?n.?rateur.*[.](tab|shp|mif)", subdir=subdir, as_url=True
|
|
54
|
+
)
|
|
55
|
+
tel_filename = find_file(
|
|
56
|
+
self.source, ".*_tel_.*g.?n.?rateur.*[.](tab|shp|mif)", subdir=subdir, as_url=True
|
|
57
|
+
)
|
|
58
|
+
if faf_filename is not None and tel_filename is not None:
|
|
59
|
+
# Saint-Quentin-en-Yvelines has two files, for phone and face-to-face surveys.
|
|
60
|
+
return [faf_filename, tel_filename]
|
|
61
|
+
else:
|
|
62
|
+
return [
|
|
63
|
+
find_file(
|
|
64
|
+
self.source,
|
|
65
|
+
".*(_gt_.*|_gt|g.?n.?rateur.*)[.](tab|shp|mif)",
|
|
66
|
+
subdir=subdir,
|
|
67
|
+
as_url=True,
|
|
68
|
+
)
|
|
69
|
+
]
|
|
70
|
+
|
|
71
|
+
def detailed_zones_filenames(self):
|
|
72
|
+
subdir = os.path.join("Doc", "SIG")
|
|
73
|
+
faf_filename = find_file(
|
|
74
|
+
self.source, ".*_faf_zones_fines[.](tab|shp|mif)", subdir=subdir, as_url=True
|
|
75
|
+
)
|
|
76
|
+
tel_filename = find_file(
|
|
77
|
+
self.source, ".*_tel_zones_fines[.](tab|shp|mif)", subdir=subdir, as_url=True
|
|
78
|
+
)
|
|
79
|
+
if faf_filename is not None and tel_filename is not None:
|
|
80
|
+
# Nice 2009 and Saint-Quentin-en-Yvelines have two files, for phone and face-to-face
|
|
81
|
+
# surveys.
|
|
82
|
+
return [faf_filename, tel_filename]
|
|
83
|
+
else:
|
|
84
|
+
return [
|
|
85
|
+
find_file(
|
|
86
|
+
self.source,
|
|
87
|
+
".*(_zf|_zf_.*|zf08_.*|(?<!tel_)zones[_ ]fines)[.](tab|shp|mif)",
|
|
88
|
+
subdir=subdir,
|
|
89
|
+
as_url=True,
|
|
90
|
+
)
|
|
91
|
+
]
|
|
92
|
+
|
|
93
|
+
def draw_zones_filenames(self):
|
|
94
|
+
return [
|
|
95
|
+
find_file(
|
|
96
|
+
self.source,
|
|
97
|
+
".*(_dtir|_secteurstirage)[.](tab|shp|mif)",
|
|
98
|
+
subdir=os.path.join("Doc", "SIG"),
|
|
99
|
+
as_url=True,
|
|
100
|
+
)
|
|
101
|
+
]
|
|
102
|
+
|
|
103
|
+
def survey_name(self):
|
|
104
|
+
filename = find_file(self.source, ".*_std_faf_men.csv", subdir="Csv", as_url=True)
|
|
105
|
+
return re.match("(.*)_std_faf_men.csv", os.path.basename(filename)).group(1)
|
|
106
|
+
|
|
107
|
+
def gt_id_columns(self):
|
|
108
|
+
return [
|
|
109
|
+
"pôle_génér", # Thanks Saint-Quentin-en-Yvelines for this monstrosity.
|
|
110
|
+
"num_generateurs08",
|
|
111
|
+
"num_generateur08",
|
|
112
|
+
"num_gene2013",
|
|
113
|
+
"num_gene_2015",
|
|
114
|
+
"cod_pgt_2",
|
|
115
|
+
"zone_fine_gt",
|
|
116
|
+
"zf2015_nouveau_codage",
|
|
117
|
+
"code_gt",
|
|
118
|
+
"gt",
|
|
119
|
+
"zonefine",
|
|
120
|
+
"dfin",
|
|
121
|
+
"zf",
|
|
122
|
+
"num_zone_fine",
|
|
123
|
+
"num_zf",
|
|
124
|
+
"code",
|
|
125
|
+
]
|
|
126
|
+
|
|
127
|
+
def zf_id_from_gt_columns(self):
|
|
128
|
+
return ["zf_rattachement", "rattacheme", "num_zf_2015", "zone_fine", "z_fine", "zf"]
|
|
129
|
+
|
|
130
|
+
def gt_name_columns(self):
|
|
131
|
+
return [
|
|
132
|
+
"pôles_gén", # Thanks again Saint-Quentin-en-Yvelines for this monstrosity.
|
|
133
|
+
"nom_generateur",
|
|
134
|
+
"nom_gt",
|
|
135
|
+
"nom_gene",
|
|
136
|
+
"nom_zone_fine",
|
|
137
|
+
"nom_zonefine",
|
|
138
|
+
"nom_zf",
|
|
139
|
+
"nom",
|
|
140
|
+
]
|
|
141
|
+
|
|
142
|
+
def gt_type_columns(self):
|
|
143
|
+
return [
|
|
144
|
+
"typegd",
|
|
145
|
+
"nature_generateur",
|
|
146
|
+
"nom_type_gt",
|
|
147
|
+
"nom_type",
|
|
148
|
+
"catégorietxt",
|
|
149
|
+
"descriptif",
|
|
150
|
+
"nature",
|
|
151
|
+
"categorie",
|
|
152
|
+
"type_zonefine",
|
|
153
|
+
]
|
|
154
|
+
|
|
155
|
+
def zf_id_columns(self):
|
|
156
|
+
return [
|
|
157
|
+
"num_zf_2013",
|
|
158
|
+
"num_zf_2015",
|
|
159
|
+
"zf2015_nouveau_codage",
|
|
160
|
+
"num_zf08",
|
|
161
|
+
"num_zf08_2",
|
|
162
|
+
"id_zf_cerema",
|
|
163
|
+
"zonefine",
|
|
164
|
+
"zone_fine",
|
|
165
|
+
"zf",
|
|
166
|
+
"dfin",
|
|
167
|
+
"num_zf",
|
|
168
|
+
"code",
|
|
169
|
+
]
|
|
170
|
+
|
|
171
|
+
def zf_name_columns(self):
|
|
172
|
+
return ["zone_fin1", "nom_zonefine", "nom_zfine", "nom_zf", "nom"]
|
|
173
|
+
|
|
174
|
+
def dtir_id_columns(self):
|
|
175
|
+
return [
|
|
176
|
+
"bayonne_dtir", # Special case for Bayonne 2010 when reading the detailed zones.
|
|
177
|
+
"id_st",
|
|
178
|
+
"dtir_2016",
|
|
179
|
+
"dtir_s",
|
|
180
|
+
"num_dtir",
|
|
181
|
+
"num_tirage",
|
|
182
|
+
"num_secteur",
|
|
183
|
+
"num_secteurs",
|
|
184
|
+
"dtir",
|
|
185
|
+
"code",
|
|
186
|
+
]
|
|
187
|
+
|
|
188
|
+
def dtir_name_columns(self):
|
|
189
|
+
return ["nom_dtir", "nom_dtir_enq", "nom_secteur"]
|
|
190
|
+
|
|
191
|
+
def insee_id_columns(self):
|
|
192
|
+
return [
|
|
193
|
+
"insee_commune",
|
|
194
|
+
"code_insee",
|
|
195
|
+
"codinsee",
|
|
196
|
+
"depcom",
|
|
197
|
+
"insee",
|
|
198
|
+
"id_commune",
|
|
199
|
+
"com_2",
|
|
200
|
+
"codcomm",
|
|
201
|
+
]
|
|
202
|
+
|
|
203
|
+
def select_zf_column(self, gdf: gpd.GeoDataFrame):
|
|
204
|
+
# For Bayonne 2010 and Metz 2017, the ZF id can be computed by summing two columns.
|
|
205
|
+
pairs = (
|
|
206
|
+
("bayonne_dtir", "CODE_ZF_VALIDE"), # Bayonne 2010.
|
|
207
|
+
("Dtir", "Num_zfin"), # Metz 2017.
|
|
208
|
+
)
|
|
209
|
+
for sec_col, zf_num_col in pairs:
|
|
210
|
+
if sec_col in gdf.columns and zf_num_col in gdf.columns:
|
|
211
|
+
gdf["detailed_zone_id"] = gdf[sec_col].astype(str) + gdf[zf_num_col].astype(str)
|
|
212
|
+
break
|
|
213
|
+
# Normal case. Find the ZF id column by name.
|
|
214
|
+
super().select_zf_column(gdf)
|
|
215
|
+
|
|
216
|
+
def select_dtir_column(self, gdf: gpd.GeoDataFrame):
|
|
217
|
+
if "Z_fine" in gdf.columns and "NUM_DTIR" in gdf.columns:
|
|
218
|
+
# For Nantes 2015, there is 1 typo in NUM_DTIR so it is better to read the draw zone id
|
|
219
|
+
# from the first 3 characters of Z_fine instead.
|
|
220
|
+
gdf["draw_zone_id"] = gdf["Z_fine"].astype(str).str.slice(0, 3)
|
|
221
|
+
assert (gdf["draw_zone_id"].astype(int) != gdf["NUM_DTIR"].astype(int)).sum() <= 1
|
|
222
|
+
# This is the standard case.
|
|
223
|
+
super().select_dtir_column(gdf)
|
|
224
|
+
if "draw_zone_id" not in gdf.columns:
|
|
225
|
+
# For Nice 2009, Saint-Denis-de-la-Réunion 2016, and Saint-Quentin-en-Yvelines 2010,
|
|
226
|
+
# the draw_zone_id can be read from the first 3 characters of the GT column.
|
|
227
|
+
# For Angers 2012, it can be read from the first 3 characters of the ZF column.
|
|
228
|
+
for dtir_col in ("num_generateur08", "num_gene_2015", "pôle_génér", "code"):
|
|
229
|
+
if matching_col := find_matching_column(dtir_col, gdf):
|
|
230
|
+
gdf["draw_zone_id"] = gdf[matching_col].astype(str).str.slice(0, 3)
|
|
231
|
+
|
|
232
|
+
def select_zf_from_gt_column(self, gdf: gpd.GeoDataFrame):
|
|
233
|
+
# For Metz 2017, the ZF id can be computed by summing two columns.
|
|
234
|
+
if "Num_dtir" in gdf.columns and "Num_zfin" in gdf.columns:
|
|
235
|
+
gdf["detailed_zone_id"] = gdf["Num_dtir"].astype(str) + gdf["Num_zfin"].astype(str)
|
|
236
|
+
# Normal case. Find the ZF id column by name.
|
|
237
|
+
super().select_zf_from_gt_column(gdf)
|
|
238
|
+
|
|
239
|
+
def preprocess_special_locations(self, gdf: gpd.GeoDataFrame):
|
|
240
|
+
# Very very special case for Metz 2017, GT 003003 is actually 003301 (don't ask me why).
|
|
241
|
+
if "Code_gt" in gdf.columns and "Nom_gt" in gdf.columns:
|
|
242
|
+
match = gdf.loc[(gdf["Code_gt"] == "003003") & (gdf["Nom_gt"] == "Aldi")].index
|
|
243
|
+
if not match.empty:
|
|
244
|
+
assert len(match) == 1
|
|
245
|
+
gdf.loc[match, "Code_gt"] = "003301"
|
|
246
|
+
return gdf
|
|
247
|
+
|
|
248
|
+
def preprocess_detailed_zones(self, gdf: gpd.GeoDataFrame):
|
|
249
|
+
# For Bayonne 2010, column "CODE_ZONE_VALIDE" contains the draw zone id but there is a type:
|
|
250
|
+
# all values are "EMDxxx" where "xxx" is the draw zone id, except for one row where the
|
|
251
|
+
# value is "004007" while it should be "EMD004" (don't ask why).
|
|
252
|
+
if "CODE_ZONE_VALIDE" in gdf.columns:
|
|
253
|
+
gdf["bayonne_dtir"] = gdf["CODE_ZONE_VALIDE"].astype(str).str.slice(-3)
|
|
254
|
+
mask = gdf["CODE_ZONE_VALIDE"].astype(str).str.slice(0, 3) != "EMD"
|
|
255
|
+
assert mask.sum() <= 1
|
|
256
|
+
gdf.loc[mask, "bayonne_dtir"] = gdf["CODE_ZONE_VALIDE"].astype(str).str.slice(0, 3)
|
|
257
|
+
# For Montpellier 2014, the external zones are included in the ZF file. We remove them.
|
|
258
|
+
if "ID_ENQ" in gdf.columns:
|
|
259
|
+
gdf = gdf.loc[gdf["ID_ENQ"] == 1].copy()
|
|
260
|
+
if "ZONE_FINE" in gdf.columns and (gdf["ZONE_FINE"].str.len() == 7).all():
|
|
261
|
+
# For Saint-Quentin-en-Yvelines 2010, there is a leading zero that needs to be removed.
|
|
262
|
+
gdf["ZONE_FINE"] = gdf["ZONE_FINE"].str.slice(1)
|
|
263
|
+
return gdf
|
|
264
|
+
|
|
265
|
+
def postprocess_special_locations_and_detailed_zones(
|
|
266
|
+
self, zfs: gpd.GeoDataFrame | None, gts: gpd.GeoDataFrame | None
|
|
267
|
+
):
|
|
268
|
+
if gts is None:
|
|
269
|
+
return zfs, gts
|
|
270
|
+
# Special case for Amiens 2010, the insee_id (from column CODE_INSEE) is not valid for GTs.
|
|
271
|
+
if "CODE_INSEE" in gts.columns and (gts["insee_id"].str.len() > 5).all():
|
|
272
|
+
gts.drop(columns=["insee_id"], inplace=True)
|
|
273
|
+
return zfs, gts
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def get_files(source: str | ZipFile, name: str):
|
|
277
|
+
# In the EDGT surveys, there are two directories with "standardized" data in the Csv directory,
|
|
278
|
+
# usually "Fichiers_Standard_Face_a_face" and "Fichiers_Standard_Telephone" but it can be
|
|
279
|
+
# something else.
|
|
280
|
+
# Within these two directories, the CSV files to read are always
|
|
281
|
+
# "[SURVEY_NAME]_std_faf_[NAME].csv" and "[SURVEY_NAME]_std_tel_[NAME].csv" where NAME is either
|
|
282
|
+
# "men", "pers", "depl", or "traj".
|
|
283
|
+
return [
|
|
284
|
+
find_file(source, f".*_std_faf_{name}.csv", subdir="Csv"),
|
|
285
|
+
find_file(source, f".*_std_tel_{name}.csv", subdir="Csv"),
|
|
286
|
+
]
|