mobisurvstd 0.4.0__tar.gz → 0.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {mobisurvstd-0.4.0/src/mobisurvstd.egg-info → mobisurvstd-0.5.0}/PKG-INFO +1 -1
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/pyproject.toml +1 -1
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/edgt/survey.py +62 -55
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/edvm/survey.py +46 -39
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/egt2010/survey.py +10 -7
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/egt2010/trajets.py +1 -1
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/egt2020/survey.py +1 -1
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/emc2/survey.py +57 -48
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/emd/survey.py +61 -53
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/emp/survey.py +1 -1
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/main.py +42 -15
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0/src/mobisurvstd.egg-info}/PKG-INFO +1 -1
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/LICENSE.md +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/README.md +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/setup.cfg +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/__init__.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/__main__.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/classes.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/clean.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/cli.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/common/__init__.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/common/cars.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/common/clean.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/common/households.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/common/legs.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/common/modes.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/common/motorcycles.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/common/persons.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/common/trips.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/common/zones.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/edgt/__init__.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/edgt/common.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/edgt/deplacements.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/edgt/menages.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/edgt/personnes.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/edgt/trajets.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/edgt/zones.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/edvm/__init__.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/edvm/common.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/edvm/deplacements.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/edvm/menages.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/edvm/personnes.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/edvm/trajets.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/edvm/zones.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/egt2010/__init__.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/egt2010/deplacements.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/egt2010/menages.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/egt2010/personnes.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/egt2010/zones.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/egt2020/__init__.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/egt2020/deplacements.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/egt2020/menages.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/egt2020/motos.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/egt2020/personnes.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/egt2020/trajets.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/egt2020/voitures.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/emc2/__init__.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/emc2/common.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/emc2/deplacements.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/emc2/menages.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/emc2/personnes.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/emc2/trajets.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/emc2/zones.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/emd/__init__.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/emd/common.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/emd/deplacements.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/emd/menages.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/emd/personnes.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/emd/trajets.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/emd/zones.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/emp/__init__.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/emp/deplacements.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/emp/menages.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/emp/motos.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/emp/personnes.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/emp/voitures.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/logger.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/resources/__init__.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/resources/admin_express.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/resources/insee_data.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/resources/nuts.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/schema.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd/utils.py +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd.egg-info/SOURCES.txt +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd.egg-info/dependency_links.txt +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd.egg-info/requires.txt +0 -0
- {mobisurvstd-0.4.0 → mobisurvstd-0.5.0}/src/mobisurvstd.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: mobisurvstd
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.5.0
|
|
4
4
|
Summary: A tool to standardize French mobility survey datasets (EMC2, EGT, EMP, etc.) into a unified format.
|
|
5
5
|
Author-email: Lucas Javaudin <lucas@lucasjavaudin.com>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
|
|
|
6
6
|
|
|
7
7
|
[project]
|
|
8
8
|
name = "mobisurvstd"
|
|
9
|
-
version = "0.
|
|
9
|
+
version = "0.5.0"
|
|
10
10
|
description = "A tool to standardize French mobility survey datasets (EMC2, EGT, EMP, etc.) into a unified format."
|
|
11
11
|
readme = "README.md"
|
|
12
12
|
authors = [{ name = "Lucas Javaudin", email = "lucas@lucasjavaudin.com" }]
|
|
@@ -23,59 +23,15 @@ from .trajets import standardize_legs
|
|
|
23
23
|
from .zones import read_detailed_zones, read_draw_zones, read_special_locations
|
|
24
24
|
|
|
25
25
|
|
|
26
|
-
def standardize(source: str | ZipFile):
|
|
26
|
+
def standardize(source: str | ZipFile, skip_spatial: bool = False):
|
|
27
27
|
source_name = source.filename if isinstance(source, ZipFile) else source
|
|
28
28
|
logger.info(f"Standardizing EDGT survey from `{source_name}`")
|
|
29
|
-
|
|
30
|
-
filename = special_locations_filename(source)
|
|
31
|
-
if filename is None:
|
|
32
|
-
special_locations = None
|
|
33
|
-
elif "EDGT_SCOTAM_2017" in source_name or "Angers12" in source_name or "EDGTFVG2016":
|
|
34
|
-
# Note. Special case for Metz 2017 and Angers 2012: I cannot figure out the ids in the GT
|
|
35
|
-
# and ZF files.
|
|
36
|
-
# For Annemasse 2016, the GT id does not seem to be specified.
|
|
37
|
-
logger.warning("Cannot read special locations: ids are invalid")
|
|
29
|
+
if skip_spatial:
|
|
38
30
|
special_locations = None
|
|
39
|
-
else:
|
|
40
|
-
logger.debug(f"Reading special locations from `{filename}`")
|
|
41
|
-
special_locations = read_special_locations(filename)
|
|
42
|
-
# Detailed zones.
|
|
43
|
-
filename = detailed_zones_filename(source)
|
|
44
|
-
if filename is None:
|
|
45
|
-
logger.debug(f"No file with detailed zones in `{source_name}`")
|
|
46
|
-
detailed_zones = None
|
|
47
|
-
elif "EDGT_SCOTAM_2017" in source_name or "Angers12" in source_name or "FaF_Zones_fines":
|
|
48
|
-
# Note. Special case for Metz 2017 and Angers 2012: I cannot figure out the ids in the GT
|
|
49
|
-
# and ZF files.
|
|
50
|
-
# For Bayonne 2010, the ids are missing.
|
|
51
|
-
logger.warning("Cannot read detailed zones: ids are invalid")
|
|
52
31
|
detailed_zones = None
|
|
53
|
-
else:
|
|
54
|
-
logger.debug(f"Reading detailed zones from `{filename}`")
|
|
55
|
-
detailed_zones = read_detailed_zones(filename)
|
|
56
|
-
# Draw zones.
|
|
57
|
-
filename = draw_zones_filename(source)
|
|
58
|
-
if filename is None:
|
|
59
|
-
logger.debug(f"No file with draw zones in `{source_name}`")
|
|
60
32
|
draw_zones = None
|
|
61
33
|
else:
|
|
62
|
-
|
|
63
|
-
draw_zones = read_draw_zones(filename)
|
|
64
|
-
|
|
65
|
-
if special_locations is not None:
|
|
66
|
-
# For the Angers 2022 survey, the ids in the detailed zones shapefile have 3 zeros removed
|
|
67
|
-
# compared to the ids in the special locations shapefile.
|
|
68
|
-
assert detailed_zones is not None, (
|
|
69
|
-
"Special locations are defined but there is no data on detailed zones"
|
|
70
|
-
)
|
|
71
|
-
if "detailed_zone_id" not in special_locations.columns:
|
|
72
|
-
special_locations = identify_detailed_zone_id(special_locations, detailed_zones)
|
|
73
|
-
dz_gcd = np.gcd.reduce(detailed_zones["detailed_zone_id"].astype(int))
|
|
74
|
-
sl_gcd = np.gcd.reduce(special_locations["detailed_zone_id"].astype(int))
|
|
75
|
-
if dz_gcd < sl_gcd and np.log10(sl_gcd).is_integer():
|
|
76
|
-
detailed_zones["detailed_zone_id"] = (
|
|
77
|
-
detailed_zones["detailed_zone_id"].astype(int) * sl_gcd
|
|
78
|
-
).astype(str)
|
|
34
|
+
special_locations, detailed_zones, draw_zones = read_spatial_data(source, source_name)
|
|
79
35
|
|
|
80
36
|
if special_locations is not None:
|
|
81
37
|
special_locations_coords = get_coords(special_locations, "special_location")
|
|
@@ -180,6 +136,61 @@ def standardize(source: str | ZipFile):
|
|
|
180
136
|
)
|
|
181
137
|
|
|
182
138
|
|
|
139
|
+
def read_spatial_data(source: str | ZipFile, source_name: str):
|
|
140
|
+
# Special locations.
|
|
141
|
+
filename = special_locations_filename(source)
|
|
142
|
+
if filename is None:
|
|
143
|
+
special_locations = None
|
|
144
|
+
elif "EDGT_SCOTAM_2017" in source_name or "Angers12" in source_name or "EDGTFVG2016":
|
|
145
|
+
# Note. Special case for Metz 2017 and Angers 2012: I cannot figure out the ids in the GT
|
|
146
|
+
# and ZF files.
|
|
147
|
+
# For Annemasse 2016, the GT id does not seem to be specified.
|
|
148
|
+
logger.warning("Cannot read special locations: ids are invalid")
|
|
149
|
+
special_locations = None
|
|
150
|
+
else:
|
|
151
|
+
logger.debug(f"Reading special locations from `{filename}`")
|
|
152
|
+
special_locations = read_special_locations(filename)
|
|
153
|
+
# Detailed zones.
|
|
154
|
+
filename = detailed_zones_filename(source)
|
|
155
|
+
if filename is None:
|
|
156
|
+
logger.debug(f"No file with detailed zones in `{source_name}`")
|
|
157
|
+
detailed_zones = None
|
|
158
|
+
elif "EDGT_SCOTAM_2017" in source_name or "Angers12" in source_name or "FaF_Zones_fines":
|
|
159
|
+
# Note. Special case for Metz 2017 and Angers 2012: I cannot figure out the ids in the GT
|
|
160
|
+
# and ZF files.
|
|
161
|
+
# For Bayonne 2010, the ids are missing.
|
|
162
|
+
logger.warning("Cannot read detailed zones: ids are invalid")
|
|
163
|
+
detailed_zones = None
|
|
164
|
+
else:
|
|
165
|
+
logger.debug(f"Reading detailed zones from `{filename}`")
|
|
166
|
+
detailed_zones = read_detailed_zones(filename)
|
|
167
|
+
# Draw zones.
|
|
168
|
+
filename = draw_zones_filename(source)
|
|
169
|
+
if filename is None:
|
|
170
|
+
logger.debug(f"No file with draw zones in `{source_name}`")
|
|
171
|
+
draw_zones = None
|
|
172
|
+
else:
|
|
173
|
+
logger.debug(f"Reading draw zones from `{filename}`")
|
|
174
|
+
draw_zones = read_draw_zones(filename)
|
|
175
|
+
|
|
176
|
+
if special_locations is not None:
|
|
177
|
+
# For the Angers 2022 survey, the ids in the detailed zones shapefile have 3 zeros removed
|
|
178
|
+
# compared to the ids in the special locations shapefile.
|
|
179
|
+
assert (
|
|
180
|
+
detailed_zones is not None
|
|
181
|
+
), "Special locations are defined but there is no data on detailed zones"
|
|
182
|
+
if "detailed_zone_id" not in special_locations.columns:
|
|
183
|
+
special_locations = identify_detailed_zone_id(special_locations, detailed_zones)
|
|
184
|
+
dz_gcd = np.gcd.reduce(detailed_zones["detailed_zone_id"].astype(int))
|
|
185
|
+
sl_gcd = np.gcd.reduce(special_locations["detailed_zone_id"].astype(int))
|
|
186
|
+
if dz_gcd < sl_gcd and np.log10(sl_gcd).is_integer():
|
|
187
|
+
detailed_zones["detailed_zone_id"] = (
|
|
188
|
+
detailed_zones["detailed_zone_id"].astype(int) * sl_gcd
|
|
189
|
+
).astype(str)
|
|
190
|
+
|
|
191
|
+
return special_locations, detailed_zones, draw_zones
|
|
192
|
+
|
|
193
|
+
|
|
183
194
|
def survey_main_insee(source: str | bytes):
|
|
184
195
|
return pl.scan_csv(source, separator=";").select("IDM4").first().collect().item()
|
|
185
196
|
|
|
@@ -267,15 +278,13 @@ def fix_special_locations_with_gcd(
|
|
|
267
278
|
pl.from_pandas(special_locations["detailed_zone_id"].astype(int)),
|
|
268
279
|
default=None,
|
|
269
280
|
)
|
|
270
|
-
)
|
|
271
|
-
.alias(f"{col}_detailed_zone"),
|
|
281
|
+
).alias(f"{col}_detailed_zone"),
|
|
272
282
|
# When the ZF is an actual ZF
|
|
273
283
|
pl.when(mask)
|
|
274
284
|
# Then the GT is null
|
|
275
285
|
.then(pl.lit(None))
|
|
276
286
|
# Otherwise use that GT as GT.
|
|
277
|
-
.otherwise(f"{col}_detailed_zone")
|
|
278
|
-
.alias(f"{col}_special_location"),
|
|
287
|
+
.otherwise(f"{col}_detailed_zone").alias(f"{col}_special_location"),
|
|
279
288
|
)
|
|
280
289
|
return df
|
|
281
290
|
|
|
@@ -298,15 +307,13 @@ def fix_special_locations(df: pl.LazyFrame, col: str, special_locations: gpd.Geo
|
|
|
298
307
|
pl.from_pandas(special_locations["detailed_zone_id"].astype(int)),
|
|
299
308
|
default=None,
|
|
300
309
|
)
|
|
301
|
-
)
|
|
302
|
-
.alias(f"{col}_detailed_zone"),
|
|
310
|
+
).alias(f"{col}_detailed_zone"),
|
|
303
311
|
# When the ZF is an actual ZF
|
|
304
312
|
pl.when(mask)
|
|
305
313
|
# Then the GT is null
|
|
306
314
|
.then(pl.lit(None))
|
|
307
315
|
# Otherwise use that GT as GT.
|
|
308
|
-
.otherwise(f"{col}_detailed_zone")
|
|
309
|
-
.alias(f"{col}_special_location"),
|
|
316
|
+
.otherwise(f"{col}_detailed_zone").alias(f"{col}_special_location"),
|
|
310
317
|
)
|
|
311
318
|
return df
|
|
312
319
|
|
|
@@ -28,40 +28,15 @@ from .zones import (
|
|
|
28
28
|
)
|
|
29
29
|
|
|
30
30
|
|
|
31
|
-
def standardize(source: str | ZipFile):
|
|
31
|
+
def standardize(source: str | ZipFile, skip_spatial: bool = False):
|
|
32
32
|
source_name = source.filename if isinstance(source, ZipFile) else source
|
|
33
33
|
logger.info(f"Standardizing EDVM survey from `{source_name}`")
|
|
34
|
-
if
|
|
35
|
-
|
|
36
|
-
detailed_zones
|
|
37
|
-
else:
|
|
38
|
-
# Special locations.
|
|
39
|
-
if filename := special_locations_filename(source):
|
|
40
|
-
logger.debug(f"Reading special locations from `{filename}`")
|
|
41
|
-
special_locations = read_special_locations(filename)
|
|
42
|
-
else:
|
|
43
|
-
special_locations = None
|
|
44
|
-
# Detailed zones.
|
|
45
|
-
if filename := detailed_zones_filename(source):
|
|
46
|
-
logger.debug(f"Reading detailed zones from `{filename}`")
|
|
47
|
-
detailed_zones = read_detailed_zones(filename)
|
|
48
|
-
else:
|
|
49
|
-
logger.debug(f"No file with detailed zones in `{source_name}`")
|
|
50
|
-
detailed_zones = None
|
|
51
|
-
# Draw zones.
|
|
52
|
-
if filename := draw_zones_filename(source):
|
|
53
|
-
logger.debug(f"Reading draw zones from `{filename}`")
|
|
54
|
-
draw_zones = read_draw_zones(filename)
|
|
55
|
-
else:
|
|
56
|
-
logger.debug(f"No file with draw zones in `{source_name}`")
|
|
34
|
+
if skip_spatial:
|
|
35
|
+
special_locations = None
|
|
36
|
+
detailed_zones = None
|
|
57
37
|
draw_zones = None
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
assert detailed_zones is not None, (
|
|
61
|
-
"Special locations are defined but there is no data on detailed zones"
|
|
62
|
-
)
|
|
63
|
-
if "detailed_zone_id" not in special_locations.columns:
|
|
64
|
-
special_locations = identify_detailed_zone_id(special_locations, detailed_zones)
|
|
38
|
+
else:
|
|
39
|
+
special_locations, detailed_zones, draw_zones = read_spatial_data(source, source_name)
|
|
65
40
|
|
|
66
41
|
if special_locations is not None:
|
|
67
42
|
special_locations_coords = get_coords(special_locations, "special_location")
|
|
@@ -133,6 +108,42 @@ def standardize(source: str | ZipFile):
|
|
|
133
108
|
)
|
|
134
109
|
|
|
135
110
|
|
|
111
|
+
def read_spatial_data(source: str | ZipFile, source_name: str):
|
|
112
|
+
if filename := special_locations_and_detailed_zones_filename(source):
|
|
113
|
+
# Special case for Beauvais 2011.
|
|
114
|
+
detailed_zones, special_locations = read_special_locations_and_detailed_zones(filename)
|
|
115
|
+
else:
|
|
116
|
+
# Special locations.
|
|
117
|
+
if filename := special_locations_filename(source):
|
|
118
|
+
logger.debug(f"Reading special locations from `{filename}`")
|
|
119
|
+
special_locations = read_special_locations(filename)
|
|
120
|
+
else:
|
|
121
|
+
special_locations = None
|
|
122
|
+
# Detailed zones.
|
|
123
|
+
if filename := detailed_zones_filename(source):
|
|
124
|
+
logger.debug(f"Reading detailed zones from `{filename}`")
|
|
125
|
+
detailed_zones = read_detailed_zones(filename)
|
|
126
|
+
else:
|
|
127
|
+
logger.debug(f"No file with detailed zones in `{source_name}`")
|
|
128
|
+
detailed_zones = None
|
|
129
|
+
# Draw zones.
|
|
130
|
+
if filename := draw_zones_filename(source):
|
|
131
|
+
logger.debug(f"Reading draw zones from `{filename}`")
|
|
132
|
+
draw_zones = read_draw_zones(filename)
|
|
133
|
+
else:
|
|
134
|
+
logger.debug(f"No file with draw zones in `{source_name}`")
|
|
135
|
+
draw_zones = None
|
|
136
|
+
|
|
137
|
+
if special_locations is not None:
|
|
138
|
+
assert (
|
|
139
|
+
detailed_zones is not None
|
|
140
|
+
), "Special locations are defined but there is no data on detailed zones"
|
|
141
|
+
if "detailed_zone_id" not in special_locations.columns:
|
|
142
|
+
special_locations = identify_detailed_zone_id(special_locations, detailed_zones)
|
|
143
|
+
|
|
144
|
+
return special_locations, detailed_zones, draw_zones
|
|
145
|
+
|
|
146
|
+
|
|
136
147
|
def survey_name(source: str | ZipFile):
|
|
137
148
|
filename = find_file(source, ".*_std_men.csv", subdir="Csv", as_url=True)
|
|
138
149
|
return re.match("(.*)_std_men.csv", os.path.basename(filename)).group(1)
|
|
@@ -212,15 +223,13 @@ def fix_special_locations_with_gcd(
|
|
|
212
223
|
pl.from_pandas(special_locations["detailed_zone_id"].astype(int)),
|
|
213
224
|
default=None,
|
|
214
225
|
)
|
|
215
|
-
)
|
|
216
|
-
.alias(f"{col}_detailed_zone"),
|
|
226
|
+
).alias(f"{col}_detailed_zone"),
|
|
217
227
|
# When the ZF is an actual ZF
|
|
218
228
|
pl.when(mask)
|
|
219
229
|
# Then the GT is null
|
|
220
230
|
.then(pl.lit(None))
|
|
221
231
|
# Otherwise use that GT as GT.
|
|
222
|
-
.otherwise(f"{col}_detailed_zone")
|
|
223
|
-
.alias(f"{col}_special_location"),
|
|
232
|
+
.otherwise(f"{col}_detailed_zone").alias(f"{col}_special_location"),
|
|
224
233
|
)
|
|
225
234
|
return df
|
|
226
235
|
|
|
@@ -243,15 +252,13 @@ def fix_special_locations(df: pl.LazyFrame, col: str, special_locations: gpd.Geo
|
|
|
243
252
|
pl.from_pandas(special_locations["detailed_zone_id"].astype(int)),
|
|
244
253
|
default=None,
|
|
245
254
|
)
|
|
246
|
-
)
|
|
247
|
-
.alias(f"{col}_detailed_zone"),
|
|
255
|
+
).alias(f"{col}_detailed_zone"),
|
|
248
256
|
# When the ZF is an actual ZF
|
|
249
257
|
pl.when(mask)
|
|
250
258
|
# Then the GT is null
|
|
251
259
|
.then(pl.lit(None))
|
|
252
260
|
# Otherwise use that GT as GT.
|
|
253
|
-
.otherwise(f"{col}_detailed_zone")
|
|
254
|
-
.alias(f"{col}_special_location"),
|
|
261
|
+
.otherwise(f"{col}_detailed_zone").alias(f"{col}_special_location"),
|
|
255
262
|
)
|
|
256
263
|
return df
|
|
257
264
|
|
|
@@ -13,15 +13,18 @@ from .trajets import standardize_legs
|
|
|
13
13
|
from .zones import read_detailed_zones
|
|
14
14
|
|
|
15
15
|
|
|
16
|
-
def standardize(source: str | ZipFile):
|
|
16
|
+
def standardize(source: str | ZipFile, skip_spatial: bool = False):
|
|
17
17
|
source_name = source.filename if isinstance(source, ZipFile) else source
|
|
18
18
|
logger.info(f"Standardizing EGT2010 survey from `{source_name}`")
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
19
|
+
if skip_spatial:
|
|
20
|
+
detailed_zones = None
|
|
21
|
+
else:
|
|
22
|
+
# Detailed zones.
|
|
23
|
+
filename = detailed_zones_filename(source)
|
|
24
|
+
if filename is None:
|
|
25
|
+
logger.error("Missing detailed zones file")
|
|
26
|
+
return None
|
|
27
|
+
detailed_zones = read_detailed_zones(filename)
|
|
25
28
|
# Households.
|
|
26
29
|
filename = households_filename(source)
|
|
27
30
|
if filename is None:
|
|
@@ -155,7 +155,7 @@ def standardize_legs(
|
|
|
155
155
|
trips: pl.LazyFrame,
|
|
156
156
|
cars: pl.LazyFrame,
|
|
157
157
|
motorcycles: pl.LazyFrame,
|
|
158
|
-
detailed_zones: gpd.GeoDataFrame,
|
|
158
|
+
detailed_zones: gpd.GeoDataFrame | None,
|
|
159
159
|
):
|
|
160
160
|
lf = scan_legs(filename)
|
|
161
161
|
# Add household_id, person_id, and trip_id.
|
|
@@ -13,7 +13,7 @@ from .trajets import standardize_legs
|
|
|
13
13
|
from .voitures import standardize_cars
|
|
14
14
|
|
|
15
15
|
|
|
16
|
-
def standardize(source: str | ZipFile):
|
|
16
|
+
def standardize(source: str | ZipFile, skip_spatial: bool = False):
|
|
17
17
|
source_name = source.filename if isinstance(source, ZipFile) else source
|
|
18
18
|
logger.info(f"Standardizing EGT2020 survey from `{source_name}`")
|
|
19
19
|
# Households.
|
|
@@ -24,57 +24,24 @@ from .trajets import standardize_legs
|
|
|
24
24
|
from .zones import read_detailed_zones, read_draw_zones, read_special_locations
|
|
25
25
|
|
|
26
26
|
|
|
27
|
-
def standardize(source: str | ZipFile):
|
|
27
|
+
def standardize(source: str | ZipFile, skip_spatial: bool = False):
|
|
28
28
|
source_name = source.filename if isinstance(source, ZipFile) else source
|
|
29
29
|
logger.info(f"Standardizing EMC2 survey from `{source_name}`")
|
|
30
|
-
|
|
31
|
-
filename = special_locations_filename(source)
|
|
32
|
-
if filename is None:
|
|
30
|
+
if skip_spatial:
|
|
33
31
|
special_locations = None
|
|
34
|
-
special_locations_coords = None
|
|
35
|
-
else:
|
|
36
|
-
logger.debug(f"Reading special locations from `{filename}`")
|
|
37
|
-
special_locations = read_special_locations(filename)
|
|
38
|
-
special_locations_coords = get_coords(special_locations, "special_location")
|
|
39
|
-
# Detailed zones.
|
|
40
|
-
filename = detailed_zones_filename(source)
|
|
41
|
-
if filename is None:
|
|
42
|
-
logger.debug(f"No file with detailed zones in `{source_name}`")
|
|
43
32
|
detailed_zones = None
|
|
44
|
-
detailed_zones_coords = None
|
|
45
|
-
else:
|
|
46
|
-
logger.debug(f"Reading detailed zones from `{filename}`")
|
|
47
|
-
detailed_zones = read_detailed_zones(filename)
|
|
48
|
-
detailed_zones_coords = get_coords(detailed_zones, "detailed_zone")
|
|
49
|
-
# Draw zones.
|
|
50
|
-
filename = draw_zones_filename(source)
|
|
51
|
-
if filename is None:
|
|
52
|
-
logger.debug(f"No file with draw zones in `{source_name}`")
|
|
53
33
|
draw_zones = None
|
|
54
34
|
else:
|
|
55
|
-
|
|
56
|
-
draw_zones = read_draw_zones(filename)
|
|
57
|
-
|
|
58
|
-
if special_locations is not None:
|
|
59
|
-
# For the Angers 2022 survey, the ids in the detailed zones shapefile have 3 zeros removed
|
|
60
|
-
# compared to the # ids in the special locations shapefile.
|
|
61
|
-
assert detailed_zones is not None, (
|
|
62
|
-
"Special locations are defined but there is no data on detailed zones"
|
|
63
|
-
)
|
|
64
|
-
assert "detailed_zone_id" in special_locations.columns, (
|
|
65
|
-
"Special locations are defined and used but the corresponding detailed zone are unknown"
|
|
66
|
-
)
|
|
67
|
-
dz_gcd = np.gcd.reduce(detailed_zones["detailed_zone_id"].astype(int))
|
|
68
|
-
sl_gcd = np.gcd.reduce(special_locations["detailed_zone_id"].astype(int))
|
|
69
|
-
if dz_gcd < sl_gcd and np.log10(sl_gcd).is_integer():
|
|
70
|
-
detailed_zones["detailed_zone_id"] = (
|
|
71
|
-
detailed_zones["detailed_zone_id"].astype(int) * sl_gcd
|
|
72
|
-
).astype(str)
|
|
35
|
+
special_locations, detailed_zones, draw_zones = read_spatial_data(source, source_name)
|
|
73
36
|
|
|
74
37
|
if special_locations is not None:
|
|
75
38
|
special_locations_coords = get_coords(special_locations, "special_location")
|
|
39
|
+
else:
|
|
40
|
+
special_locations_coords = None
|
|
76
41
|
if detailed_zones is not None:
|
|
77
42
|
detailed_zones_coords = get_coords(detailed_zones, "detailed_zone")
|
|
43
|
+
else:
|
|
44
|
+
detailed_zones_coords = None
|
|
78
45
|
|
|
79
46
|
# Households.
|
|
80
47
|
filename = households_filename(source)
|
|
@@ -112,6 +79,7 @@ def standardize(source: str | ZipFile):
|
|
|
112
79
|
|
|
113
80
|
# Fix the special locations which are being used as detailed zones.
|
|
114
81
|
if special_locations is not None:
|
|
82
|
+
assert detailed_zones is not None
|
|
115
83
|
assert (
|
|
116
84
|
len(
|
|
117
85
|
set(special_locations["special_location_id"]).intersection(
|
|
@@ -126,6 +94,7 @@ def standardize(source: str | ZipFile):
|
|
|
126
94
|
# In some cases, not all GTs are defined.
|
|
127
95
|
# For example, `origin_detailed_zone` can be defined as 70103 but there is no special
|
|
128
96
|
# location with that id. In this case, we can guess that the ZF id is 70100.
|
|
97
|
+
sl_gcd = np.gcd.reduce(special_locations["detailed_zone_id"].astype(int))
|
|
129
98
|
if np.log10(sl_gcd).is_integer() and sl_gcd > 1:
|
|
130
99
|
# All ZF ids are ending with zeros. In principle, all GT ids do _not_ end with zeros.
|
|
131
100
|
# We raise an error if at least 2 GT ids end with zeros.
|
|
@@ -170,6 +139,50 @@ def standardize(source: str | ZipFile):
|
|
|
170
139
|
)
|
|
171
140
|
|
|
172
141
|
|
|
142
|
+
def read_spatial_data(source: str | ZipFile, source_name: str):
|
|
143
|
+
# Special locations.
|
|
144
|
+
filename = special_locations_filename(source)
|
|
145
|
+
if filename is None:
|
|
146
|
+
special_locations = None
|
|
147
|
+
else:
|
|
148
|
+
logger.debug(f"Reading special locations from `{filename}`")
|
|
149
|
+
special_locations = read_special_locations(filename)
|
|
150
|
+
# Detailed zones.
|
|
151
|
+
filename = detailed_zones_filename(source)
|
|
152
|
+
if filename is None:
|
|
153
|
+
logger.debug(f"No file with detailed zones in `{source_name}`")
|
|
154
|
+
detailed_zones = None
|
|
155
|
+
else:
|
|
156
|
+
logger.debug(f"Reading detailed zones from `{filename}`")
|
|
157
|
+
detailed_zones = read_detailed_zones(filename)
|
|
158
|
+
# Draw zones.
|
|
159
|
+
filename = draw_zones_filename(source)
|
|
160
|
+
if filename is None:
|
|
161
|
+
logger.debug(f"No file with draw zones in `{source_name}`")
|
|
162
|
+
draw_zones = None
|
|
163
|
+
else:
|
|
164
|
+
logger.debug(f"Reading draw zones from `{filename}`")
|
|
165
|
+
draw_zones = read_draw_zones(filename)
|
|
166
|
+
|
|
167
|
+
if special_locations is not None:
|
|
168
|
+
# For the Angers 2022 survey, the ids in the detailed zones shapefile have 3 zeros removed
|
|
169
|
+
# compared to the # ids in the special locations shapefile.
|
|
170
|
+
assert (
|
|
171
|
+
detailed_zones is not None
|
|
172
|
+
), "Special locations are defined but there is no data on detailed zones"
|
|
173
|
+
assert (
|
|
174
|
+
"detailed_zone_id" in special_locations.columns
|
|
175
|
+
), "Special locations are defined and used but the corresponding detailed zone are unknown"
|
|
176
|
+
dz_gcd = np.gcd.reduce(detailed_zones["detailed_zone_id"].astype(int))
|
|
177
|
+
sl_gcd = np.gcd.reduce(special_locations["detailed_zone_id"].astype(int))
|
|
178
|
+
if dz_gcd < sl_gcd and np.log10(sl_gcd).is_integer():
|
|
179
|
+
detailed_zones["detailed_zone_id"] = (
|
|
180
|
+
detailed_zones["detailed_zone_id"].astype(int) * sl_gcd
|
|
181
|
+
).astype(str)
|
|
182
|
+
|
|
183
|
+
return special_locations, detailed_zones, draw_zones
|
|
184
|
+
|
|
185
|
+
|
|
173
186
|
def survey_name(source: str | ZipFile):
|
|
174
187
|
filename = find_file(
|
|
175
188
|
source, ".*_std_men.csv", subdir=os.path.join("Csv", "Fichiers_Standard"), as_url=True
|
|
@@ -238,15 +251,13 @@ def fix_special_locations_with_gcd(
|
|
|
238
251
|
pl.from_pandas(special_locations["detailed_zone_id"].astype(int)),
|
|
239
252
|
default=None,
|
|
240
253
|
)
|
|
241
|
-
)
|
|
242
|
-
.alias(f"{col}_detailed_zone"),
|
|
254
|
+
).alias(f"{col}_detailed_zone"),
|
|
243
255
|
# When the ZF is an actual ZF
|
|
244
256
|
pl.when(mask)
|
|
245
257
|
# Then the GT is null
|
|
246
258
|
.then(pl.lit(None))
|
|
247
259
|
# Otherwise use that GT as GT.
|
|
248
|
-
.otherwise(f"{col}_detailed_zone")
|
|
249
|
-
.alias(f"{col}_special_location"),
|
|
260
|
+
.otherwise(f"{col}_detailed_zone").alias(f"{col}_special_location"),
|
|
250
261
|
)
|
|
251
262
|
return df
|
|
252
263
|
|
|
@@ -269,14 +280,12 @@ def fix_special_locations(df: pl.LazyFrame, col: str, special_locations: gpd.Geo
|
|
|
269
280
|
pl.from_pandas(special_locations["detailed_zone_id"].astype(int)),
|
|
270
281
|
default=None,
|
|
271
282
|
)
|
|
272
|
-
)
|
|
273
|
-
.alias(f"{col}_detailed_zone"),
|
|
283
|
+
).alias(f"{col}_detailed_zone"),
|
|
274
284
|
# When the ZF is an actual ZF
|
|
275
285
|
pl.when(mask)
|
|
276
286
|
# Then the GT is null
|
|
277
287
|
.then(pl.lit(None))
|
|
278
288
|
# Otherwise use that GT as GT.
|
|
279
|
-
.otherwise(f"{col}_detailed_zone")
|
|
280
|
-
.alias(f"{col}_special_location"),
|
|
289
|
+
.otherwise(f"{col}_detailed_zone").alias(f"{col}_special_location"),
|
|
281
290
|
)
|
|
282
291
|
return df
|
|
@@ -28,40 +28,15 @@ from .zones import (
|
|
|
28
28
|
)
|
|
29
29
|
|
|
30
30
|
|
|
31
|
-
def standardize(source: str | ZipFile):
|
|
31
|
+
def standardize(source: str | ZipFile, skip_spatial: bool = False):
|
|
32
32
|
source_name = source.filename if isinstance(source, ZipFile) else source
|
|
33
33
|
logger.info(f"Standardizing EDVM survey from `{source_name}`")
|
|
34
|
-
if
|
|
35
|
-
|
|
36
|
-
detailed_zones
|
|
37
|
-
else:
|
|
38
|
-
# Special locations.
|
|
39
|
-
if filename := special_locations_filename(source):
|
|
40
|
-
logger.debug(f"Reading special locations from `{filename}`")
|
|
41
|
-
special_locations = read_special_locations(filename)
|
|
42
|
-
else:
|
|
43
|
-
special_locations = None
|
|
44
|
-
# Detailed zones.
|
|
45
|
-
if filename := detailed_zones_filename(source):
|
|
46
|
-
logger.debug(f"Reading detailed zones from `{filename}`")
|
|
47
|
-
detailed_zones = read_detailed_zones(filename)
|
|
48
|
-
else:
|
|
49
|
-
logger.debug(f"No file with detailed zones in `{source_name}`")
|
|
50
|
-
detailed_zones = None
|
|
51
|
-
# Draw zones.
|
|
52
|
-
if filename := draw_zones_filename(source):
|
|
53
|
-
logger.debug(f"Reading draw zones from `{filename}`")
|
|
54
|
-
draw_zones = read_draw_zones(filename)
|
|
55
|
-
else:
|
|
56
|
-
logger.debug(f"No file with draw zones in `{source_name}`")
|
|
34
|
+
if skip_spatial:
|
|
35
|
+
special_locations = None
|
|
36
|
+
detailed_zones = None
|
|
57
37
|
draw_zones = None
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
assert detailed_zones is not None, (
|
|
61
|
-
"Special locations are defined but there is no data on detailed zones"
|
|
62
|
-
)
|
|
63
|
-
if "detailed_zone_id" not in special_locations.columns:
|
|
64
|
-
special_locations = identify_detailed_zone_id(special_locations, detailed_zones)
|
|
38
|
+
else:
|
|
39
|
+
special_locations, detailed_zones, draw_zones = read_spatial_data(source, source_name)
|
|
65
40
|
|
|
66
41
|
if special_locations is not None:
|
|
67
42
|
special_locations_coords = get_coords(special_locations, "special_location")
|
|
@@ -106,9 +81,59 @@ def standardize(source: str | ZipFile):
|
|
|
106
81
|
|
|
107
82
|
households = add_survey_dates(households, persons)
|
|
108
83
|
|
|
84
|
+
return clean(
|
|
85
|
+
households=households,
|
|
86
|
+
persons=persons,
|
|
87
|
+
trips=trips,
|
|
88
|
+
legs=legs,
|
|
89
|
+
cars=cars,
|
|
90
|
+
motorcycles=motorcycles,
|
|
91
|
+
special_locations=special_locations,
|
|
92
|
+
detailed_zones=detailed_zones,
|
|
93
|
+
draw_zones=draw_zones,
|
|
94
|
+
survey_type="EMD",
|
|
95
|
+
survey_name=survey_name(source),
|
|
96
|
+
main_insee=main_insee,
|
|
97
|
+
)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def read_spatial_data(source: str | ZipFile, source_name: str):
|
|
101
|
+
if filename := special_locations_and_detailed_zones_filename(source):
|
|
102
|
+
# Special case for Beauvais 2011.
|
|
103
|
+
detailed_zones, special_locations = read_special_locations_and_detailed_zones(filename)
|
|
104
|
+
else:
|
|
105
|
+
# Special locations.
|
|
106
|
+
if filename := special_locations_filename(source):
|
|
107
|
+
logger.debug(f"Reading special locations from `{filename}`")
|
|
108
|
+
special_locations = read_special_locations(filename)
|
|
109
|
+
else:
|
|
110
|
+
special_locations = None
|
|
111
|
+
# Detailed zones.
|
|
112
|
+
if filename := detailed_zones_filename(source):
|
|
113
|
+
logger.debug(f"Reading detailed zones from `{filename}`")
|
|
114
|
+
detailed_zones = read_detailed_zones(filename)
|
|
115
|
+
else:
|
|
116
|
+
logger.debug(f"No file with detailed zones in `{source_name}`")
|
|
117
|
+
detailed_zones = None
|
|
118
|
+
# Draw zones.
|
|
119
|
+
if filename := draw_zones_filename(source):
|
|
120
|
+
logger.debug(f"Reading draw zones from `{filename}`")
|
|
121
|
+
draw_zones = read_draw_zones(filename)
|
|
122
|
+
else:
|
|
123
|
+
logger.debug(f"No file with draw zones in `{source_name}`")
|
|
124
|
+
draw_zones = None
|
|
125
|
+
|
|
126
|
+
if special_locations is not None:
|
|
127
|
+
assert (
|
|
128
|
+
detailed_zones is not None
|
|
129
|
+
), "Special locations are defined but there is no data on detailed zones"
|
|
130
|
+
if "detailed_zone_id" not in special_locations.columns:
|
|
131
|
+
special_locations = identify_detailed_zone_id(special_locations, detailed_zones)
|
|
132
|
+
|
|
109
133
|
if special_locations is not None:
|
|
110
134
|
# For Lille 2015, some GT ids also have corresponding zones. We remove these zones to
|
|
111
135
|
# prevent any issue.
|
|
136
|
+
assert detailed_zones is not None
|
|
112
137
|
detailed_zones = detailed_zones.loc[
|
|
113
138
|
~detailed_zones["detailed_zone_id"].isin(special_locations["special_location_id"])
|
|
114
139
|
]
|
|
@@ -121,20 +146,7 @@ def standardize(source: str | ZipFile):
|
|
|
121
146
|
== 0
|
|
122
147
|
), "Special locations and detailed zones have common ids"
|
|
123
148
|
|
|
124
|
-
return
|
|
125
|
-
households=households,
|
|
126
|
-
persons=persons,
|
|
127
|
-
trips=trips,
|
|
128
|
-
legs=legs,
|
|
129
|
-
cars=cars,
|
|
130
|
-
motorcycles=motorcycles,
|
|
131
|
-
special_locations=special_locations,
|
|
132
|
-
detailed_zones=detailed_zones,
|
|
133
|
-
draw_zones=draw_zones,
|
|
134
|
-
survey_type="EMD",
|
|
135
|
-
survey_name=survey_name(source),
|
|
136
|
-
main_insee=main_insee,
|
|
137
|
-
)
|
|
149
|
+
return special_locations, detailed_zones, draw_zones
|
|
138
150
|
|
|
139
151
|
|
|
140
152
|
def survey_name(source: str | ZipFile):
|
|
@@ -214,15 +226,13 @@ def fix_special_locations_with_gcd(
|
|
|
214
226
|
pl.from_pandas(special_locations["detailed_zone_id"].astype(int)),
|
|
215
227
|
default=None,
|
|
216
228
|
)
|
|
217
|
-
)
|
|
218
|
-
.alias(f"{col}_detailed_zone"),
|
|
229
|
+
).alias(f"{col}_detailed_zone"),
|
|
219
230
|
# When the ZF is an actual ZF
|
|
220
231
|
pl.when(mask)
|
|
221
232
|
# Then the GT is null
|
|
222
233
|
.then(pl.lit(None))
|
|
223
234
|
# Otherwise use that GT as GT.
|
|
224
|
-
.otherwise(f"{col}_detailed_zone")
|
|
225
|
-
.alias(f"{col}_special_location"),
|
|
235
|
+
.otherwise(f"{col}_detailed_zone").alias(f"{col}_special_location"),
|
|
226
236
|
)
|
|
227
237
|
return df
|
|
228
238
|
|
|
@@ -245,15 +255,13 @@ def fix_special_locations(df: pl.LazyFrame, col: str, special_locations: gpd.Geo
|
|
|
245
255
|
pl.from_pandas(special_locations["detailed_zone_id"].astype(int)),
|
|
246
256
|
default=None,
|
|
247
257
|
)
|
|
248
|
-
)
|
|
249
|
-
.alias(f"{col}_detailed_zone"),
|
|
258
|
+
).alias(f"{col}_detailed_zone"),
|
|
250
259
|
# When the ZF is an actual ZF
|
|
251
260
|
pl.when(mask)
|
|
252
261
|
# Then the GT is null
|
|
253
262
|
.then(pl.lit(None))
|
|
254
263
|
# Otherwise use that GT as GT.
|
|
255
|
-
.otherwise(f"{col}_detailed_zone")
|
|
256
|
-
.alias(f"{col}_special_location"),
|
|
264
|
+
.otherwise(f"{col}_detailed_zone").alias(f"{col}_special_location"),
|
|
257
265
|
)
|
|
258
266
|
return df
|
|
259
267
|
|
|
@@ -13,7 +13,7 @@ from .personnes import standardize_persons
|
|
|
13
13
|
from .voitures import standardize_cars
|
|
14
14
|
|
|
15
15
|
|
|
16
|
-
def standardize(source: str | ZipFile):
|
|
16
|
+
def standardize(source: str | ZipFile, skip_spatial: bool = False):
|
|
17
17
|
source_name = source.filename if isinstance(source, ZipFile) else source
|
|
18
18
|
logger.info(f"Standardizing EMP survey from `{source_name}`")
|
|
19
19
|
# Households.
|
|
@@ -10,9 +10,10 @@ from .utils import guess_survey_type, read_source
|
|
|
10
10
|
|
|
11
11
|
def standardize(
|
|
12
12
|
source: str,
|
|
13
|
-
output_directory: str,
|
|
13
|
+
output_directory: str | None = None,
|
|
14
14
|
survey_type: str | None = None,
|
|
15
15
|
add_name_subdir: bool = False,
|
|
16
|
+
skip_spatial: bool = False,
|
|
16
17
|
) -> SurveyData | None:
|
|
17
18
|
"""Converts a mobility survey to a clean standardized format.
|
|
18
19
|
|
|
@@ -26,6 +27,7 @@ def standardize(
|
|
|
26
27
|
output_directory
|
|
27
28
|
Path to the directory where the standardized survey should be stored.
|
|
28
29
|
If the directory does not exist, MobiSurvStd will create it (recursively).
|
|
30
|
+
If None, the standardized survey will not be saved.
|
|
29
31
|
survey_type
|
|
30
32
|
String indicating the type of the survey to be converted.
|
|
31
33
|
If the value is omitted, MobiSurvStd will do its best to guess the survey type.
|
|
@@ -36,6 +38,11 @@ def standardize(
|
|
|
36
38
|
If True, the standardized survey is stored in a subdirectory within `output_directory`. The
|
|
37
39
|
subdirectory name is the survey name.
|
|
38
40
|
If False (default), the standardized survey is stored directly in `output_directory`.
|
|
41
|
+
skip_spatial
|
|
42
|
+
If True, MobiSurvStd will not try to read spatial data from the survey.
|
|
43
|
+
This means that special locations, detailed zones, and draw zones will not be read and
|
|
44
|
+
proposed as an output.
|
|
45
|
+
Some variables (e.g., home_lng, home_lat) might also be missing as a result.
|
|
39
46
|
|
|
40
47
|
Returns
|
|
41
48
|
-------
|
|
@@ -66,19 +73,19 @@ def standardize(
|
|
|
66
73
|
# Note that I actually define here some aliases for the `survey_type` argument that are not
|
|
67
74
|
# documented (because why not?).
|
|
68
75
|
if survey_type == "emc2":
|
|
69
|
-
survey_data = emc2.standardize(dir_or_zip)
|
|
76
|
+
survey_data = emc2.standardize(dir_or_zip, skip_spatial=skip_spatial)
|
|
70
77
|
elif survey_type == "edgt":
|
|
71
|
-
survey_data = edgt.standardize(dir_or_zip)
|
|
78
|
+
survey_data = edgt.standardize(dir_or_zip, skip_spatial=skip_spatial)
|
|
72
79
|
elif survey_type == "edvm":
|
|
73
|
-
survey_data = edvm.standardize(dir_or_zip)
|
|
80
|
+
survey_data = edvm.standardize(dir_or_zip, skip_spatial=skip_spatial)
|
|
74
81
|
elif survey_type == "emd":
|
|
75
|
-
survey_data = emd.standardize(dir_or_zip)
|
|
82
|
+
survey_data = emd.standardize(dir_or_zip, skip_spatial=skip_spatial)
|
|
76
83
|
elif survey_type in ("emp", "emp2019"):
|
|
77
|
-
survey_data = emp.standardize(dir_or_zip)
|
|
84
|
+
survey_data = emp.standardize(dir_or_zip, skip_spatial=skip_spatial)
|
|
78
85
|
elif survey_type in ("egt2020", "egt20", "egt1820"):
|
|
79
|
-
survey_data = egt2020.standardize(dir_or_zip)
|
|
86
|
+
survey_data = egt2020.standardize(dir_or_zip, skip_spatial=skip_spatial)
|
|
80
87
|
elif survey_type in ("egt2010", "egt10"):
|
|
81
|
-
survey_data = egt2010.standardize(dir_or_zip)
|
|
88
|
+
survey_data = egt2010.standardize(dir_or_zip, skip_spatial=skip_spatial)
|
|
82
89
|
else:
|
|
83
90
|
logger.error(f"Unsupported survey type: {survey_type}")
|
|
84
91
|
return None
|
|
@@ -86,9 +93,10 @@ def standardize(
|
|
|
86
93
|
source_name = source.filename if isinstance(source, ZipFile) else source
|
|
87
94
|
logger.error(f"Failed to read survey from `{source_name}`")
|
|
88
95
|
return None
|
|
89
|
-
if
|
|
90
|
-
|
|
91
|
-
|
|
96
|
+
if output_directory is not None:
|
|
97
|
+
if add_name_subdir:
|
|
98
|
+
output_directory = os.path.join(output_directory, survey_data.metadata["name"])
|
|
99
|
+
survey_data.save(output_directory)
|
|
92
100
|
# survey_data = survey_data.clean()
|
|
93
101
|
return survey_data
|
|
94
102
|
|
|
@@ -97,6 +105,7 @@ def bulk_standardize(
|
|
|
97
105
|
directory: str,
|
|
98
106
|
output_directory: str,
|
|
99
107
|
survey_type: str | None = None,
|
|
108
|
+
skip_spatial: bool = False,
|
|
100
109
|
):
|
|
101
110
|
"""Standardizes mobility surveys in bulk from a given directory.
|
|
102
111
|
|
|
@@ -117,6 +126,11 @@ def bulk_standardize(
|
|
|
117
126
|
If the directory contains surveys of different types, leave this value to None and
|
|
118
127
|
MobiSurvStd will try to guess the type of each survey.
|
|
119
128
|
Possible values: "emc2", "emp2019", "egt2020", "egt2010", "edgt", "edvm", "emd".
|
|
129
|
+
skip_spatial
|
|
130
|
+
If True, MobiSurvStd will not try to read spatial data from the surveys.
|
|
131
|
+
This means that special locations, detailed zones, and draw zones will not be read and
|
|
132
|
+
proposed as an output.
|
|
133
|
+
Some variables (e.g., home_lng, home_lat) might also be missing as a result.
|
|
120
134
|
|
|
121
135
|
Examples
|
|
122
136
|
--------
|
|
@@ -127,7 +141,7 @@ def bulk_standardize(
|
|
|
127
141
|
>>> import mobisurvstd
|
|
128
142
|
>>> mobisurvstd.bulk_standardize("my_surveys", "standardized_surveys")
|
|
129
143
|
"""
|
|
130
|
-
n = bulk_standardize_impl(directory, output_directory, survey_type, n=0)
|
|
144
|
+
n = bulk_standardize_impl(directory, output_directory, survey_type, skip_spatial, n=0)
|
|
131
145
|
if n > 0:
|
|
132
146
|
logger.success(f"Successfully read {n} surveys from `{directory}`")
|
|
133
147
|
if n == 0:
|
|
@@ -138,6 +152,7 @@ def bulk_standardize_impl(
|
|
|
138
152
|
directory: str,
|
|
139
153
|
output_directory: str,
|
|
140
154
|
survey_type: str | None = None,
|
|
155
|
+
skip_spatial: bool = False,
|
|
141
156
|
n: int = 0,
|
|
142
157
|
):
|
|
143
158
|
if not os.path.isdir(directory):
|
|
@@ -150,13 +165,25 @@ def bulk_standardize_impl(
|
|
|
150
165
|
if maybe_type is None:
|
|
151
166
|
# The directory does not seem to be a valid survey.
|
|
152
167
|
# We try to iteratively read that directory.
|
|
153
|
-
n = bulk_standardize_impl(source, output_directory, survey_type, n)
|
|
168
|
+
n = bulk_standardize_impl(source, output_directory, survey_type, skip_spatial, n)
|
|
154
169
|
else:
|
|
155
|
-
data = standardize(
|
|
170
|
+
data = standardize(
|
|
171
|
+
source,
|
|
172
|
+
output_directory,
|
|
173
|
+
survey_type,
|
|
174
|
+
add_name_subdir=True,
|
|
175
|
+
skip_spatial=skip_spatial,
|
|
176
|
+
)
|
|
156
177
|
if data is not None:
|
|
157
178
|
n += 1
|
|
158
179
|
if source.endswith(".zip"):
|
|
159
|
-
data = standardize(
|
|
180
|
+
data = standardize(
|
|
181
|
+
source,
|
|
182
|
+
output_directory,
|
|
183
|
+
survey_type,
|
|
184
|
+
add_name_subdir=True,
|
|
185
|
+
skip_spatial=skip_spatial,
|
|
186
|
+
)
|
|
160
187
|
if data is not None:
|
|
161
188
|
n += 1
|
|
162
189
|
return n
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: mobisurvstd
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.5.0
|
|
4
4
|
Summary: A tool to standardize French mobility survey datasets (EMC2, EGT, EMP, etc.) into a unified format.
|
|
5
5
|
Author-email: Lucas Javaudin <lucas@lucasjavaudin.com>
|
|
6
6
|
License-Expression: MIT
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|