mobisurvstd 1.0.1__tar.gz → 1.0.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {mobisurvstd-1.0.1/src/mobisurvstd.egg-info → mobisurvstd-1.0.2}/PKG-INFO +2 -1
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/pyproject.toml +6 -3
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/cerema/deplacements.py +3 -1
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/cerema/edgt.py +26 -28
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/cerema/edvm.py +18 -21
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/cerema/emc2.py +16 -24
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/cerema/emd.py +14 -16
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/cerema/menages.py +11 -5
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/cerema/nantes_opendata.py +4 -3
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/cerema/personnes.py +3 -4
- mobisurvstd-1.0.2/src/mobisurvstd/cerema/reader.py +159 -0
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/cerema/survey.py +113 -223
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/cerema/trajets.py +3 -1
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/cerema/zones.py +8 -4
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/classes.py +14 -15
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/common/cars.py +1 -1
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/common/clean.py +11 -10
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/common/households.py +3 -3
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/common/legs.py +1 -1
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/common/motorcycles.py +1 -1
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/common/persons.py +1 -1
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/common/trips.py +2 -1
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/common/zones.py +1 -1
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/egt2010/menages.py +2 -2
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/egt2010/survey.py +4 -4
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/emp/survey.py +1 -1
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/resources/admin_express.py +22 -18
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/resources/insee_data.py +6 -5
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/schema/common.py +4 -2
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/schema/guarantees.py +8 -5
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/schema/legs.py +2 -2
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/schema/trips.py +2 -2
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/utils.py +68 -29
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2/src/mobisurvstd.egg-info}/PKG-INFO +2 -1
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd.egg-info/SOURCES.txt +1 -0
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd.egg-info/requires.txt +1 -0
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/LICENSE.md +0 -0
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/README.md +0 -0
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/setup.cfg +0 -0
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/__init__.py +0 -0
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/__main__.py +0 -0
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/cerema/__init__.py +0 -0
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/cerema/common.py +0 -0
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/cli.py +0 -0
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/common/__init__.py +0 -0
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/egt2010/__init__.py +0 -0
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/egt2010/deplacements.py +0 -0
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/egt2010/personnes.py +0 -0
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/egt2010/trajets.py +0 -0
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/egt2010/zones.py +0 -0
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/egt2020/__init__.py +0 -0
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/egt2020/deplacements.py +0 -0
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/egt2020/menages.py +0 -0
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/egt2020/motos.py +0 -0
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/egt2020/personnes.py +0 -0
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/egt2020/survey.py +0 -0
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/egt2020/trajets.py +0 -0
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/egt2020/voitures.py +0 -0
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/emp/__init__.py +0 -0
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/emp/common.py +0 -0
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/emp/deplacements.py +0 -0
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/emp/menages.py +0 -0
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/emp/motos.py +0 -0
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/emp/personnes.py +0 -0
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/emp/voitures.py +0 -0
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/logger.py +0 -0
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/main.py +0 -0
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/resources/__init__.py +0 -0
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/resources/nuts.py +0 -0
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/schema/__init__.py +0 -0
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/schema/cars.py +0 -0
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/schema/households.py +0 -0
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/schema/motorcycles.py +0 -0
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/schema/persons.py +0 -0
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd/validation.py +0 -0
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd.egg-info/dependency_links.txt +0 -0
- {mobisurvstd-1.0.1 → mobisurvstd-1.0.2}/src/mobisurvstd.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: mobisurvstd
|
|
3
|
-
Version: 1.0.
|
|
3
|
+
Version: 1.0.2
|
|
4
4
|
Summary: A tool to standardize French mobility survey datasets (EMC2, EGT, EMP, etc.) into a unified format.
|
|
5
5
|
Author-email: Lucas Javaudin <lucas@lucasjavaudin.com>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -29,6 +29,7 @@ Requires-Dist: polars>=1.33.0
|
|
|
29
29
|
Requires-Dist: py7zr>=0.20.8
|
|
30
30
|
Requires-Dist: pyarrow>=19.0.1
|
|
31
31
|
Requires-Dist: requests>=2.32.3
|
|
32
|
+
Requires-Dist: seaborn>=0.13.2
|
|
32
33
|
Requires-Dist: typer>=0.16.0
|
|
33
34
|
Dynamic: license-file
|
|
34
35
|
|
|
@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
|
|
|
6
6
|
|
|
7
7
|
[project]
|
|
8
8
|
name = "mobisurvstd"
|
|
9
|
-
version = "1.0.
|
|
9
|
+
version = "1.0.2"
|
|
10
10
|
description = "A tool to standardize French mobility survey datasets (EMC2, EGT, EMP, etc.) into a unified format."
|
|
11
11
|
readme = "README.md"
|
|
12
12
|
authors = [{ name = "Lucas Javaudin", email = "lucas@lucasjavaudin.com" }]
|
|
@@ -22,17 +22,18 @@ dependencies = [
|
|
|
22
22
|
"Cartopy >= 0.24.1",
|
|
23
23
|
"fastexcel >= 0.14.0",
|
|
24
24
|
"fsspec>=2025.9.0",
|
|
25
|
-
"geopandas
|
|
25
|
+
"geopandas>=1.0.1",
|
|
26
26
|
"humanize >= 4.13.0",
|
|
27
27
|
"loguru >= 0.7.3",
|
|
28
28
|
"matplotlib >= 3.10.1",
|
|
29
29
|
"numpy >= 2.2.4",
|
|
30
30
|
"pandas >= 2.2.3",
|
|
31
31
|
"platformdirs >= 4.3.8",
|
|
32
|
-
"polars
|
|
32
|
+
"polars>=1.33.0",
|
|
33
33
|
"py7zr >= 0.20.8",
|
|
34
34
|
"pyarrow >= 19.0.1",
|
|
35
35
|
"requests >= 2.32.3",
|
|
36
|
+
"seaborn>=0.13.2",
|
|
36
37
|
"typer >= 0.16.0",
|
|
37
38
|
]
|
|
38
39
|
requires-python = ">=3.10"
|
|
@@ -40,6 +41,8 @@ requires-python = ">=3.10"
|
|
|
40
41
|
[dependency-groups]
|
|
41
42
|
dev = [
|
|
42
43
|
"ipython>=8.37.0",
|
|
44
|
+
"pytest>=9.0.2",
|
|
45
|
+
"ty>=0.0.15",
|
|
43
46
|
]
|
|
44
47
|
|
|
45
48
|
[project.urls]
|
|
@@ -6,6 +6,8 @@ from loguru import logger
|
|
|
6
6
|
from mobisurvstd.common.trips import clean
|
|
7
7
|
from mobisurvstd.schema.common import CURRENT_YEAR
|
|
8
8
|
|
|
9
|
+
from .reader import CeremaReader
|
|
10
|
+
|
|
9
11
|
SCHEMA = {
|
|
10
12
|
"DP1": pl.UInt8, # Code fichier = 3 (déplacement)
|
|
11
13
|
"DMET": pl.UInt8, # Méthode d'enquête du ménage (EMC2 only)
|
|
@@ -114,7 +116,7 @@ def scan_trips_impl(source: str | io.BytesIO):
|
|
|
114
116
|
return pl.scan_csv(source, separator=";", schema_overrides=SCHEMA, null_values=["aa", "aaaaa"])
|
|
115
117
|
|
|
116
118
|
|
|
117
|
-
class TripsReader:
|
|
119
|
+
class TripsReader(CeremaReader):
|
|
118
120
|
def scan_trips(self):
|
|
119
121
|
lfs_iter = map(scan_trips_impl, self.trips_filenames())
|
|
120
122
|
lf = pl.concat(lfs_iter, how="vertical")
|
|
@@ -5,13 +5,13 @@ from zipfile import ZipFile
|
|
|
5
5
|
|
|
6
6
|
import geopandas as gpd
|
|
7
7
|
|
|
8
|
-
from mobisurvstd.utils import MissingFileError, find_file
|
|
8
|
+
from mobisurvstd.utils import MissingFileError, find_file, find_file_path
|
|
9
9
|
|
|
10
|
-
from .survey import
|
|
10
|
+
from .survey import CeremaStandardizer
|
|
11
11
|
from .zones import find_matching_column
|
|
12
12
|
|
|
13
13
|
|
|
14
|
-
class EDGTReader(
|
|
14
|
+
class EDGTReader(CeremaStandardizer):
|
|
15
15
|
SURVEY_TYPE = "EDGT"
|
|
16
16
|
|
|
17
17
|
def households_filenames(self):
|
|
@@ -29,53 +29,49 @@ class EDGTReader(CeremaReader):
|
|
|
29
29
|
def special_locations_and_detailed_zones_filenames(self):
|
|
30
30
|
# This should match the Amiens 2010, Clermont-Ferrand 2012, and Lyon 2015 surveys.
|
|
31
31
|
subdir = os.path.join("Doc", "SIG")
|
|
32
|
-
faf_filename =
|
|
33
|
-
self.source, ".*_faf_d.?coupagefin[.](tab|shp|mif)", subdir=subdir
|
|
32
|
+
faf_filename = find_file_path(
|
|
33
|
+
self.source, ".*_faf_d.?coupagefin[.](tab|shp|mif)", subdir=subdir
|
|
34
34
|
)
|
|
35
|
-
tel_filename =
|
|
36
|
-
self.source, ".*_tel_d.?coupagefin[.](tab|shp|mif)", subdir=subdir
|
|
35
|
+
tel_filename = find_file_path(
|
|
36
|
+
self.source, ".*_tel_d.?coupagefin[.](tab|shp|mif)", subdir=subdir
|
|
37
37
|
)
|
|
38
38
|
if faf_filename and tel_filename:
|
|
39
39
|
# Amiens 2010 has two files.
|
|
40
40
|
return [faf_filename, tel_filename]
|
|
41
41
|
else:
|
|
42
42
|
return [
|
|
43
|
-
|
|
43
|
+
find_file_path(
|
|
44
44
|
self.source,
|
|
45
45
|
".*(_zf_gt|zinterne_gen_.*)[.](tab|shp|mif)",
|
|
46
46
|
subdir=subdir,
|
|
47
|
-
as_url=True,
|
|
48
47
|
)
|
|
49
48
|
]
|
|
50
49
|
|
|
51
50
|
def special_locations_filenames(self):
|
|
52
51
|
subdir = os.path.join("Doc", "SIG")
|
|
53
|
-
faf_filename =
|
|
54
|
-
self.source, ".*_faf_.*g.?n.?rateur.*[.](tab|shp|mif)", subdir=subdir
|
|
52
|
+
faf_filename = find_file_path(
|
|
53
|
+
self.source, ".*_faf_.*g.?n.?rateur.*[.](tab|shp|mif)", subdir=subdir
|
|
55
54
|
)
|
|
56
|
-
tel_filename =
|
|
57
|
-
self.source, ".*_tel_.*g.?n.?rateur.*[.](tab|shp|mif)", subdir=subdir
|
|
55
|
+
tel_filename = find_file_path(
|
|
56
|
+
self.source, ".*_tel_.*g.?n.?rateur.*[.](tab|shp|mif)", subdir=subdir
|
|
58
57
|
)
|
|
59
58
|
if faf_filename and tel_filename:
|
|
60
59
|
# Saint-Quentin-en-Yvelines has two files, for phone and face-to-face surveys.
|
|
61
60
|
return [faf_filename, tel_filename]
|
|
62
61
|
else:
|
|
63
62
|
return [
|
|
64
|
-
|
|
65
|
-
self.source,
|
|
66
|
-
".*(_gt_.*|_gt|g.?n.?rateur.*)[.](tab|shp|mif)",
|
|
67
|
-
subdir=subdir,
|
|
68
|
-
as_url=True,
|
|
63
|
+
find_file_path(
|
|
64
|
+
self.source, ".*(_gt_.*|_gt|g.?n.?rateur.*)[.](tab|shp|mif)", subdir=subdir
|
|
69
65
|
)
|
|
70
66
|
]
|
|
71
67
|
|
|
72
68
|
def detailed_zones_filenames(self):
|
|
73
69
|
subdir = os.path.join("Doc", "SIG")
|
|
74
|
-
faf_filename =
|
|
75
|
-
self.source, ".*_faf_zones_fines[.](tab|shp|mif)", subdir=subdir
|
|
70
|
+
faf_filename = find_file_path(
|
|
71
|
+
self.source, ".*_faf_zones_fines[.](tab|shp|mif)", subdir=subdir
|
|
76
72
|
)
|
|
77
|
-
tel_filename =
|
|
78
|
-
self.source, ".*_tel_zones_fines[.](tab|shp|mif)", subdir=subdir
|
|
73
|
+
tel_filename = find_file_path(
|
|
74
|
+
self.source, ".*_tel_zones_fines[.](tab|shp|mif)", subdir=subdir
|
|
79
75
|
)
|
|
80
76
|
if faf_filename and tel_filename:
|
|
81
77
|
# Nice 2009 and Saint-Quentin-en-Yvelines have two files, for phone and face-to-face
|
|
@@ -83,27 +79,29 @@ class EDGTReader(CeremaReader):
|
|
|
83
79
|
return [faf_filename, tel_filename]
|
|
84
80
|
else:
|
|
85
81
|
return [
|
|
86
|
-
|
|
82
|
+
find_file_path(
|
|
87
83
|
self.source,
|
|
88
84
|
".*(_zf|_zf_.*|zf08_.*|(?<!tel_)zones[_ ]fines)[.](tab|shp|mif)",
|
|
89
85
|
subdir=subdir,
|
|
90
|
-
as_url=True,
|
|
91
86
|
)
|
|
92
87
|
]
|
|
93
88
|
|
|
94
89
|
def draw_zones_filenames(self):
|
|
95
90
|
return [
|
|
96
|
-
|
|
91
|
+
find_file_path(
|
|
97
92
|
self.source,
|
|
98
93
|
".*(_dtir|_secteurstirage)[.](tab|shp|mif)",
|
|
99
94
|
subdir=os.path.join("Doc", "SIG"),
|
|
100
|
-
as_url=True,
|
|
101
95
|
)
|
|
102
96
|
]
|
|
103
97
|
|
|
104
98
|
def survey_name(self):
|
|
105
|
-
filename =
|
|
106
|
-
|
|
99
|
+
filename = find_file_path(self.source, ".*_std_faf_men.csv", subdir="Csv")
|
|
100
|
+
fn_match = re.match("(.*)_std_faf_men.csv", os.path.basename(filename))
|
|
101
|
+
if fn_match is not None:
|
|
102
|
+
return fn_match.group(1)
|
|
103
|
+
else:
|
|
104
|
+
return "unkown"
|
|
107
105
|
|
|
108
106
|
def gt_id_columns(self):
|
|
109
107
|
return [
|
|
@@ -4,18 +4,18 @@ import re
|
|
|
4
4
|
|
|
5
5
|
import geopandas as gpd
|
|
6
6
|
|
|
7
|
-
from mobisurvstd.cerema.survey import
|
|
8
|
-
from mobisurvstd.utils import MissingFileError, find_file
|
|
7
|
+
from mobisurvstd.cerema.survey import CeremaStandardizer
|
|
8
|
+
from mobisurvstd.utils import MissingFileError, find_file, find_file_path
|
|
9
9
|
|
|
10
10
|
from .zones import find_matching_column
|
|
11
11
|
|
|
12
12
|
|
|
13
|
-
class EDVMReader(
|
|
13
|
+
class EDVMReader(CeremaStandardizer):
|
|
14
14
|
SURVEY_TYPE = "EDVM"
|
|
15
15
|
|
|
16
16
|
def households_filenames(self) -> list[str | io.BytesIO | MissingFileError]:
|
|
17
17
|
# The filenames are usually stored in "Csv/Fichiers_Standard/[SURVEY_NAME]_std_[NAME].csv"
|
|
18
|
-
# but in some cases (e.g., Ajaccio 2017) the directory "Fichiers_Standard" is
|
|
18
|
+
# but in some cases (e.g., Ajaccio 2017) the directory "Fichiers_Standard" is named based
|
|
19
19
|
# on the survey name (e.g., Ajaccio_2017_Standard).
|
|
20
20
|
return [find_file(self.source, ".*_std_men.csv", subdir="Csv")]
|
|
21
21
|
|
|
@@ -31,47 +31,43 @@ class EDVMReader(CeremaReader):
|
|
|
31
31
|
def special_locations_and_detailed_zones_filenames(self):
|
|
32
32
|
# This should only match the Beauvais 2011 survey.
|
|
33
33
|
return [
|
|
34
|
-
|
|
35
|
-
self.source,
|
|
36
|
-
".*beauvais.*_dfin[.](tab|shp|mif)",
|
|
37
|
-
subdir=os.path.join("Doc", "SIG"),
|
|
38
|
-
as_url=True,
|
|
34
|
+
find_file_path(
|
|
35
|
+
self.source, ".*beauvais.*_dfin[.](tab|shp|mif)", subdir=os.path.join("Doc", "SIG")
|
|
39
36
|
)
|
|
40
37
|
]
|
|
41
38
|
|
|
42
39
|
def detailed_zones_filenames(self):
|
|
43
40
|
return [
|
|
44
|
-
|
|
41
|
+
find_file_path(
|
|
45
42
|
self.source,
|
|
46
43
|
".*(_zf_.*|_zf|zones?[_ ]?fines?.*|_dfin)[.](tab|shp|mif)",
|
|
47
44
|
subdir=os.path.join("Doc", "SIG"),
|
|
48
|
-
as_url=True,
|
|
49
45
|
)
|
|
50
46
|
]
|
|
51
47
|
|
|
52
48
|
def special_locations_filenames(self):
|
|
53
49
|
return [
|
|
54
|
-
|
|
50
|
+
find_file_path(
|
|
55
51
|
self.source,
|
|
56
52
|
".*(_gt_.*|_gt|_pgt|_pg|g.?n.?rateur.*)[.](tab|shp|mif)",
|
|
57
53
|
subdir=os.path.join("Doc", "SIG"),
|
|
58
|
-
as_url=True,
|
|
59
54
|
)
|
|
60
55
|
]
|
|
61
56
|
|
|
62
57
|
def draw_zones_filenames(self):
|
|
63
58
|
return [
|
|
64
|
-
|
|
65
|
-
self.source,
|
|
66
|
-
".*_DTIR[.](tab|shp|mif)",
|
|
67
|
-
subdir=os.path.join("Doc", "SIG"),
|
|
68
|
-
as_url=True,
|
|
59
|
+
find_file_path(
|
|
60
|
+
self.source, ".*_DTIR[.](tab|shp|mif)", subdir=os.path.join("Doc", "SIG")
|
|
69
61
|
)
|
|
70
62
|
]
|
|
71
63
|
|
|
72
64
|
def survey_name(self):
|
|
73
|
-
filename =
|
|
74
|
-
|
|
65
|
+
filename = find_file_path(self.source, ".*_std_men.csv", subdir="Csv")
|
|
66
|
+
fn_match = re.match("(.*)_std_men.csv", os.path.basename(filename))
|
|
67
|
+
if fn_match is not None:
|
|
68
|
+
return fn_match.group(1)
|
|
69
|
+
else:
|
|
70
|
+
return "unkown"
|
|
75
71
|
|
|
76
72
|
def gt_id_columns(self):
|
|
77
73
|
return [
|
|
@@ -277,9 +273,10 @@ class EDVMReader(CeremaReader):
|
|
|
277
273
|
super().select_zf_from_gt_column(gdf)
|
|
278
274
|
|
|
279
275
|
def postprocess_special_locations_and_detailed_zones(
|
|
280
|
-
self, zfs: gpd.GeoDataFrame, gts: gpd.GeoDataFrame
|
|
276
|
+
self, zfs: gpd.GeoDataFrame | None, gts: gpd.GeoDataFrame | None
|
|
281
277
|
):
|
|
282
278
|
# This function should only be call for Beauvais 2011.
|
|
279
|
+
assert gts is not None
|
|
283
280
|
assert "detailed_zone_id" not in gts.columns
|
|
284
281
|
# The corresponding ZF is the GT id with the last digit replaced by 0.
|
|
285
282
|
gts["detailed_zone_id"] = gts["special_location_id"].str.replace(r"\d$", "0", regex=True)
|
|
@@ -4,13 +4,13 @@ import re
|
|
|
4
4
|
|
|
5
5
|
import geopandas as gpd
|
|
6
6
|
|
|
7
|
-
from mobisurvstd.cerema.survey import
|
|
8
|
-
from mobisurvstd.utils import MissingFileError, find_file
|
|
7
|
+
from mobisurvstd.cerema.survey import CeremaStandardizer
|
|
8
|
+
from mobisurvstd.utils import MissingFileError, find_file, find_file_path
|
|
9
9
|
|
|
10
10
|
from .zones import find_matching_column
|
|
11
11
|
|
|
12
12
|
|
|
13
|
-
class EMC2Reader(
|
|
13
|
+
class EMC2Reader(CeremaStandardizer):
|
|
14
14
|
SURVEY_TYPE = "EMC2"
|
|
15
15
|
|
|
16
16
|
def households_filenames(self) -> list[str | io.BytesIO | MissingFileError]:
|
|
@@ -43,42 +43,34 @@ class EMC2Reader(CeremaReader):
|
|
|
43
43
|
|
|
44
44
|
def detailed_zones_filenames(self):
|
|
45
45
|
return [
|
|
46
|
-
|
|
47
|
-
self.source,
|
|
48
|
-
r".*_ZF(_.*)?\.(TAB|shp)",
|
|
49
|
-
subdir=os.path.join("Doc", "SIG"),
|
|
50
|
-
as_url=True,
|
|
46
|
+
find_file_path(
|
|
47
|
+
self.source, r".*_ZF(_.*)?\.(TAB|shp)", subdir=os.path.join("Doc", "SIG")
|
|
51
48
|
)
|
|
52
49
|
]
|
|
53
50
|
|
|
54
51
|
def special_locations_filenames(self):
|
|
55
52
|
return [
|
|
56
|
-
|
|
57
|
-
self.source,
|
|
58
|
-
r".*_GT(_.*)?\.(TAB|shp)",
|
|
59
|
-
subdir=os.path.join("Doc", "SIG"),
|
|
60
|
-
as_url=True,
|
|
53
|
+
find_file_path(
|
|
54
|
+
self.source, r".*_GT(_.*)?\.(TAB|shp)", subdir=os.path.join("Doc", "SIG")
|
|
61
55
|
)
|
|
62
56
|
]
|
|
63
57
|
|
|
64
58
|
def draw_zones_filenames(self):
|
|
65
59
|
return [
|
|
66
|
-
|
|
67
|
-
self.source,
|
|
68
|
-
r".*_DTIR(_.*)?\.(TAB|shp)",
|
|
69
|
-
subdir=os.path.join("Doc", "SIG"),
|
|
70
|
-
as_url=True,
|
|
60
|
+
find_file_path(
|
|
61
|
+
self.source, r".*_DTIR(_.*)?\.(TAB|shp)", subdir=os.path.join("Doc", "SIG")
|
|
71
62
|
)
|
|
72
63
|
]
|
|
73
64
|
|
|
74
65
|
def survey_name(self):
|
|
75
|
-
filename =
|
|
76
|
-
self.source,
|
|
77
|
-
".*_std_men.csv",
|
|
78
|
-
subdir=os.path.join("Csv", "Fichiers_Standard"),
|
|
79
|
-
as_url=True,
|
|
66
|
+
filename = find_file_path(
|
|
67
|
+
self.source, ".*_std_men.csv", subdir=os.path.join("Csv", "Fichiers_Standard")
|
|
80
68
|
)
|
|
81
|
-
|
|
69
|
+
fn_match = re.match("(.*)_std_men.csv", os.path.basename(filename))
|
|
70
|
+
if fn_match is not None:
|
|
71
|
+
return fn_match.group(1)
|
|
72
|
+
else:
|
|
73
|
+
return "Unknown"
|
|
82
74
|
|
|
83
75
|
def gt_id_columns(self):
|
|
84
76
|
return ["zf_160", "zf_fusion", "codegt", "zf", "num_gt"]
|
|
@@ -4,13 +4,13 @@ import re
|
|
|
4
4
|
|
|
5
5
|
import geopandas as gpd
|
|
6
6
|
|
|
7
|
-
from mobisurvstd.cerema.survey import
|
|
8
|
-
from mobisurvstd.utils import MissingFileError, find_file
|
|
7
|
+
from mobisurvstd.cerema.survey import CeremaStandardizer
|
|
8
|
+
from mobisurvstd.utils import MissingFileError, find_file, find_file_path
|
|
9
9
|
|
|
10
10
|
from .zones import find_matching_column
|
|
11
11
|
|
|
12
12
|
|
|
13
|
-
class EMDReader(
|
|
13
|
+
class EMDReader(CeremaStandardizer):
|
|
14
14
|
SURVEY_TYPE = "EMD"
|
|
15
15
|
|
|
16
16
|
def households_filenames(self) -> list[str | io.BytesIO | MissingFileError]:
|
|
@@ -31,47 +31,45 @@ class EMDReader(CeremaReader):
|
|
|
31
31
|
def special_locations_and_detailed_zones_filenames(self):
|
|
32
32
|
# This should match the Valenciennes 2011 and Grenoble 2010 surveys.
|
|
33
33
|
return [
|
|
34
|
-
|
|
34
|
+
find_file_path(
|
|
35
35
|
self.source,
|
|
36
36
|
".*(grenobloise10_zones_fines|zf_gt)[.]mif",
|
|
37
37
|
subdir=os.path.join("Doc", "SIG"),
|
|
38
|
-
as_url=True,
|
|
39
38
|
)
|
|
40
39
|
]
|
|
41
40
|
|
|
42
41
|
def detailed_zones_filenames(self):
|
|
43
42
|
return [
|
|
44
|
-
|
|
43
|
+
find_file_path(
|
|
45
44
|
self.source,
|
|
46
45
|
".*(_zf|zones?[_ ]?fines?.*)[.](tab|mif)",
|
|
47
46
|
subdir=os.path.join("Doc", "SIG"),
|
|
48
|
-
as_url=True,
|
|
49
47
|
)
|
|
50
48
|
]
|
|
51
49
|
|
|
52
50
|
def special_locations_filenames(self):
|
|
53
51
|
return [
|
|
54
|
-
|
|
55
|
-
self.source,
|
|
56
|
-
".*(_gt|g.?n.?rateur.*)[.](tab|mif)",
|
|
57
|
-
subdir=os.path.join("Doc", "SIG"),
|
|
58
|
-
as_url=True,
|
|
52
|
+
find_file_path(
|
|
53
|
+
self.source, ".*(_gt|g.?n.?rateur.*)[.](tab|mif)", subdir=os.path.join("Doc", "SIG")
|
|
59
54
|
)
|
|
60
55
|
]
|
|
61
56
|
|
|
62
57
|
def draw_zones_filenames(self):
|
|
63
58
|
return [
|
|
64
|
-
|
|
59
|
+
find_file_path(
|
|
65
60
|
self.source,
|
|
66
61
|
".*(_DTIR|secteur_.*)[.](tab|shp|mif)",
|
|
67
62
|
subdir=os.path.join("Doc", "SIG"),
|
|
68
|
-
as_url=True,
|
|
69
63
|
)
|
|
70
64
|
]
|
|
71
65
|
|
|
72
66
|
def survey_name(self):
|
|
73
|
-
filename =
|
|
74
|
-
|
|
67
|
+
filename = find_file_path(self.source, ".*_std_men.csv", subdir="Csv")
|
|
68
|
+
fn_match = re.match("(.*)_std_men.csv", os.path.basename(filename))
|
|
69
|
+
if fn_match is not None:
|
|
70
|
+
return fn_match.group(1)
|
|
71
|
+
else:
|
|
72
|
+
return "Unknown"
|
|
75
73
|
|
|
76
74
|
def gt_id_columns(self):
|
|
77
75
|
return [
|
|
@@ -7,6 +7,8 @@ from mobisurvstd.common.households import clean as clean_households
|
|
|
7
7
|
from mobisurvstd.common.motorcycles import clean as clean_motorcycles
|
|
8
8
|
from mobisurvstd.schema.common import CURRENT_YEAR
|
|
9
9
|
|
|
10
|
+
from .reader import CeremaReader
|
|
11
|
+
|
|
10
12
|
SCHEMA = {
|
|
11
13
|
"MP1": pl.UInt8, # Code fichier = 1
|
|
12
14
|
"METH": pl.UInt8, # Méthode d'enquête du ménage (for EMC2 only)
|
|
@@ -230,8 +232,8 @@ def scan_households_impl(source: str | io.BytesIO):
|
|
|
230
232
|
return pl.scan_csv(source, separator=";", schema_overrides=SCHEMA, null_values=["a"])
|
|
231
233
|
|
|
232
234
|
|
|
233
|
-
class HouseholdsReader:
|
|
234
|
-
def scan_households(self):
|
|
235
|
+
class HouseholdsReader(CeremaReader):
|
|
236
|
+
def scan_households(self) -> pl.LazyFrame:
|
|
235
237
|
lfs_iter = map(scan_households_impl, self.households_filenames())
|
|
236
238
|
lf = pl.concat(lfs_iter, how="vertical")
|
|
237
239
|
if "METH" not in lf.collect_schema().names():
|
|
@@ -244,7 +246,7 @@ class HouseholdsReader:
|
|
|
244
246
|
return lf
|
|
245
247
|
|
|
246
248
|
def main_insee(self):
|
|
247
|
-
return self.scan_households().select("IDM4").first().collect().item()
|
|
249
|
+
return self.scan_households().select("IDM4").first().collect().item() # ty: ignore[possibly-missing-attribute]
|
|
248
250
|
|
|
249
251
|
def standardize_households(self):
|
|
250
252
|
lf = self.scan_households()
|
|
@@ -299,6 +301,8 @@ class HouseholdsReader:
|
|
|
299
301
|
how="left",
|
|
300
302
|
coalesce=True,
|
|
301
303
|
)
|
|
304
|
+
# Note: the `.lazy()` after the concat is useless (this is already a LazyFrame) but it helps
|
|
305
|
+
# the syntax checker.
|
|
302
306
|
lf = pl.concat(
|
|
303
307
|
(
|
|
304
308
|
lf.select(
|
|
@@ -326,7 +330,7 @@ class HouseholdsReader:
|
|
|
326
330
|
for i, a in enumerate(("A", "B", "C", "D"))
|
|
327
331
|
),
|
|
328
332
|
how="vertical",
|
|
329
|
-
)
|
|
333
|
+
).lazy()
|
|
330
334
|
# Drop the lines with empty car characteristics (there are always 4 cars per
|
|
331
335
|
# households even when the household has less than 4 cars).
|
|
332
336
|
lf = lf.filter(
|
|
@@ -347,6 +351,8 @@ class HouseholdsReader:
|
|
|
347
351
|
how="left",
|
|
348
352
|
coalesce=True,
|
|
349
353
|
)
|
|
354
|
+
# Note: the `.lazy()` after the concat is useless (this is already a LazyFrame) but it helps
|
|
355
|
+
# the syntax checker.
|
|
350
356
|
lf = pl.concat(
|
|
351
357
|
(
|
|
352
358
|
lf.select(
|
|
@@ -388,7 +394,7 @@ class HouseholdsReader:
|
|
|
388
394
|
for i, a in enumerate(("A", "B", "C", "D"))
|
|
389
395
|
),
|
|
390
396
|
how="vertical",
|
|
391
|
-
)
|
|
397
|
+
).lazy()
|
|
392
398
|
lf = lf.filter(
|
|
393
399
|
pl.any_horizontal(
|
|
394
400
|
pl.all()
|
|
@@ -2,13 +2,14 @@ import io
|
|
|
2
2
|
|
|
3
3
|
import pandas as pd
|
|
4
4
|
import polars as pl
|
|
5
|
+
from polars.datatypes import DataTypeClass
|
|
5
6
|
|
|
6
7
|
from mobisurvstd.utils import find_file
|
|
7
8
|
|
|
8
9
|
from .deplacements import SCHEMA as TRIP_SCHEMA
|
|
9
10
|
from .menages import SCHEMA as HOUSEHOLD_SCHEMA
|
|
10
11
|
from .personnes import SCHEMA as PERSON_SCHEMA
|
|
11
|
-
from .survey import
|
|
12
|
+
from .survey import CeremaStandardizer
|
|
12
13
|
from .trajets import SCHEMA as LEG_SCHEMA
|
|
13
14
|
|
|
14
15
|
# The anonimized open-data format for Nantes uses a fixed-width text format.
|
|
@@ -140,7 +141,7 @@ LEG_FORMAT = [
|
|
|
140
141
|
]
|
|
141
142
|
|
|
142
143
|
|
|
143
|
-
class OpenDataReader(
|
|
144
|
+
class OpenDataReader(CeremaStandardizer):
|
|
144
145
|
SURVEY_TYPE = "EDGT-opendata"
|
|
145
146
|
|
|
146
147
|
def households_filenames(self):
|
|
@@ -193,7 +194,7 @@ class OpenDataReader(CeremaReader):
|
|
|
193
194
|
def scan_fwf(
|
|
194
195
|
f: str | io.BytesIO,
|
|
195
196
|
format: list[tuple[int, int, str]],
|
|
196
|
-
schema: dict[str,
|
|
197
|
+
schema: dict[str, DataTypeClass],
|
|
197
198
|
):
|
|
198
199
|
# Decrease `start` by 1 because pandas starts counting at 0 but the survey document starts
|
|
199
200
|
# counting at 1.
|
|
@@ -4,6 +4,8 @@ import polars as pl
|
|
|
4
4
|
|
|
5
5
|
from mobisurvstd.common.persons import clean
|
|
6
6
|
|
|
7
|
+
from .reader import CeremaReader
|
|
8
|
+
|
|
7
9
|
SCHEMA = {
|
|
8
10
|
"PP1": pl.UInt8, # Code fichier = 2 (personne)
|
|
9
11
|
"PMET": pl.UInt8, # Méthode d'enquête du ménage (EMC2 only)
|
|
@@ -64,7 +66,6 @@ REFERENCE_PERSON_LINK_MAP = {
|
|
|
64
66
|
7: "other:non_relative", # Autre Non précisé
|
|
65
67
|
}
|
|
66
68
|
|
|
67
|
-
200 / 2600 / 2490 / 2809
|
|
68
69
|
DRIVING_LICENSE_MAP = {
|
|
69
70
|
1: "yes",
|
|
70
71
|
2: "no",
|
|
@@ -234,7 +235,7 @@ def scan_persons_impl(source: str | io.BytesIO):
|
|
|
234
235
|
return pl.scan_csv(source, separator=";", schema_overrides=SCHEMA, null_values=["a", "aa"])
|
|
235
236
|
|
|
236
237
|
|
|
237
|
-
class PersonsReader:
|
|
238
|
+
class PersonsReader(CeremaReader):
|
|
238
239
|
def scan_persons(self):
|
|
239
240
|
lfs_iter = map(scan_persons_impl, self.persons_filenames())
|
|
240
241
|
lf = pl.concat(lfs_iter, how="vertical")
|
|
@@ -371,8 +372,6 @@ class PersonsReader:
|
|
|
371
372
|
|
|
372
373
|
# For Arras 2014, the "MOIS" column is always 14 (for 2014) and the "DATE" column
|
|
373
374
|
# corresponds to the month (January or February). The exact date is unknown. We set to 1.
|
|
374
|
-
|
|
375
|
-
|
|
376
375
|
def fix_dates(lf: pl.LazyFrame, survey_name: str):
|
|
377
376
|
# Arras 2014 case.
|
|
378
377
|
lf = lf.with_columns(
|