mobisurvstd 1.3.0__tar.gz → 1.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {mobisurvstd-1.3.0/src/mobisurvstd.egg-info → mobisurvstd-1.4.0}/PKG-INFO +2 -2
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/README.md +1 -1
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/pyproject.toml +1 -1
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/cerema/__init__.py +2 -1
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/cerema/deplacements.py +4 -2
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/cerema/edgt.py +16 -46
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/cerema/edvm.py +14 -29
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/cerema/emc2.py +9 -8
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/cerema/emd.py +14 -35
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/cerema/menages.py +5 -3
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/cerema/personnes.py +4 -2
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/cerema/reader.py +4 -3
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/cerema/survey.py +15 -15
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/cerema/trajets.py +4 -2
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/cli.py +17 -0
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/common/cars.py +1 -1
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/common/clean.py +7 -7
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/common/households.py +1 -1
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/common/legs.py +1 -1
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/common/motorcycles.py +1 -1
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/common/persons.py +1 -1
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/common/trips.py +1 -1
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/egt2010/deplacements.py +3 -2
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/egt2010/menages.py +5 -4
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/egt2010/personnes.py +4 -2
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/egt2010/survey.py +14 -22
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/egt2010/trajets.py +4 -2
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/egt2020/survey.py +14 -21
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/emg2023/deplacements.py +2 -2
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/emg2023/personnes.py +1 -1
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/emg2023/survey.py +5 -4
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/emp/survey.py +8 -7
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/main.py +32 -26
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/resources/insee_data.py +3 -3
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/utils.py +78 -66
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0/src/mobisurvstd.egg-info}/PKG-INFO +2 -2
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/LICENSE.md +0 -0
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/setup.cfg +0 -0
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/__init__.py +0 -0
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/__main__.py +0 -0
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/cerema/common.py +0 -0
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/cerema/nantes_opendata.py +0 -0
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/cerema/zones.py +0 -0
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/classes.py +0 -0
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/common/__init__.py +0 -0
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/common/zones.py +0 -0
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/egt2010/__init__.py +0 -0
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/egt2010/zones.py +0 -0
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/egt2020/__init__.py +0 -0
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/egt2020/deplacements.py +0 -0
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/egt2020/menages.py +0 -0
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/egt2020/motos.py +0 -0
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/egt2020/personnes.py +0 -0
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/egt2020/trajets.py +0 -0
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/egt2020/voitures.py +0 -0
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/emg2023/__init__.py +0 -0
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/emg2023/menages.py +0 -0
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/emp/__init__.py +0 -0
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/emp/common.py +0 -0
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/emp/deplacements.py +0 -0
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/emp/menages.py +0 -0
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/emp/motos.py +0 -0
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/emp/personnes.py +0 -0
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/emp/voitures.py +0 -0
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/logger.py +0 -0
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/resources/__init__.py +0 -0
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/resources/admin_express.py +0 -0
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/resources/nuts.py +0 -0
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/schema/__init__.py +0 -0
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/schema/cars.py +0 -0
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/schema/common.py +0 -0
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/schema/guarantees.py +0 -0
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/schema/households.py +0 -0
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/schema/legs.py +0 -0
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/schema/motorcycles.py +0 -0
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/schema/persons.py +0 -0
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/schema/trips.py +0 -0
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd/validation.py +0 -0
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd.egg-info/SOURCES.txt +0 -0
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd.egg-info/dependency_links.txt +0 -0
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd.egg-info/entry_points.txt +0 -0
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd.egg-info/requires.txt +0 -0
- {mobisurvstd-1.3.0 → mobisurvstd-1.4.0}/src/mobisurvstd.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: mobisurvstd
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.4.0
|
|
4
4
|
Summary: A tool to standardize French mobility survey datasets (EMC2, EGT, EMP, etc.) into a unified format.
|
|
5
5
|
Author-email: Lucas Javaudin <lucas@lucasjavaudin.com>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -135,7 +135,7 @@ Download the
|
|
|
135
135
|
_(Données individuelles anonymisées (fichiers au format CSV))_, then run the following command:
|
|
136
136
|
|
|
137
137
|
```bash
|
|
138
|
-
|
|
138
|
+
mobisurvstd emp_2019_donnees_individuelles_anonymisees_novembre2024.zip standardized_emp2019 --survey-type emp2019
|
|
139
139
|
```
|
|
140
140
|
|
|
141
141
|
It will standardize the EMP 2019 survey and save the resulting Parquet files in the
|
|
@@ -102,7 +102,7 @@ Download the
|
|
|
102
102
|
_(Données individuelles anonymisées (fichiers au format CSV))_, then run the following command:
|
|
103
103
|
|
|
104
104
|
```bash
|
|
105
|
-
|
|
105
|
+
mobisurvstd emp_2019_donnees_individuelles_anonymisees_novembre2024.zip standardized_emp2019 --survey-type emp2019
|
|
106
106
|
```
|
|
107
107
|
|
|
108
108
|
It will standardize the EMP 2019 survey and save the resulting Parquet files in the
|
|
@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
|
|
|
6
6
|
|
|
7
7
|
[project]
|
|
8
8
|
name = "mobisurvstd"
|
|
9
|
-
version = "1.
|
|
9
|
+
version = "1.4.0"
|
|
10
10
|
description = "A tool to standardize French mobility survey datasets (EMC2, EGT, EMP, etc.) into a unified format."
|
|
11
11
|
readme = "README.md"
|
|
12
12
|
authors = [{ name = "Lucas Javaudin", email = "lucas@lucasjavaudin.com" }]
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
from pathlib import Path
|
|
1
2
|
from zipfile import ZipFile
|
|
2
3
|
|
|
3
4
|
from .edgt import EDGTReader
|
|
@@ -8,7 +9,7 @@ from .nantes_opendata import OpenDataReader
|
|
|
8
9
|
|
|
9
10
|
|
|
10
11
|
def standardize(
|
|
11
|
-
source:
|
|
12
|
+
source: Path | ZipFile, survey_type: str, skip_spatial: bool = False, skip_insee: bool = False
|
|
12
13
|
):
|
|
13
14
|
if survey_type == "EMC2":
|
|
14
15
|
reader = EMC2Reader(source)
|
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import io
|
|
2
|
+
from pathlib import Path
|
|
2
3
|
|
|
3
4
|
import polars as pl
|
|
4
5
|
from loguru import logger
|
|
6
|
+
from polars.datatypes import DataTypeClass
|
|
5
7
|
|
|
6
8
|
from mobisurvstd.common.trips import clean
|
|
7
9
|
from mobisurvstd.schema.common import CURRENT_YEAR
|
|
@@ -9,7 +11,7 @@ from mobisurvstd.utils import detect_csv_delimiter
|
|
|
9
11
|
|
|
10
12
|
from .reader import CeremaReader
|
|
11
13
|
|
|
12
|
-
SCHEMA = {
|
|
14
|
+
SCHEMA: dict[str, DataTypeClass] = {
|
|
13
15
|
"DP1": pl.UInt8, # Code fichier = 3 (déplacement)
|
|
14
16
|
"DMET": pl.UInt8, # Méthode d'enquête du ménage (EMC2 only)
|
|
15
17
|
"IDD3": pl.UInt16, # Année de fin d'enquête
|
|
@@ -113,7 +115,7 @@ TRIP_PERIMETER_MAP = {
|
|
|
113
115
|
}
|
|
114
116
|
|
|
115
117
|
|
|
116
|
-
def scan_trips_impl(source:
|
|
118
|
+
def scan_trips_impl(source: Path | io.BytesIO):
|
|
117
119
|
separator = detect_csv_delimiter(source)
|
|
118
120
|
return pl.scan_csv(
|
|
119
121
|
source, separator=separator, schema_overrides=SCHEMA, null_values=["aa", "aaaaa"]
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import io
|
|
2
2
|
import os
|
|
3
3
|
import re
|
|
4
|
+
from pathlib import Path
|
|
4
5
|
from zipfile import ZipFile
|
|
5
6
|
|
|
6
7
|
import geopandas as gpd
|
|
@@ -28,49 +29,26 @@ class EDGTReader(CeremaStandardizer):
|
|
|
28
29
|
|
|
29
30
|
def special_locations_and_detailed_zones_filenames(self):
|
|
30
31
|
# This should match the Amiens 2010, Clermont-Ferrand 2012, and Lyon 2015 surveys.
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
self.source, ".*_faf_d.?coupagefin[.](tab|shp|mif)", subdir=subdir
|
|
34
|
-
)
|
|
35
|
-
tel_filename = find_file_path(
|
|
36
|
-
self.source, ".*_tel_d.?coupagefin[.](tab|shp|mif)", subdir=subdir
|
|
37
|
-
)
|
|
32
|
+
faf_filename = find_file_path(self.source, ".*_faf_d.?coupagefin[.](tab|shp|mif)")
|
|
33
|
+
tel_filename = find_file_path(self.source, ".*_tel_d.?coupagefin[.](tab|shp|mif)")
|
|
38
34
|
if faf_filename and tel_filename:
|
|
39
35
|
# Amiens 2010 has two files.
|
|
40
36
|
return [faf_filename, tel_filename]
|
|
41
37
|
else:
|
|
42
|
-
return [
|
|
43
|
-
find_file_path(
|
|
44
|
-
self.source, ".*(_zf_gt|zinterne_gen_.*)[.](tab|shp|mif)", subdir=subdir
|
|
45
|
-
)
|
|
46
|
-
]
|
|
38
|
+
return [find_file_path(self.source, ".*(_zf_gt|zinterne_gen_.*)[.](tab|shp|mif)")]
|
|
47
39
|
|
|
48
40
|
def special_locations_filenames(self):
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
self.source, ".*_faf_.*g.?n.?rateur.*[.](tab|shp|mif)", subdir=subdir
|
|
52
|
-
)
|
|
53
|
-
tel_filename = find_file_path(
|
|
54
|
-
self.source, ".*_tel_.*g.?n.?rateur.*[.](tab|shp|mif)", subdir=subdir
|
|
55
|
-
)
|
|
41
|
+
faf_filename = find_file_path(self.source, ".*_faf_.*g.?n.?rateur.*[.](tab|shp|mif)")
|
|
42
|
+
tel_filename = find_file_path(self.source, ".*_tel_.*g.?n.?rateur.*[.](tab|shp|mif)")
|
|
56
43
|
if faf_filename and tel_filename:
|
|
57
44
|
# Saint-Quentin-en-Yvelines has two files, for phone and face-to-face surveys.
|
|
58
45
|
return [faf_filename, tel_filename]
|
|
59
46
|
else:
|
|
60
|
-
return [
|
|
61
|
-
find_file_path(
|
|
62
|
-
self.source, ".*(_gt_.*|_gt|g.?n.?rateur.*)[.](tab|shp|mif)", subdir=subdir
|
|
63
|
-
)
|
|
64
|
-
]
|
|
47
|
+
return [find_file_path(self.source, ".*(_gt_.*|_gt|g.?n.?rateur.*)[.](tab|shp|mif)")]
|
|
65
48
|
|
|
66
49
|
def detailed_zones_filenames(self):
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
self.source, ".*_faf_zones_fines[.](tab|shp|mif)", subdir=subdir
|
|
70
|
-
)
|
|
71
|
-
tel_filename = find_file_path(
|
|
72
|
-
self.source, ".*_tel_zones_fines[.](tab|shp|mif)", subdir=subdir
|
|
73
|
-
)
|
|
50
|
+
faf_filename = find_file_path(self.source, ".*_faf_zones_fines[.](tab|shp|mif)")
|
|
51
|
+
tel_filename = find_file_path(self.source, ".*_tel_zones_fines[.](tab|shp|mif)")
|
|
74
52
|
if faf_filename and tel_filename:
|
|
75
53
|
# Nice 2009 and Saint-Quentin-en-Yvelines have two files, for phone and face-to-face
|
|
76
54
|
# surveys.
|
|
@@ -78,28 +56,20 @@ class EDGTReader(CeremaStandardizer):
|
|
|
78
56
|
else:
|
|
79
57
|
return [
|
|
80
58
|
find_file_path(
|
|
81
|
-
self.source,
|
|
82
|
-
".*(_zf|_zf_.*|zf08_.*|(?<!tel_)zones[_ ]fines)[.](tab|shp|mif)",
|
|
83
|
-
subdir=subdir,
|
|
59
|
+
self.source, ".*(_zf|_zf_.*|zf08_.*|(?<!tel_)zones[_ ]fines)[.](tab|shp|mif)"
|
|
84
60
|
)
|
|
85
61
|
]
|
|
86
62
|
|
|
87
63
|
def draw_zones_filenames(self):
|
|
88
|
-
return [
|
|
89
|
-
find_file_path(
|
|
90
|
-
self.source,
|
|
91
|
-
".*(_dtir|_secteurstirage)[.](tab|shp|mif)",
|
|
92
|
-
subdir=os.path.join("Doc", "SIG"),
|
|
93
|
-
)
|
|
94
|
-
]
|
|
64
|
+
return [find_file_path(self.source, ".*(_dtir|_secteurstirage)[.](tab|shp|mif)")]
|
|
95
65
|
|
|
96
66
|
def survey_name(self):
|
|
97
|
-
filename = find_file_path(self.source, ".*_std_faf_men.csv"
|
|
67
|
+
filename = find_file_path(self.source, ".*_std_faf_men.csv")
|
|
98
68
|
fn_match = re.match("(.*)_std_faf_men.csv", os.path.basename(filename))
|
|
99
69
|
if fn_match is not None:
|
|
100
70
|
return fn_match.group(1)
|
|
101
71
|
else:
|
|
102
|
-
return "
|
|
72
|
+
return "unknown"
|
|
103
73
|
|
|
104
74
|
def gt_id_columns(self):
|
|
105
75
|
return [
|
|
@@ -270,7 +240,7 @@ class EDGTReader(CeremaStandardizer):
|
|
|
270
240
|
return zfs, gts
|
|
271
241
|
|
|
272
242
|
|
|
273
|
-
def get_files(source:
|
|
243
|
+
def get_files(source: Path | ZipFile, name: str) -> list[Path | io.BytesIO | MissingFileError]:
|
|
274
244
|
# In the EDGT surveys, there are two directories with "standardized" data in the Csv directory,
|
|
275
245
|
# usually "Fichiers_Standard_Face_a_face" and "Fichiers_Standard_Telephone" but it can be
|
|
276
246
|
# something else.
|
|
@@ -278,6 +248,6 @@ def get_files(source: str | ZipFile, name: str) -> list[str | io.BytesIO | Missi
|
|
|
278
248
|
# "[SURVEY_NAME]_std_faf_[NAME].csv" and "[SURVEY_NAME]_std_tel_[NAME].csv" where NAME is either
|
|
279
249
|
# "men", "pers", "depl", or "traj".
|
|
280
250
|
return [
|
|
281
|
-
find_file(source, f".*_std_faf_{name}.csv"
|
|
282
|
-
find_file(source, f".*_std_tel_{name}.csv"
|
|
251
|
+
find_file(source, f".*_std_faf_{name}.csv"),
|
|
252
|
+
find_file(source, f".*_std_tel_{name}.csv"),
|
|
283
253
|
]
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import io
|
|
2
2
|
import os
|
|
3
3
|
import re
|
|
4
|
+
from pathlib import Path
|
|
4
5
|
|
|
5
6
|
import geopandas as gpd
|
|
6
7
|
|
|
@@ -13,56 +14,40 @@ from .zones import find_matching_column
|
|
|
13
14
|
class EDVMReader(CeremaStandardizer):
|
|
14
15
|
SURVEY_TYPE = "EDVM"
|
|
15
16
|
|
|
16
|
-
def households_filenames(self) -> list[
|
|
17
|
+
def households_filenames(self) -> list[Path | io.BytesIO | MissingFileError]:
|
|
17
18
|
# The filenames are usually stored in "Csv/Fichiers_Standard/[SURVEY_NAME]_std_[NAME].csv"
|
|
18
19
|
# but in some cases (e.g., Ajaccio 2017) the directory "Fichiers_Standard" is named based
|
|
19
20
|
# on the survey name (e.g., Ajaccio_2017_Standard).
|
|
20
|
-
return [find_file(self.source, ".*_std_men.csv"
|
|
21
|
+
return [find_file(self.source, ".*_std_men.csv")]
|
|
21
22
|
|
|
22
|
-
def persons_filenames(self) -> list[
|
|
23
|
-
return [find_file(self.source, ".*_std_pers.csv"
|
|
23
|
+
def persons_filenames(self) -> list[Path | io.BytesIO | MissingFileError]:
|
|
24
|
+
return [find_file(self.source, ".*_std_pers.csv")]
|
|
24
25
|
|
|
25
|
-
def trips_filenames(self) -> list[
|
|
26
|
-
return [find_file(self.source, ".*_std_depl.csv"
|
|
26
|
+
def trips_filenames(self) -> list[Path | io.BytesIO | MissingFileError]:
|
|
27
|
+
return [find_file(self.source, ".*_std_depl.csv")]
|
|
27
28
|
|
|
28
|
-
def legs_filenames(self) -> list[
|
|
29
|
-
return [find_file(self.source, ".*_std_traj.csv"
|
|
29
|
+
def legs_filenames(self) -> list[Path | io.BytesIO | MissingFileError]:
|
|
30
|
+
return [find_file(self.source, ".*_std_traj.csv")]
|
|
30
31
|
|
|
31
32
|
def special_locations_and_detailed_zones_filenames(self):
|
|
32
33
|
# This should only match the Beauvais 2011 survey.
|
|
33
|
-
return [
|
|
34
|
-
find_file_path(
|
|
35
|
-
self.source, ".*beauvais.*_dfin[.](tab|shp|mif)", subdir=os.path.join("Doc", "SIG")
|
|
36
|
-
)
|
|
37
|
-
]
|
|
34
|
+
return [find_file_path(self.source, ".*beauvais.*_dfin[.](tab|shp|mif)")]
|
|
38
35
|
|
|
39
36
|
def detailed_zones_filenames(self):
|
|
40
37
|
return [
|
|
41
|
-
find_file_path(
|
|
42
|
-
self.source,
|
|
43
|
-
".*(_zf_.*|_zf|zones?[_ ]?fines?.*|_dfin)[.](tab|shp|mif)",
|
|
44
|
-
subdir=os.path.join("Doc", "SIG"),
|
|
45
|
-
)
|
|
38
|
+
find_file_path(self.source, ".*(_zf_.*|_zf|zones?[_ ]?fines?.*|_dfin)[.](tab|shp|mif)")
|
|
46
39
|
]
|
|
47
40
|
|
|
48
41
|
def special_locations_filenames(self):
|
|
49
42
|
return [
|
|
50
|
-
find_file_path(
|
|
51
|
-
self.source,
|
|
52
|
-
".*(_gt_.*|_gt|_pgt|_pg|g.?n.?rateur.*)[.](tab|shp|mif)",
|
|
53
|
-
subdir=os.path.join("Doc", "SIG"),
|
|
54
|
-
)
|
|
43
|
+
find_file_path(self.source, ".*(_gt_.*|_gt|_pgt|_pg|g.?n.?rateur.*)[.](tab|shp|mif)")
|
|
55
44
|
]
|
|
56
45
|
|
|
57
46
|
def draw_zones_filenames(self):
|
|
58
|
-
return [
|
|
59
|
-
find_file_path(
|
|
60
|
-
self.source, ".*_DTIR[.](tab|shp|mif)", subdir=os.path.join("Doc", "SIG")
|
|
61
|
-
)
|
|
62
|
-
]
|
|
47
|
+
return [find_file_path(self.source, ".*_DTIR[.](tab|shp|mif)")]
|
|
63
48
|
|
|
64
49
|
def survey_name(self):
|
|
65
|
-
filename = find_file_path(self.source, ".*_std_men.csv"
|
|
50
|
+
filename = find_file_path(self.source, ".*_std_men.csv")
|
|
66
51
|
fn_match = re.match("(.*)_std_men.csv", os.path.basename(filename))
|
|
67
52
|
if fn_match is not None:
|
|
68
53
|
return fn_match.group(1)
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import io
|
|
2
2
|
import os
|
|
3
3
|
import re
|
|
4
|
+
from pathlib import Path
|
|
4
5
|
|
|
5
6
|
import geopandas as gpd
|
|
6
7
|
|
|
@@ -13,31 +14,31 @@ from .zones import find_matching_column
|
|
|
13
14
|
class EMC2Reader(CeremaStandardizer):
|
|
14
15
|
SURVEY_TYPE = "EMC2"
|
|
15
16
|
|
|
16
|
-
def households_filenames(self) -> list[
|
|
17
|
+
def households_filenames(self) -> list[Path | io.BytesIO | MissingFileError]:
|
|
17
18
|
return [find_file(self.source, ".*_std_men.csv")]
|
|
18
19
|
|
|
19
|
-
def persons_filenames(self) -> list[
|
|
20
|
+
def persons_filenames(self) -> list[Path | io.BytesIO | MissingFileError]:
|
|
20
21
|
return [find_file(self.source, ".*_std_pers.csv")]
|
|
21
22
|
|
|
22
|
-
def trips_filenames(self) -> list[
|
|
23
|
+
def trips_filenames(self) -> list[Path | io.BytesIO | MissingFileError]:
|
|
23
24
|
return [find_file(self.source, ".*_std_depl.csv")]
|
|
24
25
|
|
|
25
|
-
def legs_filenames(self) -> list[
|
|
26
|
+
def legs_filenames(self) -> list[Path | io.BytesIO | MissingFileError]:
|
|
26
27
|
return [find_file(self.source, ".*_std_traj.csv")]
|
|
27
28
|
|
|
28
29
|
def detailed_zones_filenames(self):
|
|
29
|
-
return [find_file_path(self.source, r".*_ZF(_.*)?\.(TAB|shp)"
|
|
30
|
+
return [find_file_path(self.source, r".*_ZF(_.*)?\.(TAB|shp)")]
|
|
30
31
|
|
|
31
32
|
def special_locations_filenames(self):
|
|
32
|
-
base = find_file_path(self.source, r".*_GT(?!_ext)(_.*)?\.(TAB|shp)"
|
|
33
|
-
ext = find_file_path(self.source, r".*_GT_externes(_.*)?\.(TAB|shp)"
|
|
33
|
+
base = find_file_path(self.source, r".*_GT(?!_ext)(_.*)?\.(TAB|shp)")
|
|
34
|
+
ext = find_file_path(self.source, r".*_GT_externes(_.*)?\.(TAB|shp)")
|
|
34
35
|
if base and ext:
|
|
35
36
|
return [base, ext]
|
|
36
37
|
else:
|
|
37
38
|
return [base]
|
|
38
39
|
|
|
39
40
|
def draw_zones_filenames(self):
|
|
40
|
-
return [find_file_path(self.source, r".*_DTIR(_.*)?\.(TAB|shp)"
|
|
41
|
+
return [find_file_path(self.source, r".*_DTIR(_.*)?\.(TAB|shp)")]
|
|
41
42
|
|
|
42
43
|
def survey_name(self):
|
|
43
44
|
filename = find_file_path(self.source, ".*_std_men.csv")
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import io
|
|
2
2
|
import os
|
|
3
3
|
import re
|
|
4
|
+
from pathlib import Path
|
|
4
5
|
|
|
5
6
|
import geopandas as gpd
|
|
6
7
|
|
|
@@ -13,58 +14,36 @@ from .zones import find_matching_column
|
|
|
13
14
|
class EMDReader(CeremaStandardizer):
|
|
14
15
|
SURVEY_TYPE = "EMD"
|
|
15
16
|
|
|
16
|
-
def households_filenames(self) -> list[
|
|
17
|
+
def households_filenames(self) -> list[Path | io.BytesIO | MissingFileError]:
|
|
17
18
|
# The filenames are usually stored in "Csv/Fichiers_Standard/[SURVEY_NAME]_std_[NAME].csv"
|
|
18
19
|
# but in some cases (e.g., Strasbourg 2009, Rouen 2017) the directory "Fichiers_Standard" is
|
|
19
20
|
# named based on the survey name (e.g., Rouen_2017_Standard).
|
|
20
|
-
return [find_file(self.source, ".*_std_men.csv"
|
|
21
|
+
return [find_file(self.source, ".*_std_men.csv")]
|
|
21
22
|
|
|
22
|
-
def persons_filenames(self) -> list[
|
|
23
|
-
return [find_file(self.source, ".*_std_pers.csv"
|
|
23
|
+
def persons_filenames(self) -> list[Path | io.BytesIO | MissingFileError]:
|
|
24
|
+
return [find_file(self.source, ".*_std_pers.csv")]
|
|
24
25
|
|
|
25
|
-
def trips_filenames(self) -> list[
|
|
26
|
-
return [find_file(self.source, ".*_std_depl.csv"
|
|
26
|
+
def trips_filenames(self) -> list[Path | io.BytesIO | MissingFileError]:
|
|
27
|
+
return [find_file(self.source, ".*_std_depl.csv")]
|
|
27
28
|
|
|
28
|
-
def legs_filenames(self) -> list[
|
|
29
|
-
return [find_file(self.source, ".*_std_traj.csv"
|
|
29
|
+
def legs_filenames(self) -> list[Path | io.BytesIO | MissingFileError]:
|
|
30
|
+
return [find_file(self.source, ".*_std_traj.csv")]
|
|
30
31
|
|
|
31
32
|
def special_locations_and_detailed_zones_filenames(self):
|
|
32
33
|
# This should match the Valenciennes 2011 and Grenoble 2010 surveys.
|
|
33
|
-
return [
|
|
34
|
-
find_file_path(
|
|
35
|
-
self.source,
|
|
36
|
-
".*(grenobloise10_zones_fines|zf_gt)[.]mif",
|
|
37
|
-
subdir=os.path.join("Doc", "SIG"),
|
|
38
|
-
)
|
|
39
|
-
]
|
|
34
|
+
return [find_file_path(self.source, ".*(grenobloise10_zones_fines|zf_gt)[.]mif")]
|
|
40
35
|
|
|
41
36
|
def detailed_zones_filenames(self):
|
|
42
|
-
return [
|
|
43
|
-
find_file_path(
|
|
44
|
-
self.source,
|
|
45
|
-
".*(_zf|zones?[_ ]?fines?.*)[.](tab|mif)",
|
|
46
|
-
subdir=os.path.join("Doc", "SIG"),
|
|
47
|
-
)
|
|
48
|
-
]
|
|
37
|
+
return [find_file_path(self.source, ".*(_zf|zones?[_ ]?fines?.*)[.](tab|mif)")]
|
|
49
38
|
|
|
50
39
|
def special_locations_filenames(self):
|
|
51
|
-
return [
|
|
52
|
-
find_file_path(
|
|
53
|
-
self.source, ".*(_gt|g.?n.?rateur.*)[.](tab|mif)", subdir=os.path.join("Doc", "SIG")
|
|
54
|
-
)
|
|
55
|
-
]
|
|
40
|
+
return [find_file_path(self.source, ".*(_gt|g.?n.?rateur.*)[.](tab|mif)")]
|
|
56
41
|
|
|
57
42
|
def draw_zones_filenames(self):
|
|
58
|
-
return [
|
|
59
|
-
find_file_path(
|
|
60
|
-
self.source,
|
|
61
|
-
".*(_DTIR|secteur_.*)[.](tab|shp|mif)",
|
|
62
|
-
subdir=os.path.join("Doc", "SIG"),
|
|
63
|
-
)
|
|
64
|
-
]
|
|
43
|
+
return [find_file_path(self.source, ".*(_DTIR|secteur_.*)[.](tab|shp|mif)")]
|
|
65
44
|
|
|
66
45
|
def survey_name(self):
|
|
67
|
-
filename = find_file_path(self.source, ".*_std_men.csv"
|
|
46
|
+
filename = find_file_path(self.source, ".*_std_men.csv")
|
|
68
47
|
fn_match = re.match("(.*)_std_men.csv", os.path.basename(filename))
|
|
69
48
|
if fn_match is not None:
|
|
70
49
|
return fn_match.group(1)
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import io
|
|
2
|
+
from pathlib import Path
|
|
2
3
|
|
|
3
4
|
import polars as pl
|
|
5
|
+
from polars.datatypes import DataTypeClass
|
|
4
6
|
|
|
5
7
|
from mobisurvstd.common.cars import clean as clean_cars
|
|
6
8
|
from mobisurvstd.common.households import clean as clean_households
|
|
@@ -10,7 +12,7 @@ from mobisurvstd.utils import detect_csv_delimiter
|
|
|
10
12
|
|
|
11
13
|
from .reader import CeremaReader
|
|
12
14
|
|
|
13
|
-
SCHEMA = {
|
|
15
|
+
SCHEMA: dict[str, DataTypeClass] = {
|
|
14
16
|
"MP1": pl.UInt8, # Code fichier = 1
|
|
15
17
|
"METH": pl.UInt8, # Méthode d'enquête du ménage (for EMC2 only)
|
|
16
18
|
"IDM1": pl.UInt8, # Type Enquête
|
|
@@ -206,7 +208,7 @@ MOTORCYCLE_KW_UPPERBOUND_MAP = {6: 4, 7: 11}
|
|
|
206
208
|
MOTORCYCLE_THERMIC_ENGINE_TYPE_MAP = {1: "two_stroke", 2: "four_stroke"}
|
|
207
209
|
|
|
208
210
|
|
|
209
|
-
def scan_households_impl(source:
|
|
211
|
+
def scan_households_impl(source: Path | io.BytesIO):
|
|
210
212
|
separator = detect_csv_delimiter(source)
|
|
211
213
|
return pl.scan_csv(source, separator=separator, schema_overrides=SCHEMA, null_values=["a"])
|
|
212
214
|
|
|
@@ -225,7 +227,7 @@ class HouseholdsReader(CeremaReader):
|
|
|
225
227
|
return lf
|
|
226
228
|
|
|
227
229
|
def main_insee(self):
|
|
228
|
-
return self.scan_households().select("IDM4").first().collect().item() # ty: ignore[
|
|
230
|
+
return self.scan_households().select("IDM4").first().collect().item() # ty: ignore[unresolved-attribute]
|
|
229
231
|
|
|
230
232
|
def standardize_households(self, skip_insee: bool = False):
|
|
231
233
|
lf = self.scan_households()
|
|
@@ -1,13 +1,15 @@
|
|
|
1
1
|
import io
|
|
2
|
+
from pathlib import Path
|
|
2
3
|
|
|
3
4
|
import polars as pl
|
|
5
|
+
from polars.datatypes import DataTypeClass
|
|
4
6
|
|
|
5
7
|
from mobisurvstd.common.persons import clean
|
|
6
8
|
from mobisurvstd.utils import detect_csv_delimiter
|
|
7
9
|
|
|
8
10
|
from .reader import CeremaReader
|
|
9
11
|
|
|
10
|
-
SCHEMA = {
|
|
12
|
+
SCHEMA: dict[str, DataTypeClass] = {
|
|
11
13
|
"PP1": pl.UInt8, # Code fichier = 2 (personne)
|
|
12
14
|
"PMET": pl.UInt8, # Méthode d'enquête du ménage (EMC2 only)
|
|
13
15
|
"IDP3": pl.UInt16, # Année de fin d'enquête
|
|
@@ -232,7 +234,7 @@ WEEKDAY_MAP = {
|
|
|
232
234
|
}
|
|
233
235
|
|
|
234
236
|
|
|
235
|
-
def scan_persons_impl(source:
|
|
237
|
+
def scan_persons_impl(source: Path | io.BytesIO):
|
|
236
238
|
separator = detect_csv_delimiter(source)
|
|
237
239
|
return pl.scan_csv(
|
|
238
240
|
source, separator=separator, schema_overrides=SCHEMA, null_values=["a", "aa"]
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import re
|
|
2
|
+
from pathlib import Path
|
|
2
3
|
from zipfile import ZipFile
|
|
3
4
|
|
|
4
5
|
import geopandas as gpd
|
|
@@ -10,7 +11,7 @@ from .common import EMC2_MODE_MAP, MODE_MAP, NANTES_MODE_MAP
|
|
|
10
11
|
|
|
11
12
|
class CeremaReader:
|
|
12
13
|
SURVEY_TYPE = ""
|
|
13
|
-
source:
|
|
14
|
+
source: Path | ZipFile
|
|
14
15
|
households: pl.LazyFrame
|
|
15
16
|
persons: pl.LazyFrame
|
|
16
17
|
trips: pl.LazyFrame
|
|
@@ -23,7 +24,7 @@ class CeremaReader:
|
|
|
23
24
|
special_locations_coords: pl.DataFrame | None
|
|
24
25
|
detailed_zones_coords: pl.DataFrame | None
|
|
25
26
|
|
|
26
|
-
def __init__(self, source:
|
|
27
|
+
def __init__(self, source: Path | ZipFile):
|
|
27
28
|
self.source = source
|
|
28
29
|
self.special_locations = None
|
|
29
30
|
self.detailed_zones = None
|
|
@@ -35,7 +36,7 @@ class CeremaReader:
|
|
|
35
36
|
if isinstance(self.source, ZipFile):
|
|
36
37
|
return self.source.filename or "unknown zipfile"
|
|
37
38
|
else:
|
|
38
|
-
return self.source
|
|
39
|
+
return str(self.source)
|
|
39
40
|
|
|
40
41
|
def households_filenames(self):
|
|
41
42
|
raise NotImplementedError
|
|
@@ -85,12 +85,12 @@ class CeremaStandardizer(HouseholdsReader, PersonsReader, TripsReader, LegsReade
|
|
|
85
85
|
|
|
86
86
|
def finish(self):
|
|
87
87
|
# Collecting all LazyFrames at this point can speed up next computations.
|
|
88
|
-
self.households = self.households.collect().lazy() # ty: ignore[
|
|
89
|
-
self.persons = self.persons.collect().lazy() # ty: ignore[
|
|
90
|
-
self.trips = self.trips.collect().lazy() # ty: ignore[
|
|
91
|
-
self.legs = self.legs.collect().lazy() # ty: ignore[
|
|
92
|
-
self.cars = self.cars.collect().lazy() # ty: ignore[
|
|
93
|
-
self.motorcycles = self.motorcycles.collect().lazy() # ty: ignore[
|
|
88
|
+
self.households = self.households.collect().lazy() # ty: ignore[unresolved-attribute]
|
|
89
|
+
self.persons = self.persons.collect().lazy() # ty: ignore[unresolved-attribute]
|
|
90
|
+
self.trips = self.trips.collect().lazy() # ty: ignore[unresolved-attribute]
|
|
91
|
+
self.legs = self.legs.collect().lazy() # ty: ignore[unresolved-attribute]
|
|
92
|
+
self.cars = self.cars.collect().lazy() # ty: ignore[unresolved-attribute]
|
|
93
|
+
self.motorcycles = self.motorcycles.collect().lazy() # ty: ignore[unresolved-attribute]
|
|
94
94
|
self.add_survey_dates()
|
|
95
95
|
self.fix_main_mode()
|
|
96
96
|
self.fix_detailed_zones()
|
|
@@ -126,7 +126,7 @@ class CeremaStandardizer(HouseholdsReader, PersonsReader, TripsReader, LegsReade
|
|
|
126
126
|
.filter(pl.col("mode_group").eq(pl.col("main_mode_group")).any().over("trip_id").not_())
|
|
127
127
|
.select("trip_id")
|
|
128
128
|
.collect()
|
|
129
|
-
.to_series() # ty: ignore[
|
|
129
|
+
.to_series() # ty: ignore[unresolved-attribute]
|
|
130
130
|
.unique()
|
|
131
131
|
)
|
|
132
132
|
n = len(invalid_trips)
|
|
@@ -178,21 +178,21 @@ class CeremaStandardizer(HouseholdsReader, PersonsReader, TripsReader, LegsReade
|
|
|
178
178
|
self.detailed_zones["detailed_zone_id"].str.len().max(),
|
|
179
179
|
self.households.select(pl.col("home_detailed_zone").str.len_chars().max())
|
|
180
180
|
.collect()
|
|
181
|
-
.item(), # ty: ignore[
|
|
181
|
+
.item(), # ty: ignore[unresolved-attribute]
|
|
182
182
|
self.persons.select(pl.col("work_detailed_zone").str.len_chars().max())
|
|
183
183
|
.collect()
|
|
184
|
-
.item(), # ty: ignore[
|
|
184
|
+
.item(), # ty: ignore[unresolved-attribute]
|
|
185
185
|
self.persons.select(pl.col("study_detailed_zone").str.len_chars().max())
|
|
186
186
|
.collect()
|
|
187
|
-
.item(), # ty: ignore[
|
|
187
|
+
.item(), # ty: ignore[unresolved-attribute]
|
|
188
188
|
self.trips.select(pl.col("origin_detailed_zone").str.len_chars().max())
|
|
189
189
|
.collect()
|
|
190
|
-
.item(), # ty: ignore[
|
|
190
|
+
.item(), # ty: ignore[unresolved-attribute]
|
|
191
191
|
self.trips.select(pl.col("destination_detailed_zone").str.len_chars().max())
|
|
192
192
|
.collect()
|
|
193
|
-
.item(), # ty: ignore[
|
|
194
|
-
self.legs.select(pl.col("start_detailed_zone").str.len_chars().max()).collect().item(), # ty: ignore[
|
|
195
|
-
self.legs.select(pl.col("end_detailed_zone").str.len_chars().max()).collect().item(), # ty: ignore[
|
|
193
|
+
.item(), # ty: ignore[unresolved-attribute]
|
|
194
|
+
self.legs.select(pl.col("start_detailed_zone").str.len_chars().max()).collect().item(), # ty: ignore[unresolved-attribute]
|
|
195
|
+
self.legs.select(pl.col("end_detailed_zone").str.len_chars().max()).collect().item(), # ty: ignore[unresolved-attribute]
|
|
196
196
|
)
|
|
197
197
|
self.detailed_zones["detailed_zone_id"] = self.detailed_zones["detailed_zone_id"].str.pad(
|
|
198
198
|
width=max_len, fillchar="0"
|
|
@@ -268,7 +268,7 @@ class CeremaStandardizer(HouseholdsReader, PersonsReader, TripsReader, LegsReade
|
|
|
268
268
|
lambda df, prefix: remove_external_zones(df, prefix, zf_ids, gt_ids)
|
|
269
269
|
)
|
|
270
270
|
|
|
271
|
-
def apply_function_to_location_columns(self, func: Callable[[pl.DataFrame, str], pl.
|
|
271
|
+
def apply_function_to_location_columns(self, func: Callable[[pl.DataFrame, str], pl.DataFrame]):
|
|
272
272
|
"""Applies the function `func` to all the location columns (households' `home_*`, persons'
|
|
273
273
|
`work_*`, etc.
|
|
274
274
|
|
|
@@ -1,14 +1,16 @@
|
|
|
1
1
|
import io
|
|
2
|
+
from pathlib import Path
|
|
2
3
|
|
|
3
4
|
import polars as pl
|
|
4
5
|
from loguru import logger
|
|
6
|
+
from polars.datatypes import DataTypeClass
|
|
5
7
|
|
|
6
8
|
from mobisurvstd.common.legs import clean
|
|
7
9
|
from mobisurvstd.utils import detect_csv_delimiter
|
|
8
10
|
|
|
9
11
|
from .reader import CeremaReader
|
|
10
12
|
|
|
11
|
-
SCHEMA = {
|
|
13
|
+
SCHEMA: dict[str, DataTypeClass] = {
|
|
12
14
|
"TP1": pl.UInt8, # Code fichier = 4 (trajet)
|
|
13
15
|
"TMET": pl.UInt8, # Méthode d'enquête du ménage (EMC2 only)
|
|
14
16
|
"IDT3": pl.UInt16, # Année de fin d'enquête
|
|
@@ -93,7 +95,7 @@ MOTORCYCLE_MAP = {
|
|
|
93
95
|
}
|
|
94
96
|
|
|
95
97
|
|
|
96
|
-
def scan_legs_impl(source:
|
|
98
|
+
def scan_legs_impl(source: Path | io.BytesIO):
|
|
97
99
|
separator = detect_csv_delimiter(source)
|
|
98
100
|
return pl.scan_csv(
|
|
99
101
|
source, separator=separator, schema_overrides=SCHEMA, null_values=["a", "aa", "aaaaa"]
|
|
@@ -3,6 +3,8 @@ from typing import Optional
|
|
|
3
3
|
import typer
|
|
4
4
|
from typing_extensions import Annotated
|
|
5
5
|
|
|
6
|
+
import mobisurvstd
|
|
7
|
+
|
|
6
8
|
from .main import bulk_standardize, standardize
|
|
7
9
|
from .resources import clear_cache
|
|
8
10
|
|
|
@@ -13,6 +15,12 @@ def clear_cache_callback(value: bool):
|
|
|
13
15
|
raise typer.Exit()
|
|
14
16
|
|
|
15
17
|
|
|
18
|
+
def version_callback(value: bool):
|
|
19
|
+
if value:
|
|
20
|
+
print(mobisurvstd.__version__)
|
|
21
|
+
raise typer.Exit()
|
|
22
|
+
|
|
23
|
+
|
|
16
24
|
def standardizer(
|
|
17
25
|
source: Annotated[
|
|
18
26
|
str,
|
|
@@ -48,6 +56,15 @@ def standardizer(
|
|
|
48
56
|
"--clear-cache", callback=clear_cache_callback, help="Clear the cache data and exit"
|
|
49
57
|
),
|
|
50
58
|
] = None,
|
|
59
|
+
version: Annotated[
|
|
60
|
+
bool | None,
|
|
61
|
+
typer.Option(
|
|
62
|
+
"--version",
|
|
63
|
+
help="Show MobiSurvStd version and exit",
|
|
64
|
+
callback=version_callback,
|
|
65
|
+
is_eager=True,
|
|
66
|
+
),
|
|
67
|
+
] = None,
|
|
51
68
|
):
|
|
52
69
|
"""Mobility Survey Standardizer: a Python command line tool to convert mobility surveys to a
|
|
53
70
|
clean standardized format.
|
|
@@ -10,7 +10,7 @@ def clean(lf: pl.LazyFrame, extra_cols: list[str] | None = None):
|
|
|
10
10
|
columns = [variable.name for variable in CAR_SCHEMA if variable.name in existing_cols]
|
|
11
11
|
if extra_cols is not None:
|
|
12
12
|
columns.extend(extra_cols)
|
|
13
|
-
lf = lf.select(columns).sort("original_car_id").collect().lazy() # ty: ignore[
|
|
13
|
+
lf = lf.select(columns).sort("original_car_id").collect().lazy() # ty: ignore[unresolved-attribute]
|
|
14
14
|
lf = indexing(lf, existing_cols)
|
|
15
15
|
lf = add_fuel_type_group(lf, existing_cols)
|
|
16
16
|
lf = add_mileage_bounds(lf, existing_cols)
|