mobisurvstd 0.2.0__tar.gz → 0.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. {mobisurvstd-0.2.0/src/mobisurvstd.egg-info → mobisurvstd-0.4.0}/PKG-INFO +13 -3
  2. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/README.md +12 -2
  3. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/pyproject.toml +4 -4
  4. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/src/mobisurvstd/clean.py +4 -4
  5. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/src/mobisurvstd/cli.py +1 -1
  6. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/src/mobisurvstd/common/cars.py +1 -1
  7. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/src/mobisurvstd/common/persons.py +1 -1
  8. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/src/mobisurvstd/common/trips.py +1 -1
  9. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/src/mobisurvstd/edgt/survey.py +13 -15
  10. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/src/mobisurvstd/edgt/zones.py +1 -0
  11. mobisurvstd-0.4.0/src/mobisurvstd/edvm/common.py +43 -0
  12. mobisurvstd-0.4.0/src/mobisurvstd/edvm/deplacements.py +160 -0
  13. mobisurvstd-0.4.0/src/mobisurvstd/edvm/menages.py +353 -0
  14. mobisurvstd-0.4.0/src/mobisurvstd/edvm/personnes.py +308 -0
  15. mobisurvstd-0.4.0/src/mobisurvstd/edvm/survey.py +273 -0
  16. mobisurvstd-0.4.0/src/mobisurvstd/edvm/trajets.py +271 -0
  17. mobisurvstd-0.4.0/src/mobisurvstd/edvm/zones.py +374 -0
  18. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/src/mobisurvstd/egt2020/deplacements.py +4 -2
  19. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/src/mobisurvstd/egt2020/trajets.py +5 -5
  20. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/src/mobisurvstd/emc2/survey.py +3 -3
  21. mobisurvstd-0.4.0/src/mobisurvstd/emd/__init__.py +1 -0
  22. mobisurvstd-0.4.0/src/mobisurvstd/emd/common.py +42 -0
  23. mobisurvstd-0.4.0/src/mobisurvstd/emd/deplacements.py +160 -0
  24. mobisurvstd-0.4.0/src/mobisurvstd/emd/menages.py +355 -0
  25. mobisurvstd-0.4.0/src/mobisurvstd/emd/personnes.py +312 -0
  26. mobisurvstd-0.4.0/src/mobisurvstd/emd/survey.py +275 -0
  27. mobisurvstd-0.4.0/src/mobisurvstd/emd/trajets.py +271 -0
  28. mobisurvstd-0.4.0/src/mobisurvstd/emd/zones.py +222 -0
  29. mobisurvstd-0.4.0/src/mobisurvstd/emp/__init__.py +1 -0
  30. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/src/mobisurvstd/main.py +7 -3
  31. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/src/mobisurvstd/resources/admin_express.py +1 -1
  32. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/src/mobisurvstd/utils.py +44 -3
  33. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0/src/mobisurvstd.egg-info}/PKG-INFO +13 -3
  34. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/src/mobisurvstd.egg-info/SOURCES.txt +16 -0
  35. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/LICENSE.md +0 -0
  36. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/setup.cfg +0 -0
  37. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/src/mobisurvstd/__init__.py +0 -0
  38. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/src/mobisurvstd/__main__.py +0 -0
  39. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/src/mobisurvstd/classes.py +0 -0
  40. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/src/mobisurvstd/common/__init__.py +0 -0
  41. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/src/mobisurvstd/common/clean.py +0 -0
  42. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/src/mobisurvstd/common/households.py +0 -0
  43. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/src/mobisurvstd/common/legs.py +0 -0
  44. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/src/mobisurvstd/common/modes.py +0 -0
  45. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/src/mobisurvstd/common/motorcycles.py +0 -0
  46. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/src/mobisurvstd/common/zones.py +0 -0
  47. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/src/mobisurvstd/edgt/__init__.py +0 -0
  48. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/src/mobisurvstd/edgt/common.py +0 -0
  49. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/src/mobisurvstd/edgt/deplacements.py +0 -0
  50. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/src/mobisurvstd/edgt/menages.py +0 -0
  51. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/src/mobisurvstd/edgt/personnes.py +0 -0
  52. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/src/mobisurvstd/edgt/trajets.py +0 -0
  53. {mobisurvstd-0.2.0/src/mobisurvstd/egt2010 → mobisurvstd-0.4.0/src/mobisurvstd/edvm}/__init__.py +0 -0
  54. {mobisurvstd-0.2.0/src/mobisurvstd/egt2020 → mobisurvstd-0.4.0/src/mobisurvstd/egt2010}/__init__.py +0 -0
  55. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/src/mobisurvstd/egt2010/deplacements.py +0 -0
  56. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/src/mobisurvstd/egt2010/menages.py +0 -0
  57. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/src/mobisurvstd/egt2010/personnes.py +0 -0
  58. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/src/mobisurvstd/egt2010/survey.py +0 -0
  59. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/src/mobisurvstd/egt2010/trajets.py +0 -0
  60. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/src/mobisurvstd/egt2010/zones.py +0 -0
  61. {mobisurvstd-0.2.0/src/mobisurvstd/emc2 → mobisurvstd-0.4.0/src/mobisurvstd/egt2020}/__init__.py +0 -0
  62. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/src/mobisurvstd/egt2020/menages.py +0 -0
  63. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/src/mobisurvstd/egt2020/motos.py +0 -0
  64. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/src/mobisurvstd/egt2020/personnes.py +0 -0
  65. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/src/mobisurvstd/egt2020/survey.py +0 -0
  66. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/src/mobisurvstd/egt2020/voitures.py +0 -0
  67. {mobisurvstd-0.2.0/src/mobisurvstd/emp → mobisurvstd-0.4.0/src/mobisurvstd/emc2}/__init__.py +0 -0
  68. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/src/mobisurvstd/emc2/common.py +0 -0
  69. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/src/mobisurvstd/emc2/deplacements.py +0 -0
  70. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/src/mobisurvstd/emc2/menages.py +0 -0
  71. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/src/mobisurvstd/emc2/personnes.py +0 -0
  72. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/src/mobisurvstd/emc2/trajets.py +0 -0
  73. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/src/mobisurvstd/emc2/zones.py +0 -0
  74. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/src/mobisurvstd/emp/deplacements.py +0 -0
  75. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/src/mobisurvstd/emp/menages.py +0 -0
  76. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/src/mobisurvstd/emp/motos.py +0 -0
  77. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/src/mobisurvstd/emp/personnes.py +0 -0
  78. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/src/mobisurvstd/emp/survey.py +0 -0
  79. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/src/mobisurvstd/emp/voitures.py +0 -0
  80. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/src/mobisurvstd/logger.py +0 -0
  81. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/src/mobisurvstd/resources/__init__.py +0 -0
  82. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/src/mobisurvstd/resources/insee_data.py +0 -0
  83. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/src/mobisurvstd/resources/nuts.py +0 -0
  84. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/src/mobisurvstd/schema.py +0 -0
  85. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/src/mobisurvstd.egg-info/dependency_links.txt +0 -0
  86. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/src/mobisurvstd.egg-info/requires.txt +0 -0
  87. {mobisurvstd-0.2.0 → mobisurvstd-0.4.0}/src/mobisurvstd.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: mobisurvstd
3
- Version: 0.2.0
3
+ Version: 0.4.0
4
4
  Summary: A tool to standardize French mobility survey datasets (EMC2, EGT, EMP, etc.) into a unified format.
5
5
  Author-email: Lucas Javaudin <lucas@lucasjavaudin.com>
6
6
  License-Expression: MIT
@@ -78,6 +78,7 @@ Also surveys based on previous formats are still in use today:
78
78
  (until 2018);
79
79
  - **EDGT** (Enquêtes Déplacements Grands Territoires, CEREMA): mobility surveys for periphery areas
80
80
  (until 2018);
81
+ - **EMD** (Enquêtes Ménages Déplacements, CEREMA): other mobility surveys from CEREMA (until 2018);
81
82
  - [**EGT 2010**](https://omnil.fr/egt-2010) (Enquête Globale Transport, Île-de-France Mobilités):
82
83
  previous version of the Île-de-France mobility survey;
83
84
  - [**ENTD**](https://www.statistiques.developpement-durable.gouv.fr/enquete-nationale-transports-et-deplacements-entd-2008)
@@ -149,11 +150,18 @@ mobisurvstd.standardize(
149
150
 
150
151
  For more, check the [User Guide](https://mobisurvstd.github.io/MobiSurvStd/howto.html).
151
152
 
153
+ **Citing MobiSurvStd**
154
+
155
+ If you use MobiSurvStd in a publication or project, please include the following attribution:
156
+ "This work used the MobiSurvStd library developed by Lucas Javaudin, available at
157
+ [https://github.com/MobiSurvStd/MobiSurvStd]()".
158
+
152
159
  ---
153
160
 
154
161
  ## Case Study: Bicycle Use
155
162
 
156
- The following graph represents the share of bicycle trips for EMC², EDGT, and EGT surveys.
163
+ The following graph represents the share of bicycle trips for EMC², EDGT, EDVM, EMD, and EGT
164
+ surveys.
157
165
  The circle colors represent the average number of bicycles in the surveyed households.
158
166
  The circle sizes represent the expected number of trips in the surveyed area.
159
167
 
@@ -181,10 +189,12 @@ Currently, MobiSurvStd supports the following survey types:
181
189
  - `egt2020`
182
190
  - `egt2010`
183
191
  - `edgt`
192
+ - `edvm`
193
+ - `emd`
184
194
 
185
195
  See [Survey Types](https://mobisurvstd.github.io/MobiSurvStd/surveys.html) for more.
186
196
 
187
- Other survey types that are planned to be integrated are: `edvm`, `entd`, `emg`.
197
+ Other survey types that are planned to be integrated are: `entd`, `emg`.
188
198
  If you know another survey format that could be integrated, feel free to open an issue on GitHub.
189
199
 
190
200
  MobiSurvStd covers only French mobility survey formats.
@@ -46,6 +46,7 @@ Also surveys based on previous formats are still in use today:
46
46
  (until 2018);
47
47
  - **EDGT** (Enquêtes Déplacements Grands Territoires, CEREMA): mobility surveys for periphery areas
48
48
  (until 2018);
49
+ - **EMD** (Enquêtes Ménages Déplacements, CEREMA): other mobility surveys from CEREMA (until 2018);
49
50
  - [**EGT 2010**](https://omnil.fr/egt-2010) (Enquête Globale Transport, Île-de-France Mobilités):
50
51
  previous version of the Île-de-France mobility survey;
51
52
  - [**ENTD**](https://www.statistiques.developpement-durable.gouv.fr/enquete-nationale-transports-et-deplacements-entd-2008)
@@ -117,11 +118,18 @@ mobisurvstd.standardize(
117
118
 
118
119
  For more, check the [User Guide](https://mobisurvstd.github.io/MobiSurvStd/howto.html).
119
120
 
121
+ **Citing MobiSurvStd**
122
+
123
+ If you use MobiSurvStd in a publication or project, please include the following attribution:
124
+ "This work used the MobiSurvStd library developed by Lucas Javaudin, available at
125
+ [https://github.com/MobiSurvStd/MobiSurvStd]()".
126
+
120
127
  ---
121
128
 
122
129
  ## Case Study: Bicycle Use
123
130
 
124
- The following graph represents the share of bicycle trips for EMC², EDGT, and EGT surveys.
131
+ The following graph represents the share of bicycle trips for EMC², EDGT, EDVM, EMD, and EGT
132
+ surveys.
125
133
  The circle colors represent the average number of bicycles in the surveyed households.
126
134
  The circle sizes represent the expected number of trips in the surveyed area.
127
135
 
@@ -149,10 +157,12 @@ Currently, MobiSurvStd supports the following survey types:
149
157
  - `egt2020`
150
158
  - `egt2010`
151
159
  - `edgt`
160
+ - `edvm`
161
+ - `emd`
152
162
 
153
163
  See [Survey Types](https://mobisurvstd.github.io/MobiSurvStd/surveys.html) for more.
154
164
 
155
- Other survey types that are planned to be integrated are: `edvm`, `entd`, `emg`.
165
+ Other survey types that are planned to be integrated are: `entd`, `emg`.
156
166
  If you know another survey format that could be integrated, feel free to open an issue on GitHub.
157
167
 
158
168
  MobiSurvStd covers only French mobility survey formats.
@@ -1,12 +1,12 @@
1
1
  # pyproject.toml
2
2
 
3
3
  [build-system]
4
- requires = ["setuptools>=61.0.0", "wheel"]
4
+ requires = ["setuptools>=61.0.0", "wheel"]
5
5
  build-backend = "setuptools.build_meta"
6
6
 
7
7
  [project]
8
8
  name = "mobisurvstd"
9
- version = "0.2.0"
9
+ version = "0.4.0"
10
10
  description = "A tool to standardize French mobility survey datasets (EMC2, EGT, EMP, etc.) into a unified format."
11
11
  readme = "README.md"
12
12
  authors = [{ name = "Lucas Javaudin", email = "lucas@lucasjavaudin.com" }]
@@ -15,7 +15,7 @@ license-files = ["LICENSE.md"]
15
15
  classifiers = [
16
16
  "Programming Language :: Python",
17
17
  "Programming Language :: Python :: 3",
18
- "Topic :: File Formats"
18
+ "Topic :: File Formats",
19
19
  ]
20
20
  keywords = ["mobility survey", "polars", "data format"]
21
21
  dependencies = [
@@ -39,4 +39,4 @@ requires-python = ">=3.10"
39
39
  Homepage = "https://github.com/MobiSurvStd/MobiSurvStd"
40
40
  Documentation = "https://mobisurvstd.github.io/MobiSurvStd/index.html"
41
41
  Repository = "https://github.com/MobiSurvStd/MobiSurvStd"
42
- Issues = "https://github.com/MobiSurvStd/MobiSurvStd/issues"
42
+ Issues = "https://github.com/MobiSurvStd/MobiSurvStd/issues"
@@ -1017,7 +1017,7 @@ def clean(
1017
1017
  ), "`origin_purpose` is education-related for a non-student person"
1018
1018
  assert trips.select(
1019
1019
  (
1020
- pl.col("origin_purpose").cast(pl.String).str.extract("(\w+):?")
1020
+ pl.col("origin_purpose").cast(pl.String).str.extract(r"(\w+):?")
1021
1021
  == pl.col("origin_purpose_group")
1022
1022
  ).all()
1023
1023
  ).item(), "Columns `origin_purpose` and `origin_purpose_group` have some inconsistencies"
@@ -1043,7 +1043,7 @@ def clean(
1043
1043
  ), "`destination_purpose` is education-related for a non-student person"
1044
1044
  assert trips.select(
1045
1045
  (
1046
- pl.col("destination_purpose").cast(pl.String).str.extract("(\w+):?")
1046
+ pl.col("destination_purpose").cast(pl.String).str.extract(r"(\w+):?")
1047
1047
  == pl.col("destination_purpose_group")
1048
1048
  ).all()
1049
1049
  ).item(), (
@@ -1078,7 +1078,7 @@ def clean(
1078
1078
  )
1079
1079
  assert trips.select(
1080
1080
  (
1081
- pl.col("origin_escort_purpose").cast(pl.String).str.extract("(\w+):?")
1081
+ pl.col("origin_escort_purpose").cast(pl.String).str.extract(r"(\w+):?")
1082
1082
  == pl.col("origin_escort_purpose_group")
1083
1083
  ).all()
1084
1084
  ).item(), (
@@ -1101,7 +1101,7 @@ def clean(
1101
1101
  )
1102
1102
  assert trips.select(
1103
1103
  (
1104
- pl.col("destination_escort_purpose").cast(pl.String).str.extract("(\w+):?")
1104
+ pl.col("destination_escort_purpose").cast(pl.String).str.extract(r"(\w+):?")
1105
1105
  == pl.col("destination_escort_purpose_group")
1106
1106
  ).all()
1107
1107
  ).item(), (
@@ -20,7 +20,7 @@ def standardizer(
20
20
  survey_type: Annotated[
21
21
  str | None,
22
22
  typer.Option(
23
- help="Format of the original survey. Possible values: `emc2`, `emp2019`, `egt2010`, `egt2020`."
23
+ help="Format of the original survey. Possible values: `emc2`, `emp2019`, `egt2010`, `egt2020`, `edgt`, `edvm`, `emd`."
24
24
  ),
25
25
  ] = None,
26
26
  bulk: Annotated[
@@ -36,7 +36,7 @@ def indexing(lf: pl.LazyFrame, existing_cols: list[str]):
36
36
  def add_fuel_type_group(lf: pl.LazyFrame, existing_cols: list[str]):
37
37
  if "fuel_type" in existing_cols:
38
38
  lf = lf.with_columns(
39
- fuel_type_group=pl.col("fuel_type").cast(pl.String).str.extract("(\w+):?")
39
+ fuel_type_group=pl.col("fuel_type").cast(pl.String).str.extract(r"(\w+):?")
40
40
  )
41
41
  existing_cols.append("fuel_type_group")
42
42
  return lf
@@ -98,7 +98,7 @@ def add_professional_occupation(lf: pl.LazyFrame, existing_cols: list[str]):
98
98
  ):
99
99
  lf = lf.with_columns(
100
100
  professional_occupation=pl.col("detailed_professional_occupation").str.extract(
101
- "(\w+):?"
101
+ r"(\w+):?"
102
102
  )
103
103
  )
104
104
  existing_cols.append("professional_occupation")
@@ -108,7 +108,7 @@ def add_purpose_groups(lf: pl.LazyFrame, existing_cols: list[str]):
108
108
  if col in existing_cols:
109
109
  group_col = f"{col}_group"
110
110
  lf = lf.with_columns(
111
- pl.col(col).cast(pl.String).str.extract("(\w+):?").alias(group_col)
111
+ pl.col(col).cast(pl.String).str.extract(r"(\w+):?").alias(group_col)
112
112
  )
113
113
  return lf
114
114
 
@@ -30,35 +30,29 @@ def standardize(source: str | ZipFile):
30
30
  filename = special_locations_filename(source)
31
31
  if filename is None:
32
32
  special_locations = None
33
- special_locations_coords = None
34
33
  elif "EDGT_SCOTAM_2017" in source_name or "Angers12" in source_name or "EDGTFVG2016":
35
34
  # Note. Special case for Metz 2017 and Angers 2012: I cannot figure out the ids in the GT
36
35
  # and ZF files.
37
36
  # For Annemasse 2016, the GT id does not seem to be specified.
38
37
  logger.warning("Cannot read special locations: ids are invalid")
39
38
  special_locations = None
40
- special_locations_coords = None
41
39
  else:
42
40
  logger.debug(f"Reading special locations from `{filename}`")
43
41
  special_locations = read_special_locations(filename)
44
- special_locations_coords = get_coords(special_locations, "special_location")
45
42
  # Detailed zones.
46
43
  filename = detailed_zones_filename(source)
47
44
  if filename is None:
48
45
  logger.debug(f"No file with detailed zones in `{source_name}`")
49
46
  detailed_zones = None
50
- detailed_zones_coords = None
51
47
  elif "EDGT_SCOTAM_2017" in source_name or "Angers12" in source_name or "FaF_Zones_fines":
52
48
  # Note. Special case for Metz 2017 and Angers 2012: I cannot figure out the ids in the GT
53
49
  # and ZF files.
54
50
  # For Bayonne 2010, the ids are missing.
55
51
  logger.warning("Cannot read detailed zones: ids are invalid")
56
52
  detailed_zones = None
57
- detailed_zones_coords = None
58
53
  else:
59
54
  logger.debug(f"Reading detailed zones from `{filename}`")
60
55
  detailed_zones = read_detailed_zones(filename)
61
- detailed_zones_coords = get_coords(detailed_zones, "detailed_zone")
62
56
  # Draw zones.
63
57
  filename = draw_zones_filename(source)
64
58
  if filename is None:
@@ -70,7 +64,7 @@ def standardize(source: str | ZipFile):
70
64
 
71
65
  if special_locations is not None:
72
66
  # For the Angers 2022 survey, the ids in the detailed zones shapefile have 3 zeros removed
73
- # compared to the # ids in the special locations shapefile.
67
+ # compared to the ids in the special locations shapefile.
74
68
  assert detailed_zones is not None, (
75
69
  "Special locations are defined but there is no data on detailed zones"
76
70
  )
@@ -85,8 +79,12 @@ def standardize(source: str | ZipFile):
85
79
 
86
80
  if special_locations is not None:
87
81
  special_locations_coords = get_coords(special_locations, "special_location")
82
+ else:
83
+ special_locations_coords = None
88
84
  if detailed_zones is not None:
89
85
  detailed_zones_coords = get_coords(detailed_zones, "detailed_zone")
86
+ else:
87
+ detailed_zones_coords = None
90
88
 
91
89
  # Households.
92
90
  filenames = households_filename(source)
@@ -176,7 +174,7 @@ def standardize(source: str | ZipFile):
176
174
  special_locations=special_locations,
177
175
  detailed_zones=detailed_zones,
178
176
  draw_zones=draw_zones,
179
- survey_type="EMC2",
177
+ survey_type="EDGT",
180
178
  survey_name=survey_name(source),
181
179
  main_insee=main_insee,
182
180
  )
@@ -187,14 +185,14 @@ def survey_main_insee(source: str | bytes):
187
185
 
188
186
 
189
187
  def survey_name(source: str | ZipFile):
190
- filename = find_file(source, ".*_std_faf_men.csv", subdir=os.path.join("Csv"), as_url=True)
188
+ filename = find_file(source, ".*_std_faf_men.csv", subdir="Csv", as_url=True)
191
189
  return re.match("(.*)_std_faf_men.csv", os.path.basename(filename)).group(1)
192
190
 
193
191
 
194
192
  def get_files(source: str | ZipFile, name: str):
195
193
  return [
196
- find_file(source, f".*_std_faf_{name}.csv", subdir=os.path.join("Csv")),
197
- find_file(source, f".*_std_tel_{name}.csv", subdir=os.path.join("Csv")),
194
+ find_file(source, f".*_std_faf_{name}.csv", subdir="Csv"),
195
+ find_file(source, f".*_std_tel_{name}.csv", subdir="Csv"),
198
196
  ]
199
197
 
200
198
 
@@ -217,7 +215,7 @@ def legs_filename(source: str | ZipFile):
217
215
  def detailed_zones_filename(source: str | ZipFile):
218
216
  return find_file(
219
217
  source,
220
- ".*_(ZF|Zones_Fines|Zones fines|ZInterne)(.*)?\.(TAB|shp|MIF)",
218
+ r".*_(ZF|Zones_Fines|Zones fines|ZInterne)(.*)?\.(TAB|shp|MIF)",
221
219
  subdir=os.path.join("Doc", "SIG"),
222
220
  as_url=True,
223
221
  )
@@ -230,7 +228,7 @@ def special_locations_filename(source: str | ZipFile):
230
228
  # locations for Lyon.)
231
229
  return find_file(
232
230
  source,
233
- "(?!.*ZF_GT).*(_GT|Gener)(.*)?\.(TAB|shp|MIF)",
231
+ r"(?!.*ZF_GT).*(_GT|Gener)(.*)?\.(TAB|shp|MIF)",
234
232
  subdir=os.path.join("Doc", "SIG"),
235
233
  as_url=True,
236
234
  )
@@ -238,7 +236,7 @@ def special_locations_filename(source: str | ZipFile):
238
236
 
239
237
  def draw_zones_filename(source: str | ZipFile):
240
238
  return find_file(
241
- source, ".*_DTIR(_.*)?\.(TAB|shp|MIF)", subdir=os.path.join("Doc", "SIG"), as_url=True
239
+ source, r".*_DTIR(_.*)?\.(TAB|shp|MIF)", subdir=os.path.join("Doc", "SIG"), as_url=True
242
240
  )
243
241
 
244
242
 
@@ -316,7 +314,7 @@ def fix_special_locations(df: pl.LazyFrame, col: str, special_locations: gpd.Geo
316
314
  def identify_detailed_zone_id(
317
315
  special_locations: gpd.GeoDataFrame, detailed_zones: gpd.GeoDataFrame
318
316
  ):
319
- """Ads `detailed_zone_id` column to special_locations by finding the detailed zone in which the
317
+ """Adds `detailed_zone_id` column to special_locations by finding the detailed zone in which the
320
318
  special location falls.
321
319
  """
322
320
  orig_crs = special_locations.crs
@@ -15,6 +15,7 @@ ST_ID_COLUMNS = [
15
15
  "Ztir",
16
16
  "CODSECT",
17
17
  "DTIR_160",
18
+ "Secteur",
18
19
  ]
19
20
  INSEE_ID_COLUMNS = [
20
21
  "INSEE",
@@ -0,0 +1,43 @@
1
+ MODE_MAP = {
2
+ 1: "walking", # Marche à pied.
3
+ 10: "bicycle:driver:traditional:shared", # Conducteur VLS
4
+ 11: "bicycle:driver:traditional", # Conducteur de vélo.
5
+ 12: "bicycle:passenger", # Passager de vélo.
6
+ 13: "motorcycle:driver:moped", # Conducteur de deux ou trois roues motorisés < 50 cm3.
7
+ 14: "motorcycle:passenger:moped", # Passager de deux ou trois roues motorisés < 50 cm3.
8
+ 15: "motorcycle:driver:moto", # Conducteur de deux ou trois roues motorisés >= 50 cm3
9
+ 16: "motorcycle:passenger:moto", # Passager de deux ou trois roues motorisés >= 50 cm3.
10
+ 17: "bicycle:driver:electric", # Conducteur de vélo Assistance Electrique
11
+ 18: "bicycle:driver:electric:shared", # Conducteur de vélo Assistance Electrique en Libre Service
12
+ 19: "motorcycle:driver", # Conducteur de deux ou trois roues motorisés (si pas de détail sur la cylindrée).
13
+ 20: "motorcycle:passenger", # Passager de deux ou trois roues motorisés (si pas de détail sur la cylindrée)..
14
+ 21: "car:driver", # Conducteur de véhicule particulier (VP).
15
+ 22: "car:passenger", # Passager de véhicule particulier (VP).
16
+ 31: "public_transit:urban:bus", # Passager bus urbain (réseau ville centre).
17
+ 32: "public_transit:urban:tram", # Passager tramway (réseau ville centre).
18
+ 33: "public_transit:urban:metro", # Passager métro (réseau ville centre).
19
+ 34: "public_transit:urban:funicular", # Note. Not documented. Used in Le Havre (and other surveys?).
20
+ 37: "public_transit:urban:demand_responsive", # Transport à la demande (U ou IU)
21
+ 38: "public_transit:urban", # Passager autres réseaux urbains ds aire enquête.
22
+ 39: "public_transit:urban", # Passager autre réseau urbain hors AE
23
+ 41: "public_transit:urban:coach", # Passager transports interurbains routiers et autres autocars (TER routiers, lignes régulières départementales, scolaires, périscolaires, occasionnel….).
24
+ 42: "public_transit:interurban:coach", # Cars longues distances (Eurolines/Isilines, Ouibus, Flixbus…)).
25
+ 43: "public_transit:interurban:coach", # Passagers autocars anciennes définitions
26
+ 51: "public_transit:interurban:TGV", # Passager TGV
27
+ 52: "public_transit:urban:TER", # Passager train TER
28
+ 53: "public_transit:interurban:intercités", # Passagers autres trains (Intercité, TET)
29
+ 54: "public_transit:interurban:other_train", # Passager train non précisé
30
+ 61: "taxi", # Passager taxi.
31
+ 62: "VTC", # Passager VTC
32
+ 71: "employer_transport", # Transport employeur (exclusivement)
33
+ 72: "public_transit:school", # For Thionville 2012.
34
+ 81: "truck:driver", # Conducteur de fourgon, camionnette, camion (pour tournées professionnelles ou déplacements privés).
35
+ 82: "truck:passenger", # Passager de fourgon, camionnette, camion (pour tournées professionnelles ou déplacements privés)..
36
+ 91: "water_transport", # Transport Fluvial ou maritime.
37
+ 92: "airplane", # Avion.
38
+ 93: "personal_transporter:non_motorized", # Roller, skate, trottinette non électrique
39
+ 94: "wheelchair", # Fauteuil roulant.
40
+ 95: "other", # Autres modes (tracteur, engin agricole, quad, etc.).
41
+ 96: "personal_transporter:motorized", # Petits engins électriques (trottinette, segway, solowheel, etc)
42
+ 97: "personal_transporter:unspecified", # Roller, skate, trottinette électrique ou non (ancien code)
43
+ }
@@ -0,0 +1,160 @@
1
+ import polars as pl
2
+
3
+ from mobisurvstd.common.trips import clean
4
+
5
+ from .common import MODE_MAP
6
+
7
+ SCHEMA = {
8
+ "DP1": pl.UInt8, # Code fichier = 3 (déplacement)
9
+ "IDD3": pl.UInt16, # Année de fin d'enquête
10
+ "IDD4": pl.String, # Code Insee ville centre
11
+ "ZFD": pl.UInt32, # Zone fine de résidence
12
+ "ECH": pl.UInt32, # Numéro d’échantillon
13
+ "PER": pl.UInt8, # Numéro de personne
14
+ "NDEP": pl.UInt8, # Numéro de déplacement
15
+ "GD1": pl.String, # Insee Zone fine du lieu de résidence de la personne concernée par le déplacement
16
+ "STD": pl.UInt32, # Secteur de tirage dans l’enquête d’origine (résidence)
17
+ "D2A": pl.UInt8, # Motif Origine du déplacement
18
+ "D2B": pl.UInt8, # Motif Origine de la personne accompagnée
19
+ "D3": pl.UInt32, # Zone fine Origine du déplacement
20
+ "GDO1": pl.String, # Insee Zone fine Origine du déplacement
21
+ "STDO": pl.UInt32, # Secteur de tirage dans l’enquête d’origine (origine du déplacement)
22
+ "D4": pl.UInt16, # Heure de départ du déplacement
23
+ "D5A": pl.UInt8, # Motif Destination du déplacement
24
+ "D5B": pl.UInt8, # Motif Destination de la personne accompagnée
25
+ "D6": pl.UInt8, # Nombre d’arrêts sur la tournée
26
+ "D7": pl.UInt32, # Zone fine Destination du déplacement
27
+ "GDD1": pl.String, # Insee Zone fine Destination du déplacement
28
+ "STDD": pl.UInt32, # Secteur de tirage dans l’enquête d’origine (destination du déplacement)
29
+ "D8": pl.UInt16, # Heure d'arrivée du déplacement
30
+ "D9": pl.UInt16, # Durée du déplacement
31
+ "D10": pl.UInt8, # Nombre de trajets (en modes mécanisés)
32
+ "D11": pl.Float64, # Longueur à vol d'oiseau
33
+ "D12": pl.Float64, # Distance parcourue
34
+ "MODP": pl.UInt8, # Mode principal
35
+ "TYPD": pl.UInt8, # Type de déplacement
36
+ }
37
+
38
+ PURPOSE_MAP = {
39
+ 1: "home:main", # Domicile (partir de, se rendre à).
40
+ 2: "home:secondary", # Résidence secondaire, logement occasionnel, hôtel, autre domicile (partir de, se rendre à).
41
+ 11: "work:declared", # Travailler sur le lieu d’emploi déclaré.
42
+ 12: "work:telework", # Travailler sur un autre lieu – télétravail.
43
+ 13: "work:secondary", # Travailler sur un autre lieu hors télétravail
44
+ 14: "work:other", # Travailler sur un autre lieu sans distinction
45
+ # NOTE. We discard the information on the exact study location type (middle-school, high-school,
46
+ # etc.) because the information can read from the person characteristics (`student_group`).
47
+ 21: "education:childcare", # Être gardé (Nourrice, crèche...).
48
+ 22: "education:declared", # Étudier sur le lieu d'études déclaré (école maternelle et primaire).
49
+ 23: "education:declared", # Étudier sur le lieu d'études déclaré (collège).
50
+ 24: "education:declared", # Étudier sur le lieu d'études déclaré (lycée).
51
+ 25: "education:declared", # Étudier sur le lieu d'études déclaré (universités et grandes écoles).
52
+ 26: "education:other", # Étudier sur un autre lieu (école maternelle et primaire).
53
+ 27: "education:other", # Étudier sur un autre lieu (collège).
54
+ 28: "education:other", # Étudier sur un autre lieu (lycée).
55
+ 29: "education:other", # Étudier sur un autre lieu (universités et grandes écoles).
56
+ 30: "shopping:no_purchase", # Visite d’un magasin, d’un centre commercial ou d’un marché de plein vent sans effectuer d’achat
57
+ 31: "shopping:unspecified", # Réaliser plusieurs motifs en centre commercial.
58
+ 32: "shopping:unspecified", # Faire des achats en grand magasin, supermarché, hypermarché et leurs galeries marchandes.
59
+ 33: "shopping:unspecified", # Faire des achats en petit et moyen commerce et drive in
60
+ 34: "shopping:unspecified", # Faire des achats en marché couvert et de plein vent.
61
+ 35: "shopping:pickup", # Récupérer des achats faits à distance (Drive, points relais)
62
+ 41: "task:healthcare", # Recevoir des soins (santé).
63
+ 42: "task:procedure", # Faire une démarche autre que rechercher un emploi.
64
+ 43: "task:job_search", # Rechercher un emploi.
65
+ 51: "leisure:sport_or_culture", # Participer à des loisirs, des activités sportives, culturelles ou associatives.
66
+ 52: "leisure:walk_or_driving_lesson", # Faire une promenade, du « lèche-vitrines », prendre une leçon de conduite.
67
+ 53: "leisure:restaurant", # Se restaurer hors du domicile.
68
+ 54: "leisure:visiting", # Visiter des parents ou des amis.
69
+ # NOTE. We discard the information on whether the escorted person is present during pick_up /
70
+ # drop_off activity as it can be deduced (when I go pick-up someone, he's not there; when I come
71
+ # back from picking-up someone, he's there). Also, there are some errors in the data (like
72
+ # having the escorted person both before and after picking him up).
73
+ 61: "escort:activity:drop_off", # Accompagner quelqu’un (personne présente).
74
+ 62: "escort:activity:pick_up", # Aller chercher quelqu’un (personne présente).
75
+ 63: "escort:activity:drop_off", # Accompagner quelqu’un (personne absente).
76
+ 64: "escort:activity:pick_up", # Aller chercher quelqu’un (personne absente).
77
+ 68: "escort:unspecified:drop_off", # Accompagner quelqu’un ou déposer quelqu’un à un mode de transport (sans info présence personne)
78
+ 69: "escort:unspecified:pick_up", # Aller chercher quelqu’un ou reprendre quelqu’un à un mode de transport (sans info présence personne)
79
+ 71: "escort:transport:drop_off", # Déposer une personne à un mode de transport (personne présente).
80
+ 72: "escort:transport:pick_up", # Reprendre une personne à un mode de transport (personne présente).
81
+ 73: "escort:transport:drop_off", # Déposer d’une personne à un mode de transport (personne absente).
82
+ 74: "escort:transport:pick_up", # Reprendre une personne à un mode de transport( personne absente).
83
+ 81: "work:professional_tour", # Réaliser une tournée professionnelle.
84
+ 82: "shopping:tour_no_purchase", # Tournée de magasin sans achat
85
+ 91: "other", # Autres motifs
86
+ # 96: "escort:middle_high_school", # Étudier sur le lieu d'études déclaré (collège ou lycée). Cas egt personne accompagnée
87
+ # 97: "escort:middle_high_school:other", # Étudier sur un autre lieu (collège ou lycée). Cas egt personne accompagnée
88
+ # 98: "escort:shopping", # Faire des achats sans précision (egt, motif personne accompagnée)
89
+ # TODO: Fix 96, 97, and 98 if I find these values in a dataset.
90
+ }
91
+
92
+ SHOP_TYPE_MAP = {
93
+ 31: "mall", # Réaliser plusieurs motifs en centre commercial.
94
+ 32: "supermarket_or_hypermarket", # Faire des achats en grand magasin, supermarché, hypermarché et leurs galeries marchandes.
95
+ 33: "small_shop", # Faire des achats en petit et moyen commerce et drive in
96
+ 34: "market", # Faire des achats en marché couvert et de plein vent.
97
+ 35: "drive_in", # Récupérer des achats faits à distance (Drive, points relais)
98
+ }
99
+
100
+ TRIP_PERIMETER_MAP = {
101
+ 1: "internal", # interne au périmètre d'enquête
102
+ 2: "crossing", # en échange
103
+ 3: "external", # externe au périmètre d'enquête
104
+ 9: None, # inconnu (cas où origine ou destination inconnue)
105
+ }
106
+
107
+
108
+ def scan_trips(source: str | bytes):
109
+ lf = pl.scan_csv(source, separator=";", schema_overrides=SCHEMA, null_values=["aa"])
110
+ return lf
111
+
112
+
113
+ def standardize_trips(
114
+ source: str | bytes,
115
+ persons: pl.LazyFrame,
116
+ special_locations_coords: pl.DataFrame | None,
117
+ detailed_zones_coords: pl.DataFrame | None,
118
+ ):
119
+ lf = scan_trips(source)
120
+ # Add household_id, person_id, and trip date.
121
+ lf = lf.with_columns(original_person_id=pl.struct(ECH="ECH", STP="STD", PER="PER")).join(
122
+ persons.select("original_person_id", "person_id", "household_id", "trip_date"),
123
+ on="original_person_id",
124
+ how="left",
125
+ coalesce=True,
126
+ )
127
+ lf = lf.rename(
128
+ {
129
+ "D3": "origin_detailed_zone",
130
+ "GDO1": "origin_insee",
131
+ "STDO": "origin_draw_zone",
132
+ "D6": "nb_tour_stops",
133
+ "D7": "destination_detailed_zone",
134
+ "GDD1": "destination_insee",
135
+ "STDD": "destination_draw_zone",
136
+ }
137
+ )
138
+ lf = lf.with_columns(
139
+ original_trip_id=pl.struct("ECH", "STD", "PER", "NDEP"),
140
+ origin_purpose=pl.col("D2A").replace_strict(PURPOSE_MAP),
141
+ origin_escort_purpose=pl.col("D2B").replace_strict(PURPOSE_MAP),
142
+ origin_shop_type=pl.col("D2A").replace(SHOP_TYPE_MAP, default=None),
143
+ departure_time=60 * (pl.col("D4") // 100) + pl.col("D4") % 100,
144
+ destination_purpose=pl.col("D5A").replace_strict(PURPOSE_MAP),
145
+ destination_escort_purpose=pl.col("D5B").replace_strict(PURPOSE_MAP),
146
+ destination_shop_type=pl.col("D5A").replace(SHOP_TYPE_MAP, default=None),
147
+ arrival_time=60 * (pl.col("D8") // 100) + pl.col("D8") % 100,
148
+ trip_euclidean_distance_km=pl.col("D11") / 1e3,
149
+ trip_travel_distance_km=pl.col("D12") / 1e3,
150
+ main_mode=pl.col("MODP").replace_strict(MODE_MAP),
151
+ trip_perimeter=pl.col("TYPD").replace_strict(TRIP_PERIMETER_MAP),
152
+ )
153
+ year = int(lf.select(pl.col("trip_date").dt.year().mean().round()).collect().item())
154
+ lf = clean(
155
+ lf,
156
+ year=year,
157
+ special_locations=special_locations_coords,
158
+ detailed_zones=detailed_zones_coords,
159
+ )
160
+ return lf