nemobi 2.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. nemobi-2.0.0/LICENSE.md +7 -0
  2. nemobi-2.0.0/PKG-INFO +256 -0
  3. nemobi-2.0.0/README.md +226 -0
  4. nemobi-2.0.0/pyproject.toml +72 -0
  5. nemobi-2.0.0/setup.cfg +4 -0
  6. nemobi-2.0.0/src/nemobi/__init__.py +9 -0
  7. nemobi-2.0.0/src/nemobi/__main__.py +4 -0
  8. nemobi-2.0.0/src/nemobi/cerema/__init__.py +26 -0
  9. nemobi-2.0.0/src/nemobi/cerema/common.py +76 -0
  10. nemobi-2.0.0/src/nemobi/cerema/deplacements.py +254 -0
  11. nemobi-2.0.0/src/nemobi/cerema/edgt.py +254 -0
  12. nemobi-2.0.0/src/nemobi/cerema/edvm.py +269 -0
  13. nemobi-2.0.0/src/nemobi/cerema/emc2.py +141 -0
  14. nemobi-2.0.0/src/nemobi/cerema/emd.py +128 -0
  15. nemobi-2.0.0/src/nemobi/cerema/menages.py +391 -0
  16. nemobi-2.0.0/src/nemobi/cerema/nantes_opendata.py +207 -0
  17. nemobi-2.0.0/src/nemobi/cerema/personnes.py +532 -0
  18. nemobi-2.0.0/src/nemobi/cerema/reader.py +160 -0
  19. nemobi-2.0.0/src/nemobi/cerema/survey.py +451 -0
  20. nemobi-2.0.0/src/nemobi/cerema/trajets.py +423 -0
  21. nemobi-2.0.0/src/nemobi/cerema/zones.py +406 -0
  22. nemobi-2.0.0/src/nemobi/classes.py +341 -0
  23. nemobi-2.0.0/src/nemobi/cli.py +103 -0
  24. nemobi-2.0.0/src/nemobi/common/__init__.py +1 -0
  25. nemobi-2.0.0/src/nemobi/common/cars.py +119 -0
  26. nemobi-2.0.0/src/nemobi/common/clean.py +559 -0
  27. nemobi-2.0.0/src/nemobi/common/households.py +99 -0
  28. nemobi-2.0.0/src/nemobi/common/legs.py +134 -0
  29. nemobi-2.0.0/src/nemobi/common/motorcycles.py +41 -0
  30. nemobi-2.0.0/src/nemobi/common/persons.py +180 -0
  31. nemobi-2.0.0/src/nemobi/common/trips.py +265 -0
  32. nemobi-2.0.0/src/nemobi/common/zones.py +37 -0
  33. nemobi-2.0.0/src/nemobi/egt2010/__init__.py +1 -0
  34. nemobi-2.0.0/src/nemobi/egt2010/deplacements.py +227 -0
  35. nemobi-2.0.0/src/nemobi/egt2010/menages.py +403 -0
  36. nemobi-2.0.0/src/nemobi/egt2010/personnes.py +350 -0
  37. nemobi-2.0.0/src/nemobi/egt2010/survey.py +92 -0
  38. nemobi-2.0.0/src/nemobi/egt2010/trajets.py +231 -0
  39. nemobi-2.0.0/src/nemobi/egt2010/zones.py +11 -0
  40. nemobi-2.0.0/src/nemobi/egt2020/__init__.py +1 -0
  41. nemobi-2.0.0/src/nemobi/egt2020/deplacements.py +203 -0
  42. nemobi-2.0.0/src/nemobi/egt2020/menages.py +168 -0
  43. nemobi-2.0.0/src/nemobi/egt2020/motos.py +98 -0
  44. nemobi-2.0.0/src/nemobi/egt2020/personnes.py +405 -0
  45. nemobi-2.0.0/src/nemobi/egt2020/survey.py +94 -0
  46. nemobi-2.0.0/src/nemobi/egt2020/trajets.py +347 -0
  47. nemobi-2.0.0/src/nemobi/egt2020/voitures.py +131 -0
  48. nemobi-2.0.0/src/nemobi/emg2023/__init__.py +1 -0
  49. nemobi-2.0.0/src/nemobi/emg2023/deplacements.py +255 -0
  50. nemobi-2.0.0/src/nemobi/emg2023/menages.py +24 -0
  51. nemobi-2.0.0/src/nemobi/emg2023/personnes.py +170 -0
  52. nemobi-2.0.0/src/nemobi/emg2023/survey.py +66 -0
  53. nemobi-2.0.0/src/nemobi/emp/__init__.py +1 -0
  54. nemobi-2.0.0/src/nemobi/emp/common.py +31 -0
  55. nemobi-2.0.0/src/nemobi/emp/deplacements.py +403 -0
  56. nemobi-2.0.0/src/nemobi/emp/menages.py +190 -0
  57. nemobi-2.0.0/src/nemobi/emp/motos.py +105 -0
  58. nemobi-2.0.0/src/nemobi/emp/personnes.py +657 -0
  59. nemobi-2.0.0/src/nemobi/emp/survey.py +110 -0
  60. nemobi-2.0.0/src/nemobi/emp/voitures.py +184 -0
  61. nemobi-2.0.0/src/nemobi/logger.py +26 -0
  62. nemobi-2.0.0/src/nemobi/main.py +257 -0
  63. nemobi-2.0.0/src/nemobi/resources/__init__.py +42 -0
  64. nemobi-2.0.0/src/nemobi/resources/admin_express.py +96 -0
  65. nemobi-2.0.0/src/nemobi/resources/insee_data.py +446 -0
  66. nemobi-2.0.0/src/nemobi/resources/nuts.py +132 -0
  67. nemobi-2.0.0/src/nemobi/schema/__init__.py +12 -0
  68. nemobi-2.0.0/src/nemobi/schema/cars.py +95 -0
  69. nemobi-2.0.0/src/nemobi/schema/common.py +165 -0
  70. nemobi-2.0.0/src/nemobi/schema/guarantees.py +735 -0
  71. nemobi-2.0.0/src/nemobi/schema/households.py +183 -0
  72. nemobi-2.0.0/src/nemobi/schema/legs.py +277 -0
  73. nemobi-2.0.0/src/nemobi/schema/motorcycles.py +70 -0
  74. nemobi-2.0.0/src/nemobi/schema/persons.py +897 -0
  75. nemobi-2.0.0/src/nemobi/schema/trips.py +484 -0
  76. nemobi-2.0.0/src/nemobi/utils.py +275 -0
  77. nemobi-2.0.0/src/nemobi/validation.py +795 -0
  78. nemobi-2.0.0/src/nemobi.egg-info/PKG-INFO +256 -0
  79. nemobi-2.0.0/src/nemobi.egg-info/SOURCES.txt +81 -0
  80. nemobi-2.0.0/src/nemobi.egg-info/dependency_links.txt +1 -0
  81. nemobi-2.0.0/src/nemobi.egg-info/entry_points.txt +2 -0
  82. nemobi-2.0.0/src/nemobi.egg-info/requires.txt +11 -0
  83. nemobi-2.0.0/src/nemobi.egg-info/top_level.txt +1 -0
@@ -0,0 +1,7 @@
1
+ Copyright © 2025 Lucas Javaudin
2
+
3
+ Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
4
+
5
+ The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
6
+
7
+ THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
nemobi-2.0.0/PKG-INFO ADDED
@@ -0,0 +1,256 @@
1
+ Metadata-Version: 2.4
2
+ Name: nemobi
3
+ Version: 2.0.0
4
+ Summary: Nemobi (Normalisation des enquêtes mobilités): a tool to standardize French mobility survey datasets (EMC², EGT, EMP, etc.) into a unified format.
5
+ Author-email: Lucas Javaudin <lucas@lucasjavaudin.com>
6
+ License-Expression: MIT
7
+ Project-URL: Homepage, https://github.com/LucasJavaudin/Nemobi
8
+ Project-URL: Documentation, https://nemobi.lucasjavaudin.com/index.html
9
+ Project-URL: Repository, https://github.com/LucasJavaudin/Nemobi
10
+ Project-URL: Issues, https://github.com/LucasJavaudin/Nemobi/issues
11
+ Keywords: mobility survey,travel survey,France,polars,data format
12
+ Classifier: Programming Language :: Python
13
+ Classifier: Programming Language :: Python :: 3
14
+ Classifier: Topic :: File Formats
15
+ Requires-Python: >=3.11
16
+ Description-Content-Type: text/markdown
17
+ License-File: LICENSE.md
18
+ Requires-Dist: geopandas>=1.0.1
19
+ Requires-Dist: humanize>=4.13.0
20
+ Requires-Dist: loguru>=0.7.3
21
+ Requires-Dist: pandas>=2.2.3
22
+ Requires-Dist: platformdirs>=4.3.8
23
+ Requires-Dist: polars[calamine]>=2.0.0
24
+ Requires-Dist: py7zr>=1.1.0
25
+ Requires-Dist: pyarrow>=19.0.1
26
+ Requires-Dist: requests>=2.32.3
27
+ Requires-Dist: tqdm>=4.67.3
28
+ Requires-Dist: typer>=0.16.0
29
+ Dynamic: license-file
30
+
31
+ [![Contributors][contributors-shield]][contributors-url]
32
+ [![Forks][forks-shield]][forks-url]
33
+ [![Stargazers][stars-shield]][stars-url]
34
+ [![Issues][issues-shield]][issues-url]
35
+
36
+ [![GPL v3][license-shield]][license-url]
37
+ [![PyPI][pypi-shield]][pypi-url]
38
+ [![DOI][doi-shield]][doi-url]
39
+
40
+ # Nemobi
41
+
42
+ > [!NOTE]
43
+ > **MobiSurvStd is now Nemobi.** Starting with v2.0.0, the package is published as `nemobi`
44
+ > (`pip install nemobi`, `import nemobi`, CLI `nemobi`). Surveys standardized with MobiSurvStd can
45
+ > still be read by Nemobi. See the
46
+ > [migration guide](https://nemobi.lucasjavaudin.com/migration.html).
47
+
48
+ *A tool to standardize French mobility survey datasets (EMC², EGT, EMP, etc.) into a unified format.*
49
+
50
+ 📚 [Documentation](https://nemobi.lucasjavaudin.com)
51
+
52
+ <span style="color:blue">🇫🇷 [Version française](./README.fr.md)</span>
53
+
54
+ ## Table of Contents
55
+ - [Introduction](#introduction)
56
+ - [Mobility Surveys in France](#mobility-surveys-in-france)
57
+ - [Why Nemobi?](#why-nemobi)
58
+ - [Installation](#installation)
59
+ - [Getting Started](#getting-started)
60
+ - [Supported Surveys](#supported-surveys)
61
+ - [Legal Notice](#legal-notice)
62
+ - [Contributing](#contributing)
63
+
64
+ ## Introduction
65
+
66
+ Nemobi (*Normalisation des enquêtes mobilités*, i.e., standardization of mobility surveys) is an easy-to-use Python command line interface to convert many French mobility surveys
67
+ (EMC², EGT, EMP, etc.) to a unique standardized and clean format.
68
+
69
+ ---
70
+
71
+ ## Mobility Surveys in France
72
+
73
+ In France, despite recent efforts by CEREMA to create a standard format for mobility surveys – with
74
+ the EMC² surveys – various formats co-exist:
75
+
76
+ - [**EMC²**](https://www.cerema.fr/fr/activites/mobilites/connaissance-modelisation-evaluation-mobilite/enquetes-mobilite-emc2)
77
+ (Enquête mobilité certifiée CEREMA): mobility surveys for many French territories (since 2018);
78
+ - [**EGT H2020**](https://omnil.fr/actualites/les-resultats-de-la-derniere-enquete-globale-transport)
79
+ (Enquête Globale Transport, Île-de-France Mobilités): mobility survey for Île-de-France (2018–2020;
80
+ incomplete due to COVID-19);
81
+ - [**EMP**](https://www.statistiques.developpement-durable.gouv.fr/resultats-detailles-de-lenquete-mobilite-des-personnes-de-2019)
82
+ (Enquête mobilité des personnes, SDES): national mobility survey (2019).
83
+ - [**EMG**](https://www.institutparisregion.fr/mobilite-et-transports/deplacements/enquete-regionale-sur-la-mobilite-des-franciliens/)
84
+ (Enquête Mobilité par GPS, Institut Paris Région): mobility survey for Île-de-France (2022-2023)
85
+ with individuals tracked over several days through GPS.
86
+
87
+ Also surveys based on previous formats are still in use today:
88
+
89
+ - **EDVM** (Enquêtes Déplacements Villes Moyennes, CEREMA): mobility surveys for medium-size cities
90
+ (until 2018);
91
+ - **EDGT** (Enquêtes Déplacements Grands Territoires, CEREMA): mobility surveys for periphery areas
92
+ (until 2018);
93
+ - **EMD** (Enquêtes Ménages Déplacements, CEREMA): other mobility surveys from CEREMA (until 2018);
94
+ - [**EGT 2010**](https://omnil.fr/egt-2010) (Enquête Globale Transport, Île-de-France Mobilités):
95
+ previous version of the Île-de-France mobility survey;
96
+ - [**ENTD**](https://www.statistiques.developpement-durable.gouv.fr/enquete-nationale-transports-et-deplacements-entd-2008)
97
+ (Enquête nationale transports et déplacements, SDES): former national mobility survey (2008).
98
+
99
+ ---
100
+
101
+ ## Why Nemobi?
102
+
103
+ The existing formats all have the same drawbacks:
104
+
105
+ - Data are sorted in CSV files which are not always straightforward to read (Which separator?
106
+ Which encoding? What are the variable datatypes?).
107
+ - Variable names and modalities are not always clear (e.g., in the EMC² format, variable "P2"
108
+ represents the gender of the person, with modality 1 for a man and 2 for a woman).
109
+ - Joining two datasets is hard and not well documented (e.g., in the EMC² format, to join the
110
+ persons with their household, the variables to use are "METH", "ZFM" and "ECH" for the households
111
+ and "DMET", "ZFD" and "ECH" for the persons).
112
+
113
+ Additionally, when working with different territories / periods, it is often necessary to write a
114
+ similar code multiple times due to the existence of different formats.
115
+
116
+ Nemobi solves all these issues by being able to convert all survey formats to a [well-defined
117
+ Parquet format](https://nemobi.lucasjavaudin.com/format/index.html).
118
+
119
+ See [this example](https://nemobi.lucasjavaudin.com/problem-example.html) to understand how
120
+ Nemobi can simplify your workflow.
121
+
122
+ ---
123
+
124
+ ## Installation
125
+
126
+ Install the library with
127
+
128
+ ```bash
129
+ pip install nemobi
130
+ ```
131
+
132
+ ---
133
+
134
+ ## Getting Started
135
+
136
+ Download the
137
+ [EMP 2019 survey](https://www.statistiques.developpement-durable.gouv.fr/resultats-detailles-de-lenquete-mobilite-des-personnes-de-2019)
138
+ _(Données individuelles anonymisées (fichiers au format CSV))_, then run the following command:
139
+
140
+ ```bash
141
+ nemobi emp_2019_donnees_individuelles_anonymisees_novembre2024.zip standardized_emp2019
142
+ ```
143
+
144
+ It will standardize the EMP 2019 survey and save the resulting Parquet files in the
145
+ `standardized_emp2019` directory.
146
+ These Parquet files can then be analyzed using e.g., [polars](https://pola.rs/) or
147
+ [pandas](https://pandas.pydata.org) in Python or [arrow](https://arrow.apache.org/docs/r/) in R.
148
+
149
+ **Note:** During standardization, Nemobi may issue warnings when certain variable guarantees
150
+ are not initially satisfied but are automatically fixed.
151
+ These warnings can usually be safely ignored, unless they pertain to a large number of observations.
152
+
153
+ A detailed definition of the Parquet format used by Nemobi is available
154
+ [here](https://nemobi.lucasjavaudin.com/format/index.html).
155
+
156
+ You can also standardize surveys programmatically in Python with the `standardize` function:
157
+
158
+ ```python
159
+ import nemobi
160
+ nemobi.standardize(
161
+ "emp_2019_donnees_individuelles_anonymisees_novembre2024.zip",
162
+ "standardized_emp2019",
163
+ )
164
+ ```
165
+
166
+ For more, check the [User Guide](https://nemobi.lucasjavaudin.com/howto.html).
167
+
168
+ **Citing Nemobi**
169
+
170
+ If you use Nemobi in a publication or project, please include the following citation:
171
+ ```
172
+ Javaudin, L. (2025). Nemobi. Zenodo. https://doi.org/10.5281/zenodo.17710203
173
+ ```
174
+ A [CITATION.bib](CITATION.bib) is also available.
175
+
176
+ ---
177
+
178
+ ## Case Study: Bicycle Use
179
+
180
+ The following graph represents the share of bicycle trips for EMC², EDGT, EDVM, EMD, and EGT
181
+ surveys.
182
+ The circle colors represent the average number of bicycles in the surveyed households.
183
+ The circle sizes represent the expected number of trips in the surveyed area.
184
+
185
+ The graph has been generated from the code in
186
+ [analyses/bicycle_shares.py](analyses/bicycle_shares.py).
187
+
188
+ ![Graph of bicycle share by survey](https://raw.githubusercontent.com/LucasJavaudin/Nemobi/main/docs/src/images/bicycle_shares.png)
189
+
190
+ The following map represents the share of bicycle trips within INSEE municipalities.
191
+ Only municipalities with more than 30 surveyed trips are shown.
192
+
193
+ The map has been generated from the code in
194
+ [analyses/bicycle_shares_by_insee.py](analyses/bicycle_shares_by_insee.py).
195
+
196
+ ![Map of bicycle share by INSEE municipality](https://raw.githubusercontent.com/LucasJavaudin/Nemobi/main/docs/src/images/bicycle_shares_by_insee.png)
197
+
198
+ ---
199
+
200
+ ## Supported Surveys
201
+
202
+ Currently, Nemobi supports the following survey types:
203
+
204
+ - `emp2019`
205
+ - `emc2`
206
+ - `emg2023` (thanks to @sebhorl)
207
+ - `egt2020`
208
+ - `egt2010`
209
+ - `edgt` (including the [open-data version of Nantes 2015](https://www.data.gouv.fr/datasets/enquete-deplacements-2015-en-loire-atlantique))
210
+ - `edvm`
211
+ - `emd`
212
+
213
+ See [Survey Types](https://nemobi.lucasjavaudin.com/surveys.html) for more.
214
+
215
+ ![Map of all local surveys tested with Nemobi](https://raw.githubusercontent.com/LucasJavaudin/Nemobi/main/docs/src/images/survey_map.jpg)
216
+
217
+ Other survey types that are planned to be integrated are: `entd`.
218
+ If you know another survey format that could be integrated, feel free to open an issue on GitHub.
219
+
220
+ Nemobi covers only French mobility survey formats.
221
+ If other countries have similar survey formats, they might be easily integrated into Nemobi (or
222
+ a variant of it).
223
+
224
+ ---
225
+
226
+ ## Legal Notice
227
+
228
+ <span style="color:red">
229
+ ⚠️ <strong>Nemobi does not anonymize the data.</strong>
230
+ If you are working with confidential datasets (e.g., EMC² or EGT surveys), you must apply the same
231
+ confidentiality rules to the standardized data as to the original data.
232
+ In particular, <strong>you must not share the standardized data</strong> if your confidentiality
233
+ agreement prohibits sharing the original data.
234
+ </span>
235
+
236
+ ---
237
+
238
+ ## Contributing
239
+
240
+ If you think you found a bug, if you have a suggestion, or if you want to integrate a new format,
241
+ feel free to open an issue on GitHub or to create a Pull Request.
242
+
243
+ [contributors-shield]: https://img.shields.io/github/contributors/LucasJavaudin/Nemobi.svg?style=for-the-badge
244
+ [contributors-url]: https://github.com/LucasJavaudin/Nemobi/graphs/contributors
245
+ [forks-shield]: https://img.shields.io/github/forks/LucasJavaudin/Nemobi.svg?style=for-the-badge
246
+ [forks-url]: https://github.com/LucasJavaudin/Nemobi/network/members
247
+ [stars-shield]: https://img.shields.io/github/stars/LucasJavaudin/Nemobi.svg?style=for-the-badge
248
+ [stars-url]: https://github.com/LucasJavaudin/Nemobi/stargazers
249
+ [issues-shield]: https://img.shields.io/github/issues/LucasJavaudin/Nemobi.svg?style=for-the-badge
250
+ [issues-url]: https://github.com/LucasJavaudin/Nemobi/issues
251
+ [license-shield]: https://img.shields.io/github/license/LucasJavaudin/Nemobi.svg?style=for-the-badge
252
+ [license-url]: https://github.com/LucasJavaudin/Nemobi/blob/master/LICENSE.txt
253
+ [doi-shield]: https://img.shields.io/badge/doi-10.5281/zenodo.17422994-blue?style=for-the-badge
254
+ [doi-url]: https://doi.org/10.5281/zenodo.17422993
255
+ [pypi-shield]: https://img.shields.io/pypi/v/nemobi.svg?style=for-the-badge
256
+ [pypi-url]: https://pypi.org/project/nemobi/
nemobi-2.0.0/README.md ADDED
@@ -0,0 +1,226 @@
1
+ [![Contributors][contributors-shield]][contributors-url]
2
+ [![Forks][forks-shield]][forks-url]
3
+ [![Stargazers][stars-shield]][stars-url]
4
+ [![Issues][issues-shield]][issues-url]
5
+
6
+ [![GPL v3][license-shield]][license-url]
7
+ [![PyPI][pypi-shield]][pypi-url]
8
+ [![DOI][doi-shield]][doi-url]
9
+
10
+ # Nemobi
11
+
12
+ > [!NOTE]
13
+ > **MobiSurvStd is now Nemobi.** Starting with v2.0.0, the package is published as `nemobi`
14
+ > (`pip install nemobi`, `import nemobi`, CLI `nemobi`). Surveys standardized with MobiSurvStd can
15
+ > still be read by Nemobi. See the
16
+ > [migration guide](https://nemobi.lucasjavaudin.com/migration.html).
17
+
18
+ *A tool to standardize French mobility survey datasets (EMC², EGT, EMP, etc.) into a unified format.*
19
+
20
+ 📚 [Documentation](https://nemobi.lucasjavaudin.com)
21
+
22
+ <span style="color:blue">🇫🇷 [Version française](./README.fr.md)</span>
23
+
24
+ ## Table of Contents
25
+ - [Introduction](#introduction)
26
+ - [Mobility Surveys in France](#mobility-surveys-in-france)
27
+ - [Why Nemobi?](#why-nemobi)
28
+ - [Installation](#installation)
29
+ - [Getting Started](#getting-started)
30
+ - [Supported Surveys](#supported-surveys)
31
+ - [Legal Notice](#legal-notice)
32
+ - [Contributing](#contributing)
33
+
34
+ ## Introduction
35
+
36
+ Nemobi (*Normalisation des enquêtes mobilités*, i.e., standardization of mobility surveys) is an easy-to-use Python command line interface to convert many French mobility surveys
37
+ (EMC², EGT, EMP, etc.) to a unique standardized and clean format.
38
+
39
+ ---
40
+
41
+ ## Mobility Surveys in France
42
+
43
+ In France, despite recent efforts by CEREMA to create a standard format for mobility surveys – with
44
+ the EMC² surveys – various formats co-exist:
45
+
46
+ - [**EMC²**](https://www.cerema.fr/fr/activites/mobilites/connaissance-modelisation-evaluation-mobilite/enquetes-mobilite-emc2)
47
+ (Enquête mobilité certifiée CEREMA): mobility surveys for many French territories (since 2018);
48
+ - [**EGT H2020**](https://omnil.fr/actualites/les-resultats-de-la-derniere-enquete-globale-transport)
49
+ (Enquête Globale Transport, Île-de-France Mobilités): mobility survey for Île-de-France (2018–2020;
50
+ incomplete due to COVID-19);
51
+ - [**EMP**](https://www.statistiques.developpement-durable.gouv.fr/resultats-detailles-de-lenquete-mobilite-des-personnes-de-2019)
52
+ (Enquête mobilité des personnes, SDES): national mobility survey (2019).
53
+ - [**EMG**](https://www.institutparisregion.fr/mobilite-et-transports/deplacements/enquete-regionale-sur-la-mobilite-des-franciliens/)
54
+ (Enquête Mobilité par GPS, Institut Paris Région): mobility survey for Île-de-France (2022-2023)
55
+ with individuals tracked over several days through GPS.
56
+
57
+ Also surveys based on previous formats are still in use today:
58
+
59
+ - **EDVM** (Enquêtes Déplacements Villes Moyennes, CEREMA): mobility surveys for medium-size cities
60
+ (until 2018);
61
+ - **EDGT** (Enquêtes Déplacements Grands Territoires, CEREMA): mobility surveys for periphery areas
62
+ (until 2018);
63
+ - **EMD** (Enquêtes Ménages Déplacements, CEREMA): other mobility surveys from CEREMA (until 2018);
64
+ - [**EGT 2010**](https://omnil.fr/egt-2010) (Enquête Globale Transport, Île-de-France Mobilités):
65
+ previous version of the Île-de-France mobility survey;
66
+ - [**ENTD**](https://www.statistiques.developpement-durable.gouv.fr/enquete-nationale-transports-et-deplacements-entd-2008)
67
+ (Enquête nationale transports et déplacements, SDES): former national mobility survey (2008).
68
+
69
+ ---
70
+
71
+ ## Why Nemobi?
72
+
73
+ The existing formats all have the same drawbacks:
74
+
75
+ - Data are sorted in CSV files which are not always straightforward to read (Which separator?
76
+ Which encoding? What are the variable datatypes?).
77
+ - Variable names and modalities are not always clear (e.g., in the EMC² format, variable "P2"
78
+ represents the gender of the person, with modality 1 for a man and 2 for a woman).
79
+ - Joining two datasets is hard and not well documented (e.g., in the EMC² format, to join the
80
+ persons with their household, the variables to use are "METH", "ZFM" and "ECH" for the households
81
+ and "DMET", "ZFD" and "ECH" for the persons).
82
+
83
+ Additionally, when working with different territories / periods, it is often necessary to write a
84
+ similar code multiple times due to the existence of different formats.
85
+
86
+ Nemobi solves all these issues by being able to convert all survey formats to a [well-defined
87
+ Parquet format](https://nemobi.lucasjavaudin.com/format/index.html).
88
+
89
+ See [this example](https://nemobi.lucasjavaudin.com/problem-example.html) to understand how
90
+ Nemobi can simplify your workflow.
91
+
92
+ ---
93
+
94
+ ## Installation
95
+
96
+ Install the library with
97
+
98
+ ```bash
99
+ pip install nemobi
100
+ ```
101
+
102
+ ---
103
+
104
+ ## Getting Started
105
+
106
+ Download the
107
+ [EMP 2019 survey](https://www.statistiques.developpement-durable.gouv.fr/resultats-detailles-de-lenquete-mobilite-des-personnes-de-2019)
108
+ _(Données individuelles anonymisées (fichiers au format CSV))_, then run the following command:
109
+
110
+ ```bash
111
+ nemobi emp_2019_donnees_individuelles_anonymisees_novembre2024.zip standardized_emp2019
112
+ ```
113
+
114
+ It will standardize the EMP 2019 survey and save the resulting Parquet files in the
115
+ `standardized_emp2019` directory.
116
+ These Parquet files can then be analyzed using e.g., [polars](https://pola.rs/) or
117
+ [pandas](https://pandas.pydata.org) in Python or [arrow](https://arrow.apache.org/docs/r/) in R.
118
+
119
+ **Note:** During standardization, Nemobi may issue warnings when certain variable guarantees
120
+ are not initially satisfied but are automatically fixed.
121
+ These warnings can usually be safely ignored, unless they pertain to a large number of observations.
122
+
123
+ A detailed definition of the Parquet format used by Nemobi is available
124
+ [here](https://nemobi.lucasjavaudin.com/format/index.html).
125
+
126
+ You can also standardize surveys programmatically in Python with the `standardize` function:
127
+
128
+ ```python
129
+ import nemobi
130
+ nemobi.standardize(
131
+ "emp_2019_donnees_individuelles_anonymisees_novembre2024.zip",
132
+ "standardized_emp2019",
133
+ )
134
+ ```
135
+
136
+ For more, check the [User Guide](https://nemobi.lucasjavaudin.com/howto.html).
137
+
138
+ **Citing Nemobi**
139
+
140
+ If you use Nemobi in a publication or project, please include the following citation:
141
+ ```
142
+ Javaudin, L. (2025). Nemobi. Zenodo. https://doi.org/10.5281/zenodo.17710203
143
+ ```
144
+ A [CITATION.bib](CITATION.bib) is also available.
145
+
146
+ ---
147
+
148
+ ## Case Study: Bicycle Use
149
+
150
+ The following graph represents the share of bicycle trips for EMC², EDGT, EDVM, EMD, and EGT
151
+ surveys.
152
+ The circle colors represent the average number of bicycles in the surveyed households.
153
+ The circle sizes represent the expected number of trips in the surveyed area.
154
+
155
+ The graph has been generated from the code in
156
+ [analyses/bicycle_shares.py](analyses/bicycle_shares.py).
157
+
158
+ ![Graph of bicycle share by survey](https://raw.githubusercontent.com/LucasJavaudin/Nemobi/main/docs/src/images/bicycle_shares.png)
159
+
160
+ The following map represents the share of bicycle trips within INSEE municipalities.
161
+ Only municipalities with more than 30 surveyed trips are shown.
162
+
163
+ The map has been generated from the code in
164
+ [analyses/bicycle_shares_by_insee.py](analyses/bicycle_shares_by_insee.py).
165
+
166
+ ![Map of bicycle share by INSEE municipality](https://raw.githubusercontent.com/LucasJavaudin/Nemobi/main/docs/src/images/bicycle_shares_by_insee.png)
167
+
168
+ ---
169
+
170
+ ## Supported Surveys
171
+
172
+ Currently, Nemobi supports the following survey types:
173
+
174
+ - `emp2019`
175
+ - `emc2`
176
+ - `emg2023` (thanks to @sebhorl)
177
+ - `egt2020`
178
+ - `egt2010`
179
+ - `edgt` (including the [open-data version of Nantes 2015](https://www.data.gouv.fr/datasets/enquete-deplacements-2015-en-loire-atlantique))
180
+ - `edvm`
181
+ - `emd`
182
+
183
+ See [Survey Types](https://nemobi.lucasjavaudin.com/surveys.html) for more.
184
+
185
+ ![Map of all local surveys tested with Nemobi](https://raw.githubusercontent.com/LucasJavaudin/Nemobi/main/docs/src/images/survey_map.jpg)
186
+
187
+ Other survey types that are planned to be integrated are: `entd`.
188
+ If you know another survey format that could be integrated, feel free to open an issue on GitHub.
189
+
190
+ Nemobi covers only French mobility survey formats.
191
+ If other countries have similar survey formats, they might be easily integrated into Nemobi (or
192
+ a variant of it).
193
+
194
+ ---
195
+
196
+ ## Legal Notice
197
+
198
+ <span style="color:red">
199
+ ⚠️ <strong>Nemobi does not anonymize the data.</strong>
200
+ If you are working with confidential datasets (e.g., EMC² or EGT surveys), you must apply the same
201
+ confidentiality rules to the standardized data as to the original data.
202
+ In particular, <strong>you must not share the standardized data</strong> if your confidentiality
203
+ agreement prohibits sharing the original data.
204
+ </span>
205
+
206
+ ---
207
+
208
+ ## Contributing
209
+
210
+ If you think you found a bug, if you have a suggestion, or if you want to integrate a new format,
211
+ feel free to open an issue on GitHub or to create a Pull Request.
212
+
213
+ [contributors-shield]: https://img.shields.io/github/contributors/LucasJavaudin/Nemobi.svg?style=for-the-badge
214
+ [contributors-url]: https://github.com/LucasJavaudin/Nemobi/graphs/contributors
215
+ [forks-shield]: https://img.shields.io/github/forks/LucasJavaudin/Nemobi.svg?style=for-the-badge
216
+ [forks-url]: https://github.com/LucasJavaudin/Nemobi/network/members
217
+ [stars-shield]: https://img.shields.io/github/stars/LucasJavaudin/Nemobi.svg?style=for-the-badge
218
+ [stars-url]: https://github.com/LucasJavaudin/Nemobi/stargazers
219
+ [issues-shield]: https://img.shields.io/github/issues/LucasJavaudin/Nemobi.svg?style=for-the-badge
220
+ [issues-url]: https://github.com/LucasJavaudin/Nemobi/issues
221
+ [license-shield]: https://img.shields.io/github/license/LucasJavaudin/Nemobi.svg?style=for-the-badge
222
+ [license-url]: https://github.com/LucasJavaudin/Nemobi/blob/master/LICENSE.txt
223
+ [doi-shield]: https://img.shields.io/badge/doi-10.5281/zenodo.17422994-blue?style=for-the-badge
224
+ [doi-url]: https://doi.org/10.5281/zenodo.17422993
225
+ [pypi-shield]: https://img.shields.io/pypi/v/nemobi.svg?style=for-the-badge
226
+ [pypi-url]: https://pypi.org/project/nemobi/
@@ -0,0 +1,72 @@
1
+ # pyproject.toml
2
+
3
+ [build-system]
4
+ requires = ["setuptools>=61.0.0", "wheel"]
5
+ build-backend = "setuptools.build_meta"
6
+
7
+ [project]
8
+ name = "nemobi"
9
+ version = "2.0.0"
10
+ description = "Nemobi (Normalisation des enquêtes mobilités): a tool to standardize French mobility survey datasets (EMC², EGT, EMP, etc.) into a unified format."
11
+ readme = "README.md"
12
+ authors = [{ name = "Lucas Javaudin", email = "lucas@lucasjavaudin.com" }]
13
+ license = "MIT"
14
+ license-files = ["LICENSE.md"]
15
+ classifiers = [
16
+ "Programming Language :: Python",
17
+ "Programming Language :: Python :: 3",
18
+ "Topic :: File Formats",
19
+ ]
20
+ keywords = ["mobility survey", "travel survey", "France", "polars", "data format"]
21
+ dependencies = [
22
+ "geopandas>=1.0.1",
23
+ "humanize>=4.13.0",
24
+ "loguru>=0.7.3",
25
+ "pandas>=2.2.3",
26
+ "platformdirs>=4.3.8",
27
+ "polars[calamine]>=2.0.0",
28
+ "py7zr>=1.1.0",
29
+ "pyarrow>=19.0.1",
30
+ "requests>=2.32.3",
31
+ "tqdm>=4.67.3",
32
+ "typer>=0.16.0",
33
+ ]
34
+ requires-python = ">=3.11"
35
+
36
+ [project.scripts]
37
+ nemobi = "nemobi:cli.app"
38
+
39
+ [project.urls]
40
+ Homepage = "https://github.com/LucasJavaudin/Nemobi"
41
+ Documentation = "https://nemobi.lucasjavaudin.com/index.html"
42
+ Repository = "https://github.com/LucasJavaudin/Nemobi"
43
+ Issues = "https://github.com/LucasJavaudin/Nemobi/issues"
44
+
45
+ [dependency-groups]
46
+ dev = [
47
+ "Cartopy>=0.24.1",
48
+ "folium>=0.20.0",
49
+ "ipython>=8.37.0",
50
+ "matplotlib>=3.10.1",
51
+ "pytest>=9.0.2",
52
+ "ruff>=0.16.4",
53
+ "seaborn>=0.13.2",
54
+ "ty>=0.0.15",
55
+ ]
56
+
57
+ [tool.ruff]
58
+ line-length = 100
59
+ extend-exclude = ["*.md"]
60
+
61
+ [tool.ruff.format]
62
+ skip-magic-trailing-comma = true
63
+
64
+ [tool.ruff.lint]
65
+ select = ["E", "F", "I", "UP", "RUF", "PLW"]
66
+ ignore = [
67
+ "C408", # C408: allow list() and dict() calls
68
+ ]
69
+ preview = false
70
+
71
+ [tool.ruff.lint.isort]
72
+ split-on-trailing-comma = false
nemobi-2.0.0/setup.cfg ADDED
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,9 @@
1
+ from importlib.metadata import version
2
+
3
+ from .classes import SurveyDataReader as SurveyDataReader
4
+ from .classes import read_many as read_many
5
+ from .logger import setup as setup
6
+ from .main import bulk_standardize as bulk_standardize
7
+ from .main import standardize as standardize
8
+
9
+ __version__ = version("nemobi")
@@ -0,0 +1,4 @@
1
+ if __name__ == "__main__":
2
+ from .cli import app
3
+
4
+ app()
@@ -0,0 +1,26 @@
1
+ from pathlib import Path
2
+ from zipfile import ZipFile
3
+
4
+ from .edgt import EDGTReader
5
+ from .edvm import EDVMReader
6
+ from .emc2 import EMC2Reader
7
+ from .emd import EMDReader
8
+ from .nantes_opendata import OpenDataReader
9
+
10
+
11
+ def standardize(
12
+ source: Path | ZipFile, survey_type: str, skip_spatial: bool = False, skip_insee: bool = False
13
+ ):
14
+ if survey_type == "EMC2":
15
+ reader = EMC2Reader(source)
16
+ elif survey_type == "EDGT":
17
+ reader = EDGTReader(source)
18
+ elif survey_type == "EDGT-opendata":
19
+ reader = OpenDataReader(source)
20
+ elif survey_type == "EDVM":
21
+ reader = EDVMReader(source)
22
+ elif survey_type == "EMD":
23
+ reader = EMDReader(source)
24
+ else:
25
+ raise NotImplementedError
26
+ return reader.standardize(skip_spatial, skip_insee)