nemobi 2.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nemobi-2.0.0/LICENSE.md +7 -0
- nemobi-2.0.0/PKG-INFO +256 -0
- nemobi-2.0.0/README.md +226 -0
- nemobi-2.0.0/pyproject.toml +72 -0
- nemobi-2.0.0/setup.cfg +4 -0
- nemobi-2.0.0/src/nemobi/__init__.py +9 -0
- nemobi-2.0.0/src/nemobi/__main__.py +4 -0
- nemobi-2.0.0/src/nemobi/cerema/__init__.py +26 -0
- nemobi-2.0.0/src/nemobi/cerema/common.py +76 -0
- nemobi-2.0.0/src/nemobi/cerema/deplacements.py +254 -0
- nemobi-2.0.0/src/nemobi/cerema/edgt.py +254 -0
- nemobi-2.0.0/src/nemobi/cerema/edvm.py +269 -0
- nemobi-2.0.0/src/nemobi/cerema/emc2.py +141 -0
- nemobi-2.0.0/src/nemobi/cerema/emd.py +128 -0
- nemobi-2.0.0/src/nemobi/cerema/menages.py +391 -0
- nemobi-2.0.0/src/nemobi/cerema/nantes_opendata.py +207 -0
- nemobi-2.0.0/src/nemobi/cerema/personnes.py +532 -0
- nemobi-2.0.0/src/nemobi/cerema/reader.py +160 -0
- nemobi-2.0.0/src/nemobi/cerema/survey.py +451 -0
- nemobi-2.0.0/src/nemobi/cerema/trajets.py +423 -0
- nemobi-2.0.0/src/nemobi/cerema/zones.py +406 -0
- nemobi-2.0.0/src/nemobi/classes.py +341 -0
- nemobi-2.0.0/src/nemobi/cli.py +103 -0
- nemobi-2.0.0/src/nemobi/common/__init__.py +1 -0
- nemobi-2.0.0/src/nemobi/common/cars.py +119 -0
- nemobi-2.0.0/src/nemobi/common/clean.py +559 -0
- nemobi-2.0.0/src/nemobi/common/households.py +99 -0
- nemobi-2.0.0/src/nemobi/common/legs.py +134 -0
- nemobi-2.0.0/src/nemobi/common/motorcycles.py +41 -0
- nemobi-2.0.0/src/nemobi/common/persons.py +180 -0
- nemobi-2.0.0/src/nemobi/common/trips.py +265 -0
- nemobi-2.0.0/src/nemobi/common/zones.py +37 -0
- nemobi-2.0.0/src/nemobi/egt2010/__init__.py +1 -0
- nemobi-2.0.0/src/nemobi/egt2010/deplacements.py +227 -0
- nemobi-2.0.0/src/nemobi/egt2010/menages.py +403 -0
- nemobi-2.0.0/src/nemobi/egt2010/personnes.py +350 -0
- nemobi-2.0.0/src/nemobi/egt2010/survey.py +92 -0
- nemobi-2.0.0/src/nemobi/egt2010/trajets.py +231 -0
- nemobi-2.0.0/src/nemobi/egt2010/zones.py +11 -0
- nemobi-2.0.0/src/nemobi/egt2020/__init__.py +1 -0
- nemobi-2.0.0/src/nemobi/egt2020/deplacements.py +203 -0
- nemobi-2.0.0/src/nemobi/egt2020/menages.py +168 -0
- nemobi-2.0.0/src/nemobi/egt2020/motos.py +98 -0
- nemobi-2.0.0/src/nemobi/egt2020/personnes.py +405 -0
- nemobi-2.0.0/src/nemobi/egt2020/survey.py +94 -0
- nemobi-2.0.0/src/nemobi/egt2020/trajets.py +347 -0
- nemobi-2.0.0/src/nemobi/egt2020/voitures.py +131 -0
- nemobi-2.0.0/src/nemobi/emg2023/__init__.py +1 -0
- nemobi-2.0.0/src/nemobi/emg2023/deplacements.py +255 -0
- nemobi-2.0.0/src/nemobi/emg2023/menages.py +24 -0
- nemobi-2.0.0/src/nemobi/emg2023/personnes.py +170 -0
- nemobi-2.0.0/src/nemobi/emg2023/survey.py +66 -0
- nemobi-2.0.0/src/nemobi/emp/__init__.py +1 -0
- nemobi-2.0.0/src/nemobi/emp/common.py +31 -0
- nemobi-2.0.0/src/nemobi/emp/deplacements.py +403 -0
- nemobi-2.0.0/src/nemobi/emp/menages.py +190 -0
- nemobi-2.0.0/src/nemobi/emp/motos.py +105 -0
- nemobi-2.0.0/src/nemobi/emp/personnes.py +657 -0
- nemobi-2.0.0/src/nemobi/emp/survey.py +110 -0
- nemobi-2.0.0/src/nemobi/emp/voitures.py +184 -0
- nemobi-2.0.0/src/nemobi/logger.py +26 -0
- nemobi-2.0.0/src/nemobi/main.py +257 -0
- nemobi-2.0.0/src/nemobi/resources/__init__.py +42 -0
- nemobi-2.0.0/src/nemobi/resources/admin_express.py +96 -0
- nemobi-2.0.0/src/nemobi/resources/insee_data.py +446 -0
- nemobi-2.0.0/src/nemobi/resources/nuts.py +132 -0
- nemobi-2.0.0/src/nemobi/schema/__init__.py +12 -0
- nemobi-2.0.0/src/nemobi/schema/cars.py +95 -0
- nemobi-2.0.0/src/nemobi/schema/common.py +165 -0
- nemobi-2.0.0/src/nemobi/schema/guarantees.py +735 -0
- nemobi-2.0.0/src/nemobi/schema/households.py +183 -0
- nemobi-2.0.0/src/nemobi/schema/legs.py +277 -0
- nemobi-2.0.0/src/nemobi/schema/motorcycles.py +70 -0
- nemobi-2.0.0/src/nemobi/schema/persons.py +897 -0
- nemobi-2.0.0/src/nemobi/schema/trips.py +484 -0
- nemobi-2.0.0/src/nemobi/utils.py +275 -0
- nemobi-2.0.0/src/nemobi/validation.py +795 -0
- nemobi-2.0.0/src/nemobi.egg-info/PKG-INFO +256 -0
- nemobi-2.0.0/src/nemobi.egg-info/SOURCES.txt +81 -0
- nemobi-2.0.0/src/nemobi.egg-info/dependency_links.txt +1 -0
- nemobi-2.0.0/src/nemobi.egg-info/entry_points.txt +2 -0
- nemobi-2.0.0/src/nemobi.egg-info/requires.txt +11 -0
- nemobi-2.0.0/src/nemobi.egg-info/top_level.txt +1 -0
nemobi-2.0.0/LICENSE.md
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
Copyright © 2025 Lucas Javaudin
|
|
2
|
+
|
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
|
4
|
+
|
|
5
|
+
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
|
6
|
+
|
|
7
|
+
THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
nemobi-2.0.0/PKG-INFO
ADDED
|
@@ -0,0 +1,256 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: nemobi
|
|
3
|
+
Version: 2.0.0
|
|
4
|
+
Summary: Nemobi (Normalisation des enquêtes mobilités): a tool to standardize French mobility survey datasets (EMC², EGT, EMP, etc.) into a unified format.
|
|
5
|
+
Author-email: Lucas Javaudin <lucas@lucasjavaudin.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/LucasJavaudin/Nemobi
|
|
8
|
+
Project-URL: Documentation, https://nemobi.lucasjavaudin.com/index.html
|
|
9
|
+
Project-URL: Repository, https://github.com/LucasJavaudin/Nemobi
|
|
10
|
+
Project-URL: Issues, https://github.com/LucasJavaudin/Nemobi/issues
|
|
11
|
+
Keywords: mobility survey,travel survey,France,polars,data format
|
|
12
|
+
Classifier: Programming Language :: Python
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Topic :: File Formats
|
|
15
|
+
Requires-Python: >=3.11
|
|
16
|
+
Description-Content-Type: text/markdown
|
|
17
|
+
License-File: LICENSE.md
|
|
18
|
+
Requires-Dist: geopandas>=1.0.1
|
|
19
|
+
Requires-Dist: humanize>=4.13.0
|
|
20
|
+
Requires-Dist: loguru>=0.7.3
|
|
21
|
+
Requires-Dist: pandas>=2.2.3
|
|
22
|
+
Requires-Dist: platformdirs>=4.3.8
|
|
23
|
+
Requires-Dist: polars[calamine]>=2.0.0
|
|
24
|
+
Requires-Dist: py7zr>=1.1.0
|
|
25
|
+
Requires-Dist: pyarrow>=19.0.1
|
|
26
|
+
Requires-Dist: requests>=2.32.3
|
|
27
|
+
Requires-Dist: tqdm>=4.67.3
|
|
28
|
+
Requires-Dist: typer>=0.16.0
|
|
29
|
+
Dynamic: license-file
|
|
30
|
+
|
|
31
|
+
[![Contributors][contributors-shield]][contributors-url]
|
|
32
|
+
[![Forks][forks-shield]][forks-url]
|
|
33
|
+
[![Stargazers][stars-shield]][stars-url]
|
|
34
|
+
[![Issues][issues-shield]][issues-url]
|
|
35
|
+
|
|
36
|
+
[![GPL v3][license-shield]][license-url]
|
|
37
|
+
[![PyPI][pypi-shield]][pypi-url]
|
|
38
|
+
[![DOI][doi-shield]][doi-url]
|
|
39
|
+
|
|
40
|
+
# Nemobi
|
|
41
|
+
|
|
42
|
+
> [!NOTE]
|
|
43
|
+
> **MobiSurvStd is now Nemobi.** Starting with v2.0.0, the package is published as `nemobi`
|
|
44
|
+
> (`pip install nemobi`, `import nemobi`, CLI `nemobi`). Surveys standardized with MobiSurvStd can
|
|
45
|
+
> still be read by Nemobi. See the
|
|
46
|
+
> [migration guide](https://nemobi.lucasjavaudin.com/migration.html).
|
|
47
|
+
|
|
48
|
+
*A tool to standardize French mobility survey datasets (EMC², EGT, EMP, etc.) into a unified format.*
|
|
49
|
+
|
|
50
|
+
📚 [Documentation](https://nemobi.lucasjavaudin.com)
|
|
51
|
+
|
|
52
|
+
<span style="color:blue">🇫🇷 [Version française](./README.fr.md)</span>
|
|
53
|
+
|
|
54
|
+
## Table of Contents
|
|
55
|
+
- [Introduction](#introduction)
|
|
56
|
+
- [Mobility Surveys in France](#mobility-surveys-in-france)
|
|
57
|
+
- [Why Nemobi?](#why-nemobi)
|
|
58
|
+
- [Installation](#installation)
|
|
59
|
+
- [Getting Started](#getting-started)
|
|
60
|
+
- [Supported Surveys](#supported-surveys)
|
|
61
|
+
- [Legal Notice](#legal-notice)
|
|
62
|
+
- [Contributing](#contributing)
|
|
63
|
+
|
|
64
|
+
## Introduction
|
|
65
|
+
|
|
66
|
+
Nemobi (*Normalisation des enquêtes mobilités*, i.e., standardization of mobility surveys) is an easy-to-use Python command line interface to convert many French mobility surveys
|
|
67
|
+
(EMC², EGT, EMP, etc.) to a unique standardized and clean format.
|
|
68
|
+
|
|
69
|
+
---
|
|
70
|
+
|
|
71
|
+
## Mobility Surveys in France
|
|
72
|
+
|
|
73
|
+
In France, despite recent efforts by CEREMA to create a standard format for mobility surveys – with
|
|
74
|
+
the EMC² surveys – various formats co-exist:
|
|
75
|
+
|
|
76
|
+
- [**EMC²**](https://www.cerema.fr/fr/activites/mobilites/connaissance-modelisation-evaluation-mobilite/enquetes-mobilite-emc2)
|
|
77
|
+
(Enquête mobilité certifiée CEREMA): mobility surveys for many French territories (since 2018);
|
|
78
|
+
- [**EGT H2020**](https://omnil.fr/actualites/les-resultats-de-la-derniere-enquete-globale-transport)
|
|
79
|
+
(Enquête Globale Transport, Île-de-France Mobilités): mobility survey for Île-de-France (2018–2020;
|
|
80
|
+
incomplete due to COVID-19);
|
|
81
|
+
- [**EMP**](https://www.statistiques.developpement-durable.gouv.fr/resultats-detailles-de-lenquete-mobilite-des-personnes-de-2019)
|
|
82
|
+
(Enquête mobilité des personnes, SDES): national mobility survey (2019).
|
|
83
|
+
- [**EMG**](https://www.institutparisregion.fr/mobilite-et-transports/deplacements/enquete-regionale-sur-la-mobilite-des-franciliens/)
|
|
84
|
+
(Enquête Mobilité par GPS, Institut Paris Région): mobility survey for Île-de-France (2022-2023)
|
|
85
|
+
with individuals tracked over several days through GPS.
|
|
86
|
+
|
|
87
|
+
Also surveys based on previous formats are still in use today:
|
|
88
|
+
|
|
89
|
+
- **EDVM** (Enquêtes Déplacements Villes Moyennes, CEREMA): mobility surveys for medium-size cities
|
|
90
|
+
(until 2018);
|
|
91
|
+
- **EDGT** (Enquêtes Déplacements Grands Territoires, CEREMA): mobility surveys for periphery areas
|
|
92
|
+
(until 2018);
|
|
93
|
+
- **EMD** (Enquêtes Ménages Déplacements, CEREMA): other mobility surveys from CEREMA (until 2018);
|
|
94
|
+
- [**EGT 2010**](https://omnil.fr/egt-2010) (Enquête Globale Transport, Île-de-France Mobilités):
|
|
95
|
+
previous version of the Île-de-France mobility survey;
|
|
96
|
+
- [**ENTD**](https://www.statistiques.developpement-durable.gouv.fr/enquete-nationale-transports-et-deplacements-entd-2008)
|
|
97
|
+
(Enquête nationale transports et déplacements, SDES): former national mobility survey (2008).
|
|
98
|
+
|
|
99
|
+
---
|
|
100
|
+
|
|
101
|
+
## Why Nemobi?
|
|
102
|
+
|
|
103
|
+
The existing formats all have the same drawbacks:
|
|
104
|
+
|
|
105
|
+
- Data are sorted in CSV files which are not always straightforward to read (Which separator?
|
|
106
|
+
Which encoding? What are the variable datatypes?).
|
|
107
|
+
- Variable names and modalities are not always clear (e.g., in the EMC² format, variable "P2"
|
|
108
|
+
represents the gender of the person, with modality 1 for a man and 2 for a woman).
|
|
109
|
+
- Joining two datasets is hard and not well documented (e.g., in the EMC² format, to join the
|
|
110
|
+
persons with their household, the variables to use are "METH", "ZFM" and "ECH" for the households
|
|
111
|
+
and "DMET", "ZFD" and "ECH" for the persons).
|
|
112
|
+
|
|
113
|
+
Additionally, when working with different territories / periods, it is often necessary to write a
|
|
114
|
+
similar code multiple times due to the existence of different formats.
|
|
115
|
+
|
|
116
|
+
Nemobi solves all these issues by being able to convert all survey formats to a [well-defined
|
|
117
|
+
Parquet format](https://nemobi.lucasjavaudin.com/format/index.html).
|
|
118
|
+
|
|
119
|
+
See [this example](https://nemobi.lucasjavaudin.com/problem-example.html) to understand how
|
|
120
|
+
Nemobi can simplify your workflow.
|
|
121
|
+
|
|
122
|
+
---
|
|
123
|
+
|
|
124
|
+
## Installation
|
|
125
|
+
|
|
126
|
+
Install the library with
|
|
127
|
+
|
|
128
|
+
```bash
|
|
129
|
+
pip install nemobi
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
---
|
|
133
|
+
|
|
134
|
+
## Getting Started
|
|
135
|
+
|
|
136
|
+
Download the
|
|
137
|
+
[EMP 2019 survey](https://www.statistiques.developpement-durable.gouv.fr/resultats-detailles-de-lenquete-mobilite-des-personnes-de-2019)
|
|
138
|
+
_(Données individuelles anonymisées (fichiers au format CSV))_, then run the following command:
|
|
139
|
+
|
|
140
|
+
```bash
|
|
141
|
+
nemobi emp_2019_donnees_individuelles_anonymisees_novembre2024.zip standardized_emp2019
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
It will standardize the EMP 2019 survey and save the resulting Parquet files in the
|
|
145
|
+
`standardized_emp2019` directory.
|
|
146
|
+
These Parquet files can then be analyzed using e.g., [polars](https://pola.rs/) or
|
|
147
|
+
[pandas](https://pandas.pydata.org) in Python or [arrow](https://arrow.apache.org/docs/r/) in R.
|
|
148
|
+
|
|
149
|
+
**Note:** During standardization, Nemobi may issue warnings when certain variable guarantees
|
|
150
|
+
are not initially satisfied but are automatically fixed.
|
|
151
|
+
These warnings can usually be safely ignored, unless they pertain to a large number of observations.
|
|
152
|
+
|
|
153
|
+
A detailed definition of the Parquet format used by Nemobi is available
|
|
154
|
+
[here](https://nemobi.lucasjavaudin.com/format/index.html).
|
|
155
|
+
|
|
156
|
+
You can also standardize surveys programmatically in Python with the `standardize` function:
|
|
157
|
+
|
|
158
|
+
```python
|
|
159
|
+
import nemobi
|
|
160
|
+
nemobi.standardize(
|
|
161
|
+
"emp_2019_donnees_individuelles_anonymisees_novembre2024.zip",
|
|
162
|
+
"standardized_emp2019",
|
|
163
|
+
)
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
For more, check the [User Guide](https://nemobi.lucasjavaudin.com/howto.html).
|
|
167
|
+
|
|
168
|
+
**Citing Nemobi**
|
|
169
|
+
|
|
170
|
+
If you use Nemobi in a publication or project, please include the following citation:
|
|
171
|
+
```
|
|
172
|
+
Javaudin, L. (2025). Nemobi. Zenodo. https://doi.org/10.5281/zenodo.17710203
|
|
173
|
+
```
|
|
174
|
+
A [CITATION.bib](CITATION.bib) is also available.
|
|
175
|
+
|
|
176
|
+
---
|
|
177
|
+
|
|
178
|
+
## Case Study: Bicycle Use
|
|
179
|
+
|
|
180
|
+
The following graph represents the share of bicycle trips for EMC², EDGT, EDVM, EMD, and EGT
|
|
181
|
+
surveys.
|
|
182
|
+
The circle colors represent the average number of bicycles in the surveyed households.
|
|
183
|
+
The circle sizes represent the expected number of trips in the surveyed area.
|
|
184
|
+
|
|
185
|
+
The graph has been generated from the code in
|
|
186
|
+
[analyses/bicycle_shares.py](analyses/bicycle_shares.py).
|
|
187
|
+
|
|
188
|
+

|
|
189
|
+
|
|
190
|
+
The following map represents the share of bicycle trips within INSEE municipalities.
|
|
191
|
+
Only municipalities with more than 30 surveyed trips are shown.
|
|
192
|
+
|
|
193
|
+
The map has been generated from the code in
|
|
194
|
+
[analyses/bicycle_shares_by_insee.py](analyses/bicycle_shares_by_insee.py).
|
|
195
|
+
|
|
196
|
+

|
|
197
|
+
|
|
198
|
+
---
|
|
199
|
+
|
|
200
|
+
## Supported Surveys
|
|
201
|
+
|
|
202
|
+
Currently, Nemobi supports the following survey types:
|
|
203
|
+
|
|
204
|
+
- `emp2019`
|
|
205
|
+
- `emc2`
|
|
206
|
+
- `emg2023` (thanks to @sebhorl)
|
|
207
|
+
- `egt2020`
|
|
208
|
+
- `egt2010`
|
|
209
|
+
- `edgt` (including the [open-data version of Nantes 2015](https://www.data.gouv.fr/datasets/enquete-deplacements-2015-en-loire-atlantique))
|
|
210
|
+
- `edvm`
|
|
211
|
+
- `emd`
|
|
212
|
+
|
|
213
|
+
See [Survey Types](https://nemobi.lucasjavaudin.com/surveys.html) for more.
|
|
214
|
+
|
|
215
|
+

|
|
216
|
+
|
|
217
|
+
Other survey types that are planned to be integrated are: `entd`.
|
|
218
|
+
If you know another survey format that could be integrated, feel free to open an issue on GitHub.
|
|
219
|
+
|
|
220
|
+
Nemobi covers only French mobility survey formats.
|
|
221
|
+
If other countries have similar survey formats, they might be easily integrated into Nemobi (or
|
|
222
|
+
a variant of it).
|
|
223
|
+
|
|
224
|
+
---
|
|
225
|
+
|
|
226
|
+
## Legal Notice
|
|
227
|
+
|
|
228
|
+
<span style="color:red">
|
|
229
|
+
⚠️ <strong>Nemobi does not anonymize the data.</strong>
|
|
230
|
+
If you are working with confidential datasets (e.g., EMC² or EGT surveys), you must apply the same
|
|
231
|
+
confidentiality rules to the standardized data as to the original data.
|
|
232
|
+
In particular, <strong>you must not share the standardized data</strong> if your confidentiality
|
|
233
|
+
agreement prohibits sharing the original data.
|
|
234
|
+
</span>
|
|
235
|
+
|
|
236
|
+
---
|
|
237
|
+
|
|
238
|
+
## Contributing
|
|
239
|
+
|
|
240
|
+
If you think you found a bug, if you have a suggestion, or if you want to integrate a new format,
|
|
241
|
+
feel free to open an issue on GitHub or to create a Pull Request.
|
|
242
|
+
|
|
243
|
+
[contributors-shield]: https://img.shields.io/github/contributors/LucasJavaudin/Nemobi.svg?style=for-the-badge
|
|
244
|
+
[contributors-url]: https://github.com/LucasJavaudin/Nemobi/graphs/contributors
|
|
245
|
+
[forks-shield]: https://img.shields.io/github/forks/LucasJavaudin/Nemobi.svg?style=for-the-badge
|
|
246
|
+
[forks-url]: https://github.com/LucasJavaudin/Nemobi/network/members
|
|
247
|
+
[stars-shield]: https://img.shields.io/github/stars/LucasJavaudin/Nemobi.svg?style=for-the-badge
|
|
248
|
+
[stars-url]: https://github.com/LucasJavaudin/Nemobi/stargazers
|
|
249
|
+
[issues-shield]: https://img.shields.io/github/issues/LucasJavaudin/Nemobi.svg?style=for-the-badge
|
|
250
|
+
[issues-url]: https://github.com/LucasJavaudin/Nemobi/issues
|
|
251
|
+
[license-shield]: https://img.shields.io/github/license/LucasJavaudin/Nemobi.svg?style=for-the-badge
|
|
252
|
+
[license-url]: https://github.com/LucasJavaudin/Nemobi/blob/master/LICENSE.txt
|
|
253
|
+
[doi-shield]: https://img.shields.io/badge/doi-10.5281/zenodo.17422994-blue?style=for-the-badge
|
|
254
|
+
[doi-url]: https://doi.org/10.5281/zenodo.17422993
|
|
255
|
+
[pypi-shield]: https://img.shields.io/pypi/v/nemobi.svg?style=for-the-badge
|
|
256
|
+
[pypi-url]: https://pypi.org/project/nemobi/
|
nemobi-2.0.0/README.md
ADDED
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
[![Contributors][contributors-shield]][contributors-url]
|
|
2
|
+
[![Forks][forks-shield]][forks-url]
|
|
3
|
+
[![Stargazers][stars-shield]][stars-url]
|
|
4
|
+
[![Issues][issues-shield]][issues-url]
|
|
5
|
+
|
|
6
|
+
[![GPL v3][license-shield]][license-url]
|
|
7
|
+
[![PyPI][pypi-shield]][pypi-url]
|
|
8
|
+
[![DOI][doi-shield]][doi-url]
|
|
9
|
+
|
|
10
|
+
# Nemobi
|
|
11
|
+
|
|
12
|
+
> [!NOTE]
|
|
13
|
+
> **MobiSurvStd is now Nemobi.** Starting with v2.0.0, the package is published as `nemobi`
|
|
14
|
+
> (`pip install nemobi`, `import nemobi`, CLI `nemobi`). Surveys standardized with MobiSurvStd can
|
|
15
|
+
> still be read by Nemobi. See the
|
|
16
|
+
> [migration guide](https://nemobi.lucasjavaudin.com/migration.html).
|
|
17
|
+
|
|
18
|
+
*A tool to standardize French mobility survey datasets (EMC², EGT, EMP, etc.) into a unified format.*
|
|
19
|
+
|
|
20
|
+
📚 [Documentation](https://nemobi.lucasjavaudin.com)
|
|
21
|
+
|
|
22
|
+
<span style="color:blue">🇫🇷 [Version française](./README.fr.md)</span>
|
|
23
|
+
|
|
24
|
+
## Table of Contents
|
|
25
|
+
- [Introduction](#introduction)
|
|
26
|
+
- [Mobility Surveys in France](#mobility-surveys-in-france)
|
|
27
|
+
- [Why Nemobi?](#why-nemobi)
|
|
28
|
+
- [Installation](#installation)
|
|
29
|
+
- [Getting Started](#getting-started)
|
|
30
|
+
- [Supported Surveys](#supported-surveys)
|
|
31
|
+
- [Legal Notice](#legal-notice)
|
|
32
|
+
- [Contributing](#contributing)
|
|
33
|
+
|
|
34
|
+
## Introduction
|
|
35
|
+
|
|
36
|
+
Nemobi (*Normalisation des enquêtes mobilités*, i.e., standardization of mobility surveys) is an easy-to-use Python command line interface to convert many French mobility surveys
|
|
37
|
+
(EMC², EGT, EMP, etc.) to a unique standardized and clean format.
|
|
38
|
+
|
|
39
|
+
---
|
|
40
|
+
|
|
41
|
+
## Mobility Surveys in France
|
|
42
|
+
|
|
43
|
+
In France, despite recent efforts by CEREMA to create a standard format for mobility surveys – with
|
|
44
|
+
the EMC² surveys – various formats co-exist:
|
|
45
|
+
|
|
46
|
+
- [**EMC²**](https://www.cerema.fr/fr/activites/mobilites/connaissance-modelisation-evaluation-mobilite/enquetes-mobilite-emc2)
|
|
47
|
+
(Enquête mobilité certifiée CEREMA): mobility surveys for many French territories (since 2018);
|
|
48
|
+
- [**EGT H2020**](https://omnil.fr/actualites/les-resultats-de-la-derniere-enquete-globale-transport)
|
|
49
|
+
(Enquête Globale Transport, Île-de-France Mobilités): mobility survey for Île-de-France (2018–2020;
|
|
50
|
+
incomplete due to COVID-19);
|
|
51
|
+
- [**EMP**](https://www.statistiques.developpement-durable.gouv.fr/resultats-detailles-de-lenquete-mobilite-des-personnes-de-2019)
|
|
52
|
+
(Enquête mobilité des personnes, SDES): national mobility survey (2019).
|
|
53
|
+
- [**EMG**](https://www.institutparisregion.fr/mobilite-et-transports/deplacements/enquete-regionale-sur-la-mobilite-des-franciliens/)
|
|
54
|
+
(Enquête Mobilité par GPS, Institut Paris Région): mobility survey for Île-de-France (2022-2023)
|
|
55
|
+
with individuals tracked over several days through GPS.
|
|
56
|
+
|
|
57
|
+
Also surveys based on previous formats are still in use today:
|
|
58
|
+
|
|
59
|
+
- **EDVM** (Enquêtes Déplacements Villes Moyennes, CEREMA): mobility surveys for medium-size cities
|
|
60
|
+
(until 2018);
|
|
61
|
+
- **EDGT** (Enquêtes Déplacements Grands Territoires, CEREMA): mobility surveys for periphery areas
|
|
62
|
+
(until 2018);
|
|
63
|
+
- **EMD** (Enquêtes Ménages Déplacements, CEREMA): other mobility surveys from CEREMA (until 2018);
|
|
64
|
+
- [**EGT 2010**](https://omnil.fr/egt-2010) (Enquête Globale Transport, Île-de-France Mobilités):
|
|
65
|
+
previous version of the Île-de-France mobility survey;
|
|
66
|
+
- [**ENTD**](https://www.statistiques.developpement-durable.gouv.fr/enquete-nationale-transports-et-deplacements-entd-2008)
|
|
67
|
+
(Enquête nationale transports et déplacements, SDES): former national mobility survey (2008).
|
|
68
|
+
|
|
69
|
+
---
|
|
70
|
+
|
|
71
|
+
## Why Nemobi?
|
|
72
|
+
|
|
73
|
+
The existing formats all have the same drawbacks:
|
|
74
|
+
|
|
75
|
+
- Data are sorted in CSV files which are not always straightforward to read (Which separator?
|
|
76
|
+
Which encoding? What are the variable datatypes?).
|
|
77
|
+
- Variable names and modalities are not always clear (e.g., in the EMC² format, variable "P2"
|
|
78
|
+
represents the gender of the person, with modality 1 for a man and 2 for a woman).
|
|
79
|
+
- Joining two datasets is hard and not well documented (e.g., in the EMC² format, to join the
|
|
80
|
+
persons with their household, the variables to use are "METH", "ZFM" and "ECH" for the households
|
|
81
|
+
and "DMET", "ZFD" and "ECH" for the persons).
|
|
82
|
+
|
|
83
|
+
Additionally, when working with different territories / periods, it is often necessary to write a
|
|
84
|
+
similar code multiple times due to the existence of different formats.
|
|
85
|
+
|
|
86
|
+
Nemobi solves all these issues by being able to convert all survey formats to a [well-defined
|
|
87
|
+
Parquet format](https://nemobi.lucasjavaudin.com/format/index.html).
|
|
88
|
+
|
|
89
|
+
See [this example](https://nemobi.lucasjavaudin.com/problem-example.html) to understand how
|
|
90
|
+
Nemobi can simplify your workflow.
|
|
91
|
+
|
|
92
|
+
---
|
|
93
|
+
|
|
94
|
+
## Installation
|
|
95
|
+
|
|
96
|
+
Install the library with
|
|
97
|
+
|
|
98
|
+
```bash
|
|
99
|
+
pip install nemobi
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
---
|
|
103
|
+
|
|
104
|
+
## Getting Started
|
|
105
|
+
|
|
106
|
+
Download the
|
|
107
|
+
[EMP 2019 survey](https://www.statistiques.developpement-durable.gouv.fr/resultats-detailles-de-lenquete-mobilite-des-personnes-de-2019)
|
|
108
|
+
_(Données individuelles anonymisées (fichiers au format CSV))_, then run the following command:
|
|
109
|
+
|
|
110
|
+
```bash
|
|
111
|
+
nemobi emp_2019_donnees_individuelles_anonymisees_novembre2024.zip standardized_emp2019
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
It will standardize the EMP 2019 survey and save the resulting Parquet files in the
|
|
115
|
+
`standardized_emp2019` directory.
|
|
116
|
+
These Parquet files can then be analyzed using e.g., [polars](https://pola.rs/) or
|
|
117
|
+
[pandas](https://pandas.pydata.org) in Python or [arrow](https://arrow.apache.org/docs/r/) in R.
|
|
118
|
+
|
|
119
|
+
**Note:** During standardization, Nemobi may issue warnings when certain variable guarantees
|
|
120
|
+
are not initially satisfied but are automatically fixed.
|
|
121
|
+
These warnings can usually be safely ignored, unless they pertain to a large number of observations.
|
|
122
|
+
|
|
123
|
+
A detailed definition of the Parquet format used by Nemobi is available
|
|
124
|
+
[here](https://nemobi.lucasjavaudin.com/format/index.html).
|
|
125
|
+
|
|
126
|
+
You can also standardize surveys programmatically in Python with the `standardize` function:
|
|
127
|
+
|
|
128
|
+
```python
|
|
129
|
+
import nemobi
|
|
130
|
+
nemobi.standardize(
|
|
131
|
+
"emp_2019_donnees_individuelles_anonymisees_novembre2024.zip",
|
|
132
|
+
"standardized_emp2019",
|
|
133
|
+
)
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
For more, check the [User Guide](https://nemobi.lucasjavaudin.com/howto.html).
|
|
137
|
+
|
|
138
|
+
**Citing Nemobi**
|
|
139
|
+
|
|
140
|
+
If you use Nemobi in a publication or project, please include the following citation:
|
|
141
|
+
```
|
|
142
|
+
Javaudin, L. (2025). Nemobi. Zenodo. https://doi.org/10.5281/zenodo.17710203
|
|
143
|
+
```
|
|
144
|
+
A [CITATION.bib](CITATION.bib) is also available.
|
|
145
|
+
|
|
146
|
+
---
|
|
147
|
+
|
|
148
|
+
## Case Study: Bicycle Use
|
|
149
|
+
|
|
150
|
+
The following graph represents the share of bicycle trips for EMC², EDGT, EDVM, EMD, and EGT
|
|
151
|
+
surveys.
|
|
152
|
+
The circle colors represent the average number of bicycles in the surveyed households.
|
|
153
|
+
The circle sizes represent the expected number of trips in the surveyed area.
|
|
154
|
+
|
|
155
|
+
The graph has been generated from the code in
|
|
156
|
+
[analyses/bicycle_shares.py](analyses/bicycle_shares.py).
|
|
157
|
+
|
|
158
|
+

|
|
159
|
+
|
|
160
|
+
The following map represents the share of bicycle trips within INSEE municipalities.
|
|
161
|
+
Only municipalities with more than 30 surveyed trips are shown.
|
|
162
|
+
|
|
163
|
+
The map has been generated from the code in
|
|
164
|
+
[analyses/bicycle_shares_by_insee.py](analyses/bicycle_shares_by_insee.py).
|
|
165
|
+
|
|
166
|
+

|
|
167
|
+
|
|
168
|
+
---
|
|
169
|
+
|
|
170
|
+
## Supported Surveys
|
|
171
|
+
|
|
172
|
+
Currently, Nemobi supports the following survey types:
|
|
173
|
+
|
|
174
|
+
- `emp2019`
|
|
175
|
+
- `emc2`
|
|
176
|
+
- `emg2023` (thanks to @sebhorl)
|
|
177
|
+
- `egt2020`
|
|
178
|
+
- `egt2010`
|
|
179
|
+
- `edgt` (including the [open-data version of Nantes 2015](https://www.data.gouv.fr/datasets/enquete-deplacements-2015-en-loire-atlantique))
|
|
180
|
+
- `edvm`
|
|
181
|
+
- `emd`
|
|
182
|
+
|
|
183
|
+
See [Survey Types](https://nemobi.lucasjavaudin.com/surveys.html) for more.
|
|
184
|
+
|
|
185
|
+

|
|
186
|
+
|
|
187
|
+
Other survey types that are planned to be integrated are: `entd`.
|
|
188
|
+
If you know another survey format that could be integrated, feel free to open an issue on GitHub.
|
|
189
|
+
|
|
190
|
+
Nemobi covers only French mobility survey formats.
|
|
191
|
+
If other countries have similar survey formats, they might be easily integrated into Nemobi (or
|
|
192
|
+
a variant of it).
|
|
193
|
+
|
|
194
|
+
---
|
|
195
|
+
|
|
196
|
+
## Legal Notice
|
|
197
|
+
|
|
198
|
+
<span style="color:red">
|
|
199
|
+
⚠️ <strong>Nemobi does not anonymize the data.</strong>
|
|
200
|
+
If you are working with confidential datasets (e.g., EMC² or EGT surveys), you must apply the same
|
|
201
|
+
confidentiality rules to the standardized data as to the original data.
|
|
202
|
+
In particular, <strong>you must not share the standardized data</strong> if your confidentiality
|
|
203
|
+
agreement prohibits sharing the original data.
|
|
204
|
+
</span>
|
|
205
|
+
|
|
206
|
+
---
|
|
207
|
+
|
|
208
|
+
## Contributing
|
|
209
|
+
|
|
210
|
+
If you think you found a bug, if you have a suggestion, or if you want to integrate a new format,
|
|
211
|
+
feel free to open an issue on GitHub or to create a Pull Request.
|
|
212
|
+
|
|
213
|
+
[contributors-shield]: https://img.shields.io/github/contributors/LucasJavaudin/Nemobi.svg?style=for-the-badge
|
|
214
|
+
[contributors-url]: https://github.com/LucasJavaudin/Nemobi/graphs/contributors
|
|
215
|
+
[forks-shield]: https://img.shields.io/github/forks/LucasJavaudin/Nemobi.svg?style=for-the-badge
|
|
216
|
+
[forks-url]: https://github.com/LucasJavaudin/Nemobi/network/members
|
|
217
|
+
[stars-shield]: https://img.shields.io/github/stars/LucasJavaudin/Nemobi.svg?style=for-the-badge
|
|
218
|
+
[stars-url]: https://github.com/LucasJavaudin/Nemobi/stargazers
|
|
219
|
+
[issues-shield]: https://img.shields.io/github/issues/LucasJavaudin/Nemobi.svg?style=for-the-badge
|
|
220
|
+
[issues-url]: https://github.com/LucasJavaudin/Nemobi/issues
|
|
221
|
+
[license-shield]: https://img.shields.io/github/license/LucasJavaudin/Nemobi.svg?style=for-the-badge
|
|
222
|
+
[license-url]: https://github.com/LucasJavaudin/Nemobi/blob/master/LICENSE.txt
|
|
223
|
+
[doi-shield]: https://img.shields.io/badge/doi-10.5281/zenodo.17422994-blue?style=for-the-badge
|
|
224
|
+
[doi-url]: https://doi.org/10.5281/zenodo.17422993
|
|
225
|
+
[pypi-shield]: https://img.shields.io/pypi/v/nemobi.svg?style=for-the-badge
|
|
226
|
+
[pypi-url]: https://pypi.org/project/nemobi/
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
# pyproject.toml
|
|
2
|
+
|
|
3
|
+
[build-system]
|
|
4
|
+
requires = ["setuptools>=61.0.0", "wheel"]
|
|
5
|
+
build-backend = "setuptools.build_meta"
|
|
6
|
+
|
|
7
|
+
[project]
|
|
8
|
+
name = "nemobi"
|
|
9
|
+
version = "2.0.0"
|
|
10
|
+
description = "Nemobi (Normalisation des enquêtes mobilités): a tool to standardize French mobility survey datasets (EMC², EGT, EMP, etc.) into a unified format."
|
|
11
|
+
readme = "README.md"
|
|
12
|
+
authors = [{ name = "Lucas Javaudin", email = "lucas@lucasjavaudin.com" }]
|
|
13
|
+
license = "MIT"
|
|
14
|
+
license-files = ["LICENSE.md"]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Programming Language :: Python",
|
|
17
|
+
"Programming Language :: Python :: 3",
|
|
18
|
+
"Topic :: File Formats",
|
|
19
|
+
]
|
|
20
|
+
keywords = ["mobility survey", "travel survey", "France", "polars", "data format"]
|
|
21
|
+
dependencies = [
|
|
22
|
+
"geopandas>=1.0.1",
|
|
23
|
+
"humanize>=4.13.0",
|
|
24
|
+
"loguru>=0.7.3",
|
|
25
|
+
"pandas>=2.2.3",
|
|
26
|
+
"platformdirs>=4.3.8",
|
|
27
|
+
"polars[calamine]>=2.0.0",
|
|
28
|
+
"py7zr>=1.1.0",
|
|
29
|
+
"pyarrow>=19.0.1",
|
|
30
|
+
"requests>=2.32.3",
|
|
31
|
+
"tqdm>=4.67.3",
|
|
32
|
+
"typer>=0.16.0",
|
|
33
|
+
]
|
|
34
|
+
requires-python = ">=3.11"
|
|
35
|
+
|
|
36
|
+
[project.scripts]
|
|
37
|
+
nemobi = "nemobi:cli.app"
|
|
38
|
+
|
|
39
|
+
[project.urls]
|
|
40
|
+
Homepage = "https://github.com/LucasJavaudin/Nemobi"
|
|
41
|
+
Documentation = "https://nemobi.lucasjavaudin.com/index.html"
|
|
42
|
+
Repository = "https://github.com/LucasJavaudin/Nemobi"
|
|
43
|
+
Issues = "https://github.com/LucasJavaudin/Nemobi/issues"
|
|
44
|
+
|
|
45
|
+
[dependency-groups]
|
|
46
|
+
dev = [
|
|
47
|
+
"Cartopy>=0.24.1",
|
|
48
|
+
"folium>=0.20.0",
|
|
49
|
+
"ipython>=8.37.0",
|
|
50
|
+
"matplotlib>=3.10.1",
|
|
51
|
+
"pytest>=9.0.2",
|
|
52
|
+
"ruff>=0.16.4",
|
|
53
|
+
"seaborn>=0.13.2",
|
|
54
|
+
"ty>=0.0.15",
|
|
55
|
+
]
|
|
56
|
+
|
|
57
|
+
[tool.ruff]
|
|
58
|
+
line-length = 100
|
|
59
|
+
extend-exclude = ["*.md"]
|
|
60
|
+
|
|
61
|
+
[tool.ruff.format]
|
|
62
|
+
skip-magic-trailing-comma = true
|
|
63
|
+
|
|
64
|
+
[tool.ruff.lint]
|
|
65
|
+
select = ["E", "F", "I", "UP", "RUF", "PLW"]
|
|
66
|
+
ignore = [
|
|
67
|
+
"C408", # C408: allow list() and dict() calls
|
|
68
|
+
]
|
|
69
|
+
preview = false
|
|
70
|
+
|
|
71
|
+
[tool.ruff.lint.isort]
|
|
72
|
+
split-on-trailing-comma = false
|
nemobi-2.0.0/setup.cfg
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
from importlib.metadata import version
|
|
2
|
+
|
|
3
|
+
from .classes import SurveyDataReader as SurveyDataReader
|
|
4
|
+
from .classes import read_many as read_many
|
|
5
|
+
from .logger import setup as setup
|
|
6
|
+
from .main import bulk_standardize as bulk_standardize
|
|
7
|
+
from .main import standardize as standardize
|
|
8
|
+
|
|
9
|
+
__version__ = version("nemobi")
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
from zipfile import ZipFile
|
|
3
|
+
|
|
4
|
+
from .edgt import EDGTReader
|
|
5
|
+
from .edvm import EDVMReader
|
|
6
|
+
from .emc2 import EMC2Reader
|
|
7
|
+
from .emd import EMDReader
|
|
8
|
+
from .nantes_opendata import OpenDataReader
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def standardize(
|
|
12
|
+
source: Path | ZipFile, survey_type: str, skip_spatial: bool = False, skip_insee: bool = False
|
|
13
|
+
):
|
|
14
|
+
if survey_type == "EMC2":
|
|
15
|
+
reader = EMC2Reader(source)
|
|
16
|
+
elif survey_type == "EDGT":
|
|
17
|
+
reader = EDGTReader(source)
|
|
18
|
+
elif survey_type == "EDGT-opendata":
|
|
19
|
+
reader = OpenDataReader(source)
|
|
20
|
+
elif survey_type == "EDVM":
|
|
21
|
+
reader = EDVMReader(source)
|
|
22
|
+
elif survey_type == "EMD":
|
|
23
|
+
reader = EMDReader(source)
|
|
24
|
+
else:
|
|
25
|
+
raise NotImplementedError
|
|
26
|
+
return reader.standardize(skip_spatial, skip_insee)
|