es-normalizer-py 0.2.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- es_normalizer_py-0.2.4/LICENSE +21 -0
- es_normalizer_py-0.2.4/MANIFEST.in +3 -0
- es_normalizer_py-0.2.4/PKG-INFO +112 -0
- es_normalizer_py-0.2.4/README.md +97 -0
- es_normalizer_py-0.2.4/pyproject.toml +29 -0
- es_normalizer_py-0.2.4/setup.cfg +4 -0
- es_normalizer_py-0.2.4/src/es_normalizer/__init__.py +243 -0
- es_normalizer_py-0.2.4/src/es_normalizer/date_of_birth.py +128 -0
- es_normalizer_py-0.2.4/src/es_normalizer/dictionary/__init__.py +48 -0
- es_normalizer_py-0.2.4/src/es_normalizer/dictionary/address.py +110 -0
- es_normalizer_py-0.2.4/src/es_normalizer/dictionary/firstname.py +229 -0
- es_normalizer_py-0.2.4/src/es_normalizer/dictionary/title.py +41 -0
- es_normalizer_py-0.2.4/src/es_normalizer_py.egg-info/PKG-INFO +112 -0
- es_normalizer_py-0.2.4/src/es_normalizer_py.egg-info/SOURCES.txt +15 -0
- es_normalizer_py-0.2.4/src/es_normalizer_py.egg-info/dependency_links.txt +1 -0
- es_normalizer_py-0.2.4/src/es_normalizer_py.egg-info/top_level.txt +1 -0
- es_normalizer_py-0.2.4/tests/test_normalizers.py +183 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Cyril Dever
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: es-normalizer-py
|
|
3
|
+
Version: 0.2.4
|
|
4
|
+
Summary: Contact data normalization adapted from the Empreinte Sociométrique's normalizers, the Python port of es-normalizer
|
|
5
|
+
Author-email: Cyril Dever <cdever@pep-s.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/cyrildever/es-normalizer-py
|
|
8
|
+
Keywords: data,contact,normalization,empreinte sociometrique
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: Operating System :: OS Independent
|
|
11
|
+
Requires-Python: >=3.10.2
|
|
12
|
+
Description-Content-Type: text/markdown
|
|
13
|
+
License-File: LICENSE
|
|
14
|
+
Dynamic: license-file
|
|
15
|
+
|
|
16
|
+
# es-normalizer-py
|
|
17
|
+
_Contact data normalization adapted from the Empreinte Sociométrique's normalizers (in Python)_
|
|
18
|
+
|
|
19
|
+

|
|
20
|
+

|
|
21
|
+

|
|
22
|
+

|
|
23
|
+

|
|
24
|
+
|
|
25
|
+
This is the Python port of [es-normalizer](https://github.com/cyrildever/es-normalizer), the TypeScript normalization library for contact data (address, phones, etc.) adapted from [Edgewhere](https://www.edgewhere.fr)'s Empreinte Sociométrique patented normalizers.
|
|
26
|
+
|
|
27
|
+
### Motivation
|
|
28
|
+
|
|
29
|
+
When it comes to hashing data, the necessary uniqueness of the source sometimes makes it hard to compare two hashed data, mostly if dealing with postal address. This library helps normalizing any contact data (the postal or e-mail address, the mobile or landline phone number, the title, names or date of birth of an individual) that would feed any hashing algorithm to make hash comparison always trustable.
|
|
30
|
+
It's based on the work for the Empreinte Sociométrique™ patented by Cyril Dever for Edgewhere.
|
|
31
|
+
|
|
32
|
+
### TypeScript version
|
|
33
|
+
|
|
34
|
+
The TypeScript library lives at [cyrildever/es-normalizer](https://github.com/cyrildever/es-normalizer) and is published on npm as `es-normalizer`.
|
|
35
|
+
Version `0.2.4` of this port matches its version `2.4.0`: both give the same outputs for the same inputs, the test suite of es-normalizer is ported here case by case, and their dictionaries are the same.
|
|
36
|
+
An address dictionary key may hold several words (eg. `CENTRE COMMERCIAL`), translated together before the single words, the longest first.
|
|
37
|
+
|
|
38
|
+
### Usage
|
|
39
|
+
|
|
40
|
+
```
|
|
41
|
+
pip install es-normalizer-py
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
*IMPORTANT*: as of this version, most normalizers are for use on French data.
|
|
45
|
+
|
|
46
|
+
To get a normalized string, pass the data and a normalizer function to `normalize()`: it returns the normalized string, or `None` when nothing usable is left.
|
|
47
|
+
There are currently eleven specific normalizer functions and a generic one:
|
|
48
|
+
* `Any`: the generic normalizer should be used if no specific normalizer exists;
|
|
49
|
+
* `AddressLine`: pass any address line through it to get a normalized address, eg. `8, rue Henner` becomes `8 RUE HENNER`, and eventually `True` as parameter when trying to normalize a French address line 1 to also transform the title, eg. `Monsieur Cyril Dever` should then become `1 CYRIL DEVER` when passed `True` instead of `MONSIEUR CYRIL DEVER` for the default behaviour;
|
|
50
|
+
* `AddressLine6`: the French address line 6 with the department inserted in between the zip code and the city as in the Empreinte Sociométrique implementation (eg. `75009 75 PARIS`);
|
|
51
|
+
* `City`: for normalizing city names (it removes any Cedex mention in French address, for instance);
|
|
52
|
+
* `CodePostalFrance`: for French zip code;
|
|
53
|
+
* `DateOfBirth`: pass a date and up to two parameters: the input format (eg. `YYYY-MM-DD`) and the output format wished (eg. `ISO_DATE`), the default values being respectively the ISO format and the French date format;
|
|
54
|
+
* `DepartementFrance`: extract the French departement number (out of a code postal, for instance);
|
|
55
|
+
* `Email`: validates the passed e-mail and returns it in lower-case;
|
|
56
|
+
* `FirstName`: pass a first name and get a normalized one (it uses an enlarged French dictionay of first names to process it making it possible to pass from `"J.-Louis"` to `"JEAN LOUIS"`);
|
|
57
|
+
* `Mobile`: to validate a French mobile phone;
|
|
58
|
+
* `PhoneNumber`: to normalize a French phone or fax number in the international format, eg. `+33 (0) 123 456 789`;
|
|
59
|
+
* `StreetNumber`: parses the passed field to normalize it the Empreinte Sociométrique™'s way, eg. `1bis` or `1 bis` becomes `1B`;
|
|
60
|
+
* `Title`: returns a code depending on the passed string (1 for gentlemen, 2 for ladies, 0 when undefined or unknown).
|
|
61
|
+
|
|
62
|
+
```python
|
|
63
|
+
from es_normalizer import *
|
|
64
|
+
|
|
65
|
+
print(normalize('#This is any String(). ', Any))
|
|
66
|
+
# THIS IS ANY STRING
|
|
67
|
+
|
|
68
|
+
print(normalize('Monsieur Cyril Dever', AddressLine(True)))
|
|
69
|
+
# 1 CYRIL DEVER
|
|
70
|
+
|
|
71
|
+
print(normalize('24, rué de Maubeuge', AddressLine()))
|
|
72
|
+
# 24 RUE MAUBEUGE
|
|
73
|
+
|
|
74
|
+
print(normalize('75009 .# Paris', AddressLine6()))
|
|
75
|
+
# 75009 75 PARIS
|
|
76
|
+
|
|
77
|
+
print(normalize('75009 .# Paris', AddressLine6(False)))
|
|
78
|
+
# 75009 PARIS
|
|
79
|
+
|
|
80
|
+
print(normalize('Paris Cedex 09', City))
|
|
81
|
+
# PARIS
|
|
82
|
+
|
|
83
|
+
print(normalize('70/12/01', DateOfBirth('YY/MM/DD', FRENCH_DATE)))
|
|
84
|
+
# 01/12/1970
|
|
85
|
+
|
|
86
|
+
print(normalize(' 75009 ', DepartementFrance))
|
|
87
|
+
# 75
|
|
88
|
+
|
|
89
|
+
print(normalize(' Contact@GMAIL.com', Email))
|
|
90
|
+
# contact@gmail.com
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
`DateOfBirth` reads dates the way the forgiving parser of [moment.js](https://momentjs.com/) does, which es-normalizer relies on. Both give the same date for an input that fits its format; for an input that does not, eg. a numeric date read with a month name format (`DD MMM YYYY`), the guess of each may differ.
|
|
94
|
+
|
|
95
|
+
### Tests
|
|
96
|
+
|
|
97
|
+
```console
|
|
98
|
+
$ git clone https://github.com/cyrildever/es-normalizer-py.git
|
|
99
|
+
$ cd es-normalizer-py/
|
|
100
|
+
$ pip install -e .
|
|
101
|
+
$ python3 -m unittest discover
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
### License
|
|
106
|
+
|
|
107
|
+
This module is distributed under a MIT license. \
|
|
108
|
+
See the [LICENSE](LICENSE) file.
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
<hr />
|
|
112
|
+
© 2026 Cyril Dever. All rights reserved.
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
# es-normalizer-py
|
|
2
|
+
_Contact data normalization adapted from the Empreinte Sociométrique's normalizers (in Python)_
|
|
3
|
+
|
|
4
|
+

|
|
5
|
+

|
|
6
|
+

|
|
7
|
+

|
|
8
|
+

|
|
9
|
+
|
|
10
|
+
This is the Python port of [es-normalizer](https://github.com/cyrildever/es-normalizer), the TypeScript normalization library for contact data (address, phones, etc.) adapted from [Edgewhere](https://www.edgewhere.fr)'s Empreinte Sociométrique patented normalizers.
|
|
11
|
+
|
|
12
|
+
### Motivation
|
|
13
|
+
|
|
14
|
+
When it comes to hashing data, the necessary uniqueness of the source sometimes makes it hard to compare two hashed data, mostly if dealing with postal address. This library helps normalizing any contact data (the postal or e-mail address, the mobile or landline phone number, the title, names or date of birth of an individual) that would feed any hashing algorithm to make hash comparison always trustable.
|
|
15
|
+
It's based on the work for the Empreinte Sociométrique™ patented by Cyril Dever for Edgewhere.
|
|
16
|
+
|
|
17
|
+
### TypeScript version
|
|
18
|
+
|
|
19
|
+
The TypeScript library lives at [cyrildever/es-normalizer](https://github.com/cyrildever/es-normalizer) and is published on npm as `es-normalizer`.
|
|
20
|
+
Version `0.2.4` of this port matches its version `2.4.0`: both give the same outputs for the same inputs, the test suite of es-normalizer is ported here case by case, and their dictionaries are the same.
|
|
21
|
+
An address dictionary key may hold several words (eg. `CENTRE COMMERCIAL`), translated together before the single words, the longest first.
|
|
22
|
+
|
|
23
|
+
### Usage
|
|
24
|
+
|
|
25
|
+
```
|
|
26
|
+
pip install es-normalizer-py
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
*IMPORTANT*: as of this version, most normalizers are for use on French data.
|
|
30
|
+
|
|
31
|
+
To get a normalized string, pass the data and a normalizer function to `normalize()`: it returns the normalized string, or `None` when nothing usable is left.
|
|
32
|
+
There are currently eleven specific normalizer functions and a generic one:
|
|
33
|
+
* `Any`: the generic normalizer should be used if no specific normalizer exists;
|
|
34
|
+
* `AddressLine`: pass any address line through it to get a normalized address, eg. `8, rue Henner` becomes `8 RUE HENNER`, and eventually `True` as parameter when trying to normalize a French address line 1 to also transform the title, eg. `Monsieur Cyril Dever` should then become `1 CYRIL DEVER` when passed `True` instead of `MONSIEUR CYRIL DEVER` for the default behaviour;
|
|
35
|
+
* `AddressLine6`: the French address line 6 with the department inserted in between the zip code and the city as in the Empreinte Sociométrique implementation (eg. `75009 75 PARIS`);
|
|
36
|
+
* `City`: for normalizing city names (it removes any Cedex mention in French address, for instance);
|
|
37
|
+
* `CodePostalFrance`: for French zip code;
|
|
38
|
+
* `DateOfBirth`: pass a date and up to two parameters: the input format (eg. `YYYY-MM-DD`) and the output format wished (eg. `ISO_DATE`), the default values being respectively the ISO format and the French date format;
|
|
39
|
+
* `DepartementFrance`: extract the French departement number (out of a code postal, for instance);
|
|
40
|
+
* `Email`: validates the passed e-mail and returns it in lower-case;
|
|
41
|
+
* `FirstName`: pass a first name and get a normalized one (it uses an enlarged French dictionay of first names to process it making it possible to pass from `"J.-Louis"` to `"JEAN LOUIS"`);
|
|
42
|
+
* `Mobile`: to validate a French mobile phone;
|
|
43
|
+
* `PhoneNumber`: to normalize a French phone or fax number in the international format, eg. `+33 (0) 123 456 789`;
|
|
44
|
+
* `StreetNumber`: parses the passed field to normalize it the Empreinte Sociométrique™'s way, eg. `1bis` or `1 bis` becomes `1B`;
|
|
45
|
+
* `Title`: returns a code depending on the passed string (1 for gentlemen, 2 for ladies, 0 when undefined or unknown).
|
|
46
|
+
|
|
47
|
+
```python
|
|
48
|
+
from es_normalizer import *
|
|
49
|
+
|
|
50
|
+
print(normalize('#This is any String(). ', Any))
|
|
51
|
+
# THIS IS ANY STRING
|
|
52
|
+
|
|
53
|
+
print(normalize('Monsieur Cyril Dever', AddressLine(True)))
|
|
54
|
+
# 1 CYRIL DEVER
|
|
55
|
+
|
|
56
|
+
print(normalize('24, rué de Maubeuge', AddressLine()))
|
|
57
|
+
# 24 RUE MAUBEUGE
|
|
58
|
+
|
|
59
|
+
print(normalize('75009 .# Paris', AddressLine6()))
|
|
60
|
+
# 75009 75 PARIS
|
|
61
|
+
|
|
62
|
+
print(normalize('75009 .# Paris', AddressLine6(False)))
|
|
63
|
+
# 75009 PARIS
|
|
64
|
+
|
|
65
|
+
print(normalize('Paris Cedex 09', City))
|
|
66
|
+
# PARIS
|
|
67
|
+
|
|
68
|
+
print(normalize('70/12/01', DateOfBirth('YY/MM/DD', FRENCH_DATE)))
|
|
69
|
+
# 01/12/1970
|
|
70
|
+
|
|
71
|
+
print(normalize(' 75009 ', DepartementFrance))
|
|
72
|
+
# 75
|
|
73
|
+
|
|
74
|
+
print(normalize(' Contact@GMAIL.com', Email))
|
|
75
|
+
# contact@gmail.com
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
`DateOfBirth` reads dates the way the forgiving parser of [moment.js](https://momentjs.com/) does, which es-normalizer relies on. Both give the same date for an input that fits its format; for an input that does not, eg. a numeric date read with a month name format (`DD MMM YYYY`), the guess of each may differ.
|
|
79
|
+
|
|
80
|
+
### Tests
|
|
81
|
+
|
|
82
|
+
```console
|
|
83
|
+
$ git clone https://github.com/cyrildever/es-normalizer-py.git
|
|
84
|
+
$ cd es-normalizer-py/
|
|
85
|
+
$ pip install -e .
|
|
86
|
+
$ python3 -m unittest discover
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
### License
|
|
91
|
+
|
|
92
|
+
This module is distributed under a MIT license. \
|
|
93
|
+
See the [LICENSE](LICENSE) file.
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
<hr />
|
|
97
|
+
© 2026 Cyril Dever. All rights reserved.
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# pyproject.toml
|
|
2
|
+
|
|
3
|
+
[build-system]
|
|
4
|
+
requires = ["setuptools>=77.0.0", "wheel"]
|
|
5
|
+
build-backend = "setuptools.build_meta"
|
|
6
|
+
|
|
7
|
+
[project]
|
|
8
|
+
name = "es-normalizer-py"
|
|
9
|
+
version = "0.2.4"
|
|
10
|
+
description = "Contact data normalization adapted from the Empreinte Sociométrique's normalizers, the Python port of es-normalizer"
|
|
11
|
+
readme = "README.md"
|
|
12
|
+
authors = [{ name = "Cyril Dever", email = "cdever@pep-s.com" }]
|
|
13
|
+
license = "MIT"
|
|
14
|
+
license-files = ["LICENSE"]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Programming Language :: Python :: 3",
|
|
17
|
+
"Operating System :: OS Independent",
|
|
18
|
+
]
|
|
19
|
+
keywords = [
|
|
20
|
+
"data",
|
|
21
|
+
"contact",
|
|
22
|
+
"normalization",
|
|
23
|
+
"empreinte sociometrique",
|
|
24
|
+
]
|
|
25
|
+
dependencies = []
|
|
26
|
+
requires-python = ">=3.10.2"
|
|
27
|
+
|
|
28
|
+
[project.urls]
|
|
29
|
+
Homepage = "https://github.com/cyrildever/es-normalizer-py"
|
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Contact data normalization adapted from the Empreinte Sociométrique's normalizers, the Python port of es-normalizer
|
|
3
|
+
|
|
4
|
+
Every normalizer takes a string and returns the normalized string, or `None` when nothing usable is left.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import re
|
|
8
|
+
import unicodedata
|
|
9
|
+
from typing import Callable, Optional
|
|
10
|
+
|
|
11
|
+
from .dictionary import Dictionary, address_dictionary, first_name_dictionary, get_set, title_dictionary
|
|
12
|
+
|
|
13
|
+
Normalizer = Callable[[str], Optional[str]]
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def normalize(data: str, normalizer: Normalizer) -> Optional[str]:
|
|
17
|
+
return normalizer(data)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def uniformize(data: str) -> Optional[str]:
|
|
21
|
+
"""
|
|
22
|
+
Upper-cases the passed string without its accents, any run of other characters than letters, digits and dashes becoming a single space
|
|
23
|
+
"""
|
|
24
|
+
normalized = "".join(c for c in unicodedata.normalize("NFD", data.strip().lower()) if not "̀" <= c <= "ͯ")
|
|
25
|
+
if normalized == "":
|
|
26
|
+
return None
|
|
27
|
+
uniformized = normalized.replace("ß", "ss").replace("ø", "o")
|
|
28
|
+
uniformized = re.sub(r"[^a-zA-Z0-9-]+", "-", uniformized)
|
|
29
|
+
uniformized = re.sub(r"-+", "-", uniformized).replace("-", " ").strip().upper()
|
|
30
|
+
return uniformized or None
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def Any(data: str) -> Optional[str]: # noqa: N802
|
|
34
|
+
"""
|
|
35
|
+
Normalizer to use for any kind of data when no specific normalizer exists
|
|
36
|
+
"""
|
|
37
|
+
return uniformize(data)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def AddressLine(transform_title: bool = False) -> Normalizer: # noqa: N802
|
|
41
|
+
"""
|
|
42
|
+
Normalizer of an address line, eg. `8, rue Henner` becomes `8 RUE HENNER`; with `transform_title`, a French address
|
|
43
|
+
line 1 gets its title coded, eg. `Monsieur Cyril Dever` becomes `1 CYRIL DEVER`
|
|
44
|
+
"""
|
|
45
|
+
|
|
46
|
+
def normalizer(data: str) -> Optional[str]:
|
|
47
|
+
uniformized = uniformize(data)
|
|
48
|
+
if uniformized is None:
|
|
49
|
+
return None
|
|
50
|
+
return (title_dictionary if transform_title else address_dictionary).translate_text(uniformized)
|
|
51
|
+
|
|
52
|
+
return normalizer
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
_AL6 = re.compile(r"^((0[1-9]|[1-8][0-9]|9[0-5]|2[AB]|97[1-8]|98[46-9])([0-9]{3}))(.*)$")
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def AddressLine6(use_department_france: bool = True) -> Normalizer: # noqa: N802
|
|
59
|
+
"""
|
|
60
|
+
Normalizer of the French address line 6, the département inserted between the postal code and the town, eg. `75009 75 PARIS`
|
|
61
|
+
"""
|
|
62
|
+
|
|
63
|
+
def normalizer(data: str) -> Optional[str]:
|
|
64
|
+
line = AddressLine(False)(data)
|
|
65
|
+
if line is None:
|
|
66
|
+
return None
|
|
67
|
+
matches = _AL6.match(line)
|
|
68
|
+
if not matches or not matches.group(1):
|
|
69
|
+
return None
|
|
70
|
+
cp, dpt, city = matches.group(1), matches.group(2), matches.group(4)
|
|
71
|
+
if city.strip().startswith(dpt):
|
|
72
|
+
parts = [cp, city.strip()] if use_department_france else [cp, city.replace(dpt, "", 1).strip()]
|
|
73
|
+
else:
|
|
74
|
+
parts = [cp, dpt, city.strip()] if use_department_france else [cp, city.strip()]
|
|
75
|
+
return " ".join(parts).strip()
|
|
76
|
+
|
|
77
|
+
return normalizer
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def City(data: str) -> Optional[str]: # noqa: N802
|
|
81
|
+
"""
|
|
82
|
+
Normalizer of a town's name, its CEDEX mention removed
|
|
83
|
+
"""
|
|
84
|
+
uniformized = uniformize(data)
|
|
85
|
+
if uniformized is None:
|
|
86
|
+
return None
|
|
87
|
+
found = re.sub(r"CDX(\s+[0-9]+)*", "", address_dictionary.translate_text(uniformized), count=1)
|
|
88
|
+
return re.sub(r"\s+", " ", found).strip()
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
_CP = re.compile(r"^((0[1-9][0-9]|[1-8][0-9]{2})|(9[0-5][0-9])|(2[AB][0-9])|(97[1-6]))[0-9]{2}$")
|
|
92
|
+
_CORSE = re.compile(r"^2[AB][0-9]{3}$")
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def CodePostalFrance(data: str) -> Optional[str]: # noqa: N802
|
|
96
|
+
"""
|
|
97
|
+
Normalizer of a French postal code, a Corsican one written with its département (`2A165`) becoming `20165`
|
|
98
|
+
"""
|
|
99
|
+
uniformized = uniformize(data)
|
|
100
|
+
if uniformized is None:
|
|
101
|
+
return None
|
|
102
|
+
words = re.split(r"\s", uniformized)
|
|
103
|
+
candidate = words[0] if len(words) > 1 else uniformized
|
|
104
|
+
if not _CP.search(candidate):
|
|
105
|
+
return None
|
|
106
|
+
return re.sub(r"[AB]", "0", candidate, count=1) if _CORSE.search(candidate) else candidate
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
_DPT = re.compile(r"^(0[1-9]|[1-8][0-9]|9[0-5]|2[AB]|97[1-8]|98[46-9])([0-9]*)$")
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def DepartementFrance(data: str) -> Optional[str]: # noqa: N802
|
|
113
|
+
"""
|
|
114
|
+
Normalizer extracting the French département out of a postal code, eg. `2A` out of `20167`
|
|
115
|
+
"""
|
|
116
|
+
uniformized = uniformize(data)
|
|
117
|
+
if not uniformized:
|
|
118
|
+
return None
|
|
119
|
+
matches = _DPT.match(uniformized)
|
|
120
|
+
if not matches or not matches.group(1):
|
|
121
|
+
return None
|
|
122
|
+
dpt = matches.group(1)
|
|
123
|
+
if dpt != "20":
|
|
124
|
+
return dpt
|
|
125
|
+
cp = CodePostalFrance(data)
|
|
126
|
+
if cp is None:
|
|
127
|
+
return None
|
|
128
|
+
return "2A" if 19999 < int(cp) < 20200 else "2B"
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
_EMAIL = re.compile(
|
|
132
|
+
r"^(?:[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*|(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f]))"
|
|
133
|
+
r"@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+(?:[a-z]{2,}[a-z0-9-]*|([a-z][a-z0-9-]+))|(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.?){4})$"
|
|
134
|
+
)
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def Email(data: str) -> Optional[str]: # noqa: N802
|
|
138
|
+
"""
|
|
139
|
+
Normalizer validating an e-mail address and lower-casing it
|
|
140
|
+
"""
|
|
141
|
+
processed = data.strip().lower()
|
|
142
|
+
if len(processed) > 255 or not _EMAIL.search(processed):
|
|
143
|
+
return None
|
|
144
|
+
return processed
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def FirstName(data: str) -> Optional[str]: # noqa: N802
|
|
148
|
+
"""
|
|
149
|
+
Normalizer of a first name, through a French dictionary of first names, eg. `J Louis` becomes `JEAN LOUIS`
|
|
150
|
+
"""
|
|
151
|
+
uniformized = uniformize(data)
|
|
152
|
+
return None if uniformized is None else first_name_dictionary.translate_word(uniformized)
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
_MOBILE = re.compile(r"^(((00)?(33))|0)?([0]?)([0-9])([0-9]{2})([0-9]{3})([0-9]{3})$")
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def Mobile(data: str) -> Optional[str]: # noqa: N802
|
|
159
|
+
"""
|
|
160
|
+
Normalizer of a French mobile phone number in the international format, eg. `+33 (0) 623 456 789`
|
|
161
|
+
"""
|
|
162
|
+
uniformized = uniformize(data)
|
|
163
|
+
if uniformized is None:
|
|
164
|
+
return None
|
|
165
|
+
m = _MOBILE.match(re.sub(r"\s", "", uniformized))
|
|
166
|
+
if not m:
|
|
167
|
+
return None
|
|
168
|
+
if not (m.group(7) and m.group(8) and m.group(9)) or (m.group(7) + m.group(8) + m.group(9)).endswith("00000000"):
|
|
169
|
+
return None
|
|
170
|
+
if not ((m.group(6) and m.group(6) == "6") or m.group(6) == "7"):
|
|
171
|
+
return None
|
|
172
|
+
international = f"+{m.group(4)}" if m.group(4) else "+33"
|
|
173
|
+
prefix = f"({m.group(5)})" if m.group(5) else "(0)"
|
|
174
|
+
return " ".join([international, prefix, m.group(6) + m.group(7), m.group(8), m.group(9)])
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
_PHONE = re.compile(r"^(((00)?(33)|0?(262)|0?(590)|0?(594)|0?(596))([^1-9]*)|0)?([^0-9]*)([0-9]{3})([0-9]{3})([0-9]{3})$")
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def PhoneNumber(data: str) -> Optional[str]: # noqa: N802
|
|
181
|
+
"""
|
|
182
|
+
Normalizer of a French phone or fax number in the international format, eg. `+33 (0) 123 456 789`
|
|
183
|
+
"""
|
|
184
|
+
uniformized = uniformize(data)
|
|
185
|
+
if uniformized is None:
|
|
186
|
+
return None
|
|
187
|
+
m = _PHONE.match(re.sub(r"\s", "", uniformized))
|
|
188
|
+
if not m:
|
|
189
|
+
return None
|
|
190
|
+
if not (m.group(11) and m.group(12) and m.group(13)) or (m.group(11) + m.group(12) + m.group(13)).endswith("00000000"):
|
|
191
|
+
return None
|
|
192
|
+
international = f"+{m.group(4)}" if m.group(4) else "+33"
|
|
193
|
+
prefix = f"({m.group(9)})" if m.group(9) else "(0)"
|
|
194
|
+
return " ".join([international, prefix, m.group(11), m.group(12), m.group(13)])
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
_STREET_NUMBER = re.compile(r"^([0-9]*)\s*(.*)$")
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def StreetNumber(data: str) -> Optional[str]: # noqa: N802
|
|
201
|
+
"""
|
|
202
|
+
Normalizer of a street number, eg. `1bis` or `1 bis` becomes `1B`
|
|
203
|
+
"""
|
|
204
|
+
m = _STREET_NUMBER.match(data.strip())
|
|
205
|
+
if not m:
|
|
206
|
+
return None
|
|
207
|
+
if not m.group(2):
|
|
208
|
+
return m.group(1)
|
|
209
|
+
uniformized = uniformize(m.group(2))
|
|
210
|
+
if uniformized is None:
|
|
211
|
+
return None
|
|
212
|
+
return m.group(1) + address_dictionary.translate_text(uniformized).split(" ")[0]
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
_CODE_F, _CODE_M, _CODE_U = "2", "1", "0"
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def Title(data: str) -> Optional[str]: # noqa: N802
|
|
219
|
+
"""
|
|
220
|
+
Normalizer of a title into a code: 1 for gentlemen, 2 for ladies, 0 when undefined or unknown
|
|
221
|
+
"""
|
|
222
|
+
translated = uniformize(data)
|
|
223
|
+
if translated is None:
|
|
224
|
+
return None
|
|
225
|
+
if translated in ("H", "M"):
|
|
226
|
+
return _CODE_M
|
|
227
|
+
if translated == "F":
|
|
228
|
+
return _CODE_F
|
|
229
|
+
if translated == "U":
|
|
230
|
+
return _CODE_U
|
|
231
|
+
found = re.findall(r"[0-9]+", title_dictionary.translate_text(translated))
|
|
232
|
+
if not found:
|
|
233
|
+
return None
|
|
234
|
+
return found[0] if len(found) == 1 else _CODE_U
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
from .date_of_birth import FRENCH_DATE, ISO_DATE, DateOfBirth, Milliseconds, Timestamp # noqa: E402
|
|
238
|
+
|
|
239
|
+
__all__ = [
|
|
240
|
+
"Dictionary", "get_set", "normalize", "uniformize", "Any", "AddressLine", "AddressLine6", "City", "CodePostalFrance",
|
|
241
|
+
"DateOfBirth", "DepartementFrance", "Email", "FirstName", "Mobile", "PhoneNumber", "StreetNumber", "Title",
|
|
242
|
+
"FRENCH_DATE", "ISO_DATE", "Timestamp", "Milliseconds",
|
|
243
|
+
]
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
"""
|
|
2
|
+
The date of birth normalizer, which reads and writes dates the way moment.js does in its forgiving mode, as es-normalizer relies on it
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
import calendar
|
|
6
|
+
import re
|
|
7
|
+
from datetime import date, datetime
|
|
8
|
+
from typing import Callable, Optional
|
|
9
|
+
|
|
10
|
+
FRENCH_DATE = "DD/MM/YYYY"
|
|
11
|
+
ISO_DATE = "YYYYMMDD"
|
|
12
|
+
Timestamp = "timestamp"
|
|
13
|
+
Milliseconds = "timestamp_millis"
|
|
14
|
+
|
|
15
|
+
_SEPARATORS = re.compile(r"[-:/\s]+")
|
|
16
|
+
_TOKENS = re.compile(r"YYYY|YY|MMMM|MMM|MM|M|DD|D|HH|H|SS|S")
|
|
17
|
+
_MONTHS = [calendar.month_name[i] for i in range(1, 13)]
|
|
18
|
+
_SHORT_MONTHS = [calendar.month_abbr[i] for i in range(1, 13)]
|
|
19
|
+
_MONTH_NAME = re.compile("|".join(_MONTHS + _SHORT_MONTHS), re.IGNORECASE)
|
|
20
|
+
# What each token matches, searched anywhere in what is left of the input
|
|
21
|
+
_PARSE = {
|
|
22
|
+
"YYYY": re.compile(r"\d{1,4}"),
|
|
23
|
+
"YY": re.compile(r"\d\d?"),
|
|
24
|
+
"MMMM": _MONTH_NAME,
|
|
25
|
+
"MMM": _MONTH_NAME,
|
|
26
|
+
"MM": re.compile(r"\d\d?"),
|
|
27
|
+
"M": re.compile(r"\d\d?"),
|
|
28
|
+
"DD": re.compile(r"\d\d?"),
|
|
29
|
+
"D": re.compile(r"\d\d?"),
|
|
30
|
+
"HH": re.compile(r"\d\d?"),
|
|
31
|
+
"H": re.compile(r"\d\d?"),
|
|
32
|
+
"SS": re.compile(r"\d+"),
|
|
33
|
+
"S": re.compile(r"\d+"),
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _month_of(name: str) -> int:
|
|
38
|
+
name = name.lower()
|
|
39
|
+
for i, (full, short) in enumerate(zip(_MONTHS, _SHORT_MONTHS), start=1):
|
|
40
|
+
if name == full.lower() or name == short.lower():
|
|
41
|
+
return i
|
|
42
|
+
return 0
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _two_digit_year(value: str) -> int:
|
|
46
|
+
return int(value) + (1900 if int(value) > 68 else 2000)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _parse(data: str, fmt: str) -> Optional[date]:
|
|
50
|
+
year, month, day = date.today().year, 1, 1
|
|
51
|
+
rest = data
|
|
52
|
+
parsed = False
|
|
53
|
+
for token in _TOKENS.findall(fmt):
|
|
54
|
+
found = _PARSE[token].search(rest)
|
|
55
|
+
if not found:
|
|
56
|
+
continue
|
|
57
|
+
rest = rest[found.end():]
|
|
58
|
+
parsed = True
|
|
59
|
+
value = found.group(0)
|
|
60
|
+
if token == "YYYY":
|
|
61
|
+
# As moment.js: a year of two digits is read as one of `YY`
|
|
62
|
+
year = _two_digit_year(value) if len(value) == 2 else int(value)
|
|
63
|
+
elif token == "YY":
|
|
64
|
+
year = _two_digit_year(value)
|
|
65
|
+
elif token in ("MMMM", "MMM"):
|
|
66
|
+
month = _month_of(value)
|
|
67
|
+
elif token in ("MM", "M"):
|
|
68
|
+
month = int(value)
|
|
69
|
+
elif token in ("DD", "D"):
|
|
70
|
+
day = int(value)
|
|
71
|
+
if not parsed:
|
|
72
|
+
return None
|
|
73
|
+
try:
|
|
74
|
+
return date(year, month, day)
|
|
75
|
+
except ValueError:
|
|
76
|
+
return None
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _format(d: date, fmt: str) -> str:
|
|
80
|
+
def token(m: re.Match) -> str:
|
|
81
|
+
t = m.group(0)
|
|
82
|
+
return {
|
|
83
|
+
"YYYY": f"{d.year:04d}",
|
|
84
|
+
"YY": f"{d.year % 100:02d}",
|
|
85
|
+
"MMMM": _MONTHS[d.month - 1],
|
|
86
|
+
"MMM": _SHORT_MONTHS[d.month - 1],
|
|
87
|
+
"MM": f"{d.month:02d}",
|
|
88
|
+
"M": str(d.month),
|
|
89
|
+
"DD": f"{d.day:02d}",
|
|
90
|
+
"D": str(d.day),
|
|
91
|
+
"HH": "00",
|
|
92
|
+
"H": "0",
|
|
93
|
+
"SS": "00",
|
|
94
|
+
"S": "0",
|
|
95
|
+
}[t]
|
|
96
|
+
|
|
97
|
+
return _TOKENS.sub(token, fmt)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _leading_int(data: str) -> Optional[int]:
|
|
101
|
+
# As JavaScript's `parseInt`: the leading digits, sign included
|
|
102
|
+
found = re.match(r"\s*([+-]?\d+)", data)
|
|
103
|
+
return int(found.group(1)) if found else None
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def DateOfBirth(input_format: str = None, output_format: str = None) -> Callable[[str], Optional[str]]: # noqa: N802
|
|
107
|
+
"""
|
|
108
|
+
Normalizer of a date of birth, read in the input format (ISO by default) and written in the output one (French by
|
|
109
|
+
default), eg. `24/04/2010` read as `DD/MM/YYYY` and written as `ISO_DATE` becomes `20100424`
|
|
110
|
+
"""
|
|
111
|
+
inp = ISO_DATE if not input_format else input_format
|
|
112
|
+
out = FRENCH_DATE if not output_format else output_format
|
|
113
|
+
|
|
114
|
+
def normalizer(data: str) -> Optional[str]:
|
|
115
|
+
if inp.lower() in (Timestamp, Milliseconds):
|
|
116
|
+
value = _leading_int(data)
|
|
117
|
+
if value is None:
|
|
118
|
+
return None
|
|
119
|
+
d = datetime.fromtimestamp(value if inp.lower() == Timestamp else value / 1000).date()
|
|
120
|
+
else:
|
|
121
|
+
d = _parse(data, _SEPARATORS.sub("-", inp).upper())
|
|
122
|
+
if d is None:
|
|
123
|
+
return None
|
|
124
|
+
written = _format(d, _SEPARATORS.sub("-", out).upper())
|
|
125
|
+
separator = _SEPARATORS.search(out)
|
|
126
|
+
return written if separator is None else written.replace("-", separator.group(0))
|
|
127
|
+
|
|
128
|
+
return normalizer
|