rowsift 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rowsift/__init__.py +98 -0
- rowsift/clean/__init__.py +44 -0
- rowsift/clean/_common.py +30 -0
- rowsift/clean/_dates.py +283 -0
- rowsift/clean/_dispatch.py +49 -0
- rowsift/clean/_identifiers.py +161 -0
- rowsift/clean/_mask.py +117 -0
- rowsift/clean/_numbers.py +137 -0
- rowsift/clean/_options.py +75 -0
- rowsift/clean/_phone.py +202 -0
- rowsift/clean/_text.py +102 -0
- rowsift/errors.py +124 -0
- rowsift/map/__init__.py +24 -0
- rowsift/map/_fields.py +75 -0
- rowsift/map/_saved.py +132 -0
- rowsift/map/_score.py +178 -0
- rowsift/map/_suggest.py +168 -0
- rowsift/map/_types.py +169 -0
- rowsift/profile/__init__.py +17 -0
- rowsift/profile/_headers.py +44 -0
- rowsift/profile/_kinds.py +165 -0
- rowsift/profile/_options.py +69 -0
- rowsift/profile/_profile.py +119 -0
- rowsift/profile/_structure.py +157 -0
- rowsift/profile/_types.py +96 -0
- rowsift/py.typed +0 -0
- rowsift/read/__init__.py +6 -0
- rowsift/read/_cells.py +93 -0
- rowsift/read/_detect.py +124 -0
- rowsift/read/_grid.py +128 -0
- rowsift/read/_limits.py +44 -0
- rowsift/read/_read.py +99 -0
- rowsift/read/_text.py +250 -0
- rowsift/read/_workbook.py +36 -0
- rowsift/read/_xls.py +115 -0
- rowsift/read/_xlsx.py +121 -0
- rowsift/sift/__init__.py +19 -0
- rowsift/sift/_sift.py +165 -0
- rowsift/sift/_types.py +119 -0
- rowsift/types.py +378 -0
- rowsift-0.1.1.dist-info/METADATA +148 -0
- rowsift-0.1.1.dist-info/RECORD +45 -0
- rowsift-0.1.1.dist-info/WHEEL +4 -0
- rowsift-0.1.1.dist-info/licenses/LICENSE +177 -0
- rowsift-0.1.1.dist-info/licenses/NOTICE +4 -0
rowsift/__init__.py
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
"""Read spreadsheets written by people and turn them into typed, clean rows."""
|
|
2
|
+
|
|
3
|
+
from rowsift import clean
|
|
4
|
+
from rowsift.clean import CleanOptions, PhoneNumber
|
|
5
|
+
from rowsift.errors import (
|
|
6
|
+
DecodingError,
|
|
7
|
+
ExtraName,
|
|
8
|
+
LimitExceededError,
|
|
9
|
+
LimitName,
|
|
10
|
+
MissingExtraError,
|
|
11
|
+
RowsiftError,
|
|
12
|
+
UnsupportedFileError,
|
|
13
|
+
)
|
|
14
|
+
from rowsift.map import (
|
|
15
|
+
Alternative,
|
|
16
|
+
Choice,
|
|
17
|
+
Field,
|
|
18
|
+
Mapping,
|
|
19
|
+
SavedMapping,
|
|
20
|
+
apply_saved,
|
|
21
|
+
suggest,
|
|
22
|
+
)
|
|
23
|
+
from rowsift.profile import ColumnProfile, Profile, ProfileOptions, profile
|
|
24
|
+
from rowsift.read import ReadLimits, read
|
|
25
|
+
from rowsift.sift import FieldSummary, Rejection, SiftedRow, SiftResult, Summary, sift
|
|
26
|
+
from rowsift.types import (
|
|
27
|
+
Change,
|
|
28
|
+
ChangeCode,
|
|
29
|
+
Cleaned,
|
|
30
|
+
Confidence,
|
|
31
|
+
DataValue,
|
|
32
|
+
DayMonth,
|
|
33
|
+
ErrorCode,
|
|
34
|
+
FileFormat,
|
|
35
|
+
Issue,
|
|
36
|
+
IssueCode,
|
|
37
|
+
Kind,
|
|
38
|
+
LineType,
|
|
39
|
+
ReadNote,
|
|
40
|
+
ReadReport,
|
|
41
|
+
Reason,
|
|
42
|
+
ReasonCode,
|
|
43
|
+
SkipCode,
|
|
44
|
+
Table,
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
__version__ = "0.1.1"
|
|
48
|
+
|
|
49
|
+
__all__ = [
|
|
50
|
+
"Alternative",
|
|
51
|
+
"Change",
|
|
52
|
+
"ChangeCode",
|
|
53
|
+
"Choice",
|
|
54
|
+
"CleanOptions",
|
|
55
|
+
"Cleaned",
|
|
56
|
+
"ColumnProfile",
|
|
57
|
+
"Confidence",
|
|
58
|
+
"DataValue",
|
|
59
|
+
"DayMonth",
|
|
60
|
+
"DecodingError",
|
|
61
|
+
"ErrorCode",
|
|
62
|
+
"ExtraName",
|
|
63
|
+
"Field",
|
|
64
|
+
"FieldSummary",
|
|
65
|
+
"FileFormat",
|
|
66
|
+
"Issue",
|
|
67
|
+
"IssueCode",
|
|
68
|
+
"Kind",
|
|
69
|
+
"LimitExceededError",
|
|
70
|
+
"LimitName",
|
|
71
|
+
"LineType",
|
|
72
|
+
"Mapping",
|
|
73
|
+
"MissingExtraError",
|
|
74
|
+
"PhoneNumber",
|
|
75
|
+
"Profile",
|
|
76
|
+
"ProfileOptions",
|
|
77
|
+
"ReadLimits",
|
|
78
|
+
"ReadNote",
|
|
79
|
+
"ReadReport",
|
|
80
|
+
"Reason",
|
|
81
|
+
"ReasonCode",
|
|
82
|
+
"Rejection",
|
|
83
|
+
"RowsiftError",
|
|
84
|
+
"SavedMapping",
|
|
85
|
+
"SiftResult",
|
|
86
|
+
"SiftedRow",
|
|
87
|
+
"SkipCode",
|
|
88
|
+
"Summary",
|
|
89
|
+
"Table",
|
|
90
|
+
"UnsupportedFileError",
|
|
91
|
+
"__version__",
|
|
92
|
+
"apply_saved",
|
|
93
|
+
"clean",
|
|
94
|
+
"profile",
|
|
95
|
+
"read",
|
|
96
|
+
"sift",
|
|
97
|
+
"suggest",
|
|
98
|
+
]
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"""Clean one value: the normalized value and what changed, or the reason it failed.
|
|
2
|
+
|
|
3
|
+
Every cleaner takes the text of a cell and an optional ``CleanOptions`` and
|
|
4
|
+
returns a ``Cleaned``: a ``value`` or an ``issue`` code, plus the ``changes``
|
|
5
|
+
made on the way. ``value`` cleans by ``Kind``. The ``mask`` functions make a
|
|
6
|
+
value safe to display.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from rowsift.clean._dates import date, datetime, day_month, time
|
|
10
|
+
from rowsift.clean._dispatch import value
|
|
11
|
+
from rowsift.clean._identifiers import cnpj, cpf, email, postal_code
|
|
12
|
+
from rowsift.clean._mask import mask, mask_cnpj, mask_cpf, mask_email, mask_name, mask_phone
|
|
13
|
+
from rowsift.clean._numbers import decimal, integer, money
|
|
14
|
+
from rowsift.clean._options import CleanOptions
|
|
15
|
+
from rowsift.clean._phone import PhoneNumber, phone
|
|
16
|
+
from rowsift.clean._text import boolean, category, name, text
|
|
17
|
+
|
|
18
|
+
__all__ = [
|
|
19
|
+
"CleanOptions",
|
|
20
|
+
"PhoneNumber",
|
|
21
|
+
"boolean",
|
|
22
|
+
"category",
|
|
23
|
+
"cnpj",
|
|
24
|
+
"cpf",
|
|
25
|
+
"date",
|
|
26
|
+
"datetime",
|
|
27
|
+
"day_month",
|
|
28
|
+
"decimal",
|
|
29
|
+
"email",
|
|
30
|
+
"integer",
|
|
31
|
+
"mask",
|
|
32
|
+
"mask_cnpj",
|
|
33
|
+
"mask_cpf",
|
|
34
|
+
"mask_email",
|
|
35
|
+
"mask_name",
|
|
36
|
+
"mask_phone",
|
|
37
|
+
"money",
|
|
38
|
+
"name",
|
|
39
|
+
"phone",
|
|
40
|
+
"postal_code",
|
|
41
|
+
"text",
|
|
42
|
+
"time",
|
|
43
|
+
"value",
|
|
44
|
+
]
|
rowsift/clean/_common.py
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"""Small helpers shared by the cleaners. Private."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import unicodedata
|
|
6
|
+
from typing import TypeVar
|
|
7
|
+
|
|
8
|
+
from rowsift.types import Change, ChangeCode, Cleaned, DataValue, Issue, IssueCode
|
|
9
|
+
|
|
10
|
+
T = TypeVar("T")
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def fold(text: str, *, compatibility: bool = True) -> str:
|
|
14
|
+
"""Lowercase, accents removed, whitespace collapsed. For matching only.
|
|
15
|
+
|
|
16
|
+
``compatibility=False`` keeps compatibility forms such as a fullwidth digit apart.
|
|
17
|
+
"""
|
|
18
|
+
decomposed = unicodedata.normalize("NFKD" if compatibility else "NFD", text.casefold())
|
|
19
|
+
bare = "".join(char for char in decomposed if not unicodedata.combining(char))
|
|
20
|
+
return " ".join(bare.split())
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def fail(code: IssueCode, **data: DataValue) -> Cleaned[T]:
|
|
24
|
+
"""A result that carries an issue and no value."""
|
|
25
|
+
return Cleaned(issue=Issue(code, data))
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def change(code: ChangeCode, **data: DataValue) -> Change:
|
|
29
|
+
"""A change with keyword data."""
|
|
30
|
+
return Change(code, data)
|
rowsift/clean/_dates.py
ADDED
|
@@ -0,0 +1,283 @@
|
|
|
1
|
+
"""Date, day and month, time and datetime cleaners."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import datetime as dt
|
|
6
|
+
import re
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
|
|
9
|
+
from rowsift.clean._common import change, fail, fold
|
|
10
|
+
from rowsift.clean._options import CleanOptions
|
|
11
|
+
from rowsift.types import Change, ChangeCode, Cleaned, DayMonth, IssueCode
|
|
12
|
+
|
|
13
|
+
__all__ = ["date", "datetime", "day_month", "time"]
|
|
14
|
+
|
|
15
|
+
_MONTHS: dict[str, int] = {}
|
|
16
|
+
for _number, _names in enumerate(
|
|
17
|
+
(
|
|
18
|
+
("janeiro", "jan", "january"),
|
|
19
|
+
("fevereiro", "fev", "feb", "february"),
|
|
20
|
+
("marco", "mar", "march"),
|
|
21
|
+
("abril", "abr", "apr", "april"),
|
|
22
|
+
("maio", "mai", "may"),
|
|
23
|
+
("junho", "jun", "june"),
|
|
24
|
+
("julho", "jul", "july"),
|
|
25
|
+
("agosto", "ago", "aug", "august"),
|
|
26
|
+
("setembro", "set", "sep", "sept", "september"),
|
|
27
|
+
("outubro", "out", "oct", "october"),
|
|
28
|
+
("novembro", "nov", "november"),
|
|
29
|
+
("dezembro", "dez", "dec", "december"),
|
|
30
|
+
),
|
|
31
|
+
start=1,
|
|
32
|
+
):
|
|
33
|
+
for _name in _names:
|
|
34
|
+
_MONTHS[_name] = _number
|
|
35
|
+
|
|
36
|
+
_ASCII_DIGITS = "0123456789"
|
|
37
|
+
_SERIAL_MIN = 10_000 # 1927-05-18. Smaller numbers are far more likely to be something else.
|
|
38
|
+
_SERIAL_MAX = 2_958_465 # 9999-12-31
|
|
39
|
+
_SERIAL_EPOCH = dt.date(1899, 12, 30) # valid for every serial after the 1900 leap-year bug
|
|
40
|
+
|
|
41
|
+
_YMD = re.compile(r"([0-9]{4})([/.\-])([0-9]{1,2})\2([0-9]{1,2})")
|
|
42
|
+
_NUMERIC = re.compile(r"([0-9]{1,2})([/.\-])([0-9]{1,2})\2([0-9]{2}|[0-9]{4})")
|
|
43
|
+
_NUMERIC_NO_YEAR = re.compile(r"([0-9]{1,2})([/.\-])([0-9]{1,2})")
|
|
44
|
+
_WORDS_ALLOWED = re.compile(r"[a-z0-9 ,./\-]+")
|
|
45
|
+
_TOKEN = re.compile(r"[a-z]+|[0-9]+")
|
|
46
|
+
_SERIAL = re.compile(r"[0-9]+(?:\.[0-9]+)?")
|
|
47
|
+
_TIME = re.compile(
|
|
48
|
+
r"(?P<hour>[0-9]{1,2})(?:(?::(?P<minute>[0-9]{2})(?::(?P<second>[0-9]{2})(?:\.(?P<fraction>[0-9]{1,6}))?)?)"
|
|
49
|
+
r"|h(?P<hminute>[0-9]{2})?|(?=\s*[ap]\.?m\.?\Z))\s*(?P<meridiem>[ap]\.?m\.?)?",
|
|
50
|
+
re.IGNORECASE,
|
|
51
|
+
)
|
|
52
|
+
_TRAILING_TIME = re.compile(
|
|
53
|
+
r"^(?P<date>.*?)(?:\s+|T)"
|
|
54
|
+
r"(?P<time>[0-9]{1,2}(?::[0-9]{2}.*|h[0-9]{0,2}\s*|\s*[ap]\.?m\.?\s*))$",
|
|
55
|
+
re.IGNORECASE,
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@dataclass(frozen=True, slots=True)
|
|
60
|
+
class _Parts:
|
|
61
|
+
"""A date read from text: ``year`` is ``None`` when the text has none."""
|
|
62
|
+
|
|
63
|
+
day: int
|
|
64
|
+
month: int
|
|
65
|
+
year: int | None
|
|
66
|
+
expanded_from: int | None = None
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _expand_year(two_digits: int, pivot: int) -> int:
|
|
70
|
+
return 2000 + two_digits if two_digits <= pivot else 1900 + two_digits
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _parse_parts(text: str, options: CleanOptions) -> _Parts | None:
|
|
74
|
+
"""Day, month and year from numeric, ISO or month-name text. ``None`` if no shape fits."""
|
|
75
|
+
ymd = _YMD.fullmatch(text)
|
|
76
|
+
if ymd:
|
|
77
|
+
return _Parts(int(ymd.group(4)), int(ymd.group(3)), int(ymd.group(1)))
|
|
78
|
+
numeric = _NUMERIC.fullmatch(text)
|
|
79
|
+
if numeric:
|
|
80
|
+
first, second, year_text = int(numeric.group(1)), int(numeric.group(3)), numeric.group(4)
|
|
81
|
+
day, month = (first, second) if options.date_order == "dmy" else (second, first)
|
|
82
|
+
if len(year_text) == 2:
|
|
83
|
+
year = _expand_year(int(year_text), options.two_digit_year_pivot)
|
|
84
|
+
return _Parts(day, month, year, expanded_from=int(year_text))
|
|
85
|
+
return _Parts(day, month, int(year_text))
|
|
86
|
+
short = _NUMERIC_NO_YEAR.fullmatch(text)
|
|
87
|
+
if short:
|
|
88
|
+
first, second = int(short.group(1)), int(short.group(3))
|
|
89
|
+
day, month = (first, second) if options.date_order == "dmy" else (second, first)
|
|
90
|
+
return _Parts(day, month, None)
|
|
91
|
+
return _parse_words(text, options)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _parse_words(text: str, options: CleanOptions) -> _Parts | None:
|
|
95
|
+
if any(char.isdigit() and char not in _ASCII_DIGITS for char in text):
|
|
96
|
+
return None # fold() would turn other scripts' digits into ASCII ones
|
|
97
|
+
folded = fold(text)
|
|
98
|
+
if not _WORDS_ALLOWED.fullmatch(folded):
|
|
99
|
+
return None
|
|
100
|
+
tokens = [t for t in _TOKEN.findall(folded) if t not in ("de", "of")]
|
|
101
|
+
shapes = "".join("m" if t in _MONTHS else "n" if t.isdigit() else "?" for t in tokens)
|
|
102
|
+
if "?" in shapes:
|
|
103
|
+
return None
|
|
104
|
+
year_token: str | None
|
|
105
|
+
if shapes in ("nm", "nmn"):
|
|
106
|
+
day_token, month_token = tokens[0], tokens[1]
|
|
107
|
+
year_token = tokens[2] if shapes == "nmn" else None
|
|
108
|
+
elif shapes in ("mn", "mnn"):
|
|
109
|
+
month_token, day_token = tokens[0], tokens[1]
|
|
110
|
+
year_token = tokens[2] if shapes == "mnn" else None
|
|
111
|
+
else:
|
|
112
|
+
return None
|
|
113
|
+
if len(day_token) > 2:
|
|
114
|
+
return None
|
|
115
|
+
year: int | None
|
|
116
|
+
expanded: int | None
|
|
117
|
+
if year_token is None:
|
|
118
|
+
year, expanded = None, None
|
|
119
|
+
elif len(year_token) == 4:
|
|
120
|
+
year, expanded = int(year_token), None
|
|
121
|
+
elif len(year_token) == 2:
|
|
122
|
+
year = _expand_year(int(year_token), options.two_digit_year_pivot)
|
|
123
|
+
expanded = int(year_token)
|
|
124
|
+
else:
|
|
125
|
+
return None
|
|
126
|
+
return _Parts(int(day_token), _MONTHS[month_token], year, expanded)
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def _build(parts: _Parts) -> dt.date | None:
|
|
130
|
+
assert parts.year is not None
|
|
131
|
+
try:
|
|
132
|
+
return dt.date(parts.year, parts.month, parts.day)
|
|
133
|
+
except ValueError:
|
|
134
|
+
return None
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def _serial(text: str, *, allow_fraction: bool) -> tuple[dt.date, float] | None:
|
|
138
|
+
"""A spreadsheet serial number as a date and the fraction of the day."""
|
|
139
|
+
if not _SERIAL.fullmatch(text):
|
|
140
|
+
return None
|
|
141
|
+
whole, _, fraction = text.partition(".")
|
|
142
|
+
if len(whole) > len(str(_SERIAL_MAX)):
|
|
143
|
+
return None # also keeps int() away from absurdly long digit runs
|
|
144
|
+
if fraction.strip("0") and not allow_fraction:
|
|
145
|
+
return None
|
|
146
|
+
number = int(whole)
|
|
147
|
+
if not _SERIAL_MIN <= number <= _SERIAL_MAX:
|
|
148
|
+
return None
|
|
149
|
+
return _SERIAL_EPOCH + dt.timedelta(days=number), float("0." + fraction) if fraction else 0.0
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def _date_from_text(text: str, options: CleanOptions) -> Cleaned[dt.date]:
|
|
153
|
+
serial = _serial(text, allow_fraction=False)
|
|
154
|
+
if serial is not None:
|
|
155
|
+
return Cleaned(serial[0], changes=(change(ChangeCode.SERIAL_DATE_CONVERTED),))
|
|
156
|
+
parts = _parse_parts(text, options)
|
|
157
|
+
if parts is None or parts.year is None:
|
|
158
|
+
return fail(IssueCode.NOT_A_DATE)
|
|
159
|
+
built = _build(parts)
|
|
160
|
+
if built is None:
|
|
161
|
+
return fail(IssueCode.INVALID_DATE)
|
|
162
|
+
changes: tuple[Change, ...] = ()
|
|
163
|
+
if parts.expanded_from is not None:
|
|
164
|
+
changes = (change(ChangeCode.TWO_DIGIT_YEAR_EXPANDED, year=parts.year),)
|
|
165
|
+
return Cleaned(built, changes=changes)
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def date(value: str, options: CleanOptions | None = None) -> Cleaned[dt.date]:
|
|
169
|
+
"""A calendar date as ``datetime.date``.
|
|
170
|
+
|
|
171
|
+
Accepted: numeric dates with ``/``, ``-`` or ``.`` in the order of
|
|
172
|
+
``options.date_order`` (day first by default); ISO dates, which start with a
|
|
173
|
+
four-digit year; dates with month names or abbreviations in Portuguese or
|
|
174
|
+
English (``12 de março de 1988``, ``12-mar-88``, ``March 12, 1988``); and
|
|
175
|
+
spreadsheet serial numbers from 10000 to 2958465, reported as
|
|
176
|
+
``serial_date_converted``. A two-digit year is resolved with
|
|
177
|
+
``options.two_digit_year_pivot`` and reported as ``two_digit_year_expanded``.
|
|
178
|
+
A date that does not exist is the issue ``invalid_date``. Text that is not a
|
|
179
|
+
date, including a date with no year, is the issue ``not_a_date``.
|
|
180
|
+
"""
|
|
181
|
+
options = options or CleanOptions()
|
|
182
|
+
text = value.strip()
|
|
183
|
+
if not text:
|
|
184
|
+
return Cleaned()
|
|
185
|
+
return _date_from_text(text, options)
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def day_month(value: str, options: CleanOptions | None = None) -> Cleaned[DayMonth]:
|
|
189
|
+
"""A day and a month with no year, such as a birthday.
|
|
190
|
+
|
|
191
|
+
Accepts the forms of ``date`` with the year left out (``12/03``,
|
|
192
|
+
``12-mar``, ``12 de março``). A full date is accepted too: the year is
|
|
193
|
+
discarded and reported as ``year_discarded``, after checking that the date
|
|
194
|
+
exists. No year is ever invented, and 29 February is valid. A day that does
|
|
195
|
+
not exist in its month is the issue ``invalid_date``; text that is neither
|
|
196
|
+
is ``not_a_date``.
|
|
197
|
+
"""
|
|
198
|
+
options = options or CleanOptions()
|
|
199
|
+
text = value.strip()
|
|
200
|
+
if not text:
|
|
201
|
+
return Cleaned()
|
|
202
|
+
parts = _parse_parts(text, options)
|
|
203
|
+
if parts is None:
|
|
204
|
+
return fail(IssueCode.NOT_A_DATE)
|
|
205
|
+
if parts.year is not None:
|
|
206
|
+
built = _build(parts)
|
|
207
|
+
if built is None:
|
|
208
|
+
return fail(IssueCode.INVALID_DATE)
|
|
209
|
+
return Cleaned(
|
|
210
|
+
DayMonth(built.day, built.month), changes=(change(ChangeCode.YEAR_DISCARDED),)
|
|
211
|
+
)
|
|
212
|
+
try:
|
|
213
|
+
return Cleaned(DayMonth(parts.day, parts.month))
|
|
214
|
+
except ValueError:
|
|
215
|
+
return fail(IssueCode.INVALID_DATE)
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def _time_from_text(text: str) -> dt.time | None:
|
|
219
|
+
match = _TIME.fullmatch(text.strip())
|
|
220
|
+
if match is None:
|
|
221
|
+
return None
|
|
222
|
+
hour = int(match["hour"])
|
|
223
|
+
minute = int(match["minute"] or match["hminute"] or 0)
|
|
224
|
+
second = int(match["second"] or 0)
|
|
225
|
+
micro = int((match["fraction"] or "").ljust(6, "0") or 0)
|
|
226
|
+
meridiem = match["meridiem"]
|
|
227
|
+
if meridiem:
|
|
228
|
+
if not 1 <= hour <= 12:
|
|
229
|
+
return None
|
|
230
|
+
hour = hour % 12 + (12 if meridiem[0].lower() == "p" else 0)
|
|
231
|
+
try:
|
|
232
|
+
return dt.time(hour, minute, second, micro)
|
|
233
|
+
except ValueError:
|
|
234
|
+
return None
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def time(value: str, options: CleanOptions | None = None) -> Cleaned[dt.time]:
|
|
238
|
+
"""A time of day as ``datetime.time``.
|
|
239
|
+
|
|
240
|
+
Accepts ``H:MM``, ``HH:MM:SS`` with optional fractional seconds, ``9h30``,
|
|
241
|
+
``9h``, and a 12-hour clock with ``am`` or ``pm``. No time zone is read.
|
|
242
|
+
Anything else, or a value out of range, is the issue ``not_a_time``.
|
|
243
|
+
"""
|
|
244
|
+
text = value.strip()
|
|
245
|
+
if not text:
|
|
246
|
+
return Cleaned()
|
|
247
|
+
parsed = _time_from_text(text)
|
|
248
|
+
return Cleaned(parsed) if parsed is not None else fail(IssueCode.NOT_A_TIME)
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def datetime(value: str, options: CleanOptions | None = None) -> Cleaned[dt.datetime]:
|
|
252
|
+
"""A date and a time of day as a naive ``datetime.datetime``.
|
|
253
|
+
|
|
254
|
+
The date part follows ``date`` and the time part follows ``time``, separated
|
|
255
|
+
by a space or ``T``. A spreadsheet serial number with a fraction is
|
|
256
|
+
accepted. A date with no time becomes midnight and reports
|
|
257
|
+
``time_defaulted``. Time zones are not read.
|
|
258
|
+
"""
|
|
259
|
+
options = options or CleanOptions()
|
|
260
|
+
text = value.strip()
|
|
261
|
+
if not text:
|
|
262
|
+
return Cleaned()
|
|
263
|
+
serial = _serial(text, allow_fraction=True)
|
|
264
|
+
if serial is not None:
|
|
265
|
+
seconds = round(serial[1] * 86_400)
|
|
266
|
+
moment = dt.datetime.combine(serial[0], dt.time()) + dt.timedelta(seconds=seconds)
|
|
267
|
+
return Cleaned(moment, changes=(change(ChangeCode.SERIAL_DATE_CONVERTED),))
|
|
268
|
+
split = _TRAILING_TIME.fullmatch(text)
|
|
269
|
+
if split is None:
|
|
270
|
+
day = _date_from_text(text, options)
|
|
271
|
+
if day.value is None:
|
|
272
|
+
return Cleaned(issue=day.issue)
|
|
273
|
+
return Cleaned(
|
|
274
|
+
dt.datetime.combine(day.value, dt.time()),
|
|
275
|
+
changes=(*day.changes, change(ChangeCode.TIME_DEFAULTED)),
|
|
276
|
+
)
|
|
277
|
+
day = _date_from_text(split["date"].strip(), options)
|
|
278
|
+
if day.value is None:
|
|
279
|
+
return fail(day.issue.code) if day.issue else Cleaned()
|
|
280
|
+
clock = _time_from_text(split["time"])
|
|
281
|
+
if clock is None:
|
|
282
|
+
return fail(IssueCode.NOT_A_TIME)
|
|
283
|
+
return Cleaned(dt.datetime.combine(day.value, clock), changes=day.changes)
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
"""One function that cleans a value by its kind."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Callable
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
from rowsift.clean._dates import date, datetime, day_month, time
|
|
9
|
+
from rowsift.clean._identifiers import cnpj, cpf, email, postal_code
|
|
10
|
+
from rowsift.clean._numbers import decimal, integer, money
|
|
11
|
+
from rowsift.clean._options import CleanOptions
|
|
12
|
+
from rowsift.clean._phone import phone
|
|
13
|
+
from rowsift.clean._text import boolean, category, name, text
|
|
14
|
+
from rowsift.types import Cleaned, Kind
|
|
15
|
+
|
|
16
|
+
__all__ = ["value"]
|
|
17
|
+
|
|
18
|
+
_CLEANERS: dict[Kind, Callable[[str, CleanOptions | None], Cleaned[Any]]] = {
|
|
19
|
+
Kind.TEXT: text,
|
|
20
|
+
Kind.NAME: name,
|
|
21
|
+
Kind.INTEGER: integer,
|
|
22
|
+
Kind.DECIMAL: decimal,
|
|
23
|
+
Kind.MONEY: money,
|
|
24
|
+
Kind.BOOLEAN: boolean,
|
|
25
|
+
Kind.DATE: date,
|
|
26
|
+
Kind.DAY_MONTH: day_month,
|
|
27
|
+
Kind.TIME: time,
|
|
28
|
+
Kind.DATETIME: datetime,
|
|
29
|
+
Kind.PHONE: phone,
|
|
30
|
+
Kind.EMAIL: email,
|
|
31
|
+
Kind.CPF: cpf,
|
|
32
|
+
Kind.CNPJ: cnpj,
|
|
33
|
+
Kind.POSTAL_CODE: postal_code,
|
|
34
|
+
Kind.CATEGORY: category,
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def value(text: str, kind: Kind, options: CleanOptions | None = None) -> Cleaned[Any]:
|
|
39
|
+
"""Clean ``text`` with the cleaner of ``kind``.
|
|
40
|
+
|
|
41
|
+
The result is the one the kind's own cleaner returns, so every layer cleans
|
|
42
|
+
every kind through this function. ``kind`` accepts a ``Kind`` or its string
|
|
43
|
+
value.
|
|
44
|
+
|
|
45
|
+
Raises:
|
|
46
|
+
ValueError: if ``kind`` is not a ``Kind``.
|
|
47
|
+
MissingExtraError: for a phone in a region that needs the ``phone`` extra.
|
|
48
|
+
"""
|
|
49
|
+
return _CLEANERS[Kind(kind)](text, options)
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
"""CPF, CNPJ, postal code and e-mail cleaners."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
from collections.abc import Callable
|
|
7
|
+
|
|
8
|
+
from rowsift.clean._common import change, fail
|
|
9
|
+
from rowsift.clean._options import CleanOptions
|
|
10
|
+
from rowsift.types import ChangeCode, Cleaned, IssueCode
|
|
11
|
+
|
|
12
|
+
__all__ = ["cnpj", "cpf", "email", "postal_code"]
|
|
13
|
+
|
|
14
|
+
_FORMATTED_DIGITS = re.compile(r"[0-9.\-/\s]+")
|
|
15
|
+
_ASCII_DIGITS = "0123456789"
|
|
16
|
+
_CPF_LENGTH = 11
|
|
17
|
+
_CNPJ_LENGTH = 14
|
|
18
|
+
_CPF_MIN_DIGITS = 9 # at most two leading zeros are restored
|
|
19
|
+
_CNPJ_MIN_DIGITS = 12
|
|
20
|
+
_POSTAL_LENGTH = 8
|
|
21
|
+
_POSTAL_MIN_DIGITS = 7 # at most one leading zero is restored
|
|
22
|
+
_CPF_WEIGHTS = (10, 9, 8, 7, 6, 5, 4, 3, 2)
|
|
23
|
+
_CNPJ_WEIGHTS = (5, 4, 3, 2, 9, 8, 7, 6, 5, 4, 3, 2)
|
|
24
|
+
_EMAIL_LABEL = re.compile(r"[a-z0-9](?:[a-z0-9-]*[a-z0-9])?")
|
|
25
|
+
_EMAIL_MAX_LENGTH = 254
|
|
26
|
+
_EMAIL_MAX_LOCAL = 64
|
|
27
|
+
_EMAIL_MAX_LABEL = 63
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _check_digit(digits: str, weights: tuple[int, ...]) -> str:
|
|
31
|
+
total = sum(int(d) * w for d, w in zip(digits, weights, strict=True))
|
|
32
|
+
remainder = total % 11
|
|
33
|
+
return str(0 if remainder < 2 else 11 - remainder)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _valid_cpf(digits: str) -> bool:
|
|
37
|
+
if len(set(digits)) == 1:
|
|
38
|
+
return False
|
|
39
|
+
first = _check_digit(digits[:9], _CPF_WEIGHTS)
|
|
40
|
+
second = _check_digit(digits[:9] + first, (11, *_CPF_WEIGHTS))
|
|
41
|
+
return digits[9:] == first + second
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _valid_cnpj(digits: str) -> bool:
|
|
45
|
+
if len(set(digits)) == 1:
|
|
46
|
+
return False
|
|
47
|
+
first = _check_digit(digits[:12], _CNPJ_WEIGHTS)
|
|
48
|
+
second = _check_digit(digits[:12] + first, (6, *_CNPJ_WEIGHTS))
|
|
49
|
+
return digits[12:] == first + second
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _tax_id(value: str, length: int, minimum: int, valid: Callable[[str], bool]) -> Cleaned[str]:
|
|
53
|
+
text = value.strip()
|
|
54
|
+
if not text:
|
|
55
|
+
return Cleaned()
|
|
56
|
+
if not _FORMATTED_DIGITS.fullmatch(text):
|
|
57
|
+
return fail(IssueCode.NOT_A_NUMBER)
|
|
58
|
+
digits = "".join(char for char in text if char in _ASCII_DIGITS)
|
|
59
|
+
if len(digits) > length or len(digits) < minimum:
|
|
60
|
+
return fail(IssueCode.INVALID_LENGTH, digits=len(digits), expected=length)
|
|
61
|
+
if len(digits) == length:
|
|
62
|
+
if valid(digits):
|
|
63
|
+
return Cleaned(digits)
|
|
64
|
+
return fail(IssueCode.INVALID_CHECK_DIGIT)
|
|
65
|
+
padded = digits.rjust(length, "0")
|
|
66
|
+
if valid(padded):
|
|
67
|
+
return Cleaned(padded, changes=(change(ChangeCode.LEADING_ZEROS_RESTORED),))
|
|
68
|
+
return fail(IssueCode.INVALID_LENGTH, digits=len(digits), expected=length)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def cpf(value: str, options: CleanOptions | None = None) -> Cleaned[str]:
|
|
72
|
+
"""A CPF as its 11 digits.
|
|
73
|
+
|
|
74
|
+
Dots, dashes and spaces are removed. The two check digits are verified and
|
|
75
|
+
a sequence of one repeated digit is refused, both as the issue
|
|
76
|
+
``invalid_check_digit``. A value of 9 or 10 digits is padded with the
|
|
77
|
+
leading zeros a spreadsheet drops, only when the padded value passes the
|
|
78
|
+
check digits, and the change ``leading_zeros_restored`` is reported.
|
|
79
|
+
Otherwise a wrong digit count is the issue ``invalid_length`` and any other
|
|
80
|
+
character, including a digit of another script, is ``not_a_number``. Only the
|
|
81
|
+
digits 0 to 9 are supported.
|
|
82
|
+
"""
|
|
83
|
+
return _tax_id(value, _CPF_LENGTH, _CPF_MIN_DIGITS, _valid_cpf)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def cnpj(value: str, options: CleanOptions | None = None) -> Cleaned[str]:
|
|
87
|
+
"""A CNPJ as its 14 digits.
|
|
88
|
+
|
|
89
|
+
Dots, dashes, slashes and spaces are removed. The two check digits are
|
|
90
|
+
verified and a sequence of one repeated digit is refused, both as the issue
|
|
91
|
+
``invalid_check_digit``. A value of 12 or 13 digits is padded with the
|
|
92
|
+
leading zeros a spreadsheet drops, only when the padded value passes the
|
|
93
|
+
check digits, and the change ``leading_zeros_restored`` is reported.
|
|
94
|
+
Otherwise a wrong digit count is the issue ``invalid_length`` and any other
|
|
95
|
+
character, including a digit of another script, is ``not_a_number``. Only the
|
|
96
|
+
digits 0 to 9 are supported.
|
|
97
|
+
"""
|
|
98
|
+
return _tax_id(value, _CNPJ_LENGTH, _CNPJ_MIN_DIGITS, _valid_cnpj)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def postal_code(value: str, options: CleanOptions | None = None) -> Cleaned[str]:
|
|
102
|
+
"""A Brazilian postal code (CEP) as its 8 digits.
|
|
103
|
+
|
|
104
|
+
A dash, dots and spaces are removed. A 7-digit value is padded with the
|
|
105
|
+
leading zero a spreadsheet drops and the change ``leading_zeros_restored``
|
|
106
|
+
is reported. Any other digit count is the issue ``invalid_length`` and any
|
|
107
|
+
other character is ``not_a_number``.
|
|
108
|
+
|
|
109
|
+
Raises:
|
|
110
|
+
ValueError: if ``options.region`` is not ``BR``, the only region with
|
|
111
|
+
native rules.
|
|
112
|
+
"""
|
|
113
|
+
options = options or CleanOptions()
|
|
114
|
+
if options.region != "BR":
|
|
115
|
+
raise ValueError("postal codes have native rules for region BR only")
|
|
116
|
+
text = value.strip()
|
|
117
|
+
if not text:
|
|
118
|
+
return Cleaned()
|
|
119
|
+
if not _FORMATTED_DIGITS.fullmatch(text):
|
|
120
|
+
return fail(IssueCode.NOT_A_NUMBER)
|
|
121
|
+
digits = "".join(char for char in text if char in _ASCII_DIGITS)
|
|
122
|
+
if len(digits) == _POSTAL_LENGTH:
|
|
123
|
+
return Cleaned(digits)
|
|
124
|
+
if len(digits) == _POSTAL_MIN_DIGITS:
|
|
125
|
+
return Cleaned(
|
|
126
|
+
digits.rjust(_POSTAL_LENGTH, "0"),
|
|
127
|
+
changes=(change(ChangeCode.LEADING_ZEROS_RESTORED),),
|
|
128
|
+
)
|
|
129
|
+
return fail(IssueCode.INVALID_LENGTH, digits=len(digits), expected=_POSTAL_LENGTH)
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def email(value: str, options: CleanOptions | None = None) -> Cleaned[str]:
|
|
133
|
+
"""An e-mail address, trimmed, with the domain in lowercase.
|
|
134
|
+
|
|
135
|
+
The address needs exactly one ``@``, a non-empty local part without spaces
|
|
136
|
+
and a domain of at least two labels made of letters, digits and inner
|
|
137
|
+
hyphens. The whole address has at most 254 characters, the local part 64
|
|
138
|
+
and each label 63. A local part that starts or ends with a dot or holds two
|
|
139
|
+
dots in a row is refused. Anything else is the issue ``invalid_email``. The
|
|
140
|
+
local part is kept as written. Quoted local parts and internationalized domains are not
|
|
141
|
+
supported and are refused.
|
|
142
|
+
"""
|
|
143
|
+
text = value.strip()
|
|
144
|
+
if not text:
|
|
145
|
+
return Cleaned()
|
|
146
|
+
local, at, domain = text.partition("@")
|
|
147
|
+
domain = domain.lower()
|
|
148
|
+
labels = domain.split(".")
|
|
149
|
+
ok = (
|
|
150
|
+
at == "@"
|
|
151
|
+
and len(text) <= _EMAIL_MAX_LENGTH
|
|
152
|
+
and 0 < len(local) <= _EMAIL_MAX_LOCAL
|
|
153
|
+
and not (local.startswith(".") or local.endswith(".") or ".." in local)
|
|
154
|
+
and not any(char.isspace() or char in '"@,;<>()' for char in local)
|
|
155
|
+
and len(labels) >= 2 # a dotted domain
|
|
156
|
+
and all(
|
|
157
|
+
len(label) <= _EMAIL_MAX_LABEL and _EMAIL_LABEL.fullmatch(label) for label in labels
|
|
158
|
+
)
|
|
159
|
+
and not labels[-1].isdigit()
|
|
160
|
+
)
|
|
161
|
+
return Cleaned(f"{local}@{domain}") if ok else fail(IssueCode.INVALID_EMAIL)
|