rowsift 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rowsift/__init__.py +98 -0
- rowsift/clean/__init__.py +44 -0
- rowsift/clean/_common.py +30 -0
- rowsift/clean/_dates.py +283 -0
- rowsift/clean/_dispatch.py +49 -0
- rowsift/clean/_identifiers.py +161 -0
- rowsift/clean/_mask.py +117 -0
- rowsift/clean/_numbers.py +137 -0
- rowsift/clean/_options.py +75 -0
- rowsift/clean/_phone.py +202 -0
- rowsift/clean/_text.py +102 -0
- rowsift/errors.py +124 -0
- rowsift/map/__init__.py +24 -0
- rowsift/map/_fields.py +75 -0
- rowsift/map/_saved.py +132 -0
- rowsift/map/_score.py +178 -0
- rowsift/map/_suggest.py +168 -0
- rowsift/map/_types.py +169 -0
- rowsift/profile/__init__.py +17 -0
- rowsift/profile/_headers.py +44 -0
- rowsift/profile/_kinds.py +165 -0
- rowsift/profile/_options.py +69 -0
- rowsift/profile/_profile.py +119 -0
- rowsift/profile/_structure.py +157 -0
- rowsift/profile/_types.py +96 -0
- rowsift/py.typed +0 -0
- rowsift/read/__init__.py +6 -0
- rowsift/read/_cells.py +93 -0
- rowsift/read/_detect.py +124 -0
- rowsift/read/_grid.py +128 -0
- rowsift/read/_limits.py +44 -0
- rowsift/read/_read.py +99 -0
- rowsift/read/_text.py +250 -0
- rowsift/read/_workbook.py +36 -0
- rowsift/read/_xls.py +115 -0
- rowsift/read/_xlsx.py +121 -0
- rowsift/sift/__init__.py +19 -0
- rowsift/sift/_sift.py +165 -0
- rowsift/sift/_types.py +119 -0
- rowsift/types.py +378 -0
- rowsift-0.1.1.dist-info/METADATA +148 -0
- rowsift-0.1.1.dist-info/RECORD +45 -0
- rowsift-0.1.1.dist-info/WHEEL +4 -0
- rowsift-0.1.1.dist-info/licenses/LICENSE +177 -0
- rowsift-0.1.1.dist-info/licenses/NOTICE +4 -0
rowsift/clean/_mask.py
ADDED
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
"""Masking of values meant for display.
|
|
2
|
+
|
|
3
|
+
Every function takes the text as written and returns text that is safe to show.
|
|
4
|
+
A value that does not have the shape its kind expects is hidden entirely: the
|
|
5
|
+
functions fail closed, so a note, an e-mail or a number typed into the wrong
|
|
6
|
+
column does not leak.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import re
|
|
12
|
+
|
|
13
|
+
from rowsift.types import Kind
|
|
14
|
+
|
|
15
|
+
__all__ = ["mask", "mask_cnpj", "mask_cpf", "mask_email", "mask_name", "mask_phone"]
|
|
16
|
+
|
|
17
|
+
_PARTICLES = frozenset({"da", "de", "do", "das", "dos", "e"})
|
|
18
|
+
_TAX_ID_SHAPE = re.compile(r"[0-9.\-/\s]+")
|
|
19
|
+
_KEPT_TAX_DIGITS = 2
|
|
20
|
+
_NAME_PUNCTUATION = frozenset(" .'\u2019-")
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _hide(text: str) -> str:
|
|
24
|
+
return "".join("*" if char.isalnum() else char for char in text)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def mask_phone(text: str) -> str:
|
|
28
|
+
"""Keep the area code and the last 2 digits of a Brazilian phone number.
|
|
29
|
+
|
|
30
|
+
``(11) 99999-0042`` becomes ``11 *******42``. A value that does not hold a
|
|
31
|
+
10 or 11 digit number, after an optional country code, is hidden entirely.
|
|
32
|
+
"""
|
|
33
|
+
digits = "".join(char for char in text if char in "0123456789")
|
|
34
|
+
if text.lstrip().startswith("+") and not digits.startswith("55"):
|
|
35
|
+
return _hide(text) # another country: the leading digits are no Brazilian area code
|
|
36
|
+
if digits.startswith("55") and len(digits) in (12, 13):
|
|
37
|
+
digits = digits[2:]
|
|
38
|
+
letters = any(char.isalpha() for char in text)
|
|
39
|
+
if letters or len(digits) not in (10, 11):
|
|
40
|
+
return _hide(text)
|
|
41
|
+
return f"{digits[:2]} {'*' * (len(digits) - 4)}{digits[-2:]}"
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _mask_tax_id(text: str) -> str:
|
|
45
|
+
if not _TAX_ID_SHAPE.fullmatch(text):
|
|
46
|
+
return _hide(text)
|
|
47
|
+
total = sum(char in "0123456789" for char in text)
|
|
48
|
+
seen = 0
|
|
49
|
+
out: list[str] = []
|
|
50
|
+
for char in text:
|
|
51
|
+
if char in "0123456789":
|
|
52
|
+
seen += 1
|
|
53
|
+
out.append(char if seen > total - _KEPT_TAX_DIGITS else "*")
|
|
54
|
+
else:
|
|
55
|
+
out.append(char)
|
|
56
|
+
return "".join(out)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def mask_cpf(text: str) -> str:
|
|
60
|
+
"""Keep the last 2 digits of a CPF and its punctuation: ``***.***.***-09``."""
|
|
61
|
+
return _mask_tax_id(text)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def mask_cnpj(text: str) -> str:
|
|
65
|
+
"""Keep the last 2 digits of a CNPJ and its punctuation."""
|
|
66
|
+
return _mask_tax_id(text)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def mask_email(text: str) -> str:
|
|
70
|
+
"""Keep the first character of the local part and the domain: ``a***@example.com``.
|
|
71
|
+
|
|
72
|
+
A value that is not of the form ``local@domain`` is hidden entirely.
|
|
73
|
+
"""
|
|
74
|
+
local, at, domain = text.strip().partition("@")
|
|
75
|
+
if at != "@" or not local or not domain or "@" in domain or any(c.isspace() for c in text):
|
|
76
|
+
return _hide(text)
|
|
77
|
+
return f"{local[0]}***@{domain}"
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def mask_name(text: str) -> str:
|
|
81
|
+
"""Keep the first name and the initials of the rest: ``Helena P. A.``.
|
|
82
|
+
|
|
83
|
+
The particles ``da``, ``de``, ``do``, ``das``, ``dos`` and ``e`` are dropped
|
|
84
|
+
from the initials. A value with a character other than letters, spaces,
|
|
85
|
+
dots, apostrophes and hyphens (a digit, an ``@``, a colon) is not a name and
|
|
86
|
+
is hidden entirely.
|
|
87
|
+
"""
|
|
88
|
+
words = text.split()
|
|
89
|
+
if not words:
|
|
90
|
+
return text
|
|
91
|
+
if not all(char.isalpha() or char in _NAME_PUNCTUATION for char in text):
|
|
92
|
+
return _hide(text)
|
|
93
|
+
initials = [f"{word[0].upper()}." for word in words[1:] if word.lower() not in _PARTICLES]
|
|
94
|
+
return " ".join([words[0], *initials])
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def mask(text: str, kind: Kind) -> str:
|
|
98
|
+
"""Mask a value according to its kind.
|
|
99
|
+
|
|
100
|
+
Phones, CPFs, CNPJs, e-mails and names are masked by the rules of
|
|
101
|
+
``mask_phone``, ``mask_cpf``, ``mask_cnpj``, ``mask_email`` and
|
|
102
|
+
``mask_name``. Every other kind has no masking rule and is returned as
|
|
103
|
+
written.
|
|
104
|
+
"""
|
|
105
|
+
match kind:
|
|
106
|
+
case Kind.PHONE:
|
|
107
|
+
return mask_phone(text)
|
|
108
|
+
case Kind.CPF:
|
|
109
|
+
return mask_cpf(text)
|
|
110
|
+
case Kind.CNPJ:
|
|
111
|
+
return mask_cnpj(text)
|
|
112
|
+
case Kind.EMAIL:
|
|
113
|
+
return mask_email(text)
|
|
114
|
+
case Kind.NAME:
|
|
115
|
+
return mask_name(text)
|
|
116
|
+
case _:
|
|
117
|
+
return text
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
"""Integer, decimal and money cleaners."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
from decimal import Decimal
|
|
7
|
+
|
|
8
|
+
from rowsift.clean._common import change, fail
|
|
9
|
+
from rowsift.clean._options import CleanOptions
|
|
10
|
+
from rowsift.types import Change, ChangeCode, Cleaned, IssueCode
|
|
11
|
+
|
|
12
|
+
__all__ = ["decimal", "integer", "money"]
|
|
13
|
+
|
|
14
|
+
_CURRENCY = re.compile(r"R\$|US\$|BRL|USD|EUR|[$€£]", re.IGNORECASE)
|
|
15
|
+
_NUMBER_CHARS = re.compile(r"[0-9.,]+")
|
|
16
|
+
# Decimal precision of the default context, and well under the int/str digit limit.
|
|
17
|
+
_MAX_DIGITS = 38
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _groups_ok(integer_part: str, separator: str) -> bool:
|
|
21
|
+
parts = integer_part.split(separator)
|
|
22
|
+
return (
|
|
23
|
+
bool(parts[0])
|
|
24
|
+
and len(parts[0]) <= 3
|
|
25
|
+
and all(len(part) == 3 for part in parts[1:])
|
|
26
|
+
and "".join(parts).isdigit()
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _unsigned(body: str, options: CleanOptions) -> tuple[str, bool] | None:
|
|
31
|
+
"""Digits with a ``.`` decimal point, and whether the options were contradicted."""
|
|
32
|
+
decimal_sep: str = options.decimal_separator
|
|
33
|
+
group_sep = "." if decimal_sep == "," else ","
|
|
34
|
+
n_decimal, n_group = body.count(decimal_sep), body.count(group_sep)
|
|
35
|
+
|
|
36
|
+
if n_decimal and n_group:
|
|
37
|
+
point: str
|
|
38
|
+
group: str
|
|
39
|
+
if body.rfind(decimal_sep) > body.rfind(group_sep):
|
|
40
|
+
point, group, inferred = decimal_sep, group_sep, False
|
|
41
|
+
else:
|
|
42
|
+
point, group, inferred = group_sep, decimal_sep, True
|
|
43
|
+
integer_part, _, fraction = body.rpartition(point)
|
|
44
|
+
if not fraction.isdigit() or point in integer_part or not _groups_ok(integer_part, group):
|
|
45
|
+
return None
|
|
46
|
+
return integer_part.replace(group, "") + "." + fraction, inferred
|
|
47
|
+
if n_decimal == 1 and not n_group:
|
|
48
|
+
integer_part, _, fraction = body.partition(decimal_sep)
|
|
49
|
+
if not fraction.isdigit() or not (integer_part == "" or integer_part.isdigit()):
|
|
50
|
+
return None
|
|
51
|
+
return (integer_part or "0") + "." + fraction, False
|
|
52
|
+
if n_decimal > 1:
|
|
53
|
+
# The configured decimal character cannot repeat: read it as thousands.
|
|
54
|
+
return (body.replace(decimal_sep, ""), True) if _groups_ok(body, decimal_sep) else None
|
|
55
|
+
if n_group == 1:
|
|
56
|
+
integer_part, _, fraction = body.partition(group_sep)
|
|
57
|
+
grouped = len(fraction) == 3 and fraction.isdigit() and _groups_ok(integer_part, group_sep)
|
|
58
|
+
if grouped and not integer_part.startswith("0"):
|
|
59
|
+
return integer_part + fraction, False # ambiguous: the options decide
|
|
60
|
+
if not fraction.isdigit() or not (integer_part == "" or integer_part.isdigit()):
|
|
61
|
+
return None
|
|
62
|
+
return (integer_part or "0") + "." + fraction, True
|
|
63
|
+
if n_group > 1:
|
|
64
|
+
return (body.replace(group_sep, ""), False) if _groups_ok(body, group_sep) else None
|
|
65
|
+
return (body, False) if body.isdigit() else None
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _parse(value: str, options: CleanOptions) -> tuple[Decimal, tuple[Change, ...]] | None:
|
|
69
|
+
body = value.strip()
|
|
70
|
+
parenthesized = body.startswith("(") and body.endswith(")")
|
|
71
|
+
if parenthesized:
|
|
72
|
+
body = body[1:-1]
|
|
73
|
+
body = _CURRENCY.sub("", body).strip()
|
|
74
|
+
minus = 0
|
|
75
|
+
if body[:1] in ("+", "-"):
|
|
76
|
+
minus += body[0] == "-"
|
|
77
|
+
body = body[1:].strip()
|
|
78
|
+
elif body[-1:] == "-":
|
|
79
|
+
minus += 1
|
|
80
|
+
body = body[:-1].strip()
|
|
81
|
+
if parenthesized + minus > 1:
|
|
82
|
+
return None # two negative markers: never read as a positive number
|
|
83
|
+
negative = bool(parenthesized or minus)
|
|
84
|
+
if not _NUMBER_CHARS.fullmatch(body) or sum(c.isdigit() for c in body) > _MAX_DIGITS:
|
|
85
|
+
return None
|
|
86
|
+
parsed = _unsigned(body, options)
|
|
87
|
+
if parsed is None:
|
|
88
|
+
return None
|
|
89
|
+
digits, inferred = parsed
|
|
90
|
+
number = Decimal(digits)
|
|
91
|
+
changes = (change(ChangeCode.SEPARATOR_INFERRED),) if inferred else ()
|
|
92
|
+
# copy_negate keeps every digit: unary minus would round to the context precision.
|
|
93
|
+
return (number.copy_negate() if negative else number), changes
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def decimal(value: str, options: CleanOptions | None = None) -> Cleaned[Decimal]:
|
|
97
|
+
"""A number as a ``decimal.Decimal``.
|
|
98
|
+
|
|
99
|
+
Currency symbols (``R$``, ``$``, ``BRL``...) and surrounding spaces are
|
|
100
|
+
removed. A leading or trailing minus sign and parentheses mean negative.
|
|
101
|
+
Thousands and decimal separators follow ``options.decimal_separator``. An
|
|
102
|
+
input that is ambiguous under the options, such as ``1.234`` with the
|
|
103
|
+
default comma, is read by the options. An input that contradicts them
|
|
104
|
+
unambiguously, such as ``1,234.56``, is read as written and reports
|
|
105
|
+
``separator_inferred``. Exponent notation and any other text is the issue
|
|
106
|
+
``not_a_number``, and so is a number of more than 38 digits.
|
|
107
|
+
"""
|
|
108
|
+
options = options or CleanOptions()
|
|
109
|
+
if not value.strip():
|
|
110
|
+
return Cleaned()
|
|
111
|
+
parsed = _parse(value, options)
|
|
112
|
+
if parsed is None:
|
|
113
|
+
return fail(IssueCode.NOT_A_NUMBER)
|
|
114
|
+
return Cleaned(parsed[0], changes=parsed[1])
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def money(value: str, options: CleanOptions | None = None) -> Cleaned[Decimal]:
|
|
118
|
+
"""An amount of money as a ``decimal.Decimal``, parsed like ``decimal``.
|
|
119
|
+
|
|
120
|
+
The amount is not rounded: the decimals written are the decimals returned.
|
|
121
|
+
"""
|
|
122
|
+
return decimal(value, options)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def integer(value: str, options: CleanOptions | None = None) -> Cleaned[int]:
|
|
126
|
+
"""A whole number as an ``int``.
|
|
127
|
+
|
|
128
|
+
Parsed like ``decimal``. A value with a non-zero fractional part is the
|
|
129
|
+
issue ``not_a_number``, never truncated.
|
|
130
|
+
"""
|
|
131
|
+
options = options or CleanOptions()
|
|
132
|
+
if not value.strip():
|
|
133
|
+
return Cleaned()
|
|
134
|
+
parsed = _parse(value, options)
|
|
135
|
+
if parsed is None or parsed[0] != parsed[0].to_integral_value():
|
|
136
|
+
return fail(IssueCode.NOT_A_NUMBER)
|
|
137
|
+
return Cleaned(int(parsed[0]), changes=parsed[1])
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
"""Options shared by every cleaner."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from typing import Literal
|
|
7
|
+
|
|
8
|
+
__all__ = ["CleanOptions", "DateOrder", "DecimalSeparator", "NameCase"]
|
|
9
|
+
|
|
10
|
+
DateOrder = Literal["dmy", "mdy"]
|
|
11
|
+
"""Order of the day and month in a numeric date such as ``03/04/1990``."""
|
|
12
|
+
|
|
13
|
+
DecimalSeparator = Literal[",", "."]
|
|
14
|
+
"""The character that separates the integer part from the decimals."""
|
|
15
|
+
|
|
16
|
+
NameCase = Literal["title", "keep"]
|
|
17
|
+
"""``title`` recases names written entirely in one case, ``keep`` never does."""
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@dataclass(frozen=True, slots=True, kw_only=True)
|
|
21
|
+
class CleanOptions:
|
|
22
|
+
"""Every knob of the cleaners. All are passed by keyword.
|
|
23
|
+
|
|
24
|
+
Attributes:
|
|
25
|
+
region: two-letter region of phone numbers and postal codes. ``BR`` is
|
|
26
|
+
implemented natively. Another region for a phone needs the ``phone``
|
|
27
|
+
extra.
|
|
28
|
+
date_order: ``dmy`` (default) or ``mdy``, for numeric dates whose year
|
|
29
|
+
comes last. A date that starts with a four-digit year is always
|
|
30
|
+
read as year, month, day.
|
|
31
|
+
decimal_separator: ``,`` (default) or ``.``. The other character is the
|
|
32
|
+
thousands separator.
|
|
33
|
+
two_digit_year_pivot: a two-digit year from 00 up to the pivot belongs
|
|
34
|
+
to the 2000s, a larger one to the 1900s.
|
|
35
|
+
insert_ninth_digit: insert the ninth digit into an 8-digit mobile
|
|
36
|
+
number and report it. When false, such a number is an issue.
|
|
37
|
+
name_case: ``title`` or ``keep``.
|
|
38
|
+
require_mobile: a landline number is an issue.
|
|
39
|
+
one_of: allowed values of a ``category``. ``None`` means any text.
|
|
40
|
+
|
|
41
|
+
Raises:
|
|
42
|
+
ValueError: if a value is outside what the attribute allows.
|
|
43
|
+
"""
|
|
44
|
+
|
|
45
|
+
region: str = "BR"
|
|
46
|
+
date_order: DateOrder = "dmy"
|
|
47
|
+
decimal_separator: DecimalSeparator = ","
|
|
48
|
+
two_digit_year_pivot: int = 30
|
|
49
|
+
insert_ninth_digit: bool = True
|
|
50
|
+
name_case: NameCase = "title"
|
|
51
|
+
require_mobile: bool = False
|
|
52
|
+
one_of: tuple[str, ...] | None = None
|
|
53
|
+
|
|
54
|
+
def __post_init__(self) -> None:
|
|
55
|
+
if not (isinstance(self.region, str) and len(self.region) == 2 and self.region.isalpha()):
|
|
56
|
+
raise ValueError("region must be a two-letter code")
|
|
57
|
+
object.__setattr__(self, "region", self.region.upper())
|
|
58
|
+
if self.date_order not in ("dmy", "mdy"):
|
|
59
|
+
raise ValueError("date_order must be 'dmy' or 'mdy'")
|
|
60
|
+
if self.decimal_separator not in (",", "."):
|
|
61
|
+
raise ValueError("decimal_separator must be ',' or '.'")
|
|
62
|
+
pivot = self.two_digit_year_pivot
|
|
63
|
+
if type(pivot) is not int or not 0 <= pivot <= 99:
|
|
64
|
+
raise ValueError("two_digit_year_pivot must be an int between 0 and 99")
|
|
65
|
+
if self.name_case not in ("title", "keep"):
|
|
66
|
+
raise ValueError("name_case must be 'title' or 'keep'")
|
|
67
|
+
for flag in ("insert_ninth_digit", "require_mobile"):
|
|
68
|
+
if type(getattr(self, flag)) is not bool:
|
|
69
|
+
raise ValueError(f"{flag} must be a bool")
|
|
70
|
+
if self.one_of is not None:
|
|
71
|
+
if isinstance(self.one_of, str) or not all(
|
|
72
|
+
isinstance(item, str) for item in self.one_of
|
|
73
|
+
):
|
|
74
|
+
raise ValueError("one_of must be a sequence of strings")
|
|
75
|
+
object.__setattr__(self, "one_of", tuple(self.one_of))
|
rowsift/clean/_phone.py
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
"""Phone cleaner: native rules for Brazil, the ``phone`` extra for other regions."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import importlib
|
|
6
|
+
import re
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
from rowsift.clean._common import change, fail
|
|
10
|
+
from rowsift.clean._options import CleanOptions
|
|
11
|
+
from rowsift.errors import MissingExtraError
|
|
12
|
+
from rowsift.types import Change, ChangeCode, Cleaned, Issue, IssueCode, LineType
|
|
13
|
+
|
|
14
|
+
__all__ = ["PhoneNumber", "phone"]
|
|
15
|
+
|
|
16
|
+
_AREA_CODE_LIST = (
|
|
17
|
+
"11 12 13 14 15 16 17 18 19 21 22 24 27 28 31 32 33 34 35 37 38 "
|
|
18
|
+
"41 42 43 44 45 46 47 48 49 51 53 54 55 61 62 63 64 65 66 67 68 69 "
|
|
19
|
+
"71 73 74 75 77 79 81 82 83 84 85 86 87 88 89 91 92 93 94 95 96 97 98 99"
|
|
20
|
+
)
|
|
21
|
+
_AREA_CODES = frozenset(int(code) for code in _AREA_CODE_LIST.split())
|
|
22
|
+
_COUNTRY_CODE = "55"
|
|
23
|
+
_MIN_NUMBER_DIGITS = 8
|
|
24
|
+
_SPLIT = re.compile(r"\s*[/;|,\n\r]\s*|\s+(?:ou|e|or|and)\s+", re.IGNORECASE)
|
|
25
|
+
# A long number a spreadsheet turned into a float: its comma is a decimal mark, not a separator.
|
|
26
|
+
_SCIENTIFIC = re.compile(r"\s*\d+(?:[.,]\d+)?[eE][+-]?\d+\s*")
|
|
27
|
+
_FORMATTING = re.compile(r"[\s().\-+]")
|
|
28
|
+
# What another region may write around the digits. Anything else (letters, a
|
|
29
|
+
# vanity number, "tel:", an extension marker) would be altered or dropped by the
|
|
30
|
+
# phone library without a trace, so the fragment is rejected instead.
|
|
31
|
+
_PLAIN_FRAGMENT = re.compile(r"[0-9\s().\-+]+")
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class PhoneNumber(str):
|
|
35
|
+
"""A phone number in E.164 form, which is also a ``str``.
|
|
36
|
+
|
|
37
|
+
It compares, hashes and serializes as the E.164 text. ``line_type`` says
|
|
38
|
+
whether the number is a mobile or a landline, or is ``None`` when the
|
|
39
|
+
``phone`` extra cannot tell.
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
__slots__ = ("_line_type",)
|
|
43
|
+
_line_type: LineType | None
|
|
44
|
+
|
|
45
|
+
def __new__(cls, e164: str, line_type: LineType | None) -> PhoneNumber:
|
|
46
|
+
instance = super().__new__(cls, e164)
|
|
47
|
+
instance._line_type = line_type
|
|
48
|
+
return instance
|
|
49
|
+
|
|
50
|
+
@property
|
|
51
|
+
def line_type(self) -> LineType | None:
|
|
52
|
+
"""Mobile or landline."""
|
|
53
|
+
return self._line_type
|
|
54
|
+
|
|
55
|
+
def __reduce__(self) -> tuple[Any, tuple[Any, ...]]:
|
|
56
|
+
return (type(self), (str(self), self._line_type))
|
|
57
|
+
|
|
58
|
+
def __repr__(self) -> str:
|
|
59
|
+
return f"PhoneNumber({str(self)!r}, {self._line_type!r})"
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _brazil(segment: str, options: CleanOptions) -> tuple[PhoneNumber, tuple[Change, ...]] | Issue:
|
|
63
|
+
plus = segment.lstrip().startswith("+")
|
|
64
|
+
digits = _FORMATTING.sub("", segment)
|
|
65
|
+
if not (digits.isascii() and digits.isdigit()):
|
|
66
|
+
return Issue(IssueCode.INVALID_PHONE)
|
|
67
|
+
if digits.startswith("0055") and len(digits) in (14, 15):
|
|
68
|
+
digits = digits[2:]
|
|
69
|
+
if plus:
|
|
70
|
+
if not digits.startswith(_COUNTRY_CODE):
|
|
71
|
+
return Issue(IssueCode.INVALID_PHONE)
|
|
72
|
+
digits = digits[2:]
|
|
73
|
+
elif digits.startswith(_COUNTRY_CODE) and len(digits) in (12, 13):
|
|
74
|
+
digits = digits[2:]
|
|
75
|
+
if digits[:1] == "0" and digits[1:2] != "0" and len(digits) in (11, 12):
|
|
76
|
+
digits = digits[1:] # the trunk prefix some people write before the area code
|
|
77
|
+
count = len(digits)
|
|
78
|
+
if count > 11:
|
|
79
|
+
return Issue(IssueCode.INVALID_PHONE)
|
|
80
|
+
if count < 10:
|
|
81
|
+
return Issue(IssueCode.TOO_SHORT, {"digits": count})
|
|
82
|
+
if int(digits[:2]) not in _AREA_CODES:
|
|
83
|
+
return Issue(IssueCode.INVALID_AREA_CODE)
|
|
84
|
+
area, subscriber = digits[:2], digits[2:]
|
|
85
|
+
changes: tuple[Change, ...] = ()
|
|
86
|
+
if count == 11:
|
|
87
|
+
if subscriber[0] != "9":
|
|
88
|
+
return Issue(IssueCode.INVALID_PHONE)
|
|
89
|
+
line_type = LineType.MOBILE
|
|
90
|
+
elif subscriber[0] in "2345":
|
|
91
|
+
line_type = LineType.LANDLINE
|
|
92
|
+
elif subscriber[0] in "678":
|
|
93
|
+
if not options.insert_ninth_digit:
|
|
94
|
+
return Issue(IssueCode.TOO_SHORT, {"digits": count})
|
|
95
|
+
subscriber = "9" + subscriber
|
|
96
|
+
changes = (change(ChangeCode.NINTH_DIGIT_INSERTED),)
|
|
97
|
+
line_type = LineType.MOBILE
|
|
98
|
+
elif subscriber[0] == "9":
|
|
99
|
+
# Nine digits start with 9: this is a mobile missing its last digit.
|
|
100
|
+
return Issue(IssueCode.TRUNCATED, {"digits": count})
|
|
101
|
+
else:
|
|
102
|
+
return Issue(IssueCode.INVALID_PHONE)
|
|
103
|
+
if options.require_mobile and line_type is not LineType.MOBILE:
|
|
104
|
+
return Issue(IssueCode.NOT_MOBILE)
|
|
105
|
+
return PhoneNumber(f"+{_COUNTRY_CODE}{area}{subscriber}", line_type), changes
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _load_library() -> Any:
|
|
109
|
+
try:
|
|
110
|
+
return importlib.import_module("phonenumbers")
|
|
111
|
+
except ImportError:
|
|
112
|
+
raise MissingExtraError("phone") from None
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def _elsewhere(
|
|
116
|
+
segment: str, options: CleanOptions
|
|
117
|
+
) -> tuple[PhoneNumber, tuple[Change, ...]] | Issue:
|
|
118
|
+
library = _load_library()
|
|
119
|
+
if not _PLAIN_FRAGMENT.fullmatch(segment):
|
|
120
|
+
return Issue(IssueCode.INVALID_PHONE)
|
|
121
|
+
try:
|
|
122
|
+
parsed = library.parse(segment, options.region)
|
|
123
|
+
except library.NumberParseException:
|
|
124
|
+
return Issue(IssueCode.INVALID_PHONE)
|
|
125
|
+
if not library.is_valid_number(parsed):
|
|
126
|
+
return Issue(IssueCode.INVALID_PHONE)
|
|
127
|
+
if parsed.extension: # defense in depth: the fragment check already rules markers out
|
|
128
|
+
return Issue(IssueCode.INVALID_PHONE)
|
|
129
|
+
kind = library.number_type(parsed)
|
|
130
|
+
line_type: LineType | None = None
|
|
131
|
+
if kind == library.PhoneNumberType.MOBILE:
|
|
132
|
+
line_type = LineType.MOBILE
|
|
133
|
+
elif kind == library.PhoneNumberType.FIXED_LINE:
|
|
134
|
+
line_type = LineType.LANDLINE
|
|
135
|
+
if options.require_mobile and line_type is not LineType.MOBILE:
|
|
136
|
+
return Issue(IssueCode.NOT_MOBILE)
|
|
137
|
+
e164 = library.format_number(parsed, library.PhoneNumberFormat.E164)
|
|
138
|
+
return PhoneNumber(e164, line_type), ()
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def phone(value: str, options: CleanOptions | None = None) -> Cleaned[PhoneNumber]:
|
|
142
|
+
"""A phone number in E.164 form.
|
|
143
|
+
|
|
144
|
+
With region ``BR`` (the default) the rules are native: formatting and an
|
|
145
|
+
optional country code (``+55``, ``55``, ``0055``) and trunk zero are
|
|
146
|
+
removed, the area code must be one of the national list, and the number is
|
|
147
|
+
classified as mobile (11 digits, or 10 digits whose subscriber part starts
|
|
148
|
+
with 6, 7 or 8) or landline (starts with 2 to 5). A mobile written with 8
|
|
149
|
+
subscriber digits gets the ninth digit when ``options.insert_ninth_digit``
|
|
150
|
+
is true, reported as ``ninth_digit_inserted``. A number is never completed
|
|
151
|
+
otherwise: a 10-digit number whose subscriber part starts with 9 is the
|
|
152
|
+
issue ``truncated``, fewer than 10 digits is ``too_short`` (so is an 8-digit mobile pattern when
|
|
153
|
+
``insert_ninth_digit`` is false), an unknown area code is
|
|
154
|
+
``invalid_area_code``, a landline with ``require_mobile`` is
|
|
155
|
+
``not_mobile``, and any other shape (letters, scientific notation, a
|
|
156
|
+
foreign country code, too many digits) is ``invalid_phone``.
|
|
157
|
+
|
|
158
|
+
Several numbers in one cell, separated by ``/``, ``;``, ``|``, ``,``, a line
|
|
159
|
+
break or the words ``ou`` and ``e``, give the first valid one and the change
|
|
160
|
+
``multiple_values`` with the count of fragments that have at least 8 digits
|
|
161
|
+
(an extension such as ``ramal 12`` after a separator is not counted). An
|
|
162
|
+
extension written in the same fragment as the number, as in
|
|
163
|
+
``(11) 99999-0042 ramal 12``, is the issue ``invalid_phone``, and so is an
|
|
164
|
+
extension in another region (``212 555 0123 x99``). For another region a
|
|
165
|
+
fragment may hold only digits, spaces and ``( ) . - +``: letters, a vanity
|
|
166
|
+
number (``1-800-FLOWERS``), a prefix such as ``tel:``, and a dangling
|
|
167
|
+
extension marker are ``invalid_phone`` too, because the ``phone`` extra
|
|
168
|
+
would otherwise alter or drop them silently. If none is valid the issue is
|
|
169
|
+
the first number's.
|
|
170
|
+
|
|
171
|
+
The result's value is a ``str`` with a ``line_type`` attribute. For another
|
|
172
|
+
region the ``phone`` extra is used, and ``MissingExtraError`` is raised when
|
|
173
|
+
it is not installed.
|
|
174
|
+
|
|
175
|
+
Raises:
|
|
176
|
+
MissingExtraError: for a region other than ``BR`` without the extra,
|
|
177
|
+
whatever the cell holds.
|
|
178
|
+
"""
|
|
179
|
+
options = options or CleanOptions()
|
|
180
|
+
if options.region != "BR":
|
|
181
|
+
_load_library() # fail on a missing extra whatever the cell holds
|
|
182
|
+
if _SCIENTIFIC.fullmatch(value):
|
|
183
|
+
return fail(IssueCode.INVALID_PHONE)
|
|
184
|
+
candidates = [part for part in _SPLIT.split(value) if any(c.isdigit() for c in part)]
|
|
185
|
+
if not candidates:
|
|
186
|
+
if not value.strip():
|
|
187
|
+
return Cleaned()
|
|
188
|
+
return fail(IssueCode.INVALID_PHONE)
|
|
189
|
+
# Only a fragment with enough digits to be a number counts: "ramal 12" does not.
|
|
190
|
+
count = sum(sum(c.isdigit() for c in part) >= _MIN_NUMBER_DIGITS for part in candidates)
|
|
191
|
+
clean_one = _brazil if options.region == "BR" else _elsewhere
|
|
192
|
+
first_issue: Issue | None = None
|
|
193
|
+
for candidate in candidates:
|
|
194
|
+
outcome = clean_one(candidate, options)
|
|
195
|
+
if isinstance(outcome, Issue):
|
|
196
|
+
first_issue = first_issue or outcome
|
|
197
|
+
continue
|
|
198
|
+
number, changes = outcome
|
|
199
|
+
if count > 1:
|
|
200
|
+
changes = (*changes, change(ChangeCode.MULTIPLE_VALUES, count=count))
|
|
201
|
+
return Cleaned(number, changes=changes)
|
|
202
|
+
return Cleaned(issue=first_issue)
|
rowsift/clean/_text.py
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
"""Text, name, boolean and category cleaners."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
|
|
7
|
+
from rowsift.clean._common import change, fail, fold
|
|
8
|
+
from rowsift.clean._options import CleanOptions
|
|
9
|
+
from rowsift.types import ChangeCode, Cleaned, IssueCode
|
|
10
|
+
|
|
11
|
+
__all__ = ["boolean", "category", "name", "text"]
|
|
12
|
+
|
|
13
|
+
_PARTICLES = frozenset({"da", "de", "do", "das", "dos", "e"})
|
|
14
|
+
_TRUE_WORDS = frozenset({"sim", "s", "yes", "y", "true", "t", "1", "verdadeiro"})
|
|
15
|
+
_FALSE_WORDS = frozenset({"nao", "n", "no", "false", "f", "0", "falso"})
|
|
16
|
+
_WORD_START = re.compile(r"(^|[-'\u2019])([^\W\d_])")
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def text(value: str, options: CleanOptions | None = None) -> Cleaned[str]:
|
|
20
|
+
"""Free text, trimmed.
|
|
21
|
+
|
|
22
|
+
Returns no value for an empty or whitespace-only input. Interior
|
|
23
|
+
whitespace is kept as written.
|
|
24
|
+
"""
|
|
25
|
+
stripped = value.strip()
|
|
26
|
+
return Cleaned(stripped) if stripped else Cleaned()
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _capitalize(word: str) -> str:
|
|
30
|
+
return _WORD_START.sub(lambda m: m.group(1) + m.group(2).upper(), word.lower())
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _same_letters(recased: str, original: str) -> bool:
|
|
34
|
+
"""Recasing must not change the letters: ``ß`` to ``SS`` is not a case change."""
|
|
35
|
+
return len(recased) == len(original) and recased.casefold() == original.casefold()
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def name(value: str, options: CleanOptions | None = None) -> Cleaned[str]:
|
|
39
|
+
"""A person's name with whitespace collapsed.
|
|
40
|
+
|
|
41
|
+
With ``name_case="title"`` (the default) a name written entirely in
|
|
42
|
+
uppercase or entirely in lowercase is recased, keeping the particles
|
|
43
|
+
``da``, ``de``, ``do``, ``das``, ``dos`` and ``e`` in lowercase after the
|
|
44
|
+
first word, and the change ``case_changed`` is reported. A name in mixed
|
|
45
|
+
case is kept as written, and so is a name whose recasing would change the
|
|
46
|
+
letters, such as ``ß``.
|
|
47
|
+
"""
|
|
48
|
+
options = options or CleanOptions()
|
|
49
|
+
words = value.split()
|
|
50
|
+
if not words:
|
|
51
|
+
return Cleaned()
|
|
52
|
+
collapsed = " ".join(words)
|
|
53
|
+
letters = [char for char in collapsed if char.isalpha()]
|
|
54
|
+
single_case = bool(letters) and (collapsed.isupper() or collapsed.islower())
|
|
55
|
+
if options.name_case != "title" or not single_case:
|
|
56
|
+
return Cleaned(collapsed)
|
|
57
|
+
recased = " ".join(
|
|
58
|
+
word.lower() if index and word.lower() in _PARTICLES else _capitalize(word)
|
|
59
|
+
for index, word in enumerate(words)
|
|
60
|
+
)
|
|
61
|
+
if recased == collapsed or not _same_letters(recased, collapsed):
|
|
62
|
+
return Cleaned(collapsed)
|
|
63
|
+
return Cleaned(recased, changes=(change(ChangeCode.CASE_CHANGED),))
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def boolean(value: str, options: CleanOptions | None = None) -> Cleaned[bool]:
|
|
67
|
+
"""A yes or no.
|
|
68
|
+
|
|
69
|
+
True words: ``sim``, ``s``, ``yes``, ``y``, ``true``, ``t``, ``1``,
|
|
70
|
+
``verdadeiro``. False words: ``nao``, ``n``, ``no``, ``false``, ``f``,
|
|
71
|
+
``0``, ``falso``. Case and accents are ignored, and only ASCII digits
|
|
72
|
+
count. Anything else is the issue ``not_a_boolean``.
|
|
73
|
+
"""
|
|
74
|
+
stripped = value.strip()
|
|
75
|
+
if not stripped:
|
|
76
|
+
return Cleaned()
|
|
77
|
+
word = fold(stripped, compatibility=False)
|
|
78
|
+
if word in _TRUE_WORDS:
|
|
79
|
+
return Cleaned(True)
|
|
80
|
+
if word in _FALSE_WORDS:
|
|
81
|
+
return Cleaned(False)
|
|
82
|
+
return fail(IssueCode.NOT_A_BOOLEAN)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def category(value: str, options: CleanOptions | None = None) -> Cleaned[str]:
|
|
86
|
+
"""One of the allowed values, in its declared spelling.
|
|
87
|
+
|
|
88
|
+
Matching ignores case, accents and repeated whitespace. A value that
|
|
89
|
+
matches no entry of ``options.one_of`` is the issue
|
|
90
|
+
``not_in_allowed_values``. Without ``one_of`` any text is accepted, trimmed.
|
|
91
|
+
"""
|
|
92
|
+
options = options or CleanOptions()
|
|
93
|
+
stripped = value.strip()
|
|
94
|
+
if not stripped:
|
|
95
|
+
return Cleaned()
|
|
96
|
+
if options.one_of is None:
|
|
97
|
+
return Cleaned(stripped)
|
|
98
|
+
wanted = fold(stripped)
|
|
99
|
+
for allowed in options.one_of:
|
|
100
|
+
if fold(allowed) == wanted:
|
|
101
|
+
return Cleaned(allowed)
|
|
102
|
+
return fail(IssueCode.NOT_IN_ALLOWED_VALUES)
|