rowsift 0.1.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
rowsift/clean/_mask.py ADDED
@@ -0,0 +1,117 @@
1
+ """Masking of values meant for display.
2
+
3
+ Every function takes the text as written and returns text that is safe to show.
4
+ A value that does not have the shape its kind expects is hidden entirely: the
5
+ functions fail closed, so a note, an e-mail or a number typed into the wrong
6
+ column does not leak.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import re
12
+
13
+ from rowsift.types import Kind
14
+
15
+ __all__ = ["mask", "mask_cnpj", "mask_cpf", "mask_email", "mask_name", "mask_phone"]
16
+
17
+ _PARTICLES = frozenset({"da", "de", "do", "das", "dos", "e"})
18
+ _TAX_ID_SHAPE = re.compile(r"[0-9.\-/\s]+")
19
+ _KEPT_TAX_DIGITS = 2
20
+ _NAME_PUNCTUATION = frozenset(" .'\u2019-")
21
+
22
+
23
+ def _hide(text: str) -> str:
24
+ return "".join("*" if char.isalnum() else char for char in text)
25
+
26
+
27
+ def mask_phone(text: str) -> str:
28
+ """Keep the area code and the last 2 digits of a Brazilian phone number.
29
+
30
+ ``(11) 99999-0042`` becomes ``11 *******42``. A value that does not hold a
31
+ 10 or 11 digit number, after an optional country code, is hidden entirely.
32
+ """
33
+ digits = "".join(char for char in text if char in "0123456789")
34
+ if text.lstrip().startswith("+") and not digits.startswith("55"):
35
+ return _hide(text) # another country: the leading digits are no Brazilian area code
36
+ if digits.startswith("55") and len(digits) in (12, 13):
37
+ digits = digits[2:]
38
+ letters = any(char.isalpha() for char in text)
39
+ if letters or len(digits) not in (10, 11):
40
+ return _hide(text)
41
+ return f"{digits[:2]} {'*' * (len(digits) - 4)}{digits[-2:]}"
42
+
43
+
44
+ def _mask_tax_id(text: str) -> str:
45
+ if not _TAX_ID_SHAPE.fullmatch(text):
46
+ return _hide(text)
47
+ total = sum(char in "0123456789" for char in text)
48
+ seen = 0
49
+ out: list[str] = []
50
+ for char in text:
51
+ if char in "0123456789":
52
+ seen += 1
53
+ out.append(char if seen > total - _KEPT_TAX_DIGITS else "*")
54
+ else:
55
+ out.append(char)
56
+ return "".join(out)
57
+
58
+
59
+ def mask_cpf(text: str) -> str:
60
+ """Keep the last 2 digits of a CPF and its punctuation: ``***.***.***-09``."""
61
+ return _mask_tax_id(text)
62
+
63
+
64
+ def mask_cnpj(text: str) -> str:
65
+ """Keep the last 2 digits of a CNPJ and its punctuation."""
66
+ return _mask_tax_id(text)
67
+
68
+
69
+ def mask_email(text: str) -> str:
70
+ """Keep the first character of the local part and the domain: ``a***@example.com``.
71
+
72
+ A value that is not of the form ``local@domain`` is hidden entirely.
73
+ """
74
+ local, at, domain = text.strip().partition("@")
75
+ if at != "@" or not local or not domain or "@" in domain or any(c.isspace() for c in text):
76
+ return _hide(text)
77
+ return f"{local[0]}***@{domain}"
78
+
79
+
80
+ def mask_name(text: str) -> str:
81
+ """Keep the first name and the initials of the rest: ``Helena P. A.``.
82
+
83
+ The particles ``da``, ``de``, ``do``, ``das``, ``dos`` and ``e`` are dropped
84
+ from the initials. A value with a character other than letters, spaces,
85
+ dots, apostrophes and hyphens (a digit, an ``@``, a colon) is not a name and
86
+ is hidden entirely.
87
+ """
88
+ words = text.split()
89
+ if not words:
90
+ return text
91
+ if not all(char.isalpha() or char in _NAME_PUNCTUATION for char in text):
92
+ return _hide(text)
93
+ initials = [f"{word[0].upper()}." for word in words[1:] if word.lower() not in _PARTICLES]
94
+ return " ".join([words[0], *initials])
95
+
96
+
97
+ def mask(text: str, kind: Kind) -> str:
98
+ """Mask a value according to its kind.
99
+
100
+ Phones, CPFs, CNPJs, e-mails and names are masked by the rules of
101
+ ``mask_phone``, ``mask_cpf``, ``mask_cnpj``, ``mask_email`` and
102
+ ``mask_name``. Every other kind has no masking rule and is returned as
103
+ written.
104
+ """
105
+ match kind:
106
+ case Kind.PHONE:
107
+ return mask_phone(text)
108
+ case Kind.CPF:
109
+ return mask_cpf(text)
110
+ case Kind.CNPJ:
111
+ return mask_cnpj(text)
112
+ case Kind.EMAIL:
113
+ return mask_email(text)
114
+ case Kind.NAME:
115
+ return mask_name(text)
116
+ case _:
117
+ return text
@@ -0,0 +1,137 @@
1
+ """Integer, decimal and money cleaners."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import re
6
+ from decimal import Decimal
7
+
8
+ from rowsift.clean._common import change, fail
9
+ from rowsift.clean._options import CleanOptions
10
+ from rowsift.types import Change, ChangeCode, Cleaned, IssueCode
11
+
12
+ __all__ = ["decimal", "integer", "money"]
13
+
14
+ _CURRENCY = re.compile(r"R\$|US\$|BRL|USD|EUR|[$€£]", re.IGNORECASE)
15
+ _NUMBER_CHARS = re.compile(r"[0-9.,]+")
16
+ # Decimal precision of the default context, and well under the int/str digit limit.
17
+ _MAX_DIGITS = 38
18
+
19
+
20
+ def _groups_ok(integer_part: str, separator: str) -> bool:
21
+ parts = integer_part.split(separator)
22
+ return (
23
+ bool(parts[0])
24
+ and len(parts[0]) <= 3
25
+ and all(len(part) == 3 for part in parts[1:])
26
+ and "".join(parts).isdigit()
27
+ )
28
+
29
+
30
+ def _unsigned(body: str, options: CleanOptions) -> tuple[str, bool] | None:
31
+ """Digits with a ``.`` decimal point, and whether the options were contradicted."""
32
+ decimal_sep: str = options.decimal_separator
33
+ group_sep = "." if decimal_sep == "," else ","
34
+ n_decimal, n_group = body.count(decimal_sep), body.count(group_sep)
35
+
36
+ if n_decimal and n_group:
37
+ point: str
38
+ group: str
39
+ if body.rfind(decimal_sep) > body.rfind(group_sep):
40
+ point, group, inferred = decimal_sep, group_sep, False
41
+ else:
42
+ point, group, inferred = group_sep, decimal_sep, True
43
+ integer_part, _, fraction = body.rpartition(point)
44
+ if not fraction.isdigit() or point in integer_part or not _groups_ok(integer_part, group):
45
+ return None
46
+ return integer_part.replace(group, "") + "." + fraction, inferred
47
+ if n_decimal == 1 and not n_group:
48
+ integer_part, _, fraction = body.partition(decimal_sep)
49
+ if not fraction.isdigit() or not (integer_part == "" or integer_part.isdigit()):
50
+ return None
51
+ return (integer_part or "0") + "." + fraction, False
52
+ if n_decimal > 1:
53
+ # The configured decimal character cannot repeat: read it as thousands.
54
+ return (body.replace(decimal_sep, ""), True) if _groups_ok(body, decimal_sep) else None
55
+ if n_group == 1:
56
+ integer_part, _, fraction = body.partition(group_sep)
57
+ grouped = len(fraction) == 3 and fraction.isdigit() and _groups_ok(integer_part, group_sep)
58
+ if grouped and not integer_part.startswith("0"):
59
+ return integer_part + fraction, False # ambiguous: the options decide
60
+ if not fraction.isdigit() or not (integer_part == "" or integer_part.isdigit()):
61
+ return None
62
+ return (integer_part or "0") + "." + fraction, True
63
+ if n_group > 1:
64
+ return (body.replace(group_sep, ""), False) if _groups_ok(body, group_sep) else None
65
+ return (body, False) if body.isdigit() else None
66
+
67
+
68
+ def _parse(value: str, options: CleanOptions) -> tuple[Decimal, tuple[Change, ...]] | None:
69
+ body = value.strip()
70
+ parenthesized = body.startswith("(") and body.endswith(")")
71
+ if parenthesized:
72
+ body = body[1:-1]
73
+ body = _CURRENCY.sub("", body).strip()
74
+ minus = 0
75
+ if body[:1] in ("+", "-"):
76
+ minus += body[0] == "-"
77
+ body = body[1:].strip()
78
+ elif body[-1:] == "-":
79
+ minus += 1
80
+ body = body[:-1].strip()
81
+ if parenthesized + minus > 1:
82
+ return None # two negative markers: never read as a positive number
83
+ negative = bool(parenthesized or minus)
84
+ if not _NUMBER_CHARS.fullmatch(body) or sum(c.isdigit() for c in body) > _MAX_DIGITS:
85
+ return None
86
+ parsed = _unsigned(body, options)
87
+ if parsed is None:
88
+ return None
89
+ digits, inferred = parsed
90
+ number = Decimal(digits)
91
+ changes = (change(ChangeCode.SEPARATOR_INFERRED),) if inferred else ()
92
+ # copy_negate keeps every digit: unary minus would round to the context precision.
93
+ return (number.copy_negate() if negative else number), changes
94
+
95
+
96
+ def decimal(value: str, options: CleanOptions | None = None) -> Cleaned[Decimal]:
97
+ """A number as a ``decimal.Decimal``.
98
+
99
+ Currency symbols (``R$``, ``$``, ``BRL``...) and surrounding spaces are
100
+ removed. A leading or trailing minus sign and parentheses mean negative.
101
+ Thousands and decimal separators follow ``options.decimal_separator``. An
102
+ input that is ambiguous under the options, such as ``1.234`` with the
103
+ default comma, is read by the options. An input that contradicts them
104
+ unambiguously, such as ``1,234.56``, is read as written and reports
105
+ ``separator_inferred``. Exponent notation and any other text is the issue
106
+ ``not_a_number``, and so is a number of more than 38 digits.
107
+ """
108
+ options = options or CleanOptions()
109
+ if not value.strip():
110
+ return Cleaned()
111
+ parsed = _parse(value, options)
112
+ if parsed is None:
113
+ return fail(IssueCode.NOT_A_NUMBER)
114
+ return Cleaned(parsed[0], changes=parsed[1])
115
+
116
+
117
+ def money(value: str, options: CleanOptions | None = None) -> Cleaned[Decimal]:
118
+ """An amount of money as a ``decimal.Decimal``, parsed like ``decimal``.
119
+
120
+ The amount is not rounded: the decimals written are the decimals returned.
121
+ """
122
+ return decimal(value, options)
123
+
124
+
125
+ def integer(value: str, options: CleanOptions | None = None) -> Cleaned[int]:
126
+ """A whole number as an ``int``.
127
+
128
+ Parsed like ``decimal``. A value with a non-zero fractional part is the
129
+ issue ``not_a_number``, never truncated.
130
+ """
131
+ options = options or CleanOptions()
132
+ if not value.strip():
133
+ return Cleaned()
134
+ parsed = _parse(value, options)
135
+ if parsed is None or parsed[0] != parsed[0].to_integral_value():
136
+ return fail(IssueCode.NOT_A_NUMBER)
137
+ return Cleaned(int(parsed[0]), changes=parsed[1])
@@ -0,0 +1,75 @@
1
+ """Options shared by every cleaner."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from dataclasses import dataclass
6
+ from typing import Literal
7
+
8
+ __all__ = ["CleanOptions", "DateOrder", "DecimalSeparator", "NameCase"]
9
+
10
+ DateOrder = Literal["dmy", "mdy"]
11
+ """Order of the day and month in a numeric date such as ``03/04/1990``."""
12
+
13
+ DecimalSeparator = Literal[",", "."]
14
+ """The character that separates the integer part from the decimals."""
15
+
16
+ NameCase = Literal["title", "keep"]
17
+ """``title`` recases names written entirely in one case, ``keep`` never does."""
18
+
19
+
20
+ @dataclass(frozen=True, slots=True, kw_only=True)
21
+ class CleanOptions:
22
+ """Every knob of the cleaners. All are passed by keyword.
23
+
24
+ Attributes:
25
+ region: two-letter region of phone numbers and postal codes. ``BR`` is
26
+ implemented natively. Another region for a phone needs the ``phone``
27
+ extra.
28
+ date_order: ``dmy`` (default) or ``mdy``, for numeric dates whose year
29
+ comes last. A date that starts with a four-digit year is always
30
+ read as year, month, day.
31
+ decimal_separator: ``,`` (default) or ``.``. The other character is the
32
+ thousands separator.
33
+ two_digit_year_pivot: a two-digit year from 00 up to the pivot belongs
34
+ to the 2000s, a larger one to the 1900s.
35
+ insert_ninth_digit: insert the ninth digit into an 8-digit mobile
36
+ number and report it. When false, such a number is an issue.
37
+ name_case: ``title`` or ``keep``.
38
+ require_mobile: a landline number is an issue.
39
+ one_of: allowed values of a ``category``. ``None`` means any text.
40
+
41
+ Raises:
42
+ ValueError: if a value is outside what the attribute allows.
43
+ """
44
+
45
+ region: str = "BR"
46
+ date_order: DateOrder = "dmy"
47
+ decimal_separator: DecimalSeparator = ","
48
+ two_digit_year_pivot: int = 30
49
+ insert_ninth_digit: bool = True
50
+ name_case: NameCase = "title"
51
+ require_mobile: bool = False
52
+ one_of: tuple[str, ...] | None = None
53
+
54
+ def __post_init__(self) -> None:
55
+ if not (isinstance(self.region, str) and len(self.region) == 2 and self.region.isalpha()):
56
+ raise ValueError("region must be a two-letter code")
57
+ object.__setattr__(self, "region", self.region.upper())
58
+ if self.date_order not in ("dmy", "mdy"):
59
+ raise ValueError("date_order must be 'dmy' or 'mdy'")
60
+ if self.decimal_separator not in (",", "."):
61
+ raise ValueError("decimal_separator must be ',' or '.'")
62
+ pivot = self.two_digit_year_pivot
63
+ if type(pivot) is not int or not 0 <= pivot <= 99:
64
+ raise ValueError("two_digit_year_pivot must be an int between 0 and 99")
65
+ if self.name_case not in ("title", "keep"):
66
+ raise ValueError("name_case must be 'title' or 'keep'")
67
+ for flag in ("insert_ninth_digit", "require_mobile"):
68
+ if type(getattr(self, flag)) is not bool:
69
+ raise ValueError(f"{flag} must be a bool")
70
+ if self.one_of is not None:
71
+ if isinstance(self.one_of, str) or not all(
72
+ isinstance(item, str) for item in self.one_of
73
+ ):
74
+ raise ValueError("one_of must be a sequence of strings")
75
+ object.__setattr__(self, "one_of", tuple(self.one_of))
@@ -0,0 +1,202 @@
1
+ """Phone cleaner: native rules for Brazil, the ``phone`` extra for other regions."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import importlib
6
+ import re
7
+ from typing import Any
8
+
9
+ from rowsift.clean._common import change, fail
10
+ from rowsift.clean._options import CleanOptions
11
+ from rowsift.errors import MissingExtraError
12
+ from rowsift.types import Change, ChangeCode, Cleaned, Issue, IssueCode, LineType
13
+
14
+ __all__ = ["PhoneNumber", "phone"]
15
+
16
+ _AREA_CODE_LIST = (
17
+ "11 12 13 14 15 16 17 18 19 21 22 24 27 28 31 32 33 34 35 37 38 "
18
+ "41 42 43 44 45 46 47 48 49 51 53 54 55 61 62 63 64 65 66 67 68 69 "
19
+ "71 73 74 75 77 79 81 82 83 84 85 86 87 88 89 91 92 93 94 95 96 97 98 99"
20
+ )
21
+ _AREA_CODES = frozenset(int(code) for code in _AREA_CODE_LIST.split())
22
+ _COUNTRY_CODE = "55"
23
+ _MIN_NUMBER_DIGITS = 8
24
+ _SPLIT = re.compile(r"\s*[/;|,\n\r]\s*|\s+(?:ou|e|or|and)\s+", re.IGNORECASE)
25
+ # A long number a spreadsheet turned into a float: its comma is a decimal mark, not a separator.
26
+ _SCIENTIFIC = re.compile(r"\s*\d+(?:[.,]\d+)?[eE][+-]?\d+\s*")
27
+ _FORMATTING = re.compile(r"[\s().\-+]")
28
+ # What another region may write around the digits. Anything else (letters, a
29
+ # vanity number, "tel:", an extension marker) would be altered or dropped by the
30
+ # phone library without a trace, so the fragment is rejected instead.
31
+ _PLAIN_FRAGMENT = re.compile(r"[0-9\s().\-+]+")
32
+
33
+
34
+ class PhoneNumber(str):
35
+ """A phone number in E.164 form, which is also a ``str``.
36
+
37
+ It compares, hashes and serializes as the E.164 text. ``line_type`` says
38
+ whether the number is a mobile or a landline, or is ``None`` when the
39
+ ``phone`` extra cannot tell.
40
+ """
41
+
42
+ __slots__ = ("_line_type",)
43
+ _line_type: LineType | None
44
+
45
+ def __new__(cls, e164: str, line_type: LineType | None) -> PhoneNumber:
46
+ instance = super().__new__(cls, e164)
47
+ instance._line_type = line_type
48
+ return instance
49
+
50
+ @property
51
+ def line_type(self) -> LineType | None:
52
+ """Mobile or landline."""
53
+ return self._line_type
54
+
55
+ def __reduce__(self) -> tuple[Any, tuple[Any, ...]]:
56
+ return (type(self), (str(self), self._line_type))
57
+
58
+ def __repr__(self) -> str:
59
+ return f"PhoneNumber({str(self)!r}, {self._line_type!r})"
60
+
61
+
62
+ def _brazil(segment: str, options: CleanOptions) -> tuple[PhoneNumber, tuple[Change, ...]] | Issue:
63
+ plus = segment.lstrip().startswith("+")
64
+ digits = _FORMATTING.sub("", segment)
65
+ if not (digits.isascii() and digits.isdigit()):
66
+ return Issue(IssueCode.INVALID_PHONE)
67
+ if digits.startswith("0055") and len(digits) in (14, 15):
68
+ digits = digits[2:]
69
+ if plus:
70
+ if not digits.startswith(_COUNTRY_CODE):
71
+ return Issue(IssueCode.INVALID_PHONE)
72
+ digits = digits[2:]
73
+ elif digits.startswith(_COUNTRY_CODE) and len(digits) in (12, 13):
74
+ digits = digits[2:]
75
+ if digits[:1] == "0" and digits[1:2] != "0" and len(digits) in (11, 12):
76
+ digits = digits[1:] # the trunk prefix some people write before the area code
77
+ count = len(digits)
78
+ if count > 11:
79
+ return Issue(IssueCode.INVALID_PHONE)
80
+ if count < 10:
81
+ return Issue(IssueCode.TOO_SHORT, {"digits": count})
82
+ if int(digits[:2]) not in _AREA_CODES:
83
+ return Issue(IssueCode.INVALID_AREA_CODE)
84
+ area, subscriber = digits[:2], digits[2:]
85
+ changes: tuple[Change, ...] = ()
86
+ if count == 11:
87
+ if subscriber[0] != "9":
88
+ return Issue(IssueCode.INVALID_PHONE)
89
+ line_type = LineType.MOBILE
90
+ elif subscriber[0] in "2345":
91
+ line_type = LineType.LANDLINE
92
+ elif subscriber[0] in "678":
93
+ if not options.insert_ninth_digit:
94
+ return Issue(IssueCode.TOO_SHORT, {"digits": count})
95
+ subscriber = "9" + subscriber
96
+ changes = (change(ChangeCode.NINTH_DIGIT_INSERTED),)
97
+ line_type = LineType.MOBILE
98
+ elif subscriber[0] == "9":
99
+ # Nine digits start with 9: this is a mobile missing its last digit.
100
+ return Issue(IssueCode.TRUNCATED, {"digits": count})
101
+ else:
102
+ return Issue(IssueCode.INVALID_PHONE)
103
+ if options.require_mobile and line_type is not LineType.MOBILE:
104
+ return Issue(IssueCode.NOT_MOBILE)
105
+ return PhoneNumber(f"+{_COUNTRY_CODE}{area}{subscriber}", line_type), changes
106
+
107
+
108
+ def _load_library() -> Any:
109
+ try:
110
+ return importlib.import_module("phonenumbers")
111
+ except ImportError:
112
+ raise MissingExtraError("phone") from None
113
+
114
+
115
+ def _elsewhere(
116
+ segment: str, options: CleanOptions
117
+ ) -> tuple[PhoneNumber, tuple[Change, ...]] | Issue:
118
+ library = _load_library()
119
+ if not _PLAIN_FRAGMENT.fullmatch(segment):
120
+ return Issue(IssueCode.INVALID_PHONE)
121
+ try:
122
+ parsed = library.parse(segment, options.region)
123
+ except library.NumberParseException:
124
+ return Issue(IssueCode.INVALID_PHONE)
125
+ if not library.is_valid_number(parsed):
126
+ return Issue(IssueCode.INVALID_PHONE)
127
+ if parsed.extension: # defense in depth: the fragment check already rules markers out
128
+ return Issue(IssueCode.INVALID_PHONE)
129
+ kind = library.number_type(parsed)
130
+ line_type: LineType | None = None
131
+ if kind == library.PhoneNumberType.MOBILE:
132
+ line_type = LineType.MOBILE
133
+ elif kind == library.PhoneNumberType.FIXED_LINE:
134
+ line_type = LineType.LANDLINE
135
+ if options.require_mobile and line_type is not LineType.MOBILE:
136
+ return Issue(IssueCode.NOT_MOBILE)
137
+ e164 = library.format_number(parsed, library.PhoneNumberFormat.E164)
138
+ return PhoneNumber(e164, line_type), ()
139
+
140
+
141
+ def phone(value: str, options: CleanOptions | None = None) -> Cleaned[PhoneNumber]:
142
+ """A phone number in E.164 form.
143
+
144
+ With region ``BR`` (the default) the rules are native: formatting and an
145
+ optional country code (``+55``, ``55``, ``0055``) and trunk zero are
146
+ removed, the area code must be one of the national list, and the number is
147
+ classified as mobile (11 digits, or 10 digits whose subscriber part starts
148
+ with 6, 7 or 8) or landline (starts with 2 to 5). A mobile written with 8
149
+ subscriber digits gets the ninth digit when ``options.insert_ninth_digit``
150
+ is true, reported as ``ninth_digit_inserted``. A number is never completed
151
+ otherwise: a 10-digit number whose subscriber part starts with 9 is the
152
+ issue ``truncated``, fewer than 10 digits is ``too_short`` (so is an 8-digit mobile pattern when
153
+ ``insert_ninth_digit`` is false), an unknown area code is
154
+ ``invalid_area_code``, a landline with ``require_mobile`` is
155
+ ``not_mobile``, and any other shape (letters, scientific notation, a
156
+ foreign country code, too many digits) is ``invalid_phone``.
157
+
158
+ Several numbers in one cell, separated by ``/``, ``;``, ``|``, ``,``, a line
159
+ break or the words ``ou`` and ``e``, give the first valid one and the change
160
+ ``multiple_values`` with the count of fragments that have at least 8 digits
161
+ (an extension such as ``ramal 12`` after a separator is not counted). An
162
+ extension written in the same fragment as the number, as in
163
+ ``(11) 99999-0042 ramal 12``, is the issue ``invalid_phone``, and so is an
164
+ extension in another region (``212 555 0123 x99``). For another region a
165
+ fragment may hold only digits, spaces and ``( ) . - +``: letters, a vanity
166
+ number (``1-800-FLOWERS``), a prefix such as ``tel:``, and a dangling
167
+ extension marker are ``invalid_phone`` too, because the ``phone`` extra
168
+ would otherwise alter or drop them silently. If none is valid the issue is
169
+ the first number's.
170
+
171
+ The result's value is a ``str`` with a ``line_type`` attribute. For another
172
+ region the ``phone`` extra is used, and ``MissingExtraError`` is raised when
173
+ it is not installed.
174
+
175
+ Raises:
176
+ MissingExtraError: for a region other than ``BR`` without the extra,
177
+ whatever the cell holds.
178
+ """
179
+ options = options or CleanOptions()
180
+ if options.region != "BR":
181
+ _load_library() # fail on a missing extra whatever the cell holds
182
+ if _SCIENTIFIC.fullmatch(value):
183
+ return fail(IssueCode.INVALID_PHONE)
184
+ candidates = [part for part in _SPLIT.split(value) if any(c.isdigit() for c in part)]
185
+ if not candidates:
186
+ if not value.strip():
187
+ return Cleaned()
188
+ return fail(IssueCode.INVALID_PHONE)
189
+ # Only a fragment with enough digits to be a number counts: "ramal 12" does not.
190
+ count = sum(sum(c.isdigit() for c in part) >= _MIN_NUMBER_DIGITS for part in candidates)
191
+ clean_one = _brazil if options.region == "BR" else _elsewhere
192
+ first_issue: Issue | None = None
193
+ for candidate in candidates:
194
+ outcome = clean_one(candidate, options)
195
+ if isinstance(outcome, Issue):
196
+ first_issue = first_issue or outcome
197
+ continue
198
+ number, changes = outcome
199
+ if count > 1:
200
+ changes = (*changes, change(ChangeCode.MULTIPLE_VALUES, count=count))
201
+ return Cleaned(number, changes=changes)
202
+ return Cleaned(issue=first_issue)
rowsift/clean/_text.py ADDED
@@ -0,0 +1,102 @@
1
+ """Text, name, boolean and category cleaners."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import re
6
+
7
+ from rowsift.clean._common import change, fail, fold
8
+ from rowsift.clean._options import CleanOptions
9
+ from rowsift.types import ChangeCode, Cleaned, IssueCode
10
+
11
+ __all__ = ["boolean", "category", "name", "text"]
12
+
13
+ _PARTICLES = frozenset({"da", "de", "do", "das", "dos", "e"})
14
+ _TRUE_WORDS = frozenset({"sim", "s", "yes", "y", "true", "t", "1", "verdadeiro"})
15
+ _FALSE_WORDS = frozenset({"nao", "n", "no", "false", "f", "0", "falso"})
16
+ _WORD_START = re.compile(r"(^|[-'\u2019])([^\W\d_])")
17
+
18
+
19
+ def text(value: str, options: CleanOptions | None = None) -> Cleaned[str]:
20
+ """Free text, trimmed.
21
+
22
+ Returns no value for an empty or whitespace-only input. Interior
23
+ whitespace is kept as written.
24
+ """
25
+ stripped = value.strip()
26
+ return Cleaned(stripped) if stripped else Cleaned()
27
+
28
+
29
+ def _capitalize(word: str) -> str:
30
+ return _WORD_START.sub(lambda m: m.group(1) + m.group(2).upper(), word.lower())
31
+
32
+
33
+ def _same_letters(recased: str, original: str) -> bool:
34
+ """Recasing must not change the letters: ``ß`` to ``SS`` is not a case change."""
35
+ return len(recased) == len(original) and recased.casefold() == original.casefold()
36
+
37
+
38
+ def name(value: str, options: CleanOptions | None = None) -> Cleaned[str]:
39
+ """A person's name with whitespace collapsed.
40
+
41
+ With ``name_case="title"`` (the default) a name written entirely in
42
+ uppercase or entirely in lowercase is recased, keeping the particles
43
+ ``da``, ``de``, ``do``, ``das``, ``dos`` and ``e`` in lowercase after the
44
+ first word, and the change ``case_changed`` is reported. A name in mixed
45
+ case is kept as written, and so is a name whose recasing would change the
46
+ letters, such as ``ß``.
47
+ """
48
+ options = options or CleanOptions()
49
+ words = value.split()
50
+ if not words:
51
+ return Cleaned()
52
+ collapsed = " ".join(words)
53
+ letters = [char for char in collapsed if char.isalpha()]
54
+ single_case = bool(letters) and (collapsed.isupper() or collapsed.islower())
55
+ if options.name_case != "title" or not single_case:
56
+ return Cleaned(collapsed)
57
+ recased = " ".join(
58
+ word.lower() if index and word.lower() in _PARTICLES else _capitalize(word)
59
+ for index, word in enumerate(words)
60
+ )
61
+ if recased == collapsed or not _same_letters(recased, collapsed):
62
+ return Cleaned(collapsed)
63
+ return Cleaned(recased, changes=(change(ChangeCode.CASE_CHANGED),))
64
+
65
+
66
+ def boolean(value: str, options: CleanOptions | None = None) -> Cleaned[bool]:
67
+ """A yes or no.
68
+
69
+ True words: ``sim``, ``s``, ``yes``, ``y``, ``true``, ``t``, ``1``,
70
+ ``verdadeiro``. False words: ``nao``, ``n``, ``no``, ``false``, ``f``,
71
+ ``0``, ``falso``. Case and accents are ignored, and only ASCII digits
72
+ count. Anything else is the issue ``not_a_boolean``.
73
+ """
74
+ stripped = value.strip()
75
+ if not stripped:
76
+ return Cleaned()
77
+ word = fold(stripped, compatibility=False)
78
+ if word in _TRUE_WORDS:
79
+ return Cleaned(True)
80
+ if word in _FALSE_WORDS:
81
+ return Cleaned(False)
82
+ return fail(IssueCode.NOT_A_BOOLEAN)
83
+
84
+
85
+ def category(value: str, options: CleanOptions | None = None) -> Cleaned[str]:
86
+ """One of the allowed values, in its declared spelling.
87
+
88
+ Matching ignores case, accents and repeated whitespace. A value that
89
+ matches no entry of ``options.one_of`` is the issue
90
+ ``not_in_allowed_values``. Without ``one_of`` any text is accepted, trimmed.
91
+ """
92
+ options = options or CleanOptions()
93
+ stripped = value.strip()
94
+ if not stripped:
95
+ return Cleaned()
96
+ if options.one_of is None:
97
+ return Cleaned(stripped)
98
+ wanted = fold(stripped)
99
+ for allowed in options.one_of:
100
+ if fold(allowed) == wanted:
101
+ return Cleaned(allowed)
102
+ return fail(IssueCode.NOT_IN_ALLOWED_VALUES)