rowsift 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rowsift/__init__.py +98 -0
- rowsift/clean/__init__.py +44 -0
- rowsift/clean/_common.py +30 -0
- rowsift/clean/_dates.py +283 -0
- rowsift/clean/_dispatch.py +49 -0
- rowsift/clean/_identifiers.py +161 -0
- rowsift/clean/_mask.py +117 -0
- rowsift/clean/_numbers.py +137 -0
- rowsift/clean/_options.py +75 -0
- rowsift/clean/_phone.py +202 -0
- rowsift/clean/_text.py +102 -0
- rowsift/errors.py +124 -0
- rowsift/map/__init__.py +24 -0
- rowsift/map/_fields.py +75 -0
- rowsift/map/_saved.py +132 -0
- rowsift/map/_score.py +178 -0
- rowsift/map/_suggest.py +168 -0
- rowsift/map/_types.py +169 -0
- rowsift/profile/__init__.py +17 -0
- rowsift/profile/_headers.py +44 -0
- rowsift/profile/_kinds.py +165 -0
- rowsift/profile/_options.py +69 -0
- rowsift/profile/_profile.py +119 -0
- rowsift/profile/_structure.py +157 -0
- rowsift/profile/_types.py +96 -0
- rowsift/py.typed +0 -0
- rowsift/read/__init__.py +6 -0
- rowsift/read/_cells.py +93 -0
- rowsift/read/_detect.py +124 -0
- rowsift/read/_grid.py +128 -0
- rowsift/read/_limits.py +44 -0
- rowsift/read/_read.py +99 -0
- rowsift/read/_text.py +250 -0
- rowsift/read/_workbook.py +36 -0
- rowsift/read/_xls.py +115 -0
- rowsift/read/_xlsx.py +121 -0
- rowsift/sift/__init__.py +19 -0
- rowsift/sift/_sift.py +165 -0
- rowsift/sift/_types.py +119 -0
- rowsift/types.py +378 -0
- rowsift-0.1.1.dist-info/METADATA +148 -0
- rowsift-0.1.1.dist-info/RECORD +45 -0
- rowsift-0.1.1.dist-info/WHEEL +4 -0
- rowsift-0.1.1.dist-info/licenses/LICENSE +177 -0
- rowsift-0.1.1.dist-info/licenses/NOTICE +4 -0
rowsift/map/_suggest.py
ADDED
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
"""Suggest a column for each declared field."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
from collections import Counter
|
|
7
|
+
from collections.abc import Sequence
|
|
8
|
+
|
|
9
|
+
from rowsift.map._fields import Field
|
|
10
|
+
from rowsift.map._score import ACCEPTS, Candidate, alias_table, score_column
|
|
11
|
+
from rowsift.map._types import Alternative, Choice, Mapping
|
|
12
|
+
from rowsift.profile import ColumnProfile, Profile
|
|
13
|
+
from rowsift.types import Confidence, Kind, Reason, ReasonCode
|
|
14
|
+
|
|
15
|
+
__all__ = ["check_fields", "fingerprint_of", "suggest"]
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def check_fields(fields: Sequence[Field]) -> None:
|
|
19
|
+
"""Raise ``ValueError`` naming the first key declared twice."""
|
|
20
|
+
seen: set[str] = set()
|
|
21
|
+
for field in fields:
|
|
22
|
+
if field.key in seen:
|
|
23
|
+
raise ValueError(f"duplicate field key {field.key!r}")
|
|
24
|
+
seen.add(field.key)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def fingerprint_of(profile: Profile) -> str:
|
|
28
|
+
"""SHA-256 in hexadecimal of the normalized headers, joined by a newline."""
|
|
29
|
+
joined = "\n".join(column.normalized_header for column in profile.columns)
|
|
30
|
+
return hashlib.sha256(joined.encode("utf-8")).hexdigest()
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _order(item: tuple[int, Candidate]) -> tuple[int, float, int, int, int]:
|
|
34
|
+
field_index, candidate = item
|
|
35
|
+
return (
|
|
36
|
+
-candidate.score,
|
|
37
|
+
-candidate.share,
|
|
38
|
+
-candidate.alias_tokens,
|
|
39
|
+
candidate.column,
|
|
40
|
+
field_index,
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def suggest(profile: Profile, fields: Sequence[Field]) -> Mapping:
|
|
45
|
+
"""Suggest a column for each declared field.
|
|
46
|
+
|
|
47
|
+
Each column is scored against each field from two signals: the header (an
|
|
48
|
+
alias equal to it, or an alias all of whose words it contains) and the
|
|
49
|
+
content (the column kind and its share against the field kind). Confidence
|
|
50
|
+
is ``high`` only when the header equals an alias and the content fits the
|
|
51
|
+
kind, ``medium`` with one strong signal and the other neutral, ``low``
|
|
52
|
+
otherwise. A column whose header says nothing is suggested only when its
|
|
53
|
+
kind is one that values alone reveal (a phone, an e-mail, a date and so on)
|
|
54
|
+
and equals the field kind.
|
|
55
|
+
|
|
56
|
+
A column feeds at most one field. Candidates are taken from the highest
|
|
57
|
+
score down, ties going to the larger share, the longer alias, the leftmost
|
|
58
|
+
column and the earlier field. A field that loses a column takes its next
|
|
59
|
+
best candidate, and lists the columns it did not get in ``alternatives``.
|
|
60
|
+
A required field that nothing relates to takes the one remaining column its
|
|
61
|
+
kind accepts, with confidence ``low`` and the reason ``only_candidate``, and
|
|
62
|
+
has no column and the reason ``required_field_unmatched`` when there is not
|
|
63
|
+
exactly one.
|
|
64
|
+
|
|
65
|
+
A profile with no header row ignores headers: only content signals count.
|
|
66
|
+
|
|
67
|
+
Args:
|
|
68
|
+
profile: the profile of the table.
|
|
69
|
+
fields: the fields to fill. The library has no predefined field.
|
|
70
|
+
|
|
71
|
+
Returns:
|
|
72
|
+
A ``Mapping`` with one ``Choice`` per field, in declaration order, the
|
|
73
|
+
indexes of the columns no field uses, and the header fingerprint.
|
|
74
|
+
|
|
75
|
+
Raises:
|
|
76
|
+
ValueError: if two fields have the same key.
|
|
77
|
+
"""
|
|
78
|
+
check_fields(fields)
|
|
79
|
+
has_header = profile.header_row is not None
|
|
80
|
+
kind_fields = Counter(field.kind for field in fields)
|
|
81
|
+
kind_columns = Counter(column.kind for column in profile.columns)
|
|
82
|
+
|
|
83
|
+
candidates: list[tuple[int, Candidate]] = []
|
|
84
|
+
for field_index, field in enumerate(fields):
|
|
85
|
+
aliases = alias_table(field)
|
|
86
|
+
unique = kind_fields[field.kind] == 1 and kind_columns[field.kind] == 1
|
|
87
|
+
for column in profile.columns:
|
|
88
|
+
found = score_column(
|
|
89
|
+
column, field, aliases, has_header=has_header, kind_is_unique=unique
|
|
90
|
+
)
|
|
91
|
+
if found is not None:
|
|
92
|
+
candidates.append((field_index, found))
|
|
93
|
+
candidates.sort(key=_order)
|
|
94
|
+
|
|
95
|
+
taken_fields: dict[int, Candidate] = {}
|
|
96
|
+
taken_columns: set[int] = set()
|
|
97
|
+
for field_index, candidate in candidates:
|
|
98
|
+
if field_index in taken_fields or candidate.column in taken_columns:
|
|
99
|
+
continue
|
|
100
|
+
taken_fields[field_index] = candidate
|
|
101
|
+
taken_columns.add(candidate.column)
|
|
102
|
+
|
|
103
|
+
fallback = _only_candidates(profile.columns, fields, taken_fields, taken_columns)
|
|
104
|
+
choices = []
|
|
105
|
+
for field_index, field in enumerate(fields):
|
|
106
|
+
alternatives = tuple(
|
|
107
|
+
Alternative(c.column, c.confidence)
|
|
108
|
+
for i, c in candidates
|
|
109
|
+
if i == field_index
|
|
110
|
+
and (field_index not in taken_fields or c.column != taken_fields[field_index].column)
|
|
111
|
+
)
|
|
112
|
+
won = taken_fields.get(field_index)
|
|
113
|
+
if won is not None:
|
|
114
|
+
choices.append(Choice(field.key, won.column, won.confidence, won.reasons, alternatives))
|
|
115
|
+
elif field_index in fallback:
|
|
116
|
+
choices.append(
|
|
117
|
+
Choice(
|
|
118
|
+
field.key, fallback[field_index][0], Confidence.LOW, fallback[field_index][1]
|
|
119
|
+
)
|
|
120
|
+
)
|
|
121
|
+
else:
|
|
122
|
+
reasons = (Reason(ReasonCode.REQUIRED_FIELD_UNMATCHED),) if field.required else ()
|
|
123
|
+
choices.append(Choice(field.key, None, None, reasons, alternatives))
|
|
124
|
+
|
|
125
|
+
used = {choice.column for choice in choices if choice.column is not None}
|
|
126
|
+
unmapped = tuple(column.index for column in profile.columns if column.index not in used)
|
|
127
|
+
return Mapping(tuple(choices), unmapped, fingerprint_of(profile))
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
_TEXT_FALLBACK = frozenset({Kind.TEXT, Kind.NAME, Kind.CATEGORY, Kind.INTEGER})
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def _fallback_kinds(kind: Kind) -> frozenset[Kind]:
|
|
134
|
+
"""Column kinds a required field may take as the only candidate.
|
|
135
|
+
|
|
136
|
+
A text field accepts every kind when a header matches, but as a fallback with
|
|
137
|
+
no header signal it takes only text-like columns, so a lone date column is
|
|
138
|
+
never guessed into it.
|
|
139
|
+
"""
|
|
140
|
+
if kind is Kind.TEXT:
|
|
141
|
+
return _TEXT_FALLBACK
|
|
142
|
+
return frozenset({kind, *ACCEPTS.get(kind, frozenset())})
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def _only_candidates(
|
|
146
|
+
columns: tuple[ColumnProfile, ...],
|
|
147
|
+
fields: Sequence[Field],
|
|
148
|
+
taken_fields: dict[int, Candidate],
|
|
149
|
+
taken_columns: set[int],
|
|
150
|
+
) -> dict[int, tuple[int, tuple[Reason, ...]]]:
|
|
151
|
+
"""For required fields left unmatched, the single remaining column they accept."""
|
|
152
|
+
result: dict[int, tuple[int, tuple[Reason, ...]]] = {}
|
|
153
|
+
used = set(taken_columns)
|
|
154
|
+
for field_index, field in enumerate(fields):
|
|
155
|
+
if field_index in taken_fields or not field.required:
|
|
156
|
+
continue
|
|
157
|
+
accepted = _fallback_kinds(field.kind)
|
|
158
|
+
remaining = [c for c in columns if c.index not in used and c.filled and c.kind in accepted]
|
|
159
|
+
if len(remaining) != 1:
|
|
160
|
+
continue
|
|
161
|
+
column = remaining[0]
|
|
162
|
+
used.add(column.index)
|
|
163
|
+
fits = Reason(
|
|
164
|
+
ReasonCode.CONTENT_FITS_KIND,
|
|
165
|
+
{"kind": str(column.kind), "share": round(column.kind_share, 4)},
|
|
166
|
+
)
|
|
167
|
+
result[field_index] = (column.index, (fits, Reason(ReasonCode.ONLY_CANDIDATE)))
|
|
168
|
+
return result
|
rowsift/map/_types.py
ADDED
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
"""Value types of the mapping layer."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass, replace
|
|
6
|
+
|
|
7
|
+
from rowsift.types import Confidence, Reason, ReasonCode
|
|
8
|
+
|
|
9
|
+
__all__ = ["Alternative", "Choice", "Mapping"]
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@dataclass(frozen=True, slots=True)
|
|
13
|
+
class Alternative:
|
|
14
|
+
"""Another column a field could take, with the confidence it would carry."""
|
|
15
|
+
|
|
16
|
+
column: int
|
|
17
|
+
confidence: Confidence
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@dataclass(frozen=True, slots=True)
|
|
21
|
+
class Choice:
|
|
22
|
+
"""The column suggested for one field.
|
|
23
|
+
|
|
24
|
+
Attributes:
|
|
25
|
+
field: the key of the field.
|
|
26
|
+
column: the 0-based index of the suggested column, or ``None`` when no
|
|
27
|
+
column is assigned.
|
|
28
|
+
confidence: how sure the suggestion is, or ``None`` when there is no
|
|
29
|
+
column.
|
|
30
|
+
reasons: the codes, with their data, that produced the suggestion or
|
|
31
|
+
explain why there is none.
|
|
32
|
+
alternatives: the other columns that were candidates for the field, best
|
|
33
|
+
first, each with its own confidence.
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
field: str
|
|
37
|
+
column: int | None
|
|
38
|
+
confidence: Confidence | None
|
|
39
|
+
reasons: tuple[Reason, ...] = ()
|
|
40
|
+
alternatives: tuple[Alternative, ...] = ()
|
|
41
|
+
|
|
42
|
+
def __post_init__(self) -> None:
|
|
43
|
+
if (self.column is None) != (self.confidence is None):
|
|
44
|
+
raise ValueError("a choice has a column and a confidence, or neither")
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@dataclass(frozen=True, slots=True)
|
|
48
|
+
class Mapping:
|
|
49
|
+
"""Which column feeds which field.
|
|
50
|
+
|
|
51
|
+
Attributes:
|
|
52
|
+
choices: one per declared field, in declaration order.
|
|
53
|
+
unmapped: the indexes of the columns no field uses, in table order. The
|
|
54
|
+
caller can offer them as extra attributes, using their profile.
|
|
55
|
+
fingerprint: SHA-256 in hexadecimal of the normalized headers in column
|
|
56
|
+
order, joined by a newline. It does not change with case, accents,
|
|
57
|
+
punctuation or spacing in the headers.
|
|
58
|
+
|
|
59
|
+
A column feeds at most one field. ``assign`` and ``unassign`` return a new
|
|
60
|
+
mapping and keep that rule.
|
|
61
|
+
|
|
62
|
+
Raises:
|
|
63
|
+
ValueError: if two choices share a field key or a column, a column index
|
|
64
|
+
is negative, ``unmapped`` is not sorted and free of repeats, or a
|
|
65
|
+
column is both assigned and listed as unmapped.
|
|
66
|
+
"""
|
|
67
|
+
|
|
68
|
+
choices: tuple[Choice, ...]
|
|
69
|
+
unmapped: tuple[int, ...]
|
|
70
|
+
fingerprint: str
|
|
71
|
+
|
|
72
|
+
def __post_init__(self) -> None:
|
|
73
|
+
keys = [choice.field for choice in self.choices]
|
|
74
|
+
columns = [choice.column for choice in self.choices if choice.column is not None]
|
|
75
|
+
if len(set(keys)) != len(keys):
|
|
76
|
+
raise ValueError("two choices share a field key")
|
|
77
|
+
if len(set(columns)) != len(columns):
|
|
78
|
+
raise ValueError("a column feeds more than one field")
|
|
79
|
+
if any(index < 0 for index in (*columns, *self.unmapped)):
|
|
80
|
+
raise ValueError("a column index is negative")
|
|
81
|
+
if list(self.unmapped) != sorted(set(self.unmapped)):
|
|
82
|
+
raise ValueError("unmapped must be sorted, without repeats")
|
|
83
|
+
if set(columns) & set(self.unmapped):
|
|
84
|
+
raise ValueError("a column is both assigned and unmapped")
|
|
85
|
+
|
|
86
|
+
def assign(self, field_key: str, column_index: int) -> Mapping:
|
|
87
|
+
"""Point a field at a column, as the caller decided.
|
|
88
|
+
|
|
89
|
+
The new choice has confidence ``high`` and the reason ``set_by_caller``.
|
|
90
|
+
The column the field had before goes to ``unmapped``. A field that held
|
|
91
|
+
the new column loses it, and its choice carries the reason
|
|
92
|
+
``set_by_caller`` with ``displaced_by`` set to ``field_key``. The column a
|
|
93
|
+
choice gave up, in either case, is added to its alternatives, so a review
|
|
94
|
+
screen can offer it back.
|
|
95
|
+
|
|
96
|
+
Raises:
|
|
97
|
+
ValueError: if no choice has the key, or ``column_index`` is not a
|
|
98
|
+
column of the mapping.
|
|
99
|
+
TypeError: if ``column_index`` is not an ``int``.
|
|
100
|
+
"""
|
|
101
|
+
if type(column_index) is not int:
|
|
102
|
+
raise TypeError("column_index must be int")
|
|
103
|
+
previous = self._choice(field_key)
|
|
104
|
+
if column_index not in self._columns():
|
|
105
|
+
raise ValueError(f"column {column_index} is not a column of the mapping")
|
|
106
|
+
choices: list[Choice] = []
|
|
107
|
+
for choice in self.choices:
|
|
108
|
+
if choice.field == field_key:
|
|
109
|
+
kept = tuple(a for a in choice.alternatives if a.column != column_index)
|
|
110
|
+
if choice.column not in (None, column_index) and choice.confidence is not None:
|
|
111
|
+
kept = (*kept, Alternative(choice.column, choice.confidence))
|
|
112
|
+
choice = replace(
|
|
113
|
+
choice,
|
|
114
|
+
column=column_index,
|
|
115
|
+
confidence=Confidence.HIGH,
|
|
116
|
+
reasons=(Reason(ReasonCode.SET_BY_CALLER),),
|
|
117
|
+
alternatives=kept,
|
|
118
|
+
)
|
|
119
|
+
elif choice.column == column_index and choice.confidence is not None:
|
|
120
|
+
lost = choice.confidence
|
|
121
|
+
choice = replace(
|
|
122
|
+
choice,
|
|
123
|
+
column=None,
|
|
124
|
+
confidence=None,
|
|
125
|
+
reasons=(Reason(ReasonCode.SET_BY_CALLER, {"displaced_by": field_key}),),
|
|
126
|
+
alternatives=(*choice.alternatives, Alternative(column_index, lost)),
|
|
127
|
+
)
|
|
128
|
+
choices.append(choice)
|
|
129
|
+
freed = {previous.column} - {None, column_index}
|
|
130
|
+
return self._rebuilt(choices, freed)
|
|
131
|
+
|
|
132
|
+
def unassign(self, field_key: str) -> Mapping:
|
|
133
|
+
"""Take the column away from a field, as the caller decided.
|
|
134
|
+
|
|
135
|
+
The choice has no column and the reason ``set_by_caller``, and the column
|
|
136
|
+
goes to ``unmapped``.
|
|
137
|
+
|
|
138
|
+
Raises:
|
|
139
|
+
ValueError: if no choice has the key.
|
|
140
|
+
"""
|
|
141
|
+
previous = self._choice(field_key)
|
|
142
|
+
choices = [
|
|
143
|
+
replace(
|
|
144
|
+
choice,
|
|
145
|
+
column=None,
|
|
146
|
+
confidence=None,
|
|
147
|
+
reasons=(Reason(ReasonCode.SET_BY_CALLER),),
|
|
148
|
+
)
|
|
149
|
+
if choice.field == field_key
|
|
150
|
+
else choice
|
|
151
|
+
for choice in self.choices
|
|
152
|
+
]
|
|
153
|
+
freed = {previous.column} - {None}
|
|
154
|
+
return self._rebuilt(choices, freed)
|
|
155
|
+
|
|
156
|
+
def _choice(self, field_key: str) -> Choice:
|
|
157
|
+
for choice in self.choices:
|
|
158
|
+
if choice.field == field_key:
|
|
159
|
+
return choice
|
|
160
|
+
raise ValueError(f"no field with the key {field_key!r}")
|
|
161
|
+
|
|
162
|
+
def _columns(self) -> set[int]:
|
|
163
|
+
assigned = {choice.column for choice in self.choices if choice.column is not None}
|
|
164
|
+
return assigned | set(self.unmapped)
|
|
165
|
+
|
|
166
|
+
def _rebuilt(self, choices: list[Choice], freed: set[int | None]) -> Mapping:
|
|
167
|
+
assigned = {choice.column for choice in choices if choice.column is not None}
|
|
168
|
+
unmapped = sorted((set(self.unmapped) | {c for c in freed if c is not None}) - assigned)
|
|
169
|
+
return Mapping(tuple(choices), tuple(unmapped), self.fingerprint)
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"""Profile a table: where the header is, which rows are data, what each column holds."""
|
|
2
|
+
|
|
3
|
+
from rowsift.profile._headers import normalize_header
|
|
4
|
+
from rowsift.profile._options import ProfileOptions
|
|
5
|
+
from rowsift.profile._profile import profile
|
|
6
|
+
from rowsift.profile._types import ColumnProfile, Profile, RunnerUp, SkippedRow, TopValue
|
|
7
|
+
|
|
8
|
+
__all__ = [
|
|
9
|
+
"ColumnProfile",
|
|
10
|
+
"Profile",
|
|
11
|
+
"ProfileOptions",
|
|
12
|
+
"RunnerUp",
|
|
13
|
+
"SkippedRow",
|
|
14
|
+
"TopValue",
|
|
15
|
+
"normalize_header",
|
|
16
|
+
"profile",
|
|
17
|
+
]
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"""Header normalization and column names."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
import unicodedata
|
|
7
|
+
|
|
8
|
+
__all__ = ["normalize_header"]
|
|
9
|
+
|
|
10
|
+
_NOT_ALPHANUMERIC = re.compile(r"[^0-9a-z]+")
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def normalize_header(text: str) -> str:
|
|
14
|
+
"""Fold a header for matching.
|
|
15
|
+
|
|
16
|
+
Lowercase, accents removed, every run of characters that are not letters or
|
|
17
|
+
digits replaced by one space, and the ends trimmed: ``Dt. Nascimento `` is
|
|
18
|
+
``dt nascimento``. Only ASCII letters and digits survive, so a header written
|
|
19
|
+
in a script with no Latin decomposition folds to an empty string.
|
|
20
|
+
"""
|
|
21
|
+
decomposed = unicodedata.normalize("NFKD", text.casefold())
|
|
22
|
+
bare = "".join(char for char in decomposed if not unicodedata.combining(char))
|
|
23
|
+
return _NOT_ALPHANUMERIC.sub(" ", bare).strip()
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def column_names(headers: list[str] | None, width: int) -> list[str]:
|
|
27
|
+
"""Unique normalized names for ``width`` columns.
|
|
28
|
+
|
|
29
|
+
``headers`` is the header row as written, or ``None`` when the table has no
|
|
30
|
+
header. An empty header is named ``column_<position>``. A name already taken
|
|
31
|
+
gets the suffix `` 2``, `` 3`` and so on.
|
|
32
|
+
"""
|
|
33
|
+
taken: set[str] = set()
|
|
34
|
+
names: list[str] = []
|
|
35
|
+
for position in range(width):
|
|
36
|
+
base = normalize_header(headers[position]) if headers is not None else ""
|
|
37
|
+
base = base or f"column_{position + 1}"
|
|
38
|
+
name, suffix = base, 1
|
|
39
|
+
while name in taken:
|
|
40
|
+
suffix += 1
|
|
41
|
+
name = f"{base} {suffix}"
|
|
42
|
+
taken.add(name)
|
|
43
|
+
names.append(name)
|
|
44
|
+
return names
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
"""Kind inference: which kind most of a column's values fit."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
from collections.abc import Iterable
|
|
7
|
+
|
|
8
|
+
from rowsift import clean
|
|
9
|
+
from rowsift.clean import CleanOptions, mask
|
|
10
|
+
from rowsift.profile._types import RunnerUp
|
|
11
|
+
from rowsift.types import ChangeCode, Kind
|
|
12
|
+
|
|
13
|
+
__all__ = ["display", "infer_kind", "looks_like_data"]
|
|
14
|
+
|
|
15
|
+
# Most specific first. At equal footing the earlier kind wins, so a column of
|
|
16
|
+
# CPFs is a cpf and not an integer, and phones beat integers.
|
|
17
|
+
_ORDER: tuple[Kind, ...] = (
|
|
18
|
+
Kind.CPF,
|
|
19
|
+
Kind.CNPJ,
|
|
20
|
+
Kind.EMAIL,
|
|
21
|
+
Kind.PHONE,
|
|
22
|
+
Kind.DATETIME,
|
|
23
|
+
Kind.DATE,
|
|
24
|
+
Kind.DAY_MONTH,
|
|
25
|
+
Kind.TIME,
|
|
26
|
+
Kind.BOOLEAN,
|
|
27
|
+
Kind.POSTAL_CODE,
|
|
28
|
+
Kind.MONEY,
|
|
29
|
+
Kind.INTEGER,
|
|
30
|
+
Kind.DECIMAL,
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
# What a header cell never looks like. A boolean word is a plausible header.
|
|
34
|
+
_DATA_KINDS = tuple(kind for kind in _ORDER if kind is not Kind.BOOLEAN)
|
|
35
|
+
|
|
36
|
+
# Kinds that identify a person. A value of a column of another kind that fits one of
|
|
37
|
+
# them is masked as that kind, so a mixed column does not show it in clear.
|
|
38
|
+
_PERSONAL: tuple[Kind, ...] = (Kind.CPF, Kind.CNPJ, Kind.EMAIL, Kind.PHONE)
|
|
39
|
+
_HIDDEN = "***"
|
|
40
|
+
|
|
41
|
+
# No value of any kind is longer than an e-mail address can be, so a longer cell is
|
|
42
|
+
# never run through the cleaners. It bounds the cost of a column of long text.
|
|
43
|
+
_MAX_VALUE_CHARS = 254
|
|
44
|
+
# Personal data inside free text: an e-mail address, or a run of 10 or more digits
|
|
45
|
+
# that a phone number or a tax id would make, written with the usual separators.
|
|
46
|
+
_EMBEDDED = re.compile(r"@|[0-9](?:[ .()\-]?[0-9]){9}")
|
|
47
|
+
|
|
48
|
+
_POSTAL_SHAPE = re.compile(r"[0-9]{5}-?[0-9]{3}")
|
|
49
|
+
_CURRENCY = re.compile(r"R\$|US\$|BRL|USD|EUR|[$€£]", re.IGNORECASE)
|
|
50
|
+
|
|
51
|
+
# A cleaner may accept a value by a route that says nothing about the column:
|
|
52
|
+
# these changes mark a fit that is not evidence for the kind.
|
|
53
|
+
_NOT_EVIDENCE: dict[Kind, frozenset[ChangeCode]] = {
|
|
54
|
+
Kind.DATE: frozenset({ChangeCode.SERIAL_DATE_CONVERTED}),
|
|
55
|
+
Kind.DATETIME: frozenset({ChangeCode.SERIAL_DATE_CONVERTED, ChangeCode.TIME_DEFAULTED}),
|
|
56
|
+
Kind.DAY_MONTH: frozenset({ChangeCode.YEAR_DISCARDED}),
|
|
57
|
+
Kind.POSTAL_CODE: frozenset({ChangeCode.LEADING_ZEROS_RESTORED}),
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def fits(text: str, kind: Kind, options: CleanOptions) -> bool:
|
|
62
|
+
"""Whether ``text`` is a value of ``kind`` that the column can be said to hold.
|
|
63
|
+
|
|
64
|
+
A spreadsheet serial number is not a date here, a bare date is not a
|
|
65
|
+
datetime, a full date is not a day and month, and a postal code that lost
|
|
66
|
+
its leading zero is not a postal code: each would turn a column of plain
|
|
67
|
+
numbers or dates into something else. Money needs a currency symbol and a
|
|
68
|
+
postal code is written as 8 digits, with or without one hyphen.
|
|
69
|
+
"""
|
|
70
|
+
if len(text) > _MAX_VALUE_CHARS:
|
|
71
|
+
return False
|
|
72
|
+
if kind is Kind.EMAIL:
|
|
73
|
+
if "@" not in text:
|
|
74
|
+
return False
|
|
75
|
+
elif kind is not Kind.BOOLEAN and not any("0" <= char <= "9" for char in text):
|
|
76
|
+
return False # every other kind is written with at least one digit
|
|
77
|
+
if kind is Kind.MONEY and not _CURRENCY.search(text):
|
|
78
|
+
return False
|
|
79
|
+
if kind is Kind.POSTAL_CODE and not _POSTAL_SHAPE.fullmatch(text.strip()):
|
|
80
|
+
return False # the cleaner tolerates any separator, a date of 8 digits would pass
|
|
81
|
+
result = clean.value(text, kind, options)
|
|
82
|
+
if result.issue is not None or result.value is None:
|
|
83
|
+
return False
|
|
84
|
+
blocked = _NOT_EVIDENCE.get(kind, frozenset())
|
|
85
|
+
return not any(change.code in blocked for change in result.changes)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def looks_like_data(text: str, options: CleanOptions) -> bool:
|
|
89
|
+
"""Whether a cell holds a value that a header would not: a number, date, id and so on."""
|
|
90
|
+
return any(fits(text, kind, options) for kind in _DATA_KINDS)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def infer_kind(
|
|
94
|
+
values: Iterable[str], threshold: float, options: CleanOptions
|
|
95
|
+
) -> tuple[Kind, float, RunnerUp | None]:
|
|
96
|
+
"""The kind of a column from its non-empty values, with its share and runner-up.
|
|
97
|
+
|
|
98
|
+
The kind is the first of the candidates, most specific first, that at least
|
|
99
|
+
``threshold`` of ``values`` fit. When none does it is ``text``, which every
|
|
100
|
+
value fits. The runner-up is the other candidate with the highest share
|
|
101
|
+
above zero, the more specific one on a tie.
|
|
102
|
+
|
|
103
|
+
``0`` and ``1`` are boolean literals and ``boolean`` comes before ``integer``,
|
|
104
|
+
so a column holding only 0 and 1 is a ``boolean``, and the kind turns
|
|
105
|
+
``integer`` as soon as another number appears. The ``integer`` share and the
|
|
106
|
+
runner-up report the other reading.
|
|
107
|
+
"""
|
|
108
|
+
counts: dict[str, int] = {}
|
|
109
|
+
for value in values:
|
|
110
|
+
counts[value] = counts.get(value, 0) + 1
|
|
111
|
+
total = sum(counts.values())
|
|
112
|
+
if total == 0:
|
|
113
|
+
return Kind.TEXT, 0.0, None
|
|
114
|
+
shares: dict[Kind, float] = {}
|
|
115
|
+
for kind in _ORDER:
|
|
116
|
+
fitting = sum(count for value, count in counts.items() if fits(value, kind, options))
|
|
117
|
+
shares[kind] = fitting / total
|
|
118
|
+
chosen = next((kind for kind in _ORDER if shares[kind] >= threshold), None)
|
|
119
|
+
others = [kind for kind in _ORDER if kind is not chosen and shares[kind] > 0]
|
|
120
|
+
best = max(others, key=lambda kind: shares[kind], default=None)
|
|
121
|
+
runner_up = RunnerUp(best, shares[best]) if best is not None else None
|
|
122
|
+
if chosen is None:
|
|
123
|
+
return Kind.TEXT, 1.0, runner_up
|
|
124
|
+
return chosen, shares[chosen], runner_up
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _accepted(text: str, kind: Kind, options: CleanOptions) -> bool:
|
|
128
|
+
"""Whether the cleaner of ``kind`` takes ``text``, whatever route it takes."""
|
|
129
|
+
if len(text) > _MAX_VALUE_CHARS:
|
|
130
|
+
return False
|
|
131
|
+
result = clean.value(text, kind, options)
|
|
132
|
+
return result.issue is None and result.value is not None
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def display(text: str, kind: Kind, options: CleanOptions) -> str:
|
|
136
|
+
"""``text`` as it is safe to show, given the kind of its column.
|
|
137
|
+
|
|
138
|
+
A column of a personal kind (phone, CPF, CNPJ, e-mail, name) masks each
|
|
139
|
+
value by that kind, and a value that does not have the shape of the kind is
|
|
140
|
+
replaced by a fixed placeholder, so neither its length nor its words show.
|
|
141
|
+
|
|
142
|
+
In any other column each value is judged by itself. A value that fits a
|
|
143
|
+
personal kind is masked as that kind, so a column that fell short of the
|
|
144
|
+
threshold, or mixes phones with notes, does not show the phones in clear. A
|
|
145
|
+
value with an e-mail address or a run of 10 or more digits inside it is the
|
|
146
|
+
placeholder. In a ``text`` column every other value is returned as written;
|
|
147
|
+
in a column of a typed kind (a date, an integer) a value the kind does not
|
|
148
|
+
take is the placeholder too, so a stray name is not shown.
|
|
149
|
+
|
|
150
|
+
A person's name in a ``text`` column is not recognized: ``profile`` never
|
|
151
|
+
infers ``name``, because no rule tells a name from free text with certainty.
|
|
152
|
+
"""
|
|
153
|
+
if kind is Kind.NAME:
|
|
154
|
+
return mask(text, kind)
|
|
155
|
+
if kind in _PERSONAL:
|
|
156
|
+
return mask(text, kind) if fits(text, kind, options) else _HIDDEN
|
|
157
|
+
embedded = _EMBEDDED.search(text) is not None
|
|
158
|
+
if kind is not Kind.TEXT and not embedded and _accepted(text, kind, options):
|
|
159
|
+
return text
|
|
160
|
+
personal = next((k for k in _PERSONAL if fits(text, k, options)), None)
|
|
161
|
+
if personal is not None:
|
|
162
|
+
return mask(text, personal)
|
|
163
|
+
if embedded:
|
|
164
|
+
return _HIDDEN
|
|
165
|
+
return text if kind is Kind.TEXT else _HIDDEN
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
"""Options of the profile layer."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
|
|
7
|
+
from rowsift.clean import CleanOptions
|
|
8
|
+
|
|
9
|
+
__all__ = ["ProfileOptions"]
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@dataclass(frozen=True, slots=True, kw_only=True)
|
|
13
|
+
class ProfileOptions:
|
|
14
|
+
"""Every knob of ``profile``. All are passed by keyword.
|
|
15
|
+
|
|
16
|
+
Attributes:
|
|
17
|
+
header_scan_rows: how many non-empty rows are examined as header
|
|
18
|
+
candidates.
|
|
19
|
+
kind_threshold: the share of non-empty sampled values a kind must fit
|
|
20
|
+
for a column to take it, above 0 and at most 1.
|
|
21
|
+
sample_rows: how many data rows are read to infer the kind.
|
|
22
|
+
low_cardinality_min_filled: a column needs at least this many filled
|
|
23
|
+
cells to be low-cardinality.
|
|
24
|
+
low_cardinality_max_distinct: and at most this many distinct values.
|
|
25
|
+
low_cardinality_max_share: and at most this share of distinct values
|
|
26
|
+
over filled cells.
|
|
27
|
+
sample_count: how many distinct sample values to keep per column.
|
|
28
|
+
mask_samples: mask samples and top values by the kind of the column.
|
|
29
|
+
Turn it off only for values that are safe to display.
|
|
30
|
+
clean: the options the cleaners run with while inferring kinds, such as
|
|
31
|
+
the order of the day and the month.
|
|
32
|
+
|
|
33
|
+
Raises:
|
|
34
|
+
ValueError: if a value is outside what the attribute allows.
|
|
35
|
+
"""
|
|
36
|
+
|
|
37
|
+
header_scan_rows: int = 20
|
|
38
|
+
kind_threshold: float = 0.8
|
|
39
|
+
sample_rows: int = 5000
|
|
40
|
+
low_cardinality_min_filled: int = 20
|
|
41
|
+
low_cardinality_max_distinct: int = 12
|
|
42
|
+
low_cardinality_max_share: float = 0.05
|
|
43
|
+
sample_count: int = 3
|
|
44
|
+
mask_samples: bool = True
|
|
45
|
+
clean: CleanOptions = field(default_factory=CleanOptions)
|
|
46
|
+
|
|
47
|
+
def __post_init__(self) -> None:
|
|
48
|
+
for name in (
|
|
49
|
+
"header_scan_rows",
|
|
50
|
+
"sample_rows",
|
|
51
|
+
"low_cardinality_min_filled",
|
|
52
|
+
"low_cardinality_max_distinct",
|
|
53
|
+
"sample_count",
|
|
54
|
+
):
|
|
55
|
+
value = getattr(self, name)
|
|
56
|
+
if type(value) is not int or value < (1 if name.endswith("rows") else 0):
|
|
57
|
+
raise ValueError(f"{name} must be a whole number in range")
|
|
58
|
+
for name in ("kind_threshold", "low_cardinality_max_share"):
|
|
59
|
+
value = getattr(self, name)
|
|
60
|
+
if isinstance(value, bool) or not isinstance(value, int | float):
|
|
61
|
+
raise ValueError(f"{name} must be a number")
|
|
62
|
+
if not 0 < self.kind_threshold <= 1:
|
|
63
|
+
raise ValueError("kind_threshold must be above 0 and at most 1")
|
|
64
|
+
if not 0 <= self.low_cardinality_max_share <= 1:
|
|
65
|
+
raise ValueError("low_cardinality_max_share must be between 0 and 1")
|
|
66
|
+
if type(self.mask_samples) is not bool:
|
|
67
|
+
raise ValueError("mask_samples must be a bool")
|
|
68
|
+
if not isinstance(self.clean, CleanOptions):
|
|
69
|
+
raise ValueError("clean must be a CleanOptions")
|