rowsift 0.1.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,168 @@
1
+ """Suggest a column for each declared field."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import hashlib
6
+ from collections import Counter
7
+ from collections.abc import Sequence
8
+
9
+ from rowsift.map._fields import Field
10
+ from rowsift.map._score import ACCEPTS, Candidate, alias_table, score_column
11
+ from rowsift.map._types import Alternative, Choice, Mapping
12
+ from rowsift.profile import ColumnProfile, Profile
13
+ from rowsift.types import Confidence, Kind, Reason, ReasonCode
14
+
15
+ __all__ = ["check_fields", "fingerprint_of", "suggest"]
16
+
17
+
18
+ def check_fields(fields: Sequence[Field]) -> None:
19
+ """Raise ``ValueError`` naming the first key declared twice."""
20
+ seen: set[str] = set()
21
+ for field in fields:
22
+ if field.key in seen:
23
+ raise ValueError(f"duplicate field key {field.key!r}")
24
+ seen.add(field.key)
25
+
26
+
27
+ def fingerprint_of(profile: Profile) -> str:
28
+ """SHA-256 in hexadecimal of the normalized headers, joined by a newline."""
29
+ joined = "\n".join(column.normalized_header for column in profile.columns)
30
+ return hashlib.sha256(joined.encode("utf-8")).hexdigest()
31
+
32
+
33
+ def _order(item: tuple[int, Candidate]) -> tuple[int, float, int, int, int]:
34
+ field_index, candidate = item
35
+ return (
36
+ -candidate.score,
37
+ -candidate.share,
38
+ -candidate.alias_tokens,
39
+ candidate.column,
40
+ field_index,
41
+ )
42
+
43
+
44
+ def suggest(profile: Profile, fields: Sequence[Field]) -> Mapping:
45
+ """Suggest a column for each declared field.
46
+
47
+ Each column is scored against each field from two signals: the header (an
48
+ alias equal to it, or an alias all of whose words it contains) and the
49
+ content (the column kind and its share against the field kind). Confidence
50
+ is ``high`` only when the header equals an alias and the content fits the
51
+ kind, ``medium`` with one strong signal and the other neutral, ``low``
52
+ otherwise. A column whose header says nothing is suggested only when its
53
+ kind is one that values alone reveal (a phone, an e-mail, a date and so on)
54
+ and equals the field kind.
55
+
56
+ A column feeds at most one field. Candidates are taken from the highest
57
+ score down, ties going to the larger share, the longer alias, the leftmost
58
+ column and the earlier field. A field that loses a column takes its next
59
+ best candidate, and lists the columns it did not get in ``alternatives``.
60
+ A required field that nothing relates to takes the one remaining column its
61
+ kind accepts, with confidence ``low`` and the reason ``only_candidate``, and
62
+ has no column and the reason ``required_field_unmatched`` when there is not
63
+ exactly one.
64
+
65
+ A profile with no header row ignores headers: only content signals count.
66
+
67
+ Args:
68
+ profile: the profile of the table.
69
+ fields: the fields to fill. The library has no predefined field.
70
+
71
+ Returns:
72
+ A ``Mapping`` with one ``Choice`` per field, in declaration order, the
73
+ indexes of the columns no field uses, and the header fingerprint.
74
+
75
+ Raises:
76
+ ValueError: if two fields have the same key.
77
+ """
78
+ check_fields(fields)
79
+ has_header = profile.header_row is not None
80
+ kind_fields = Counter(field.kind for field in fields)
81
+ kind_columns = Counter(column.kind for column in profile.columns)
82
+
83
+ candidates: list[tuple[int, Candidate]] = []
84
+ for field_index, field in enumerate(fields):
85
+ aliases = alias_table(field)
86
+ unique = kind_fields[field.kind] == 1 and kind_columns[field.kind] == 1
87
+ for column in profile.columns:
88
+ found = score_column(
89
+ column, field, aliases, has_header=has_header, kind_is_unique=unique
90
+ )
91
+ if found is not None:
92
+ candidates.append((field_index, found))
93
+ candidates.sort(key=_order)
94
+
95
+ taken_fields: dict[int, Candidate] = {}
96
+ taken_columns: set[int] = set()
97
+ for field_index, candidate in candidates:
98
+ if field_index in taken_fields or candidate.column in taken_columns:
99
+ continue
100
+ taken_fields[field_index] = candidate
101
+ taken_columns.add(candidate.column)
102
+
103
+ fallback = _only_candidates(profile.columns, fields, taken_fields, taken_columns)
104
+ choices = []
105
+ for field_index, field in enumerate(fields):
106
+ alternatives = tuple(
107
+ Alternative(c.column, c.confidence)
108
+ for i, c in candidates
109
+ if i == field_index
110
+ and (field_index not in taken_fields or c.column != taken_fields[field_index].column)
111
+ )
112
+ won = taken_fields.get(field_index)
113
+ if won is not None:
114
+ choices.append(Choice(field.key, won.column, won.confidence, won.reasons, alternatives))
115
+ elif field_index in fallback:
116
+ choices.append(
117
+ Choice(
118
+ field.key, fallback[field_index][0], Confidence.LOW, fallback[field_index][1]
119
+ )
120
+ )
121
+ else:
122
+ reasons = (Reason(ReasonCode.REQUIRED_FIELD_UNMATCHED),) if field.required else ()
123
+ choices.append(Choice(field.key, None, None, reasons, alternatives))
124
+
125
+ used = {choice.column for choice in choices if choice.column is not None}
126
+ unmapped = tuple(column.index for column in profile.columns if column.index not in used)
127
+ return Mapping(tuple(choices), unmapped, fingerprint_of(profile))
128
+
129
+
130
+ _TEXT_FALLBACK = frozenset({Kind.TEXT, Kind.NAME, Kind.CATEGORY, Kind.INTEGER})
131
+
132
+
133
+ def _fallback_kinds(kind: Kind) -> frozenset[Kind]:
134
+ """Column kinds a required field may take as the only candidate.
135
+
136
+ A text field accepts every kind when a header matches, but as a fallback with
137
+ no header signal it takes only text-like columns, so a lone date column is
138
+ never guessed into it.
139
+ """
140
+ if kind is Kind.TEXT:
141
+ return _TEXT_FALLBACK
142
+ return frozenset({kind, *ACCEPTS.get(kind, frozenset())})
143
+
144
+
145
+ def _only_candidates(
146
+ columns: tuple[ColumnProfile, ...],
147
+ fields: Sequence[Field],
148
+ taken_fields: dict[int, Candidate],
149
+ taken_columns: set[int],
150
+ ) -> dict[int, tuple[int, tuple[Reason, ...]]]:
151
+ """For required fields left unmatched, the single remaining column they accept."""
152
+ result: dict[int, tuple[int, tuple[Reason, ...]]] = {}
153
+ used = set(taken_columns)
154
+ for field_index, field in enumerate(fields):
155
+ if field_index in taken_fields or not field.required:
156
+ continue
157
+ accepted = _fallback_kinds(field.kind)
158
+ remaining = [c for c in columns if c.index not in used and c.filled and c.kind in accepted]
159
+ if len(remaining) != 1:
160
+ continue
161
+ column = remaining[0]
162
+ used.add(column.index)
163
+ fits = Reason(
164
+ ReasonCode.CONTENT_FITS_KIND,
165
+ {"kind": str(column.kind), "share": round(column.kind_share, 4)},
166
+ )
167
+ result[field_index] = (column.index, (fits, Reason(ReasonCode.ONLY_CANDIDATE)))
168
+ return result
rowsift/map/_types.py ADDED
@@ -0,0 +1,169 @@
1
+ """Value types of the mapping layer."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from dataclasses import dataclass, replace
6
+
7
+ from rowsift.types import Confidence, Reason, ReasonCode
8
+
9
+ __all__ = ["Alternative", "Choice", "Mapping"]
10
+
11
+
12
+ @dataclass(frozen=True, slots=True)
13
+ class Alternative:
14
+ """Another column a field could take, with the confidence it would carry."""
15
+
16
+ column: int
17
+ confidence: Confidence
18
+
19
+
20
+ @dataclass(frozen=True, slots=True)
21
+ class Choice:
22
+ """The column suggested for one field.
23
+
24
+ Attributes:
25
+ field: the key of the field.
26
+ column: the 0-based index of the suggested column, or ``None`` when no
27
+ column is assigned.
28
+ confidence: how sure the suggestion is, or ``None`` when there is no
29
+ column.
30
+ reasons: the codes, with their data, that produced the suggestion or
31
+ explain why there is none.
32
+ alternatives: the other columns that were candidates for the field, best
33
+ first, each with its own confidence.
34
+ """
35
+
36
+ field: str
37
+ column: int | None
38
+ confidence: Confidence | None
39
+ reasons: tuple[Reason, ...] = ()
40
+ alternatives: tuple[Alternative, ...] = ()
41
+
42
+ def __post_init__(self) -> None:
43
+ if (self.column is None) != (self.confidence is None):
44
+ raise ValueError("a choice has a column and a confidence, or neither")
45
+
46
+
47
+ @dataclass(frozen=True, slots=True)
48
+ class Mapping:
49
+ """Which column feeds which field.
50
+
51
+ Attributes:
52
+ choices: one per declared field, in declaration order.
53
+ unmapped: the indexes of the columns no field uses, in table order. The
54
+ caller can offer them as extra attributes, using their profile.
55
+ fingerprint: SHA-256 in hexadecimal of the normalized headers in column
56
+ order, joined by a newline. It does not change with case, accents,
57
+ punctuation or spacing in the headers.
58
+
59
+ A column feeds at most one field. ``assign`` and ``unassign`` return a new
60
+ mapping and keep that rule.
61
+
62
+ Raises:
63
+ ValueError: if two choices share a field key or a column, a column index
64
+ is negative, ``unmapped`` is not sorted and free of repeats, or a
65
+ column is both assigned and listed as unmapped.
66
+ """
67
+
68
+ choices: tuple[Choice, ...]
69
+ unmapped: tuple[int, ...]
70
+ fingerprint: str
71
+
72
+ def __post_init__(self) -> None:
73
+ keys = [choice.field for choice in self.choices]
74
+ columns = [choice.column for choice in self.choices if choice.column is not None]
75
+ if len(set(keys)) != len(keys):
76
+ raise ValueError("two choices share a field key")
77
+ if len(set(columns)) != len(columns):
78
+ raise ValueError("a column feeds more than one field")
79
+ if any(index < 0 for index in (*columns, *self.unmapped)):
80
+ raise ValueError("a column index is negative")
81
+ if list(self.unmapped) != sorted(set(self.unmapped)):
82
+ raise ValueError("unmapped must be sorted, without repeats")
83
+ if set(columns) & set(self.unmapped):
84
+ raise ValueError("a column is both assigned and unmapped")
85
+
86
+ def assign(self, field_key: str, column_index: int) -> Mapping:
87
+ """Point a field at a column, as the caller decided.
88
+
89
+ The new choice has confidence ``high`` and the reason ``set_by_caller``.
90
+ The column the field had before goes to ``unmapped``. A field that held
91
+ the new column loses it, and its choice carries the reason
92
+ ``set_by_caller`` with ``displaced_by`` set to ``field_key``. The column a
93
+ choice gave up, in either case, is added to its alternatives, so a review
94
+ screen can offer it back.
95
+
96
+ Raises:
97
+ ValueError: if no choice has the key, or ``column_index`` is not a
98
+ column of the mapping.
99
+ TypeError: if ``column_index`` is not an ``int``.
100
+ """
101
+ if type(column_index) is not int:
102
+ raise TypeError("column_index must be int")
103
+ previous = self._choice(field_key)
104
+ if column_index not in self._columns():
105
+ raise ValueError(f"column {column_index} is not a column of the mapping")
106
+ choices: list[Choice] = []
107
+ for choice in self.choices:
108
+ if choice.field == field_key:
109
+ kept = tuple(a for a in choice.alternatives if a.column != column_index)
110
+ if choice.column not in (None, column_index) and choice.confidence is not None:
111
+ kept = (*kept, Alternative(choice.column, choice.confidence))
112
+ choice = replace(
113
+ choice,
114
+ column=column_index,
115
+ confidence=Confidence.HIGH,
116
+ reasons=(Reason(ReasonCode.SET_BY_CALLER),),
117
+ alternatives=kept,
118
+ )
119
+ elif choice.column == column_index and choice.confidence is not None:
120
+ lost = choice.confidence
121
+ choice = replace(
122
+ choice,
123
+ column=None,
124
+ confidence=None,
125
+ reasons=(Reason(ReasonCode.SET_BY_CALLER, {"displaced_by": field_key}),),
126
+ alternatives=(*choice.alternatives, Alternative(column_index, lost)),
127
+ )
128
+ choices.append(choice)
129
+ freed = {previous.column} - {None, column_index}
130
+ return self._rebuilt(choices, freed)
131
+
132
+ def unassign(self, field_key: str) -> Mapping:
133
+ """Take the column away from a field, as the caller decided.
134
+
135
+ The choice has no column and the reason ``set_by_caller``, and the column
136
+ goes to ``unmapped``.
137
+
138
+ Raises:
139
+ ValueError: if no choice has the key.
140
+ """
141
+ previous = self._choice(field_key)
142
+ choices = [
143
+ replace(
144
+ choice,
145
+ column=None,
146
+ confidence=None,
147
+ reasons=(Reason(ReasonCode.SET_BY_CALLER),),
148
+ )
149
+ if choice.field == field_key
150
+ else choice
151
+ for choice in self.choices
152
+ ]
153
+ freed = {previous.column} - {None}
154
+ return self._rebuilt(choices, freed)
155
+
156
+ def _choice(self, field_key: str) -> Choice:
157
+ for choice in self.choices:
158
+ if choice.field == field_key:
159
+ return choice
160
+ raise ValueError(f"no field with the key {field_key!r}")
161
+
162
+ def _columns(self) -> set[int]:
163
+ assigned = {choice.column for choice in self.choices if choice.column is not None}
164
+ return assigned | set(self.unmapped)
165
+
166
+ def _rebuilt(self, choices: list[Choice], freed: set[int | None]) -> Mapping:
167
+ assigned = {choice.column for choice in choices if choice.column is not None}
168
+ unmapped = sorted((set(self.unmapped) | {c for c in freed if c is not None}) - assigned)
169
+ return Mapping(tuple(choices), tuple(unmapped), self.fingerprint)
@@ -0,0 +1,17 @@
1
+ """Profile a table: where the header is, which rows are data, what each column holds."""
2
+
3
+ from rowsift.profile._headers import normalize_header
4
+ from rowsift.profile._options import ProfileOptions
5
+ from rowsift.profile._profile import profile
6
+ from rowsift.profile._types import ColumnProfile, Profile, RunnerUp, SkippedRow, TopValue
7
+
8
+ __all__ = [
9
+ "ColumnProfile",
10
+ "Profile",
11
+ "ProfileOptions",
12
+ "RunnerUp",
13
+ "SkippedRow",
14
+ "TopValue",
15
+ "normalize_header",
16
+ "profile",
17
+ ]
@@ -0,0 +1,44 @@
1
+ """Header normalization and column names."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import re
6
+ import unicodedata
7
+
8
+ __all__ = ["normalize_header"]
9
+
10
+ _NOT_ALPHANUMERIC = re.compile(r"[^0-9a-z]+")
11
+
12
+
13
+ def normalize_header(text: str) -> str:
14
+ """Fold a header for matching.
15
+
16
+ Lowercase, accents removed, every run of characters that are not letters or
17
+ digits replaced by one space, and the ends trimmed: ``Dt. Nascimento `` is
18
+ ``dt nascimento``. Only ASCII letters and digits survive, so a header written
19
+ in a script with no Latin decomposition folds to an empty string.
20
+ """
21
+ decomposed = unicodedata.normalize("NFKD", text.casefold())
22
+ bare = "".join(char for char in decomposed if not unicodedata.combining(char))
23
+ return _NOT_ALPHANUMERIC.sub(" ", bare).strip()
24
+
25
+
26
+ def column_names(headers: list[str] | None, width: int) -> list[str]:
27
+ """Unique normalized names for ``width`` columns.
28
+
29
+ ``headers`` is the header row as written, or ``None`` when the table has no
30
+ header. An empty header is named ``column_<position>``. A name already taken
31
+ gets the suffix `` 2``, `` 3`` and so on.
32
+ """
33
+ taken: set[str] = set()
34
+ names: list[str] = []
35
+ for position in range(width):
36
+ base = normalize_header(headers[position]) if headers is not None else ""
37
+ base = base or f"column_{position + 1}"
38
+ name, suffix = base, 1
39
+ while name in taken:
40
+ suffix += 1
41
+ name = f"{base} {suffix}"
42
+ taken.add(name)
43
+ names.append(name)
44
+ return names
@@ -0,0 +1,165 @@
1
+ """Kind inference: which kind most of a column's values fit."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import re
6
+ from collections.abc import Iterable
7
+
8
+ from rowsift import clean
9
+ from rowsift.clean import CleanOptions, mask
10
+ from rowsift.profile._types import RunnerUp
11
+ from rowsift.types import ChangeCode, Kind
12
+
13
+ __all__ = ["display", "infer_kind", "looks_like_data"]
14
+
15
+ # Most specific first. At equal footing the earlier kind wins, so a column of
16
+ # CPFs is a cpf and not an integer, and phones beat integers.
17
+ _ORDER: tuple[Kind, ...] = (
18
+ Kind.CPF,
19
+ Kind.CNPJ,
20
+ Kind.EMAIL,
21
+ Kind.PHONE,
22
+ Kind.DATETIME,
23
+ Kind.DATE,
24
+ Kind.DAY_MONTH,
25
+ Kind.TIME,
26
+ Kind.BOOLEAN,
27
+ Kind.POSTAL_CODE,
28
+ Kind.MONEY,
29
+ Kind.INTEGER,
30
+ Kind.DECIMAL,
31
+ )
32
+
33
+ # What a header cell never looks like. A boolean word is a plausible header.
34
+ _DATA_KINDS = tuple(kind for kind in _ORDER if kind is not Kind.BOOLEAN)
35
+
36
+ # Kinds that identify a person. A value of a column of another kind that fits one of
37
+ # them is masked as that kind, so a mixed column does not show it in clear.
38
+ _PERSONAL: tuple[Kind, ...] = (Kind.CPF, Kind.CNPJ, Kind.EMAIL, Kind.PHONE)
39
+ _HIDDEN = "***"
40
+
41
+ # No value of any kind is longer than an e-mail address can be, so a longer cell is
42
+ # never run through the cleaners. It bounds the cost of a column of long text.
43
+ _MAX_VALUE_CHARS = 254
44
+ # Personal data inside free text: an e-mail address, or a run of 10 or more digits
45
+ # that a phone number or a tax id would make, written with the usual separators.
46
+ _EMBEDDED = re.compile(r"@|[0-9](?:[ .()\-]?[0-9]){9}")
47
+
48
+ _POSTAL_SHAPE = re.compile(r"[0-9]{5}-?[0-9]{3}")
49
+ _CURRENCY = re.compile(r"R\$|US\$|BRL|USD|EUR|[$€£]", re.IGNORECASE)
50
+
51
+ # A cleaner may accept a value by a route that says nothing about the column:
52
+ # these changes mark a fit that is not evidence for the kind.
53
+ _NOT_EVIDENCE: dict[Kind, frozenset[ChangeCode]] = {
54
+ Kind.DATE: frozenset({ChangeCode.SERIAL_DATE_CONVERTED}),
55
+ Kind.DATETIME: frozenset({ChangeCode.SERIAL_DATE_CONVERTED, ChangeCode.TIME_DEFAULTED}),
56
+ Kind.DAY_MONTH: frozenset({ChangeCode.YEAR_DISCARDED}),
57
+ Kind.POSTAL_CODE: frozenset({ChangeCode.LEADING_ZEROS_RESTORED}),
58
+ }
59
+
60
+
61
+ def fits(text: str, kind: Kind, options: CleanOptions) -> bool:
62
+ """Whether ``text`` is a value of ``kind`` that the column can be said to hold.
63
+
64
+ A spreadsheet serial number is not a date here, a bare date is not a
65
+ datetime, a full date is not a day and month, and a postal code that lost
66
+ its leading zero is not a postal code: each would turn a column of plain
67
+ numbers or dates into something else. Money needs a currency symbol and a
68
+ postal code is written as 8 digits, with or without one hyphen.
69
+ """
70
+ if len(text) > _MAX_VALUE_CHARS:
71
+ return False
72
+ if kind is Kind.EMAIL:
73
+ if "@" not in text:
74
+ return False
75
+ elif kind is not Kind.BOOLEAN and not any("0" <= char <= "9" for char in text):
76
+ return False # every other kind is written with at least one digit
77
+ if kind is Kind.MONEY and not _CURRENCY.search(text):
78
+ return False
79
+ if kind is Kind.POSTAL_CODE and not _POSTAL_SHAPE.fullmatch(text.strip()):
80
+ return False # the cleaner tolerates any separator, a date of 8 digits would pass
81
+ result = clean.value(text, kind, options)
82
+ if result.issue is not None or result.value is None:
83
+ return False
84
+ blocked = _NOT_EVIDENCE.get(kind, frozenset())
85
+ return not any(change.code in blocked for change in result.changes)
86
+
87
+
88
+ def looks_like_data(text: str, options: CleanOptions) -> bool:
89
+ """Whether a cell holds a value that a header would not: a number, date, id and so on."""
90
+ return any(fits(text, kind, options) for kind in _DATA_KINDS)
91
+
92
+
93
+ def infer_kind(
94
+ values: Iterable[str], threshold: float, options: CleanOptions
95
+ ) -> tuple[Kind, float, RunnerUp | None]:
96
+ """The kind of a column from its non-empty values, with its share and runner-up.
97
+
98
+ The kind is the first of the candidates, most specific first, that at least
99
+ ``threshold`` of ``values`` fit. When none does it is ``text``, which every
100
+ value fits. The runner-up is the other candidate with the highest share
101
+ above zero, the more specific one on a tie.
102
+
103
+ ``0`` and ``1`` are boolean literals and ``boolean`` comes before ``integer``,
104
+ so a column holding only 0 and 1 is a ``boolean``, and the kind turns
105
+ ``integer`` as soon as another number appears. The ``integer`` share and the
106
+ runner-up report the other reading.
107
+ """
108
+ counts: dict[str, int] = {}
109
+ for value in values:
110
+ counts[value] = counts.get(value, 0) + 1
111
+ total = sum(counts.values())
112
+ if total == 0:
113
+ return Kind.TEXT, 0.0, None
114
+ shares: dict[Kind, float] = {}
115
+ for kind in _ORDER:
116
+ fitting = sum(count for value, count in counts.items() if fits(value, kind, options))
117
+ shares[kind] = fitting / total
118
+ chosen = next((kind for kind in _ORDER if shares[kind] >= threshold), None)
119
+ others = [kind for kind in _ORDER if kind is not chosen and shares[kind] > 0]
120
+ best = max(others, key=lambda kind: shares[kind], default=None)
121
+ runner_up = RunnerUp(best, shares[best]) if best is not None else None
122
+ if chosen is None:
123
+ return Kind.TEXT, 1.0, runner_up
124
+ return chosen, shares[chosen], runner_up
125
+
126
+
127
+ def _accepted(text: str, kind: Kind, options: CleanOptions) -> bool:
128
+ """Whether the cleaner of ``kind`` takes ``text``, whatever route it takes."""
129
+ if len(text) > _MAX_VALUE_CHARS:
130
+ return False
131
+ result = clean.value(text, kind, options)
132
+ return result.issue is None and result.value is not None
133
+
134
+
135
+ def display(text: str, kind: Kind, options: CleanOptions) -> str:
136
+ """``text`` as it is safe to show, given the kind of its column.
137
+
138
+ A column of a personal kind (phone, CPF, CNPJ, e-mail, name) masks each
139
+ value by that kind, and a value that does not have the shape of the kind is
140
+ replaced by a fixed placeholder, so neither its length nor its words show.
141
+
142
+ In any other column each value is judged by itself. A value that fits a
143
+ personal kind is masked as that kind, so a column that fell short of the
144
+ threshold, or mixes phones with notes, does not show the phones in clear. A
145
+ value with an e-mail address or a run of 10 or more digits inside it is the
146
+ placeholder. In a ``text`` column every other value is returned as written;
147
+ in a column of a typed kind (a date, an integer) a value the kind does not
148
+ take is the placeholder too, so a stray name is not shown.
149
+
150
+ A person's name in a ``text`` column is not recognized: ``profile`` never
151
+ infers ``name``, because no rule tells a name from free text with certainty.
152
+ """
153
+ if kind is Kind.NAME:
154
+ return mask(text, kind)
155
+ if kind in _PERSONAL:
156
+ return mask(text, kind) if fits(text, kind, options) else _HIDDEN
157
+ embedded = _EMBEDDED.search(text) is not None
158
+ if kind is not Kind.TEXT and not embedded and _accepted(text, kind, options):
159
+ return text
160
+ personal = next((k for k in _PERSONAL if fits(text, k, options)), None)
161
+ if personal is not None:
162
+ return mask(text, personal)
163
+ if embedded:
164
+ return _HIDDEN
165
+ return text if kind is Kind.TEXT else _HIDDEN
@@ -0,0 +1,69 @@
1
+ """Options of the profile layer."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from dataclasses import dataclass, field
6
+
7
+ from rowsift.clean import CleanOptions
8
+
9
+ __all__ = ["ProfileOptions"]
10
+
11
+
12
+ @dataclass(frozen=True, slots=True, kw_only=True)
13
+ class ProfileOptions:
14
+ """Every knob of ``profile``. All are passed by keyword.
15
+
16
+ Attributes:
17
+ header_scan_rows: how many non-empty rows are examined as header
18
+ candidates.
19
+ kind_threshold: the share of non-empty sampled values a kind must fit
20
+ for a column to take it, above 0 and at most 1.
21
+ sample_rows: how many data rows are read to infer the kind.
22
+ low_cardinality_min_filled: a column needs at least this many filled
23
+ cells to be low-cardinality.
24
+ low_cardinality_max_distinct: and at most this many distinct values.
25
+ low_cardinality_max_share: and at most this share of distinct values
26
+ over filled cells.
27
+ sample_count: how many distinct sample values to keep per column.
28
+ mask_samples: mask samples and top values by the kind of the column.
29
+ Turn it off only for values that are safe to display.
30
+ clean: the options the cleaners run with while inferring kinds, such as
31
+ the order of the day and the month.
32
+
33
+ Raises:
34
+ ValueError: if a value is outside what the attribute allows.
35
+ """
36
+
37
+ header_scan_rows: int = 20
38
+ kind_threshold: float = 0.8
39
+ sample_rows: int = 5000
40
+ low_cardinality_min_filled: int = 20
41
+ low_cardinality_max_distinct: int = 12
42
+ low_cardinality_max_share: float = 0.05
43
+ sample_count: int = 3
44
+ mask_samples: bool = True
45
+ clean: CleanOptions = field(default_factory=CleanOptions)
46
+
47
+ def __post_init__(self) -> None:
48
+ for name in (
49
+ "header_scan_rows",
50
+ "sample_rows",
51
+ "low_cardinality_min_filled",
52
+ "low_cardinality_max_distinct",
53
+ "sample_count",
54
+ ):
55
+ value = getattr(self, name)
56
+ if type(value) is not int or value < (1 if name.endswith("rows") else 0):
57
+ raise ValueError(f"{name} must be a whole number in range")
58
+ for name in ("kind_threshold", "low_cardinality_max_share"):
59
+ value = getattr(self, name)
60
+ if isinstance(value, bool) or not isinstance(value, int | float):
61
+ raise ValueError(f"{name} must be a number")
62
+ if not 0 < self.kind_threshold <= 1:
63
+ raise ValueError("kind_threshold must be above 0 and at most 1")
64
+ if not 0 <= self.low_cardinality_max_share <= 1:
65
+ raise ValueError("low_cardinality_max_share must be between 0 and 1")
66
+ if type(self.mask_samples) is not bool:
67
+ raise ValueError("mask_samples must be a bool")
68
+ if not isinstance(self.clean, CleanOptions):
69
+ raise ValueError("clean must be a CleanOptions")