rowsift 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rowsift/__init__.py +98 -0
- rowsift/clean/__init__.py +44 -0
- rowsift/clean/_common.py +30 -0
- rowsift/clean/_dates.py +283 -0
- rowsift/clean/_dispatch.py +49 -0
- rowsift/clean/_identifiers.py +161 -0
- rowsift/clean/_mask.py +117 -0
- rowsift/clean/_numbers.py +137 -0
- rowsift/clean/_options.py +75 -0
- rowsift/clean/_phone.py +202 -0
- rowsift/clean/_text.py +102 -0
- rowsift/errors.py +124 -0
- rowsift/map/__init__.py +24 -0
- rowsift/map/_fields.py +75 -0
- rowsift/map/_saved.py +132 -0
- rowsift/map/_score.py +178 -0
- rowsift/map/_suggest.py +168 -0
- rowsift/map/_types.py +169 -0
- rowsift/profile/__init__.py +17 -0
- rowsift/profile/_headers.py +44 -0
- rowsift/profile/_kinds.py +165 -0
- rowsift/profile/_options.py +69 -0
- rowsift/profile/_profile.py +119 -0
- rowsift/profile/_structure.py +157 -0
- rowsift/profile/_types.py +96 -0
- rowsift/py.typed +0 -0
- rowsift/read/__init__.py +6 -0
- rowsift/read/_cells.py +93 -0
- rowsift/read/_detect.py +124 -0
- rowsift/read/_grid.py +128 -0
- rowsift/read/_limits.py +44 -0
- rowsift/read/_read.py +99 -0
- rowsift/read/_text.py +250 -0
- rowsift/read/_workbook.py +36 -0
- rowsift/read/_xls.py +115 -0
- rowsift/read/_xlsx.py +121 -0
- rowsift/sift/__init__.py +19 -0
- rowsift/sift/_sift.py +165 -0
- rowsift/sift/_types.py +119 -0
- rowsift/types.py +378 -0
- rowsift-0.1.1.dist-info/METADATA +148 -0
- rowsift-0.1.1.dist-info/RECORD +45 -0
- rowsift-0.1.1.dist-info/WHEEL +4 -0
- rowsift-0.1.1.dist-info/licenses/LICENSE +177 -0
- rowsift-0.1.1.dist-info/licenses/NOTICE +4 -0
rowsift/errors.py
ADDED
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
"""Exceptions raised on purpose by the library.
|
|
2
|
+
|
|
3
|
+
Every one derives from ``RowsiftError`` and carries a code. Messages are built
|
|
4
|
+
from codes and constant names only: they never include cell contents. Every
|
|
5
|
+
one survives ``pickle`` and ``copy``, so it can cross process boundaries.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from typing import Any, Literal, get_args
|
|
11
|
+
|
|
12
|
+
from rowsift.types import ErrorCode
|
|
13
|
+
|
|
14
|
+
__all__ = [
|
|
15
|
+
"DecodingError",
|
|
16
|
+
"ExtraName",
|
|
17
|
+
"LimitExceededError",
|
|
18
|
+
"LimitName",
|
|
19
|
+
"MissingExtraError",
|
|
20
|
+
"RowsiftError",
|
|
21
|
+
"UnsupportedFileError",
|
|
22
|
+
]
|
|
23
|
+
|
|
24
|
+
LimitName = Literal[
|
|
25
|
+
"max_bytes",
|
|
26
|
+
"max_rows",
|
|
27
|
+
"max_columns",
|
|
28
|
+
"max_cell_chars",
|
|
29
|
+
"max_decompressed_bytes",
|
|
30
|
+
]
|
|
31
|
+
"""The limits a read can exceed. Constants only, never a value taken from a file."""
|
|
32
|
+
|
|
33
|
+
ExtraName = Literal["excel", "phone"]
|
|
34
|
+
"""The optional extras. Constants only, never a value taken from a file."""
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
_UNSUPPORTED_CODES = (
|
|
38
|
+
ErrorCode.UNSUPPORTED_FILE,
|
|
39
|
+
ErrorCode.SHEET_NOT_FOUND,
|
|
40
|
+
ErrorCode.PASSWORD_PROTECTED,
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class RowsiftError(Exception):
|
|
45
|
+
"""Base class of every exception the library raises on purpose.
|
|
46
|
+
|
|
47
|
+
Attributes:
|
|
48
|
+
code: the machine-readable reason.
|
|
49
|
+
"""
|
|
50
|
+
|
|
51
|
+
code: ErrorCode
|
|
52
|
+
|
|
53
|
+
def __init__(self, code: ErrorCode) -> None:
|
|
54
|
+
if not isinstance(code, ErrorCode):
|
|
55
|
+
raise TypeError("code must be an ErrorCode")
|
|
56
|
+
super().__init__(code.value)
|
|
57
|
+
self.code = code
|
|
58
|
+
|
|
59
|
+
def __reduce__(self) -> tuple[Any, tuple[Any, ...]]:
|
|
60
|
+
return (type(self), (self.code,))
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class UnsupportedFileError(RowsiftError):
|
|
64
|
+
"""The bytes are no supported format, or a requested sheet does not exist.
|
|
65
|
+
|
|
66
|
+
The code is ``unsupported_file`` by default, ``sheet_not_found`` when a
|
|
67
|
+
named sheet is absent and ``password_protected`` when the workbook is
|
|
68
|
+
encrypted with a password, which the library does not open.
|
|
69
|
+
"""
|
|
70
|
+
|
|
71
|
+
def __init__(self, code: ErrorCode = ErrorCode.UNSUPPORTED_FILE) -> None:
|
|
72
|
+
if code not in _UNSUPPORTED_CODES:
|
|
73
|
+
raise ValueError("code must be unsupported_file, sheet_not_found or password_protected")
|
|
74
|
+
super().__init__(code)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
class LimitExceededError(RowsiftError):
|
|
78
|
+
"""A read limit was exceeded.
|
|
79
|
+
|
|
80
|
+
Attributes:
|
|
81
|
+
limit: the name of the limit, such as ``max_rows``.
|
|
82
|
+
allowed: the value the limit allows.
|
|
83
|
+
"""
|
|
84
|
+
|
|
85
|
+
def __init__(self, limit: LimitName, allowed: int) -> None:
|
|
86
|
+
if limit not in get_args(LimitName):
|
|
87
|
+
raise ValueError("limit must be a known limit name")
|
|
88
|
+
if type(allowed) is not int or allowed < 0:
|
|
89
|
+
raise ValueError("allowed must be a non-negative int")
|
|
90
|
+
super().__init__(ErrorCode.LIMIT_EXCEEDED)
|
|
91
|
+
self.limit = limit
|
|
92
|
+
self.allowed = allowed
|
|
93
|
+
self.args = (f"{ErrorCode.LIMIT_EXCEEDED.value}: {limit} (allowed {allowed})",)
|
|
94
|
+
|
|
95
|
+
def __reduce__(self) -> tuple[Any, tuple[Any, ...]]:
|
|
96
|
+
return (type(self), (self.limit, self.allowed))
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
class MissingExtraError(RowsiftError):
|
|
100
|
+
"""A feature needs an optional extra that is not installed.
|
|
101
|
+
|
|
102
|
+
Attributes:
|
|
103
|
+
extra: the extra to install, such as ``excel`` or ``phone``.
|
|
104
|
+
"""
|
|
105
|
+
|
|
106
|
+
def __init__(self, extra: ExtraName) -> None:
|
|
107
|
+
if extra not in get_args(ExtraName):
|
|
108
|
+
raise ValueError("extra must be a known extra name")
|
|
109
|
+
super().__init__(ErrorCode.MISSING_EXTRA)
|
|
110
|
+
self.extra = extra
|
|
111
|
+
self.args = (f"{ErrorCode.MISSING_EXTRA.value}: install rowsift[{extra}]",)
|
|
112
|
+
|
|
113
|
+
def __reduce__(self) -> tuple[Any, tuple[Any, ...]]:
|
|
114
|
+
return (type(self), (self.extra,))
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
class DecodingError(RowsiftError):
|
|
118
|
+
"""The text of a file could not be decoded."""
|
|
119
|
+
|
|
120
|
+
def __init__(self) -> None:
|
|
121
|
+
super().__init__(ErrorCode.DECODING_FAILED)
|
|
122
|
+
|
|
123
|
+
def __reduce__(self) -> tuple[Any, tuple[Any, ...]]:
|
|
124
|
+
return (type(self), ())
|
rowsift/map/__init__.py
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
"""Map: suggest which column feeds each field the application declares.
|
|
2
|
+
|
|
3
|
+
``suggest`` scores the columns of a profile against the declared ``Field`` values
|
|
4
|
+
and returns a ``Mapping``: one ``Choice`` per field with a confidence and the
|
|
5
|
+
reasons as codes, the columns no field uses, and the fingerprint of the headers.
|
|
6
|
+
A person can change the result with ``Mapping.assign`` and ``Mapping.unassign``.
|
|
7
|
+
``SavedMapping`` keeps a mapping between files and ``apply_saved`` restores it
|
|
8
|
+
without guessing.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from rowsift.map._fields import Field
|
|
12
|
+
from rowsift.map._saved import SavedMapping, apply_saved
|
|
13
|
+
from rowsift.map._suggest import suggest
|
|
14
|
+
from rowsift.map._types import Alternative, Choice, Mapping
|
|
15
|
+
|
|
16
|
+
__all__ = [
|
|
17
|
+
"Alternative",
|
|
18
|
+
"Choice",
|
|
19
|
+
"Field",
|
|
20
|
+
"Mapping",
|
|
21
|
+
"SavedMapping",
|
|
22
|
+
"apply_saved",
|
|
23
|
+
"suggest",
|
|
24
|
+
]
|
rowsift/map/_fields.py
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
"""The fields a caller declares."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Iterable
|
|
6
|
+
from dataclasses import KW_ONLY, dataclass
|
|
7
|
+
|
|
8
|
+
from rowsift.clean import CleanOptions
|
|
9
|
+
from rowsift.types import Kind
|
|
10
|
+
|
|
11
|
+
__all__ = ["Field"]
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _texts(name: str, values: Iterable[str]) -> tuple[str, ...]:
|
|
15
|
+
if isinstance(values, str):
|
|
16
|
+
raise TypeError(f"{name} must be a sequence of str, not a bare str")
|
|
17
|
+
items = tuple(values)
|
|
18
|
+
if any(type(item) is not str for item in items):
|
|
19
|
+
raise TypeError(f"{name} must hold only str")
|
|
20
|
+
return items
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dataclass(frozen=True, slots=True)
|
|
24
|
+
class Field:
|
|
25
|
+
"""A field the application wants filled from a column.
|
|
26
|
+
|
|
27
|
+
The library declares no field: every field comes from the caller. The key
|
|
28
|
+
of a field also counts as one of its aliases, so a field ``phone`` matches
|
|
29
|
+
a column headed ``Phone`` without declaring the alias.
|
|
30
|
+
|
|
31
|
+
Attributes:
|
|
32
|
+
key: the name the application uses for the field. Unique among the
|
|
33
|
+
fields passed together.
|
|
34
|
+
kind: what the field holds.
|
|
35
|
+
required: whether a row with no value for the field is rejected when the
|
|
36
|
+
rows are sifted. A required field that no column fits is reported by
|
|
37
|
+
``suggest``.
|
|
38
|
+
aliases: other headers the column may carry. Compared after the same
|
|
39
|
+
folding as the headers (case, accents and punctuation ignored).
|
|
40
|
+
one_of: the allowed values of a ``Kind.CATEGORY`` field, or ``None`` for
|
|
41
|
+
any text. Only a category field may set it.
|
|
42
|
+
options: the cleaning options for this field alone, or ``None`` to use
|
|
43
|
+
the options of the call.
|
|
44
|
+
|
|
45
|
+
Raises:
|
|
46
|
+
TypeError: if the key is not ``str``, ``aliases`` or ``one_of`` is a bare
|
|
47
|
+
``str`` or holds a non-``str``, ``required`` is not ``bool`` or
|
|
48
|
+
``options`` is not a ``CleanOptions``.
|
|
49
|
+
ValueError: if the key is empty, ``kind`` is not a ``Kind``, or ``one_of``
|
|
50
|
+
is set on a field that is not a category.
|
|
51
|
+
"""
|
|
52
|
+
|
|
53
|
+
key: str
|
|
54
|
+
kind: Kind
|
|
55
|
+
_: KW_ONLY
|
|
56
|
+
required: bool = False
|
|
57
|
+
aliases: tuple[str, ...] = ()
|
|
58
|
+
one_of: tuple[str, ...] | None = None
|
|
59
|
+
options: CleanOptions | None = None
|
|
60
|
+
|
|
61
|
+
def __post_init__(self) -> None:
|
|
62
|
+
if type(self.key) is not str:
|
|
63
|
+
raise TypeError("key must be str")
|
|
64
|
+
if not self.key.strip():
|
|
65
|
+
raise ValueError("key must not be empty")
|
|
66
|
+
object.__setattr__(self, "kind", Kind(self.kind))
|
|
67
|
+
if type(self.required) is not bool:
|
|
68
|
+
raise TypeError("required must be bool")
|
|
69
|
+
object.__setattr__(self, "aliases", _texts("aliases", self.aliases))
|
|
70
|
+
if self.one_of is not None:
|
|
71
|
+
if self.kind is not Kind.CATEGORY:
|
|
72
|
+
raise ValueError("one_of is only for a category field")
|
|
73
|
+
object.__setattr__(self, "one_of", _texts("one_of", self.one_of))
|
|
74
|
+
if self.options is not None and not isinstance(self.options, CleanOptions):
|
|
75
|
+
raise TypeError("options must be a CleanOptions or None")
|
rowsift/map/_saved.py
ADDED
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
"""Saved mappings: keep a person's choices and apply them to the next file."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Mapping as MappingABC
|
|
6
|
+
from collections.abc import Sequence
|
|
7
|
+
from dataclasses import dataclass, field
|
|
8
|
+
from types import MappingProxyType
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
from rowsift.map._fields import Field
|
|
12
|
+
from rowsift.map._suggest import check_fields, fingerprint_of
|
|
13
|
+
from rowsift.map._types import Choice, Mapping
|
|
14
|
+
from rowsift.profile import Profile
|
|
15
|
+
from rowsift.types import Confidence, Reason, ReasonCode
|
|
16
|
+
|
|
17
|
+
__all__ = ["SavedMapping", "apply_saved"]
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@dataclass(frozen=True, slots=True)
|
|
21
|
+
class SavedMapping:
|
|
22
|
+
"""A mapping reduced to what survives between files: header names.
|
|
23
|
+
|
|
24
|
+
Attributes:
|
|
25
|
+
fingerprint: the header fingerprint of the file it was saved from.
|
|
26
|
+
assignments: for each field key, the normalized header of the column it
|
|
27
|
+
was mapped to. A field with no column is not listed. Copied into a
|
|
28
|
+
read-only mapping.
|
|
29
|
+
|
|
30
|
+
Raises:
|
|
31
|
+
TypeError: if the fingerprint is not ``str`` or an assignment is not a
|
|
32
|
+
``str`` to ``str`` pair.
|
|
33
|
+
ValueError: if two fields point at the same header.
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
fingerprint: str
|
|
37
|
+
assignments: MappingABC[str, str] = field(default_factory=dict)
|
|
38
|
+
|
|
39
|
+
def __post_init__(self) -> None:
|
|
40
|
+
if type(self.fingerprint) is not str:
|
|
41
|
+
raise TypeError("fingerprint must be str")
|
|
42
|
+
copied = dict(self.assignments)
|
|
43
|
+
for key, header in copied.items():
|
|
44
|
+
if type(key) is not str or type(header) is not str:
|
|
45
|
+
raise TypeError("assignments must map str to str")
|
|
46
|
+
if len(set(copied.values())) != len(copied):
|
|
47
|
+
raise ValueError("two fields are saved on the same header")
|
|
48
|
+
object.__setattr__(self, "assignments", MappingProxyType(copied))
|
|
49
|
+
|
|
50
|
+
def __hash__(self) -> int:
|
|
51
|
+
return hash((self.fingerprint, frozenset(self.assignments.items())))
|
|
52
|
+
|
|
53
|
+
def __reduce__(self) -> tuple[Any, tuple[Any, ...]]:
|
|
54
|
+
return (type(self), (self.fingerprint, dict(self.assignments)))
|
|
55
|
+
|
|
56
|
+
@classmethod
|
|
57
|
+
def from_mapping(cls, mapping: Mapping, profile: Profile) -> SavedMapping:
|
|
58
|
+
"""Save the assigned columns of ``mapping`` by their normalized header.
|
|
59
|
+
|
|
60
|
+
Raises:
|
|
61
|
+
ValueError: if the fingerprint of ``profile`` differs from the one of
|
|
62
|
+
``mapping``, that is, the mapping was not made from this profile.
|
|
63
|
+
"""
|
|
64
|
+
if fingerprint_of(profile) != mapping.fingerprint:
|
|
65
|
+
raise ValueError("the mapping was not made from this profile: fingerprints differ")
|
|
66
|
+
headers = {column.index: column.normalized_header for column in profile.columns}
|
|
67
|
+
assignments: dict[str, str] = {}
|
|
68
|
+
for choice in mapping.choices:
|
|
69
|
+
if choice.column is None:
|
|
70
|
+
continue
|
|
71
|
+
assignments[choice.field] = headers[choice.column]
|
|
72
|
+
return cls(mapping.fingerprint, assignments)
|
|
73
|
+
|
|
74
|
+
def to_dict(self) -> dict[str, Any]:
|
|
75
|
+
"""A plain dictionary of text, ready for JSON: ``fingerprint`` and ``assignments``."""
|
|
76
|
+
return {"fingerprint": self.fingerprint, "assignments": dict(self.assignments)}
|
|
77
|
+
|
|
78
|
+
@classmethod
|
|
79
|
+
def from_dict(cls, data: MappingABC[str, Any]) -> SavedMapping:
|
|
80
|
+
"""Read what ``to_dict`` wrote.
|
|
81
|
+
|
|
82
|
+
Raises:
|
|
83
|
+
ValueError: if a key is missing, or two fields share a header.
|
|
84
|
+
TypeError: if a value is not of the expected type.
|
|
85
|
+
"""
|
|
86
|
+
try:
|
|
87
|
+
fingerprint = data["fingerprint"]
|
|
88
|
+
assignments = data["assignments"]
|
|
89
|
+
except KeyError as error:
|
|
90
|
+
raise ValueError(f"a saved mapping needs the key {error.args[0]!r}") from None
|
|
91
|
+
if not isinstance(assignments, MappingABC):
|
|
92
|
+
raise TypeError("assignments must be a mapping")
|
|
93
|
+
return cls(fingerprint, dict(assignments))
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def apply_saved(profile: Profile, fields: Sequence[Field], saved: SavedMapping) -> Mapping:
|
|
97
|
+
"""Restore a saved mapping onto a new profile, without guessing.
|
|
98
|
+
|
|
99
|
+
When the fingerprint of the profile equals the saved one, every assignment is
|
|
100
|
+
restored with confidence ``high`` and the reason ``saved_mapping``. When it
|
|
101
|
+
differs, an assignment is restored, with confidence ``medium`` and the same
|
|
102
|
+
reason, only if a column still has the saved normalized header. The others
|
|
103
|
+
come back with no column and the reason ``saved_header_missing``, carrying the
|
|
104
|
+
saved header. A renamed header is never matched by similarity.
|
|
105
|
+
|
|
106
|
+
A field the saved mapping does not list comes back with no column, and the
|
|
107
|
+
reason ``required_field_unmatched`` when it is required. An optional field
|
|
108
|
+
the saved mapping does not list has no reasons, as in ``suggest``. A saved key that no
|
|
109
|
+
field declares is ignored.
|
|
110
|
+
|
|
111
|
+
Raises:
|
|
112
|
+
ValueError: if two fields have the same key.
|
|
113
|
+
"""
|
|
114
|
+
check_fields(fields)
|
|
115
|
+
same_layout = fingerprint_of(profile) == saved.fingerprint
|
|
116
|
+
by_header = {column.normalized_header: column.index for column in profile.columns}
|
|
117
|
+
confidence = Confidence.HIGH if same_layout else Confidence.MEDIUM
|
|
118
|
+
choices: list[Choice] = []
|
|
119
|
+
for field_ in fields:
|
|
120
|
+
header = saved.assignments.get(field_.key)
|
|
121
|
+
if header is None:
|
|
122
|
+
reasons = (Reason(ReasonCode.REQUIRED_FIELD_UNMATCHED),) if field_.required else ()
|
|
123
|
+
choices.append(Choice(field_.key, None, None, reasons))
|
|
124
|
+
elif header in by_header:
|
|
125
|
+
reason = Reason(ReasonCode.SAVED_MAPPING)
|
|
126
|
+
choices.append(Choice(field_.key, by_header[header], confidence, (reason,)))
|
|
127
|
+
else:
|
|
128
|
+
reason = Reason(ReasonCode.SAVED_HEADER_MISSING, {"header": header})
|
|
129
|
+
choices.append(Choice(field_.key, None, None, (reason,)))
|
|
130
|
+
used = {choice.column for choice in choices if choice.column is not None}
|
|
131
|
+
unmapped = tuple(column.index for column in profile.columns if column.index not in used)
|
|
132
|
+
return Mapping(tuple(choices), unmapped, fingerprint_of(profile))
|
rowsift/map/_score.py
ADDED
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
"""Scoring one column against one field, from the header and from the content."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from enum import Enum, auto
|
|
7
|
+
|
|
8
|
+
from rowsift.map._fields import Field
|
|
9
|
+
from rowsift.profile import ColumnProfile, normalize_header
|
|
10
|
+
from rowsift.types import Confidence, Kind, Reason, ReasonCode
|
|
11
|
+
|
|
12
|
+
__all__ = ["Candidate", "HeaderMatch", "alias_table", "score_column"]
|
|
13
|
+
|
|
14
|
+
SPECIFIC_KINDS = frozenset(
|
|
15
|
+
{
|
|
16
|
+
Kind.PHONE,
|
|
17
|
+
Kind.EMAIL,
|
|
18
|
+
Kind.CPF,
|
|
19
|
+
Kind.CNPJ,
|
|
20
|
+
Kind.POSTAL_CODE,
|
|
21
|
+
Kind.DATE,
|
|
22
|
+
Kind.DAY_MONTH,
|
|
23
|
+
Kind.TIME,
|
|
24
|
+
Kind.DATETIME,
|
|
25
|
+
}
|
|
26
|
+
)
|
|
27
|
+
"""Kinds a column can only take by what its values look like. A column of one of
|
|
28
|
+
these kinds relates to a field of the same kind even when the header says nothing."""
|
|
29
|
+
|
|
30
|
+
ACCEPTS: dict[Kind, frozenset[Kind]] = {
|
|
31
|
+
Kind.TEXT: frozenset(Kind) - {Kind.TEXT},
|
|
32
|
+
Kind.NAME: frozenset({Kind.TEXT, Kind.CATEGORY}),
|
|
33
|
+
Kind.CATEGORY: frozenset({Kind.TEXT, Kind.NAME, Kind.INTEGER, Kind.BOOLEAN}),
|
|
34
|
+
Kind.INTEGER: frozenset({Kind.BOOLEAN}),
|
|
35
|
+
Kind.DECIMAL: frozenset({Kind.INTEGER}),
|
|
36
|
+
Kind.MONEY: frozenset({Kind.DECIMAL, Kind.INTEGER}),
|
|
37
|
+
Kind.DATE: frozenset({Kind.DATETIME}),
|
|
38
|
+
Kind.DATETIME: frozenset({Kind.DATE}),
|
|
39
|
+
Kind.DAY_MONTH: frozenset({Kind.DATE}),
|
|
40
|
+
}
|
|
41
|
+
"""For a field kind, the other column kinds whose values the cleaner can still read.
|
|
42
|
+
A text field reads every kind, since any cell is text."""
|
|
43
|
+
|
|
44
|
+
_EXACT_POINTS = 4
|
|
45
|
+
_CONTAINS_POINTS = 2
|
|
46
|
+
_FIT_EXACT_POINTS = 2
|
|
47
|
+
_FIT_ACCEPTED_POINTS = 1
|
|
48
|
+
_CONFLICT_POINTS = -1
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class HeaderMatch(Enum):
|
|
52
|
+
"""How a header relates to the aliases of a field."""
|
|
53
|
+
|
|
54
|
+
NONE = auto()
|
|
55
|
+
CONTAINS = auto()
|
|
56
|
+
EXACT = auto()
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class _Content(Enum):
|
|
60
|
+
NEUTRAL = auto()
|
|
61
|
+
FITS = auto()
|
|
62
|
+
CONFLICTS = auto()
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
@dataclass(frozen=True, slots=True)
|
|
66
|
+
class Candidate:
|
|
67
|
+
"""One column as a candidate for one field.
|
|
68
|
+
|
|
69
|
+
Candidates are ordered by the higher score, then the larger share, then the
|
|
70
|
+
longer alias. ``confidence`` and ``reasons`` are what the choice carries if
|
|
71
|
+
the candidate wins.
|
|
72
|
+
"""
|
|
73
|
+
|
|
74
|
+
column: int
|
|
75
|
+
score: int
|
|
76
|
+
share: float
|
|
77
|
+
alias_tokens: int
|
|
78
|
+
confidence: Confidence
|
|
79
|
+
reasons: tuple[Reason, ...]
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def alias_table(field: Field) -> tuple[str, ...]:
|
|
83
|
+
"""The folded aliases of a field: its declared aliases and its key."""
|
|
84
|
+
folded = (normalize_header(alias) for alias in (*field.aliases, field.key))
|
|
85
|
+
return tuple(dict.fromkeys(alias for alias in folded if alias))
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _header_match(column: ColumnProfile, aliases: tuple[str, ...]) -> tuple[HeaderMatch, str, int]:
|
|
89
|
+
"""The best match of the header against the aliases: kind, alias, alias tokens."""
|
|
90
|
+
if not column.header or not column.normalized_header:
|
|
91
|
+
return HeaderMatch.NONE, "", 0
|
|
92
|
+
header = column.normalized_header
|
|
93
|
+
header_tokens = set(header.split())
|
|
94
|
+
best = (HeaderMatch.NONE, "", 0)
|
|
95
|
+
for alias in aliases:
|
|
96
|
+
tokens = alias.split()
|
|
97
|
+
if alias == header:
|
|
98
|
+
return HeaderMatch.EXACT, alias, len(tokens)
|
|
99
|
+
if set(tokens) <= header_tokens and len(tokens) > best[2]:
|
|
100
|
+
best = (HeaderMatch.CONTAINS, alias, len(tokens))
|
|
101
|
+
return best
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _content(column: ColumnProfile, field: Field) -> tuple[_Content, float, bool]:
|
|
105
|
+
"""Whether the content fits the field: verdict, share, and whether the kind is equal."""
|
|
106
|
+
if column.filled == 0:
|
|
107
|
+
return _Content.NEUTRAL, 0.0, False
|
|
108
|
+
if column.kind is field.kind:
|
|
109
|
+
return _Content.FITS, column.kind_share, True
|
|
110
|
+
if column.kind in ACCEPTS.get(field.kind, frozenset()):
|
|
111
|
+
return _Content.FITS, column.kind_share, False
|
|
112
|
+
runner = column.runner_up
|
|
113
|
+
share = runner.share if runner is not None and runner.kind is field.kind else 0.0
|
|
114
|
+
return _Content.CONFLICTS, share, False
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def score_column(
|
|
118
|
+
column: ColumnProfile,
|
|
119
|
+
field: Field,
|
|
120
|
+
aliases: tuple[str, ...],
|
|
121
|
+
*,
|
|
122
|
+
has_header: bool,
|
|
123
|
+
kind_is_unique: bool,
|
|
124
|
+
) -> Candidate | None:
|
|
125
|
+
"""Score a column for a field, or ``None`` when nothing relates them.
|
|
126
|
+
|
|
127
|
+
A header signal is an alias equal to the header (exact) or all of whose words
|
|
128
|
+
appear in it (contains). A content signal is the column kind fitting the
|
|
129
|
+
field kind. ``high`` needs an exact header and fitting content. With no
|
|
130
|
+
header signal, only a column whose kind is the field kind, and a kind that
|
|
131
|
+
values alone can reveal, is a candidate: ``medium`` when it is the only
|
|
132
|
+
column and the only field of that kind, ``low`` otherwise.
|
|
133
|
+
|
|
134
|
+
``has_header`` is false for a profile with no header row, where headers carry
|
|
135
|
+
no meaning. ``kind_is_unique`` says whether the field is the only declared
|
|
136
|
+
field of its kind and the column the only column of that kind.
|
|
137
|
+
"""
|
|
138
|
+
match, alias, alias_tokens = (
|
|
139
|
+
_header_match(column, aliases) if has_header else (HeaderMatch.NONE, "", 0)
|
|
140
|
+
)
|
|
141
|
+
content, share, exact_kind = _content(column, field)
|
|
142
|
+
fits = Reason(
|
|
143
|
+
ReasonCode.CONTENT_FITS_KIND, {"kind": str(column.kind), "share": round(share, 4)}
|
|
144
|
+
)
|
|
145
|
+
conflicts = Reason(
|
|
146
|
+
ReasonCode.CONTENT_CONFLICTS_WITH_KIND,
|
|
147
|
+
{
|
|
148
|
+
"column_kind": str(column.kind),
|
|
149
|
+
"field_kind": str(field.kind),
|
|
150
|
+
"share": round(share, 4),
|
|
151
|
+
},
|
|
152
|
+
)
|
|
153
|
+
|
|
154
|
+
if match is HeaderMatch.NONE:
|
|
155
|
+
if content is not _Content.FITS or not exact_kind or field.kind not in SPECIFIC_KINDS:
|
|
156
|
+
return None
|
|
157
|
+
confidence = Confidence.MEDIUM if kind_is_unique else Confidence.LOW
|
|
158
|
+
return Candidate(column.index, _FIT_EXACT_POINTS, share, 0, confidence, (fits,))
|
|
159
|
+
|
|
160
|
+
exact = match is HeaderMatch.EXACT
|
|
161
|
+
header_reason = Reason(
|
|
162
|
+
ReasonCode.HEADER_MATCHES_ALIAS if exact else ReasonCode.HEADER_CONTAINS_ALIAS,
|
|
163
|
+
{"alias": alias},
|
|
164
|
+
)
|
|
165
|
+
score = _EXACT_POINTS if exact else _CONTAINS_POINTS
|
|
166
|
+
if content is _Content.FITS:
|
|
167
|
+
score += _FIT_EXACT_POINTS if exact_kind else _FIT_ACCEPTED_POINTS
|
|
168
|
+
confidence = Confidence.HIGH if exact else Confidence.MEDIUM
|
|
169
|
+
return Candidate(
|
|
170
|
+
column.index, score, share, alias_tokens, confidence, (header_reason, fits)
|
|
171
|
+
)
|
|
172
|
+
if content is _Content.NEUTRAL:
|
|
173
|
+
confidence = Confidence.MEDIUM if exact else Confidence.LOW
|
|
174
|
+
return Candidate(column.index, score, 0.0, alias_tokens, confidence, (header_reason,))
|
|
175
|
+
score += _CONFLICT_POINTS
|
|
176
|
+
return Candidate(
|
|
177
|
+
column.index, score, share, alias_tokens, Confidence.LOW, (header_reason, conflicts)
|
|
178
|
+
)
|