rowsift 0.1.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
rowsift/errors.py ADDED
@@ -0,0 +1,124 @@
1
+ """Exceptions raised on purpose by the library.
2
+
3
+ Every one derives from ``RowsiftError`` and carries a code. Messages are built
4
+ from codes and constant names only: they never include cell contents. Every
5
+ one survives ``pickle`` and ``copy``, so it can cross process boundaries.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from typing import Any, Literal, get_args
11
+
12
+ from rowsift.types import ErrorCode
13
+
14
+ __all__ = [
15
+ "DecodingError",
16
+ "ExtraName",
17
+ "LimitExceededError",
18
+ "LimitName",
19
+ "MissingExtraError",
20
+ "RowsiftError",
21
+ "UnsupportedFileError",
22
+ ]
23
+
24
+ LimitName = Literal[
25
+ "max_bytes",
26
+ "max_rows",
27
+ "max_columns",
28
+ "max_cell_chars",
29
+ "max_decompressed_bytes",
30
+ ]
31
+ """The limits a read can exceed. Constants only, never a value taken from a file."""
32
+
33
+ ExtraName = Literal["excel", "phone"]
34
+ """The optional extras. Constants only, never a value taken from a file."""
35
+
36
+
37
+ _UNSUPPORTED_CODES = (
38
+ ErrorCode.UNSUPPORTED_FILE,
39
+ ErrorCode.SHEET_NOT_FOUND,
40
+ ErrorCode.PASSWORD_PROTECTED,
41
+ )
42
+
43
+
44
+ class RowsiftError(Exception):
45
+ """Base class of every exception the library raises on purpose.
46
+
47
+ Attributes:
48
+ code: the machine-readable reason.
49
+ """
50
+
51
+ code: ErrorCode
52
+
53
+ def __init__(self, code: ErrorCode) -> None:
54
+ if not isinstance(code, ErrorCode):
55
+ raise TypeError("code must be an ErrorCode")
56
+ super().__init__(code.value)
57
+ self.code = code
58
+
59
+ def __reduce__(self) -> tuple[Any, tuple[Any, ...]]:
60
+ return (type(self), (self.code,))
61
+
62
+
63
+ class UnsupportedFileError(RowsiftError):
64
+ """The bytes are no supported format, or a requested sheet does not exist.
65
+
66
+ The code is ``unsupported_file`` by default, ``sheet_not_found`` when a
67
+ named sheet is absent and ``password_protected`` when the workbook is
68
+ encrypted with a password, which the library does not open.
69
+ """
70
+
71
+ def __init__(self, code: ErrorCode = ErrorCode.UNSUPPORTED_FILE) -> None:
72
+ if code not in _UNSUPPORTED_CODES:
73
+ raise ValueError("code must be unsupported_file, sheet_not_found or password_protected")
74
+ super().__init__(code)
75
+
76
+
77
+ class LimitExceededError(RowsiftError):
78
+ """A read limit was exceeded.
79
+
80
+ Attributes:
81
+ limit: the name of the limit, such as ``max_rows``.
82
+ allowed: the value the limit allows.
83
+ """
84
+
85
+ def __init__(self, limit: LimitName, allowed: int) -> None:
86
+ if limit not in get_args(LimitName):
87
+ raise ValueError("limit must be a known limit name")
88
+ if type(allowed) is not int or allowed < 0:
89
+ raise ValueError("allowed must be a non-negative int")
90
+ super().__init__(ErrorCode.LIMIT_EXCEEDED)
91
+ self.limit = limit
92
+ self.allowed = allowed
93
+ self.args = (f"{ErrorCode.LIMIT_EXCEEDED.value}: {limit} (allowed {allowed})",)
94
+
95
+ def __reduce__(self) -> tuple[Any, tuple[Any, ...]]:
96
+ return (type(self), (self.limit, self.allowed))
97
+
98
+
99
+ class MissingExtraError(RowsiftError):
100
+ """A feature needs an optional extra that is not installed.
101
+
102
+ Attributes:
103
+ extra: the extra to install, such as ``excel`` or ``phone``.
104
+ """
105
+
106
+ def __init__(self, extra: ExtraName) -> None:
107
+ if extra not in get_args(ExtraName):
108
+ raise ValueError("extra must be a known extra name")
109
+ super().__init__(ErrorCode.MISSING_EXTRA)
110
+ self.extra = extra
111
+ self.args = (f"{ErrorCode.MISSING_EXTRA.value}: install rowsift[{extra}]",)
112
+
113
+ def __reduce__(self) -> tuple[Any, tuple[Any, ...]]:
114
+ return (type(self), (self.extra,))
115
+
116
+
117
+ class DecodingError(RowsiftError):
118
+ """The text of a file could not be decoded."""
119
+
120
+ def __init__(self) -> None:
121
+ super().__init__(ErrorCode.DECODING_FAILED)
122
+
123
+ def __reduce__(self) -> tuple[Any, tuple[Any, ...]]:
124
+ return (type(self), ())
@@ -0,0 +1,24 @@
1
+ """Map: suggest which column feeds each field the application declares.
2
+
3
+ ``suggest`` scores the columns of a profile against the declared ``Field`` values
4
+ and returns a ``Mapping``: one ``Choice`` per field with a confidence and the
5
+ reasons as codes, the columns no field uses, and the fingerprint of the headers.
6
+ A person can change the result with ``Mapping.assign`` and ``Mapping.unassign``.
7
+ ``SavedMapping`` keeps a mapping between files and ``apply_saved`` restores it
8
+ without guessing.
9
+ """
10
+
11
+ from rowsift.map._fields import Field
12
+ from rowsift.map._saved import SavedMapping, apply_saved
13
+ from rowsift.map._suggest import suggest
14
+ from rowsift.map._types import Alternative, Choice, Mapping
15
+
16
+ __all__ = [
17
+ "Alternative",
18
+ "Choice",
19
+ "Field",
20
+ "Mapping",
21
+ "SavedMapping",
22
+ "apply_saved",
23
+ "suggest",
24
+ ]
rowsift/map/_fields.py ADDED
@@ -0,0 +1,75 @@
1
+ """The fields a caller declares."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from collections.abc import Iterable
6
+ from dataclasses import KW_ONLY, dataclass
7
+
8
+ from rowsift.clean import CleanOptions
9
+ from rowsift.types import Kind
10
+
11
+ __all__ = ["Field"]
12
+
13
+
14
+ def _texts(name: str, values: Iterable[str]) -> tuple[str, ...]:
15
+ if isinstance(values, str):
16
+ raise TypeError(f"{name} must be a sequence of str, not a bare str")
17
+ items = tuple(values)
18
+ if any(type(item) is not str for item in items):
19
+ raise TypeError(f"{name} must hold only str")
20
+ return items
21
+
22
+
23
+ @dataclass(frozen=True, slots=True)
24
+ class Field:
25
+ """A field the application wants filled from a column.
26
+
27
+ The library declares no field: every field comes from the caller. The key
28
+ of a field also counts as one of its aliases, so a field ``phone`` matches
29
+ a column headed ``Phone`` without declaring the alias.
30
+
31
+ Attributes:
32
+ key: the name the application uses for the field. Unique among the
33
+ fields passed together.
34
+ kind: what the field holds.
35
+ required: whether a row with no value for the field is rejected when the
36
+ rows are sifted. A required field that no column fits is reported by
37
+ ``suggest``.
38
+ aliases: other headers the column may carry. Compared after the same
39
+ folding as the headers (case, accents and punctuation ignored).
40
+ one_of: the allowed values of a ``Kind.CATEGORY`` field, or ``None`` for
41
+ any text. Only a category field may set it.
42
+ options: the cleaning options for this field alone, or ``None`` to use
43
+ the options of the call.
44
+
45
+ Raises:
46
+ TypeError: if the key is not ``str``, ``aliases`` or ``one_of`` is a bare
47
+ ``str`` or holds a non-``str``, ``required`` is not ``bool`` or
48
+ ``options`` is not a ``CleanOptions``.
49
+ ValueError: if the key is empty, ``kind`` is not a ``Kind``, or ``one_of``
50
+ is set on a field that is not a category.
51
+ """
52
+
53
+ key: str
54
+ kind: Kind
55
+ _: KW_ONLY
56
+ required: bool = False
57
+ aliases: tuple[str, ...] = ()
58
+ one_of: tuple[str, ...] | None = None
59
+ options: CleanOptions | None = None
60
+
61
+ def __post_init__(self) -> None:
62
+ if type(self.key) is not str:
63
+ raise TypeError("key must be str")
64
+ if not self.key.strip():
65
+ raise ValueError("key must not be empty")
66
+ object.__setattr__(self, "kind", Kind(self.kind))
67
+ if type(self.required) is not bool:
68
+ raise TypeError("required must be bool")
69
+ object.__setattr__(self, "aliases", _texts("aliases", self.aliases))
70
+ if self.one_of is not None:
71
+ if self.kind is not Kind.CATEGORY:
72
+ raise ValueError("one_of is only for a category field")
73
+ object.__setattr__(self, "one_of", _texts("one_of", self.one_of))
74
+ if self.options is not None and not isinstance(self.options, CleanOptions):
75
+ raise TypeError("options must be a CleanOptions or None")
rowsift/map/_saved.py ADDED
@@ -0,0 +1,132 @@
1
+ """Saved mappings: keep a person's choices and apply them to the next file."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from collections.abc import Mapping as MappingABC
6
+ from collections.abc import Sequence
7
+ from dataclasses import dataclass, field
8
+ from types import MappingProxyType
9
+ from typing import Any
10
+
11
+ from rowsift.map._fields import Field
12
+ from rowsift.map._suggest import check_fields, fingerprint_of
13
+ from rowsift.map._types import Choice, Mapping
14
+ from rowsift.profile import Profile
15
+ from rowsift.types import Confidence, Reason, ReasonCode
16
+
17
+ __all__ = ["SavedMapping", "apply_saved"]
18
+
19
+
20
+ @dataclass(frozen=True, slots=True)
21
+ class SavedMapping:
22
+ """A mapping reduced to what survives between files: header names.
23
+
24
+ Attributes:
25
+ fingerprint: the header fingerprint of the file it was saved from.
26
+ assignments: for each field key, the normalized header of the column it
27
+ was mapped to. A field with no column is not listed. Copied into a
28
+ read-only mapping.
29
+
30
+ Raises:
31
+ TypeError: if the fingerprint is not ``str`` or an assignment is not a
32
+ ``str`` to ``str`` pair.
33
+ ValueError: if two fields point at the same header.
34
+ """
35
+
36
+ fingerprint: str
37
+ assignments: MappingABC[str, str] = field(default_factory=dict)
38
+
39
+ def __post_init__(self) -> None:
40
+ if type(self.fingerprint) is not str:
41
+ raise TypeError("fingerprint must be str")
42
+ copied = dict(self.assignments)
43
+ for key, header in copied.items():
44
+ if type(key) is not str or type(header) is not str:
45
+ raise TypeError("assignments must map str to str")
46
+ if len(set(copied.values())) != len(copied):
47
+ raise ValueError("two fields are saved on the same header")
48
+ object.__setattr__(self, "assignments", MappingProxyType(copied))
49
+
50
+ def __hash__(self) -> int:
51
+ return hash((self.fingerprint, frozenset(self.assignments.items())))
52
+
53
+ def __reduce__(self) -> tuple[Any, tuple[Any, ...]]:
54
+ return (type(self), (self.fingerprint, dict(self.assignments)))
55
+
56
+ @classmethod
57
+ def from_mapping(cls, mapping: Mapping, profile: Profile) -> SavedMapping:
58
+ """Save the assigned columns of ``mapping`` by their normalized header.
59
+
60
+ Raises:
61
+ ValueError: if the fingerprint of ``profile`` differs from the one of
62
+ ``mapping``, that is, the mapping was not made from this profile.
63
+ """
64
+ if fingerprint_of(profile) != mapping.fingerprint:
65
+ raise ValueError("the mapping was not made from this profile: fingerprints differ")
66
+ headers = {column.index: column.normalized_header for column in profile.columns}
67
+ assignments: dict[str, str] = {}
68
+ for choice in mapping.choices:
69
+ if choice.column is None:
70
+ continue
71
+ assignments[choice.field] = headers[choice.column]
72
+ return cls(mapping.fingerprint, assignments)
73
+
74
+ def to_dict(self) -> dict[str, Any]:
75
+ """A plain dictionary of text, ready for JSON: ``fingerprint`` and ``assignments``."""
76
+ return {"fingerprint": self.fingerprint, "assignments": dict(self.assignments)}
77
+
78
+ @classmethod
79
+ def from_dict(cls, data: MappingABC[str, Any]) -> SavedMapping:
80
+ """Read what ``to_dict`` wrote.
81
+
82
+ Raises:
83
+ ValueError: if a key is missing, or two fields share a header.
84
+ TypeError: if a value is not of the expected type.
85
+ """
86
+ try:
87
+ fingerprint = data["fingerprint"]
88
+ assignments = data["assignments"]
89
+ except KeyError as error:
90
+ raise ValueError(f"a saved mapping needs the key {error.args[0]!r}") from None
91
+ if not isinstance(assignments, MappingABC):
92
+ raise TypeError("assignments must be a mapping")
93
+ return cls(fingerprint, dict(assignments))
94
+
95
+
96
+ def apply_saved(profile: Profile, fields: Sequence[Field], saved: SavedMapping) -> Mapping:
97
+ """Restore a saved mapping onto a new profile, without guessing.
98
+
99
+ When the fingerprint of the profile equals the saved one, every assignment is
100
+ restored with confidence ``high`` and the reason ``saved_mapping``. When it
101
+ differs, an assignment is restored, with confidence ``medium`` and the same
102
+ reason, only if a column still has the saved normalized header. The others
103
+ come back with no column and the reason ``saved_header_missing``, carrying the
104
+ saved header. A renamed header is never matched by similarity.
105
+
106
+ A field the saved mapping does not list comes back with no column, and the
107
+ reason ``required_field_unmatched`` when it is required. An optional field
108
+ the saved mapping does not list has no reasons, as in ``suggest``. A saved key that no
109
+ field declares is ignored.
110
+
111
+ Raises:
112
+ ValueError: if two fields have the same key.
113
+ """
114
+ check_fields(fields)
115
+ same_layout = fingerprint_of(profile) == saved.fingerprint
116
+ by_header = {column.normalized_header: column.index for column in profile.columns}
117
+ confidence = Confidence.HIGH if same_layout else Confidence.MEDIUM
118
+ choices: list[Choice] = []
119
+ for field_ in fields:
120
+ header = saved.assignments.get(field_.key)
121
+ if header is None:
122
+ reasons = (Reason(ReasonCode.REQUIRED_FIELD_UNMATCHED),) if field_.required else ()
123
+ choices.append(Choice(field_.key, None, None, reasons))
124
+ elif header in by_header:
125
+ reason = Reason(ReasonCode.SAVED_MAPPING)
126
+ choices.append(Choice(field_.key, by_header[header], confidence, (reason,)))
127
+ else:
128
+ reason = Reason(ReasonCode.SAVED_HEADER_MISSING, {"header": header})
129
+ choices.append(Choice(field_.key, None, None, (reason,)))
130
+ used = {choice.column for choice in choices if choice.column is not None}
131
+ unmapped = tuple(column.index for column in profile.columns if column.index not in used)
132
+ return Mapping(tuple(choices), unmapped, fingerprint_of(profile))
rowsift/map/_score.py ADDED
@@ -0,0 +1,178 @@
1
+ """Scoring one column against one field, from the header and from the content."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from dataclasses import dataclass
6
+ from enum import Enum, auto
7
+
8
+ from rowsift.map._fields import Field
9
+ from rowsift.profile import ColumnProfile, normalize_header
10
+ from rowsift.types import Confidence, Kind, Reason, ReasonCode
11
+
12
+ __all__ = ["Candidate", "HeaderMatch", "alias_table", "score_column"]
13
+
14
+ SPECIFIC_KINDS = frozenset(
15
+ {
16
+ Kind.PHONE,
17
+ Kind.EMAIL,
18
+ Kind.CPF,
19
+ Kind.CNPJ,
20
+ Kind.POSTAL_CODE,
21
+ Kind.DATE,
22
+ Kind.DAY_MONTH,
23
+ Kind.TIME,
24
+ Kind.DATETIME,
25
+ }
26
+ )
27
+ """Kinds a column can only take by what its values look like. A column of one of
28
+ these kinds relates to a field of the same kind even when the header says nothing."""
29
+
30
+ ACCEPTS: dict[Kind, frozenset[Kind]] = {
31
+ Kind.TEXT: frozenset(Kind) - {Kind.TEXT},
32
+ Kind.NAME: frozenset({Kind.TEXT, Kind.CATEGORY}),
33
+ Kind.CATEGORY: frozenset({Kind.TEXT, Kind.NAME, Kind.INTEGER, Kind.BOOLEAN}),
34
+ Kind.INTEGER: frozenset({Kind.BOOLEAN}),
35
+ Kind.DECIMAL: frozenset({Kind.INTEGER}),
36
+ Kind.MONEY: frozenset({Kind.DECIMAL, Kind.INTEGER}),
37
+ Kind.DATE: frozenset({Kind.DATETIME}),
38
+ Kind.DATETIME: frozenset({Kind.DATE}),
39
+ Kind.DAY_MONTH: frozenset({Kind.DATE}),
40
+ }
41
+ """For a field kind, the other column kinds whose values the cleaner can still read.
42
+ A text field reads every kind, since any cell is text."""
43
+
44
+ _EXACT_POINTS = 4
45
+ _CONTAINS_POINTS = 2
46
+ _FIT_EXACT_POINTS = 2
47
+ _FIT_ACCEPTED_POINTS = 1
48
+ _CONFLICT_POINTS = -1
49
+
50
+
51
+ class HeaderMatch(Enum):
52
+ """How a header relates to the aliases of a field."""
53
+
54
+ NONE = auto()
55
+ CONTAINS = auto()
56
+ EXACT = auto()
57
+
58
+
59
+ class _Content(Enum):
60
+ NEUTRAL = auto()
61
+ FITS = auto()
62
+ CONFLICTS = auto()
63
+
64
+
65
+ @dataclass(frozen=True, slots=True)
66
+ class Candidate:
67
+ """One column as a candidate for one field.
68
+
69
+ Candidates are ordered by the higher score, then the larger share, then the
70
+ longer alias. ``confidence`` and ``reasons`` are what the choice carries if
71
+ the candidate wins.
72
+ """
73
+
74
+ column: int
75
+ score: int
76
+ share: float
77
+ alias_tokens: int
78
+ confidence: Confidence
79
+ reasons: tuple[Reason, ...]
80
+
81
+
82
+ def alias_table(field: Field) -> tuple[str, ...]:
83
+ """The folded aliases of a field: its declared aliases and its key."""
84
+ folded = (normalize_header(alias) for alias in (*field.aliases, field.key))
85
+ return tuple(dict.fromkeys(alias for alias in folded if alias))
86
+
87
+
88
+ def _header_match(column: ColumnProfile, aliases: tuple[str, ...]) -> tuple[HeaderMatch, str, int]:
89
+ """The best match of the header against the aliases: kind, alias, alias tokens."""
90
+ if not column.header or not column.normalized_header:
91
+ return HeaderMatch.NONE, "", 0
92
+ header = column.normalized_header
93
+ header_tokens = set(header.split())
94
+ best = (HeaderMatch.NONE, "", 0)
95
+ for alias in aliases:
96
+ tokens = alias.split()
97
+ if alias == header:
98
+ return HeaderMatch.EXACT, alias, len(tokens)
99
+ if set(tokens) <= header_tokens and len(tokens) > best[2]:
100
+ best = (HeaderMatch.CONTAINS, alias, len(tokens))
101
+ return best
102
+
103
+
104
+ def _content(column: ColumnProfile, field: Field) -> tuple[_Content, float, bool]:
105
+ """Whether the content fits the field: verdict, share, and whether the kind is equal."""
106
+ if column.filled == 0:
107
+ return _Content.NEUTRAL, 0.0, False
108
+ if column.kind is field.kind:
109
+ return _Content.FITS, column.kind_share, True
110
+ if column.kind in ACCEPTS.get(field.kind, frozenset()):
111
+ return _Content.FITS, column.kind_share, False
112
+ runner = column.runner_up
113
+ share = runner.share if runner is not None and runner.kind is field.kind else 0.0
114
+ return _Content.CONFLICTS, share, False
115
+
116
+
117
+ def score_column(
118
+ column: ColumnProfile,
119
+ field: Field,
120
+ aliases: tuple[str, ...],
121
+ *,
122
+ has_header: bool,
123
+ kind_is_unique: bool,
124
+ ) -> Candidate | None:
125
+ """Score a column for a field, or ``None`` when nothing relates them.
126
+
127
+ A header signal is an alias equal to the header (exact) or all of whose words
128
+ appear in it (contains). A content signal is the column kind fitting the
129
+ field kind. ``high`` needs an exact header and fitting content. With no
130
+ header signal, only a column whose kind is the field kind, and a kind that
131
+ values alone can reveal, is a candidate: ``medium`` when it is the only
132
+ column and the only field of that kind, ``low`` otherwise.
133
+
134
+ ``has_header`` is false for a profile with no header row, where headers carry
135
+ no meaning. ``kind_is_unique`` says whether the field is the only declared
136
+ field of its kind and the column the only column of that kind.
137
+ """
138
+ match, alias, alias_tokens = (
139
+ _header_match(column, aliases) if has_header else (HeaderMatch.NONE, "", 0)
140
+ )
141
+ content, share, exact_kind = _content(column, field)
142
+ fits = Reason(
143
+ ReasonCode.CONTENT_FITS_KIND, {"kind": str(column.kind), "share": round(share, 4)}
144
+ )
145
+ conflicts = Reason(
146
+ ReasonCode.CONTENT_CONFLICTS_WITH_KIND,
147
+ {
148
+ "column_kind": str(column.kind),
149
+ "field_kind": str(field.kind),
150
+ "share": round(share, 4),
151
+ },
152
+ )
153
+
154
+ if match is HeaderMatch.NONE:
155
+ if content is not _Content.FITS or not exact_kind or field.kind not in SPECIFIC_KINDS:
156
+ return None
157
+ confidence = Confidence.MEDIUM if kind_is_unique else Confidence.LOW
158
+ return Candidate(column.index, _FIT_EXACT_POINTS, share, 0, confidence, (fits,))
159
+
160
+ exact = match is HeaderMatch.EXACT
161
+ header_reason = Reason(
162
+ ReasonCode.HEADER_MATCHES_ALIAS if exact else ReasonCode.HEADER_CONTAINS_ALIAS,
163
+ {"alias": alias},
164
+ )
165
+ score = _EXACT_POINTS if exact else _CONTAINS_POINTS
166
+ if content is _Content.FITS:
167
+ score += _FIT_EXACT_POINTS if exact_kind else _FIT_ACCEPTED_POINTS
168
+ confidence = Confidence.HIGH if exact else Confidence.MEDIUM
169
+ return Candidate(
170
+ column.index, score, share, alias_tokens, confidence, (header_reason, fits)
171
+ )
172
+ if content is _Content.NEUTRAL:
173
+ confidence = Confidence.MEDIUM if exact else Confidence.LOW
174
+ return Candidate(column.index, score, 0.0, alias_tokens, confidence, (header_reason,))
175
+ score += _CONFLICT_POINTS
176
+ return Candidate(
177
+ column.index, score, share, alias_tokens, Confidence.LOW, (header_reason, conflicts)
178
+ )