rowsift 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rowsift/__init__.py +98 -0
- rowsift/clean/__init__.py +44 -0
- rowsift/clean/_common.py +30 -0
- rowsift/clean/_dates.py +283 -0
- rowsift/clean/_dispatch.py +49 -0
- rowsift/clean/_identifiers.py +161 -0
- rowsift/clean/_mask.py +117 -0
- rowsift/clean/_numbers.py +137 -0
- rowsift/clean/_options.py +75 -0
- rowsift/clean/_phone.py +202 -0
- rowsift/clean/_text.py +102 -0
- rowsift/errors.py +124 -0
- rowsift/map/__init__.py +24 -0
- rowsift/map/_fields.py +75 -0
- rowsift/map/_saved.py +132 -0
- rowsift/map/_score.py +178 -0
- rowsift/map/_suggest.py +168 -0
- rowsift/map/_types.py +169 -0
- rowsift/profile/__init__.py +17 -0
- rowsift/profile/_headers.py +44 -0
- rowsift/profile/_kinds.py +165 -0
- rowsift/profile/_options.py +69 -0
- rowsift/profile/_profile.py +119 -0
- rowsift/profile/_structure.py +157 -0
- rowsift/profile/_types.py +96 -0
- rowsift/py.typed +0 -0
- rowsift/read/__init__.py +6 -0
- rowsift/read/_cells.py +93 -0
- rowsift/read/_detect.py +124 -0
- rowsift/read/_grid.py +128 -0
- rowsift/read/_limits.py +44 -0
- rowsift/read/_read.py +99 -0
- rowsift/read/_text.py +250 -0
- rowsift/read/_workbook.py +36 -0
- rowsift/read/_xls.py +115 -0
- rowsift/read/_xlsx.py +121 -0
- rowsift/sift/__init__.py +19 -0
- rowsift/sift/_sift.py +165 -0
- rowsift/sift/_types.py +119 -0
- rowsift/types.py +378 -0
- rowsift-0.1.1.dist-info/METADATA +148 -0
- rowsift-0.1.1.dist-info/RECORD +45 -0
- rowsift-0.1.1.dist-info/WHEEL +4 -0
- rowsift-0.1.1.dist-info/licenses/LICENSE +177 -0
- rowsift-0.1.1.dist-info/licenses/NOTICE +4 -0
rowsift/read/_xlsx.py
ADDED
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
"""XLSX workbooks, read with the ``excel`` extra."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import io
|
|
6
|
+
import re
|
|
7
|
+
import zipfile
|
|
8
|
+
from collections.abc import Iterator
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
from rowsift.errors import LimitExceededError, MissingExtraError, RowsiftError, UnsupportedFileError
|
|
12
|
+
from rowsift.read._cells import cell_text
|
|
13
|
+
from rowsift.read._grid import Grid, RowCollector, fill_merged, select_sheets
|
|
14
|
+
from rowsift.read._limits import ReadLimits
|
|
15
|
+
from rowsift.read._workbook import WorkbookResult
|
|
16
|
+
|
|
17
|
+
__all__ = ["check_archive", "read_xlsx"]
|
|
18
|
+
|
|
19
|
+
# `mergeCell` elements sit after the cell data. The attribute holds a cell range.
|
|
20
|
+
_MERGE_CELL = re.compile(
|
|
21
|
+
rb'<(?:\w+:)?mergeCell\s[^>]*?\bref="([A-Za-z]+[0-9]+(?::[A-Za-z]+[0-9]+)?)"'
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def check_archive(data: bytes, limits: ReadLimits) -> None:
|
|
26
|
+
"""Refuse an archive whose declared uncompressed size is beyond the limit.
|
|
27
|
+
|
|
28
|
+
Only the central directory is read: no part is expanded. The declared size
|
|
29
|
+
also bounds what the ZIP reader gives back for a part, so an archive that
|
|
30
|
+
understates a size cannot expand past it.
|
|
31
|
+
|
|
32
|
+
Raises:
|
|
33
|
+
LimitExceededError: with ``max_decompressed_bytes``.
|
|
34
|
+
UnsupportedFileError: if the archive directory cannot be read.
|
|
35
|
+
"""
|
|
36
|
+
try:
|
|
37
|
+
with zipfile.ZipFile(io.BytesIO(data)) as archive:
|
|
38
|
+
declared = sum(info.file_size for info in archive.infolist())
|
|
39
|
+
except (zipfile.BadZipFile, OSError, ValueError, NotImplementedError):
|
|
40
|
+
raise UnsupportedFileError() from None
|
|
41
|
+
if declared > limits.max_decompressed_bytes:
|
|
42
|
+
raise LimitExceededError("max_decompressed_bytes", limits.max_decompressed_bytes)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _merged_areas(data: bytes, part: str) -> Iterator[tuple[int, int, int, int]]:
|
|
46
|
+
from openpyxl.utils.cell import range_boundaries
|
|
47
|
+
|
|
48
|
+
with zipfile.ZipFile(io.BytesIO(data)) as archive:
|
|
49
|
+
xml = archive.read(part.lstrip("/"))
|
|
50
|
+
for match in _MERGE_CELL.finditer(xml):
|
|
51
|
+
bounds = range_boundaries(match.group(1).decode("ascii"))
|
|
52
|
+
first_col, first_row, last_col, last_row = (int(bound or 1) for bound in bounds)
|
|
53
|
+
yield first_row - 1, first_col - 1, last_row - 1, last_col - 1
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _part_of(worksheet: Any) -> str:
|
|
57
|
+
# openpyxl has no public accessor for the archive part of a sheet. The
|
|
58
|
+
# attribute is covered by a test and the dependency is pinned to one major version.
|
|
59
|
+
part: str = worksheet._worksheet_path
|
|
60
|
+
return part
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _rows(worksheet: Any, limits: ReadLimits) -> Grid:
|
|
64
|
+
# The declared dimension is often wrong or reaches far past the data.
|
|
65
|
+
worksheet.reset_dimensions()
|
|
66
|
+
collector = RowCollector(limits, trim_trailing_rows=True)
|
|
67
|
+
for values in worksheet.iter_rows(values_only=True):
|
|
68
|
+
collector.add(cell_text(value) for value in values)
|
|
69
|
+
return collector.finish()
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def read_xlsx(data: bytes, sheet: str | int | None, limits: ReadLimits) -> WorkbookResult:
|
|
73
|
+
"""Read an XLSX workbook into a grid of cell text.
|
|
74
|
+
|
|
75
|
+
Formulas give the result the file stored and are never evaluated. Merged
|
|
76
|
+
areas repeat the value of their top-left cell, clipped to the data.
|
|
77
|
+
|
|
78
|
+
Raises:
|
|
79
|
+
LimitExceededError: if the archive, the rows, the columns or a cell go
|
|
80
|
+
beyond the limits.
|
|
81
|
+
MissingExtraError: if the ``excel`` extra is not installed.
|
|
82
|
+
UnsupportedFileError: if the workbook cannot be read, or the sheet does
|
|
83
|
+
not exist (``sheet_not_found``).
|
|
84
|
+
"""
|
|
85
|
+
check_archive(data, limits)
|
|
86
|
+
try:
|
|
87
|
+
from openpyxl import load_workbook
|
|
88
|
+
except ImportError:
|
|
89
|
+
raise MissingExtraError("excel") from None
|
|
90
|
+
try:
|
|
91
|
+
workbook = load_workbook(io.BytesIO(data), read_only=True, data_only=True, keep_links=False)
|
|
92
|
+
except (MemoryError, RecursionError):
|
|
93
|
+
raise
|
|
94
|
+
except Exception:
|
|
95
|
+
raise UnsupportedFileError() from None
|
|
96
|
+
try:
|
|
97
|
+
titles = [worksheet.title for worksheet in workbook.worksheets]
|
|
98
|
+
candidates = select_sheets(titles, sheet)
|
|
99
|
+
first_empty: Grid | None = None
|
|
100
|
+
for index in candidates:
|
|
101
|
+
worksheet = workbook[titles[index]]
|
|
102
|
+
try:
|
|
103
|
+
grid = _rows(worksheet, limits)
|
|
104
|
+
if not grid.rows:
|
|
105
|
+
first_empty = first_empty or grid
|
|
106
|
+
continue
|
|
107
|
+
changed = fill_merged(grid.rows, _merged_areas(data, _part_of(worksheet)))
|
|
108
|
+
except RowsiftError:
|
|
109
|
+
raise
|
|
110
|
+
except (MemoryError, RecursionError):
|
|
111
|
+
raise
|
|
112
|
+
except Exception:
|
|
113
|
+
raise UnsupportedFileError() from None
|
|
114
|
+
return WorkbookResult(grid, titles[index], tuple(titles), merged_filled=changed)
|
|
115
|
+
if first_empty is None:
|
|
116
|
+
return WorkbookResult(Grid([], False), None, tuple(titles), merged_filled=False)
|
|
117
|
+
return WorkbookResult(
|
|
118
|
+
first_empty, titles[candidates[0]], tuple(titles), merged_filled=False
|
|
119
|
+
)
|
|
120
|
+
finally:
|
|
121
|
+
workbook.close()
|
rowsift/sift/__init__.py
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"""Sift: apply a mapping to a table and get typed rows, rejections and counts.
|
|
2
|
+
|
|
3
|
+
``sift`` cleans every data row with the kind of each mapped field. It keeps the
|
|
4
|
+
rows whose required fields are good, lists the others as rejections with the
|
|
5
|
+
issue code, and counts every outcome. It has no side effect, so a caller can
|
|
6
|
+
run it as a preview and again as the final pass.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from rowsift.sift._sift import sift
|
|
10
|
+
from rowsift.sift._types import FieldSummary, Rejection, SiftedRow, SiftResult, Summary
|
|
11
|
+
|
|
12
|
+
__all__ = [
|
|
13
|
+
"FieldSummary",
|
|
14
|
+
"Rejection",
|
|
15
|
+
"SiftResult",
|
|
16
|
+
"SiftedRow",
|
|
17
|
+
"Summary",
|
|
18
|
+
"sift",
|
|
19
|
+
]
|
rowsift/sift/_sift.py
ADDED
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
"""Apply a mapping to a table."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections import Counter
|
|
6
|
+
from collections.abc import Sequence
|
|
7
|
+
from dataclasses import replace
|
|
8
|
+
|
|
9
|
+
from rowsift.clean import CleanOptions
|
|
10
|
+
from rowsift.clean import value as clean_value
|
|
11
|
+
from rowsift.map import Field, Mapping
|
|
12
|
+
from rowsift.profile import Profile
|
|
13
|
+
from rowsift.sift._types import FieldSummary, Rejection, SiftedRow, SiftResult, Summary
|
|
14
|
+
from rowsift.types import Change, Issue, IssueCode, Table
|
|
15
|
+
|
|
16
|
+
__all__ = ["sift"]
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def sift(
|
|
20
|
+
table: Table,
|
|
21
|
+
profile: Profile,
|
|
22
|
+
mapping: Mapping,
|
|
23
|
+
fields: Sequence[Field],
|
|
24
|
+
*,
|
|
25
|
+
options: CleanOptions | None = None,
|
|
26
|
+
include_extras: bool = True,
|
|
27
|
+
) -> SiftResult:
|
|
28
|
+
"""Clean the data rows of ``table`` into the declared fields.
|
|
29
|
+
|
|
30
|
+
Each data row is cleaned, field by field, with the kind of the field. A row
|
|
31
|
+
is rejected only when a required field is empty (``required_missing``) or
|
|
32
|
+
its value has an issue. An optional field that is empty or has an issue is
|
|
33
|
+
``None`` in the row, and the issue is recorded on the row. Rows the profile
|
|
34
|
+
skipped (rows above the header, empty rows, repeated headers, totals) are
|
|
35
|
+
not data and are not counted.
|
|
36
|
+
|
|
37
|
+
Nothing is modified and nothing is read beyond the arguments, so the same
|
|
38
|
+
arguments give an equal result.
|
|
39
|
+
|
|
40
|
+
Args:
|
|
41
|
+
table: the grid from ``read``.
|
|
42
|
+
profile: the profile of ``table``.
|
|
43
|
+
mapping: which column feeds each field, from ``suggest``, ``apply_saved``
|
|
44
|
+
or changed by the caller.
|
|
45
|
+
fields: the declared fields. Each must have a choice in ``mapping``.
|
|
46
|
+
options: the cleaning options of the call. A field's own ``options``
|
|
47
|
+
replace them, and a field's ``one_of`` replaces ``one_of`` in both.
|
|
48
|
+
``None`` means the defaults.
|
|
49
|
+
include_extras: return the text of the unmapped columns in each row.
|
|
50
|
+
|
|
51
|
+
Returns:
|
|
52
|
+
The kept rows, the rejections and the summary.
|
|
53
|
+
|
|
54
|
+
Raises:
|
|
55
|
+
ValueError: if two fields share a key, a field has no choice in the
|
|
56
|
+
mapping, the mapping has a choice for no field, a choice points to a
|
|
57
|
+
column the profile does not have, a required field has no column, or
|
|
58
|
+
the profile does not match the table (another column count or a data
|
|
59
|
+
start beyond the rows).
|
|
60
|
+
Raised before any row is processed.
|
|
61
|
+
"""
|
|
62
|
+
plan = _plan(profile, mapping, fields)
|
|
63
|
+
if len(profile.columns) != table.report.column_count or profile.data_start > len(table.rows):
|
|
64
|
+
raise ValueError("the profile does not match the table")
|
|
65
|
+
base = options if options is not None else CleanOptions()
|
|
66
|
+
per_field = {item.key: _options_of(item, base) for item in fields}
|
|
67
|
+
extras_columns = _extras_columns(profile, mapping) if include_extras else ()
|
|
68
|
+
|
|
69
|
+
rows: list[SiftedRow] = []
|
|
70
|
+
rejected: list[Rejection] = []
|
|
71
|
+
changes_count: dict[str, Counter[str]] = {item.key: Counter() for item in fields}
|
|
72
|
+
issues_count: dict[str, Counter[str]] = {item.key: Counter() for item in fields}
|
|
73
|
+
filled: Counter[str] = Counter()
|
|
74
|
+
data_rows = 0
|
|
75
|
+
rejected_rows = 0
|
|
76
|
+
|
|
77
|
+
for index in _data_indexes(table, profile):
|
|
78
|
+
data_rows += 1
|
|
79
|
+
cells = table.rows[index]
|
|
80
|
+
values: dict[str, object] = {}
|
|
81
|
+
changes: dict[str, tuple[Change, ...]] = {}
|
|
82
|
+
issues: dict[str, Issue] = {}
|
|
83
|
+
failures: list[Rejection] = []
|
|
84
|
+
for item, column in plan:
|
|
85
|
+
cleaned = clean_value(
|
|
86
|
+
cells[column] if column is not None else "", item.kind, per_field[item.key]
|
|
87
|
+
)
|
|
88
|
+
values[item.key] = cleaned.value
|
|
89
|
+
if cleaned.value is not None:
|
|
90
|
+
filled[item.key] += 1
|
|
91
|
+
if cleaned.changes:
|
|
92
|
+
changes[item.key] = cleaned.changes
|
|
93
|
+
changes_count[item.key].update(change.code for change in cleaned.changes)
|
|
94
|
+
issue = cleaned.issue
|
|
95
|
+
if issue is None and cleaned.value is None and item.required:
|
|
96
|
+
issue = Issue(IssueCode.REQUIRED_MISSING)
|
|
97
|
+
if issue is None:
|
|
98
|
+
continue
|
|
99
|
+
issues_count[item.key][issue.code] += 1
|
|
100
|
+
if item.required:
|
|
101
|
+
failures.append(Rejection(index + 1, item.key, issue))
|
|
102
|
+
else:
|
|
103
|
+
issues[item.key] = issue
|
|
104
|
+
if failures:
|
|
105
|
+
rejected.extend(failures)
|
|
106
|
+
rejected_rows += 1
|
|
107
|
+
continue
|
|
108
|
+
extras = {profile.columns[c].normalized_header: cells[c] for c in extras_columns}
|
|
109
|
+
rows.append(SiftedRow(index + 1, values, extras, changes, issues))
|
|
110
|
+
|
|
111
|
+
summary = Summary(
|
|
112
|
+
data_rows=data_rows,
|
|
113
|
+
kept=len(rows),
|
|
114
|
+
rejected=rejected_rows,
|
|
115
|
+
fields={
|
|
116
|
+
item.key: FieldSummary(
|
|
117
|
+
filled[item.key], changes_count[item.key], issues_count[item.key]
|
|
118
|
+
)
|
|
119
|
+
for item in fields
|
|
120
|
+
},
|
|
121
|
+
)
|
|
122
|
+
return SiftResult(tuple(rows), tuple(rejected), summary)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def _plan(
|
|
126
|
+
profile: Profile, mapping: Mapping, fields: Sequence[Field]
|
|
127
|
+
) -> list[tuple[Field, int | None]]:
|
|
128
|
+
keys = [item.key for item in fields]
|
|
129
|
+
if len(set(keys)) != len(keys):
|
|
130
|
+
raise ValueError("two fields share a key")
|
|
131
|
+
choices = {choice.field: choice for choice in mapping.choices}
|
|
132
|
+
for key in keys:
|
|
133
|
+
if key not in choices:
|
|
134
|
+
raise ValueError(f"the mapping has no choice for the field {key!r}")
|
|
135
|
+
for key in choices:
|
|
136
|
+
if key not in keys:
|
|
137
|
+
raise ValueError(f"the mapping has a choice for {key!r}, which is not a declared field")
|
|
138
|
+
plan: list[tuple[Field, int | None]] = []
|
|
139
|
+
for item in fields:
|
|
140
|
+
column = choices[item.key].column
|
|
141
|
+
if column is None and item.required:
|
|
142
|
+
raise ValueError(f"the required field {item.key!r} has no column")
|
|
143
|
+
if column is not None and column >= len(profile.columns):
|
|
144
|
+
raise ValueError(
|
|
145
|
+
f"the field {item.key!r} points to column {column}, which is not in the profile"
|
|
146
|
+
)
|
|
147
|
+
plan.append((item, column))
|
|
148
|
+
return plan
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _options_of(item: Field, base: CleanOptions) -> CleanOptions:
|
|
152
|
+
chosen = item.options if item.options is not None else base
|
|
153
|
+
if item.one_of is not None:
|
|
154
|
+
return replace(chosen, one_of=item.one_of)
|
|
155
|
+
return chosen
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def _extras_columns(profile: Profile, mapping: Mapping) -> tuple[int, ...]:
|
|
159
|
+
assigned = {choice.column for choice in mapping.choices if choice.column is not None}
|
|
160
|
+
return tuple(column.index for column in profile.columns if column.index not in assigned)
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def _data_indexes(table: Table, profile: Profile) -> list[int]:
|
|
164
|
+
skipped = {row.index for row in profile.skipped_rows}
|
|
165
|
+
return [i for i in range(profile.data_start, len(table.rows)) if i not in skipped]
|
rowsift/sift/_types.py
ADDED
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
"""Value types of the sifting layer."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Mapping
|
|
6
|
+
from dataclasses import dataclass, field
|
|
7
|
+
from types import MappingProxyType
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
from rowsift.types import Change, Issue
|
|
11
|
+
|
|
12
|
+
__all__ = ["FieldSummary", "Rejection", "SiftResult", "SiftedRow", "Summary"]
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _frozen(mapping: Mapping[Any, Any]) -> Mapping[Any, Any]:
|
|
16
|
+
return MappingProxyType(dict(mapping))
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@dataclass(frozen=True, slots=True)
|
|
20
|
+
class SiftedRow:
|
|
21
|
+
"""A data row that was kept, with its fields cleaned.
|
|
22
|
+
|
|
23
|
+
Attributes:
|
|
24
|
+
number: the 1-based position of the row in the file, counting the rows
|
|
25
|
+
above the header. The first row of the table is ``1``.
|
|
26
|
+
values: the cleaned value of every declared field, keyed by field key.
|
|
27
|
+
A field with no column, an empty cell or an optional value that
|
|
28
|
+
failed holds ``None``.
|
|
29
|
+
extras: the text of every column no field uses, keyed by normalized
|
|
30
|
+
header. Empty when the caller asked for no extras.
|
|
31
|
+
changes: what the cleaners altered, keyed by field key. Only fields with
|
|
32
|
+
at least one change appear.
|
|
33
|
+
issues: why an optional field became ``None``, keyed by field key. Only
|
|
34
|
+
fields that failed appear.
|
|
35
|
+
"""
|
|
36
|
+
|
|
37
|
+
number: int
|
|
38
|
+
values: Mapping[str, object]
|
|
39
|
+
extras: Mapping[str, str] = field(default_factory=dict)
|
|
40
|
+
changes: Mapping[str, tuple[Change, ...]] = field(default_factory=dict)
|
|
41
|
+
issues: Mapping[str, Issue] = field(default_factory=dict)
|
|
42
|
+
|
|
43
|
+
def __post_init__(self) -> None:
|
|
44
|
+
for name in ("values", "extras", "changes", "issues"):
|
|
45
|
+
object.__setattr__(self, name, _frozen(getattr(self, name)))
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@dataclass(frozen=True, slots=True)
|
|
49
|
+
class Rejection:
|
|
50
|
+
"""Why a row was left out: a required field that failed.
|
|
51
|
+
|
|
52
|
+
A row with several failing required fields has one rejection per field.
|
|
53
|
+
|
|
54
|
+
Attributes:
|
|
55
|
+
number: the 1-based position of the row in the file.
|
|
56
|
+
field: the key of the required field.
|
|
57
|
+
issue: the issue code and its data. ``required_missing`` when the cell
|
|
58
|
+
was empty.
|
|
59
|
+
"""
|
|
60
|
+
|
|
61
|
+
number: int
|
|
62
|
+
field: str
|
|
63
|
+
issue: Issue
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
@dataclass(frozen=True, slots=True)
|
|
67
|
+
class FieldSummary:
|
|
68
|
+
"""The outcomes of one field over every data row, kept or rejected.
|
|
69
|
+
|
|
70
|
+
Attributes:
|
|
71
|
+
filled: how many rows got a value for the field.
|
|
72
|
+
changes: how many times each change code was reported, keyed by the
|
|
73
|
+
code (a ``ChangeCode`` or its string).
|
|
74
|
+
issues: how many rows failed with each issue code, keyed likewise.
|
|
75
|
+
"""
|
|
76
|
+
|
|
77
|
+
filled: int = 0
|
|
78
|
+
changes: Mapping[str, int] = field(default_factory=dict)
|
|
79
|
+
issues: Mapping[str, int] = field(default_factory=dict)
|
|
80
|
+
|
|
81
|
+
def __post_init__(self) -> None:
|
|
82
|
+
object.__setattr__(self, "changes", _frozen(self.changes))
|
|
83
|
+
object.__setattr__(self, "issues", _frozen(self.issues))
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
@dataclass(frozen=True, slots=True)
|
|
87
|
+
class Summary:
|
|
88
|
+
"""Counts of every outcome of a sift.
|
|
89
|
+
|
|
90
|
+
Attributes:
|
|
91
|
+
data_rows: the data rows that were cleaned.
|
|
92
|
+
kept: the rows in ``SiftResult.rows``.
|
|
93
|
+
rejected: the rows left out. A row with several failing required fields
|
|
94
|
+
counts once.
|
|
95
|
+
fields: one summary per declared field, in declaration order.
|
|
96
|
+
"""
|
|
97
|
+
|
|
98
|
+
data_rows: int
|
|
99
|
+
kept: int
|
|
100
|
+
rejected: int
|
|
101
|
+
fields: Mapping[str, FieldSummary]
|
|
102
|
+
|
|
103
|
+
def __post_init__(self) -> None:
|
|
104
|
+
object.__setattr__(self, "fields", _frozen(self.fields))
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
@dataclass(frozen=True, slots=True)
|
|
108
|
+
class SiftResult:
|
|
109
|
+
"""The outcome of applying a mapping to a table.
|
|
110
|
+
|
|
111
|
+
Attributes:
|
|
112
|
+
rows: the kept rows, in file order.
|
|
113
|
+
rejected: the rejections, in file order and then field order.
|
|
114
|
+
summary: the counts.
|
|
115
|
+
"""
|
|
116
|
+
|
|
117
|
+
rows: tuple[SiftedRow, ...]
|
|
118
|
+
rejected: tuple[Rejection, ...]
|
|
119
|
+
summary: Summary
|