rowsift 0.1.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
rowsift/read/_xlsx.py ADDED
@@ -0,0 +1,121 @@
1
+ """XLSX workbooks, read with the ``excel`` extra."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import io
6
+ import re
7
+ import zipfile
8
+ from collections.abc import Iterator
9
+ from typing import Any
10
+
11
+ from rowsift.errors import LimitExceededError, MissingExtraError, RowsiftError, UnsupportedFileError
12
+ from rowsift.read._cells import cell_text
13
+ from rowsift.read._grid import Grid, RowCollector, fill_merged, select_sheets
14
+ from rowsift.read._limits import ReadLimits
15
+ from rowsift.read._workbook import WorkbookResult
16
+
17
+ __all__ = ["check_archive", "read_xlsx"]
18
+
19
+ # `mergeCell` elements sit after the cell data. The attribute holds a cell range.
20
+ _MERGE_CELL = re.compile(
21
+ rb'<(?:\w+:)?mergeCell\s[^>]*?\bref="([A-Za-z]+[0-9]+(?::[A-Za-z]+[0-9]+)?)"'
22
+ )
23
+
24
+
25
+ def check_archive(data: bytes, limits: ReadLimits) -> None:
26
+ """Refuse an archive whose declared uncompressed size is beyond the limit.
27
+
28
+ Only the central directory is read: no part is expanded. The declared size
29
+ also bounds what the ZIP reader gives back for a part, so an archive that
30
+ understates a size cannot expand past it.
31
+
32
+ Raises:
33
+ LimitExceededError: with ``max_decompressed_bytes``.
34
+ UnsupportedFileError: if the archive directory cannot be read.
35
+ """
36
+ try:
37
+ with zipfile.ZipFile(io.BytesIO(data)) as archive:
38
+ declared = sum(info.file_size for info in archive.infolist())
39
+ except (zipfile.BadZipFile, OSError, ValueError, NotImplementedError):
40
+ raise UnsupportedFileError() from None
41
+ if declared > limits.max_decompressed_bytes:
42
+ raise LimitExceededError("max_decompressed_bytes", limits.max_decompressed_bytes)
43
+
44
+
45
+ def _merged_areas(data: bytes, part: str) -> Iterator[tuple[int, int, int, int]]:
46
+ from openpyxl.utils.cell import range_boundaries
47
+
48
+ with zipfile.ZipFile(io.BytesIO(data)) as archive:
49
+ xml = archive.read(part.lstrip("/"))
50
+ for match in _MERGE_CELL.finditer(xml):
51
+ bounds = range_boundaries(match.group(1).decode("ascii"))
52
+ first_col, first_row, last_col, last_row = (int(bound or 1) for bound in bounds)
53
+ yield first_row - 1, first_col - 1, last_row - 1, last_col - 1
54
+
55
+
56
+ def _part_of(worksheet: Any) -> str:
57
+ # openpyxl has no public accessor for the archive part of a sheet. The
58
+ # attribute is covered by a test and the dependency is pinned to one major version.
59
+ part: str = worksheet._worksheet_path
60
+ return part
61
+
62
+
63
+ def _rows(worksheet: Any, limits: ReadLimits) -> Grid:
64
+ # The declared dimension is often wrong or reaches far past the data.
65
+ worksheet.reset_dimensions()
66
+ collector = RowCollector(limits, trim_trailing_rows=True)
67
+ for values in worksheet.iter_rows(values_only=True):
68
+ collector.add(cell_text(value) for value in values)
69
+ return collector.finish()
70
+
71
+
72
+ def read_xlsx(data: bytes, sheet: str | int | None, limits: ReadLimits) -> WorkbookResult:
73
+ """Read an XLSX workbook into a grid of cell text.
74
+
75
+ Formulas give the result the file stored and are never evaluated. Merged
76
+ areas repeat the value of their top-left cell, clipped to the data.
77
+
78
+ Raises:
79
+ LimitExceededError: if the archive, the rows, the columns or a cell go
80
+ beyond the limits.
81
+ MissingExtraError: if the ``excel`` extra is not installed.
82
+ UnsupportedFileError: if the workbook cannot be read, or the sheet does
83
+ not exist (``sheet_not_found``).
84
+ """
85
+ check_archive(data, limits)
86
+ try:
87
+ from openpyxl import load_workbook
88
+ except ImportError:
89
+ raise MissingExtraError("excel") from None
90
+ try:
91
+ workbook = load_workbook(io.BytesIO(data), read_only=True, data_only=True, keep_links=False)
92
+ except (MemoryError, RecursionError):
93
+ raise
94
+ except Exception:
95
+ raise UnsupportedFileError() from None
96
+ try:
97
+ titles = [worksheet.title for worksheet in workbook.worksheets]
98
+ candidates = select_sheets(titles, sheet)
99
+ first_empty: Grid | None = None
100
+ for index in candidates:
101
+ worksheet = workbook[titles[index]]
102
+ try:
103
+ grid = _rows(worksheet, limits)
104
+ if not grid.rows:
105
+ first_empty = first_empty or grid
106
+ continue
107
+ changed = fill_merged(grid.rows, _merged_areas(data, _part_of(worksheet)))
108
+ except RowsiftError:
109
+ raise
110
+ except (MemoryError, RecursionError):
111
+ raise
112
+ except Exception:
113
+ raise UnsupportedFileError() from None
114
+ return WorkbookResult(grid, titles[index], tuple(titles), merged_filled=changed)
115
+ if first_empty is None:
116
+ return WorkbookResult(Grid([], False), None, tuple(titles), merged_filled=False)
117
+ return WorkbookResult(
118
+ first_empty, titles[candidates[0]], tuple(titles), merged_filled=False
119
+ )
120
+ finally:
121
+ workbook.close()
@@ -0,0 +1,19 @@
1
+ """Sift: apply a mapping to a table and get typed rows, rejections and counts.
2
+
3
+ ``sift`` cleans every data row with the kind of each mapped field. It keeps the
4
+ rows whose required fields are good, lists the others as rejections with the
5
+ issue code, and counts every outcome. It has no side effect, so a caller can
6
+ run it as a preview and again as the final pass.
7
+ """
8
+
9
+ from rowsift.sift._sift import sift
10
+ from rowsift.sift._types import FieldSummary, Rejection, SiftedRow, SiftResult, Summary
11
+
12
+ __all__ = [
13
+ "FieldSummary",
14
+ "Rejection",
15
+ "SiftResult",
16
+ "SiftedRow",
17
+ "Summary",
18
+ "sift",
19
+ ]
rowsift/sift/_sift.py ADDED
@@ -0,0 +1,165 @@
1
+ """Apply a mapping to a table."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from collections import Counter
6
+ from collections.abc import Sequence
7
+ from dataclasses import replace
8
+
9
+ from rowsift.clean import CleanOptions
10
+ from rowsift.clean import value as clean_value
11
+ from rowsift.map import Field, Mapping
12
+ from rowsift.profile import Profile
13
+ from rowsift.sift._types import FieldSummary, Rejection, SiftedRow, SiftResult, Summary
14
+ from rowsift.types import Change, Issue, IssueCode, Table
15
+
16
+ __all__ = ["sift"]
17
+
18
+
19
+ def sift(
20
+ table: Table,
21
+ profile: Profile,
22
+ mapping: Mapping,
23
+ fields: Sequence[Field],
24
+ *,
25
+ options: CleanOptions | None = None,
26
+ include_extras: bool = True,
27
+ ) -> SiftResult:
28
+ """Clean the data rows of ``table`` into the declared fields.
29
+
30
+ Each data row is cleaned, field by field, with the kind of the field. A row
31
+ is rejected only when a required field is empty (``required_missing``) or
32
+ its value has an issue. An optional field that is empty or has an issue is
33
+ ``None`` in the row, and the issue is recorded on the row. Rows the profile
34
+ skipped (rows above the header, empty rows, repeated headers, totals) are
35
+ not data and are not counted.
36
+
37
+ Nothing is modified and nothing is read beyond the arguments, so the same
38
+ arguments give an equal result.
39
+
40
+ Args:
41
+ table: the grid from ``read``.
42
+ profile: the profile of ``table``.
43
+ mapping: which column feeds each field, from ``suggest``, ``apply_saved``
44
+ or changed by the caller.
45
+ fields: the declared fields. Each must have a choice in ``mapping``.
46
+ options: the cleaning options of the call. A field's own ``options``
47
+ replace them, and a field's ``one_of`` replaces ``one_of`` in both.
48
+ ``None`` means the defaults.
49
+ include_extras: return the text of the unmapped columns in each row.
50
+
51
+ Returns:
52
+ The kept rows, the rejections and the summary.
53
+
54
+ Raises:
55
+ ValueError: if two fields share a key, a field has no choice in the
56
+ mapping, the mapping has a choice for no field, a choice points to a
57
+ column the profile does not have, a required field has no column, or
58
+ the profile does not match the table (another column count or a data
59
+ start beyond the rows).
60
+ Raised before any row is processed.
61
+ """
62
+ plan = _plan(profile, mapping, fields)
63
+ if len(profile.columns) != table.report.column_count or profile.data_start > len(table.rows):
64
+ raise ValueError("the profile does not match the table")
65
+ base = options if options is not None else CleanOptions()
66
+ per_field = {item.key: _options_of(item, base) for item in fields}
67
+ extras_columns = _extras_columns(profile, mapping) if include_extras else ()
68
+
69
+ rows: list[SiftedRow] = []
70
+ rejected: list[Rejection] = []
71
+ changes_count: dict[str, Counter[str]] = {item.key: Counter() for item in fields}
72
+ issues_count: dict[str, Counter[str]] = {item.key: Counter() for item in fields}
73
+ filled: Counter[str] = Counter()
74
+ data_rows = 0
75
+ rejected_rows = 0
76
+
77
+ for index in _data_indexes(table, profile):
78
+ data_rows += 1
79
+ cells = table.rows[index]
80
+ values: dict[str, object] = {}
81
+ changes: dict[str, tuple[Change, ...]] = {}
82
+ issues: dict[str, Issue] = {}
83
+ failures: list[Rejection] = []
84
+ for item, column in plan:
85
+ cleaned = clean_value(
86
+ cells[column] if column is not None else "", item.kind, per_field[item.key]
87
+ )
88
+ values[item.key] = cleaned.value
89
+ if cleaned.value is not None:
90
+ filled[item.key] += 1
91
+ if cleaned.changes:
92
+ changes[item.key] = cleaned.changes
93
+ changes_count[item.key].update(change.code for change in cleaned.changes)
94
+ issue = cleaned.issue
95
+ if issue is None and cleaned.value is None and item.required:
96
+ issue = Issue(IssueCode.REQUIRED_MISSING)
97
+ if issue is None:
98
+ continue
99
+ issues_count[item.key][issue.code] += 1
100
+ if item.required:
101
+ failures.append(Rejection(index + 1, item.key, issue))
102
+ else:
103
+ issues[item.key] = issue
104
+ if failures:
105
+ rejected.extend(failures)
106
+ rejected_rows += 1
107
+ continue
108
+ extras = {profile.columns[c].normalized_header: cells[c] for c in extras_columns}
109
+ rows.append(SiftedRow(index + 1, values, extras, changes, issues))
110
+
111
+ summary = Summary(
112
+ data_rows=data_rows,
113
+ kept=len(rows),
114
+ rejected=rejected_rows,
115
+ fields={
116
+ item.key: FieldSummary(
117
+ filled[item.key], changes_count[item.key], issues_count[item.key]
118
+ )
119
+ for item in fields
120
+ },
121
+ )
122
+ return SiftResult(tuple(rows), tuple(rejected), summary)
123
+
124
+
125
+ def _plan(
126
+ profile: Profile, mapping: Mapping, fields: Sequence[Field]
127
+ ) -> list[tuple[Field, int | None]]:
128
+ keys = [item.key for item in fields]
129
+ if len(set(keys)) != len(keys):
130
+ raise ValueError("two fields share a key")
131
+ choices = {choice.field: choice for choice in mapping.choices}
132
+ for key in keys:
133
+ if key not in choices:
134
+ raise ValueError(f"the mapping has no choice for the field {key!r}")
135
+ for key in choices:
136
+ if key not in keys:
137
+ raise ValueError(f"the mapping has a choice for {key!r}, which is not a declared field")
138
+ plan: list[tuple[Field, int | None]] = []
139
+ for item in fields:
140
+ column = choices[item.key].column
141
+ if column is None and item.required:
142
+ raise ValueError(f"the required field {item.key!r} has no column")
143
+ if column is not None and column >= len(profile.columns):
144
+ raise ValueError(
145
+ f"the field {item.key!r} points to column {column}, which is not in the profile"
146
+ )
147
+ plan.append((item, column))
148
+ return plan
149
+
150
+
151
+ def _options_of(item: Field, base: CleanOptions) -> CleanOptions:
152
+ chosen = item.options if item.options is not None else base
153
+ if item.one_of is not None:
154
+ return replace(chosen, one_of=item.one_of)
155
+ return chosen
156
+
157
+
158
+ def _extras_columns(profile: Profile, mapping: Mapping) -> tuple[int, ...]:
159
+ assigned = {choice.column for choice in mapping.choices if choice.column is not None}
160
+ return tuple(column.index for column in profile.columns if column.index not in assigned)
161
+
162
+
163
+ def _data_indexes(table: Table, profile: Profile) -> list[int]:
164
+ skipped = {row.index for row in profile.skipped_rows}
165
+ return [i for i in range(profile.data_start, len(table.rows)) if i not in skipped]
rowsift/sift/_types.py ADDED
@@ -0,0 +1,119 @@
1
+ """Value types of the sifting layer."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from collections.abc import Mapping
6
+ from dataclasses import dataclass, field
7
+ from types import MappingProxyType
8
+ from typing import Any
9
+
10
+ from rowsift.types import Change, Issue
11
+
12
+ __all__ = ["FieldSummary", "Rejection", "SiftResult", "SiftedRow", "Summary"]
13
+
14
+
15
+ def _frozen(mapping: Mapping[Any, Any]) -> Mapping[Any, Any]:
16
+ return MappingProxyType(dict(mapping))
17
+
18
+
19
+ @dataclass(frozen=True, slots=True)
20
+ class SiftedRow:
21
+ """A data row that was kept, with its fields cleaned.
22
+
23
+ Attributes:
24
+ number: the 1-based position of the row in the file, counting the rows
25
+ above the header. The first row of the table is ``1``.
26
+ values: the cleaned value of every declared field, keyed by field key.
27
+ A field with no column, an empty cell or an optional value that
28
+ failed holds ``None``.
29
+ extras: the text of every column no field uses, keyed by normalized
30
+ header. Empty when the caller asked for no extras.
31
+ changes: what the cleaners altered, keyed by field key. Only fields with
32
+ at least one change appear.
33
+ issues: why an optional field became ``None``, keyed by field key. Only
34
+ fields that failed appear.
35
+ """
36
+
37
+ number: int
38
+ values: Mapping[str, object]
39
+ extras: Mapping[str, str] = field(default_factory=dict)
40
+ changes: Mapping[str, tuple[Change, ...]] = field(default_factory=dict)
41
+ issues: Mapping[str, Issue] = field(default_factory=dict)
42
+
43
+ def __post_init__(self) -> None:
44
+ for name in ("values", "extras", "changes", "issues"):
45
+ object.__setattr__(self, name, _frozen(getattr(self, name)))
46
+
47
+
48
+ @dataclass(frozen=True, slots=True)
49
+ class Rejection:
50
+ """Why a row was left out: a required field that failed.
51
+
52
+ A row with several failing required fields has one rejection per field.
53
+
54
+ Attributes:
55
+ number: the 1-based position of the row in the file.
56
+ field: the key of the required field.
57
+ issue: the issue code and its data. ``required_missing`` when the cell
58
+ was empty.
59
+ """
60
+
61
+ number: int
62
+ field: str
63
+ issue: Issue
64
+
65
+
66
+ @dataclass(frozen=True, slots=True)
67
+ class FieldSummary:
68
+ """The outcomes of one field over every data row, kept or rejected.
69
+
70
+ Attributes:
71
+ filled: how many rows got a value for the field.
72
+ changes: how many times each change code was reported, keyed by the
73
+ code (a ``ChangeCode`` or its string).
74
+ issues: how many rows failed with each issue code, keyed likewise.
75
+ """
76
+
77
+ filled: int = 0
78
+ changes: Mapping[str, int] = field(default_factory=dict)
79
+ issues: Mapping[str, int] = field(default_factory=dict)
80
+
81
+ def __post_init__(self) -> None:
82
+ object.__setattr__(self, "changes", _frozen(self.changes))
83
+ object.__setattr__(self, "issues", _frozen(self.issues))
84
+
85
+
86
+ @dataclass(frozen=True, slots=True)
87
+ class Summary:
88
+ """Counts of every outcome of a sift.
89
+
90
+ Attributes:
91
+ data_rows: the data rows that were cleaned.
92
+ kept: the rows in ``SiftResult.rows``.
93
+ rejected: the rows left out. A row with several failing required fields
94
+ counts once.
95
+ fields: one summary per declared field, in declaration order.
96
+ """
97
+
98
+ data_rows: int
99
+ kept: int
100
+ rejected: int
101
+ fields: Mapping[str, FieldSummary]
102
+
103
+ def __post_init__(self) -> None:
104
+ object.__setattr__(self, "fields", _frozen(self.fields))
105
+
106
+
107
+ @dataclass(frozen=True, slots=True)
108
+ class SiftResult:
109
+ """The outcome of applying a mapping to a table.
110
+
111
+ Attributes:
112
+ rows: the kept rows, in file order.
113
+ rejected: the rejections, in file order and then field order.
114
+ summary: the counts.
115
+ """
116
+
117
+ rows: tuple[SiftedRow, ...]
118
+ rejected: tuple[Rejection, ...]
119
+ summary: Summary