rowsift 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rowsift/__init__.py +98 -0
- rowsift/clean/__init__.py +44 -0
- rowsift/clean/_common.py +30 -0
- rowsift/clean/_dates.py +283 -0
- rowsift/clean/_dispatch.py +49 -0
- rowsift/clean/_identifiers.py +161 -0
- rowsift/clean/_mask.py +117 -0
- rowsift/clean/_numbers.py +137 -0
- rowsift/clean/_options.py +75 -0
- rowsift/clean/_phone.py +202 -0
- rowsift/clean/_text.py +102 -0
- rowsift/errors.py +124 -0
- rowsift/map/__init__.py +24 -0
- rowsift/map/_fields.py +75 -0
- rowsift/map/_saved.py +132 -0
- rowsift/map/_score.py +178 -0
- rowsift/map/_suggest.py +168 -0
- rowsift/map/_types.py +169 -0
- rowsift/profile/__init__.py +17 -0
- rowsift/profile/_headers.py +44 -0
- rowsift/profile/_kinds.py +165 -0
- rowsift/profile/_options.py +69 -0
- rowsift/profile/_profile.py +119 -0
- rowsift/profile/_structure.py +157 -0
- rowsift/profile/_types.py +96 -0
- rowsift/py.typed +0 -0
- rowsift/read/__init__.py +6 -0
- rowsift/read/_cells.py +93 -0
- rowsift/read/_detect.py +124 -0
- rowsift/read/_grid.py +128 -0
- rowsift/read/_limits.py +44 -0
- rowsift/read/_read.py +99 -0
- rowsift/read/_text.py +250 -0
- rowsift/read/_workbook.py +36 -0
- rowsift/read/_xls.py +115 -0
- rowsift/read/_xlsx.py +121 -0
- rowsift/sift/__init__.py +19 -0
- rowsift/sift/_sift.py +165 -0
- rowsift/sift/_types.py +119 -0
- rowsift/types.py +378 -0
- rowsift-0.1.1.dist-info/METADATA +148 -0
- rowsift-0.1.1.dist-info/RECORD +45 -0
- rowsift-0.1.1.dist-info/WHEEL +4 -0
- rowsift-0.1.1.dist-info/licenses/LICENSE +177 -0
- rowsift-0.1.1.dist-info/licenses/NOTICE +4 -0
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
"""The entry point of the profile layer."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections import Counter
|
|
6
|
+
from itertools import islice
|
|
7
|
+
|
|
8
|
+
from rowsift.profile._headers import column_names
|
|
9
|
+
from rowsift.profile._kinds import display, infer_kind
|
|
10
|
+
from rowsift.profile._options import ProfileOptions
|
|
11
|
+
from rowsift.profile._structure import find_structure
|
|
12
|
+
from rowsift.profile._types import ColumnProfile, Profile, TopValue
|
|
13
|
+
from rowsift.types import Kind, Table
|
|
14
|
+
|
|
15
|
+
__all__ = ["profile"]
|
|
16
|
+
|
|
17
|
+
# Distinct values looked at for samples: a column that masks alike is not read to its end.
|
|
18
|
+
_SCAN_LIMIT = 100
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _display(value: str, kind: Kind, options: ProfileOptions) -> str:
|
|
22
|
+
return display(value, kind, options.clean) if options.mask_samples else value
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _samples(counts: Counter[str], kind: Kind, options: ProfileOptions) -> tuple[str, ...]:
|
|
26
|
+
"""Up to ``sample_count`` distinct values as shown, in order of first appearance."""
|
|
27
|
+
shown: list[str] = []
|
|
28
|
+
for value in islice(counts, _SCAN_LIMIT):
|
|
29
|
+
if len(shown) >= options.sample_count:
|
|
30
|
+
break
|
|
31
|
+
text = _display(value, kind, options)
|
|
32
|
+
if text not in shown:
|
|
33
|
+
shown.append(text)
|
|
34
|
+
return tuple(shown)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _top_values(
|
|
38
|
+
ranked: list[tuple[str, int]], kind: Kind, options: ProfileOptions
|
|
39
|
+
) -> tuple[TopValue, ...]:
|
|
40
|
+
"""Values as shown with their counts: values that show alike are one, counts summed."""
|
|
41
|
+
merged: Counter[str] = Counter()
|
|
42
|
+
for value, count in ranked:
|
|
43
|
+
merged[_display(value, kind, options)] += count
|
|
44
|
+
ordered = sorted(merged.items(), key=lambda item: (-item[1], item[0]))
|
|
45
|
+
return tuple(TopValue(value, count) for value, count in ordered)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def profile(table: Table, options: ProfileOptions | None = None) -> Profile:
|
|
49
|
+
"""Describe the structure of ``table`` and what each column holds.
|
|
50
|
+
|
|
51
|
+
Finds the header among the first ``options.header_scan_rows`` non-empty
|
|
52
|
+
rows (no header is a valid answer), separates data rows from rows that are
|
|
53
|
+
skipped with a code, and gives each column a kind with the share of values
|
|
54
|
+
that fit it, a runner-up, counts of filled and distinct cells, the
|
|
55
|
+
low-cardinality signal with its values, and masked samples.
|
|
56
|
+
|
|
57
|
+
The kind is inferred from the first ``options.sample_rows`` data rows. The
|
|
58
|
+
counts and top values cover every data row. Samples and top values are
|
|
59
|
+
masked by the kind of their column and by the personal kind each value fits,
|
|
60
|
+
and values that show alike once masked are one, unless ``options.mask_samples``
|
|
61
|
+
is off (see ``display`` in the kinds module for the rule).
|
|
62
|
+
|
|
63
|
+
Limits, reported and not guessed: a table of text only, with no header, takes
|
|
64
|
+
its first row as the header. A header of years (``2023``, ``2024``) is
|
|
65
|
+
recognized only above numbers, with at least one text cell beside the years.
|
|
66
|
+
A total row is skipped only at the very end of the data, so a footer line
|
|
67
|
+
after it leaves both as data. A cell longer than 254 characters is never a
|
|
68
|
+
typed value. The time spent grows with ``options.sample_rows`` times the
|
|
69
|
+
number of columns, so lowering ``sample_rows`` bounds it.
|
|
70
|
+
Pure: the same table and options give the same profile.
|
|
71
|
+
|
|
72
|
+
Raises:
|
|
73
|
+
MissingExtraError: when ``options.clean`` names a region whose phone
|
|
74
|
+
rules need the ``phone`` extra.
|
|
75
|
+
"""
|
|
76
|
+
options = options or ProfileOptions()
|
|
77
|
+
rows = table.rows
|
|
78
|
+
structure = find_structure(rows, options)
|
|
79
|
+
headers = list(rows[structure.header_row]) if structure.header_row is not None else None
|
|
80
|
+
width = table.report.column_count
|
|
81
|
+
names = column_names(headers, width)
|
|
82
|
+
|
|
83
|
+
columns: list[ColumnProfile] = []
|
|
84
|
+
for index in range(width):
|
|
85
|
+
values = [rows[i][index].strip() for i in structure.data]
|
|
86
|
+
filled = [value for value in values if value]
|
|
87
|
+
kind, share, runner_up = infer_kind(
|
|
88
|
+
[v for v in values[: options.sample_rows] if v], options.kind_threshold, options.clean
|
|
89
|
+
)
|
|
90
|
+
counts = Counter(filled)
|
|
91
|
+
distinct = len(counts)
|
|
92
|
+
low = (
|
|
93
|
+
len(filled) >= options.low_cardinality_min_filled
|
|
94
|
+
and distinct <= options.low_cardinality_max_distinct
|
|
95
|
+
and distinct <= options.low_cardinality_max_share * len(filled)
|
|
96
|
+
)
|
|
97
|
+
ranked = list(counts.items()) if low else []
|
|
98
|
+
columns.append(
|
|
99
|
+
ColumnProfile(
|
|
100
|
+
index=index,
|
|
101
|
+
header=headers[index] if headers is not None else "",
|
|
102
|
+
normalized_header=names[index],
|
|
103
|
+
kind=kind,
|
|
104
|
+
kind_share=share,
|
|
105
|
+
runner_up=runner_up,
|
|
106
|
+
filled=len(filled),
|
|
107
|
+
distinct=distinct,
|
|
108
|
+
low_cardinality=low,
|
|
109
|
+
top_values=_top_values(ranked, kind, options),
|
|
110
|
+
samples=_samples(counts, kind, options),
|
|
111
|
+
)
|
|
112
|
+
)
|
|
113
|
+
return Profile(
|
|
114
|
+
header_row=structure.header_row,
|
|
115
|
+
data_start=structure.data_start,
|
|
116
|
+
data_rows=len(structure.data),
|
|
117
|
+
skipped_rows=structure.skipped,
|
|
118
|
+
columns=tuple(columns),
|
|
119
|
+
)
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
"""Where the header is, and which rows are data."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
|
|
8
|
+
from rowsift.clean import CleanOptions
|
|
9
|
+
from rowsift.profile._headers import normalize_header
|
|
10
|
+
from rowsift.profile._kinds import looks_like_data
|
|
11
|
+
from rowsift.profile._options import ProfileOptions
|
|
12
|
+
from rowsift.profile._types import SkippedRow
|
|
13
|
+
from rowsift.types import SkipCode
|
|
14
|
+
|
|
15
|
+
__all__ = ["Structure", "find_structure"]
|
|
16
|
+
|
|
17
|
+
_TOTAL_WORDS = frozenset(
|
|
18
|
+
{"total", "totais", "subtotal", "total geral", "grand total", "soma", "sum"}
|
|
19
|
+
)
|
|
20
|
+
_MIN_FILLED_SHARE = 0.5 # of the widest of the first rows: a header fills more
|
|
21
|
+
_MAX_DATA_SHARE = 0.2 # of a header's cells that may look like data
|
|
22
|
+
_MIN_DISTINCT_SHARE = 0.5 # of a header's cells that must differ: a header may repeat a name
|
|
23
|
+
_DATA_COLUMN_SHARE = 0.5 # of the cells below that make a column "data-shaped"
|
|
24
|
+
_ROWS_BELOW = 20
|
|
25
|
+
_WIDTH_ROWS = 20 # non-empty rows that set the widest row, whatever the scan window
|
|
26
|
+
_YEAR = re.compile(r"(?:19|20)[0-9]{2}")
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass(frozen=True, slots=True)
|
|
30
|
+
class Structure:
|
|
31
|
+
"""Header position, first data row, skipped rows and the indexes of the data rows."""
|
|
32
|
+
|
|
33
|
+
header_row: int | None
|
|
34
|
+
data_start: int
|
|
35
|
+
skipped: tuple[SkippedRow, ...]
|
|
36
|
+
data: tuple[int, ...]
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _filled(row: tuple[str, ...]) -> list[str]:
|
|
40
|
+
return [cell.strip() for cell in row if cell.strip()]
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _numbers_below(below: list[tuple[str, ...]], column: int, options: CleanOptions) -> bool:
|
|
44
|
+
"""Whether most of the filled cells under a column are numbers."""
|
|
45
|
+
under = [r[column].strip() for r in below if r[column].strip()]
|
|
46
|
+
numbers = sum(looks_like_data(v, options) for v in under)
|
|
47
|
+
return bool(under) and numbers >= _DATA_COLUMN_SHARE * len(under)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _is_header(
|
|
51
|
+
rows: tuple[tuple[str, ...], ...],
|
|
52
|
+
position: int,
|
|
53
|
+
non_empty: list[int],
|
|
54
|
+
widest: int,
|
|
55
|
+
options: CleanOptions,
|
|
56
|
+
) -> bool:
|
|
57
|
+
"""Whether the ``position``-th non-empty row reads as a header."""
|
|
58
|
+
row = rows[non_empty[position]]
|
|
59
|
+
cells = _filled(row)
|
|
60
|
+
if len(cells) <= _MIN_FILLED_SHARE * widest:
|
|
61
|
+
return False
|
|
62
|
+
below = [rows[i] for i in non_empty[position + 1 : position + 1 + _ROWS_BELOW]]
|
|
63
|
+
data_like = {
|
|
64
|
+
i: looks_like_data(cell.strip(), options) for i, cell in enumerate(row) if cell.strip()
|
|
65
|
+
}
|
|
66
|
+
# A year is a label when the rows under it are data: a pivot layout (Name, 2023, 2024).
|
|
67
|
+
labels = [
|
|
68
|
+
i
|
|
69
|
+
for i, flag in data_like.items()
|
|
70
|
+
if flag and _YEAR.fullmatch(row[i].strip()) and _numbers_below(below, i, options)
|
|
71
|
+
]
|
|
72
|
+
if len(labels) < len(data_like) and len(labels) == sum(data_like.values()):
|
|
73
|
+
data_like = {i: flag and i not in labels for i, flag in data_like.items()}
|
|
74
|
+
if sum(data_like.values()) > _MAX_DATA_SHARE * len(cells):
|
|
75
|
+
return False
|
|
76
|
+
if len({cell.casefold() for cell in cells}) < _MIN_DISTINCT_SHARE * len(cells):
|
|
77
|
+
return False
|
|
78
|
+
for column, cell in enumerate(row):
|
|
79
|
+
if not cell.strip() or data_like[column]:
|
|
80
|
+
continue
|
|
81
|
+
under = [r[column].strip() for r in below if r[column].strip()]
|
|
82
|
+
if under and (
|
|
83
|
+
sum(looks_like_data(v, options) for v in under) >= _DATA_COLUMN_SHARE * len(under)
|
|
84
|
+
):
|
|
85
|
+
return True
|
|
86
|
+
# A table of text only has no column to tell a header apart by shape. Its first
|
|
87
|
+
# row is the header when none of its cells comes back below it.
|
|
88
|
+
if position == 0:
|
|
89
|
+
return all(
|
|
90
|
+
cell.strip().casefold() not in {r[column].strip().casefold() for r in below}
|
|
91
|
+
for column, cell in enumerate(row)
|
|
92
|
+
if cell.strip()
|
|
93
|
+
)
|
|
94
|
+
return False
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
# A total word, then a colon or hyphen, then a count: "Total: 25", "Subtotal - 3".
|
|
98
|
+
_TOTAL_WITH_COUNT = re.compile(r"\s*(?P<word>[^\d:\-]+?)\s*[:\-]\s*\d[\d.,]*\s*")
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _is_total(row: tuple[str, ...]) -> bool:
|
|
102
|
+
"""Whether the first filled cell is a total word, alone or followed by a count.
|
|
103
|
+
|
|
104
|
+
``Total`` and ``Total: 25`` qualify. The count needs a colon or a hyphen
|
|
105
|
+
after the word, so a label such as ``Total 2024`` is data. Only the ASCII
|
|
106
|
+
colon and hyphen separate: ``Total - 2024`` is a total row, while
|
|
107
|
+
``Total`` followed by an en dash and a count, and ``Total: -5`` are data. A total word
|
|
108
|
+
followed by anything but a count (``Total de vendas``) does not qualify.
|
|
109
|
+
"""
|
|
110
|
+
first = next((cell for cell in row if cell.strip()), "")
|
|
111
|
+
if normalize_header(first) in _TOTAL_WORDS:
|
|
112
|
+
return True
|
|
113
|
+
match = _TOTAL_WITH_COUNT.fullmatch(first)
|
|
114
|
+
return match is not None and normalize_header(match["word"]) in _TOTAL_WORDS
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def find_structure(rows: tuple[tuple[str, ...], ...], options: ProfileOptions) -> Structure:
|
|
118
|
+
"""Find the header and classify every row as header, data or skipped."""
|
|
119
|
+
non_empty = [i for i, row in enumerate(rows) if _filled(row)]
|
|
120
|
+
window = non_empty[: options.header_scan_rows]
|
|
121
|
+
widest = max(
|
|
122
|
+
(len(_filled(rows[i])) for i in non_empty[: max(options.header_scan_rows, _WIDTH_ROWS)]),
|
|
123
|
+
default=0,
|
|
124
|
+
)
|
|
125
|
+
header_row: int | None = None
|
|
126
|
+
for position, index in enumerate(window):
|
|
127
|
+
if _is_header(rows, position, non_empty, widest, options.clean):
|
|
128
|
+
header_row = index
|
|
129
|
+
break
|
|
130
|
+
|
|
131
|
+
skipped: dict[int, SkipCode] = {}
|
|
132
|
+
data_start = 0 if header_row is None else header_row + 1
|
|
133
|
+
for index, row in enumerate(rows[:data_start]):
|
|
134
|
+
if index == header_row:
|
|
135
|
+
continue
|
|
136
|
+
skipped[index] = SkipCode.EMPTY if not _filled(row) else SkipCode.BEFORE_HEADER
|
|
137
|
+
|
|
138
|
+
last = len(rows)
|
|
139
|
+
while last > data_start and (not _filled(rows[last - 1]) or _is_total(rows[last - 1])):
|
|
140
|
+
last -= 1
|
|
141
|
+
skipped[last] = SkipCode.EMPTY if not _filled(rows[last]) else SkipCode.TOTAL_ROW
|
|
142
|
+
|
|
143
|
+
repeated = (
|
|
144
|
+
tuple(normalize_header(cell) for cell in rows[header_row])
|
|
145
|
+
if header_row is not None
|
|
146
|
+
else None
|
|
147
|
+
)
|
|
148
|
+
for index in range(data_start, last):
|
|
149
|
+
row = rows[index]
|
|
150
|
+
if not _filled(row):
|
|
151
|
+
skipped[index] = SkipCode.EMPTY
|
|
152
|
+
elif repeated and any(repeated) and tuple(normalize_header(c) for c in row) == repeated:
|
|
153
|
+
skipped[index] = SkipCode.REPEATED_HEADER
|
|
154
|
+
|
|
155
|
+
data = tuple(i for i in range(data_start, len(rows)) if i not in skipped)
|
|
156
|
+
ordered = tuple(SkippedRow(i, skipped[i]) for i in sorted(skipped))
|
|
157
|
+
return Structure(header_row, data_start, ordered, data)
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
"""Value types of the profile layer."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
|
|
7
|
+
from rowsift.types import Kind, SkipCode
|
|
8
|
+
|
|
9
|
+
__all__ = ["ColumnProfile", "Profile", "RunnerUp", "SkippedRow", "TopValue"]
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@dataclass(frozen=True, slots=True)
|
|
13
|
+
class SkippedRow:
|
|
14
|
+
"""A row of the table that is not a data row, and why.
|
|
15
|
+
|
|
16
|
+
``index`` is the 0-based position of the row in ``Table.rows``.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
index: int
|
|
20
|
+
code: SkipCode
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dataclass(frozen=True, slots=True)
|
|
24
|
+
class RunnerUp:
|
|
25
|
+
"""The kind that came closest to the one a column was given, and its share."""
|
|
26
|
+
|
|
27
|
+
kind: Kind
|
|
28
|
+
share: float
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@dataclass(frozen=True, slots=True)
|
|
32
|
+
class TopValue:
|
|
33
|
+
"""A value of a low-cardinality column and how many times it appears."""
|
|
34
|
+
|
|
35
|
+
value: str
|
|
36
|
+
count: int
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@dataclass(frozen=True, slots=True)
|
|
40
|
+
class ColumnProfile:
|
|
41
|
+
"""What one column holds.
|
|
42
|
+
|
|
43
|
+
Attributes:
|
|
44
|
+
index: 0-based position of the column in the table.
|
|
45
|
+
header: the header cell as written, or ``""`` when the cell is empty or
|
|
46
|
+
the table has no header.
|
|
47
|
+
normalized_header: the header folded for matching (see
|
|
48
|
+
``normalize_header``), unique among the columns. An empty header and
|
|
49
|
+
a table with no header give ``column_<position>``, with the position
|
|
50
|
+
counted from 1.
|
|
51
|
+
kind: the most specific kind that at least ``kind_threshold`` of the
|
|
52
|
+
non-empty sampled values fit, or ``text``.
|
|
53
|
+
kind_share: the share of the non-empty sampled values that fit ``kind``.
|
|
54
|
+
``0.0`` for a column with no filled cell.
|
|
55
|
+
runner_up: the closest other kind with a share above zero, or ``None``.
|
|
56
|
+
filled: the number of data rows where the cell is not empty.
|
|
57
|
+
distinct: the number of distinct non-empty values among the data rows.
|
|
58
|
+
low_cardinality: whether the column holds a small set of repeated values.
|
|
59
|
+
top_values: for a low-cardinality column, each value with its count, in
|
|
60
|
+
descending order of count. Empty otherwise.
|
|
61
|
+
samples: up to ``sample_count`` distinct values, masked by ``kind``
|
|
62
|
+
unless ``mask_samples`` is off.
|
|
63
|
+
"""
|
|
64
|
+
|
|
65
|
+
index: int
|
|
66
|
+
header: str
|
|
67
|
+
normalized_header: str
|
|
68
|
+
kind: Kind
|
|
69
|
+
kind_share: float
|
|
70
|
+
runner_up: RunnerUp | None
|
|
71
|
+
filled: int
|
|
72
|
+
distinct: int
|
|
73
|
+
low_cardinality: bool
|
|
74
|
+
top_values: tuple[TopValue, ...]
|
|
75
|
+
samples: tuple[str, ...]
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
@dataclass(frozen=True, slots=True)
|
|
79
|
+
class Profile:
|
|
80
|
+
"""The structure of a table: where the data is and what each column holds.
|
|
81
|
+
|
|
82
|
+
Attributes:
|
|
83
|
+
header_row: index in ``Table.rows`` of the header, or ``None`` when the
|
|
84
|
+
table has no header.
|
|
85
|
+
data_start: index of the first row after the header, ``0`` with no header.
|
|
86
|
+
data_rows: the number of rows that count as data.
|
|
87
|
+
skipped_rows: every row that is not a header and not data, with its code,
|
|
88
|
+
in table order.
|
|
89
|
+
columns: one profile per column, in table order.
|
|
90
|
+
"""
|
|
91
|
+
|
|
92
|
+
header_row: int | None
|
|
93
|
+
data_start: int
|
|
94
|
+
data_rows: int
|
|
95
|
+
skipped_rows: tuple[SkippedRow, ...]
|
|
96
|
+
columns: tuple[ColumnProfile, ...]
|
rowsift/py.typed
ADDED
|
File without changes
|
rowsift/read/__init__.py
ADDED
rowsift/read/_cells.py
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
"""The stable rules that turn a workbook cell value into text."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import datetime
|
|
6
|
+
from decimal import Decimal
|
|
7
|
+
|
|
8
|
+
__all__ = ["cell_text"]
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
# Every integer up to 2**53 is exact in a float.
|
|
12
|
+
_EXACT_INTEGER = 2**53
|
|
13
|
+
# Half a millisecond: the float error of a spreadsheet serial that is absorbed.
|
|
14
|
+
_FLOAT_ERROR = datetime.timedelta(microseconds=500)
|
|
15
|
+
_ZERO = datetime.timedelta(0)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _float_text(value: float) -> str:
|
|
19
|
+
if value != value or value in (float("inf"), float("-inf")):
|
|
20
|
+
return str(value)
|
|
21
|
+
if value.is_integer() and abs(value) <= _EXACT_INTEGER:
|
|
22
|
+
return str(int(value))
|
|
23
|
+
# Past 2**53 a float no longer holds every digit of the integer it prints as,
|
|
24
|
+
# so only the first 15 significant digits are kept.
|
|
25
|
+
text = format(value, ".15g")
|
|
26
|
+
if "e" in text:
|
|
27
|
+
# Plain digits, never scientific notation, so a long number stays readable.
|
|
28
|
+
return format(Decimal(text), "f")
|
|
29
|
+
return text
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _duration_text(value: datetime.timedelta) -> str:
|
|
33
|
+
# The same half millisecond of float error is absorbed as for a time of day.
|
|
34
|
+
magnitude = abs(value) + _FLOAT_ERROR
|
|
35
|
+
total = int(magnitude.total_seconds())
|
|
36
|
+
sign = "-" if value < _ZERO and total > 0 else ""
|
|
37
|
+
hours, remainder = divmod(total, 3600)
|
|
38
|
+
minutes, seconds = divmod(remainder, 60)
|
|
39
|
+
return f"{sign}{hours:02d}:{minutes:02d}:{seconds:02d}"
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _to_second(value: datetime.datetime) -> datetime.datetime:
|
|
43
|
+
# Spreadsheet serials are floats: a stored 13:45:30 comes back a hair off.
|
|
44
|
+
# Only that error is absorbed (half a millisecond, upward), then the fraction
|
|
45
|
+
# is cut. A value within half a millisecond below a second boundary reads as
|
|
46
|
+
# that boundary, so it can carry into the next second, minute or day. The
|
|
47
|
+
# last representable datetime has nothing after it and is only cut.
|
|
48
|
+
if value > datetime.datetime.max - _FLOAT_ERROR:
|
|
49
|
+
return value.replace(microsecond=0)
|
|
50
|
+
return (value + _FLOAT_ERROR).replace(microsecond=0)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def cell_text(value: object) -> str:
|
|
54
|
+
"""Text of one workbook cell value.
|
|
55
|
+
|
|
56
|
+
A date is ``YYYY-MM-DD`` and a date with a time of day is
|
|
57
|
+
``YYYY-MM-DDTHH:MM:SS``. A whole number has no decimal part, a decimal
|
|
58
|
+
number has at most 15 significant digits and never uses scientific
|
|
59
|
+
notation (a whole number beyond 2**53 keeps its first 15 significant digits),
|
|
60
|
+
a boolean is ``true`` or ``false`` and an empty cell is an empty string. A
|
|
61
|
+
datetime at exactly midnight is the date alone, a time of day is
|
|
62
|
+
``HH:MM:SS`` and a duration is ``HH:MM:SS`` with hours that may pass 24.
|
|
63
|
+
Fractions of a second are cut, after absorbing half a millisecond of float
|
|
64
|
+
error upward: a value within half a millisecond below a second boundary reads
|
|
65
|
+
as that boundary, so a time of day of 23:59:59.9995 or later wraps to
|
|
66
|
+
``00:00:00`` and a datetime carries into the next day. A value of any other
|
|
67
|
+
type is converted with ``str``. An integer of more than 4300 digits raises
|
|
68
|
+
``ValueError`` (the limit of ``str`` on an ``int``), which the workbook readers
|
|
69
|
+
report as an unsupported file.
|
|
70
|
+
"""
|
|
71
|
+
if value is None:
|
|
72
|
+
return ""
|
|
73
|
+
if isinstance(value, bool):
|
|
74
|
+
return "true" if value else "false"
|
|
75
|
+
if isinstance(value, str):
|
|
76
|
+
return value
|
|
77
|
+
if isinstance(value, int):
|
|
78
|
+
return str(value)
|
|
79
|
+
if isinstance(value, float):
|
|
80
|
+
return _float_text(value)
|
|
81
|
+
if isinstance(value, datetime.datetime):
|
|
82
|
+
value = _to_second(value)
|
|
83
|
+
if value.time() == datetime.time(0):
|
|
84
|
+
return value.date().isoformat()
|
|
85
|
+
return value.isoformat()
|
|
86
|
+
if isinstance(value, datetime.date):
|
|
87
|
+
return value.isoformat()
|
|
88
|
+
if isinstance(value, datetime.time):
|
|
89
|
+
moment = datetime.datetime.combine(datetime.date(2000, 1, 1), value)
|
|
90
|
+
return _to_second(moment).time().isoformat()
|
|
91
|
+
if isinstance(value, datetime.timedelta):
|
|
92
|
+
return _duration_text(value)
|
|
93
|
+
return str(value)
|
rowsift/read/_detect.py
ADDED
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
"""Decide the format of the bytes from their content, never from a filename."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import io
|
|
6
|
+
import zipfile
|
|
7
|
+
|
|
8
|
+
from rowsift.errors import UnsupportedFileError
|
|
9
|
+
from rowsift.read._text import looks_like_text
|
|
10
|
+
from rowsift.types import ErrorCode, FileFormat
|
|
11
|
+
|
|
12
|
+
__all__ = ["XLSX_WORKBOOK_PART", "detect_format"]
|
|
13
|
+
|
|
14
|
+
XLSX_WORKBOOK_PART = "xl/workbook.xml"
|
|
15
|
+
|
|
16
|
+
_ZIP_SIGNATURES = (b"PK\x03\x04", b"PK\x05\x06")
|
|
17
|
+
_COMPOUND_FILE_SIGNATURE = bytes.fromhex("d0cf11e0a1b11ae1")
|
|
18
|
+
# The directory of a workbook encrypted with a password holds a stream with this name.
|
|
19
|
+
_ENCRYPTED_PACKAGE = "EncryptedPackage"
|
|
20
|
+
_HEADER_SIZE = 512
|
|
21
|
+
_MAX_DIFAT_SECTORS = 4096
|
|
22
|
+
_DIRECTORY_ENTRY_SIZE = 128
|
|
23
|
+
_DIFAT_IN_HEADER = 109
|
|
24
|
+
_END_OF_CHAIN = 0xFFFFFFFE
|
|
25
|
+
_MAX_DIRECTORY_SECTORS = 4096
|
|
26
|
+
_OTHER_BINARY_SIGNATURES = (
|
|
27
|
+
b"%PDF-",
|
|
28
|
+
b"\x89PNG\r\n\x1a\n",
|
|
29
|
+
b"GIF87a",
|
|
30
|
+
b"GIF89a",
|
|
31
|
+
b"\xff\xd8\xff",
|
|
32
|
+
b"\x1f\x8b",
|
|
33
|
+
b"Rar!\x1a\x07",
|
|
34
|
+
b"7z\xbc\xaf\x27\x1c",
|
|
35
|
+
b"\x7fELF",
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _u32(data: bytes, offset: int) -> int:
|
|
40
|
+
return int.from_bytes(data[offset : offset + 4], "little")
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _directory_names(data: bytes) -> list[str]:
|
|
44
|
+
"""The entry names of the compound file directory, or none when it cannot be walked.
|
|
45
|
+
|
|
46
|
+
Reads the header, follows the directory chain through the allocation table
|
|
47
|
+
and decodes each 128-byte entry name by its length field. Sector offsets
|
|
48
|
+
account for the header filling a whole sector (512 or 4096 bytes), and the
|
|
49
|
+
extended DIFAT chain is followed, bounded. A damaged or
|
|
50
|
+
truncated container yields what could be read, never an error: the XLS
|
|
51
|
+
reader applies its own checks afterwards.
|
|
52
|
+
"""
|
|
53
|
+
if len(data) < _HEADER_SIZE:
|
|
54
|
+
return []
|
|
55
|
+
shift = int.from_bytes(data[30:32], "little")
|
|
56
|
+
if not 7 <= shift <= 16:
|
|
57
|
+
return []
|
|
58
|
+
size = 1 << shift
|
|
59
|
+
|
|
60
|
+
def sector_at(sector: int) -> bytes:
|
|
61
|
+
# The header fills the first sector, so sector n starts at (n + 1) * size.
|
|
62
|
+
if sector >= _END_OF_CHAIN:
|
|
63
|
+
return b""
|
|
64
|
+
start = (sector + 1) * size
|
|
65
|
+
return data[start : start + size]
|
|
66
|
+
|
|
67
|
+
fat_sectors = [_u32(data, 76 + 4 * i) for i in range(_DIFAT_IN_HEADER)]
|
|
68
|
+
difat = _u32(data, 68)
|
|
69
|
+
for _ in range(_MAX_DIFAT_SECTORS):
|
|
70
|
+
block = sector_at(difat)
|
|
71
|
+
if not block:
|
|
72
|
+
break
|
|
73
|
+
count = len(block) // 4
|
|
74
|
+
fat_sectors.extend(_u32(block, 4 * i) for i in range(count - 1))
|
|
75
|
+
difat = _u32(block, 4 * (count - 1))
|
|
76
|
+
fat: list[int] = []
|
|
77
|
+
listed = [sector for sector in fat_sectors if sector < _END_OF_CHAIN]
|
|
78
|
+
for sector in listed[: _u32(data, 44)]:
|
|
79
|
+
block = sector_at(sector)
|
|
80
|
+
fat.extend(_u32(block, i) for i in range(0, len(block) - 3, 4))
|
|
81
|
+
names: list[str] = []
|
|
82
|
+
sector = _u32(data, 48)
|
|
83
|
+
for _ in range(_MAX_DIRECTORY_SECTORS):
|
|
84
|
+
if sector >= _END_OF_CHAIN:
|
|
85
|
+
break
|
|
86
|
+
block = sector_at(sector)
|
|
87
|
+
for offset in range(0, len(block) - _DIRECTORY_ENTRY_SIZE + 1, _DIRECTORY_ENTRY_SIZE):
|
|
88
|
+
length = int.from_bytes(block[offset + 64 : offset + 66], "little")
|
|
89
|
+
if 2 <= length <= 64:
|
|
90
|
+
raw = block[offset : offset + length - 2]
|
|
91
|
+
names.append(raw.decode("utf-16-le", errors="replace"))
|
|
92
|
+
sector = fat[sector] if sector < len(fat) else _END_OF_CHAIN
|
|
93
|
+
return names
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def detect_format(data: bytes) -> FileFormat:
|
|
97
|
+
"""The format of ``data``: an XLSX workbook, an XLS workbook or delimited text.
|
|
98
|
+
|
|
99
|
+
An XLSX is a ZIP archive that holds the workbook part. The XLS signature is
|
|
100
|
+
the compound file header. Everything else is delimited text when it has the
|
|
101
|
+
look of text, which means no NUL bytes except in the pattern of UTF-16.
|
|
102
|
+
|
|
103
|
+
Raises:
|
|
104
|
+
UnsupportedFileError: if the bytes are a ZIP archive that is no workbook,
|
|
105
|
+
a known binary format, or have the look of binary data
|
|
106
|
+
(``unsupported_file``), or a compound file that holds an encrypted
|
|
107
|
+
workbook (``password_protected``).
|
|
108
|
+
"""
|
|
109
|
+
if data.startswith(_ZIP_SIGNATURES):
|
|
110
|
+
try:
|
|
111
|
+
with zipfile.ZipFile(io.BytesIO(data)) as archive:
|
|
112
|
+
names = archive.namelist()
|
|
113
|
+
except (zipfile.BadZipFile, OSError, ValueError, NotImplementedError):
|
|
114
|
+
raise UnsupportedFileError() from None
|
|
115
|
+
if XLSX_WORKBOOK_PART in names:
|
|
116
|
+
return FileFormat.XLSX
|
|
117
|
+
raise UnsupportedFileError()
|
|
118
|
+
if data.startswith(_COMPOUND_FILE_SIGNATURE):
|
|
119
|
+
if _ENCRYPTED_PACKAGE in _directory_names(data):
|
|
120
|
+
raise UnsupportedFileError(ErrorCode.PASSWORD_PROTECTED)
|
|
121
|
+
return FileFormat.XLS
|
|
122
|
+
if data.startswith(_OTHER_BINARY_SIGNATURES) or not looks_like_text(data):
|
|
123
|
+
raise UnsupportedFileError()
|
|
124
|
+
return FileFormat.CSV
|