rowsift 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rowsift/__init__.py +98 -0
- rowsift/clean/__init__.py +44 -0
- rowsift/clean/_common.py +30 -0
- rowsift/clean/_dates.py +283 -0
- rowsift/clean/_dispatch.py +49 -0
- rowsift/clean/_identifiers.py +161 -0
- rowsift/clean/_mask.py +117 -0
- rowsift/clean/_numbers.py +137 -0
- rowsift/clean/_options.py +75 -0
- rowsift/clean/_phone.py +202 -0
- rowsift/clean/_text.py +102 -0
- rowsift/errors.py +124 -0
- rowsift/map/__init__.py +24 -0
- rowsift/map/_fields.py +75 -0
- rowsift/map/_saved.py +132 -0
- rowsift/map/_score.py +178 -0
- rowsift/map/_suggest.py +168 -0
- rowsift/map/_types.py +169 -0
- rowsift/profile/__init__.py +17 -0
- rowsift/profile/_headers.py +44 -0
- rowsift/profile/_kinds.py +165 -0
- rowsift/profile/_options.py +69 -0
- rowsift/profile/_profile.py +119 -0
- rowsift/profile/_structure.py +157 -0
- rowsift/profile/_types.py +96 -0
- rowsift/py.typed +0 -0
- rowsift/read/__init__.py +6 -0
- rowsift/read/_cells.py +93 -0
- rowsift/read/_detect.py +124 -0
- rowsift/read/_grid.py +128 -0
- rowsift/read/_limits.py +44 -0
- rowsift/read/_read.py +99 -0
- rowsift/read/_text.py +250 -0
- rowsift/read/_workbook.py +36 -0
- rowsift/read/_xls.py +115 -0
- rowsift/read/_xlsx.py +121 -0
- rowsift/sift/__init__.py +19 -0
- rowsift/sift/_sift.py +165 -0
- rowsift/sift/_types.py +119 -0
- rowsift/types.py +378 -0
- rowsift-0.1.1.dist-info/METADATA +148 -0
- rowsift-0.1.1.dist-info/RECORD +45 -0
- rowsift-0.1.1.dist-info/WHEEL +4 -0
- rowsift-0.1.1.dist-info/licenses/LICENSE +177 -0
- rowsift-0.1.1.dist-info/licenses/NOTICE +4 -0
rowsift/read/_grid.py
ADDED
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
"""Shared bookkeeping that turns rows of cell text into a rectangular grid."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Iterable
|
|
6
|
+
|
|
7
|
+
from rowsift.errors import LimitExceededError, UnsupportedFileError
|
|
8
|
+
from rowsift.read._limits import ReadLimits
|
|
9
|
+
from rowsift.types import ErrorCode
|
|
10
|
+
|
|
11
|
+
__all__ = ["Grid", "RowCollector", "fill_merged", "select_sheets"]
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class Grid:
|
|
15
|
+
"""The result of collecting rows: padded rows and whether columns were dropped."""
|
|
16
|
+
|
|
17
|
+
__slots__ = ("columns_dropped", "rows")
|
|
18
|
+
|
|
19
|
+
def __init__(self, rows: list[list[str]], columns_dropped: bool) -> None:
|
|
20
|
+
self.rows = rows
|
|
21
|
+
self.columns_dropped = columns_dropped
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class RowCollector:
|
|
25
|
+
"""Collect rows one at a time, enforcing the row, column and cell limits.
|
|
26
|
+
|
|
27
|
+
Every row is cut at its last non-empty cell, so ``max_columns`` counts
|
|
28
|
+
columns that hold data. With ``trim_trailing_rows`` the rows after the last
|
|
29
|
+
non-empty one are dropped, which is what a workbook needs because its sheet
|
|
30
|
+
dimension often reaches far past the data.
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
def __init__(self, limits: ReadLimits, *, trim_trailing_rows: bool) -> None:
|
|
34
|
+
self._limits = limits
|
|
35
|
+
self._trim_trailing_rows = trim_trailing_rows
|
|
36
|
+
self._rows: list[list[str]] = []
|
|
37
|
+
self._pending_empty = 0
|
|
38
|
+
self._raw_width = 0
|
|
39
|
+
|
|
40
|
+
def add(self, cells: Iterable[str]) -> None:
|
|
41
|
+
"""Add the next row of cell text."""
|
|
42
|
+
row = list(cells)
|
|
43
|
+
self._raw_width = max(self._raw_width, len(row))
|
|
44
|
+
for cell in row:
|
|
45
|
+
if len(cell) > self._limits.max_cell_chars:
|
|
46
|
+
raise LimitExceededError("max_cell_chars", self._limits.max_cell_chars)
|
|
47
|
+
while row and not row[-1]:
|
|
48
|
+
row.pop()
|
|
49
|
+
if len(row) > self._limits.max_columns:
|
|
50
|
+
raise LimitExceededError("max_columns", self._limits.max_columns)
|
|
51
|
+
if not row and self._trim_trailing_rows:
|
|
52
|
+
self._pending_empty += 1
|
|
53
|
+
return
|
|
54
|
+
for _ in range(self._pending_empty):
|
|
55
|
+
self._append([])
|
|
56
|
+
self._pending_empty = 0
|
|
57
|
+
self._append(row)
|
|
58
|
+
|
|
59
|
+
def _append(self, row: list[str]) -> None:
|
|
60
|
+
if len(self._rows) >= self._limits.max_rows:
|
|
61
|
+
raise LimitExceededError("max_rows", self._limits.max_rows)
|
|
62
|
+
self._rows.append(row)
|
|
63
|
+
|
|
64
|
+
def finish(self) -> Grid:
|
|
65
|
+
"""Pad every row to the widest one and report whether columns were dropped."""
|
|
66
|
+
if not self._trim_trailing_rows:
|
|
67
|
+
for _ in range(self._pending_empty):
|
|
68
|
+
self._append([])
|
|
69
|
+
self._pending_empty = 0
|
|
70
|
+
width = max((len(row) for row in self._rows), default=0)
|
|
71
|
+
for row in self._rows:
|
|
72
|
+
row.extend([""] * (width - len(row)))
|
|
73
|
+
return Grid(self._rows, columns_dropped=self._raw_width > width)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def fill_merged(rows: list[list[str]], areas: Iterable[tuple[int, int, int, int]]) -> bool:
|
|
77
|
+
"""Give every cell of each merged area the value of its top-left cell.
|
|
78
|
+
|
|
79
|
+
``areas`` are ``(first_row, first_column, last_row, last_column)``, zero
|
|
80
|
+
based and inclusive. An area is clipped to the grid, so an area that runs
|
|
81
|
+
past the data never grows the table. Returns whether any cell changed.
|
|
82
|
+
|
|
83
|
+
Raises:
|
|
84
|
+
UnsupportedFileError: if the areas, taken together, cover more cells than
|
|
85
|
+
the grid holds, which only overlapping areas can do.
|
|
86
|
+
"""
|
|
87
|
+
height = len(rows)
|
|
88
|
+
width = len(rows[0]) if rows else 0
|
|
89
|
+
budget = height * width
|
|
90
|
+
changed = False
|
|
91
|
+
for first_row, first_col, last_row, last_col in areas:
|
|
92
|
+
last_row = min(last_row, height - 1)
|
|
93
|
+
last_col = min(last_col, width - 1)
|
|
94
|
+
if first_row > last_row or first_col > last_col:
|
|
95
|
+
continue
|
|
96
|
+
budget -= (last_row - first_row + 1) * (last_col - first_col + 1)
|
|
97
|
+
if budget < 0:
|
|
98
|
+
raise UnsupportedFileError()
|
|
99
|
+
value = rows[first_row][first_col]
|
|
100
|
+
if not value:
|
|
101
|
+
continue
|
|
102
|
+
for row_index in range(first_row, last_row + 1):
|
|
103
|
+
row = rows[row_index]
|
|
104
|
+
for col_index in range(first_col, last_col + 1):
|
|
105
|
+
if row[col_index] != value:
|
|
106
|
+
row[col_index] = value
|
|
107
|
+
changed = True
|
|
108
|
+
return changed
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def select_sheets(names: list[str], sheet: str | int | None) -> list[int]:
|
|
112
|
+
"""Indexes of the sheets to try, in order.
|
|
113
|
+
|
|
114
|
+
With no ``sheet`` every sheet is a candidate, so the first non-empty one
|
|
115
|
+
wins. A name or a zero-based index selects exactly one.
|
|
116
|
+
|
|
117
|
+
Raises:
|
|
118
|
+
UnsupportedFileError: with the code ``sheet_not_found`` if the name or the
|
|
119
|
+
index does not exist.
|
|
120
|
+
"""
|
|
121
|
+
if sheet is None:
|
|
122
|
+
return list(range(len(names)))
|
|
123
|
+
if isinstance(sheet, str):
|
|
124
|
+
if sheet in names:
|
|
125
|
+
return [names.index(sheet)]
|
|
126
|
+
elif 0 <= sheet < len(names):
|
|
127
|
+
return [sheet]
|
|
128
|
+
raise UnsupportedFileError(ErrorCode.SHEET_NOT_FOUND)
|
rowsift/read/_limits.py
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"""The limits of a read."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass, fields
|
|
6
|
+
|
|
7
|
+
__all__ = ["ReadLimits"]
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@dataclass(frozen=True, slots=True, kw_only=True)
|
|
11
|
+
class ReadLimits:
|
|
12
|
+
"""Upper bounds a read enforces before it does the work. All are passed by keyword.
|
|
13
|
+
|
|
14
|
+
A file that goes beyond one of them stops the read with a
|
|
15
|
+
``LimitExceededError`` that names the limit. Sizes are in bytes, with a
|
|
16
|
+
megabyte being 1,048,576 bytes.
|
|
17
|
+
|
|
18
|
+
Attributes:
|
|
19
|
+
max_bytes: size of the input. Default 10 MB.
|
|
20
|
+
max_rows: rows of the table, empty rows included. Default 50,000.
|
|
21
|
+
max_columns: columns that hold data. Trailing columns that are empty in
|
|
22
|
+
every row do not count. Default 200.
|
|
23
|
+
max_decompressed_bytes: sum of the uncompressed sizes a workbook
|
|
24
|
+
archive declares. Default 100 MB.
|
|
25
|
+
max_cell_chars: characters in one cell. Default 10,000.
|
|
26
|
+
|
|
27
|
+
Raises:
|
|
28
|
+
TypeError: if a limit is not an ``int`` (a ``bool`` is not).
|
|
29
|
+
ValueError: if a limit is not positive.
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
max_bytes: int = 10 * 1024 * 1024
|
|
33
|
+
max_rows: int = 50_000
|
|
34
|
+
max_columns: int = 200
|
|
35
|
+
max_decompressed_bytes: int = 100 * 1024 * 1024
|
|
36
|
+
max_cell_chars: int = 10_000
|
|
37
|
+
|
|
38
|
+
def __post_init__(self) -> None:
|
|
39
|
+
for field in fields(self):
|
|
40
|
+
value = getattr(self, field.name)
|
|
41
|
+
if type(value) is not int:
|
|
42
|
+
raise TypeError(f"{field.name} must be an int")
|
|
43
|
+
if value < 1:
|
|
44
|
+
raise ValueError(f"{field.name} must be positive")
|
rowsift/read/_read.py
ADDED
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
"""The entry point of the reading layer."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from rowsift.errors import LimitExceededError
|
|
6
|
+
from rowsift.read._detect import detect_format
|
|
7
|
+
from rowsift.read._grid import select_sheets
|
|
8
|
+
from rowsift.read._limits import ReadLimits
|
|
9
|
+
from rowsift.read._text import read_delimited
|
|
10
|
+
from rowsift.read._workbook import WorkbookResult
|
|
11
|
+
from rowsift.read._xls import read_xls
|
|
12
|
+
from rowsift.read._xlsx import read_xlsx
|
|
13
|
+
from rowsift.types import FileFormat, ReadReport, Table
|
|
14
|
+
|
|
15
|
+
__all__ = ["read"]
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def read(
|
|
19
|
+
data: bytes | bytearray | memoryview,
|
|
20
|
+
*,
|
|
21
|
+
filename: str | None = None,
|
|
22
|
+
sheet: str | int | None = None,
|
|
23
|
+
limits: ReadLimits | None = None,
|
|
24
|
+
) -> Table:
|
|
25
|
+
"""Read the bytes of a spreadsheet into a grid of text cells.
|
|
26
|
+
|
|
27
|
+
The format comes from the content: an XLSX workbook, an XLS workbook or
|
|
28
|
+
delimited text. XLSX and XLS need the ``excel`` extra. Every cell is text,
|
|
29
|
+
short rows are padded and trailing columns that are empty in every row are
|
|
30
|
+
dropped. Formulas give the result the file stored and are never evaluated.
|
|
31
|
+
The ``report`` of the table says how the file was read.
|
|
32
|
+
|
|
33
|
+
Args:
|
|
34
|
+
data: the bytes of the file (``bytes``, ``bytearray`` or ``memoryview``).
|
|
35
|
+
filename: accepted for the caller's convenience. It never changes the
|
|
36
|
+
result, because the format is decided by the content.
|
|
37
|
+
sheet: the sheet of a workbook, by name or zero-based index. By default
|
|
38
|
+
the first sheet that holds data. A file of delimited text has no
|
|
39
|
+
sheets, so asking for one raises ``sheet_not_found``.
|
|
40
|
+
limits: the bounds to enforce. ``ReadLimits()`` when omitted.
|
|
41
|
+
|
|
42
|
+
Returns:
|
|
43
|
+
The table and its report.
|
|
44
|
+
|
|
45
|
+
Raises:
|
|
46
|
+
LimitExceededError: if the input, the rows, the columns, a cell or the
|
|
47
|
+
decompressed size of a workbook go beyond the limits.
|
|
48
|
+
MissingExtraError: if a workbook is read without the ``excel`` extra.
|
|
49
|
+
UnsupportedFileError: if the bytes are no supported format, the sheet
|
|
50
|
+
does not exist (``sheet_not_found``) or the workbook is encrypted
|
|
51
|
+
with a password (``password_protected``).
|
|
52
|
+
DecodingError: if a byte order mark promises an encoding the bytes do not
|
|
53
|
+
follow, or the delimited text cannot be split into records.
|
|
54
|
+
TypeError: if an argument has the wrong type.
|
|
55
|
+
"""
|
|
56
|
+
if not isinstance(data, bytes | bytearray | memoryview):
|
|
57
|
+
raise TypeError("data must be bytes")
|
|
58
|
+
if filename is not None and not isinstance(filename, str):
|
|
59
|
+
raise TypeError("filename must be a str or None")
|
|
60
|
+
if sheet is not None and (isinstance(sheet, bool) or not isinstance(sheet, str | int)):
|
|
61
|
+
raise TypeError("sheet must be a str, an int or None")
|
|
62
|
+
if limits is None:
|
|
63
|
+
limits = ReadLimits()
|
|
64
|
+
elif not isinstance(limits, ReadLimits):
|
|
65
|
+
raise TypeError("limits must be a ReadLimits")
|
|
66
|
+
if memoryview(data).nbytes > limits.max_bytes:
|
|
67
|
+
raise LimitExceededError("max_bytes", limits.max_bytes)
|
|
68
|
+
content = bytes(data)
|
|
69
|
+
|
|
70
|
+
file_format = detect_format(content) if content else FileFormat.CSV
|
|
71
|
+
if file_format is FileFormat.CSV:
|
|
72
|
+
if sheet is not None:
|
|
73
|
+
select_sheets([], sheet)
|
|
74
|
+
text = read_delimited(content, limits)
|
|
75
|
+
rows = tuple(tuple(row) for row in text.grid.rows)
|
|
76
|
+
report = ReadReport(
|
|
77
|
+
format=FileFormat.CSV,
|
|
78
|
+
encoding=text.encoding,
|
|
79
|
+
delimiter=text.delimiter,
|
|
80
|
+
row_count=len(rows),
|
|
81
|
+
column_count=len(rows[0]) if rows else 0,
|
|
82
|
+
notes=text.notes,
|
|
83
|
+
)
|
|
84
|
+
return Table(rows, report)
|
|
85
|
+
workbook: WorkbookResult
|
|
86
|
+
if file_format is FileFormat.XLSX:
|
|
87
|
+
workbook = read_xlsx(content, sheet, limits)
|
|
88
|
+
else:
|
|
89
|
+
workbook = read_xls(content, sheet, limits)
|
|
90
|
+
rows = tuple(tuple(row) for row in workbook.grid.rows)
|
|
91
|
+
report = ReadReport(
|
|
92
|
+
format=file_format,
|
|
93
|
+
sheet=workbook.sheet,
|
|
94
|
+
sheets=workbook.sheets,
|
|
95
|
+
row_count=len(rows),
|
|
96
|
+
column_count=len(rows[0]) if rows else 0,
|
|
97
|
+
notes=workbook.notes(),
|
|
98
|
+
)
|
|
99
|
+
return Table(rows, report)
|
rowsift/read/_text.py
ADDED
|
@@ -0,0 +1,250 @@
|
|
|
1
|
+
"""Delimited text: encoding, delimiter and quoting."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import codecs
|
|
6
|
+
import csv
|
|
7
|
+
import io
|
|
8
|
+
import re
|
|
9
|
+
import threading
|
|
10
|
+
from collections import Counter
|
|
11
|
+
from collections.abc import Iterator
|
|
12
|
+
from contextlib import contextmanager
|
|
13
|
+
|
|
14
|
+
from rowsift.errors import DecodingError, LimitExceededError, UnsupportedFileError
|
|
15
|
+
from rowsift.read._grid import Grid, RowCollector
|
|
16
|
+
from rowsift.read._limits import ReadLimits
|
|
17
|
+
from rowsift.types import ReadNote
|
|
18
|
+
|
|
19
|
+
__all__ = ["DelimitedResult", "decode", "detect_delimiter", "looks_like_text", "read_delimited"]
|
|
20
|
+
|
|
21
|
+
_NUL_SAMPLE = 8192
|
|
22
|
+
_UTF16_NUL_SHARE = 0.3
|
|
23
|
+
_DELIMITER_SAMPLE_RECORDS = 100
|
|
24
|
+
# The csv module keeps its field size limit for the whole process.
|
|
25
|
+
_FIELD_LIMIT_LOCK = threading.Lock()
|
|
26
|
+
# Tie-break order when two delimiters split the sample equally well.
|
|
27
|
+
_CANDIDATES = (";", "\t", "|", ",")
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _utf16_without_mark(data: bytes) -> str | None:
|
|
31
|
+
"""``utf-16-le`` or ``utf-16-be`` when the NUL bytes sit in one parity only."""
|
|
32
|
+
sample = data[:_NUL_SAMPLE]
|
|
33
|
+
size = len(sample) // 2 * 2
|
|
34
|
+
if size < 2:
|
|
35
|
+
return None
|
|
36
|
+
even_nuls = sample[0:size:2].count(0)
|
|
37
|
+
odd_nuls = sample[1:size:2].count(0)
|
|
38
|
+
half = size // 2
|
|
39
|
+
if even_nuls == 0 and odd_nuls >= half * _UTF16_NUL_SHARE:
|
|
40
|
+
return "utf-16-le"
|
|
41
|
+
if odd_nuls == 0 and even_nuls >= half * _UTF16_NUL_SHARE:
|
|
42
|
+
return "utf-16-be"
|
|
43
|
+
return None
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def looks_like_text(data: bytes) -> bool:
|
|
47
|
+
"""Whether ``data`` has the look of text: no NUL bytes, or the pattern of UTF-16."""
|
|
48
|
+
if data.startswith((codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE)):
|
|
49
|
+
return True
|
|
50
|
+
if b"\x00" not in data[:_NUL_SAMPLE]:
|
|
51
|
+
return True
|
|
52
|
+
return _utf16_without_mark(data) is not None
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _strict(data: bytes, encoding: str) -> str | None:
|
|
56
|
+
try:
|
|
57
|
+
return data.decode(encoding)
|
|
58
|
+
except UnicodeDecodeError:
|
|
59
|
+
return None
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def decode(data: bytes) -> tuple[str, str, bool]:
|
|
63
|
+
"""Decode delimited text. Returns the text, the encoding used and whether a mark was removed.
|
|
64
|
+
|
|
65
|
+
The order is a byte order mark (UTF-8, UTF-16 LE, UTF-16 BE), UTF-16 without
|
|
66
|
+
a mark when the NUL pattern says so, strict UTF-8, cp1252 and latin-1. The
|
|
67
|
+
NUL pattern comes before UTF-8 because UTF-16 text of Latin characters is
|
|
68
|
+
also valid UTF-8. The mark is removed from the text.
|
|
69
|
+
|
|
70
|
+
Raises:
|
|
71
|
+
DecodingError: if a byte order mark promises an encoding the bytes do not follow.
|
|
72
|
+
"""
|
|
73
|
+
marks = (
|
|
74
|
+
(codecs.BOM_UTF8, "utf-8"),
|
|
75
|
+
(codecs.BOM_UTF16_LE, "utf-16-le"),
|
|
76
|
+
(codecs.BOM_UTF16_BE, "utf-16-be"),
|
|
77
|
+
)
|
|
78
|
+
for mark, encoding in marks:
|
|
79
|
+
if data.startswith(mark):
|
|
80
|
+
text = _strict(data[len(mark) :], encoding)
|
|
81
|
+
if text is None:
|
|
82
|
+
raise DecodingError()
|
|
83
|
+
return text, encoding, True
|
|
84
|
+
# UTF-16 text is also valid UTF-8, with a NUL beside every character, so the
|
|
85
|
+
# NUL pattern has to be checked before UTF-8 can claim the bytes.
|
|
86
|
+
wide = _utf16_without_mark(data)
|
|
87
|
+
if wide is not None:
|
|
88
|
+
text = _strict(data, wide)
|
|
89
|
+
if text is not None:
|
|
90
|
+
return text, wide, False
|
|
91
|
+
text = _strict(data, "utf-8")
|
|
92
|
+
if text is not None:
|
|
93
|
+
return text, "utf-8", False
|
|
94
|
+
for encoding in ("cp1252", "latin-1"):
|
|
95
|
+
text = _strict(data, encoding)
|
|
96
|
+
if text is not None:
|
|
97
|
+
return text, encoding, False
|
|
98
|
+
raise DecodingError() # pragma: no cover - latin-1 decodes every byte
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
@contextmanager
|
|
102
|
+
def _field_limit(limits: ReadLimits) -> Iterator[None]:
|
|
103
|
+
"""Set the csv field size limit to one past ``max_cell_chars`` for the block.
|
|
104
|
+
|
|
105
|
+
The csv module has its own default of 131072 characters, which would refuse
|
|
106
|
+
a cell the caller allows. One past the limit lets the row collector see an
|
|
107
|
+
oversized cell and refuse it with the cell limit it reports. The previous
|
|
108
|
+
value is restored on the way out.
|
|
109
|
+
"""
|
|
110
|
+
with _FIELD_LIMIT_LOCK:
|
|
111
|
+
previous = csv.field_size_limit(limits.max_cell_chars + 1)
|
|
112
|
+
try:
|
|
113
|
+
yield
|
|
114
|
+
finally:
|
|
115
|
+
csv.field_size_limit(previous)
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _ends_inside_quotes(text: str, delimiter: str) -> bool:
|
|
119
|
+
"""Whether the text ends in the middle of a quoted field.
|
|
120
|
+
|
|
121
|
+
The scan follows the quoting rules of the csv module in its lenient mode: a
|
|
122
|
+
quote opens a quoted field only at the start of a field, a doubled quote
|
|
123
|
+
inside one is a literal quote, and a quote anywhere else is plain text. A
|
|
124
|
+
stray quote earlier in the file therefore does not hide a quote that is
|
|
125
|
+
never closed.
|
|
126
|
+
"""
|
|
127
|
+
if '"' not in text:
|
|
128
|
+
return False
|
|
129
|
+
boundary = re.compile(f"[{re.escape(delimiter)}\r\n]")
|
|
130
|
+
size = len(text)
|
|
131
|
+
position = 0
|
|
132
|
+
while position < size:
|
|
133
|
+
if text[position] == '"':
|
|
134
|
+
position += 1
|
|
135
|
+
while True:
|
|
136
|
+
quote = text.find('"', position)
|
|
137
|
+
if quote < 0:
|
|
138
|
+
return True
|
|
139
|
+
if text.startswith('""', quote):
|
|
140
|
+
position = quote + 2
|
|
141
|
+
continue
|
|
142
|
+
position = quote + 1
|
|
143
|
+
break
|
|
144
|
+
found = boundary.search(text, position)
|
|
145
|
+
if found is None:
|
|
146
|
+
return False
|
|
147
|
+
position = found.end()
|
|
148
|
+
return False
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _field_counts(text: str, delimiter: str) -> list[int]:
|
|
152
|
+
counts: list[int] = []
|
|
153
|
+
reader = csv.reader(io.StringIO(text, newline=""), delimiter=delimiter)
|
|
154
|
+
try:
|
|
155
|
+
for record in reader:
|
|
156
|
+
if any(cell.strip() for cell in record):
|
|
157
|
+
counts.append(len(record))
|
|
158
|
+
if len(counts) >= _DELIMITER_SAMPLE_RECORDS:
|
|
159
|
+
break
|
|
160
|
+
except csv.Error:
|
|
161
|
+
pass
|
|
162
|
+
return counts
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def detect_delimiter(text: str) -> str | None:
|
|
166
|
+
"""The delimiter among ``;`` tab ``|`` and ``,``, or ``None`` for a single column.
|
|
167
|
+
|
|
168
|
+
Each candidate splits the first non-empty records, quoted fields respected.
|
|
169
|
+
A candidate needs at least two fields in its most common count. The winner
|
|
170
|
+
is the one with the most records at that count, weighted by the number of
|
|
171
|
+
cells it separates, so a comma inside free text does not beat a semicolon
|
|
172
|
+
that splits every row into many cells. A tie goes to the first of
|
|
173
|
+
``;`` tab ``|`` ``,``.
|
|
174
|
+
"""
|
|
175
|
+
best: str | None = None
|
|
176
|
+
best_score = 0
|
|
177
|
+
for candidate in _CANDIDATES:
|
|
178
|
+
counts = _field_counts(text, candidate)
|
|
179
|
+
if not counts:
|
|
180
|
+
continue
|
|
181
|
+
mode, records = Counter(counts).most_common(1)[0]
|
|
182
|
+
if mode < 2:
|
|
183
|
+
continue
|
|
184
|
+
score = records * (mode - 1)
|
|
185
|
+
if score > best_score:
|
|
186
|
+
best, best_score = candidate, score
|
|
187
|
+
return best
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def _unused_character(text: str) -> str:
|
|
191
|
+
for code in range(1, 32):
|
|
192
|
+
character = chr(code)
|
|
193
|
+
if character not in "\r\n" and character not in text:
|
|
194
|
+
return character
|
|
195
|
+
raise UnsupportedFileError()
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
class DelimitedResult:
|
|
199
|
+
"""What reading delimited text produced."""
|
|
200
|
+
|
|
201
|
+
__slots__ = ("delimiter", "encoding", "grid", "notes")
|
|
202
|
+
|
|
203
|
+
def __init__(
|
|
204
|
+
self,
|
|
205
|
+
grid: Grid,
|
|
206
|
+
encoding: str,
|
|
207
|
+
delimiter: str | None,
|
|
208
|
+
notes: tuple[ReadNote, ...],
|
|
209
|
+
) -> None:
|
|
210
|
+
self.grid = grid
|
|
211
|
+
self.encoding = encoding
|
|
212
|
+
self.delimiter = delimiter
|
|
213
|
+
self.notes = notes
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def read_delimited(data: bytes, limits: ReadLimits) -> DelimitedResult:
|
|
217
|
+
"""Read delimited text into a grid of cell text.
|
|
218
|
+
|
|
219
|
+
A quoted field that is never closed swallows the rest of the file into one
|
|
220
|
+
cell, so the result carries the ``unterminated_quote`` note. When that
|
|
221
|
+
cell is longer than ``max_cell_chars`` the read raises the cell limit error
|
|
222
|
+
instead, because the limit comes first.
|
|
223
|
+
|
|
224
|
+
Raises:
|
|
225
|
+
DecodingError: if a byte order mark promises an encoding the bytes do not
|
|
226
|
+
follow, or the text cannot be split into records.
|
|
227
|
+
LimitExceededError: if the rows, columns or a cell go beyond the limits.
|
|
228
|
+
UnsupportedFileError: if the text cannot be split at all.
|
|
229
|
+
"""
|
|
230
|
+
text, encoding, mark_removed = decode(data)
|
|
231
|
+
with _field_limit(limits):
|
|
232
|
+
delimiter = detect_delimiter(text)
|
|
233
|
+
splitter = delimiter if delimiter is not None else _unused_character(text)
|
|
234
|
+
collector = RowCollector(limits, trim_trailing_rows=False)
|
|
235
|
+
reader = csv.reader(io.StringIO(text, newline=""), delimiter=splitter)
|
|
236
|
+
try:
|
|
237
|
+
for record in reader:
|
|
238
|
+
collector.add(record)
|
|
239
|
+
except csv.Error as error:
|
|
240
|
+
if "field larger than field limit" in str(error):
|
|
241
|
+
raise LimitExceededError("max_cell_chars", limits.max_cell_chars) from None
|
|
242
|
+
raise DecodingError() from None
|
|
243
|
+
unterminated = _ends_inside_quotes(text, splitter)
|
|
244
|
+
grid = collector.finish()
|
|
245
|
+
notes = [ReadNote.BOM_REMOVED] if mark_removed else []
|
|
246
|
+
if unterminated:
|
|
247
|
+
notes.append(ReadNote.UNTERMINATED_QUOTE)
|
|
248
|
+
if grid.columns_dropped:
|
|
249
|
+
notes.append(ReadNote.TRAILING_EMPTY_COLUMNS_DROPPED)
|
|
250
|
+
return DelimitedResult(grid, encoding, delimiter, tuple(notes))
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
"""What reading a workbook produced, shared by the XLSX and XLS readers."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from rowsift.read._grid import Grid
|
|
6
|
+
from rowsift.types import ReadNote
|
|
7
|
+
|
|
8
|
+
__all__ = ["WorkbookResult"]
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class WorkbookResult:
|
|
12
|
+
"""The grid of the chosen sheet, its name and the names of every sheet."""
|
|
13
|
+
|
|
14
|
+
__slots__ = ("grid", "merged_filled", "sheet", "sheets")
|
|
15
|
+
|
|
16
|
+
def __init__(
|
|
17
|
+
self,
|
|
18
|
+
grid: Grid,
|
|
19
|
+
sheet: str | None,
|
|
20
|
+
sheets: tuple[str, ...],
|
|
21
|
+
*,
|
|
22
|
+
merged_filled: bool,
|
|
23
|
+
) -> None:
|
|
24
|
+
self.grid = grid
|
|
25
|
+
self.sheet = sheet
|
|
26
|
+
self.sheets = sheets
|
|
27
|
+
self.merged_filled = merged_filled
|
|
28
|
+
|
|
29
|
+
def notes(self) -> tuple[ReadNote, ...]:
|
|
30
|
+
"""The notes of this read, in a fixed order."""
|
|
31
|
+
notes: list[ReadNote] = []
|
|
32
|
+
if self.merged_filled:
|
|
33
|
+
notes.append(ReadNote.MERGED_CELLS_FILLED)
|
|
34
|
+
if self.grid.columns_dropped:
|
|
35
|
+
notes.append(ReadNote.TRAILING_EMPTY_COLUMNS_DROPPED)
|
|
36
|
+
return tuple(notes)
|
rowsift/read/_xls.py
ADDED
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
"""XLS workbooks, read with the ``excel`` extra."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import datetime
|
|
6
|
+
import io
|
|
7
|
+
import re
|
|
8
|
+
from collections.abc import Iterator
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
from rowsift.errors import MissingExtraError, RowsiftError, UnsupportedFileError
|
|
12
|
+
from rowsift.read._cells import cell_text
|
|
13
|
+
from rowsift.read._grid import Grid, RowCollector, fill_merged, select_sheets
|
|
14
|
+
from rowsift.read._limits import ReadLimits
|
|
15
|
+
from rowsift.read._workbook import WorkbookResult
|
|
16
|
+
|
|
17
|
+
__all__ = ["read_xls"]
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
_QUOTED = re.compile(r'"[^"]*"|\\.|\[(?![hms])[^\]]*\]', re.IGNORECASE)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _is_time_only(book: Any, cell: Any) -> bool:
|
|
24
|
+
"""Whether the number format of a date cell shows hours, minutes or seconds only."""
|
|
25
|
+
try:
|
|
26
|
+
xf = book.xf_list[cell.xf_index]
|
|
27
|
+
pattern: str = book.format_map[xf.format_key].format_str
|
|
28
|
+
except (AttributeError, IndexError, KeyError):
|
|
29
|
+
return False
|
|
30
|
+
shown = _QUOTED.sub("", pattern).lower()
|
|
31
|
+
return not any(letter in shown for letter in "yd") and "mmm" not in shown
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _cell_value(xlrd: Any, book: Any, cell: Any) -> object:
|
|
35
|
+
kind = cell.ctype
|
|
36
|
+
if kind == xlrd.XL_CELL_DATE:
|
|
37
|
+
value: float = cell.value
|
|
38
|
+
if value >= 1 and _is_time_only(book, cell):
|
|
39
|
+
# A duration such as [h]:mm:ss: the serial counts days, not a date.
|
|
40
|
+
return datetime.timedelta(days=value)
|
|
41
|
+
moment: datetime.datetime = xlrd.xldate.xldate_as_datetime(value, book.datemode)
|
|
42
|
+
# A value below one day is a time of day: the serial has no date part.
|
|
43
|
+
return moment.time() if value < 1 else moment
|
|
44
|
+
if kind == xlrd.XL_CELL_BOOLEAN:
|
|
45
|
+
return bool(cell.value)
|
|
46
|
+
if kind == xlrd.XL_CELL_ERROR:
|
|
47
|
+
return xlrd.error_text_from_code.get(cell.value, "")
|
|
48
|
+
if kind in (xlrd.XL_CELL_EMPTY, xlrd.XL_CELL_BLANK):
|
|
49
|
+
return None
|
|
50
|
+
return cell.value
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _areas(worksheet: Any) -> Iterator[tuple[int, int, int, int]]:
|
|
54
|
+
# xlrd gives bounds as (first row, last row + 1, first column, last column + 1).
|
|
55
|
+
for first_row, end_row, first_col, end_col in worksheet.merged_cells:
|
|
56
|
+
yield first_row, first_col, end_row - 1, end_col - 1
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _rows(xlrd: Any, book: Any, worksheet: Any, limits: ReadLimits) -> Grid:
|
|
60
|
+
collector = RowCollector(limits, trim_trailing_rows=True)
|
|
61
|
+
for row_index in range(worksheet.nrows):
|
|
62
|
+
cells = worksheet.row(row_index)
|
|
63
|
+
collector.add(cell_text(_cell_value(xlrd, book, cell)) for cell in cells)
|
|
64
|
+
return collector.finish()
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def read_xls(data: bytes, sheet: str | int | None, limits: ReadLimits) -> WorkbookResult:
|
|
68
|
+
"""Read an XLS workbook into a grid of cell text.
|
|
69
|
+
|
|
70
|
+
Formulas give the result the file stored and are never evaluated. Merged
|
|
71
|
+
areas repeat the value of their top-left cell, clipped to the data. An
|
|
72
|
+
encrypted workbook is unsupported.
|
|
73
|
+
|
|
74
|
+
Raises:
|
|
75
|
+
LimitExceededError: if the rows, the columns or a cell go beyond the limits.
|
|
76
|
+
MissingExtraError: if the ``excel`` extra is not installed.
|
|
77
|
+
UnsupportedFileError: if the workbook cannot be read, or the sheet does
|
|
78
|
+
not exist (``sheet_not_found``).
|
|
79
|
+
"""
|
|
80
|
+
try:
|
|
81
|
+
import xlrd
|
|
82
|
+
except ImportError:
|
|
83
|
+
raise MissingExtraError("excel") from None
|
|
84
|
+
try:
|
|
85
|
+
book = xlrd.open_workbook(file_contents=data, formatting_info=True, logfile=io.StringIO())
|
|
86
|
+
except (MemoryError, RecursionError):
|
|
87
|
+
raise
|
|
88
|
+
except Exception:
|
|
89
|
+
raise UnsupportedFileError() from None
|
|
90
|
+
try:
|
|
91
|
+
titles: list[str] = list(book.sheet_names())
|
|
92
|
+
candidates = select_sheets(titles, sheet)
|
|
93
|
+
first_empty: Grid | None = None
|
|
94
|
+
for index in candidates:
|
|
95
|
+
worksheet = book.sheet_by_index(index)
|
|
96
|
+
try:
|
|
97
|
+
grid = _rows(xlrd, book, worksheet, limits)
|
|
98
|
+
if not grid.rows:
|
|
99
|
+
first_empty = first_empty or grid
|
|
100
|
+
continue
|
|
101
|
+
changed = fill_merged(grid.rows, _areas(worksheet))
|
|
102
|
+
except RowsiftError:
|
|
103
|
+
raise
|
|
104
|
+
except (MemoryError, RecursionError):
|
|
105
|
+
raise
|
|
106
|
+
except Exception:
|
|
107
|
+
raise UnsupportedFileError() from None
|
|
108
|
+
return WorkbookResult(grid, titles[index], tuple(titles), merged_filled=changed)
|
|
109
|
+
if first_empty is None:
|
|
110
|
+
return WorkbookResult(Grid([], False), None, tuple(titles), merged_filled=False)
|
|
111
|
+
return WorkbookResult(
|
|
112
|
+
first_empty, titles[candidates[0]], tuple(titles), merged_filled=False
|
|
113
|
+
)
|
|
114
|
+
finally:
|
|
115
|
+
book.release_resources()
|