rowsift 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rowsift/__init__.py +98 -0
- rowsift/clean/__init__.py +44 -0
- rowsift/clean/_common.py +30 -0
- rowsift/clean/_dates.py +283 -0
- rowsift/clean/_dispatch.py +49 -0
- rowsift/clean/_identifiers.py +161 -0
- rowsift/clean/_mask.py +117 -0
- rowsift/clean/_numbers.py +137 -0
- rowsift/clean/_options.py +75 -0
- rowsift/clean/_phone.py +202 -0
- rowsift/clean/_text.py +102 -0
- rowsift/errors.py +124 -0
- rowsift/map/__init__.py +24 -0
- rowsift/map/_fields.py +75 -0
- rowsift/map/_saved.py +132 -0
- rowsift/map/_score.py +178 -0
- rowsift/map/_suggest.py +168 -0
- rowsift/map/_types.py +169 -0
- rowsift/profile/__init__.py +17 -0
- rowsift/profile/_headers.py +44 -0
- rowsift/profile/_kinds.py +165 -0
- rowsift/profile/_options.py +69 -0
- rowsift/profile/_profile.py +119 -0
- rowsift/profile/_structure.py +157 -0
- rowsift/profile/_types.py +96 -0
- rowsift/py.typed +0 -0
- rowsift/read/__init__.py +6 -0
- rowsift/read/_cells.py +93 -0
- rowsift/read/_detect.py +124 -0
- rowsift/read/_grid.py +128 -0
- rowsift/read/_limits.py +44 -0
- rowsift/read/_read.py +99 -0
- rowsift/read/_text.py +250 -0
- rowsift/read/_workbook.py +36 -0
- rowsift/read/_xls.py +115 -0
- rowsift/read/_xlsx.py +121 -0
- rowsift/sift/__init__.py +19 -0
- rowsift/sift/_sift.py +165 -0
- rowsift/sift/_types.py +119 -0
- rowsift/types.py +378 -0
- rowsift-0.1.1.dist-info/METADATA +148 -0
- rowsift-0.1.1.dist-info/RECORD +45 -0
- rowsift-0.1.1.dist-info/WHEEL +4 -0
- rowsift-0.1.1.dist-info/licenses/LICENSE +177 -0
- rowsift-0.1.1.dist-info/licenses/NOTICE +4 -0
rowsift/types.py
ADDED
|
@@ -0,0 +1,378 @@
|
|
|
1
|
+
"""Value types and code enumerations shared by every layer.
|
|
2
|
+
|
|
3
|
+
Everything here is a frozen dataclass with slots or a ``StrEnum``, so values
|
|
4
|
+
are immutable and codes serialize as plain strings. This module imports
|
|
5
|
+
nothing else from the package.
|
|
6
|
+
|
|
7
|
+
Every explanation the library gives is a code plus structured data. There are
|
|
8
|
+
no sentences: the application writes the message. Removing or renaming a code
|
|
9
|
+
is a breaking change, adding one is a minor change.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import calendar
|
|
15
|
+
from collections.abc import Mapping
|
|
16
|
+
from dataclasses import dataclass, field
|
|
17
|
+
from enum import StrEnum
|
|
18
|
+
from types import MappingProxyType
|
|
19
|
+
from typing import Any, Generic, TypeVar
|
|
20
|
+
|
|
21
|
+
__all__ = [
|
|
22
|
+
"Change",
|
|
23
|
+
"ChangeCode",
|
|
24
|
+
"Cleaned",
|
|
25
|
+
"Confidence",
|
|
26
|
+
"DataValue",
|
|
27
|
+
"DayMonth",
|
|
28
|
+
"ErrorCode",
|
|
29
|
+
"FileFormat",
|
|
30
|
+
"Issue",
|
|
31
|
+
"IssueCode",
|
|
32
|
+
"Kind",
|
|
33
|
+
"LineType",
|
|
34
|
+
"ReadNote",
|
|
35
|
+
"ReadReport",
|
|
36
|
+
"Reason",
|
|
37
|
+
"ReasonCode",
|
|
38
|
+
"SkipCode",
|
|
39
|
+
"Table",
|
|
40
|
+
]
|
|
41
|
+
|
|
42
|
+
DataValue = str | int | float | bool
|
|
43
|
+
"""A scalar carried in the ``data`` of a code: a share, a count, a name."""
|
|
44
|
+
|
|
45
|
+
T = TypeVar("T")
|
|
46
|
+
|
|
47
|
+
_LEAP_YEAR = 2000
|
|
48
|
+
"""A leap year, so that 29 February is accepted as a day and month."""
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class Kind(StrEnum):
|
|
52
|
+
"""What a column or a value holds."""
|
|
53
|
+
|
|
54
|
+
TEXT = "text"
|
|
55
|
+
NAME = "name"
|
|
56
|
+
INTEGER = "integer"
|
|
57
|
+
DECIMAL = "decimal"
|
|
58
|
+
MONEY = "money"
|
|
59
|
+
BOOLEAN = "boolean"
|
|
60
|
+
DATE = "date"
|
|
61
|
+
DAY_MONTH = "day_month"
|
|
62
|
+
TIME = "time"
|
|
63
|
+
DATETIME = "datetime"
|
|
64
|
+
PHONE = "phone"
|
|
65
|
+
EMAIL = "email"
|
|
66
|
+
CPF = "cpf"
|
|
67
|
+
CNPJ = "cnpj"
|
|
68
|
+
POSTAL_CODE = "postal_code"
|
|
69
|
+
CATEGORY = "category"
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
class Confidence(StrEnum):
|
|
73
|
+
"""How sure a mapping suggestion is."""
|
|
74
|
+
|
|
75
|
+
HIGH = "high"
|
|
76
|
+
MEDIUM = "medium"
|
|
77
|
+
LOW = "low"
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
class FileFormat(StrEnum):
|
|
81
|
+
"""The format a file was detected as, from its content."""
|
|
82
|
+
|
|
83
|
+
CSV = "csv"
|
|
84
|
+
XLSX = "xlsx"
|
|
85
|
+
XLS = "xls"
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
class LineType(StrEnum):
|
|
89
|
+
"""Whether a phone number is a mobile or a landline."""
|
|
90
|
+
|
|
91
|
+
MOBILE = "mobile"
|
|
92
|
+
LANDLINE = "landline"
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
class ChangeCode(StrEnum):
|
|
96
|
+
"""A transformation that altered the meaning-bearing content of a value."""
|
|
97
|
+
|
|
98
|
+
NINTH_DIGIT_INSERTED = "ninth_digit_inserted"
|
|
99
|
+
MULTIPLE_VALUES = "multiple_values"
|
|
100
|
+
TWO_DIGIT_YEAR_EXPANDED = "two_digit_year_expanded"
|
|
101
|
+
YEAR_DISCARDED = "year_discarded"
|
|
102
|
+
LEADING_ZEROS_RESTORED = "leading_zeros_restored"
|
|
103
|
+
SEPARATOR_INFERRED = "separator_inferred"
|
|
104
|
+
CASE_CHANGED = "case_changed"
|
|
105
|
+
SERIAL_DATE_CONVERTED = "serial_date_converted"
|
|
106
|
+
TIME_DEFAULTED = "time_defaulted"
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
class IssueCode(StrEnum):
|
|
110
|
+
"""Why a value could not be normalized, or why a row was rejected."""
|
|
111
|
+
|
|
112
|
+
TOO_SHORT = "too_short"
|
|
113
|
+
TRUNCATED = "truncated"
|
|
114
|
+
INVALID_AREA_CODE = "invalid_area_code"
|
|
115
|
+
NOT_MOBILE = "not_mobile"
|
|
116
|
+
INVALID_DATE = "invalid_date"
|
|
117
|
+
NOT_A_DATE = "not_a_date"
|
|
118
|
+
NOT_A_NUMBER = "not_a_number"
|
|
119
|
+
INVALID_CHECK_DIGIT = "invalid_check_digit"
|
|
120
|
+
INVALID_EMAIL = "invalid_email"
|
|
121
|
+
NOT_IN_ALLOWED_VALUES = "not_in_allowed_values"
|
|
122
|
+
REQUIRED_MISSING = "required_missing"
|
|
123
|
+
INVALID_PHONE = "invalid_phone"
|
|
124
|
+
INVALID_LENGTH = "invalid_length"
|
|
125
|
+
NOT_A_TIME = "not_a_time"
|
|
126
|
+
NOT_A_BOOLEAN = "not_a_boolean"
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
class ReasonCode(StrEnum):
|
|
130
|
+
"""Why a column was, or was not, suggested for a field."""
|
|
131
|
+
|
|
132
|
+
HEADER_MATCHES_ALIAS = "header_matches_alias"
|
|
133
|
+
HEADER_CONTAINS_ALIAS = "header_contains_alias"
|
|
134
|
+
CONTENT_FITS_KIND = "content_fits_kind"
|
|
135
|
+
CONTENT_CONFLICTS_WITH_KIND = "content_conflicts_with_kind"
|
|
136
|
+
ONLY_CANDIDATE = "only_candidate"
|
|
137
|
+
REQUIRED_FIELD_UNMATCHED = "required_field_unmatched"
|
|
138
|
+
SAVED_MAPPING = "saved_mapping"
|
|
139
|
+
SAVED_HEADER_MISSING = "saved_header_missing"
|
|
140
|
+
SET_BY_CALLER = "set_by_caller"
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
class SkipCode(StrEnum):
|
|
144
|
+
"""Why a row is not a data row."""
|
|
145
|
+
|
|
146
|
+
EMPTY = "empty"
|
|
147
|
+
BEFORE_HEADER = "before_header"
|
|
148
|
+
REPEATED_HEADER = "repeated_header"
|
|
149
|
+
TOTAL_ROW = "total_row"
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
class ReadNote(StrEnum):
|
|
153
|
+
"""Something the reader did that the caller may want to know about."""
|
|
154
|
+
|
|
155
|
+
BOM_REMOVED = "bom_removed"
|
|
156
|
+
MERGED_CELLS_FILLED = "merged_cells_filled"
|
|
157
|
+
TRAILING_EMPTY_COLUMNS_DROPPED = "trailing_empty_columns_dropped"
|
|
158
|
+
UNTERMINATED_QUOTE = "unterminated_quote"
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
class ErrorCode(StrEnum):
|
|
162
|
+
"""The code carried by every exception the library raises on purpose."""
|
|
163
|
+
|
|
164
|
+
UNSUPPORTED_FILE = "unsupported_file"
|
|
165
|
+
SHEET_NOT_FOUND = "sheet_not_found"
|
|
166
|
+
PASSWORD_PROTECTED = "password_protected"
|
|
167
|
+
LIMIT_EXCEEDED = "limit_exceeded"
|
|
168
|
+
MISSING_EXTRA = "missing_extra"
|
|
169
|
+
DECODING_FAILED = "decoding_failed"
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
_DATA_TYPES = (str, int, float, bool)
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def _freeze(data: Mapping[str, DataValue]) -> Mapping[str, DataValue]:
|
|
176
|
+
"""A read-only copy of ``data``, so a code cannot be altered after the fact.
|
|
177
|
+
|
|
178
|
+
Raises:
|
|
179
|
+
TypeError: if a key is not text or a value is not a str, int, float or bool.
|
|
180
|
+
"""
|
|
181
|
+
copied = dict(data)
|
|
182
|
+
for key, value in copied.items():
|
|
183
|
+
if type(key) is not str:
|
|
184
|
+
raise TypeError("data keys must be str")
|
|
185
|
+
if not isinstance(value, _DATA_TYPES):
|
|
186
|
+
raise TypeError("data values must be str, int, float or bool")
|
|
187
|
+
return MappingProxyType(copied)
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def _hash_data(code: StrEnum, data: Mapping[str, DataValue]) -> int:
|
|
191
|
+
return hash((code, frozenset(data.items())))
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
@dataclass(frozen=True, slots=True)
|
|
195
|
+
class Change:
|
|
196
|
+
"""A transformation applied to a value, as a code plus its data.
|
|
197
|
+
|
|
198
|
+
``data`` is copied into a read-only mapping, and the value is hashable.
|
|
199
|
+
Data values compare as Python compares numbers, so ``1``, ``1.0`` and
|
|
200
|
+
``True`` are equal.
|
|
201
|
+
|
|
202
|
+
Raises:
|
|
203
|
+
ValueError: if ``code`` is not a ``ChangeCode``.
|
|
204
|
+
TypeError: if ``data`` holds a key that is not text or a non-scalar value.
|
|
205
|
+
"""
|
|
206
|
+
|
|
207
|
+
code: ChangeCode
|
|
208
|
+
data: Mapping[str, DataValue] = field(default_factory=dict)
|
|
209
|
+
|
|
210
|
+
def __post_init__(self) -> None:
|
|
211
|
+
object.__setattr__(self, "code", ChangeCode(self.code))
|
|
212
|
+
object.__setattr__(self, "data", _freeze(self.data))
|
|
213
|
+
|
|
214
|
+
def __hash__(self) -> int:
|
|
215
|
+
return _hash_data(self.code, self.data)
|
|
216
|
+
|
|
217
|
+
def __reduce__(self) -> tuple[Any, tuple[Any, ...]]:
|
|
218
|
+
return (type(self), (self.code, dict(self.data)))
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
@dataclass(frozen=True, slots=True)
|
|
222
|
+
class Issue:
|
|
223
|
+
"""The reason a value failed, as a code plus its data.
|
|
224
|
+
|
|
225
|
+
``data`` is copied into a read-only mapping, and the value is hashable.
|
|
226
|
+
Data values compare as Python compares numbers, so ``1``, ``1.0`` and
|
|
227
|
+
``True`` are equal.
|
|
228
|
+
|
|
229
|
+
Raises:
|
|
230
|
+
ValueError: if ``code`` is not an ``IssueCode``.
|
|
231
|
+
TypeError: if ``data`` holds a key that is not text or a non-scalar value.
|
|
232
|
+
"""
|
|
233
|
+
|
|
234
|
+
code: IssueCode
|
|
235
|
+
data: Mapping[str, DataValue] = field(default_factory=dict)
|
|
236
|
+
|
|
237
|
+
def __post_init__(self) -> None:
|
|
238
|
+
object.__setattr__(self, "code", IssueCode(self.code))
|
|
239
|
+
object.__setattr__(self, "data", _freeze(self.data))
|
|
240
|
+
|
|
241
|
+
def __hash__(self) -> int:
|
|
242
|
+
return _hash_data(self.code, self.data)
|
|
243
|
+
|
|
244
|
+
def __reduce__(self) -> tuple[Any, tuple[Any, ...]]:
|
|
245
|
+
return (type(self), (self.code, dict(self.data)))
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
@dataclass(frozen=True, slots=True)
|
|
249
|
+
class Reason:
|
|
250
|
+
"""Why a column was, or was not, suggested for a field, as a code plus its data.
|
|
251
|
+
|
|
252
|
+
``data`` is copied into a read-only mapping, and the value is hashable.
|
|
253
|
+
Data values compare as Python compares numbers, so ``1``, ``1.0`` and
|
|
254
|
+
``True`` are equal.
|
|
255
|
+
|
|
256
|
+
Raises:
|
|
257
|
+
ValueError: if ``code`` is not a ``ReasonCode``.
|
|
258
|
+
TypeError: if ``data`` holds a key that is not text or a non-scalar value.
|
|
259
|
+
"""
|
|
260
|
+
|
|
261
|
+
code: ReasonCode
|
|
262
|
+
data: Mapping[str, DataValue] = field(default_factory=dict)
|
|
263
|
+
|
|
264
|
+
def __post_init__(self) -> None:
|
|
265
|
+
object.__setattr__(self, "code", ReasonCode(self.code))
|
|
266
|
+
object.__setattr__(self, "data", _freeze(self.data))
|
|
267
|
+
|
|
268
|
+
def __hash__(self) -> int:
|
|
269
|
+
return _hash_data(self.code, self.data)
|
|
270
|
+
|
|
271
|
+
def __reduce__(self) -> tuple[Any, tuple[Any, ...]]:
|
|
272
|
+
return (type(self), (self.code, dict(self.data)))
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
@dataclass(frozen=True, slots=True)
|
|
276
|
+
class Cleaned(Generic[T]): # PEP 695 syntax needs Python 3.12, the floor is 3.11
|
|
277
|
+
"""The outcome of cleaning one value.
|
|
278
|
+
|
|
279
|
+
A non-empty input gives either a ``value`` or an ``issue``. An empty input
|
|
280
|
+
gives neither, and no changes. ``changes`` lists what was altered on the way.
|
|
281
|
+
|
|
282
|
+
Raises:
|
|
283
|
+
ValueError: if both ``value`` and ``issue`` are set, or if there are
|
|
284
|
+
changes and neither is set.
|
|
285
|
+
"""
|
|
286
|
+
|
|
287
|
+
value: T | None = None
|
|
288
|
+
changes: tuple[Change, ...] = ()
|
|
289
|
+
issue: Issue | None = None
|
|
290
|
+
|
|
291
|
+
def __post_init__(self) -> None:
|
|
292
|
+
if self.value is not None and self.issue is not None:
|
|
293
|
+
raise ValueError("a Cleaned carries a value or an issue, never both")
|
|
294
|
+
if self.value is None and self.issue is None and self.changes:
|
|
295
|
+
raise ValueError("changes need a value or an issue")
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
@dataclass(frozen=True, slots=True)
|
|
299
|
+
class DayMonth:
|
|
300
|
+
"""A day and a month with no year, such as a birthday.
|
|
301
|
+
|
|
302
|
+
29 February is valid. No year is ever invented.
|
|
303
|
+
|
|
304
|
+
Raises:
|
|
305
|
+
TypeError: if the day or the month is not an ``int`` (a ``bool`` is not).
|
|
306
|
+
ValueError: if the day does not exist in the month.
|
|
307
|
+
"""
|
|
308
|
+
|
|
309
|
+
day: int
|
|
310
|
+
month: int
|
|
311
|
+
|
|
312
|
+
def __post_init__(self) -> None:
|
|
313
|
+
for part in (self.day, self.month):
|
|
314
|
+
if type(part) is not int:
|
|
315
|
+
raise TypeError("day and month must be int")
|
|
316
|
+
if not 1 <= self.month <= 12:
|
|
317
|
+
raise ValueError("month must be between 1 and 12")
|
|
318
|
+
if not 1 <= self.day <= calendar.monthrange(_LEAP_YEAR, self.month)[1]:
|
|
319
|
+
raise ValueError("day does not exist in the month")
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
@dataclass(frozen=True, slots=True, kw_only=True)
|
|
323
|
+
class ReadReport:
|
|
324
|
+
"""How a file was read. Every field is passed by keyword.
|
|
325
|
+
|
|
326
|
+
``encoding`` and ``delimiter`` are ``None`` for workbooks. ``sheet`` is
|
|
327
|
+
``None`` for delimited text. ``sheets`` lists the names available.
|
|
328
|
+
|
|
329
|
+
Raises:
|
|
330
|
+
TypeError: if a count is not an ``int`` (a ``bool`` is not).
|
|
331
|
+
ValueError: if a count is negative.
|
|
332
|
+
"""
|
|
333
|
+
|
|
334
|
+
format: FileFormat
|
|
335
|
+
encoding: str | None = None
|
|
336
|
+
delimiter: str | None = None
|
|
337
|
+
sheet: str | None = None
|
|
338
|
+
sheets: tuple[str, ...] = ()
|
|
339
|
+
row_count: int
|
|
340
|
+
column_count: int
|
|
341
|
+
notes: tuple[ReadNote, ...] = ()
|
|
342
|
+
|
|
343
|
+
def __post_init__(self) -> None:
|
|
344
|
+
for count in (self.row_count, self.column_count):
|
|
345
|
+
if type(count) is not int:
|
|
346
|
+
raise TypeError("row_count and column_count must be int")
|
|
347
|
+
if count < 0:
|
|
348
|
+
raise ValueError("row_count and column_count must not be negative")
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
@dataclass(frozen=True, slots=True)
|
|
352
|
+
class Table:
|
|
353
|
+
"""A grid of text cells and the report of how it was read.
|
|
354
|
+
|
|
355
|
+
Every cell is text, with an empty string for an empty cell. Every row has
|
|
356
|
+
``report.column_count`` cells and there are ``report.row_count`` rows: the
|
|
357
|
+
reader pads short rows.
|
|
358
|
+
|
|
359
|
+
Raises:
|
|
360
|
+
TypeError: if ``rows`` is not a tuple of tuples of ``str``.
|
|
361
|
+
ValueError: if the rows are not rectangular or disagree with the report.
|
|
362
|
+
"""
|
|
363
|
+
|
|
364
|
+
rows: tuple[tuple[str, ...], ...]
|
|
365
|
+
report: ReadReport
|
|
366
|
+
|
|
367
|
+
def __post_init__(self) -> None:
|
|
368
|
+
if type(self.rows) is not tuple or any(
|
|
369
|
+
type(row) is not tuple or any(type(cell) is not str for cell in row)
|
|
370
|
+
for row in self.rows
|
|
371
|
+
):
|
|
372
|
+
raise TypeError("rows must be a tuple of tuples of str")
|
|
373
|
+
if len(self.rows) != self.report.row_count:
|
|
374
|
+
raise ValueError("the number of rows must equal report.row_count")
|
|
375
|
+
if any(len(row) != self.report.column_count for row in self.rows):
|
|
376
|
+
raise ValueError("every row must have report.column_count cells")
|
|
377
|
+
if not self.rows and self.report.column_count:
|
|
378
|
+
raise ValueError("a table with no rows has no columns")
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: rowsift
|
|
3
|
+
Version: 0.1.1
|
|
4
|
+
Summary: Read spreadsheets written by people: find the table, infer what each column holds, clean the values and map them to the fields you declare.
|
|
5
|
+
Project-URL: Repository, https://github.com/msantino/rowsift
|
|
6
|
+
Project-URL: Issues, https://github.com/msantino/rowsift/issues
|
|
7
|
+
Project-URL: Documentation, https://github.com/msantino/rowsift/tree/main/docs
|
|
8
|
+
Author: Marcelo Santino
|
|
9
|
+
License-Expression: Apache-2.0
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
License-File: NOTICE
|
|
12
|
+
Keywords: column-mapping,csv,data-cleaning,excel,import,spreadsheet
|
|
13
|
+
Classifier: Development Status :: 3 - Alpha
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Operating System :: OS Independent
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
21
|
+
Classifier: Topic :: Office/Business :: Financial :: Spreadsheet
|
|
22
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
23
|
+
Classifier: Typing :: Typed
|
|
24
|
+
Requires-Python: >=3.11
|
|
25
|
+
Provides-Extra: all
|
|
26
|
+
Requires-Dist: openpyxl<4,>=3.1; extra == 'all'
|
|
27
|
+
Requires-Dist: phonenumbers<10,>=8.13; extra == 'all'
|
|
28
|
+
Requires-Dist: xlrd<3,>=2.0.1; extra == 'all'
|
|
29
|
+
Provides-Extra: excel
|
|
30
|
+
Requires-Dist: openpyxl<4,>=3.1; extra == 'excel'
|
|
31
|
+
Requires-Dist: xlrd<3,>=2.0.1; extra == 'excel'
|
|
32
|
+
Provides-Extra: phone
|
|
33
|
+
Requires-Dist: phonenumbers<10,>=8.13; extra == 'phone'
|
|
34
|
+
Description-Content-Type: text/markdown
|
|
35
|
+
|
|
36
|
+
# rowsift
|
|
37
|
+
|
|
38
|
+
rowsift reads spreadsheets written by people and turns them into typed, clean rows mapped to the fields your application declares.
|
|
39
|
+
|
|
40
|
+
A file exported by a person is rarely a clean table. The header sits on the fourth row under a title and a blank line. The encoding is whatever the spreadsheet program chose. Phone numbers come in five formats, dates have no year, and the column you need is called `Cel 2`. rowsift handles that part of an import, on the server, and reports every decision it made so the application can show it to the person who sent the file.
|
|
41
|
+
|
|
42
|
+
## What it does
|
|
43
|
+
|
|
44
|
+
The work is split into four steps. Each one can be used on its own.
|
|
45
|
+
|
|
46
|
+
| Step | Input | Output |
|
|
47
|
+
|---|---|---|
|
|
48
|
+
| **Read** | bytes of a CSV, XLSX or XLS file | a grid of text cells, plus how it was read: format, encoding, delimiter, sheet |
|
|
49
|
+
| **Profile** | the grid | where the header is, which rows are data, and what each column holds: text, number, money, date, day and month, phone, e-mail, tax id, postal code, or a small set of repeated values |
|
|
50
|
+
| **Clean** | one value and the kind it should be | the normalized value, or the reason it could not be normalized |
|
|
51
|
+
| **Map** | the profile and the fields you declare | a suggested column for each field, with a confidence level and the reason for it |
|
|
52
|
+
|
|
53
|
+
Applying a mapping to a file gives typed rows, the rows that were rejected with the reason for each, and counts of what was changed.
|
|
54
|
+
|
|
55
|
+
## What it does not do
|
|
56
|
+
|
|
57
|
+
- It does not store anything. Models, jobs, uploads and screens belong to the application.
|
|
58
|
+
- It does not guess silently. A phone number cut short by an export is reported as cut short, never completed.
|
|
59
|
+
- It does not write user-facing text. Reasons are returned as codes with their data, so the application writes the message in its own language and tone.
|
|
60
|
+
- It does not depend on a web framework.
|
|
61
|
+
|
|
62
|
+
## Formats and conventions
|
|
63
|
+
|
|
64
|
+
Brazilian conventions are supported from the first version, because that is where the library started: day-first dates, decimal comma, CPF and CNPJ check digits, CEP, and mobile numbers written with or without the ninth digit. Region-specific behavior is selected through options, and the defaults are documented with each function.
|
|
65
|
+
|
|
66
|
+
## Example
|
|
67
|
+
|
|
68
|
+
```python
|
|
69
|
+
import rowsift
|
|
70
|
+
|
|
71
|
+
data = (
|
|
72
|
+
"Cadastro de clientes\n"
|
|
73
|
+
"\n"
|
|
74
|
+
"Nome;Celular;Nascimento\n"
|
|
75
|
+
"ana souza;(11) 91234-5678;03/04/1990\n"
|
|
76
|
+
"bruno lima;(11) 9123;15/08/1985\n"
|
|
77
|
+
).encode("utf-8")
|
|
78
|
+
|
|
79
|
+
table = rowsift.read(data)
|
|
80
|
+
profile = rowsift.profile(table)
|
|
81
|
+
|
|
82
|
+
fields = [
|
|
83
|
+
rowsift.Field("name", rowsift.Kind.NAME, required=True, aliases=("Nome",)),
|
|
84
|
+
rowsift.Field("phone", rowsift.Kind.PHONE, aliases=("Celular",)),
|
|
85
|
+
rowsift.Field("birthday", rowsift.Kind.DATE, aliases=("Nascimento",)),
|
|
86
|
+
]
|
|
87
|
+
mapping = rowsift.suggest(profile, fields)
|
|
88
|
+
result = rowsift.sift(table, profile, mapping, fields)
|
|
89
|
+
|
|
90
|
+
assert result.summary.kept == 2
|
|
91
|
+
assert result.rows[0].values["name"] == "Ana Souza"
|
|
92
|
+
assert result.rows[0].values["phone"] == "+5511912345678"
|
|
93
|
+
# A phone number cut short is reported, not completed.
|
|
94
|
+
assert result.rows[1].values["phone"] is None
|
|
95
|
+
assert result.rows[1].issues["phone"].code == "too_short"
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
The header was found under a title and a blank line, each column was matched to a field, and the second phone number came back with a reason code instead of a guess. The [quick start](https://github.com/msantino/rowsift/blob/main/docs/quick-start.md) walks through the same file step by step.
|
|
99
|
+
|
|
100
|
+
## Documentation
|
|
101
|
+
|
|
102
|
+
- [Quick start](https://github.com/msantino/rowsift/blob/main/docs/quick-start.md)
|
|
103
|
+
- [Reading](https://github.com/msantino/rowsift/blob/main/docs/reading.md): formats, encodings, limits, errors
|
|
104
|
+
- [Profiling](https://github.com/msantino/rowsift/blob/main/docs/profiling.md): the header, skipped rows, kinds
|
|
105
|
+
- [Cleaning](https://github.com/msantino/rowsift/blob/main/docs/cleaning.md): one function per kind, options, masking
|
|
106
|
+
- [Mapping](https://github.com/msantino/rowsift/blob/main/docs/mapping.md): suggestions, confidence, saved mappings
|
|
107
|
+
- [Sifting](https://github.com/msantino/rowsift/blob/main/docs/sifting.md): rows, rejections, counts
|
|
108
|
+
- [Codes](https://github.com/msantino/rowsift/blob/main/docs/codes.md): every code and what it means
|
|
109
|
+
|
|
110
|
+
## Status
|
|
111
|
+
|
|
112
|
+
Early development, version `0.1.1`. The API can still change between minor versions. Before `1.0.0`, a breaking change raises the minor version. Removing or renaming a public name or a code is a breaking change, and adding a code is a minor change.
|
|
113
|
+
|
|
114
|
+
| Step | State |
|
|
115
|
+
|---|---|
|
|
116
|
+
| Read | available: `rowsift.read(...)` turns CSV, XLSX and XLS bytes into a grid of text cells and a report, under explicit limits. XLSX and XLS need the `excel` extra |
|
|
117
|
+
| Profile | available: `rowsift.profile(table)` finds the header, skips rows that are not data with a code, and gives each column a kind with its share, counts, the low-cardinality signal and masked samples |
|
|
118
|
+
| Clean | available: one cleaner per kind, `rowsift.clean.value` to dispatch by kind, masking functions. Phone numbers of a region other than Brazil need the `phone` extra |
|
|
119
|
+
| Map | available: `rowsift.suggest(profile, fields)` suggests a column for each declared `Field`, with a confidence level and reason codes. `SavedMapping` and `rowsift.apply_saved` carry a person's choices to the next file without guessing |
|
|
120
|
+
| Sift | available: `rowsift.sift(table, profile, mapping, fields)` returns the kept rows with their cleaned values and changes, the rejected rows with the field and issue code, and counts per field. It has no side effect, so it serves as a preview and as the final pass |
|
|
121
|
+
|
|
122
|
+
## Requirements
|
|
123
|
+
|
|
124
|
+
Python 3.11 to 3.14. The core has no runtime dependency.
|
|
125
|
+
|
|
126
|
+
```bash
|
|
127
|
+
pip install rowsift # core: CSV and delimited text
|
|
128
|
+
pip install "rowsift[excel]" # adds XLSX and XLS
|
|
129
|
+
pip install "rowsift[phone]" # adds phone numbers outside Brazil
|
|
130
|
+
pip install "rowsift[all]" # every extra
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
A function that needs an extra that is not installed raises `rowsift.MissingExtraError`, which names the extra.
|
|
134
|
+
|
|
135
|
+
## Development
|
|
136
|
+
|
|
137
|
+
You need [uv](https://docs.astral.sh/uv/) and [just](https://just.systems/).
|
|
138
|
+
|
|
139
|
+
```bash
|
|
140
|
+
just setup # install the project and its tools into .venv
|
|
141
|
+
just verify # lint, type check, tests, text check and build: exactly what CI runs
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
See [CONTRIBUTING.md](CONTRIBUTING.md) before opening a pull request, and [SECURITY.md](SECURITY.md) to report a vulnerability.
|
|
145
|
+
|
|
146
|
+
## License
|
|
147
|
+
|
|
148
|
+
[Apache License 2.0](LICENSE). You can use, modify and distribute the library, including in commercial and closed-source software, as long as you keep the license and the notices.
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
rowsift/__init__.py,sha256=LbR9Ao8QmyjZrZaT4XP3ZgYjCf-gMY8kS57p_hFYFhg,1810
|
|
2
|
+
rowsift/errors.py,sha256=7MegU_kdTZu6Wei_ZxPdykGHODnd9OO0BpPGICN10qg,3818
|
|
3
|
+
rowsift/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
4
|
+
rowsift/types.py,sha256=4BhZvCaBSLjUOTRkJwvKdUNnUjVrY5D-GTBA0Zz4Cow,11835
|
|
5
|
+
rowsift/clean/__init__.py,sha256=ZHT0-v2arLxQvA6-1LD_ZbSSuplLtPMH3J5Sl3yzegU,1222
|
|
6
|
+
rowsift/clean/_common.py,sha256=ZggIXlti1rxUTugViZPaPSKRRMNzTQQwxJSVD9KWTcc,970
|
|
7
|
+
rowsift/clean/_dates.py,sha256=m3dHZsDRs88BHLU8tkJH05RK4GBlXkRfbwd_IS28SyY,11063
|
|
8
|
+
rowsift/clean/_dispatch.py,sha256=OJhQnjTCIuiytXzrsigg41ZoR1xyyg8jcfRBf24htYY,1548
|
|
9
|
+
rowsift/clean/_identifiers.py,sha256=m-0fSM8UOGE9LirZWFX0Y6qcEAjiFQYGM4kr699UQhw,6575
|
|
10
|
+
rowsift/clean/_mask.py,sha256=BHi-cyGv67kBAeHHqTak_L-gKjW3REY6nRzVGEBBgHQ,3999
|
|
11
|
+
rowsift/clean/_numbers.py,sha256=Eyp9iK_eH1vtyMgjn0kEtpG92NOhCSYZpwImOAmHa5M,5701
|
|
12
|
+
rowsift/clean/_options.py,sha256=GqS2qXONT_PwYWBTvWaE76OpBIAG8xGbA2mK7_X_jeM,3302
|
|
13
|
+
rowsift/clean/_phone.py,sha256=htPK7M4hyWLFkayQxs1l8dJ36TcmGmSqSSfvzSdE7D0,8949
|
|
14
|
+
rowsift/clean/_text.py,sha256=caKf_d4cBqLtWkwOnIwvz1OCJBePCZCZ440kYDo0qfA,3873
|
|
15
|
+
rowsift/map/__init__.py,sha256=VYE95lhWzZulgll4KJVUBl6p_SbBhTXGe521m2Mr1yo,823
|
|
16
|
+
rowsift/map/_fields.py,sha256=eG4qXKMyKxNwpEwbwoXyuB7X0UCrbKdY9KNRj3Q23x8,3038
|
|
17
|
+
rowsift/map/_saved.py,sha256=3KU9tK45OoAIrA5dBf-SR1V3MaDZ9-TCQVbucjDdtQc,5978
|
|
18
|
+
rowsift/map/_score.py,sha256=-jQzIyWqChKSV6y0PxWTQNWAyOHKUvRWO1PZUL5zHno,6476
|
|
19
|
+
rowsift/map/_suggest.py,sha256=11xtPzXd96HJ_Mg6aUQTdOdEW3bLcOonfCEUtBC0f1k,6870
|
|
20
|
+
rowsift/map/_types.py,sha256=JMDZ1t4GpX1QtWsH2tWQQtDPotEygDFGZRIm1JjlVjY,6897
|
|
21
|
+
rowsift/profile/__init__.py,sha256=MDN-CMjNkiFFqFXQD9teZnmEi1IsNt3aLkp7xY0y2xY,493
|
|
22
|
+
rowsift/profile/_headers.py,sha256=UwVQdIXJNZOfonz7_9wetum6F09csKsY03Cm7Fz3Z90,1536
|
|
23
|
+
rowsift/profile/_kinds.py,sha256=NFmMhFzZ7lPdUlReoZEUIIZut-xyqWS5n_o2aX-AghU,7246
|
|
24
|
+
rowsift/profile/_options.py,sha256=a8tUsq9EtUbQD2XqO045pwKey9k_oiIdagf-Yo3yvfU,2892
|
|
25
|
+
rowsift/profile/_profile.py,sha256=D3CRD_lku5_LZz_asvkkyjUwq0jaOYcfSZvRBSPBOuw,5055
|
|
26
|
+
rowsift/profile/_structure.py,sha256=9pyRqWRIS2ZJZmcDIV1iT5vj4SdEgMxod9Keb30pYpM,6400
|
|
27
|
+
rowsift/profile/_types.py,sha256=dbuAPzTV48D3ruo2EFYzTDHhRtntHtyjqrWok14m3tU,3145
|
|
28
|
+
rowsift/read/__init__.py,sha256=0l-bvqaTlwrcSFhOF8CttjAtO7TC5AqMJp6aDMZW0BQ,209
|
|
29
|
+
rowsift/read/_cells.py,sha256=Z7gBCbNEv_dUl02ds6lHfZoY-Il3_F3xxSoxhNRTmGg,3957
|
|
30
|
+
rowsift/read/_detect.py,sha256=rWazRjiqUCu6WTOUwhQo685Vk6AB7IlUAJI6fuk90e8,4716
|
|
31
|
+
rowsift/read/_grid.py,sha256=mpw3Z6sjTV8GncNw0-QIwnFffaNiodGlInizas6pQr4,4888
|
|
32
|
+
rowsift/read/_limits.py,sha256=avPLT9XY2yLgLyeEdYYASllKtVlo9sxoKz9BR8vWftk,1553
|
|
33
|
+
rowsift/read/_read.py,sha256=y50UuxkW2M3uqpNLffb0TcgKQVuMJv09YWCYwTtwrdE,4141
|
|
34
|
+
rowsift/read/_text.py,sha256=LE5lEeRjOl_46Ex6oSOfwHGqGAaICt8WT0E4m1OrsQ0,9026
|
|
35
|
+
rowsift/read/_workbook.py,sha256=MGwuu_pU29zt5XU_JK_N9f3xkHw5JWEjrH-lYLvIk9w,1026
|
|
36
|
+
rowsift/read/_xls.py,sha256=1oLONmYKA8bWY3ivSSvMXOgtAK1bONdTBpU9J39bzDQ,4550
|
|
37
|
+
rowsift/read/_xlsx.py,sha256=js639v1hKN_UlfXi3_JCupSbrZT-eocj4MtiXm0VRkA,4911
|
|
38
|
+
rowsift/sift/__init__.py,sha256=76xj0msvNimabqaYV-OyRKuf3Ku6gwHU9CKoQ1Tg0DI,606
|
|
39
|
+
rowsift/sift/_sift.py,sha256=IZ5Wdo1T7YF5MRo-7gvzyEGVztdKnbr1Zwj9F6Kk4yA,6724
|
|
40
|
+
rowsift/sift/_types.py,sha256=jU-wUhrXTszj0C_V-qa8-5zwr9osa3iem_XrnRlWouI,3871
|
|
41
|
+
rowsift-0.1.1.dist-info/METADATA,sha256=8az2FDIoI_4t6WAdvv5mxlRMOr6Iq53jInC2sqXatCU,8111
|
|
42
|
+
rowsift-0.1.1.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
|
|
43
|
+
rowsift-0.1.1.dist-info/licenses/LICENSE,sha256=DVQuDIgE45qn836wDaWnYhSdxoLXgpRRKH4RuTjpRZQ,10174
|
|
44
|
+
rowsift-0.1.1.dist-info/licenses/NOTICE,sha256=LYkFZ4Dj_tMy4aUPR9cGFNhjKVv4G4eOF7GdHPdl8m0,104
|
|
45
|
+
rowsift-0.1.1.dist-info/RECORD,,
|