dataspector 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dataspector/__init__.py +64 -0
- dataspector/__main__.py +10 -0
- dataspector/_connectors.py +260 -0
- dataspector/_reporting.py +480 -0
- dataspector/checkers/__init__.py +85 -0
- dataspector/checkers/record.py +671 -0
- dataspector/checkers/source.py +344 -0
- dataspector/checkers/utils.py +342 -0
- dataspector/cli.py +207 -0
- dataspector/coboller/__init__.py +123 -0
- dataspector/coboller/codecs.py +841 -0
- dataspector/coboller/constants.py +324 -0
- dataspector/coboller/copybook.py +1448 -0
- dataspector/coboller/parser.py +201 -0
- dataspector/core.py +740 -0
- dataspector/py.typed +0 -0
- dataspector/settings.py +101 -0
- dataspector-1.0.0.dist-info/METADATA +181 -0
- dataspector-1.0.0.dist-info/RECORD +22 -0
- dataspector-1.0.0.dist-info/WHEEL +4 -0
- dataspector-1.0.0.dist-info/entry_points.txt +3 -0
- dataspector-1.0.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,671 @@
|
|
|
1
|
+
"""Record-level integrity check functions operating on DataFrame chunks (alphabetical)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import pathlib
|
|
6
|
+
from collections.abc import Sequence
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
import pandas as pd
|
|
10
|
+
import pandera.pandas as pa
|
|
11
|
+
from pandera.errors import SchemaError, SchemaErrors
|
|
12
|
+
|
|
13
|
+
from dataspector.checkers.utils import (
|
|
14
|
+
CheckOutcome,
|
|
15
|
+
check_series_between,
|
|
16
|
+
missing_field_note,
|
|
17
|
+
parse_date_bound,
|
|
18
|
+
requires_all_data,
|
|
19
|
+
resolve_pandera_schema,
|
|
20
|
+
split_constraint,
|
|
21
|
+
truncate_lines,
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@requires_all_data
|
|
26
|
+
def record_all_data_not_null(
|
|
27
|
+
data: pd.DataFrame,
|
|
28
|
+
_last_chunk: bool = True,
|
|
29
|
+
) -> CheckOutcome:
|
|
30
|
+
"""Check that no cell anywhere in *data* is null/NaN.
|
|
31
|
+
|
|
32
|
+
Parameters
|
|
33
|
+
----------
|
|
34
|
+
data
|
|
35
|
+
Chunk of records.
|
|
36
|
+
_last_chunk
|
|
37
|
+
``True`` on the final chunk.
|
|
38
|
+
|
|
39
|
+
Returns
|
|
40
|
+
-------
|
|
41
|
+
CheckOutcome
|
|
42
|
+
``False`` immediately on any null; ``None`` while streaming; ``True`` at end.
|
|
43
|
+
"""
|
|
44
|
+
if bool(data.isnull().any().any()):
|
|
45
|
+
bad_rows = truncate_lines(data.index[data.isnull().any(axis=1)].tolist())
|
|
46
|
+
return CheckOutcome(passed=False, notes=f"Field at line(s) {bad_rows} is null.")
|
|
47
|
+
return CheckOutcome(passed=True if _last_chunk else None)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def record_contains(data: pd.DataFrame, field: str) -> CheckOutcome:
|
|
51
|
+
"""Check that *data* contains column *field*.
|
|
52
|
+
|
|
53
|
+
Parameters
|
|
54
|
+
----------
|
|
55
|
+
data
|
|
56
|
+
Chunk of records.
|
|
57
|
+
field
|
|
58
|
+
Expected column name.
|
|
59
|
+
|
|
60
|
+
Returns
|
|
61
|
+
-------
|
|
62
|
+
CheckOutcome
|
|
63
|
+
Outcome listing available columns on failure.
|
|
64
|
+
"""
|
|
65
|
+
present = field in data
|
|
66
|
+
notes = (
|
|
67
|
+
f"Source has fields: {list(data.columns)}"
|
|
68
|
+
if not present and hasattr(data, "columns")
|
|
69
|
+
else None
|
|
70
|
+
)
|
|
71
|
+
return CheckOutcome(passed=present, notes=notes, kwargs={"field": field})
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def record_contains_at_least(data: pd.DataFrame, fields: Sequence[str]) -> CheckOutcome:
|
|
75
|
+
"""Check that *data* contains every column listed in *fields*.
|
|
76
|
+
|
|
77
|
+
Parameters
|
|
78
|
+
----------
|
|
79
|
+
data
|
|
80
|
+
Chunk of records.
|
|
81
|
+
fields
|
|
82
|
+
Required column names.
|
|
83
|
+
|
|
84
|
+
Returns
|
|
85
|
+
-------
|
|
86
|
+
CheckOutcome
|
|
87
|
+
Outcome listing missing columns on failure.
|
|
88
|
+
"""
|
|
89
|
+
at_least = all(f in data for f in fields)
|
|
90
|
+
notes = None if at_least else f"Source is missing: {sorted(set(fields) - set(data.columns))}"
|
|
91
|
+
return CheckOutcome(passed=at_least, notes=notes, kwargs={"fields": list(fields)})
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
@requires_all_data
|
|
95
|
+
def record_count(
|
|
96
|
+
data: pd.DataFrame,
|
|
97
|
+
nrows: int,
|
|
98
|
+
_previous: int = 0,
|
|
99
|
+
_last_chunk: bool = True,
|
|
100
|
+
) -> CheckOutcome:
|
|
101
|
+
"""Check that the dataset has exactly *nrows* rows across all streamed chunks.
|
|
102
|
+
|
|
103
|
+
Parameters
|
|
104
|
+
----------
|
|
105
|
+
data
|
|
106
|
+
Chunk of records.
|
|
107
|
+
nrows
|
|
108
|
+
Expected total row count.
|
|
109
|
+
_previous
|
|
110
|
+
Accumulated row count from prior chunks.
|
|
111
|
+
_last_chunk
|
|
112
|
+
``True`` on the final chunk.
|
|
113
|
+
|
|
114
|
+
Returns
|
|
115
|
+
-------
|
|
116
|
+
CheckOutcome
|
|
117
|
+
``passed=None`` while streaming; ``True``/``False`` on the final chunk.
|
|
118
|
+
"""
|
|
119
|
+
nrecords = len(data) + _previous
|
|
120
|
+
passed: bool | None = None
|
|
121
|
+
notes: str | None = None
|
|
122
|
+
if _last_chunk:
|
|
123
|
+
passed = nrecords == nrows
|
|
124
|
+
if not passed:
|
|
125
|
+
notes = f"Found {nrecords} records"
|
|
126
|
+
return CheckOutcome(passed=passed, notes=notes, kwargs={"nrows": nrows, "_previous": nrecords})
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
@requires_all_data
|
|
130
|
+
def record_count_greater_than(
|
|
131
|
+
data: pd.DataFrame,
|
|
132
|
+
nrows: int,
|
|
133
|
+
_previous: int = 0,
|
|
134
|
+
_last_chunk: bool = True,
|
|
135
|
+
) -> CheckOutcome:
|
|
136
|
+
"""Check that the dataset has strictly more than *nrows* rows.
|
|
137
|
+
|
|
138
|
+
Parameters
|
|
139
|
+
----------
|
|
140
|
+
data
|
|
141
|
+
Chunk of records.
|
|
142
|
+
nrows
|
|
143
|
+
Exclusive minimum row count.
|
|
144
|
+
_previous
|
|
145
|
+
Accumulated row count from prior chunks.
|
|
146
|
+
_last_chunk
|
|
147
|
+
``True`` on the final chunk.
|
|
148
|
+
|
|
149
|
+
Returns
|
|
150
|
+
-------
|
|
151
|
+
CheckOutcome
|
|
152
|
+
``True`` as soon as *nrows* is exceeded; ``None`` while streaming; ``False`` at end.
|
|
153
|
+
"""
|
|
154
|
+
nrecords = len(data) + _previous
|
|
155
|
+
notes: str | None = None
|
|
156
|
+
passed: bool | None
|
|
157
|
+
if nrecords > nrows:
|
|
158
|
+
passed = True
|
|
159
|
+
elif _last_chunk:
|
|
160
|
+
passed = False
|
|
161
|
+
notes = f"Found {nrecords} records"
|
|
162
|
+
else:
|
|
163
|
+
passed = None
|
|
164
|
+
return CheckOutcome(passed=passed, notes=notes, kwargs={"nrows": nrows, "_previous": nrecords})
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
@requires_all_data
|
|
168
|
+
def record_field_contains(
|
|
169
|
+
data: pd.DataFrame,
|
|
170
|
+
field: str,
|
|
171
|
+
value: Any,
|
|
172
|
+
_last_chunk: bool = True,
|
|
173
|
+
) -> CheckOutcome:
|
|
174
|
+
"""Check that column *field* contains *value* at least once across chunks.
|
|
175
|
+
|
|
176
|
+
Parameters
|
|
177
|
+
----------
|
|
178
|
+
data
|
|
179
|
+
Chunk of records.
|
|
180
|
+
field
|
|
181
|
+
Target column name.
|
|
182
|
+
value
|
|
183
|
+
Value to search for.
|
|
184
|
+
_last_chunk
|
|
185
|
+
``True`` on the final chunk.
|
|
186
|
+
|
|
187
|
+
Returns
|
|
188
|
+
-------
|
|
189
|
+
CheckOutcome
|
|
190
|
+
``True`` as soon as *value* is seen; ``None`` while streaming; ``False`` if absent at end.
|
|
191
|
+
"""
|
|
192
|
+
kwargs: dict[str, Any] = {"field": field, "value": value}
|
|
193
|
+
if field not in data:
|
|
194
|
+
return CheckOutcome(passed=False, notes=missing_field_note(field, data), kwargs=kwargs)
|
|
195
|
+
|
|
196
|
+
if value in data[field].unique():
|
|
197
|
+
return CheckOutcome(passed=True, kwargs=kwargs)
|
|
198
|
+
if _last_chunk:
|
|
199
|
+
return CheckOutcome(
|
|
200
|
+
passed=False,
|
|
201
|
+
notes=f"Value {value!r} not found in field {field!r}.",
|
|
202
|
+
kwargs=kwargs,
|
|
203
|
+
)
|
|
204
|
+
return CheckOutcome(passed=None, kwargs=kwargs)
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
@requires_all_data
|
|
208
|
+
def record_field_contains_at_least(
|
|
209
|
+
data: pd.DataFrame,
|
|
210
|
+
field: str,
|
|
211
|
+
values: Sequence[Any],
|
|
212
|
+
_last_chunk: bool = True,
|
|
213
|
+
_found: list[Any] | None = None,
|
|
214
|
+
) -> CheckOutcome:
|
|
215
|
+
"""Check that column *field* contains every value in *values* across the dataset.
|
|
216
|
+
|
|
217
|
+
Parameters
|
|
218
|
+
----------
|
|
219
|
+
data
|
|
220
|
+
Chunk of records.
|
|
221
|
+
field
|
|
222
|
+
Target column name.
|
|
223
|
+
values
|
|
224
|
+
Required values.
|
|
225
|
+
_last_chunk
|
|
226
|
+
``True`` on the final chunk.
|
|
227
|
+
_found
|
|
228
|
+
Accumulated unique values observed in prior chunks.
|
|
229
|
+
|
|
230
|
+
Returns
|
|
231
|
+
-------
|
|
232
|
+
CheckOutcome
|
|
233
|
+
``True`` once all values are observed; ``None`` while streaming; ``False`` at end.
|
|
234
|
+
"""
|
|
235
|
+
found_set = set(_found) if _found is not None else set()
|
|
236
|
+
passed: bool | None = None
|
|
237
|
+
notes: str | None = None
|
|
238
|
+
|
|
239
|
+
if field not in data:
|
|
240
|
+
passed = False
|
|
241
|
+
notes = missing_field_note(field, data)
|
|
242
|
+
else:
|
|
243
|
+
found_set.update(data[field].dropna().unique().tolist())
|
|
244
|
+
all_found = all(val in found_set for val in values)
|
|
245
|
+
if _last_chunk or all_found:
|
|
246
|
+
passed = all_found
|
|
247
|
+
if not all_found and len(found_set) < 10:
|
|
248
|
+
notes = f"Found the following unique values: {sorted(found_set, key=str)}"
|
|
249
|
+
|
|
250
|
+
return CheckOutcome(
|
|
251
|
+
passed=passed,
|
|
252
|
+
notes=notes,
|
|
253
|
+
kwargs={"field": field, "values": list(values), "_found": list(found_set)},
|
|
254
|
+
)
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
@requires_all_data
|
|
258
|
+
def record_field_contains_only(
|
|
259
|
+
data: pd.DataFrame,
|
|
260
|
+
field: str,
|
|
261
|
+
values: Sequence[Any],
|
|
262
|
+
_last_chunk: bool = True,
|
|
263
|
+
_found: list[Any] | None = None,
|
|
264
|
+
) -> CheckOutcome:
|
|
265
|
+
"""Check that column *field* contains no values outside *values*.
|
|
266
|
+
|
|
267
|
+
Parameters
|
|
268
|
+
----------
|
|
269
|
+
data
|
|
270
|
+
Chunk of records.
|
|
271
|
+
field
|
|
272
|
+
Target column name.
|
|
273
|
+
values
|
|
274
|
+
Permitted values.
|
|
275
|
+
_last_chunk
|
|
276
|
+
``True`` on the final chunk.
|
|
277
|
+
_found
|
|
278
|
+
Accumulated unique values observed in prior chunks.
|
|
279
|
+
|
|
280
|
+
Returns
|
|
281
|
+
-------
|
|
282
|
+
CheckOutcome
|
|
283
|
+
``False`` immediately on unexpected values; ``None`` while streaming; ``True`` at end.
|
|
284
|
+
"""
|
|
285
|
+
found_set = set(_found) if _found is not None else set()
|
|
286
|
+
allowed = set(values)
|
|
287
|
+
passed: bool | None = None
|
|
288
|
+
notes: str | None = None
|
|
289
|
+
|
|
290
|
+
if field not in data:
|
|
291
|
+
passed = False
|
|
292
|
+
notes = missing_field_note(field, data)
|
|
293
|
+
else:
|
|
294
|
+
found_set.update(data[field].unique().tolist())
|
|
295
|
+
extra_vals = [v for v in found_set if v not in allowed]
|
|
296
|
+
if extra_vals:
|
|
297
|
+
passed = False
|
|
298
|
+
bad_lines = data.index[data[field].isin(extra_vals)].tolist()
|
|
299
|
+
notes = (
|
|
300
|
+
f"Found the following extra values: {extra_vals} "
|
|
301
|
+
f"at line(s) {truncate_lines(bad_lines)}"
|
|
302
|
+
)
|
|
303
|
+
elif _last_chunk:
|
|
304
|
+
passed = True
|
|
305
|
+
|
|
306
|
+
return CheckOutcome(
|
|
307
|
+
passed=passed,
|
|
308
|
+
notes=notes,
|
|
309
|
+
kwargs={"field": field, "values": list(values), "_found": list(found_set)},
|
|
310
|
+
)
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
@requires_all_data
|
|
314
|
+
def record_field_date_between(
|
|
315
|
+
data: pd.DataFrame,
|
|
316
|
+
field: str,
|
|
317
|
+
constraint: str | Sequence[Any],
|
|
318
|
+
include_bounds: bool = False,
|
|
319
|
+
_last_chunk: bool = True,
|
|
320
|
+
) -> CheckOutcome:
|
|
321
|
+
"""Check that all date values in *field* lie within *constraint*.
|
|
322
|
+
|
|
323
|
+
Parameters
|
|
324
|
+
----------
|
|
325
|
+
data
|
|
326
|
+
Chunk of records.
|
|
327
|
+
field
|
|
328
|
+
Target column name.
|
|
329
|
+
constraint
|
|
330
|
+
Lower and upper date bounds as a 2-sequence or bracketed string
|
|
331
|
+
(e.g. ``'[January 2015, 2018-01-01]'`` or ``(2015, 2018)``).
|
|
332
|
+
include_bounds
|
|
333
|
+
Use inclusive bounds when ``True``, strict otherwise.
|
|
334
|
+
_last_chunk
|
|
335
|
+
``True`` on the final chunk.
|
|
336
|
+
|
|
337
|
+
Returns
|
|
338
|
+
-------
|
|
339
|
+
CheckOutcome
|
|
340
|
+
``False`` immediately on out-of-range dates; ``None`` while streaming; ``True`` at end.
|
|
341
|
+
"""
|
|
342
|
+
try:
|
|
343
|
+
raw_lo, raw_hi = split_constraint(constraint)
|
|
344
|
+
bounds = (parse_date_bound(raw_lo), parse_date_bound(raw_hi))
|
|
345
|
+
except (ValueError, TypeError):
|
|
346
|
+
return CheckOutcome(
|
|
347
|
+
passed=False,
|
|
348
|
+
notes=f"Cannot convert {constraint} to dates. Stopping this check.",
|
|
349
|
+
kwargs={"field": field, "constraint": constraint, "include_bounds": include_bounds},
|
|
350
|
+
)
|
|
351
|
+
|
|
352
|
+
kwargs: dict[str, Any] = {
|
|
353
|
+
"field": field,
|
|
354
|
+
"constraint": bounds,
|
|
355
|
+
"include_bounds": include_bounds,
|
|
356
|
+
}
|
|
357
|
+
if field not in data:
|
|
358
|
+
return CheckOutcome(
|
|
359
|
+
passed=False,
|
|
360
|
+
notes=f"{field} not found in {list(data.columns)}. Stopping this check.",
|
|
361
|
+
kwargs=kwargs,
|
|
362
|
+
)
|
|
363
|
+
|
|
364
|
+
if data.empty:
|
|
365
|
+
return CheckOutcome(passed=True if _last_chunk else None, kwargs=kwargs)
|
|
366
|
+
|
|
367
|
+
try:
|
|
368
|
+
date_series = pd.to_datetime(data[field], format="mixed")
|
|
369
|
+
except (ValueError, TypeError):
|
|
370
|
+
return CheckOutcome(
|
|
371
|
+
passed=False,
|
|
372
|
+
notes=f"Cannot convert {field} to dates. Stopping this check.",
|
|
373
|
+
kwargs=kwargs,
|
|
374
|
+
)
|
|
375
|
+
|
|
376
|
+
passed, notes = check_series_between(
|
|
377
|
+
date_series,
|
|
378
|
+
bounds[0],
|
|
379
|
+
bounds[1],
|
|
380
|
+
include_bounds=include_bounds,
|
|
381
|
+
last_chunk=_last_chunk,
|
|
382
|
+
)
|
|
383
|
+
return CheckOutcome(passed=passed, notes=notes, kwargs=kwargs)
|
|
384
|
+
|
|
385
|
+
|
|
386
|
+
def record_field_is_numeric(data: pd.DataFrame, field: str) -> CheckOutcome:
|
|
387
|
+
"""Check that column *field* has a numeric pandas dtype.
|
|
388
|
+
|
|
389
|
+
Parameters
|
|
390
|
+
----------
|
|
391
|
+
data
|
|
392
|
+
Chunk of records.
|
|
393
|
+
field
|
|
394
|
+
Target column name.
|
|
395
|
+
|
|
396
|
+
Returns
|
|
397
|
+
-------
|
|
398
|
+
CheckOutcome
|
|
399
|
+
Outcome detailing actual dtype on failure.
|
|
400
|
+
"""
|
|
401
|
+
kwargs: dict[str, Any] = {"field": field}
|
|
402
|
+
if field not in data:
|
|
403
|
+
return CheckOutcome(passed=False, notes=missing_field_note(field, data), kwargs=kwargs)
|
|
404
|
+
|
|
405
|
+
is_num = bool(pd.api.types.is_numeric_dtype(data[field]))
|
|
406
|
+
notes = None if is_num else f"{field} data type: {data[field].dtype}"
|
|
407
|
+
return CheckOutcome(passed=is_num, notes=notes, kwargs=kwargs)
|
|
408
|
+
|
|
409
|
+
|
|
410
|
+
@requires_all_data
|
|
411
|
+
def record_field_is_unique(
|
|
412
|
+
data: pd.DataFrame,
|
|
413
|
+
field: str,
|
|
414
|
+
_last_chunk: bool = True,
|
|
415
|
+
_found: list[Any] | None = None,
|
|
416
|
+
) -> CheckOutcome:
|
|
417
|
+
"""Check that every value in *field* is unique across all chunks.
|
|
418
|
+
|
|
419
|
+
Parameters
|
|
420
|
+
----------
|
|
421
|
+
data
|
|
422
|
+
Chunk of records.
|
|
423
|
+
field
|
|
424
|
+
Target column name.
|
|
425
|
+
_last_chunk
|
|
426
|
+
``True`` on the final chunk.
|
|
427
|
+
_found
|
|
428
|
+
Accumulated values observed in prior chunks.
|
|
429
|
+
|
|
430
|
+
Returns
|
|
431
|
+
-------
|
|
432
|
+
CheckOutcome
|
|
433
|
+
``False`` immediately on duplicate values; ``None`` while streaming; ``True`` at end.
|
|
434
|
+
"""
|
|
435
|
+
seen = set(_found) if _found is not None else set()
|
|
436
|
+
passed: bool | None = None
|
|
437
|
+
notes: str | None = None
|
|
438
|
+
|
|
439
|
+
if field not in data:
|
|
440
|
+
passed = False
|
|
441
|
+
notes = missing_field_note(field, data)
|
|
442
|
+
else:
|
|
443
|
+
col = data[field]
|
|
444
|
+
dups_within = set(col[col.duplicated()].tolist())
|
|
445
|
+
dups_across = set(col[col.isin(seen)].tolist())
|
|
446
|
+
dups = sorted(dups_within | dups_across, key=str)
|
|
447
|
+
seen.update(col.tolist())
|
|
448
|
+
|
|
449
|
+
if dups:
|
|
450
|
+
passed = False
|
|
451
|
+
bad_lines = data.index[col.isin(dups)].tolist()
|
|
452
|
+
notes = (
|
|
453
|
+
f"Found the following duplicate values: {dups} "
|
|
454
|
+
f"at line(s) {truncate_lines(bad_lines)}"
|
|
455
|
+
)
|
|
456
|
+
elif _last_chunk:
|
|
457
|
+
passed = True
|
|
458
|
+
|
|
459
|
+
return CheckOutcome(passed=passed, notes=notes, kwargs={"field": field, "_found": list(seen)})
|
|
460
|
+
|
|
461
|
+
|
|
462
|
+
@requires_all_data
|
|
463
|
+
def record_field_not_null(
|
|
464
|
+
data: pd.DataFrame,
|
|
465
|
+
field: str,
|
|
466
|
+
_last_chunk: bool = True,
|
|
467
|
+
) -> CheckOutcome:
|
|
468
|
+
"""Check that column *field* contains no null/NaN values.
|
|
469
|
+
|
|
470
|
+
Parameters
|
|
471
|
+
----------
|
|
472
|
+
data
|
|
473
|
+
Chunk of records.
|
|
474
|
+
field
|
|
475
|
+
Target column name.
|
|
476
|
+
_last_chunk
|
|
477
|
+
``True`` on the final chunk.
|
|
478
|
+
|
|
479
|
+
Returns
|
|
480
|
+
-------
|
|
481
|
+
CheckOutcome
|
|
482
|
+
``False`` immediately on any null; ``None`` while streaming; ``True`` at end.
|
|
483
|
+
"""
|
|
484
|
+
kwargs: dict[str, Any] = {"field": field}
|
|
485
|
+
if field not in data:
|
|
486
|
+
return CheckOutcome(passed=False, notes=missing_field_note(field, data), kwargs=kwargs)
|
|
487
|
+
|
|
488
|
+
null_mask = data[field].isnull()
|
|
489
|
+
if bool(null_mask.any()):
|
|
490
|
+
lines = truncate_lines(data.index[null_mask].tolist())
|
|
491
|
+
return CheckOutcome(passed=False, notes=f"Field at line(s) {lines} is null.", kwargs=kwargs)
|
|
492
|
+
return CheckOutcome(passed=True if _last_chunk else None, kwargs=kwargs)
|
|
493
|
+
|
|
494
|
+
|
|
495
|
+
@requires_all_data
|
|
496
|
+
def record_field_numeric_value_between(
|
|
497
|
+
data: pd.DataFrame,
|
|
498
|
+
field: str,
|
|
499
|
+
constraint: str | Sequence[float | int | str],
|
|
500
|
+
include_bounds: bool = False,
|
|
501
|
+
_last_chunk: bool = True,
|
|
502
|
+
) -> CheckOutcome:
|
|
503
|
+
"""Check that all numeric values in *field* lie within *constraint*.
|
|
504
|
+
|
|
505
|
+
Parameters
|
|
506
|
+
----------
|
|
507
|
+
data
|
|
508
|
+
Chunk of records.
|
|
509
|
+
field
|
|
510
|
+
Target column name.
|
|
511
|
+
constraint
|
|
512
|
+
Lower and upper bounds as a 2-sequence or string (e.g. ``'(0.1, 100)'``).
|
|
513
|
+
include_bounds
|
|
514
|
+
Use inclusive ``<=``/``>=`` bounds when ``True``, strict otherwise.
|
|
515
|
+
_last_chunk
|
|
516
|
+
``True`` on the final chunk.
|
|
517
|
+
|
|
518
|
+
Returns
|
|
519
|
+
-------
|
|
520
|
+
CheckOutcome
|
|
521
|
+
``False`` immediately on out-of-range values; ``None`` while streaming; ``True`` at end.
|
|
522
|
+
"""
|
|
523
|
+
try:
|
|
524
|
+
raw_lo, raw_hi = split_constraint(constraint)
|
|
525
|
+
bounds = (float(raw_lo), float(raw_hi))
|
|
526
|
+
except (ValueError, TypeError):
|
|
527
|
+
return CheckOutcome(
|
|
528
|
+
passed=False,
|
|
529
|
+
notes=f"Cannot convert {constraint} to numeric bounds. Stopping this check.",
|
|
530
|
+
kwargs={"field": field, "constraint": constraint, "include_bounds": include_bounds},
|
|
531
|
+
)
|
|
532
|
+
|
|
533
|
+
kwargs: dict[str, Any] = {
|
|
534
|
+
"field": field,
|
|
535
|
+
"constraint": bounds,
|
|
536
|
+
"include_bounds": include_bounds,
|
|
537
|
+
}
|
|
538
|
+
if field not in data:
|
|
539
|
+
return CheckOutcome(
|
|
540
|
+
passed=False,
|
|
541
|
+
notes=f"{field} not found in {list(data.columns)}. Stopping this check.",
|
|
542
|
+
kwargs=kwargs,
|
|
543
|
+
)
|
|
544
|
+
|
|
545
|
+
passed, notes = check_series_between(
|
|
546
|
+
data[field],
|
|
547
|
+
bounds[0],
|
|
548
|
+
bounds[1],
|
|
549
|
+
include_bounds=include_bounds,
|
|
550
|
+
last_chunk=_last_chunk,
|
|
551
|
+
)
|
|
552
|
+
return CheckOutcome(passed=passed, notes=notes, kwargs=kwargs)
|
|
553
|
+
|
|
554
|
+
|
|
555
|
+
@requires_all_data
|
|
556
|
+
def record_if_field1_startswith_field2_contains_only(
|
|
557
|
+
data: pd.DataFrame,
|
|
558
|
+
field1: str,
|
|
559
|
+
value1: str,
|
|
560
|
+
field2: str,
|
|
561
|
+
value2: Any,
|
|
562
|
+
_last_chunk: bool = True,
|
|
563
|
+
) -> CheckOutcome:
|
|
564
|
+
"""Check conditional dependency: if *field1* starts with *value1*, *field2* equals *value2*.
|
|
565
|
+
|
|
566
|
+
Parameters
|
|
567
|
+
----------
|
|
568
|
+
data
|
|
569
|
+
Chunk of records.
|
|
570
|
+
field1
|
|
571
|
+
Condition column name.
|
|
572
|
+
value1
|
|
573
|
+
Required prefix on *field1*.
|
|
574
|
+
field2
|
|
575
|
+
Dependent column name.
|
|
576
|
+
value2
|
|
577
|
+
Required value for *field2* whenever *field1* matches *value1*.
|
|
578
|
+
_last_chunk
|
|
579
|
+
``True`` on the final chunk.
|
|
580
|
+
|
|
581
|
+
Returns
|
|
582
|
+
-------
|
|
583
|
+
CheckOutcome
|
|
584
|
+
``False`` immediately on violations; ``None`` while streaming; ``True`` at end.
|
|
585
|
+
"""
|
|
586
|
+
kwargs: dict[str, Any] = {
|
|
587
|
+
"field1": field1,
|
|
588
|
+
"value1": value1,
|
|
589
|
+
"field2": field2,
|
|
590
|
+
"value2": value2,
|
|
591
|
+
}
|
|
592
|
+
if field1 not in data or field2 not in data:
|
|
593
|
+
return CheckOutcome(
|
|
594
|
+
passed=False,
|
|
595
|
+
notes=f"Fields {field1} or {field2} not available in {list(data.columns)}",
|
|
596
|
+
kwargs=kwargs,
|
|
597
|
+
)
|
|
598
|
+
|
|
599
|
+
if data.empty:
|
|
600
|
+
return CheckOutcome(passed=True if _last_chunk else None, kwargs=kwargs)
|
|
601
|
+
|
|
602
|
+
matches_prefix = data[field1].astype(str).str.startswith(str(value1))
|
|
603
|
+
mismatches = matches_prefix & (data[field2] != value2)
|
|
604
|
+
if bool(mismatches.any()):
|
|
605
|
+
extra_vals = data.loc[mismatches, field2].unique().tolist()
|
|
606
|
+
bad_lines = data.index[mismatches].tolist()
|
|
607
|
+
note = (
|
|
608
|
+
f"Found the following extra values: {extra_vals} at line(s) {truncate_lines(bad_lines)}"
|
|
609
|
+
)
|
|
610
|
+
return CheckOutcome(passed=False, notes=note, kwargs=kwargs)
|
|
611
|
+
|
|
612
|
+
return CheckOutcome(passed=True if _last_chunk else None, kwargs=kwargs)
|
|
613
|
+
|
|
614
|
+
|
|
615
|
+
@requires_all_data
|
|
616
|
+
def record_validate_schema(
|
|
617
|
+
data: pd.DataFrame,
|
|
618
|
+
schema: pa.DataFrameSchema | dict[str, Any] | str | pathlib.Path,
|
|
619
|
+
lazy: bool = True,
|
|
620
|
+
_last_chunk: bool = True,
|
|
621
|
+
) -> CheckOutcome:
|
|
622
|
+
"""Validate each chunk of *data* against a Pandera :class:`DataFrameSchema`.
|
|
623
|
+
|
|
624
|
+
Accepts a :class:`pandera.pandas.DataFrameSchema` object, a concise dict
|
|
625
|
+
specification, a YAML/JSON schema file path, or a COBOL copybook path
|
|
626
|
+
(auto-converted via :meth:`~dataspector.coboller.copybook.Copybook.to_pandera_schema`).
|
|
627
|
+
|
|
628
|
+
Parameters
|
|
629
|
+
----------
|
|
630
|
+
data
|
|
631
|
+
Chunk of records to validate.
|
|
632
|
+
schema
|
|
633
|
+
Declarative Pandera schema definition or path.
|
|
634
|
+
lazy
|
|
635
|
+
Collect all schema violations per chunk before failing.
|
|
636
|
+
_last_chunk
|
|
637
|
+
``True`` on the final chunk.
|
|
638
|
+
|
|
639
|
+
Returns
|
|
640
|
+
-------
|
|
641
|
+
CheckOutcome
|
|
642
|
+
Outcome with concise violation summary in ``notes`` on failure.
|
|
643
|
+
"""
|
|
644
|
+
try:
|
|
645
|
+
resolved = resolve_pandera_schema(schema)
|
|
646
|
+
except Exception as exc:
|
|
647
|
+
return CheckOutcome(
|
|
648
|
+
passed=False,
|
|
649
|
+
notes=f"Invalid Pandera schema: {exc}",
|
|
650
|
+
kwargs={"schema": schema, "lazy": lazy},
|
|
651
|
+
)
|
|
652
|
+
|
|
653
|
+
kwargs: dict[str, Any] = {"schema": resolved, "lazy": lazy}
|
|
654
|
+
if data.empty:
|
|
655
|
+
return CheckOutcome(passed=True if _last_chunk else None, kwargs=kwargs)
|
|
656
|
+
|
|
657
|
+
try:
|
|
658
|
+
resolved.validate(data, lazy=lazy)
|
|
659
|
+
except SchemaErrors as exc:
|
|
660
|
+
cases = exc.failure_cases
|
|
661
|
+
summary = "; ".join(
|
|
662
|
+
f"{row.get('column', '<df>')}: {row.get('check', 'failed')} (case={row.get('failure_case')})"
|
|
663
|
+
for _, row in cases.head(5).iterrows()
|
|
664
|
+
)
|
|
665
|
+
if len(cases) > 5:
|
|
666
|
+
summary += f"; ... ({len(cases)} total failures)"
|
|
667
|
+
return CheckOutcome(passed=False, notes=summary, kwargs=kwargs)
|
|
668
|
+
except SchemaError as exc:
|
|
669
|
+
return CheckOutcome(passed=False, notes=str(exc), kwargs=kwargs)
|
|
670
|
+
|
|
671
|
+
return CheckOutcome(passed=True if _last_chunk else None, kwargs=kwargs)
|