dataspector 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,671 @@
1
+ """Record-level integrity check functions operating on DataFrame chunks (alphabetical)."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import pathlib
6
+ from collections.abc import Sequence
7
+ from typing import Any
8
+
9
+ import pandas as pd
10
+ import pandera.pandas as pa
11
+ from pandera.errors import SchemaError, SchemaErrors
12
+
13
+ from dataspector.checkers.utils import (
14
+ CheckOutcome,
15
+ check_series_between,
16
+ missing_field_note,
17
+ parse_date_bound,
18
+ requires_all_data,
19
+ resolve_pandera_schema,
20
+ split_constraint,
21
+ truncate_lines,
22
+ )
23
+
24
+
25
+ @requires_all_data
26
+ def record_all_data_not_null(
27
+ data: pd.DataFrame,
28
+ _last_chunk: bool = True,
29
+ ) -> CheckOutcome:
30
+ """Check that no cell anywhere in *data* is null/NaN.
31
+
32
+ Parameters
33
+ ----------
34
+ data
35
+ Chunk of records.
36
+ _last_chunk
37
+ ``True`` on the final chunk.
38
+
39
+ Returns
40
+ -------
41
+ CheckOutcome
42
+ ``False`` immediately on any null; ``None`` while streaming; ``True`` at end.
43
+ """
44
+ if bool(data.isnull().any().any()):
45
+ bad_rows = truncate_lines(data.index[data.isnull().any(axis=1)].tolist())
46
+ return CheckOutcome(passed=False, notes=f"Field at line(s) {bad_rows} is null.")
47
+ return CheckOutcome(passed=True if _last_chunk else None)
48
+
49
+
50
+ def record_contains(data: pd.DataFrame, field: str) -> CheckOutcome:
51
+ """Check that *data* contains column *field*.
52
+
53
+ Parameters
54
+ ----------
55
+ data
56
+ Chunk of records.
57
+ field
58
+ Expected column name.
59
+
60
+ Returns
61
+ -------
62
+ CheckOutcome
63
+ Outcome listing available columns on failure.
64
+ """
65
+ present = field in data
66
+ notes = (
67
+ f"Source has fields: {list(data.columns)}"
68
+ if not present and hasattr(data, "columns")
69
+ else None
70
+ )
71
+ return CheckOutcome(passed=present, notes=notes, kwargs={"field": field})
72
+
73
+
74
+ def record_contains_at_least(data: pd.DataFrame, fields: Sequence[str]) -> CheckOutcome:
75
+ """Check that *data* contains every column listed in *fields*.
76
+
77
+ Parameters
78
+ ----------
79
+ data
80
+ Chunk of records.
81
+ fields
82
+ Required column names.
83
+
84
+ Returns
85
+ -------
86
+ CheckOutcome
87
+ Outcome listing missing columns on failure.
88
+ """
89
+ at_least = all(f in data for f in fields)
90
+ notes = None if at_least else f"Source is missing: {sorted(set(fields) - set(data.columns))}"
91
+ return CheckOutcome(passed=at_least, notes=notes, kwargs={"fields": list(fields)})
92
+
93
+
94
+ @requires_all_data
95
+ def record_count(
96
+ data: pd.DataFrame,
97
+ nrows: int,
98
+ _previous: int = 0,
99
+ _last_chunk: bool = True,
100
+ ) -> CheckOutcome:
101
+ """Check that the dataset has exactly *nrows* rows across all streamed chunks.
102
+
103
+ Parameters
104
+ ----------
105
+ data
106
+ Chunk of records.
107
+ nrows
108
+ Expected total row count.
109
+ _previous
110
+ Accumulated row count from prior chunks.
111
+ _last_chunk
112
+ ``True`` on the final chunk.
113
+
114
+ Returns
115
+ -------
116
+ CheckOutcome
117
+ ``passed=None`` while streaming; ``True``/``False`` on the final chunk.
118
+ """
119
+ nrecords = len(data) + _previous
120
+ passed: bool | None = None
121
+ notes: str | None = None
122
+ if _last_chunk:
123
+ passed = nrecords == nrows
124
+ if not passed:
125
+ notes = f"Found {nrecords} records"
126
+ return CheckOutcome(passed=passed, notes=notes, kwargs={"nrows": nrows, "_previous": nrecords})
127
+
128
+
129
+ @requires_all_data
130
+ def record_count_greater_than(
131
+ data: pd.DataFrame,
132
+ nrows: int,
133
+ _previous: int = 0,
134
+ _last_chunk: bool = True,
135
+ ) -> CheckOutcome:
136
+ """Check that the dataset has strictly more than *nrows* rows.
137
+
138
+ Parameters
139
+ ----------
140
+ data
141
+ Chunk of records.
142
+ nrows
143
+ Exclusive minimum row count.
144
+ _previous
145
+ Accumulated row count from prior chunks.
146
+ _last_chunk
147
+ ``True`` on the final chunk.
148
+
149
+ Returns
150
+ -------
151
+ CheckOutcome
152
+ ``True`` as soon as *nrows* is exceeded; ``None`` while streaming; ``False`` at end.
153
+ """
154
+ nrecords = len(data) + _previous
155
+ notes: str | None = None
156
+ passed: bool | None
157
+ if nrecords > nrows:
158
+ passed = True
159
+ elif _last_chunk:
160
+ passed = False
161
+ notes = f"Found {nrecords} records"
162
+ else:
163
+ passed = None
164
+ return CheckOutcome(passed=passed, notes=notes, kwargs={"nrows": nrows, "_previous": nrecords})
165
+
166
+
167
+ @requires_all_data
168
+ def record_field_contains(
169
+ data: pd.DataFrame,
170
+ field: str,
171
+ value: Any,
172
+ _last_chunk: bool = True,
173
+ ) -> CheckOutcome:
174
+ """Check that column *field* contains *value* at least once across chunks.
175
+
176
+ Parameters
177
+ ----------
178
+ data
179
+ Chunk of records.
180
+ field
181
+ Target column name.
182
+ value
183
+ Value to search for.
184
+ _last_chunk
185
+ ``True`` on the final chunk.
186
+
187
+ Returns
188
+ -------
189
+ CheckOutcome
190
+ ``True`` as soon as *value* is seen; ``None`` while streaming; ``False`` if absent at end.
191
+ """
192
+ kwargs: dict[str, Any] = {"field": field, "value": value}
193
+ if field not in data:
194
+ return CheckOutcome(passed=False, notes=missing_field_note(field, data), kwargs=kwargs)
195
+
196
+ if value in data[field].unique():
197
+ return CheckOutcome(passed=True, kwargs=kwargs)
198
+ if _last_chunk:
199
+ return CheckOutcome(
200
+ passed=False,
201
+ notes=f"Value {value!r} not found in field {field!r}.",
202
+ kwargs=kwargs,
203
+ )
204
+ return CheckOutcome(passed=None, kwargs=kwargs)
205
+
206
+
207
+ @requires_all_data
208
+ def record_field_contains_at_least(
209
+ data: pd.DataFrame,
210
+ field: str,
211
+ values: Sequence[Any],
212
+ _last_chunk: bool = True,
213
+ _found: list[Any] | None = None,
214
+ ) -> CheckOutcome:
215
+ """Check that column *field* contains every value in *values* across the dataset.
216
+
217
+ Parameters
218
+ ----------
219
+ data
220
+ Chunk of records.
221
+ field
222
+ Target column name.
223
+ values
224
+ Required values.
225
+ _last_chunk
226
+ ``True`` on the final chunk.
227
+ _found
228
+ Accumulated unique values observed in prior chunks.
229
+
230
+ Returns
231
+ -------
232
+ CheckOutcome
233
+ ``True`` once all values are observed; ``None`` while streaming; ``False`` at end.
234
+ """
235
+ found_set = set(_found) if _found is not None else set()
236
+ passed: bool | None = None
237
+ notes: str | None = None
238
+
239
+ if field not in data:
240
+ passed = False
241
+ notes = missing_field_note(field, data)
242
+ else:
243
+ found_set.update(data[field].dropna().unique().tolist())
244
+ all_found = all(val in found_set for val in values)
245
+ if _last_chunk or all_found:
246
+ passed = all_found
247
+ if not all_found and len(found_set) < 10:
248
+ notes = f"Found the following unique values: {sorted(found_set, key=str)}"
249
+
250
+ return CheckOutcome(
251
+ passed=passed,
252
+ notes=notes,
253
+ kwargs={"field": field, "values": list(values), "_found": list(found_set)},
254
+ )
255
+
256
+
257
+ @requires_all_data
258
+ def record_field_contains_only(
259
+ data: pd.DataFrame,
260
+ field: str,
261
+ values: Sequence[Any],
262
+ _last_chunk: bool = True,
263
+ _found: list[Any] | None = None,
264
+ ) -> CheckOutcome:
265
+ """Check that column *field* contains no values outside *values*.
266
+
267
+ Parameters
268
+ ----------
269
+ data
270
+ Chunk of records.
271
+ field
272
+ Target column name.
273
+ values
274
+ Permitted values.
275
+ _last_chunk
276
+ ``True`` on the final chunk.
277
+ _found
278
+ Accumulated unique values observed in prior chunks.
279
+
280
+ Returns
281
+ -------
282
+ CheckOutcome
283
+ ``False`` immediately on unexpected values; ``None`` while streaming; ``True`` at end.
284
+ """
285
+ found_set = set(_found) if _found is not None else set()
286
+ allowed = set(values)
287
+ passed: bool | None = None
288
+ notes: str | None = None
289
+
290
+ if field not in data:
291
+ passed = False
292
+ notes = missing_field_note(field, data)
293
+ else:
294
+ found_set.update(data[field].unique().tolist())
295
+ extra_vals = [v for v in found_set if v not in allowed]
296
+ if extra_vals:
297
+ passed = False
298
+ bad_lines = data.index[data[field].isin(extra_vals)].tolist()
299
+ notes = (
300
+ f"Found the following extra values: {extra_vals} "
301
+ f"at line(s) {truncate_lines(bad_lines)}"
302
+ )
303
+ elif _last_chunk:
304
+ passed = True
305
+
306
+ return CheckOutcome(
307
+ passed=passed,
308
+ notes=notes,
309
+ kwargs={"field": field, "values": list(values), "_found": list(found_set)},
310
+ )
311
+
312
+
313
+ @requires_all_data
314
+ def record_field_date_between(
315
+ data: pd.DataFrame,
316
+ field: str,
317
+ constraint: str | Sequence[Any],
318
+ include_bounds: bool = False,
319
+ _last_chunk: bool = True,
320
+ ) -> CheckOutcome:
321
+ """Check that all date values in *field* lie within *constraint*.
322
+
323
+ Parameters
324
+ ----------
325
+ data
326
+ Chunk of records.
327
+ field
328
+ Target column name.
329
+ constraint
330
+ Lower and upper date bounds as a 2-sequence or bracketed string
331
+ (e.g. ``'[January 2015, 2018-01-01]'`` or ``(2015, 2018)``).
332
+ include_bounds
333
+ Use inclusive bounds when ``True``, strict otherwise.
334
+ _last_chunk
335
+ ``True`` on the final chunk.
336
+
337
+ Returns
338
+ -------
339
+ CheckOutcome
340
+ ``False`` immediately on out-of-range dates; ``None`` while streaming; ``True`` at end.
341
+ """
342
+ try:
343
+ raw_lo, raw_hi = split_constraint(constraint)
344
+ bounds = (parse_date_bound(raw_lo), parse_date_bound(raw_hi))
345
+ except (ValueError, TypeError):
346
+ return CheckOutcome(
347
+ passed=False,
348
+ notes=f"Cannot convert {constraint} to dates. Stopping this check.",
349
+ kwargs={"field": field, "constraint": constraint, "include_bounds": include_bounds},
350
+ )
351
+
352
+ kwargs: dict[str, Any] = {
353
+ "field": field,
354
+ "constraint": bounds,
355
+ "include_bounds": include_bounds,
356
+ }
357
+ if field not in data:
358
+ return CheckOutcome(
359
+ passed=False,
360
+ notes=f"{field} not found in {list(data.columns)}. Stopping this check.",
361
+ kwargs=kwargs,
362
+ )
363
+
364
+ if data.empty:
365
+ return CheckOutcome(passed=True if _last_chunk else None, kwargs=kwargs)
366
+
367
+ try:
368
+ date_series = pd.to_datetime(data[field], format="mixed")
369
+ except (ValueError, TypeError):
370
+ return CheckOutcome(
371
+ passed=False,
372
+ notes=f"Cannot convert {field} to dates. Stopping this check.",
373
+ kwargs=kwargs,
374
+ )
375
+
376
+ passed, notes = check_series_between(
377
+ date_series,
378
+ bounds[0],
379
+ bounds[1],
380
+ include_bounds=include_bounds,
381
+ last_chunk=_last_chunk,
382
+ )
383
+ return CheckOutcome(passed=passed, notes=notes, kwargs=kwargs)
384
+
385
+
386
+ def record_field_is_numeric(data: pd.DataFrame, field: str) -> CheckOutcome:
387
+ """Check that column *field* has a numeric pandas dtype.
388
+
389
+ Parameters
390
+ ----------
391
+ data
392
+ Chunk of records.
393
+ field
394
+ Target column name.
395
+
396
+ Returns
397
+ -------
398
+ CheckOutcome
399
+ Outcome detailing actual dtype on failure.
400
+ """
401
+ kwargs: dict[str, Any] = {"field": field}
402
+ if field not in data:
403
+ return CheckOutcome(passed=False, notes=missing_field_note(field, data), kwargs=kwargs)
404
+
405
+ is_num = bool(pd.api.types.is_numeric_dtype(data[field]))
406
+ notes = None if is_num else f"{field} data type: {data[field].dtype}"
407
+ return CheckOutcome(passed=is_num, notes=notes, kwargs=kwargs)
408
+
409
+
410
+ @requires_all_data
411
+ def record_field_is_unique(
412
+ data: pd.DataFrame,
413
+ field: str,
414
+ _last_chunk: bool = True,
415
+ _found: list[Any] | None = None,
416
+ ) -> CheckOutcome:
417
+ """Check that every value in *field* is unique across all chunks.
418
+
419
+ Parameters
420
+ ----------
421
+ data
422
+ Chunk of records.
423
+ field
424
+ Target column name.
425
+ _last_chunk
426
+ ``True`` on the final chunk.
427
+ _found
428
+ Accumulated values observed in prior chunks.
429
+
430
+ Returns
431
+ -------
432
+ CheckOutcome
433
+ ``False`` immediately on duplicate values; ``None`` while streaming; ``True`` at end.
434
+ """
435
+ seen = set(_found) if _found is not None else set()
436
+ passed: bool | None = None
437
+ notes: str | None = None
438
+
439
+ if field not in data:
440
+ passed = False
441
+ notes = missing_field_note(field, data)
442
+ else:
443
+ col = data[field]
444
+ dups_within = set(col[col.duplicated()].tolist())
445
+ dups_across = set(col[col.isin(seen)].tolist())
446
+ dups = sorted(dups_within | dups_across, key=str)
447
+ seen.update(col.tolist())
448
+
449
+ if dups:
450
+ passed = False
451
+ bad_lines = data.index[col.isin(dups)].tolist()
452
+ notes = (
453
+ f"Found the following duplicate values: {dups} "
454
+ f"at line(s) {truncate_lines(bad_lines)}"
455
+ )
456
+ elif _last_chunk:
457
+ passed = True
458
+
459
+ return CheckOutcome(passed=passed, notes=notes, kwargs={"field": field, "_found": list(seen)})
460
+
461
+
462
+ @requires_all_data
463
+ def record_field_not_null(
464
+ data: pd.DataFrame,
465
+ field: str,
466
+ _last_chunk: bool = True,
467
+ ) -> CheckOutcome:
468
+ """Check that column *field* contains no null/NaN values.
469
+
470
+ Parameters
471
+ ----------
472
+ data
473
+ Chunk of records.
474
+ field
475
+ Target column name.
476
+ _last_chunk
477
+ ``True`` on the final chunk.
478
+
479
+ Returns
480
+ -------
481
+ CheckOutcome
482
+ ``False`` immediately on any null; ``None`` while streaming; ``True`` at end.
483
+ """
484
+ kwargs: dict[str, Any] = {"field": field}
485
+ if field not in data:
486
+ return CheckOutcome(passed=False, notes=missing_field_note(field, data), kwargs=kwargs)
487
+
488
+ null_mask = data[field].isnull()
489
+ if bool(null_mask.any()):
490
+ lines = truncate_lines(data.index[null_mask].tolist())
491
+ return CheckOutcome(passed=False, notes=f"Field at line(s) {lines} is null.", kwargs=kwargs)
492
+ return CheckOutcome(passed=True if _last_chunk else None, kwargs=kwargs)
493
+
494
+
495
+ @requires_all_data
496
+ def record_field_numeric_value_between(
497
+ data: pd.DataFrame,
498
+ field: str,
499
+ constraint: str | Sequence[float | int | str],
500
+ include_bounds: bool = False,
501
+ _last_chunk: bool = True,
502
+ ) -> CheckOutcome:
503
+ """Check that all numeric values in *field* lie within *constraint*.
504
+
505
+ Parameters
506
+ ----------
507
+ data
508
+ Chunk of records.
509
+ field
510
+ Target column name.
511
+ constraint
512
+ Lower and upper bounds as a 2-sequence or string (e.g. ``'(0.1, 100)'``).
513
+ include_bounds
514
+ Use inclusive ``<=``/``>=`` bounds when ``True``, strict otherwise.
515
+ _last_chunk
516
+ ``True`` on the final chunk.
517
+
518
+ Returns
519
+ -------
520
+ CheckOutcome
521
+ ``False`` immediately on out-of-range values; ``None`` while streaming; ``True`` at end.
522
+ """
523
+ try:
524
+ raw_lo, raw_hi = split_constraint(constraint)
525
+ bounds = (float(raw_lo), float(raw_hi))
526
+ except (ValueError, TypeError):
527
+ return CheckOutcome(
528
+ passed=False,
529
+ notes=f"Cannot convert {constraint} to numeric bounds. Stopping this check.",
530
+ kwargs={"field": field, "constraint": constraint, "include_bounds": include_bounds},
531
+ )
532
+
533
+ kwargs: dict[str, Any] = {
534
+ "field": field,
535
+ "constraint": bounds,
536
+ "include_bounds": include_bounds,
537
+ }
538
+ if field not in data:
539
+ return CheckOutcome(
540
+ passed=False,
541
+ notes=f"{field} not found in {list(data.columns)}. Stopping this check.",
542
+ kwargs=kwargs,
543
+ )
544
+
545
+ passed, notes = check_series_between(
546
+ data[field],
547
+ bounds[0],
548
+ bounds[1],
549
+ include_bounds=include_bounds,
550
+ last_chunk=_last_chunk,
551
+ )
552
+ return CheckOutcome(passed=passed, notes=notes, kwargs=kwargs)
553
+
554
+
555
+ @requires_all_data
556
+ def record_if_field1_startswith_field2_contains_only(
557
+ data: pd.DataFrame,
558
+ field1: str,
559
+ value1: str,
560
+ field2: str,
561
+ value2: Any,
562
+ _last_chunk: bool = True,
563
+ ) -> CheckOutcome:
564
+ """Check conditional dependency: if *field1* starts with *value1*, *field2* equals *value2*.
565
+
566
+ Parameters
567
+ ----------
568
+ data
569
+ Chunk of records.
570
+ field1
571
+ Condition column name.
572
+ value1
573
+ Required prefix on *field1*.
574
+ field2
575
+ Dependent column name.
576
+ value2
577
+ Required value for *field2* whenever *field1* matches *value1*.
578
+ _last_chunk
579
+ ``True`` on the final chunk.
580
+
581
+ Returns
582
+ -------
583
+ CheckOutcome
584
+ ``False`` immediately on violations; ``None`` while streaming; ``True`` at end.
585
+ """
586
+ kwargs: dict[str, Any] = {
587
+ "field1": field1,
588
+ "value1": value1,
589
+ "field2": field2,
590
+ "value2": value2,
591
+ }
592
+ if field1 not in data or field2 not in data:
593
+ return CheckOutcome(
594
+ passed=False,
595
+ notes=f"Fields {field1} or {field2} not available in {list(data.columns)}",
596
+ kwargs=kwargs,
597
+ )
598
+
599
+ if data.empty:
600
+ return CheckOutcome(passed=True if _last_chunk else None, kwargs=kwargs)
601
+
602
+ matches_prefix = data[field1].astype(str).str.startswith(str(value1))
603
+ mismatches = matches_prefix & (data[field2] != value2)
604
+ if bool(mismatches.any()):
605
+ extra_vals = data.loc[mismatches, field2].unique().tolist()
606
+ bad_lines = data.index[mismatches].tolist()
607
+ note = (
608
+ f"Found the following extra values: {extra_vals} at line(s) {truncate_lines(bad_lines)}"
609
+ )
610
+ return CheckOutcome(passed=False, notes=note, kwargs=kwargs)
611
+
612
+ return CheckOutcome(passed=True if _last_chunk else None, kwargs=kwargs)
613
+
614
+
615
+ @requires_all_data
616
+ def record_validate_schema(
617
+ data: pd.DataFrame,
618
+ schema: pa.DataFrameSchema | dict[str, Any] | str | pathlib.Path,
619
+ lazy: bool = True,
620
+ _last_chunk: bool = True,
621
+ ) -> CheckOutcome:
622
+ """Validate each chunk of *data* against a Pandera :class:`DataFrameSchema`.
623
+
624
+ Accepts a :class:`pandera.pandas.DataFrameSchema` object, a concise dict
625
+ specification, a YAML/JSON schema file path, or a COBOL copybook path
626
+ (auto-converted via :meth:`~dataspector.coboller.copybook.Copybook.to_pandera_schema`).
627
+
628
+ Parameters
629
+ ----------
630
+ data
631
+ Chunk of records to validate.
632
+ schema
633
+ Declarative Pandera schema definition or path.
634
+ lazy
635
+ Collect all schema violations per chunk before failing.
636
+ _last_chunk
637
+ ``True`` on the final chunk.
638
+
639
+ Returns
640
+ -------
641
+ CheckOutcome
642
+ Outcome with concise violation summary in ``notes`` on failure.
643
+ """
644
+ try:
645
+ resolved = resolve_pandera_schema(schema)
646
+ except Exception as exc:
647
+ return CheckOutcome(
648
+ passed=False,
649
+ notes=f"Invalid Pandera schema: {exc}",
650
+ kwargs={"schema": schema, "lazy": lazy},
651
+ )
652
+
653
+ kwargs: dict[str, Any] = {"schema": resolved, "lazy": lazy}
654
+ if data.empty:
655
+ return CheckOutcome(passed=True if _last_chunk else None, kwargs=kwargs)
656
+
657
+ try:
658
+ resolved.validate(data, lazy=lazy)
659
+ except SchemaErrors as exc:
660
+ cases = exc.failure_cases
661
+ summary = "; ".join(
662
+ f"{row.get('column', '<df>')}: {row.get('check', 'failed')} (case={row.get('failure_case')})"
663
+ for _, row in cases.head(5).iterrows()
664
+ )
665
+ if len(cases) > 5:
666
+ summary += f"; ... ({len(cases)} total failures)"
667
+ return CheckOutcome(passed=False, notes=summary, kwargs=kwargs)
668
+ except SchemaError as exc:
669
+ return CheckOutcome(passed=False, notes=str(exc), kwargs=kwargs)
670
+
671
+ return CheckOutcome(passed=True if _last_chunk else None, kwargs=kwargs)