dataspector 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dataspector/__init__.py +64 -0
- dataspector/__main__.py +10 -0
- dataspector/_connectors.py +260 -0
- dataspector/_reporting.py +480 -0
- dataspector/checkers/__init__.py +85 -0
- dataspector/checkers/record.py +671 -0
- dataspector/checkers/source.py +344 -0
- dataspector/checkers/utils.py +342 -0
- dataspector/cli.py +207 -0
- dataspector/coboller/__init__.py +123 -0
- dataspector/coboller/codecs.py +841 -0
- dataspector/coboller/constants.py +324 -0
- dataspector/coboller/copybook.py +1448 -0
- dataspector/coboller/parser.py +201 -0
- dataspector/core.py +740 -0
- dataspector/py.typed +0 -0
- dataspector/settings.py +101 -0
- dataspector-1.0.0.dist-info/METADATA +181 -0
- dataspector-1.0.0.dist-info/RECORD +22 -0
- dataspector-1.0.0.dist-info/WHEEL +4 -0
- dataspector-1.0.0.dist-info/entry_points.txt +3 -0
- dataspector-1.0.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,344 @@
|
|
|
1
|
+
"""Source-level and cross-source integrity check functions (alphabetical)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import filecmp
|
|
6
|
+
import pathlib
|
|
7
|
+
from collections.abc import Sized
|
|
8
|
+
from typing import TYPE_CHECKING, Any
|
|
9
|
+
|
|
10
|
+
import pandas as pd
|
|
11
|
+
|
|
12
|
+
from dataspector.checkers.utils import (
|
|
13
|
+
CheckOutcome,
|
|
14
|
+
as_generic_data,
|
|
15
|
+
requires_all_data,
|
|
16
|
+
source_byte_size,
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
if TYPE_CHECKING:
|
|
20
|
+
from dataspector.core import GenericData
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def source_exists(source: GenericData) -> CheckOutcome:
|
|
24
|
+
"""Check whether *source* exists.
|
|
25
|
+
|
|
26
|
+
Parameters
|
|
27
|
+
----------
|
|
28
|
+
source
|
|
29
|
+
Data source wrapper.
|
|
30
|
+
|
|
31
|
+
Returns
|
|
32
|
+
-------
|
|
33
|
+
CheckOutcome
|
|
34
|
+
``passed=True`` if *source* exists, else ``False``.
|
|
35
|
+
"""
|
|
36
|
+
return CheckOutcome(passed=source.exists)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def source_fuzzy_find(source: GenericData, pattern: str) -> CheckOutcome:
|
|
40
|
+
"""Search *source* for files, sheets, or tables matching shell *pattern*.
|
|
41
|
+
|
|
42
|
+
Parameters
|
|
43
|
+
----------
|
|
44
|
+
source
|
|
45
|
+
Directory, workbook, or database source.
|
|
46
|
+
pattern
|
|
47
|
+
Shell-style wildcard pattern (``*``, ``?``).
|
|
48
|
+
|
|
49
|
+
Returns
|
|
50
|
+
-------
|
|
51
|
+
CheckOutcome
|
|
52
|
+
Outcome listing matching entries in ``notes``.
|
|
53
|
+
"""
|
|
54
|
+
matches = source.fuzzy_find(pattern)
|
|
55
|
+
if matches is None:
|
|
56
|
+
passed, notes = False, "Pattern search not applicable."
|
|
57
|
+
elif matches:
|
|
58
|
+
passed, notes = True, f"Found: {matches}"
|
|
59
|
+
else:
|
|
60
|
+
passed, notes = False, "No matches found."
|
|
61
|
+
return CheckOutcome(passed=passed, notes=notes, kwargs={"pattern": pattern})
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def source_has_N_fields(source: GenericData, n_fields: int) -> CheckOutcome:
|
|
65
|
+
"""Check that *source* has exactly *n_fields* columns.
|
|
66
|
+
|
|
67
|
+
Parameters
|
|
68
|
+
----------
|
|
69
|
+
source
|
|
70
|
+
Data source wrapper.
|
|
71
|
+
n_fields
|
|
72
|
+
Expected column count.
|
|
73
|
+
|
|
74
|
+
Returns
|
|
75
|
+
-------
|
|
76
|
+
CheckOutcome
|
|
77
|
+
Outcome with actual field count in ``notes`` on mismatch.
|
|
78
|
+
"""
|
|
79
|
+
cols = source.columns
|
|
80
|
+
n = len(cols) if cols is not None else 0
|
|
81
|
+
has_n = n == n_fields
|
|
82
|
+
notes = None if has_n else f"Source has {n} fields"
|
|
83
|
+
return CheckOutcome(passed=has_n, notes=notes, kwargs={"n_fields": n_fields})
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def source_modified_on_date(
|
|
87
|
+
source: GenericData,
|
|
88
|
+
date: Any,
|
|
89
|
+
tolerance: str = "12h",
|
|
90
|
+
) -> CheckOutcome:
|
|
91
|
+
"""Check that *source* was last modified within *tolerance* of *date*.
|
|
92
|
+
|
|
93
|
+
Parameters
|
|
94
|
+
----------
|
|
95
|
+
source
|
|
96
|
+
Data source wrapper.
|
|
97
|
+
date
|
|
98
|
+
Expected modification timestamp.
|
|
99
|
+
tolerance
|
|
100
|
+
Pandas Timedelta string (e.g. ``'1d'``, ``'2.5h'``).
|
|
101
|
+
|
|
102
|
+
Returns
|
|
103
|
+
-------
|
|
104
|
+
CheckOutcome
|
|
105
|
+
Outcome with source modification timestamp in ``notes``.
|
|
106
|
+
"""
|
|
107
|
+
kwargs: dict[str, Any] = {"date": date, "tolerance": tolerance}
|
|
108
|
+
try:
|
|
109
|
+
target = pd.to_datetime(date)
|
|
110
|
+
except (ValueError, TypeError):
|
|
111
|
+
return CheckOutcome(
|
|
112
|
+
passed=False,
|
|
113
|
+
notes=f"Cannot convert {date} to dates. Stopping this check.",
|
|
114
|
+
kwargs=kwargs,
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
try:
|
|
118
|
+
delta = pd.Timedelta(tolerance)
|
|
119
|
+
except (ValueError, TypeError):
|
|
120
|
+
return CheckOutcome(
|
|
121
|
+
passed=False,
|
|
122
|
+
notes=f"Cannot convert {tolerance} to time difference. Stopping this check.",
|
|
123
|
+
kwargs=kwargs,
|
|
124
|
+
)
|
|
125
|
+
|
|
126
|
+
modded_on = source.last_modified
|
|
127
|
+
if modded_on is None:
|
|
128
|
+
return CheckOutcome(
|
|
129
|
+
passed=False,
|
|
130
|
+
notes="Could not determine source last_modified timestamp.",
|
|
131
|
+
kwargs=kwargs,
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
mod_ts = pd.Timestamp(modded_on)
|
|
135
|
+
if mod_ts.tzinfo is not None and target.tzinfo is None:
|
|
136
|
+
target = target.tz_localize(mod_ts.tzinfo)
|
|
137
|
+
elif mod_ts.tzinfo is None and target.tzinfo is not None:
|
|
138
|
+
mod_ts = mod_ts.tz_localize(target.tzinfo)
|
|
139
|
+
|
|
140
|
+
passed = bool(abs(mod_ts - target) < delta)
|
|
141
|
+
return CheckOutcome(passed=passed, notes=f"Source modified on {modded_on}", kwargs=kwargs)
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def source_modified_recently(source: GenericData, days_ago: float = 1) -> CheckOutcome:
|
|
145
|
+
"""Check that *source* was modified within the last *days_ago* days.
|
|
146
|
+
|
|
147
|
+
Parameters
|
|
148
|
+
----------
|
|
149
|
+
source
|
|
150
|
+
Data source wrapper.
|
|
151
|
+
days_ago
|
|
152
|
+
Maximum allowed age in days.
|
|
153
|
+
|
|
154
|
+
Returns
|
|
155
|
+
-------
|
|
156
|
+
CheckOutcome
|
|
157
|
+
Outcome with last modification timestamp in ``notes`` when stale.
|
|
158
|
+
"""
|
|
159
|
+
kwargs: dict[str, Any] = {"days_ago": days_ago}
|
|
160
|
+
recent = source.is_modified_recently(days_ago)
|
|
161
|
+
notes = f"Source modified on {source.last_modified}" if recent is False else None
|
|
162
|
+
return CheckOutcome(passed=recent, notes=notes, kwargs=kwargs)
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def source_source_identical(
|
|
166
|
+
source: GenericData,
|
|
167
|
+
source2: Any,
|
|
168
|
+
shallow: bool = False,
|
|
169
|
+
) -> CheckOutcome:
|
|
170
|
+
"""Check whether *source* and *source2* are identical files on disk.
|
|
171
|
+
|
|
172
|
+
Parameters
|
|
173
|
+
----------
|
|
174
|
+
source
|
|
175
|
+
Primary file-backed data source.
|
|
176
|
+
source2
|
|
177
|
+
Comparison file-backed data source.
|
|
178
|
+
shallow
|
|
179
|
+
Compare only file metadata when ``True``; byte-by-byte when ``False``.
|
|
180
|
+
|
|
181
|
+
Returns
|
|
182
|
+
-------
|
|
183
|
+
CheckOutcome
|
|
184
|
+
Outcome indicating file identity.
|
|
185
|
+
"""
|
|
186
|
+
kwargs: dict[str, Any] = {"source2": source2, "shallow": shallow}
|
|
187
|
+
f1 = source.path
|
|
188
|
+
f2 = as_generic_data(source2).path
|
|
189
|
+
if not isinstance(f1, pathlib.Path) or not isinstance(f2, pathlib.Path):
|
|
190
|
+
return CheckOutcome(
|
|
191
|
+
passed=False,
|
|
192
|
+
notes="Both sources must be file paths for byte-level comparison.",
|
|
193
|
+
kwargs=kwargs,
|
|
194
|
+
)
|
|
195
|
+
return CheckOutcome(passed=filecmp.cmp(f1, f2, shallow=shallow), kwargs=kwargs)
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
@requires_all_data
|
|
199
|
+
def source_source_roughly_same_number_of_records(
|
|
200
|
+
data: Sized,
|
|
201
|
+
source2: Any,
|
|
202
|
+
tolerance: float = 0.0,
|
|
203
|
+
_last_chunk: bool = True,
|
|
204
|
+
_nrecords: int = 0,
|
|
205
|
+
) -> CheckOutcome:
|
|
206
|
+
"""Check that the streamed source has roughly the same row count as *source2*.
|
|
207
|
+
|
|
208
|
+
Passes when ``abs(n1 - n2) <= tolerance * n2`` (or both have zero records).
|
|
209
|
+
Because this check is decorated with :func:`~dataspector.checkers.utils.requires_all_data`,
|
|
210
|
+
it accumulates ``_nrecords`` and returns ``passed=None`` on intermediate chunks
|
|
211
|
+
(``_last_chunk=False``) so the runner keeps streaming until the final chunk.
|
|
212
|
+
|
|
213
|
+
Parameters
|
|
214
|
+
----------
|
|
215
|
+
data
|
|
216
|
+
Chunk of records from the primary source (any sized sequence/DataFrame).
|
|
217
|
+
source2
|
|
218
|
+
Reference data source to compare against.
|
|
219
|
+
tolerance
|
|
220
|
+
Relative difference threshold (default ``0.0`` requires exact equality).
|
|
221
|
+
_last_chunk
|
|
222
|
+
``True`` on the final chunk, triggering the comparison against *source2*.
|
|
223
|
+
_nrecords
|
|
224
|
+
Accumulated record count carried across chunks.
|
|
225
|
+
|
|
226
|
+
Returns
|
|
227
|
+
-------
|
|
228
|
+
CheckOutcome
|
|
229
|
+
``passed=None`` while streaming; ``True`` or ``False`` on the final chunk.
|
|
230
|
+
"""
|
|
231
|
+
passed: bool | None = None
|
|
232
|
+
notes: str | None = None
|
|
233
|
+
_nrecords += len(data)
|
|
234
|
+
|
|
235
|
+
if _last_chunk:
|
|
236
|
+
n2 = as_generic_data(source2).number_of_records
|
|
237
|
+
if _nrecords == 0 and n2 == 0:
|
|
238
|
+
passed = True
|
|
239
|
+
notes = "Number of records is zero."
|
|
240
|
+
elif n2 > 0 and abs(_nrecords - n2) <= tolerance * n2:
|
|
241
|
+
passed = True
|
|
242
|
+
else:
|
|
243
|
+
passed = False
|
|
244
|
+
notes = f"Number of records in (source, source2): ({_nrecords}, {n2})"
|
|
245
|
+
|
|
246
|
+
return CheckOutcome(
|
|
247
|
+
passed=passed,
|
|
248
|
+
notes=notes,
|
|
249
|
+
kwargs={"source2": source2, "tolerance": tolerance, "_nrecords": _nrecords},
|
|
250
|
+
)
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def source_source_same_fields(source: GenericData, source2: Any) -> CheckOutcome:
|
|
254
|
+
"""Check that *source* and *source2* expose the same set of fields.
|
|
255
|
+
|
|
256
|
+
Parameters
|
|
257
|
+
----------
|
|
258
|
+
source
|
|
259
|
+
Primary data source.
|
|
260
|
+
source2
|
|
261
|
+
Comparison data source.
|
|
262
|
+
|
|
263
|
+
Returns
|
|
264
|
+
-------
|
|
265
|
+
CheckOutcome
|
|
266
|
+
Outcome detailing missing/extra columns in ``notes`` on mismatch.
|
|
267
|
+
"""
|
|
268
|
+
f1 = list(source.columns or [])
|
|
269
|
+
f2 = list(as_generic_data(source2).columns or [])
|
|
270
|
+
same = set(f1) == set(f2)
|
|
271
|
+
notes: str | None = None
|
|
272
|
+
if not same:
|
|
273
|
+
missing = sorted(set(f1) - set(f2))
|
|
274
|
+
extra = sorted(set(f2) - set(f1))
|
|
275
|
+
parts: list[str] = []
|
|
276
|
+
if missing:
|
|
277
|
+
parts.append(f"Missing in source2: {missing}")
|
|
278
|
+
if extra:
|
|
279
|
+
parts.append(f"Extra in source2: {extra}")
|
|
280
|
+
notes = "; ".join(parts)
|
|
281
|
+
return CheckOutcome(passed=same, notes=notes, kwargs={"source2": source2})
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
def source_source_size_roughly_same(
|
|
285
|
+
source: GenericData,
|
|
286
|
+
source2: Any,
|
|
287
|
+
tolerance: float = 0.2,
|
|
288
|
+
) -> CheckOutcome:
|
|
289
|
+
"""Check that *source* and *source2* byte sizes are within *tolerance*.
|
|
290
|
+
|
|
291
|
+
Parameters
|
|
292
|
+
----------
|
|
293
|
+
source
|
|
294
|
+
Primary data source.
|
|
295
|
+
source2
|
|
296
|
+
Comparison data source.
|
|
297
|
+
tolerance
|
|
298
|
+
Maximum allowed relative difference ``abs(size1 - size2) / size1``.
|
|
299
|
+
|
|
300
|
+
Returns
|
|
301
|
+
-------
|
|
302
|
+
CheckOutcome
|
|
303
|
+
Outcome detailing relative size ratio in ``notes`` on failure.
|
|
304
|
+
"""
|
|
305
|
+
size1 = source_byte_size(source)
|
|
306
|
+
size2 = source_byte_size(as_generic_data(source2))
|
|
307
|
+
|
|
308
|
+
if size1 == 0:
|
|
309
|
+
passed = size2 == 0
|
|
310
|
+
note = None if passed else f"Source is empty (0 bytes) while source2 is {size2} bytes"
|
|
311
|
+
else:
|
|
312
|
+
passed = abs(size1 - size2) / size1 <= tolerance
|
|
313
|
+
note = None if passed else f"Source2 is {size2 / size1} times the size of source"
|
|
314
|
+
|
|
315
|
+
return CheckOutcome(
|
|
316
|
+
passed=passed,
|
|
317
|
+
notes=note,
|
|
318
|
+
kwargs={"source2": source2, "tolerance": tolerance},
|
|
319
|
+
)
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
def source_storage_size_roughly(
|
|
323
|
+
source: GenericData,
|
|
324
|
+
rough_size: int,
|
|
325
|
+
tolerance: float = 0.2,
|
|
326
|
+
) -> CheckOutcome:
|
|
327
|
+
"""Check that *source* byte size is within *tolerance* of *rough_size*.
|
|
328
|
+
|
|
329
|
+
Parameters
|
|
330
|
+
----------
|
|
331
|
+
source
|
|
332
|
+
Data source wrapper.
|
|
333
|
+
rough_size
|
|
334
|
+
Expected size in bytes.
|
|
335
|
+
tolerance
|
|
336
|
+
Maximum allowed relative difference.
|
|
337
|
+
|
|
338
|
+
Returns
|
|
339
|
+
-------
|
|
340
|
+
CheckOutcome
|
|
341
|
+
Outcome indicating whether the byte size is within tolerance.
|
|
342
|
+
"""
|
|
343
|
+
kwargs: dict[str, Any] = {"rough_size": rough_size, "tolerance": tolerance}
|
|
344
|
+
return CheckOutcome(passed=source.storage_size_roughly(**kwargs), kwargs=kwargs)
|
|
@@ -0,0 +1,342 @@
|
|
|
1
|
+
"""Shared types, decorators, and internal helpers for :mod:`dataspector.checkers`."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import pathlib
|
|
7
|
+
from collections.abc import Callable, Sequence
|
|
8
|
+
from dataclasses import dataclass, field
|
|
9
|
+
from typing import TYPE_CHECKING, Any, TypeVar
|
|
10
|
+
|
|
11
|
+
import pandas as pd
|
|
12
|
+
import pandera.pandas as pa
|
|
13
|
+
|
|
14
|
+
if TYPE_CHECKING:
|
|
15
|
+
from dataspector.core import GenericData
|
|
16
|
+
|
|
17
|
+
_F = TypeVar("_F", bound=Callable[..., Any])
|
|
18
|
+
|
|
19
|
+
REQUIRES_ALL_DATA: set[str] = set()
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@dataclass(slots=True)
|
|
23
|
+
class CheckOutcome:
|
|
24
|
+
"""Structured result returned by every check function in :mod:`dataspector.checkers`.
|
|
25
|
+
|
|
26
|
+
Attributes
|
|
27
|
+
----------
|
|
28
|
+
passed
|
|
29
|
+
``True`` if the check passed, ``False`` if it failed, or ``None`` if
|
|
30
|
+
more chunks must be streamed before deciding.
|
|
31
|
+
notes
|
|
32
|
+
Optional human-readable diagnostic details (e.g. failing row indices).
|
|
33
|
+
kwargs
|
|
34
|
+
Keyword arguments (including updated accumulator state) forwarded to
|
|
35
|
+
subsequent chunk invocations or recorded in reports.
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
passed: bool | None
|
|
39
|
+
notes: str | None = None
|
|
40
|
+
kwargs: dict[str, Any] = field(default_factory=dict)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def requires_all_data(func: _F) -> _F:
|
|
44
|
+
"""Mark a check function as requiring the full streamed dataset before finalizing.
|
|
45
|
+
|
|
46
|
+
Decorated checks accept a ``_last_chunk: bool = True`` parameter. While
|
|
47
|
+
streaming chunks with ``_last_chunk=False``, the check may return
|
|
48
|
+
``CheckOutcome(passed=None, kwargs=...)`` to instruct
|
|
49
|
+
:class:`~dataspector.core.CheckeredHistory` to keep streaming and pass the
|
|
50
|
+
updated ``kwargs`` into the next chunk. On the final flush
|
|
51
|
+
(``_last_chunk=True``), the check must emit a definite ``True`` or ``False``.
|
|
52
|
+
|
|
53
|
+
Parameters
|
|
54
|
+
----------
|
|
55
|
+
func
|
|
56
|
+
Check function to register in ``REQUIRES_ALL_DATA``.
|
|
57
|
+
|
|
58
|
+
Returns
|
|
59
|
+
-------
|
|
60
|
+
Callable
|
|
61
|
+
The original check function, unmodified.
|
|
62
|
+
"""
|
|
63
|
+
name = getattr(func, "__name__", None)
|
|
64
|
+
if isinstance(name, str):
|
|
65
|
+
REQUIRES_ALL_DATA.add(name)
|
|
66
|
+
return func
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def as_generic_data(source: Any) -> GenericData:
|
|
70
|
+
"""Coerce a path, tuple, or :class:`~dataspector.core.GenericData` into :class:`~dataspector.core.GenericData`.
|
|
71
|
+
|
|
72
|
+
Parameters
|
|
73
|
+
----------
|
|
74
|
+
source
|
|
75
|
+
File path, locator tuple, or existing :class:`~dataspector.core.GenericData` wrapper.
|
|
76
|
+
|
|
77
|
+
Returns
|
|
78
|
+
-------
|
|
79
|
+
GenericData
|
|
80
|
+
Wrapped data source.
|
|
81
|
+
"""
|
|
82
|
+
from dataspector.core import GenericData
|
|
83
|
+
|
|
84
|
+
return source if isinstance(source, GenericData) else GenericData(source)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def build_pandera_checks(spec: dict[str, Any] | list[pa.Check] | None) -> list[pa.Check]:
|
|
88
|
+
"""Convert a simple dict of check constraints into :class:`pandera.Check` objects.
|
|
89
|
+
|
|
90
|
+
Parameters
|
|
91
|
+
----------
|
|
92
|
+
spec
|
|
93
|
+
Mapping of constraint names (e.g. ``gt``, ``in_range``, ``isin``) to
|
|
94
|
+
arguments, an existing list of :class:`pandera.Check`, or ``None``.
|
|
95
|
+
|
|
96
|
+
Returns
|
|
97
|
+
-------
|
|
98
|
+
list[pandera.Check]
|
|
99
|
+
Constructed Pandera check instances.
|
|
100
|
+
"""
|
|
101
|
+
if spec is None:
|
|
102
|
+
return []
|
|
103
|
+
if isinstance(spec, list):
|
|
104
|
+
return spec
|
|
105
|
+
|
|
106
|
+
builders: dict[str, Callable[[Any], pa.Check]] = {
|
|
107
|
+
"gt": pa.Check.greater_than,
|
|
108
|
+
"ge": pa.Check.greater_than_or_equal_to,
|
|
109
|
+
"lt": pa.Check.less_than,
|
|
110
|
+
"le": pa.Check.less_than_or_equal_to,
|
|
111
|
+
"isin": pa.Check.isin,
|
|
112
|
+
"notin": pa.Check.notin,
|
|
113
|
+
"str_matches": pa.Check.str_matches,
|
|
114
|
+
"str_startswith": pa.Check.str_startswith,
|
|
115
|
+
"str_endswith": pa.Check.str_endswith,
|
|
116
|
+
"str_contains": pa.Check.str_contains,
|
|
117
|
+
}
|
|
118
|
+
checks: list[pa.Check] = []
|
|
119
|
+
for name, arg in spec.items():
|
|
120
|
+
if name == "in_range" and isinstance(arg, (list, tuple)) and len(arg) == 2:
|
|
121
|
+
checks.append(pa.Check.in_range(min_value=arg[0], max_value=arg[1]))
|
|
122
|
+
elif name == "str_length" and isinstance(arg, (list, tuple)) and len(arg) == 2:
|
|
123
|
+
checks.append(pa.Check.str_length(min_value=arg[0], max_value=arg[1]))
|
|
124
|
+
elif name in builders:
|
|
125
|
+
checks.append(builders[name](arg))
|
|
126
|
+
else:
|
|
127
|
+
raise ValueError(f"Unsupported Pandera check key: {name!r}")
|
|
128
|
+
return checks
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def check_series_between(
|
|
132
|
+
series: pd.Series,
|
|
133
|
+
lower: Any,
|
|
134
|
+
upper: Any,
|
|
135
|
+
include_bounds: bool = False,
|
|
136
|
+
last_chunk: bool = True,
|
|
137
|
+
) -> tuple[bool | None, str | None]:
|
|
138
|
+
"""Evaluate range bounds on a numeric or datetime :class:`pandas.Series`.
|
|
139
|
+
|
|
140
|
+
Parameters
|
|
141
|
+
----------
|
|
142
|
+
series
|
|
143
|
+
Values to test against the interval bounds.
|
|
144
|
+
lower
|
|
145
|
+
Lower bound.
|
|
146
|
+
upper
|
|
147
|
+
Upper bound.
|
|
148
|
+
include_bounds
|
|
149
|
+
Treat *lower* and *upper* as inclusive when ``True`` (exclusive when ``False``).
|
|
150
|
+
last_chunk
|
|
151
|
+
Whether this is the final chunk in a streaming check.
|
|
152
|
+
|
|
153
|
+
Returns
|
|
154
|
+
-------
|
|
155
|
+
tuple[bool | None, str | None]
|
|
156
|
+
Pair of ``(passed, diagnostic_note)``; ``passed`` is ``None`` while
|
|
157
|
+
more chunks remain and no violation has occurred.
|
|
158
|
+
"""
|
|
159
|
+
if series.empty:
|
|
160
|
+
return (True, None) if last_chunk else (None, None)
|
|
161
|
+
|
|
162
|
+
if include_bounds:
|
|
163
|
+
outside = (series < lower) | (series > upper)
|
|
164
|
+
else:
|
|
165
|
+
outside = (series <= lower) | (series >= upper)
|
|
166
|
+
|
|
167
|
+
if bool(outside.any()):
|
|
168
|
+
minv = series.min()
|
|
169
|
+
maxv = series.max()
|
|
170
|
+
below = minv < lower if include_bounds else minv <= lower
|
|
171
|
+
above = maxv > upper if include_bounds else maxv >= upper
|
|
172
|
+
if below and above:
|
|
173
|
+
note = (
|
|
174
|
+
f"Found (min, max) values: ({minv}, {maxv}) "
|
|
175
|
+
f"at lines ({series.idxmin()}, {series.idxmax()})"
|
|
176
|
+
)
|
|
177
|
+
elif below:
|
|
178
|
+
note = f"Found value of {minv}, below constraint {lower} at line {series.idxmin()}"
|
|
179
|
+
else:
|
|
180
|
+
note = f"Found value of {maxv}, above constraint {upper} at line {series.idxmax()}"
|
|
181
|
+
return False, note
|
|
182
|
+
|
|
183
|
+
return (True, None) if last_chunk else (None, None)
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def missing_field_note(field_name: str, data: pd.DataFrame) -> str:
|
|
187
|
+
"""Format a consistent diagnostic note when a required column is absent.
|
|
188
|
+
|
|
189
|
+
Parameters
|
|
190
|
+
----------
|
|
191
|
+
field_name
|
|
192
|
+
Name of the missing column.
|
|
193
|
+
data
|
|
194
|
+
DataFrame whose columns were inspected.
|
|
195
|
+
|
|
196
|
+
Returns
|
|
197
|
+
-------
|
|
198
|
+
str
|
|
199
|
+
Human-readable diagnostic message listing available columns.
|
|
200
|
+
"""
|
|
201
|
+
return f"Field {field_name} not available in {list(data.columns)}"
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def parse_date_bound(value: Any) -> pd.Timestamp:
|
|
205
|
+
"""Convert a string, year integer, or datetime-like bound to :class:`pd.Timestamp`.
|
|
206
|
+
|
|
207
|
+
Parameters
|
|
208
|
+
----------
|
|
209
|
+
value
|
|
210
|
+
Date string, integer year, or datetime-like object.
|
|
211
|
+
|
|
212
|
+
Returns
|
|
213
|
+
-------
|
|
214
|
+
pandas.Timestamp
|
|
215
|
+
Parsed timestamp bound.
|
|
216
|
+
"""
|
|
217
|
+
if isinstance(value, pd.Timestamp):
|
|
218
|
+
return value
|
|
219
|
+
if isinstance(value, int):
|
|
220
|
+
return pd.to_datetime(str(value))
|
|
221
|
+
return pd.to_datetime(value)
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def resolve_pandera_schema(
|
|
225
|
+
schema: pa.DataFrameSchema | dict[str, Any] | str | pathlib.Path,
|
|
226
|
+
) -> pa.DataFrameSchema:
|
|
227
|
+
"""Resolve a Pandera schema from an object, dict, YAML/JSON file, or COBOL copybook.
|
|
228
|
+
|
|
229
|
+
Parameters
|
|
230
|
+
----------
|
|
231
|
+
schema
|
|
232
|
+
Existing :class:`pandera.pandas.DataFrameSchema`, column-spec dictionary,
|
|
233
|
+
JSON string, or path to a ``.json``, ``.yaml``/``.yml``, or COBOL copybook file.
|
|
234
|
+
|
|
235
|
+
Returns
|
|
236
|
+
-------
|
|
237
|
+
pandera.pandas.DataFrameSchema
|
|
238
|
+
Resolved Pandera DataFrame schema.
|
|
239
|
+
"""
|
|
240
|
+
if isinstance(schema, pa.DataFrameSchema):
|
|
241
|
+
return schema
|
|
242
|
+
|
|
243
|
+
if isinstance(schema, dict):
|
|
244
|
+
columns: dict[str, pa.Column] = {}
|
|
245
|
+
for col_name, spec in schema.items():
|
|
246
|
+
if isinstance(spec, pa.Column):
|
|
247
|
+
columns[col_name] = spec
|
|
248
|
+
elif isinstance(spec, dict):
|
|
249
|
+
col_checks = build_pandera_checks(spec.get("checks"))
|
|
250
|
+
columns[col_name] = pa.Column(
|
|
251
|
+
dtype=spec.get("dtype"),
|
|
252
|
+
nullable=bool(spec.get("nullable", False)),
|
|
253
|
+
unique=bool(spec.get("unique", False)),
|
|
254
|
+
coerce=bool(spec.get("coerce", False)),
|
|
255
|
+
checks=col_checks,
|
|
256
|
+
)
|
|
257
|
+
else:
|
|
258
|
+
columns[col_name] = pa.Column(dtype=spec)
|
|
259
|
+
return pa.DataFrameSchema(columns=columns)
|
|
260
|
+
|
|
261
|
+
path = pathlib.Path(schema)
|
|
262
|
+
suffix = path.suffix.lower()
|
|
263
|
+
if suffix in (".cbl", ".cpy", ".cob"):
|
|
264
|
+
from dataspector.coboller import Copybook
|
|
265
|
+
|
|
266
|
+
return Copybook(path).to_pandera_schema()
|
|
267
|
+
if suffix == ".json":
|
|
268
|
+
return pa.DataFrameSchema.from_json(path)
|
|
269
|
+
if suffix in (".yml", ".yaml"):
|
|
270
|
+
return pa.DataFrameSchema.from_yaml(path)
|
|
271
|
+
|
|
272
|
+
parsed = json.loads(str(schema))
|
|
273
|
+
return resolve_pandera_schema(parsed)
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def source_byte_size(source: GenericData) -> int:
|
|
277
|
+
"""Return file byte size for a file-backed or COBOL :class:`~dataspector.core.GenericData`.
|
|
278
|
+
|
|
279
|
+
Parameters
|
|
280
|
+
----------
|
|
281
|
+
source
|
|
282
|
+
Data source wrapper backed by a local file or COBOL data file.
|
|
283
|
+
|
|
284
|
+
Returns
|
|
285
|
+
-------
|
|
286
|
+
int
|
|
287
|
+
Size of the underlying data file in bytes.
|
|
288
|
+
"""
|
|
289
|
+
from dataspector.coboller import Copybook
|
|
290
|
+
|
|
291
|
+
if isinstance(source.path, Copybook):
|
|
292
|
+
if isinstance(source.engine, pathlib.Path):
|
|
293
|
+
return source.engine.stat().st_size
|
|
294
|
+
raise ValueError("COBOL size check requires an underlying data file path.")
|
|
295
|
+
if source.path is not None and source.path.exists():
|
|
296
|
+
return source.path.stat().st_size
|
|
297
|
+
raise FileNotFoundError(f"Cannot determine byte size for source {source!r}.")
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
def split_constraint(constraint: str | Sequence[Any]) -> tuple[Any, Any]:
|
|
301
|
+
"""Split a string like ``'(0, 10)'`` / ``'[2020-01-01, 2021-01-01]'`` or 2-sequence.
|
|
302
|
+
|
|
303
|
+
Parameters
|
|
304
|
+
----------
|
|
305
|
+
constraint
|
|
306
|
+
Two-element sequence or bracketed comma-separated string.
|
|
307
|
+
|
|
308
|
+
Returns
|
|
309
|
+
-------
|
|
310
|
+
tuple[Any, Any]
|
|
311
|
+
``(lower, upper)`` raw bound values.
|
|
312
|
+
"""
|
|
313
|
+
if isinstance(constraint, str):
|
|
314
|
+
cleaned = constraint.strip().strip("()[]")
|
|
315
|
+
parts = [p.strip() for p in cleaned.split(",")]
|
|
316
|
+
if len(parts) != 2:
|
|
317
|
+
raise ValueError(f"Expected 2 bounds in constraint {constraint!r}.")
|
|
318
|
+
return parts[0], parts[1]
|
|
319
|
+
if len(constraint) != 2:
|
|
320
|
+
raise ValueError(f"Expected 2 bounds in constraint {constraint!r}.")
|
|
321
|
+
return constraint[0], constraint[1]
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
def truncate_lines(indices: Sequence[Any], limit: int = 10) -> list[Any]:
|
|
325
|
+
"""Return up to *limit* row indices with an overflow marker if truncated.
|
|
326
|
+
|
|
327
|
+
Parameters
|
|
328
|
+
----------
|
|
329
|
+
indices
|
|
330
|
+
Sequence of failing row indices.
|
|
331
|
+
limit
|
|
332
|
+
Maximum number of indices to retain before appending ``"and more"``.
|
|
333
|
+
|
|
334
|
+
Returns
|
|
335
|
+
-------
|
|
336
|
+
list[Any]
|
|
337
|
+
List of row indices, optionally followed by ``"and more"``.
|
|
338
|
+
"""
|
|
339
|
+
items = list(indices)
|
|
340
|
+
if len(items) > limit:
|
|
341
|
+
return [*items[:limit], "and more"]
|
|
342
|
+
return items
|