dataspector 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dataspector/__init__.py +64 -0
- dataspector/__main__.py +10 -0
- dataspector/_connectors.py +260 -0
- dataspector/_reporting.py +480 -0
- dataspector/checkers/__init__.py +85 -0
- dataspector/checkers/record.py +671 -0
- dataspector/checkers/source.py +344 -0
- dataspector/checkers/utils.py +342 -0
- dataspector/cli.py +207 -0
- dataspector/coboller/__init__.py +123 -0
- dataspector/coboller/codecs.py +841 -0
- dataspector/coboller/constants.py +324 -0
- dataspector/coboller/copybook.py +1448 -0
- dataspector/coboller/parser.py +201 -0
- dataspector/core.py +740 -0
- dataspector/py.typed +0 -0
- dataspector/settings.py +101 -0
- dataspector-1.0.0.dist-info/METADATA +181 -0
- dataspector-1.0.0.dist-info/RECORD +22 -0
- dataspector-1.0.0.dist-info/WHEEL +4 -0
- dataspector-1.0.0.dist-info/entry_points.txt +3 -0
- dataspector-1.0.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,480 @@
|
|
|
1
|
+
"""Excel driver workbook loading, terminal formatting, and Excel report export."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import ast
|
|
6
|
+
import inspect
|
|
7
|
+
import json
|
|
8
|
+
import pathlib
|
|
9
|
+
from collections import defaultdict
|
|
10
|
+
from collections.abc import Iterator, Sequence
|
|
11
|
+
from typing import TYPE_CHECKING, Any
|
|
12
|
+
|
|
13
|
+
import pandas as pd
|
|
14
|
+
import xlsxwriter
|
|
15
|
+
from loguru import logger
|
|
16
|
+
|
|
17
|
+
from dataspector import checkers
|
|
18
|
+
|
|
19
|
+
if TYPE_CHECKING:
|
|
20
|
+
from dataspector.core import CheckeredHistory
|
|
21
|
+
|
|
22
|
+
available_checks: list[str] = [
|
|
23
|
+
name
|
|
24
|
+
for name, obj in checkers.__dict__.items()
|
|
25
|
+
if callable(obj) and (name.startswith("source_") or name.startswith("record_"))
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
SUPPORTED_EXPORT_FORMATS: frozenset[str] = frozenset(
|
|
29
|
+
{".csv", ".tsv", ".jsonl", ".json", ".xlsx", ".parquet"}
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def chunks_to_dataframe(
|
|
34
|
+
chunks: Iterator[Any],
|
|
35
|
+
nrows: int | None = None,
|
|
36
|
+
empty_columns: Sequence[Any] | None = None,
|
|
37
|
+
) -> pd.DataFrame:
|
|
38
|
+
"""Materialize streamed DataFrame chunks into a single :class:`pandas.DataFrame`.
|
|
39
|
+
|
|
40
|
+
Parameters
|
|
41
|
+
----------
|
|
42
|
+
chunks
|
|
43
|
+
Iterator yielding :class:`pandas.DataFrame` chunks.
|
|
44
|
+
nrows
|
|
45
|
+
Optional maximum number of rows to collect before stopping iteration.
|
|
46
|
+
empty_columns
|
|
47
|
+
Column names to use if *chunks* yields no rows.
|
|
48
|
+
|
|
49
|
+
Returns
|
|
50
|
+
-------
|
|
51
|
+
pandas.DataFrame
|
|
52
|
+
Concatenated records.
|
|
53
|
+
"""
|
|
54
|
+
frames: list[pd.DataFrame] = []
|
|
55
|
+
collected = 0
|
|
56
|
+
|
|
57
|
+
for chunk in chunks:
|
|
58
|
+
if not isinstance(chunk, pd.DataFrame):
|
|
59
|
+
continue
|
|
60
|
+
if nrows is not None:
|
|
61
|
+
remaining = nrows - collected
|
|
62
|
+
if remaining <= 0:
|
|
63
|
+
break
|
|
64
|
+
if len(chunk) > remaining:
|
|
65
|
+
chunk = chunk.iloc[:remaining]
|
|
66
|
+
frames.append(chunk)
|
|
67
|
+
collected += len(chunk)
|
|
68
|
+
if nrows is not None and collected >= nrows:
|
|
69
|
+
break
|
|
70
|
+
|
|
71
|
+
if not frames:
|
|
72
|
+
return pd.DataFrame(columns=list(empty_columns) if empty_columns is not None else [])
|
|
73
|
+
return pd.concat(frames, ignore_index=True)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def export_chunks(
|
|
77
|
+
chunks: Iterator[Any],
|
|
78
|
+
output: str | pathlib.Path,
|
|
79
|
+
empty_columns: Sequence[Any] | None = None,
|
|
80
|
+
) -> pathlib.Path:
|
|
81
|
+
"""Stream DataFrame chunks to CSV, TSV, JSONL, JSON, Excel, or Parquet.
|
|
82
|
+
|
|
83
|
+
Parameters
|
|
84
|
+
----------
|
|
85
|
+
chunks
|
|
86
|
+
Iterator yielding :class:`pandas.DataFrame` chunks.
|
|
87
|
+
output
|
|
88
|
+
Destination file path whose extension selects the output format.
|
|
89
|
+
empty_columns
|
|
90
|
+
Column names written when *chunks* yields no rows.
|
|
91
|
+
|
|
92
|
+
Returns
|
|
93
|
+
-------
|
|
94
|
+
pathlib.Path
|
|
95
|
+
Path to the written output file.
|
|
96
|
+
|
|
97
|
+
Raises
|
|
98
|
+
------
|
|
99
|
+
ValueError
|
|
100
|
+
If the destination extension is not in :data:`SUPPORTED_EXPORT_FORMATS`.
|
|
101
|
+
"""
|
|
102
|
+
out_path = pathlib.Path(output)
|
|
103
|
+
suffix = out_path.suffix.lower()
|
|
104
|
+
if suffix not in SUPPORTED_EXPORT_FORMATS:
|
|
105
|
+
raise ValueError(
|
|
106
|
+
f"Unsupported output format {suffix!r}; choose from {sorted(SUPPORTED_EXPORT_FORMATS)}."
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
out_path.parent.mkdir(parents=True, exist_ok=True)
|
|
110
|
+
|
|
111
|
+
if suffix in (".csv", ".tsv", ".jsonl"):
|
|
112
|
+
sep = "\t" if suffix == ".tsv" else ","
|
|
113
|
+
first = True
|
|
114
|
+
for chunk in chunks:
|
|
115
|
+
if not isinstance(chunk, pd.DataFrame):
|
|
116
|
+
continue
|
|
117
|
+
mode = "w" if first else "a"
|
|
118
|
+
if suffix == ".jsonl":
|
|
119
|
+
chunk.to_json(out_path, orient="records", lines=True, mode=mode, force_ascii=False)
|
|
120
|
+
else:
|
|
121
|
+
chunk.to_csv(out_path, sep=sep, index=False, header=first, mode=mode)
|
|
122
|
+
first = False
|
|
123
|
+
if first:
|
|
124
|
+
empty_df = pd.DataFrame(
|
|
125
|
+
columns=list(empty_columns) if empty_columns is not None else []
|
|
126
|
+
)
|
|
127
|
+
if suffix == ".jsonl":
|
|
128
|
+
out_path.write_text("", encoding="utf-8")
|
|
129
|
+
else:
|
|
130
|
+
empty_df.to_csv(out_path, sep=sep, index=False)
|
|
131
|
+
return out_path
|
|
132
|
+
|
|
133
|
+
df = chunks_to_dataframe(chunks, empty_columns=empty_columns)
|
|
134
|
+
if suffix == ".json":
|
|
135
|
+
df.to_json(out_path, orient="records", indent=2, force_ascii=False)
|
|
136
|
+
elif suffix == ".xlsx":
|
|
137
|
+
df.to_excel(out_path, index=False)
|
|
138
|
+
elif suffix == ".parquet":
|
|
139
|
+
df.to_parquet(out_path, index=False)
|
|
140
|
+
return out_path
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def load_driverfile(
|
|
144
|
+
driver: str | pathlib.Path,
|
|
145
|
+
) -> dict[str, list[tuple[str, dict[str, Any] | None, Any, dict[str, Any]]]]:
|
|
146
|
+
"""Load an Excel driver workbook containing ``Location Details`` and ``Checks to Run`` sheets.
|
|
147
|
+
|
|
148
|
+
Parameters
|
|
149
|
+
----------
|
|
150
|
+
driver
|
|
151
|
+
Path to the Excel driver workbook.
|
|
152
|
+
|
|
153
|
+
Returns
|
|
154
|
+
-------
|
|
155
|
+
dict[str, list[tuple[str, dict[str, Any] | None, Any, dict[str, Any]]]]
|
|
156
|
+
Mapping of resolved source locator to list of
|
|
157
|
+
``(check_name, kwargs, severity, reader_args)`` tuples.
|
|
158
|
+
"""
|
|
159
|
+
file_info = pd.read_excel(driver, sheet_name="Location Details", header=1)
|
|
160
|
+
checks_df = pd.read_excel(driver, sheet_name="Checks to Run", header=3)
|
|
161
|
+
|
|
162
|
+
sources = file_info["Source Name"].astype(str).copy()
|
|
163
|
+
locations = file_info["Location"].fillna("").astype(str)
|
|
164
|
+
full_paths = locations + sources
|
|
165
|
+
|
|
166
|
+
dup_mask = sources.duplicated(keep=False)
|
|
167
|
+
if bool(dup_mask.any()):
|
|
168
|
+
logger.error(
|
|
169
|
+
"Duplicate Source Name(s) {} in Location Details; falling back to full paths.",
|
|
170
|
+
sources[dup_mask].unique().tolist(),
|
|
171
|
+
)
|
|
172
|
+
sources.loc[dup_mask] = full_paths.loc[dup_mask]
|
|
173
|
+
|
|
174
|
+
to_check: dict[str, list[tuple[str, dict[str, Any] | None, Any, dict[str, Any]]]] = defaultdict(
|
|
175
|
+
list
|
|
176
|
+
)
|
|
177
|
+
|
|
178
|
+
for row in checks_df.to_dict("records"):
|
|
179
|
+
if row.get("Process") not in ("Y", "y", "1", 1, True):
|
|
180
|
+
continue
|
|
181
|
+
|
|
182
|
+
source_ref = str(row.get("Source Name", ""))
|
|
183
|
+
match_mask = (sources == source_ref) | (full_paths == source_ref)
|
|
184
|
+
if not bool(match_mask.any()):
|
|
185
|
+
logger.error(
|
|
186
|
+
"Skipping check with unknown/ambiguous source: {} {}",
|
|
187
|
+
source_ref,
|
|
188
|
+
row.get("Check"),
|
|
189
|
+
)
|
|
190
|
+
continue
|
|
191
|
+
|
|
192
|
+
location = str(full_paths.loc[match_mask].iloc[0])
|
|
193
|
+
reader_series = file_info.loc[match_mask, "Reader Arguments"].dropna()
|
|
194
|
+
if len(reader_series) == 1:
|
|
195
|
+
reader_args = parse_cell_to_dict(reader_series.iloc[0]) or {}
|
|
196
|
+
else:
|
|
197
|
+
if len(reader_series) > 1:
|
|
198
|
+
logger.error("Source {} has duplicate Reader Arguments; ignoring.", source_ref)
|
|
199
|
+
reader_args = {}
|
|
200
|
+
|
|
201
|
+
check_name = str(row.get("Check", ""))
|
|
202
|
+
if check_name not in available_checks:
|
|
203
|
+
logger.error("Check '{}' does not exist for {}.", check_name, source_ref)
|
|
204
|
+
|
|
205
|
+
default_key: str | None = None
|
|
206
|
+
check_fn = getattr(checkers, check_name, None)
|
|
207
|
+
if callable(check_fn):
|
|
208
|
+
params = list(inspect.signature(check_fn).parameters)
|
|
209
|
+
if len(params) > 1:
|
|
210
|
+
default_key = params[1]
|
|
211
|
+
|
|
212
|
+
kwargs = parse_cell_to_dict(row.get("Keyword Arguments"), default_key=default_key)
|
|
213
|
+
severity = row.get("Severity", "")
|
|
214
|
+
to_check[location].append((check_name, kwargs, severity, reader_args))
|
|
215
|
+
|
|
216
|
+
return dict(to_check)
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def parse_cell_to_dict(cell: Any, default_key: str | None = None) -> dict[str, Any] | None:
|
|
220
|
+
"""Parse flexible Excel cell text (JSON, Python literal, ``key=val``, or scalar) into a dict.
|
|
221
|
+
|
|
222
|
+
Parameters
|
|
223
|
+
----------
|
|
224
|
+
cell
|
|
225
|
+
Cell value read from an Excel driver workbook.
|
|
226
|
+
default_key
|
|
227
|
+
Key name assigned when *cell* holds a single scalar value.
|
|
228
|
+
|
|
229
|
+
Returns
|
|
230
|
+
-------
|
|
231
|
+
dict[str, Any] | None
|
|
232
|
+
Parsed keyword arguments dictionary, or ``None`` if empty/unparseable.
|
|
233
|
+
"""
|
|
234
|
+
if cell is None or (isinstance(cell, float) and pd.isna(cell)):
|
|
235
|
+
return None
|
|
236
|
+
|
|
237
|
+
if isinstance(cell, dict):
|
|
238
|
+
return cell
|
|
239
|
+
|
|
240
|
+
if isinstance(cell, str):
|
|
241
|
+
stripped = cell.strip()
|
|
242
|
+
if not stripped:
|
|
243
|
+
return None
|
|
244
|
+
|
|
245
|
+
for parser in (json.loads, ast.literal_eval):
|
|
246
|
+
for candidate in (stripped, f"{{{stripped}}}"):
|
|
247
|
+
try:
|
|
248
|
+
parsed = parser(candidate)
|
|
249
|
+
if isinstance(parsed, dict):
|
|
250
|
+
return parsed
|
|
251
|
+
except Exception:
|
|
252
|
+
continue
|
|
253
|
+
|
|
254
|
+
if "=" in stripped:
|
|
255
|
+
kv_dict: dict[str, Any] = {}
|
|
256
|
+
for line in stripped.splitlines():
|
|
257
|
+
for part in line.split(","):
|
|
258
|
+
if "=" in part:
|
|
259
|
+
raw_k, raw_v = part.split("=", 1)
|
|
260
|
+
k_str = raw_k.strip()
|
|
261
|
+
try:
|
|
262
|
+
key = str(ast.literal_eval(k_str))
|
|
263
|
+
except Exception:
|
|
264
|
+
key = k_str
|
|
265
|
+
sub = parse_cell_to_dict(raw_v.strip(), default_key=key)
|
|
266
|
+
if sub:
|
|
267
|
+
kv_dict.update(sub)
|
|
268
|
+
if kv_dict:
|
|
269
|
+
return kv_dict
|
|
270
|
+
|
|
271
|
+
if default_key is not None:
|
|
272
|
+
try:
|
|
273
|
+
return {default_key: ast.literal_eval(stripped)}
|
|
274
|
+
except Exception:
|
|
275
|
+
return {default_key: stripped}
|
|
276
|
+
return None
|
|
277
|
+
|
|
278
|
+
if default_key is not None:
|
|
279
|
+
return {default_key: cell}
|
|
280
|
+
return None
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
def pretty_print_results(
|
|
284
|
+
results: CheckeredHistory | Sequence[CheckeredHistory],
|
|
285
|
+
maxlen: int = 40,
|
|
286
|
+
) -> str:
|
|
287
|
+
"""Format and print check results as an aligned ASCII table.
|
|
288
|
+
|
|
289
|
+
Parameters
|
|
290
|
+
----------
|
|
291
|
+
results
|
|
292
|
+
Results to render.
|
|
293
|
+
maxlen
|
|
294
|
+
Maximum column width before wrapping onto additional lines.
|
|
295
|
+
|
|
296
|
+
Returns
|
|
297
|
+
-------
|
|
298
|
+
str
|
|
299
|
+
Rendered table text (also printed to stdout).
|
|
300
|
+
"""
|
|
301
|
+
from dataspector.core import CheckeredHistory
|
|
302
|
+
|
|
303
|
+
result_list: list[CheckeredHistory] = (
|
|
304
|
+
[results] if isinstance(results, CheckeredHistory) else list(results)
|
|
305
|
+
)
|
|
306
|
+
lines: list[str] = [
|
|
307
|
+
f"Printing results of checks across {len(result_list)} locations",
|
|
308
|
+
"",
|
|
309
|
+
]
|
|
310
|
+
|
|
311
|
+
table: list[tuple[str, ...]] = [("Pass", "Location", "Check", "kwargs", "Notes")]
|
|
312
|
+
for res in result_list:
|
|
313
|
+
loc_display = getattr(res.location, "_location", res.location)
|
|
314
|
+
for idx, check in enumerate(res.checklist):
|
|
315
|
+
if isinstance(check, tuple):
|
|
316
|
+
checkname = check[0]
|
|
317
|
+
kwargs: Any = check[1] if len(check) > 1 else {}
|
|
318
|
+
if isinstance(kwargs, dict) and len(kwargs) == 1:
|
|
319
|
+
kwargs = next(iter(kwargs.values()))
|
|
320
|
+
else:
|
|
321
|
+
checkname = check
|
|
322
|
+
kwargs = {}
|
|
323
|
+
|
|
324
|
+
notes_text = " ".join(
|
|
325
|
+
part for part in (res.notes.get(idx), res.exceptions.get(idx)) if part
|
|
326
|
+
)
|
|
327
|
+
row = (
|
|
328
|
+
str(res.results.get(idx)),
|
|
329
|
+
str(loc_display),
|
|
330
|
+
str(checkname),
|
|
331
|
+
str(kwargs),
|
|
332
|
+
notes_text,
|
|
333
|
+
)
|
|
334
|
+
n_wraps = max((len(col) - 1) // maxlen + 1 if col else 1 for col in row)
|
|
335
|
+
for wrap_idx in range(n_wraps):
|
|
336
|
+
sl = slice(wrap_idx * maxlen, (wrap_idx + 1) * maxlen)
|
|
337
|
+
table.append(tuple(col[sl] for col in row))
|
|
338
|
+
|
|
339
|
+
widths = [max(len(r[col_i]) for r in table) for col_i in range(len(table[0]))]
|
|
340
|
+
for r_idx, row in enumerate(table):
|
|
341
|
+
parts = [f"{row[0].ljust(widths[0] + 1)}||"]
|
|
342
|
+
for col_i in range(1, len(row)):
|
|
343
|
+
parts.append(f"{row[col_i].rjust(widths[col_i] + 1)} |")
|
|
344
|
+
lines.append("".join(parts))
|
|
345
|
+
if r_idx == 0:
|
|
346
|
+
lines.append("-" * (sum(widths) + 14))
|
|
347
|
+
|
|
348
|
+
output = "\n".join(lines)
|
|
349
|
+
print(output)
|
|
350
|
+
return output
|
|
351
|
+
|
|
352
|
+
|
|
353
|
+
def version_adder(fout: str | pathlib.Path, extension: str | None = None) -> str:
|
|
354
|
+
"""Return a non-colliding output path, appending ``_v01``, ``_v02``, ... if needed.
|
|
355
|
+
|
|
356
|
+
Parameters
|
|
357
|
+
----------
|
|
358
|
+
fout
|
|
359
|
+
Desired output file path.
|
|
360
|
+
extension
|
|
361
|
+
Optional required file extension (e.g. ``'.xlsx'``).
|
|
362
|
+
|
|
363
|
+
Returns
|
|
364
|
+
-------
|
|
365
|
+
str
|
|
366
|
+
Unique output path string.
|
|
367
|
+
"""
|
|
368
|
+
path = pathlib.Path(fout)
|
|
369
|
+
if extension is not None:
|
|
370
|
+
ext = f".{extension.lstrip('.')}"
|
|
371
|
+
if path.suffix.lower() != ext.lower():
|
|
372
|
+
logger.info("Appending {} extension to {}", ext, path)
|
|
373
|
+
path = path.with_suffix(ext) if path.suffix else pathlib.Path(f"{path}{ext}")
|
|
374
|
+
|
|
375
|
+
candidate = path
|
|
376
|
+
n = 1
|
|
377
|
+
while candidate.exists():
|
|
378
|
+
candidate = path.with_name(f"{path.stem}_v{n:02d}{path.suffix}")
|
|
379
|
+
n += 1
|
|
380
|
+
return str(candidate)
|
|
381
|
+
|
|
382
|
+
|
|
383
|
+
def write_excel_report(
|
|
384
|
+
results: CheckeredHistory | Sequence[CheckeredHistory],
|
|
385
|
+
excel_file_name: str | pathlib.Path,
|
|
386
|
+
) -> pathlib.Path:
|
|
387
|
+
"""Write formatted check results to an Excel workbook with conditional highlighting.
|
|
388
|
+
|
|
389
|
+
Parameters
|
|
390
|
+
----------
|
|
391
|
+
results
|
|
392
|
+
Check results produced by :func:`~dataspector.core.run_checks_on_location` or
|
|
393
|
+
:func:`~dataspector.core.load_and_run_checks`.
|
|
394
|
+
excel_file_name
|
|
395
|
+
Target ``.xlsx`` output path (auto-versioned if it already exists).
|
|
396
|
+
|
|
397
|
+
Returns
|
|
398
|
+
-------
|
|
399
|
+
pathlib.Path
|
|
400
|
+
Path to the written Excel report.
|
|
401
|
+
"""
|
|
402
|
+
from dataspector.core import CheckeredHistory
|
|
403
|
+
|
|
404
|
+
result_list: list[CheckeredHistory] = (
|
|
405
|
+
[results] if isinstance(results, CheckeredHistory) else list(results)
|
|
406
|
+
)
|
|
407
|
+
out_path = pathlib.Path(version_adder(excel_file_name, extension=".xlsx"))
|
|
408
|
+
|
|
409
|
+
workbook = xlsxwriter.Workbook(str(out_path))
|
|
410
|
+
worksheet = workbook.add_worksheet("Results")
|
|
411
|
+
header_row = ["Pass", "Severity", "Location", "Check", "kwargs", "Notes"]
|
|
412
|
+
header = [{"header": col} for col in header_row]
|
|
413
|
+
|
|
414
|
+
format_fail = workbook.add_format({"bg_color": "#FF6B6B", "font_color": "#000000"})
|
|
415
|
+
format_warn = workbook.add_format({"bg_color": "#FADA5E", "font_color": "#000000"})
|
|
416
|
+
format_pass = workbook.add_format({"bg_color": "#ADD8E6", "font_color": "#000000"})
|
|
417
|
+
|
|
418
|
+
row_idx = 1
|
|
419
|
+
for res in result_list:
|
|
420
|
+
loc_display = getattr(res.location, "_location", res.location)
|
|
421
|
+
for idx, check in enumerate(res.checklist):
|
|
422
|
+
if isinstance(check, tuple):
|
|
423
|
+
checkname = check[0]
|
|
424
|
+
kwargs = check[1] if len(check) > 1 else {}
|
|
425
|
+
severity = check[2] if len(check) > 2 else ""
|
|
426
|
+
else:
|
|
427
|
+
checkname = check
|
|
428
|
+
kwargs = {}
|
|
429
|
+
severity = ""
|
|
430
|
+
|
|
431
|
+
notes_text = " ".join(
|
|
432
|
+
part for part in (res.notes.get(idx), res.exceptions.get(idx)) if part
|
|
433
|
+
)
|
|
434
|
+
worksheet.write_row(
|
|
435
|
+
row_idx,
|
|
436
|
+
0,
|
|
437
|
+
[
|
|
438
|
+
str(res.results.get(idx)),
|
|
439
|
+
str(severity).upper(),
|
|
440
|
+
str(loc_display),
|
|
441
|
+
str(checkname),
|
|
442
|
+
str(kwargs),
|
|
443
|
+
notes_text,
|
|
444
|
+
],
|
|
445
|
+
)
|
|
446
|
+
row_idx += 1
|
|
447
|
+
|
|
448
|
+
last_row = max(row_idx - 1, 1)
|
|
449
|
+
worksheet.add_table(
|
|
450
|
+
0,
|
|
451
|
+
0,
|
|
452
|
+
last_row,
|
|
453
|
+
len(header_row) - 1,
|
|
454
|
+
{"header_row": True, "first_column": True, "columns": header},
|
|
455
|
+
)
|
|
456
|
+
cell_range = f"A2:F{last_row + 1}"
|
|
457
|
+
worksheet.conditional_format(
|
|
458
|
+
cell_range,
|
|
459
|
+
{"type": "formula", "criteria": '=LEFT($A2, 4)="TRUE"', "format": format_pass},
|
|
460
|
+
)
|
|
461
|
+
worksheet.conditional_format(
|
|
462
|
+
cell_range,
|
|
463
|
+
{
|
|
464
|
+
"type": "formula",
|
|
465
|
+
"criteria": '=AND(LEFT($A2, 5)="FALSE", LEFT($B2, 7)="WARNING")',
|
|
466
|
+
"format": format_warn,
|
|
467
|
+
},
|
|
468
|
+
)
|
|
469
|
+
worksheet.conditional_format(
|
|
470
|
+
cell_range,
|
|
471
|
+
{
|
|
472
|
+
"type": "formula",
|
|
473
|
+
"criteria": '=AND(LEFT($A2, 5)="FALSE", NOT(LEFT($B2, 7)="WARNING"))',
|
|
474
|
+
"format": format_fail,
|
|
475
|
+
},
|
|
476
|
+
)
|
|
477
|
+
worksheet.set_column(0, 1, 12)
|
|
478
|
+
worksheet.set_column(2, 5, 40)
|
|
479
|
+
workbook.close()
|
|
480
|
+
return out_path
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
"""Format-agnostic data-integrity checkers.
|
|
2
|
+
|
|
3
|
+
Public facade re-exporting:
|
|
4
|
+
|
|
5
|
+
* **Source-level checks** (``source_*``) from :mod:`dataspector.checkers.source`
|
|
6
|
+
* **Record-level checks** (``record_*``) from :mod:`dataspector.checkers.record`
|
|
7
|
+
* :class:`~dataspector.checkers.utils.CheckOutcome`,
|
|
8
|
+
:func:`~dataspector.checkers.utils.requires_all_data`, and
|
|
9
|
+
``REQUIRES_ALL_DATA`` from :mod:`dataspector.checkers.utils`
|
|
10
|
+
|
|
11
|
+
Every check returns a :class:`~dataspector.checkers.utils.CheckOutcome` whose
|
|
12
|
+
``passed`` attribute is:
|
|
13
|
+
|
|
14
|
+
* ``True`` — check passed;
|
|
15
|
+
* ``False`` — check failed;
|
|
16
|
+
* ``None`` — indeterminate so far; keep streaming subsequent chunks.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
from dataspector.checkers.record import (
|
|
22
|
+
record_all_data_not_null,
|
|
23
|
+
record_contains,
|
|
24
|
+
record_contains_at_least,
|
|
25
|
+
record_count,
|
|
26
|
+
record_count_greater_than,
|
|
27
|
+
record_field_contains,
|
|
28
|
+
record_field_contains_at_least,
|
|
29
|
+
record_field_contains_only,
|
|
30
|
+
record_field_date_between,
|
|
31
|
+
record_field_is_numeric,
|
|
32
|
+
record_field_is_unique,
|
|
33
|
+
record_field_not_null,
|
|
34
|
+
record_field_numeric_value_between,
|
|
35
|
+
record_if_field1_startswith_field2_contains_only,
|
|
36
|
+
record_validate_schema,
|
|
37
|
+
)
|
|
38
|
+
from dataspector.checkers.source import (
|
|
39
|
+
source_exists,
|
|
40
|
+
source_fuzzy_find,
|
|
41
|
+
source_has_N_fields,
|
|
42
|
+
source_modified_on_date,
|
|
43
|
+
source_modified_recently,
|
|
44
|
+
source_source_identical,
|
|
45
|
+
source_source_roughly_same_number_of_records,
|
|
46
|
+
source_source_same_fields,
|
|
47
|
+
source_source_size_roughly_same,
|
|
48
|
+
source_storage_size_roughly,
|
|
49
|
+
)
|
|
50
|
+
from dataspector.checkers.utils import (
|
|
51
|
+
REQUIRES_ALL_DATA,
|
|
52
|
+
CheckOutcome,
|
|
53
|
+
requires_all_data,
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
__all__ = [
|
|
57
|
+
"REQUIRES_ALL_DATA",
|
|
58
|
+
"CheckOutcome",
|
|
59
|
+
"record_all_data_not_null",
|
|
60
|
+
"record_contains",
|
|
61
|
+
"record_contains_at_least",
|
|
62
|
+
"record_count",
|
|
63
|
+
"record_count_greater_than",
|
|
64
|
+
"record_field_contains",
|
|
65
|
+
"record_field_contains_at_least",
|
|
66
|
+
"record_field_contains_only",
|
|
67
|
+
"record_field_date_between",
|
|
68
|
+
"record_field_is_numeric",
|
|
69
|
+
"record_field_is_unique",
|
|
70
|
+
"record_field_not_null",
|
|
71
|
+
"record_field_numeric_value_between",
|
|
72
|
+
"record_if_field1_startswith_field2_contains_only",
|
|
73
|
+
"record_validate_schema",
|
|
74
|
+
"requires_all_data",
|
|
75
|
+
"source_exists",
|
|
76
|
+
"source_fuzzy_find",
|
|
77
|
+
"source_has_N_fields",
|
|
78
|
+
"source_modified_on_date",
|
|
79
|
+
"source_modified_recently",
|
|
80
|
+
"source_source_identical",
|
|
81
|
+
"source_source_roughly_same_number_of_records",
|
|
82
|
+
"source_source_same_fields",
|
|
83
|
+
"source_source_size_roughly_same",
|
|
84
|
+
"source_storage_size_roughly",
|
|
85
|
+
]
|