dataspector 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,480 @@
1
+ """Excel driver workbook loading, terminal formatting, and Excel report export."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import ast
6
+ import inspect
7
+ import json
8
+ import pathlib
9
+ from collections import defaultdict
10
+ from collections.abc import Iterator, Sequence
11
+ from typing import TYPE_CHECKING, Any
12
+
13
+ import pandas as pd
14
+ import xlsxwriter
15
+ from loguru import logger
16
+
17
+ from dataspector import checkers
18
+
19
+ if TYPE_CHECKING:
20
+ from dataspector.core import CheckeredHistory
21
+
22
+ available_checks: list[str] = [
23
+ name
24
+ for name, obj in checkers.__dict__.items()
25
+ if callable(obj) and (name.startswith("source_") or name.startswith("record_"))
26
+ ]
27
+
28
+ SUPPORTED_EXPORT_FORMATS: frozenset[str] = frozenset(
29
+ {".csv", ".tsv", ".jsonl", ".json", ".xlsx", ".parquet"}
30
+ )
31
+
32
+
33
+ def chunks_to_dataframe(
34
+ chunks: Iterator[Any],
35
+ nrows: int | None = None,
36
+ empty_columns: Sequence[Any] | None = None,
37
+ ) -> pd.DataFrame:
38
+ """Materialize streamed DataFrame chunks into a single :class:`pandas.DataFrame`.
39
+
40
+ Parameters
41
+ ----------
42
+ chunks
43
+ Iterator yielding :class:`pandas.DataFrame` chunks.
44
+ nrows
45
+ Optional maximum number of rows to collect before stopping iteration.
46
+ empty_columns
47
+ Column names to use if *chunks* yields no rows.
48
+
49
+ Returns
50
+ -------
51
+ pandas.DataFrame
52
+ Concatenated records.
53
+ """
54
+ frames: list[pd.DataFrame] = []
55
+ collected = 0
56
+
57
+ for chunk in chunks:
58
+ if not isinstance(chunk, pd.DataFrame):
59
+ continue
60
+ if nrows is not None:
61
+ remaining = nrows - collected
62
+ if remaining <= 0:
63
+ break
64
+ if len(chunk) > remaining:
65
+ chunk = chunk.iloc[:remaining]
66
+ frames.append(chunk)
67
+ collected += len(chunk)
68
+ if nrows is not None and collected >= nrows:
69
+ break
70
+
71
+ if not frames:
72
+ return pd.DataFrame(columns=list(empty_columns) if empty_columns is not None else [])
73
+ return pd.concat(frames, ignore_index=True)
74
+
75
+
76
+ def export_chunks(
77
+ chunks: Iterator[Any],
78
+ output: str | pathlib.Path,
79
+ empty_columns: Sequence[Any] | None = None,
80
+ ) -> pathlib.Path:
81
+ """Stream DataFrame chunks to CSV, TSV, JSONL, JSON, Excel, or Parquet.
82
+
83
+ Parameters
84
+ ----------
85
+ chunks
86
+ Iterator yielding :class:`pandas.DataFrame` chunks.
87
+ output
88
+ Destination file path whose extension selects the output format.
89
+ empty_columns
90
+ Column names written when *chunks* yields no rows.
91
+
92
+ Returns
93
+ -------
94
+ pathlib.Path
95
+ Path to the written output file.
96
+
97
+ Raises
98
+ ------
99
+ ValueError
100
+ If the destination extension is not in :data:`SUPPORTED_EXPORT_FORMATS`.
101
+ """
102
+ out_path = pathlib.Path(output)
103
+ suffix = out_path.suffix.lower()
104
+ if suffix not in SUPPORTED_EXPORT_FORMATS:
105
+ raise ValueError(
106
+ f"Unsupported output format {suffix!r}; choose from {sorted(SUPPORTED_EXPORT_FORMATS)}."
107
+ )
108
+
109
+ out_path.parent.mkdir(parents=True, exist_ok=True)
110
+
111
+ if suffix in (".csv", ".tsv", ".jsonl"):
112
+ sep = "\t" if suffix == ".tsv" else ","
113
+ first = True
114
+ for chunk in chunks:
115
+ if not isinstance(chunk, pd.DataFrame):
116
+ continue
117
+ mode = "w" if first else "a"
118
+ if suffix == ".jsonl":
119
+ chunk.to_json(out_path, orient="records", lines=True, mode=mode, force_ascii=False)
120
+ else:
121
+ chunk.to_csv(out_path, sep=sep, index=False, header=first, mode=mode)
122
+ first = False
123
+ if first:
124
+ empty_df = pd.DataFrame(
125
+ columns=list(empty_columns) if empty_columns is not None else []
126
+ )
127
+ if suffix == ".jsonl":
128
+ out_path.write_text("", encoding="utf-8")
129
+ else:
130
+ empty_df.to_csv(out_path, sep=sep, index=False)
131
+ return out_path
132
+
133
+ df = chunks_to_dataframe(chunks, empty_columns=empty_columns)
134
+ if suffix == ".json":
135
+ df.to_json(out_path, orient="records", indent=2, force_ascii=False)
136
+ elif suffix == ".xlsx":
137
+ df.to_excel(out_path, index=False)
138
+ elif suffix == ".parquet":
139
+ df.to_parquet(out_path, index=False)
140
+ return out_path
141
+
142
+
143
+ def load_driverfile(
144
+ driver: str | pathlib.Path,
145
+ ) -> dict[str, list[tuple[str, dict[str, Any] | None, Any, dict[str, Any]]]]:
146
+ """Load an Excel driver workbook containing ``Location Details`` and ``Checks to Run`` sheets.
147
+
148
+ Parameters
149
+ ----------
150
+ driver
151
+ Path to the Excel driver workbook.
152
+
153
+ Returns
154
+ -------
155
+ dict[str, list[tuple[str, dict[str, Any] | None, Any, dict[str, Any]]]]
156
+ Mapping of resolved source locator to list of
157
+ ``(check_name, kwargs, severity, reader_args)`` tuples.
158
+ """
159
+ file_info = pd.read_excel(driver, sheet_name="Location Details", header=1)
160
+ checks_df = pd.read_excel(driver, sheet_name="Checks to Run", header=3)
161
+
162
+ sources = file_info["Source Name"].astype(str).copy()
163
+ locations = file_info["Location"].fillna("").astype(str)
164
+ full_paths = locations + sources
165
+
166
+ dup_mask = sources.duplicated(keep=False)
167
+ if bool(dup_mask.any()):
168
+ logger.error(
169
+ "Duplicate Source Name(s) {} in Location Details; falling back to full paths.",
170
+ sources[dup_mask].unique().tolist(),
171
+ )
172
+ sources.loc[dup_mask] = full_paths.loc[dup_mask]
173
+
174
+ to_check: dict[str, list[tuple[str, dict[str, Any] | None, Any, dict[str, Any]]]] = defaultdict(
175
+ list
176
+ )
177
+
178
+ for row in checks_df.to_dict("records"):
179
+ if row.get("Process") not in ("Y", "y", "1", 1, True):
180
+ continue
181
+
182
+ source_ref = str(row.get("Source Name", ""))
183
+ match_mask = (sources == source_ref) | (full_paths == source_ref)
184
+ if not bool(match_mask.any()):
185
+ logger.error(
186
+ "Skipping check with unknown/ambiguous source: {} {}",
187
+ source_ref,
188
+ row.get("Check"),
189
+ )
190
+ continue
191
+
192
+ location = str(full_paths.loc[match_mask].iloc[0])
193
+ reader_series = file_info.loc[match_mask, "Reader Arguments"].dropna()
194
+ if len(reader_series) == 1:
195
+ reader_args = parse_cell_to_dict(reader_series.iloc[0]) or {}
196
+ else:
197
+ if len(reader_series) > 1:
198
+ logger.error("Source {} has duplicate Reader Arguments; ignoring.", source_ref)
199
+ reader_args = {}
200
+
201
+ check_name = str(row.get("Check", ""))
202
+ if check_name not in available_checks:
203
+ logger.error("Check '{}' does not exist for {}.", check_name, source_ref)
204
+
205
+ default_key: str | None = None
206
+ check_fn = getattr(checkers, check_name, None)
207
+ if callable(check_fn):
208
+ params = list(inspect.signature(check_fn).parameters)
209
+ if len(params) > 1:
210
+ default_key = params[1]
211
+
212
+ kwargs = parse_cell_to_dict(row.get("Keyword Arguments"), default_key=default_key)
213
+ severity = row.get("Severity", "")
214
+ to_check[location].append((check_name, kwargs, severity, reader_args))
215
+
216
+ return dict(to_check)
217
+
218
+
219
+ def parse_cell_to_dict(cell: Any, default_key: str | None = None) -> dict[str, Any] | None:
220
+ """Parse flexible Excel cell text (JSON, Python literal, ``key=val``, or scalar) into a dict.
221
+
222
+ Parameters
223
+ ----------
224
+ cell
225
+ Cell value read from an Excel driver workbook.
226
+ default_key
227
+ Key name assigned when *cell* holds a single scalar value.
228
+
229
+ Returns
230
+ -------
231
+ dict[str, Any] | None
232
+ Parsed keyword arguments dictionary, or ``None`` if empty/unparseable.
233
+ """
234
+ if cell is None or (isinstance(cell, float) and pd.isna(cell)):
235
+ return None
236
+
237
+ if isinstance(cell, dict):
238
+ return cell
239
+
240
+ if isinstance(cell, str):
241
+ stripped = cell.strip()
242
+ if not stripped:
243
+ return None
244
+
245
+ for parser in (json.loads, ast.literal_eval):
246
+ for candidate in (stripped, f"{{{stripped}}}"):
247
+ try:
248
+ parsed = parser(candidate)
249
+ if isinstance(parsed, dict):
250
+ return parsed
251
+ except Exception:
252
+ continue
253
+
254
+ if "=" in stripped:
255
+ kv_dict: dict[str, Any] = {}
256
+ for line in stripped.splitlines():
257
+ for part in line.split(","):
258
+ if "=" in part:
259
+ raw_k, raw_v = part.split("=", 1)
260
+ k_str = raw_k.strip()
261
+ try:
262
+ key = str(ast.literal_eval(k_str))
263
+ except Exception:
264
+ key = k_str
265
+ sub = parse_cell_to_dict(raw_v.strip(), default_key=key)
266
+ if sub:
267
+ kv_dict.update(sub)
268
+ if kv_dict:
269
+ return kv_dict
270
+
271
+ if default_key is not None:
272
+ try:
273
+ return {default_key: ast.literal_eval(stripped)}
274
+ except Exception:
275
+ return {default_key: stripped}
276
+ return None
277
+
278
+ if default_key is not None:
279
+ return {default_key: cell}
280
+ return None
281
+
282
+
283
+ def pretty_print_results(
284
+ results: CheckeredHistory | Sequence[CheckeredHistory],
285
+ maxlen: int = 40,
286
+ ) -> str:
287
+ """Format and print check results as an aligned ASCII table.
288
+
289
+ Parameters
290
+ ----------
291
+ results
292
+ Results to render.
293
+ maxlen
294
+ Maximum column width before wrapping onto additional lines.
295
+
296
+ Returns
297
+ -------
298
+ str
299
+ Rendered table text (also printed to stdout).
300
+ """
301
+ from dataspector.core import CheckeredHistory
302
+
303
+ result_list: list[CheckeredHistory] = (
304
+ [results] if isinstance(results, CheckeredHistory) else list(results)
305
+ )
306
+ lines: list[str] = [
307
+ f"Printing results of checks across {len(result_list)} locations",
308
+ "",
309
+ ]
310
+
311
+ table: list[tuple[str, ...]] = [("Pass", "Location", "Check", "kwargs", "Notes")]
312
+ for res in result_list:
313
+ loc_display = getattr(res.location, "_location", res.location)
314
+ for idx, check in enumerate(res.checklist):
315
+ if isinstance(check, tuple):
316
+ checkname = check[0]
317
+ kwargs: Any = check[1] if len(check) > 1 else {}
318
+ if isinstance(kwargs, dict) and len(kwargs) == 1:
319
+ kwargs = next(iter(kwargs.values()))
320
+ else:
321
+ checkname = check
322
+ kwargs = {}
323
+
324
+ notes_text = " ".join(
325
+ part for part in (res.notes.get(idx), res.exceptions.get(idx)) if part
326
+ )
327
+ row = (
328
+ str(res.results.get(idx)),
329
+ str(loc_display),
330
+ str(checkname),
331
+ str(kwargs),
332
+ notes_text,
333
+ )
334
+ n_wraps = max((len(col) - 1) // maxlen + 1 if col else 1 for col in row)
335
+ for wrap_idx in range(n_wraps):
336
+ sl = slice(wrap_idx * maxlen, (wrap_idx + 1) * maxlen)
337
+ table.append(tuple(col[sl] for col in row))
338
+
339
+ widths = [max(len(r[col_i]) for r in table) for col_i in range(len(table[0]))]
340
+ for r_idx, row in enumerate(table):
341
+ parts = [f"{row[0].ljust(widths[0] + 1)}||"]
342
+ for col_i in range(1, len(row)):
343
+ parts.append(f"{row[col_i].rjust(widths[col_i] + 1)} |")
344
+ lines.append("".join(parts))
345
+ if r_idx == 0:
346
+ lines.append("-" * (sum(widths) + 14))
347
+
348
+ output = "\n".join(lines)
349
+ print(output)
350
+ return output
351
+
352
+
353
+ def version_adder(fout: str | pathlib.Path, extension: str | None = None) -> str:
354
+ """Return a non-colliding output path, appending ``_v01``, ``_v02``, ... if needed.
355
+
356
+ Parameters
357
+ ----------
358
+ fout
359
+ Desired output file path.
360
+ extension
361
+ Optional required file extension (e.g. ``'.xlsx'``).
362
+
363
+ Returns
364
+ -------
365
+ str
366
+ Unique output path string.
367
+ """
368
+ path = pathlib.Path(fout)
369
+ if extension is not None:
370
+ ext = f".{extension.lstrip('.')}"
371
+ if path.suffix.lower() != ext.lower():
372
+ logger.info("Appending {} extension to {}", ext, path)
373
+ path = path.with_suffix(ext) if path.suffix else pathlib.Path(f"{path}{ext}")
374
+
375
+ candidate = path
376
+ n = 1
377
+ while candidate.exists():
378
+ candidate = path.with_name(f"{path.stem}_v{n:02d}{path.suffix}")
379
+ n += 1
380
+ return str(candidate)
381
+
382
+
383
+ def write_excel_report(
384
+ results: CheckeredHistory | Sequence[CheckeredHistory],
385
+ excel_file_name: str | pathlib.Path,
386
+ ) -> pathlib.Path:
387
+ """Write formatted check results to an Excel workbook with conditional highlighting.
388
+
389
+ Parameters
390
+ ----------
391
+ results
392
+ Check results produced by :func:`~dataspector.core.run_checks_on_location` or
393
+ :func:`~dataspector.core.load_and_run_checks`.
394
+ excel_file_name
395
+ Target ``.xlsx`` output path (auto-versioned if it already exists).
396
+
397
+ Returns
398
+ -------
399
+ pathlib.Path
400
+ Path to the written Excel report.
401
+ """
402
+ from dataspector.core import CheckeredHistory
403
+
404
+ result_list: list[CheckeredHistory] = (
405
+ [results] if isinstance(results, CheckeredHistory) else list(results)
406
+ )
407
+ out_path = pathlib.Path(version_adder(excel_file_name, extension=".xlsx"))
408
+
409
+ workbook = xlsxwriter.Workbook(str(out_path))
410
+ worksheet = workbook.add_worksheet("Results")
411
+ header_row = ["Pass", "Severity", "Location", "Check", "kwargs", "Notes"]
412
+ header = [{"header": col} for col in header_row]
413
+
414
+ format_fail = workbook.add_format({"bg_color": "#FF6B6B", "font_color": "#000000"})
415
+ format_warn = workbook.add_format({"bg_color": "#FADA5E", "font_color": "#000000"})
416
+ format_pass = workbook.add_format({"bg_color": "#ADD8E6", "font_color": "#000000"})
417
+
418
+ row_idx = 1
419
+ for res in result_list:
420
+ loc_display = getattr(res.location, "_location", res.location)
421
+ for idx, check in enumerate(res.checklist):
422
+ if isinstance(check, tuple):
423
+ checkname = check[0]
424
+ kwargs = check[1] if len(check) > 1 else {}
425
+ severity = check[2] if len(check) > 2 else ""
426
+ else:
427
+ checkname = check
428
+ kwargs = {}
429
+ severity = ""
430
+
431
+ notes_text = " ".join(
432
+ part for part in (res.notes.get(idx), res.exceptions.get(idx)) if part
433
+ )
434
+ worksheet.write_row(
435
+ row_idx,
436
+ 0,
437
+ [
438
+ str(res.results.get(idx)),
439
+ str(severity).upper(),
440
+ str(loc_display),
441
+ str(checkname),
442
+ str(kwargs),
443
+ notes_text,
444
+ ],
445
+ )
446
+ row_idx += 1
447
+
448
+ last_row = max(row_idx - 1, 1)
449
+ worksheet.add_table(
450
+ 0,
451
+ 0,
452
+ last_row,
453
+ len(header_row) - 1,
454
+ {"header_row": True, "first_column": True, "columns": header},
455
+ )
456
+ cell_range = f"A2:F{last_row + 1}"
457
+ worksheet.conditional_format(
458
+ cell_range,
459
+ {"type": "formula", "criteria": '=LEFT($A2, 4)="TRUE"', "format": format_pass},
460
+ )
461
+ worksheet.conditional_format(
462
+ cell_range,
463
+ {
464
+ "type": "formula",
465
+ "criteria": '=AND(LEFT($A2, 5)="FALSE", LEFT($B2, 7)="WARNING")',
466
+ "format": format_warn,
467
+ },
468
+ )
469
+ worksheet.conditional_format(
470
+ cell_range,
471
+ {
472
+ "type": "formula",
473
+ "criteria": '=AND(LEFT($A2, 5)="FALSE", NOT(LEFT($B2, 7)="WARNING"))',
474
+ "format": format_fail,
475
+ },
476
+ )
477
+ worksheet.set_column(0, 1, 12)
478
+ worksheet.set_column(2, 5, 40)
479
+ workbook.close()
480
+ return out_path
@@ -0,0 +1,85 @@
1
+ """Format-agnostic data-integrity checkers.
2
+
3
+ Public facade re-exporting:
4
+
5
+ * **Source-level checks** (``source_*``) from :mod:`dataspector.checkers.source`
6
+ * **Record-level checks** (``record_*``) from :mod:`dataspector.checkers.record`
7
+ * :class:`~dataspector.checkers.utils.CheckOutcome`,
8
+ :func:`~dataspector.checkers.utils.requires_all_data`, and
9
+ ``REQUIRES_ALL_DATA`` from :mod:`dataspector.checkers.utils`
10
+
11
+ Every check returns a :class:`~dataspector.checkers.utils.CheckOutcome` whose
12
+ ``passed`` attribute is:
13
+
14
+ * ``True`` — check passed;
15
+ * ``False`` — check failed;
16
+ * ``None`` — indeterminate so far; keep streaming subsequent chunks.
17
+ """
18
+
19
+ from __future__ import annotations
20
+
21
+ from dataspector.checkers.record import (
22
+ record_all_data_not_null,
23
+ record_contains,
24
+ record_contains_at_least,
25
+ record_count,
26
+ record_count_greater_than,
27
+ record_field_contains,
28
+ record_field_contains_at_least,
29
+ record_field_contains_only,
30
+ record_field_date_between,
31
+ record_field_is_numeric,
32
+ record_field_is_unique,
33
+ record_field_not_null,
34
+ record_field_numeric_value_between,
35
+ record_if_field1_startswith_field2_contains_only,
36
+ record_validate_schema,
37
+ )
38
+ from dataspector.checkers.source import (
39
+ source_exists,
40
+ source_fuzzy_find,
41
+ source_has_N_fields,
42
+ source_modified_on_date,
43
+ source_modified_recently,
44
+ source_source_identical,
45
+ source_source_roughly_same_number_of_records,
46
+ source_source_same_fields,
47
+ source_source_size_roughly_same,
48
+ source_storage_size_roughly,
49
+ )
50
+ from dataspector.checkers.utils import (
51
+ REQUIRES_ALL_DATA,
52
+ CheckOutcome,
53
+ requires_all_data,
54
+ )
55
+
56
+ __all__ = [
57
+ "REQUIRES_ALL_DATA",
58
+ "CheckOutcome",
59
+ "record_all_data_not_null",
60
+ "record_contains",
61
+ "record_contains_at_least",
62
+ "record_count",
63
+ "record_count_greater_than",
64
+ "record_field_contains",
65
+ "record_field_contains_at_least",
66
+ "record_field_contains_only",
67
+ "record_field_date_between",
68
+ "record_field_is_numeric",
69
+ "record_field_is_unique",
70
+ "record_field_not_null",
71
+ "record_field_numeric_value_between",
72
+ "record_if_field1_startswith_field2_contains_only",
73
+ "record_validate_schema",
74
+ "requires_all_data",
75
+ "source_exists",
76
+ "source_fuzzy_find",
77
+ "source_has_N_fields",
78
+ "source_modified_on_date",
79
+ "source_modified_recently",
80
+ "source_source_identical",
81
+ "source_source_roughly_same_number_of_records",
82
+ "source_source_same_fields",
83
+ "source_source_size_roughly_same",
84
+ "source_storage_size_roughly",
85
+ ]