sheets-data-validation-kit 0.1.0__py3-none-any.whl

sheets_data_validation_kit-0.1.0.dist-info/METADATA

@@ -0,0 +1,133 @@
+Metadata-Version: 2.4
+Name: sheets-data-validation-kit
+Version: 0.1.0
+Summary: Validation, reconciliation and entity-resolution primitives for spreadsheet and CSV pipelines
+License: MIT
+Project-URL: Homepage, https://github.com/devops-gm88/sheets-data-validation-kit
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Software Development :: Quality Assurance
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == "dev"
+Requires-Dist: ruff>=0.6; extra == "dev"
+Dynamic: license-file
+
+# sheets-data-validation-kit
+
+[![CI](https://github.com/devops-gm88/sheets-data-validation-kit/actions/workflows/ci.yml/badge.svg)](https://github.com/devops-gm88/sheets-data-validation-kit/actions/workflows/ci.yml)
+[![Python versions](https://img.shields.io/badge/python-3.9%20%7C%203.10%20%7C%203.11%20%7C%203.12%20%7C%203.13%20%7C%203.14-blue)](pyproject.toml)
+[![License: MIT](https://img.shields.io/badge/license-MIT-black.svg)](LICENSE)
+
+Validation, reconciliation and entity-resolution primitives for spreadsheet and CSV pipelines.
+
+Most data pipelines fail quietly. A row lands, a number looks plausible, and nobody notices it is wrong
+until it has been reported to someone. This kit exists to make that failure mode loud.
+
+It is deliberately **standard-library only** — no dependencies, nothing to install, nothing to upgrade.
+Copy the `src/validation_kit` folder into your project, or install the package in one line:
+
+```bash
+pip install git+https://github.com/devops-gm88/sheets-data-validation-kit.git
+```
+
+Not on PyPI yet - the GitHub URL above is the install. From a clone:
+`pip install -e .`.
+
+## The three ideas
+
+**1. Rows pass or they are quarantined. They are never silently dropped.**
+A row that fails a contract goes to quarantine with the reasons attached. You can always answer
+"where did this record go?" — which is the question that kills trust in a pipeline.
+
+**2. Arithmetic is a second, independent check.**
+Field-level validation confirms each value looks right. Reconciliation confirms the values are
+*consistent with each other*. An extracted balance sheet must balance; a cash flow statement must tie to
+the change in cash. If the arithmetic does not close, the extraction failed regardless of how confident
+each individual field was. This catches the failure nobody sees: clean extraction, quietly wrong numbers.
+
+**3. Exact match or human review. Never a fuzzy merge.**
+Duplicates are detected on a canonical key. Suspected duplicates are queued for a person, because a
+silent merge corrupts counts invisibly and is very hard to unpick later.
+
+## Quick start
+
+```python
+from validation_kit import RowContract, Field, Min, Range, Pattern, Reconcile, Assertion
+from validation_kit.entities import resolve
+
+contract = RowContract([
+    Field("account_id", required=True),
+    Field("amount", rules=[Min(0)], coerce=float,
+          on_fail="quarantine"),
+    Field("period", rules=[Pattern(r"^\d{4}-Q[1-4]$")]),
+])
+
+result = contract.validate_rows(rows)
+print(result.summary())
+for row, reasons in result.quarantined:
+    print(row.get("account_id"), reasons)
+```
+
+```python
+# Arithmetic self-consistency: the check that does not depend on the model
+suite = Reconcile([
+    Assertion("balance_sheet_balances",
+              lambda r: r["total_assets"], lambda r: r["total_liabilities"] + r["total_equity"],
+              tolerance=0.01),
+])
+
+for outcome in suite.check(parsed_statement):
+    print(outcome)   # PASS / FAIL with both sides of the equation printed
+```
+
+```python
+# Entity resolution: exact key, review queue, no auto-merge
+outcome = resolve(records, key="child_id")
+print(outcome.unique_count, len(outcome.duplicate_candidates))
+```
+
+## What this is not
+
+- It is not a data-quality *scoring* tool. There is no 0-100 number and no dashboard.
+- It does not fix your source data. It tells you which records are untrustworthy and why.
+- It will not merge duplicates for you. That is the point.
+
+## Why it exists
+
+Built from repeatedly solving the same problem on real reporting systems: a monthly pack that was
+*almost* right, and no way to prove which numbers were which. These are the primitives that were
+written more than once, extracted so they are not written again.
+
+## Tests
+
+Install first, then run:
+
+```bash
+pip install -e .
+python3 -m unittest discover -s tests -v
+```
+
+Or without installing anything:
+
+```bash
+PYTHONPATH=src python3 -m unittest discover -s tests -v
+```
+
+Both give **24 tests, OK**. Running `unittest` from a fresh clone without one of
+those two steps fails to import the package — that is a packaging path, not a
+test failure.
+
+## License
+
+MIT

sheets_data_validation_kit-0.1.0.dist-info/RECORD

@@ -0,0 +1,9 @@
+sheets_data_validation_kit-0.1.0.dist-info/licenses/LICENSE,sha256=vIy64iu9eupgwG0r71cAIixKKKiYfb6oIMT6a9X7X-4,1066
+validation_kit/__init__.py,sha256=_evSY2lmY_Fq7p7AoMY9yJ6xkDH7qa5L_tHv6vdSm9g,532
+validation_kit/contract.py,sha256=K7RZ0Owkz2BkFrtJ2_mUESxKTwPTKVDQYDuyYJybuqQ,6393
+validation_kit/entities.py,sha256=PVLP6JTayoWGPZMTtyvcxrWZtydm5NQzTarvEdy4SOA,4402
+validation_kit/reconcile.py,sha256=XIx9h34PY0vM2AfK_lJ8t9CEvnBVpItvqdySp40R7Dg,3526
+sheets_data_validation_kit-0.1.0.dist-info/METADATA,sha256=9PBvUlhnQ0E0Nd8COHPRhoZyRq7jmci4F-hPuw7IMCQ,5154
+sheets_data_validation_kit-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+sheets_data_validation_kit-0.1.0.dist-info/top_level.txt,sha256=mTLk3dltJ_9j3CTZaHM9VCt91W8N4gPo9TeTYqqtSck,15
+sheets_data_validation_kit-0.1.0.dist-info/RECORD,,

sheets_data_validation_kit-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

sheets_data_validation_kit-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 SK Saimon
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

sheets_data_validation_kit-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+validation_kit

validation_kit/__init__.py

@@ -0,0 +1,16 @@
+"""Validation, reconciliation and entity-resolution primitives.
+
+Standard library only, deliberately.
+"""
+from .contract import Field, Max, Min, NotBlank, OneOf, Pattern, Range, RowContract, ValidationResult
+from .entities import ResolutionOutcome, resolve
+from .reconcile import Assertion, Outcome, Reconcile
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "Field", "RowContract", "ValidationResult",
+    "Min", "Max", "Range", "Pattern", "OneOf", "NotBlank",
+    "Assertion", "Reconcile", "Outcome",
+    "resolve", "ResolutionOutcome",
+]

validation_kit/contract.py

@@ -0,0 +1,201 @@
+"""Field-level contracts.
+
+A row either satisfies its contract or it is quarantined with the reasons attached.
+Quarantine is the default failure mode: nothing is ever silently dropped.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple
+
+
+class Rule:
+    """Base class for a single field rule."""
+
+    name = "rule"
+
+    def check(self, value: Any) -> Optional[str]:
+        """Return None if the value passes, else a human-readable reason."""
+        raise NotImplementedError
+
+
+@dataclass
+class Min(Rule):
+    bound: float
+    name = "min"
+
+    def check(self, value):
+        try:
+            return None if float(value) >= self.bound else f"below minimum {self.bound}"
+        except (TypeError, ValueError):
+            return f"not numeric, cannot compare to minimum {self.bound}"
+
+
+@dataclass
+class Max(Rule):
+    bound: float
+    name = "max"
+
+    def check(self, value):
+        try:
+            return None if float(value) <= self.bound else f"above maximum {self.bound}"
+        except (TypeError, ValueError):
+            return f"not numeric, cannot compare to maximum {self.bound}"
+
+
+@dataclass
+class Range(Rule):
+    low: float
+    high: float
+    name = "range"
+
+    def check(self, value):
+        try:
+            number = float(value)
+        except (TypeError, ValueError):
+            return f"not numeric, cannot range-check {self.low}..{self.high}"
+        if number < self.low or number > self.high:
+            return f"{number} outside plausible range {self.low}..{self.high}"
+        return None
+
+
+@dataclass
+class Pattern(Rule):
+    regex: str
+    name = "pattern"
+
+    def check(self, value):
+        import re
+
+        return None if re.match(self.regex, str(value)) else f"does not match {self.regex}"
+
+
+@dataclass
+class OneOf(Rule):
+    allowed: Sequence[Any]
+    name = "one_of"
+
+    def check(self, value):
+        return None if value in self.allowed else f"{value!r} not one of {list(self.allowed)}"
+
+
+class NotBlank(Rule):
+    name = "not_blank"
+
+    def check(self, value):
+        if value is None or (isinstance(value, str) and not value.strip()):
+            return "blank"
+        return None
+
+
+@dataclass
+class Field:
+    """One column in a row contract.
+
+    on_fail:
+        "quarantine" (default) - hold the whole row back for review.
+        "flag"                 - record the problem but keep the row.
+    """
+
+    name: str
+    required: bool = False
+    rules: List[Rule] = field(default_factory=list)
+    coerce: Optional[Callable[[Any], Any]] = None
+    on_fail: str = "quarantine"
+
+
+@dataclass
+class ValidationResult:
+    accepted: List[Dict[str, Any]] = field(default_factory=list)
+    quarantined: List[Tuple[Dict[str, Any], List[str]]] = field(default_factory=list)
+    flagged: List[Tuple[Dict[str, Any], List[str]]] = field(default_factory=list)
+    field_error_counts: Dict[str, int] = field(default_factory=dict)
+
+    @property
+    def total(self) -> int:
+        return len(self.accepted) + len(self.quarantined)
+
+    @property
+    def pass_rate(self) -> float:
+        return 0.0 if self.total == 0 else len(self.accepted) / self.total
+
+    def summary(self) -> str:
+        lines = [
+            f"rows in        : {self.total}",
+            f"accepted       : {len(self.accepted)}  ({self.pass_rate:.1%})",
+            f"quarantined    : {len(self.quarantined)}",
+            f"flagged        : {len(self.flagged)}",
+        ]
+        if self.field_error_counts:
+            lines.append("errors by field:")
+            for name, count in sorted(self.field_error_counts.items(), key=lambda kv: -kv[1]):
+                lines.append(f"  {name:<24} {count}")
+        return "\n".join(lines)
+
+
+class RowContract:
+    """A set of field rules applied to every row.
+
+    Rows that fail a quarantine field are held back. Rows that fail a flag field are
+    kept, with the problem recorded, so a pipeline can decide for itself how strict to be
+    on fields it does not depend on.
+    """
+
+    def __init__(self, fields: Sequence[Field]):
+        self.fields = list(fields)
+
+    def validate_row(self, row: Dict[str, Any]):
+        """Returns (cleaned_row, reasons, verdict) where verdict is pass|quarantine|flag."""
+        cleaned: Dict[str, Any] = dict(row)
+        reasons: List[str] = []
+        severity = "pass"
+
+        for spec in self.fields:
+            value = row.get(spec.name)
+
+            if spec.required and (value is None or (isinstance(value, str) and not value.strip())):
+                reasons.append(f"{spec.name}: required but blank")
+                severity = "quarantine"
+                continue
+
+            if value is None or (isinstance(value, str) and not value.strip()):
+                continue
+
+            if spec.coerce is not None:
+                try:
+                    value = spec.coerce(value)
+                    cleaned[spec.name] = value
+                except (TypeError, ValueError) as exc:
+                    reasons.append(f"{spec.name}: cannot coerce ({exc})")
+                    severity = "quarantine" if spec.on_fail == "quarantine" else "flag"
+                    continue
+
+            for rule in spec.rules:
+                problem = rule.check(value)
+                if problem:
+                    reasons.append(f"{spec.name}: {problem}")
+                    if spec.on_fail == "quarantine":
+                        severity = "quarantine"
+                    elif severity == "pass":
+                        severity = "flag"
+
+        return cleaned, reasons, severity
+
+    def validate_rows(self, rows: Iterable[Dict[str, Any]]) -> ValidationResult:
+        result = ValidationResult()
+        for row in rows:
+            cleaned, reasons, verdict = self.validate_row(row)
+            if verdict == "pass":
+                result.accepted.append(cleaned)
+            elif verdict == "quarantine":
+                result.quarantined.append((cleaned, reasons))
+                _bump(result.field_error_counts, reasons)
+            else:
+                result.flagged.append((cleaned, reasons))
+                _bump(result.field_error_counts, reasons)
+        return result
+
+
+def _bump(counter: Dict[str, int], reasons: Sequence[str]) -> None:
+    for reason in reasons:
+        counter[reason.split(":", 1)[0]] = counter.get(reason.split(":", 1)[0], 0) + 1

validation_kit/entities.py

@@ -0,0 +1,113 @@
+"""Exact-match entity resolution with a human review queue.
+
+The rule this module enforces: **never merge automatically.**
+
+Name-based and fuzzy matching is the usual cause of inflated counts in reporting
+pipelines, because common names collide and re-registrations create a second record.
+Suspects are surfaced for a person to decide, not resolved by a heuristic.
+"""
+from __future__ import annotations
+
+from collections import defaultdict
+from dataclasses import dataclass, field
+from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple
+
+
+@dataclass
+class DuplicateCandidate:
+    """Two or more records that look like the same real entity but carry different keys."""
+
+    reason: str
+    records: List[Dict[str, Any]]
+
+    def keys(self) -> List[Any]:
+        return [r.get("_key") for r in self.records]
+
+
+@dataclass
+class ResolutionOutcome:
+    unique: List[Dict[str, Any]] = field(default_factory=list)
+    duplicate_keys: List[Tuple[Any, int]] = field(default_factory=list)
+    duplicate_candidates: List[DuplicateCandidate] = field(default_factory=list)
+    blank_keys: List[Dict[str, Any]] = field(default_factory=list)
+
+    @property
+    def unique_count(self) -> int:
+        return len(self.unique)
+
+    @property
+    def input_count(self) -> int:
+        return self.unique_count + sum(count - 1 for _, count in self.duplicate_keys)
+
+    @property
+    def inflation(self) -> int:
+        """How many phantom entities a naive row count would have reported."""
+        return sum(count - 1 for _, count in self.duplicate_keys)
+
+    def summary(self) -> str:
+        return (
+            f"records in            : {self.input_count}\n"
+            f"distinct entities     : {self.unique_count}\n"
+            f"phantom entities      : {self.inflation}\n"
+            f"blank keys held back  : {len(self.blank_keys)}\n"
+            f"suspects for review   : {len(self.duplicate_candidates)}"
+        )
+
+
+def resolve(records: Iterable[Dict[str, Any]],
+            key: str,
+            signature: Optional[Sequence[str]] = None,
+            normalise: Optional[Callable[[Any], Any]] = None) -> ResolutionOutcome:
+    """Resolve records to distinct entities on an exact key.
+
+    key        the canonical identifier. Rows without one are held back, never counted.
+    signature  optional tuple of fields used only to *suggest* duplicates. Suggestions
+               are queued for review; they are never merged.
+    normalise  optional callable applied to the key (strip/casefold and similar).
+    """
+    outcome = ResolutionOutcome()
+    norm = normalise or (lambda v: str(v).strip().casefold() if v is not None else None)
+
+    buckets: Dict[Any, List[Dict[str, Any]]] = defaultdict(list)
+    for record in records:
+        raw = record.get(key)
+        if raw is None or (isinstance(raw, str) and not raw.strip()):
+            outcome.blank_keys.append(record)
+            continue
+        record = dict(record)
+        record["_key"] = norm(raw)
+        buckets[record["_key"]].append(record)
+
+    for entity_key, group in buckets.items():
+        group_sorted = sorted(group, key=lambda r: str(r.get("_source", "")))
+        outcome.unique.append(group_sorted[0])
+        if len(group) > 1:
+            outcome.duplicate_keys.append((entity_key, len(group)))
+
+    outcome.duplicate_keys.sort(key=lambda kv: -kv[1])
+
+    if signature:
+        seen: Dict[Tuple[Any, ...], List[Dict[str, Any]]] = defaultdict(list)
+        for record in outcome.unique:
+            token = tuple(norm(record.get(f)) for f in signature)
+            if any(part is None for part in token):
+                continue
+            seen[token].append(record)
+        for token, group in seen.items():
+            if len(group) > 1:
+                outcome.duplicate_candidates.append(
+                    DuplicateCandidate(
+                        reason=f"same {'/'.join(signature)}: {token}",
+                        records=group,
+                    )
+                )
+        outcome.duplicate_candidates.sort(key=lambda c: -len(c.records))
+
+    return outcome
+
+
+def count_distinct(records: Iterable[Dict[str, Any]], key: str) -> int:
+    """Distinct entity count. Blank keys are excluded rather than counted as one entity."""
+    values = {str(r.get(key)).strip().casefold() for r in records
+              if r.get(key) is not None and str(r.get(key)).strip()}
+    return len(values)

validation_kit/reconcile.py

@@ -0,0 +1,97 @@
+"""Arithmetic reconciliation.
+
+Field-level validation asks "does this value look right?".
+Reconciliation asks "are these values consistent with each other?".
+
+It is a genuinely independent check: it needs no model, no confidence score and no
+reference data, only arithmetic that must hold. If a statement extracts cleanly and the
+arithmetic does not close, the extraction failed.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Callable, Dict, List, Sequence
+
+
+@dataclass
+class Outcome:
+    name: str
+    passed: bool
+    left: float
+    right: float
+    difference: float
+    tolerance: float
+    note: str = ""
+
+    def __str__(self) -> str:
+        mark = "PASS" if self.passed else "FAIL"
+        detail = f"{self.left:,.2f} vs {self.right:,.2f} (diff {self.difference:,.4f}, tol {self.tolerance})"
+        return f"[{mark}] {self.name}: {detail}" + (f"  -- {self.note}" if self.note else "")
+
+
+@dataclass
+class Assertion:
+    """One equation that must hold for a record.
+
+    left and right are callables taking the record and returning a number, so the
+    assertion reads like the equation it tests:
+
+        Assertion("assets = liabilities + equity",
+                  lambda r: r["total_assets"],
+                  lambda r: r["total_liabilities"] + r["total_equity"])
+    """
+
+    name: str
+    left: Callable[[Dict[str, Any]], float]
+    right: Callable[[Dict[str, Any]], float]
+    tolerance: float = 0.01
+    note: str = ""
+
+
+class Reconcile:
+    def __init__(self, assertions: Sequence[Assertion]):
+        self.assertions = list(assertions)
+
+    def check(self, record: Dict[str, Any]) -> List[Outcome]:
+        outcomes: List[Outcome] = []
+        for assertion in self.assertions:
+            try:
+                left = float(assertion.left(record))
+                right = float(assertion.right(record))
+            except (KeyError, TypeError, ValueError) as exc:
+                outcomes.append(Outcome(assertion.name, False, float("nan"), float("nan"),
+                                        float("nan"), assertion.tolerance,
+                                        note=f"cannot evaluate: {exc}"))
+                continue
+            difference = abs(left - right)
+            outcomes.append(Outcome(assertion.name, difference <= assertion.tolerance,
+                                    left, right, difference, assertion.tolerance, assertion.note))
+        return outcomes
+
+    def all_pass(self, record: Dict[str, Any]) -> bool:
+        return all(o.passed for o in self.check(record))
+
+    def failures(self, record: Dict[str, Any]) -> List[Outcome]:
+        return [o for o in self.check(record) if not o.passed]
+
+
+def closing_balance_check(opening: str, inflow: str, outflow: str, closing: str,
+                          tolerance: float = 0.01) -> Assertion:
+    """closing == opening + inflow - outflow."""
+    return Assertion(
+        "closing = opening + inflow - outflow",
+        lambda r: r[closing],
+        lambda r: r[opening] + r[inflow] - r[outflow],
+        tolerance=tolerance,
+    )
+
+
+def rollforward_check(opening: str, additions: str, reductions: str, closing: str,
+                      tolerance: float = 0.01) -> Assertion:
+    """closing == opening + additions - reductions. Used for inventory, PP&E, capital assets."""
+    return Assertion(
+        "closing = opening + additions - reductions",
+        lambda r: r[closing],
+        lambda r: r[opening] + r[additions] - r[reductions],
+        tolerance=tolerance,
+    )