dataassay 0.7.0__py3-none-any.whl

dataassay/__init__.py

@@ -0,0 +1,11 @@
+"""dataassay — audit a tabular dataset.
+
+An assay characterizes a sample before it makes any claim about it. This tool
+works in that order too: establish what kind of thing each column is, run only
+the checks that property makes valid, and report what could NOT be checked as
+plainly as what failed.
+"""
+
+__version__ = "0.7.0"
+
+__all__ = ["__version__"]

dataassay/__main__.py

@@ -0,0 +1,4 @@
+from dataassay.cli import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())

dataassay/audit.py

@@ -0,0 +1,166 @@
+"""Orchestration: profile, infer structure, gate the checks, rank what survives.
+
+Coverage is a first-class output, not a footnote. A reader cannot interpret an
+empty findings list without knowing how many checks ran, which were withheld,
+and which are waiting on an answer -- zero findings at 30% coverage and zero at
+95% are entirely different objects, and one number cannot tell them apart.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+
+import duckdb
+
+from dataassay import manifest as manifest_mod
+from dataassay import structure as structure_mod
+from dataassay.checks.base import Confidence, Finding
+from dataassay.checks.registry import CATALOG, CATALOG_VERSION
+from dataassay.profile import Profile, build
+from dataassay.provenance import reader_for, source_expr
+from dataassay.structure import Structure
+
+
+@dataclass
+class Coverage:
+    ran: list[str] = field(default_factory=list)
+    withheld: list[tuple[str, str]] = field(default_factory=list)
+    blocked: list[tuple[str, str]] = field(default_factory=list)
+
+    @property
+    def total(self) -> int:
+        return len(self.ran) + len(self.withheld) + len(self.blocked)
+
+    def to_dict(self) -> dict:
+        return {
+            "checks_total": self.total,
+            "ran": self.ran,
+            "withheld": [{"check": c, "reason": r} for c, r in self.withheld],
+            "blocked": [{"check": c, "question": qn} for c, qn in self.blocked],
+        }
+
+
+@dataclass
+class Audit:
+    profile: Profile
+    structure: Structure
+    findings: list[Finding]
+    coverage: Coverage
+    catalog_version: str = CATALOG_VERSION
+    manifest_path: str | None = None
+    # Question codes a person has seen and deliberately declined to answer.
+    # An unanswered question and a declined one are different states and the
+    # report must not show them as the same thing: the first is work nobody has
+    # done yet, the second is a decision, and presenting a decision as an
+    # outstanding task is how a report trains people to ignore it.
+    skipped_questions: list[str] = field(default_factory=list)
+
+    @property
+    def open_questions(self) -> list:
+        return [q for q in self.profile.questions
+                if q.code not in set(self.skipped_questions)]
+
+    @property
+    def declined_questions(self) -> list:
+        return [q for q in self.profile.questions
+                if q.code in set(self.skipped_questions)]
+
+    def to_dict(self) -> dict:
+        d = self.profile.to_dict()
+        d |= {
+            "catalog_version": self.catalog_version,
+            "structure": self.structure.to_dict(),
+            "coverage": self.coverage.to_dict(),
+            "manifest": self.manifest_path,
+            "skipped_questions": self.skipped_questions,
+            "findings": [f.to_dict() for f in self.findings],
+        }
+        return d
+
+
+def _corroborate(findings: list[Finding]) -> list[Finding]:
+    """Promote findings that two independent checks agree on.
+
+    Agreement is the strongest confidence signal available, and the only one
+    that does not depend on trusting a single detector's threshold. A column
+    flagged by both a saturation check and a level-shift check is real in a way
+    that either alone is not.
+    """
+    by_column: dict[str, set[str]] = {}
+    for f in findings:
+        if f.column:
+            by_column.setdefault(f.column, set()).add(f.check_id)
+
+    out = []
+    for f in findings:
+        others = by_column.get(f.column or "", set()) - {f.check_id}
+        if others and f.confidence.level != "high":
+            f = Finding(
+                check_id=f.check_id,
+                column=f.column,
+                disposition=f.disposition,
+                summary=f.summary,
+                evidence=f.evidence,
+                predicate=f.predicate,
+                confidence=f.confidence.corroborated_by(", ".join(sorted(others))),
+                raw_values=f.raw_values,
+            )
+        out.append(f)
+    return out
+
+
+def run(
+    path: Path,
+    byte_cap: int | None = None,
+    manifest_path: Path | None = None,
+    use_manifest: bool = True,
+) -> Audit:
+    from dataassay.checks.base import CheckContext
+    from dataassay.rawscan import BYTE_CAP
+
+    reader = reader_for(path)
+    con = duckdb.connect(":memory:")
+    try:
+        profile = build(path, byte_cap=byte_cap or BYTE_CAP, con=con)
+        source = source_expr(reader, profile.provenance.read_mode)
+        params = [str(path)]
+        manifest = (
+            manifest_mod.discover(path, manifest_path) if use_manifest else None
+        )
+        struct = structure_mod.infer(
+            profile.columns, con, source, params, profile.provenance.row_count,
+            manifest=manifest,
+        )
+        ctx = CheckContext(
+            profile=profile, structure=struct, con=con, source=source,
+            params=params, manifest=manifest,
+        )
+
+        coverage = Coverage()
+        findings: list[Finding] = []
+        for check in CATALOG:
+            verdict = check.applies(ctx)
+            if not verdict.applicable:
+                if verdict.blocked:
+                    coverage.blocked.append((check.spec.id, verdict.reason))
+                else:
+                    coverage.withheld.append((check.spec.id, verdict.reason))
+                continue
+            coverage.ran.append(check.spec.id)
+            findings.extend(check.run(ctx))
+    finally:
+        con.close()
+
+    findings = _corroborate(findings)
+    findings.sort(key=lambda f: f.sort_key)
+    return Audit(
+        profile=profile, structure=struct, findings=findings, coverage=coverage,
+        manifest_path=(
+            str(manifest.source_path) if manifest and manifest.source_path else None
+        ),
+        skipped_questions=list(manifest.skipped) if manifest else [],
+    )
+
+
+__all__ = ["Audit", "Coverage", "Confidence", "run"]

dataassay/checks/base.py

@@ -0,0 +1,217 @@
+"""What a check is, and what it is allowed to claim.
+
+Three ideas carry the design:
+
+  A check is GATED. It declares the properties it needs, and the engine refuses
+  to run it when the profile has not established them. A sigma rule on a
+  tail-inflated column is not a weak check, it is an invalid one, and the way to
+  stop it producing confident nonsense is to never let it run.
+
+  A check that could not run is REPORTED. Withheld and blocked checks travel
+  beside the findings, because "seasonality undetermined -- 1.2 cycles of
+  history" is something the reader needs in order to know what the silence
+  means.
+
+  A finding carries its own PREDICATE. Whatever produced it must be re-runnable
+  by the person reading it, without our code. An audit nobody can reproduce is
+  an opinion.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Protocol
+
+if TYPE_CHECKING:
+    import duckdb
+
+    from dataassay.columns import ColumnProfile
+    from dataassay.manifest import Manifest
+    from dataassay.profile import Profile
+    from dataassay.structure import Structure
+
+# -- disposition ---------------------------------------------------------------
+# Most anomalies in real data are the source's own bookkeeping. A tool that
+# cannot say so gets ignored within a week, so the disposition is part of the
+# finding rather than left to the reader.
+DEFECT = "defect"            # very likely wrong
+SUSPECT = "suspect"          # wrong-looking, but a benign explanation exists
+BOOKKEEPING = "bookkeeping"  # the source doing something legitimate
+
+SEVERITY_ORDER = {DEFECT: 0, SUSPECT: 1, BOOKKEEPING: 2}
+
+# -- confidence ----------------------------------------------------------------
+HIGH = "high"
+MEDIUM = "medium"
+LOW = "low"
+
+_CONFIDENCE_ORDER = {HIGH: 0, MEDIUM: 1, LOW: 2}
+
+
+@dataclass(frozen=True)
+class Confidence:
+    """Never an opaque score. Each input is named and shown.
+
+    Agreement between independent checks is the strongest signal available and
+    is applied by the engine after all checks have run -- a point flagged by two
+    unrelated detectors is real in a way that one 3-sigma hit never is.
+    """
+
+    level: str
+    inputs: list[str] = field(default_factory=list)
+
+    def to_dict(self) -> dict:
+        return {"level": self.level, "inputs": list(self.inputs)}
+
+    def corroborated_by(self, other_check: str) -> Confidence:
+        promoted = HIGH if self.level == MEDIUM else self.level
+        return Confidence(
+            level=promoted,
+            inputs=[*self.inputs, f"independently flagged by {other_check}"],
+        )
+
+
+@dataclass(frozen=True)
+class Finding:
+    check_id: str
+    column: str | None
+    disposition: str
+    summary: str
+    evidence: dict
+    predicate: str
+    confidence: Confidence
+    # Set when `evidence` carries actual cell values. Acting on a finding
+    # usually requires seeing the offending key, so the report shows them -- but
+    # anything crossing a network boundary later has to know they are in there.
+    raw_values: bool = False
+
+    @property
+    def sort_key(self) -> tuple[int, int]:
+        return (
+            SEVERITY_ORDER.get(self.disposition, 9),
+            _CONFIDENCE_ORDER.get(self.confidence.level, 9),
+        )
+
+    def to_dict(self) -> dict:
+        return {
+            "check_id": self.check_id,
+            "column": self.column,
+            "disposition": self.disposition,
+            "summary": self.summary,
+            "evidence": self.evidence,
+            "predicate": self.predicate,
+            "confidence": self.confidence.to_dict(),
+            "raw_values": self.raw_values,
+        }
+
+
+@dataclass(frozen=True)
+class Applicability:
+    """Why a check will or will not run here.
+
+    `blocked` distinguishes the two kinds of no. A withheld check is impossible
+    on this data and nothing can change that; a blocked one is waiting on an
+    answer, and so belongs in the interview queue rather than the limitations
+    list.
+    """
+
+    applicable: bool
+    reason: str = ""
+    blocked: bool = False
+
+    @staticmethod
+    def yes() -> Applicability:
+        return Applicability(True)
+
+    @staticmethod
+    def no(reason: str) -> Applicability:
+        return Applicability(False, reason)
+
+    @staticmethod
+    def needs_answer(question: str) -> Applicability:
+        return Applicability(False, question, blocked=True)
+
+
+@dataclass(frozen=True)
+class CheckSpec:
+    """The catalog entry. This is the part that is worth more than the code.
+
+    `not_the_obvious` exists because the most expensive lesson in this corpus is
+    that the obvious detector is often the wrong one: counting states finds
+    nothing in a panel where absence is encoded as zero, and watching zeros move
+    finds nothing in a ragged panel. Where a check has a near neighbour that
+    fails, the catalog says so.
+    """
+
+    id: str
+    name: str
+    detects: str
+    gate: str
+    default_disposition: str
+    not_the_obvious: str = ""
+    traces_to: str = ""  # the real finding that earned this check a place
+
+    def to_dict(self) -> dict:
+        return {
+            "id": self.id,
+            "name": self.name,
+            "detects": self.detects,
+            "gate": self.gate,
+            "default_disposition": self.default_disposition,
+            "not_the_obvious": self.not_the_obvious,
+            "traces_to": self.traces_to,
+        }
+
+
+@dataclass
+class CheckContext:
+    """Everything a check is allowed to look at.
+
+    The connection is shared: checks run against one open handle over the same
+    source, so a file larger than memory is not re-opened per check.
+    """
+
+    profile: Profile
+    structure: Structure
+    con: duckdb.DuckDBPyConnection
+    source: str
+    params: list[str]
+    manifest: Manifest | None = None
+
+    def columns(self, kind: str | None = None) -> list[ColumnProfile]:
+        cols = self.profile.columns
+        return [c for c in cols if kind is None or c.kind == kind]
+
+    def column(self, name: str) -> ColumnProfile | None:
+        return next((c for c in self.profile.columns if c.name == name), None)
+
+    def holds(self, column: str, prop: str) -> bool:
+        """Has the profile established `prop` for this column?"""
+        col = self.column(column)
+        if col is None:
+            return False
+        return any(
+            p["property"] == prop and p["holds"] for p in col.observed_properties()
+        )
+
+    def fetch(self, sql: str) -> list[tuple]:
+        """Run SQL against the source, binding the path once per reference.
+
+        A check that names the source twice needs the parameter twice, and
+        forgetting that surfaces as an opaque binder error rather than a wrong
+        answer -- so the repetition is handled here instead of in every check.
+        """
+        placeholders = sql.count("?")
+        params = self.params * max(1, placeholders // max(1, len(self.params)))
+        return self.con.execute(sql, params).fetchall()
+
+
+class Check(Protocol):
+    spec: CheckSpec
+
+    def applies(self, ctx: CheckContext) -> Applicability:
+        """Decide from the profile and structure alone, before touching data."""
+        ...
+
+    def run(self, ctx: CheckContext) -> list[Finding]:
+        ...