datadoom 0.1.0.dev0__py3-none-any.whl

datadoom/engine/advice.py

@@ -0,0 +1,289 @@
+"""ML-handling advice for the issues a column carries (exploratory guidance).
+
+DataDoom knows *exactly* what it did to every column — which failure mode hit it,
+the realized magnitude, and (for derived columns) how it was generated. This
+module turns that ground truth into **actionable guidance for the engineer or
+student who will model the data**: for each issue, a plain-language explanation,
+the single best handling approach, and a short menu of concrete techniques.
+
+This is static knowledge keyed on ``(mechanism, column type)`` plus the realized
+magnitude — no randomness, no model fitting. It exists so the Results screen can
+answer "what should I focus on, and how do I deal with it?" without the user
+having to recognise an MNAR pattern from a histogram themselves.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+# Severity ranks the urgency of an issue for someone building a model on the data.
+# "critical" = will silently break the model (leakage); "high" = caps achievable
+# performance or biases estimates if mishandled; "medium"/"low" = handle with care.
+Severity = str  # "critical" | "high" | "medium" | "low"
+_SEVERITY_RANK = {"critical": 3, "high": 2, "medium": 1, "low": 0}
+
+
+@dataclass
+class Issue:
+    """One thing about a column an ML engineer should know and handle."""
+
+    mode: str  # the failure mechanism ("mnar", "leakage", …) or "class_imbalance"
+    title: str  # short headline ("Missing not at random")
+    severity: Severity
+    magnitude: str  # human-readable realized effect ("12.0% of values missing")
+    explanation: str  # what the issue is, in plain language
+    recommendation: str  # the single best way to handle it
+    techniques: list[str] = field(default_factory=list)  # concrete options
+    detail: dict[str, Any] = field(default_factory=dict)  # raw numbers for the UI
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "mode": self.mode,
+            "title": self.title,
+            "severity": self.severity,
+            "magnitude": self.magnitude,
+            "explanation": self.explanation,
+            "recommendation": self.recommendation,
+            "techniques": self.techniques,
+            "detail": self.detail,
+        }
+
+
+# --- the knowledge base --------------------------------------------------------------
+# Each entry is the fixed guidance for a mechanism. ``base_severity`` is escalated
+# by realized magnitude in the builders below where it makes sense.
+
+_GUIDE: dict[str, dict[str, Any]] = {
+    "mcar": {
+        "title": "Missing completely at random",
+        "base_severity": "low",
+        "explanation": (
+            "Values are missing independently of every column, including their own "
+            "value. This is the most benign kind of missingness: the observed rows "
+            "stay a fair, unbiased sample of the data."
+        ),
+        "recommendation": (
+            "Simple imputation is unbiased here. Use the median (numeric) or mode "
+            "(categorical), or drop rows if the rate is small (<5%)."
+        ),
+        "techniques": [
+            "Median/mean imputation for numeric columns",
+            "Mode (most-frequent) imputation for categoricals",
+            "Listwise row deletion — unbiased under MCAR when the rate is low",
+        ],
+    },
+    "mar": {
+        "title": "Missing at random",
+        "base_severity": "medium",
+        "explanation": (
+            "Whether a value is missing depends on *other observed* columns, not on "
+            "the missing value itself. Naive mean/median imputation biases the column "
+            "because it ignores those drivers; conditioning on them fixes it."
+        ),
+        "recommendation": (
+            "Use conditional, model-based imputation that learns from the observed "
+            "drivers (e.g. IterativeImputer / MICE or KNN), not a global mean."
+        ),
+        "techniques": [
+            "IterativeImputer (MICE) — regresses the column on the others",
+            "KNN imputation on the correlated features",
+            "Add a binary missingness-indicator feature alongside the imputed value",
+            "Avoid plain mean/median imputation — it biases under MAR",
+        ],
+    },
+    "mnar": {
+        "title": "Missing not at random",
+        "base_severity": "high",
+        "explanation": (
+            "Whether a value is missing depends on the *unobserved value itself* "
+            "(e.g. high earners hide income). No imputation is unbiased without "
+            "modelling the missingness mechanism — the missing rows are a skewed "
+            "sample, so filling them in from the observed rows distorts the column."
+        ),
+        "recommendation": (
+            "Treat missingness as informative: add an explicit missing-indicator "
+            "feature (it is often predictive on its own) and avoid pretending the "
+            "data is MCAR/MAR. Consider selection / pattern-mixture models."
+        ),
+        "techniques": [
+            "Add a binary 'is-missing' indicator — frequently predictive itself",
+            "Pattern-mixture or selection models for the missingness mechanism",
+            "Domain-informed bounds instead of point imputation",
+            "Run a sensitivity analysis over plausible imputed values",
+        ],
+    },
+    "label_noise": {
+        "title": "Label noise",
+        "base_severity": "high",
+        "explanation": (
+            "A fraction of the target labels are wrong. This caps the accuracy any "
+            "model can honestly reach and, with high-capacity models, gets memorised "
+            "as if it were signal — hurting generalisation."
+        ),
+        "recommendation": (
+            "Train with noise-robust losses and strong regularisation, and use "
+            "confident-learning tools to find and prune likely-mislabelled rows."
+        ),
+        "techniques": [
+            "Confident learning (e.g. cleanlab) to flag mislabelled rows",
+            "Noise-robust losses (label smoothing, symmetric / MAE loss)",
+            "Early stopping and regularisation to resist memorising errors",
+            "Ensemble disagreement to surface suspect labels",
+        ],
+    },
+    "feature_noise": {
+        "title": "Feature measurement noise",
+        "base_severity": "medium",
+        "explanation": (
+            "Additive measurement noise was injected into this feature, lowering its "
+            "signal-to-noise ratio. Linear models will attenuate its coefficient "
+            "(regression dilution); the feature looks weaker than it truly is."
+        ),
+        "recommendation": (
+            "Prefer noise-tolerant models and regularisation; aggregate or smooth if "
+            "repeated measurements exist for the same entity."
+        ),
+        "techniques": [
+            "L2 regularisation to stabilise the attenuated coefficient",
+            "Tree ensembles — more tolerant of feature noise than linear models",
+            "Aggregate/denoise if multiple readings per entity are available",
+            "Robust scaling so the noise does not dominate distance metrics",
+        ],
+    },
+    "drift": {
+        "title": "Distribution drift over row order",
+        "base_severity": "medium",
+        "explanation": (
+            "This feature's distribution shifts across the dataset index (its "
+            "early rows differ from its late rows). A random train/test split leaks "
+            "the late regime into training and overstates real-world performance."
+        ),
+        "recommendation": (
+            "Split by row/time order rather than randomly, and validate on the later "
+            "regime. Detrend the feature or add the time index as a feature."
+        ),
+        "techniques": [
+            "Time-ordered train/test split (do not shuffle)",
+            "Detrend / difference the feature to remove the systematic shift",
+            "Add the row index or timestamp as an explicit feature",
+            "Rolling or periodic retraining for a deployed model",
+        ],
+    },
+    "covariate_shift": {
+        "title": "Covariate shift",
+        "base_severity": "medium",
+        "explanation": (
+            "This feature's marginal distribution was moved away from the original "
+            "(train ≠ deployment distribution) while its relationship to the label is "
+            "preserved. Models tuned on the original distribution mis-calibrate."
+        ),
+        "recommendation": (
+            "Re-weight training rows by the density ratio (importance weighting), or "
+            "re-standardise using deployment statistics, and validate on the shifted "
+            "distribution."
+        ),
+        "techniques": [
+            "Importance weighting via density-ratio estimation",
+            "Domain-adaptation methods",
+            "Re-standardise the feature using deployment-time statistics",
+            "Validate explicitly on the shifted distribution",
+        ],
+    },
+    "leakage": {
+        "title": "Target leakage",
+        "base_severity": "critical",
+        "explanation": (
+            "This column is a near-perfect proxy for the target that would NOT be "
+            "available at prediction time. It inflates offline metrics to look great "
+            "and then collapses in production. This is leakage, not signal."
+        ),
+        "recommendation": (
+            "Drop this column before training. If a single feature gives suspiciously "
+            "high accuracy/AUROC, treat it as leakage until proven otherwise."
+        ),
+        "techniques": [
+            "Remove the leaking column from the feature set",
+            "Audit all features for ones derived from the target",
+            "Be suspicious of any single feature with near-perfect predictive power",
+        ],
+    },
+    "class_imbalance": {
+        "title": "Class imbalance",
+        "base_severity": "medium",
+        "explanation": (
+            "The target classes are far from balanced. A model can score high "
+            "accuracy by predicting the majority class while being useless on the "
+            "minority class that usually matters most."
+        ),
+        "recommendation": (
+            "Stop using raw accuracy — evaluate with PR-AUC / F1 / recall on the "
+            "minority class, and rebalance via class weights or resampling."
+        ),
+        "techniques": [
+            "Class weights (e.g. class_weight='balanced')",
+            "Resampling: SMOTE / oversample minority or undersample majority",
+            "Stratified train/test splitting and cross-validation",
+            "Evaluate with PR-AUC, F1, recall — not accuracy",
+            "Tune the decision threshold rather than defaulting to 0.5",
+        ],
+    },
+}
+
+
+def _escalate(base: Severity, fraction: float | None) -> Severity:
+    """Bump severity up one tier when a corruption rate is large."""
+    if fraction is None:
+        return base
+    if fraction >= 0.30:
+        return _bump(base, 2)
+    if fraction >= 0.15:
+        return _bump(base, 1)
+    return base
+
+
+def _bump(sev: Severity, by: int) -> Severity:
+    rank = min(3, _SEVERITY_RANK.get(sev, 1) + by)
+    for name, value in _SEVERITY_RANK.items():
+        if value == rank:
+            return name
+    return sev
+
+
+def build_issue(
+    mode: str, *, magnitude: str, fraction: float | None = None, detail: dict[str, Any] | None = None
+) -> Issue:
+    """Assemble the guidance :class:`Issue` for a mechanism + realized magnitude.
+
+    ``fraction`` (a corruption rate in [0, 1]) escalates the base severity for
+    rate-like mechanisms; pass ``None`` for mechanisms where rate is irrelevant.
+    Unknown mechanisms fall back to a generic medium-severity entry so a new
+    plugin failure mode still renders coherently.
+    """
+    guide = _GUIDE.get(mode)
+    if guide is None:
+        return Issue(
+            mode=mode,
+            title=mode.replace("_", " ").title(),
+            severity="medium",
+            magnitude=magnitude,
+            explanation=f"Column was modified by the {mode!r} mechanism.",
+            recommendation="Inspect the realized effect and handle accordingly.",
+            techniques=[],
+            detail=detail or {},
+        )
+    return Issue(
+        mode=mode,
+        title=guide["title"],
+        severity=_escalate(guide["base_severity"], fraction),
+        magnitude=magnitude,
+        explanation=guide["explanation"],
+        recommendation=guide["recommendation"],
+        techniques=list(guide["techniques"]),
+        detail=detail or {},
+    )
+
+
+def severity_rank(sev: Severity) -> int:
+    """Numeric rank for sorting issues by urgency (higher = more urgent)."""
+    return _SEVERITY_RANK.get(sev, 1)

datadoom/engine/audit.py

@@ -0,0 +1,290 @@
+"""Human-readable audit report for a generation (bound into the export bundle).
+
+Renders the full :class:`~datadoom.engine.reports.ReportBundle` — compliance, the
+per-column guide (stats + data-quality issues + ML advice), injected failures,
+causal truth, difficulty, and the determinism record — into a single Markdown
+document, ``audit_report.md``, opened with a clickable **table of contents** so a
+reader can jump straight to a section (or a specific column). It ships alongside
+the data, ``metadata.json`` and the locked ``spec.resolved.yaml`` so a downloaded
+bundle is self-describing.
+
+Pure, deterministic, **timestamp-free** (invariant #6): the same
+``(spec_hash, seed)`` renders byte-identical Markdown.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from .reports import ReportBundle
+    from .spec.models import Spec
+
+_SEVERITY_MARK = {"critical": "🔴", "high": "🟠", "medium": "🟡", "low": "⚪"}
+
+
+def _slug(text: str) -> str:
+    """GitHub-style heading anchor: lowercase, drop punctuation, spaces → hyphens.
+
+    Matches the id common Markdown renderers (GitHub, VS Code) derive from a
+    heading, so the table-of-contents links resolve. Headings in this document
+    are kept free of decorative characters so the mapping stays unambiguous.
+    """
+    s = text.strip().lower().replace("`", "")
+    s = re.sub(r"[^\w\s-]", "", s)  # keep word chars, whitespace, hyphens
+    return s.replace(" ", "-")
+
+
+def _fmt(x: Any) -> str:
+    """Compact, stable formatting for a number (or pass-through for non-numbers)."""
+    if x is None:
+        return "—"
+    if isinstance(x, bool):
+        return str(x)
+    if isinstance(x, int):
+        return str(x)
+    if isinstance(x, float):
+        if x != x:  # NaN
+            return "—"
+        if abs(x) >= 1e5 or (x != 0 and abs(x) < 1e-3):
+            return f"{x:.3e}"
+        return f"{x:.4g}"
+    return str(x)
+
+
+def _pct(x: float | None) -> str:
+    return "—" if x is None else f"{x * 100:.1f}%"
+
+
+def render_audit_markdown(
+    spec: Spec, report: ReportBundle, *, package_version: str | None = None
+) -> str:
+    """Render the audit report for ``report`` as a Markdown string."""
+    r = report.to_dict()
+    det = r.get("determinism") or {}
+    dist = r.get("distribution")
+    profile = r.get("profile")
+    failures = r.get("failures")
+    truth = r.get("causal_truth")
+    difficulty = r.get("difficulty")
+
+    lines: list[str] = []
+    w = lines.append
+
+    # --- title -------------------------------------------------------------------
+    w(f"# DataDoom Audit Report — {spec.name}")
+    w("")
+    if spec.description:
+        w(f"_{spec.description}_")
+        w("")
+    w("> Regenerate byte-identical data from the locked `spec.resolved.yaml` in this")
+    w("> bundle. This report is deterministic — no timestamps or ambient state.")
+    w("")
+
+    # --- table of contents -------------------------------------------------------
+    # (title, present?) — only list sections that are actually rendered.
+    toc: list[tuple[str, bool]] = [
+        ("Overview", True),
+        ("Distribution compliance", bool(dist and dist.get("features"))),
+        ("Column guide", bool(profile and profile.get("columns"))),
+        ("Injected failures", bool(failures and failures.get("modes"))),
+        ("Causal truth", bool(truth and truth.get("edges"))),
+        ("Difficulty calibration", bool(difficulty)),
+        ("Determinism", bool(det.get("artifact_checksums"))),
+    ]
+    w("## Contents")
+    w("")
+    for title, present in toc:
+        if not present:
+            continue
+        w(f"- [{title}](#{_slug(title)})")
+        if title == "Column guide" and profile:
+            for col in profile["columns"]:
+                n_issues = len(col.get("issues") or [])
+                suffix = f" — {n_issues} issue{'s' if n_issues != 1 else ''}" if n_issues else ""
+                w(f"  - [{col['name']}](#{_slug(col['name'])}){suffix}")
+    w("")
+
+    # --- overview ----------------------------------------------------------------
+    w("## Overview")
+    w("")
+    w("| Field | Value |")
+    w("|---|---|")
+    w(f"| Spec hash | `{det.get('spec_hash', '—')}` |")
+    w(f"| Seed | `{det.get('seed', '—')}` |")
+    w(f"| Rows | {spec.rows} |")
+    w(f"| Features (declared) | {len(spec.features)} |")
+    w(f"| datadoom_version | {spec.datadoom_version} |")
+    if package_version:
+        w(f"| Engine version | {package_version} |")
+    score = r.get("compliance_score")
+    if score is not None:
+        w(f"| Compliance score | {_pct(score)} |")
+    w("")
+
+    _compliance(w, dist)
+    _column_guide(w, profile)
+    _failures(w, failures)
+    _causal(w, truth)
+    _difficulty(w, difficulty)
+    _determinism(w, det)
+
+    w("---")
+    w("_Generated by DataDoom — controllable, reproducible synthetic data._")
+    w("")
+    return "\n".join(lines)
+
+
+def _compliance(w: Any, dist: dict[str, Any] | None) -> None:
+    if not dist or not dist.get("features"):
+        return
+    w("## Distribution compliance")
+    w("")
+    w("Honest fit of each realized column against its requested distribution (KS or")
+    w("chi-square goodness-of-fit). Parameters are never refit to the sample.")
+    w("")
+    w("| Feature | Distribution | Emp. mean | Emp. std | Clamped | p-value | Verdict |")
+    w("|---|---|---|---|---|---|---|")
+    for c in dist["features"]:
+        emp = c.get("empirical") or {}
+        applicable = c.get("applicable", True)
+        verdict = "n/a" if not applicable else ("pass" if c.get("passed") else "review")
+        w(
+            f"| {c.get('feature')} | {c.get('dist')} | {_fmt(emp.get('mean'))} | "
+            f"{_fmt(emp.get('std'))} | {_pct(c.get('clamped_fraction'))} | "
+            f"{_fmt(c.get('p_value')) if applicable else '—'} | {verdict} |"
+        )
+    w("")
+
+
+def _column_guide(w: Any, profile: dict[str, Any] | None) -> None:
+    if not profile or not profile.get("columns"):
+        return
+    s = profile.get("summary") or {}
+    w("## Column guide")
+    w("")
+    w(
+        f"{s.get('n_columns', 0)} columns · {s.get('columns_with_issues', 0)} with "
+        f"data-quality issues · {s.get('critical_issues', 0)} critical · "
+        f"{s.get('high_issues', 0)} high severity."
+    )
+    if s.get("label"):
+        w("")
+        w(f"Detected target column: **{s['label']}**.")
+    w("")
+    for col in profile["columns"]:
+        # Heading is the bare column name so its anchor is stable for the TOC; the
+        # role/type/derivation goes on the line below.
+        w(f"### {col['name']}")
+        meta = f"_{col.get('role', 'feature')} · {col.get('feature_type')} ({col.get('dtype')})_"
+        parents = col.get("parents") or []
+        if parents:
+            meta += f" · _derived from: {', '.join(parents)}_"
+        w(meta)
+        w("")
+        bits = [
+            f"rows {col.get('count')}",
+            f"missing {_pct(col.get('missing_pct'))}",
+            f"unique {col.get('unique')}",
+        ]
+        stats = col.get("stats")
+        if stats:
+            bits += [
+                f"mean {_fmt(stats.get('mean'))}",
+                f"std {_fmt(stats.get('std'))}",
+                f"min {_fmt(stats.get('min'))}",
+                f"median {_fmt(stats.get('median'))}",
+                f"max {_fmt(stats.get('max'))}",
+            ]
+        imb = col.get("imbalance")
+        if imb:
+            bits.append(f"balance {_pct(imb.get('majority_pct'))}/{_pct(imb.get('minority_pct'))}")
+        inj = col.get("injected")
+        if inj and inj.get("missing_pct"):
+            bits.append(f"missing after injection {_pct(inj.get('missing_pct'))}")
+        w("- " + " · ".join(bits))
+
+        cats = col.get("categories")
+        if cats:
+            preview = ", ".join(f"{c['value']} {_pct(c['pct'])}" for c in cats[:6])
+            w(f"- classes: {preview}")
+
+        for issue in col.get("issues") or []:
+            mark = _SEVERITY_MARK.get(issue.get("severity", "medium"), "•")
+            w("")
+            w(f"- {mark} **{issue.get('title')}** ({issue.get('severity')}) — {issue.get('magnitude')}")
+            w(f"  - {issue.get('explanation')}")
+            w(f"  - **How to handle it:** {issue.get('recommendation')}")
+            for t in issue.get("techniques") or []:
+                w(f"    - {t}")
+        w("")
+
+
+def _failures(w: Any, failures: dict[str, Any] | None) -> None:
+    if not failures or not failures.get("modes"):
+        return
+    w("## Injected failures")
+    w("")
+    w(f"{failures.get('count', 0)} corruption(s) applied to the injected variant.")
+    w("")
+    for m in failures["modes"]:
+        target = m.get("column") or m.get("into") or ", ".join((m.get("nullified_fraction") or {}).keys())
+        w(f"- **{m.get('type')}** → `{target}`")
+        for k, v in m.items():
+            if k in ("index", "type", "mechanism"):
+                continue
+            w(f"  - {k}: {_fmt(v) if isinstance(v, (int, float)) else v}")
+    w("")
+
+
+def _causal(w: Any, truth: dict[str, Any] | None) -> None:
+    if not truth or not truth.get("edges"):
+        return
+    w("## Causal truth")
+    w("")
+    w("The true generating graph (structural equations). Edges into an intervened")
+    w("node are detached (`active: false`).")
+    w("")
+    w("| From | To | Function | Active |")
+    w("|---|---|---|---|")
+    for e in truth["edges"]:
+        w(f"| {e.get('from')} | {e.get('to')} | {e.get('fn')} | {e.get('active', True)} |")
+    interventions = truth.get("interventions") or {}
+    if interventions:
+        w("")
+        w("Interventions: " + ", ".join(f"do({k}={_fmt(v)})" for k, v in interventions.items()))
+    w("")
+
+
+def _difficulty(w: Any, diff: dict[str, Any] | None) -> None:
+    if not diff:
+        return
+    target = diff.get("target") or {}
+    band = target.get("band") or [None, None]
+    w("## Difficulty calibration")
+    w("")
+    w(
+        f"Achieved **{_fmt(diff.get('achieved_metric'))} "
+        f"{str(diff.get('metric_name', '')).upper()}** against target band "
+        f"{_fmt(band[0])}–{_fmt(band[1])} "
+        f"({'in band' if diff.get('band_met') else 'closest reached'}) "
+        f"in {diff.get('iterations', 0)} iteration(s)."
+    )
+    w("")
+
+
+def _determinism(w: Any, det: dict[str, Any]) -> None:
+    checks = det.get("artifact_checksums") or {}
+    if not checks:
+        return
+    w("## Determinism")
+    w("")
+    w("SHA-256 checksums of the data artifacts — the same `(spec_hash, seed)`")
+    w("reproduces these byte-for-byte.")
+    w("")
+    w("| File | Checksum |")
+    w("|---|---|")
+    for name, digest in checks.items():
+        w(f"| `{name}` | `{digest}` |")
+    w("")

datadoom/engine/causal/__init__.py

@@ -0,0 +1,15 @@
+"""Causal engine — DAG construction, structural functions, SEM execution (05 §3)."""
+
+from __future__ import annotations
+
+from .execute import execute_causal, resolve_interventions
+from .functions import STRUCTURAL_FNS, StructuralFn
+from .graph import CausalDag
+
+__all__ = [
+    "CausalDag",
+    "StructuralFn",
+    "STRUCTURAL_FNS",
+    "execute_causal",
+    "resolve_interventions",
+]