tabaudit 0.1.0__py3-none-any.whl

tabaudit/__init__.py

@@ -0,0 +1,8 @@
+"""tabaudit — audit tabular ML datasets before you train."""
+
+__version__ = "0.1.0"
+
+from tabaudit.audit import run_audit
+from tabaudit.findings import AuditReport, Finding, Severity
+
+__all__ = ["AuditReport", "Finding", "Severity", "__version__", "run_audit"]

tabaudit/audit.py

@@ -0,0 +1,102 @@
+"""Orchestrates loading, task inference and running every registered check."""
+
+from __future__ import annotations
+
+import time
+from collections.abc import Callable, Iterable
+from pathlib import Path
+
+import pandas as pd
+
+from tabaudit import __version__
+from tabaudit.checks import REGISTRY
+from tabaudit.context import AuditContext
+from tabaudit.findings import AuditReport, CheckRun, DatasetSummary, Finding
+from tabaudit.loader import dtype_kinds, infer_task, load_table
+
+ProgressFn = Callable[[str, str], None]  # (check_name, status) -> None
+
+
+def run_audit(
+    data: str | Path | pd.DataFrame,
+    target: str | None = None,
+    test: str | Path | pd.DataFrame | None = None,
+    checks: Iterable[str] | None = None,
+    max_rows: int = 50_000,
+    random_state: int = 42,
+    on_progress: ProgressFn | None = None,
+) -> AuditReport:
+    """Audit a dataset and return an :class:`AuditReport`.
+
+    Parameters
+    ----------
+    data:    path to CSV/Parquet/… or a DataFrame.
+    target:  name of the label column (omit for unsupervised checks only).
+    test:    optional held-out set, used to detect train/test contamination.
+    checks:  subset of check names to run (default: all).
+    """
+    df, path = (
+        (data, "<DataFrame>") if isinstance(data, pd.DataFrame) else (load_table(data), str(data))
+    )
+    test_df, test_path = (None, None)
+    if test is not None:
+        test_df, test_path = (
+            (test, "<DataFrame>")
+            if isinstance(test, pd.DataFrame)
+            else (load_table(test), str(test))
+        )
+
+    if target is not None and target not in df.columns:
+        close = [c for c in df.columns if c.lower() == target.lower()]
+        hint = f" Did you mean '{close[0]}'?" if close else ""
+        raise KeyError(f"Target column '{target}' not found.{hint}")
+
+    task = infer_task(df[target]) if target else "unsupervised"
+    n_classes = (
+        int(df[target].nunique(dropna=True)) if target and task == "classification" else None
+    )
+
+    ctx = AuditContext(
+        df=df,
+        target=target,
+        task=task,
+        test_df=test_df,
+        max_rows=max_rows,
+        random_state=random_state,
+    )
+    selected = list(checks) if checks else list(REGISTRY)
+    unknown = [c for c in selected if c not in REGISTRY]
+    if unknown:
+        raise ValueError(f"Unknown check(s): {unknown}. Available: {list(REGISTRY)}")
+
+    findings: list[Finding] = []
+    runs: list[CheckRun] = []
+    for name in selected:
+        fn = REGISTRY[name]
+        if on_progress:
+            on_progress(name, "running")
+        t0 = time.perf_counter()
+        try:
+            out = fn(ctx)
+            findings.extend(out)
+            runs.append(CheckRun(name, "ok", time.perf_counter() - t0, len(out)))
+        except Exception as exc:
+            runs.append(
+                CheckRun(name, "error", time.perf_counter() - t0, 0, f"{type(exc).__name__}: {exc}")
+            )
+        if on_progress:
+            on_progress(name, runs[-1].status)
+
+    summary = DatasetSummary(
+        path=path,
+        n_rows=len(df),
+        n_cols=int(df.shape[1]),
+        target=target,
+        task=task,
+        n_classes=n_classes,
+        test_path=test_path,
+        n_test_rows=len(test_df) if test_df is not None else None,
+        memory_mb=round(float(df.memory_usage(deep=True).sum()) / 1e6, 2),
+        dtypes=dtype_kinds(df),
+    )
+    return AuditReport(summary=summary, findings=findings, checks=runs, version=__version__)

tabaudit/checks/__init__.py

@@ -0,0 +1,22 @@
+"""Check registry. Each check is a callable (AuditContext) -> list[Finding]."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+
+from tabaudit.checks import duplicates, imbalance, label_noise, leakage, schema
+from tabaudit.context import AuditContext
+from tabaudit.findings import Finding
+
+CheckFn = Callable[[AuditContext], list[Finding]]
+
+# Ordered: cheap structural checks first, model-based checks last.
+REGISTRY: dict[str, CheckFn] = {
+    "schema": schema.run,
+    "duplicates": duplicates.run,
+    "imbalance": imbalance.run,
+    "leakage": leakage.run,
+    "label_noise": label_noise.run,
+}
+
+__all__ = ["REGISTRY", "CheckFn"]

tabaudit/checks/duplicates.py

@@ -0,0 +1,83 @@
+"""Duplicate rows, conflicting labels, and train/test contamination."""
+
+from __future__ import annotations
+
+import pandas as pd
+
+from tabaudit.context import AuditContext
+from tabaudit.findings import Finding, Severity
+
+CHECK = "duplicates"
+
+
+def _row_hashes(df: pd.DataFrame) -> pd.Series:
+    return pd.util.hash_pandas_object(df, index=False)
+
+
+def run(ctx: AuditContext) -> list[Finding]:
+    df = ctx.df
+    findings: list[Finding] = []
+    n = len(df)
+    feats = ctx.feature_cols
+
+    # ---- exact duplicate rows (all columns) -----------------------------
+    full_dup = df.duplicated(keep="first")
+    n_full = int(full_dup.sum())
+    if n_full:
+        frac = n_full / n
+        sev = Severity.HIGH if frac >= 0.05 else Severity.MEDIUM if frac >= 0.01 else Severity.LOW
+        findings.append(
+            Finding(
+                check=CHECK,
+                severity=sev,
+                title=f"{n_full:,} exact duplicate rows ({frac:.1%})",
+                detail="Identical rows will land in both train and validation folds, "
+                "inflating every cross-validation metric.",
+                recommendation="`df.drop_duplicates()` before splitting.",
+                evidence={"n_duplicates": n_full, "fraction": round(frac, 4)},
+            )
+        )
+
+    # ---- same features, different label --------------------------------
+    if ctx.target and feats:
+        fh = _row_hashes(df[feats])
+        grp = df.groupby(fh.values)[ctx.target].nunique(dropna=False)
+        conflicting = grp[grp > 1]
+        if not conflicting.empty:
+            n_rows = int(fh.isin(conflicting.index).sum())
+            findings.append(
+                Finding(
+                    check=CHECK,
+                    severity=Severity.MEDIUM,
+                    title=f"{len(conflicting):,} feature-identical group(s) carry conflicting labels",
+                    detail=f"{n_rows:,} rows share identical features with another row but have a "
+                    "different target value. No model can fit these; they cap achievable accuracy.",
+                    recommendation="Investigate the labelling process, or add the feature that "
+                    "actually distinguishes them.",
+                    evidence={"n_groups": len(conflicting), "n_rows": n_rows},
+                )
+            )
+
+    # ---- train/test overlap --------------------------------------------
+    if ctx.test_df is not None:
+        common = [c for c in feats if c in ctx.test_df.columns]
+        if common:
+            train_h = set(_row_hashes(df[common]))
+            test_h = _row_hashes(ctx.test_df[common])
+            n_leak = int(test_h.isin(train_h).sum())
+            if n_leak:
+                frac = n_leak / len(ctx.test_df)
+                sev = Severity.CRITICAL if frac >= 0.01 else Severity.HIGH
+                findings.append(
+                    Finding(
+                        check=CHECK,
+                        severity=sev,
+                        title=f"{n_leak:,} test rows ({frac:.1%}) also appear in the training set",
+                        detail="The model has already seen these rows. Any test-set score is "
+                        "partly memorisation, not generalisation.",
+                        recommendation="Remove overlapping rows from the test set, or re-split "
+                        "with a group-aware splitter if rows belong to entities.",
+                        evidence={"n_overlap": n_leak, "fraction_of_test": round(frac, 4)},
+                    )
+                )
+    return findings

tabaudit/checks/imbalance.py

@@ -0,0 +1,70 @@
+"""Class imbalance and rare classes."""
+
+from __future__ import annotations
+
+from tabaudit.context import AuditContext
+from tabaudit.findings import Finding, Severity
+
+CHECK = "imbalance"
+
+
+def run(ctx: AuditContext) -> list[Finding]:
+    if ctx.task != "classification" or ctx.y is None:
+        return []
+    y = ctx.y.dropna()
+    counts = y.value_counts()
+    if len(counts) < 2:
+        return [
+            Finding(
+                check=CHECK,
+                severity=Severity.CRITICAL,
+                title="Target has only one class",
+                detail=f"All {len(y):,} labelled rows are '{counts.index[0]}'.",
+                recommendation="This is not a classification dataset as-is.",
+                columns=[ctx.target or ""],
+            )
+        ]
+    dist = {str(k): int(v) for k, v in counts.items()}
+    minority_frac = float(counts.min() / counts.sum())
+    ratio = float(counts.max() / counts.min())
+    findings: list[Finding] = []
+
+    sev: Severity | None
+    if ratio >= 100:
+        sev = Severity.HIGH
+    elif ratio >= 10:
+        sev = Severity.MEDIUM
+    elif ratio >= 3:
+        sev = Severity.LOW
+    else:
+        sev = None
+    if sev is not None:
+        findings.append(
+            Finding(
+                check=CHECK,
+                severity=sev,
+                title=f"Class imbalance {ratio:,.0f}:1 (minority class = {minority_frac:.2%})",
+                detail="Accuracy is meaningless here - a model predicting the majority class "
+                f"scores {1 - minority_frac:.1%} without learning anything.",
+                recommendation="Report precision/recall, PR-AUC or balanced accuracy; use "
+                "stratified splits and class weights. Never oversample before splitting.",
+                columns=[ctx.target or ""],
+                evidence={"class_counts": dist, "imbalance_ratio": round(ratio, 2)},
+            )
+        )
+
+    rare = counts[counts < 10]
+    if not rare.empty:
+        findings.append(
+            Finding(
+                check=CHECK,
+                severity=Severity.MEDIUM,
+                title=f"{len(rare)} class(es) have fewer than 10 examples",
+                detail=", ".join(f"'{k}' ({v})" for k, v in rare.items()),
+                recommendation="Merge into an 'other' class or collect more data; "
+                "stratified CV will fail or be meaningless for these.",
+                columns=[ctx.target or ""],
+                evidence={"rare_classes": {str(k): int(v) for k, v in rare.items()}},
+            )
+        )
+    return findings

tabaudit/checks/label_noise.py

@@ -0,0 +1,134 @@
+"""Likely mislabeled rows, found with confident learning (cleanlab).
+
+We train an out-of-fold gradient-boosting model, obtain predicted class probabilities for
+every row, and let cleanlab compare them with the given labels. Rows whose given label is
+confidently contradicted by the model are flagged.
+"""
+
+from __future__ import annotations
+
+import warnings
+
+import numpy as np
+import pandas as pd
+from sklearn.ensemble import HistGradientBoostingClassifier
+from sklearn.model_selection import StratifiedKFold, cross_val_predict
+
+from tabaudit.context import AuditContext
+from tabaudit.findings import Finding, Severity
+
+CHECK = "label_noise"
+TOP_N = 25
+# A suspect counts as *likely* mislabeled when the out-of-fold model gives the given label
+# less than this probability. Chosen on the synthetic benchmark (examples/validate_label_noise.py)
+# as the point where precision is useful without collapsing recall.
+LIKELY_MAX_SELF_CONFIDENCE = 0.2
+
+
+def make_model(random_state: int) -> HistGradientBoostingClassifier:
+    """Deliberately regularised: an over-confident model makes cleanlab over-flag."""
+    return HistGradientBoostingClassifier(
+        max_iter=200,
+        learning_rate=0.05,
+        max_leaf_nodes=15,
+        min_samples_leaf=40,
+        l2_regularization=1.0,
+        early_stopping=True,
+        validation_fraction=0.15,
+        random_state=random_state,
+    )
+
+
+def run(ctx: AuditContext) -> list[Finding]:
+    if ctx.task != "classification" or ctx.y is None:
+        return []
+    try:
+        from cleanlab.filter import find_label_issues
+    except ImportError:  # pragma: no cover
+        return []
+
+    idx = ctx.sample_index()
+    # Leaky / identifier columns are excluded on purpose: a leak would make the model agree
+    # with every wrong label and hide the noise we are trying to find.
+    Xe = ctx.X_encoded_clean.loc[idx]
+    y = ctx.y.loc[idx]
+    keep = y.notna().to_numpy()
+    Xe, y = Xe[keep], y[keep]
+    if len(y) < 50 or Xe.shape[1] == 0:
+        return []
+
+    y_codes, classes = pd.factorize(y)
+    counts = np.bincount(y_codes)
+    if len(classes) < 2 or counts.min() < 5:
+        return []
+    n_splits = int(min(5, counts.min()))
+
+    model = make_model(ctx.random_state)
+    cv = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=ctx.random_state)
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore")
+        pred_probs = cross_val_predict(model, Xe.to_numpy(), y_codes, cv=cv, method="predict_proba")
+        # n_jobs=1: cleanlab's worker pool misbehaves on Windows and buys nothing at this size.
+        suspected = find_label_issues(
+            labels=y_codes, pred_probs=pred_probs, filter_by="confident_learning", n_jobs=1
+        )
+    self_conf = pred_probs[np.arange(len(y_codes)), y_codes]
+    likely = suspected & (self_conf < LIKELY_MAX_SELF_CONFIDENCE)
+    # Rank all suspects by how little the model believes the given label.
+    issue_idx = np.flatnonzero(suspected)
+    issue_idx = issue_idx[np.argsort(self_conf[issue_idx])]
+
+    n_suspected, n_likely = int(suspected.sum()), int(likely.sum())
+    if n_suspected == 0:
+        return []
+    frac_likely = n_likely / len(y)
+    frac_suspected = n_suspected / len(y)
+    if frac_likely >= 0.08:
+        sev = Severity.HIGH
+    elif frac_likely >= 0.03:
+        sev = Severity.MEDIUM
+    elif frac_likely >= 0.005:
+        sev = Severity.LOW
+    else:
+        sev = Severity.INFO
+
+    # Top suspects, for the report.
+    suspects = []
+    for i in issue_idx[:TOP_N]:
+        given = y_codes[i]
+        suggested = int(np.argmax(pred_probs[i]))
+        suspects.append(
+            {
+                "row": int(Xe.index[i]),
+                "given_label": str(classes[given]),
+                "suggested_label": str(classes[suggested]),
+                "confidence_in_given": round(float(pred_probs[i, given]), 3),
+                "confidence_in_suggested": round(float(pred_probs[i, suggested]), 3),
+            }
+        )
+    scope = f" (of a {len(y):,}-row sample)" if len(idx) < len(ctx.df) else ""
+    excluded = sorted(ctx.excluded_features & set(ctx.X_encoded.columns))
+    if excluded:
+        scope += f"; estimated with {', '.join(excluded)} excluded as leaky/identifier"
+    return [
+        Finding(
+            check=CHECK,
+            severity=sev,
+            title=f"~{n_likely:,} rows ({frac_likely:.1%}) are likely mislabeled, "
+            f"{n_suspected - n_likely:,} more suspected",
+            detail="An out-of-fold model confidently disagrees with the given label on these rows"
+            f"{scope}. 'Likely' = model gives the given label <{LIKELY_MAX_SELF_CONFIDENCE:.0%} "
+            "probability; 'suspected' = flagged by confident learning (cleanlab). Label noise "
+            "caps achievable accuracy and misleads model selection.",
+            recommendation="Review the top suspects below (ranked most-confident first). If they are "
+            "real errors, relabel or drop them; do not silently trust the reported test accuracy.",
+            columns=[ctx.target or ""],
+            evidence={
+                "n_likely": n_likely,
+                "n_suspected": n_suspected,
+                "fraction_likely": round(frac_likely, 4),
+                "fraction_suspected": round(frac_suspected, 4),
+                "top_suspects": suspects,
+            },
+        )
+    ]