confound-controls 0.3.2__py3-none-any.whl

confound_controls/__init__.py

@@ -0,0 +1,102 @@
+"""Controls for "did my classifier learn the signal, or a confound?"
+
+Extracted from the aim3 control battery shared byte-identically between
+two internal research projects.
+"""
+
+from .ablation import (
+    INCONCLUSIVE as ABLATION_INCONCLUSIVE,
+)
+from .ablation import (
+    STRUCTURE_DEPENDENT,
+    STRUCTURE_INDEPENDENT,
+    AblationResult,
+    ablation_control,
+    assert_ablation_changed_input,
+)
+from .battery import (
+    ConfoundResult,
+    battery_passes,
+    evaluate_confound,
+    format_battery,
+    run_battery,
+)
+from .incremental import (
+    ADDS,
+    HARMS,
+    NO_EFFECT,
+    UNDERPOWERED,
+    DeltaResult,
+    incremental_validity,
+    paired_delta_auroc,
+)
+from .matching import MatchResult, match_negatives
+from .metrics import (
+    DRIVEN,
+    INCONCLUSIVE,
+    INVERTED,
+    PARTIAL,
+    ROBUST,
+    AurocCI,
+    bootstrap_auroc,
+    recovery,
+    verdict,
+)
+from .sequence import (
+    CONFIRMED,
+    NOT_CONFIRMED,
+    ControlSpan,
+    GroupedDelta,
+    ShuffleResult,
+    confirm_knockout,
+    dinucleotide_counts,
+    dinucleotide_shuffle,
+    grouped_delta_ci,
+    knockout_span,
+    sample_control_span,
+)
+
+__version__ = "0.3.2"
+
+__all__ = [
+    "ABLATION_INCONCLUSIVE",
+    "ADDS",
+    "CONFIRMED",
+    "DRIVEN",
+    "HARMS",
+    "INCONCLUSIVE",
+    "INVERTED",
+    "NOT_CONFIRMED",
+    "NO_EFFECT",
+    "PARTIAL",
+    "ROBUST",
+    "STRUCTURE_DEPENDENT",
+    "STRUCTURE_INDEPENDENT",
+    "UNDERPOWERED",
+    "AblationResult",
+    "AurocCI",
+    "ConfoundResult",
+    "ControlSpan",
+    "DeltaResult",
+    "GroupedDelta",
+    "MatchResult",
+    "ShuffleResult",
+    "ablation_control",
+    "assert_ablation_changed_input",
+    "battery_passes",
+    "bootstrap_auroc",
+    "confirm_knockout",
+    "dinucleotide_counts",
+    "dinucleotide_shuffle",
+    "evaluate_confound",
+    "format_battery",
+    "grouped_delta_ci",
+    "incremental_validity",
+    "knockout_span",
+    "match_negatives",
+    "paired_delta_auroc",
+    "recovery",
+    "run_battery",
+    "sample_control_span",
+    "verdict",
+]

confound_controls/ablation.py

@@ -0,0 +1,165 @@
+"""Score an ablated input with the same model and see what survives.
+
+Extracted from an internal grammar-shuffle script
+(byte-identical in a second internal project). There the ablation was a
+dinucleotide-preserving shuffle of promoter sequences, applied zero-shot to a
+frozen probe: if AUROC collapses toward chance, the model was reading order
+rather than composition.
+
+Nothing about that argument is specific to sequences. The general shape is:
+take a model's scores on real input, take its scores on input with the
+structure of interest destroyed but the nuisance properties preserved, and ask
+whether the difference clears zero.
+
+The source got one thing right that is worth keeping and generalising -- it
+refused to run when the ablation had not changed anything:
+
+    assert not np.allclose(shuffled, real), \\
+        "shuffled embeddings identical to real - shuffle/embeds broken"
+
+Without that check, a broken ablation produces scores identical to the real
+ones, the delta is exactly zero, the interval hugs zero, and the module reports
+"the model survives ablation" -- the strongest possible result, obtained by
+doing nothing. That failure is silent, and it is the same shape as the vacuous
+matched control in `matching.py`: an inert control returns the original answer,
+which reads as the hypothesis surviving.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+import numpy as np
+from sklearn.metrics import roc_auc_score
+
+# What survived the ablation.
+STRUCTURE_DEPENDENT = "structure-dependent"  # ablation collapsed it to chance
+STRUCTURE_INDEPENDENT = "structure-independent"  # ablated scores still separate
+PARTIAL = "partial"
+INCONCLUSIVE = "inconclusive"
+
+
+@dataclass(frozen=True)
+class AblationResult:
+    auroc_real: float
+    auroc_ablated: float
+    delta: float
+    delta_lo: float
+    delta_hi: float
+    ablated_lo: float
+    ablated_hi: float
+    verdict: str
+    n_resamples_used: int
+
+    @property
+    def ablated_includes_chance(self) -> bool:
+        return self.ablated_lo <= 0.5 <= self.ablated_hi
+
+
+def assert_ablation_changed_input(real, ablated, *, name: str = "ablation") -> None:
+    """Refuse an ablation that did not alter its input.
+
+    Call this on the ABLATED REPRESENTATION (sequences, embeddings, features) --
+    not on the scores. Identical scores can also arise from a model that
+    genuinely ignores the ablated structure, which is a finding; identical
+    inputs are a broken pipeline, and the two must not be confused.
+    """
+    real = np.asarray(real)
+    ablated = np.asarray(ablated)
+    if real.shape != ablated.shape:
+        raise ValueError(
+            f"{name}: shapes differ, real={real.shape} ablated={ablated.shape}; "
+            f"these are meant to be the same inputs with structure destroyed"
+        )
+    if real.dtype.kind in "fc" or ablated.dtype.kind in "fc":
+        # equal_nan=True: NaN != NaN by default, so two IDENTICAL arrays
+        # containing NaN slipped past the guard built to catch identical input.
+        identical = np.allclose(real, ablated, equal_nan=True)
+    else:
+        identical = np.array_equal(real, ablated)
+    if identical:
+        raise ValueError(
+            f"{name}: the ablated input is identical to the real input, so the "
+            f"ablation did nothing. Every downstream comparison would report "
+            f"the model as surviving it -- the strongest possible result, "
+            f"obtained by doing nothing."
+        )
+
+
+def ablation_control(
+    y,
+    p_real,
+    p_ablated,
+    *,
+    n: int = 2000,
+    seed: int = 42,
+    max_ci_width: float | None = None,
+) -> AblationResult:
+    """Compare real vs ablated scores with a paired bootstrap.
+
+    Both the delta interval and the ablated AUROC's own interval are reported,
+    because they answer different questions: the delta says the ablation
+    changed something, the ablated interval says whether anything is left.
+    """
+    y = np.asarray(y)
+    p_real = np.asarray(p_real)
+    p_ablated = np.asarray(p_ablated)
+    if not (len(y) == len(p_real) == len(p_ablated)):
+        raise ValueError(
+            f"lengths differ: y={len(y)}, real={len(p_real)}, ablated={len(p_ablated)}"
+        )
+    if len(np.unique(y)) < 2:
+        raise ValueError("AUROC needs both classes present in y")
+
+    real_auc = roc_auc_score(y, p_real)
+    abl_auc = roc_auc_score(y, p_ablated)
+
+    rng = np.random.RandomState(seed)
+    idx = np.arange(len(y))
+    deltas, ablated_aucs = [], []
+    for _ in range(n):
+        b = rng.choice(idx, len(idx), replace=True)
+        if len(np.unique(y[b])) < 2:
+            continue
+        a = roc_auc_score(y[b], p_ablated[b])
+        deltas.append(roc_auc_score(y[b], p_real[b]) - a)
+        ablated_aucs.append(a)
+
+    if len(deltas) < n // 2:
+        raise ValueError(
+            f"only {len(deltas)} of {n} resamples contained both classes; the "
+            f"intervals would rest on too few replicates to mean anything"
+        )
+
+    d_lo, d_hi = np.percentile(deltas, [2.5, 97.5])
+    a_lo, a_hi = np.percentile(ablated_aucs, [2.5, 97.5])
+    delta_excludes_zero = d_lo > 0 or d_hi < 0
+    # Separation is DISTANCE from chance, not height above it: a perfectly
+    # reversed ranking (auroc 0) discriminates exactly as well as auroc 1. The
+    # source asked `a_lo > 0.5`, so an ablation that changed nothing at all read
+    # as a partial collapse whenever the scores ran the other way.
+    ablated_excludes_chance = a_lo > 0.5 or a_hi < 0.5
+    ablated_includes_chance = not ablated_excludes_chance
+
+    if max_ci_width is not None and (d_hi - d_lo) > max_ci_width:
+        verdict = INCONCLUSIVE
+    elif ablated_includes_chance and delta_excludes_zero:
+        verdict = STRUCTURE_DEPENDENT
+    elif ablated_excludes_chance and not delta_excludes_zero:
+        # Ablated scores still separate, and the drop is not distinguishable
+        # from zero: the structure was never what the model was using.
+        verdict = STRUCTURE_INDEPENDENT
+    else:
+        verdict = PARTIAL
+
+    return AblationResult(
+        auroc_real=float(real_auc),
+        auroc_ablated=float(abl_auc),
+        delta=float(real_auc - abl_auc),
+        delta_lo=float(d_lo),
+        delta_hi=float(d_hi),
+        ablated_lo=float(a_lo),
+        ablated_hi=float(a_hi),
+        verdict=verdict,
+        n_resamples_used=len(deltas),
+    )

confound_controls/battery.py

@@ -0,0 +1,166 @@
+"""Run a confound battery: for each confound, match on it and re-score.
+
+Extracted from an internal matched-negatives script, whose
+main() hardcoded the study in four separate places:
+
+    CONFOUNDS = os.path.join(REPO, "results/aim3_control/confounds.tsv")
+    ANCHOR    = 0.629
+    KMER_COLS = [f"kmer3_{i}" for i in range(64)]
+    CONFOUNDS_SPEC = {"gc": ["gc"], "expression": ["log_basemean"], ...}
+
+-- input paths derived from __file__, the study's own baseline AUROC, and the
+study's column schema. None of it was reachable from the command line, so the
+module ran against exactly one dataset in exactly one repo layout. The logic in
+between was general the whole time.
+
+Here the caller passes the frame, the probabilities, the spec, and the anchor.
+Nothing is read from disk and nothing is written to it.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass, field
+
+import numpy as np
+import pandas as pd
+
+from .matching import match_negatives
+from .metrics import AurocCI, bootstrap_auroc, recovery, verdict
+
+
+@dataclass(frozen=True)
+class ConfoundResult:
+    name: str
+    ci: AurocCI
+    recovery: float
+    verdict: str
+    n_positives: int
+    n_matched_negatives: int
+    match_complete: bool
+    match_selective: bool
+    unmatched_positions: list = field(default_factory=list)
+
+
+def evaluate_confound(
+    df: pd.DataFrame,
+    prob_map: Mapping,
+    columns: Sequence[str],
+    anchor: float,
+    *,
+    name: str = "confound",
+    id_column: str = "id",
+    label_column: str = "label",
+    recovery_threshold: float = 0.70,
+    require_complete_match: bool = True,
+    require_selective_match: bool = True,
+    bootstrap_n: int = 2000,
+    seed: int = 42,
+) -> ConfoundResult:
+    """Match negatives to positives on `columns`, then re-score on the subset."""
+    missing = [c for c in list(columns) + [id_column, label_column] if c not in df.columns]
+    if missing:
+        raise ValueError(
+            f"{name}: columns not in frame: {missing}. Present: {list(df.columns)[:12]}"
+        )
+
+    pos = df[df[label_column] == 1]
+    neg = df[df[label_column] == 0]
+    if pos.empty or neg.empty:
+        raise ValueError(
+            f"{name}: need both classes; got {len(pos)} positive and {len(neg)} negative rows"
+        )
+
+    dupes = df[id_column][df[id_column].duplicated()].unique()
+    if len(dupes):
+        raise ValueError(
+            f"{name}: {len(dupes)} duplicate id(s) in column '{id_column}' "
+            f"(e.g. {list(dupes[:3])}). An id must name exactly one row: "
+            f"probabilities are looked up by id and the evaluated set is rebuilt "
+            f"by id, so a duplicate silently drags its twin into the evaluation "
+            f"and the reported 1:1 design is not 1:1."
+        )
+
+    match = match_negatives(
+        pos[list(columns)].to_numpy(),
+        neg[id_column].tolist(),
+        neg[list(columns)].to_numpy(),
+    )
+    if require_complete_match:
+        match.require_complete()
+    if require_selective_match:
+        match.require_selective()
+
+    eval_ids = set(pos[id_column].tolist()) | set(match.matched_ids)
+    sub = df[df[id_column].isin(eval_ids)]
+
+    unknown = [g for g in sub[id_column] if g not in prob_map]
+    if unknown:
+        raise ValueError(
+            f"{name}: {len(unknown)} evaluated ids have no probability "
+            f"(e.g. {unknown[:3]}). Scoring them as anything would invent data."
+        )
+
+    y = sub[label_column].to_numpy()
+    p = np.array([prob_map[g] for g in sub[id_column]])
+    ci = bootstrap_auroc(y, p, n=bootstrap_n, seed=seed)
+
+    return ConfoundResult(
+        name=name,
+        ci=ci,
+        recovery=recovery(ci.point, anchor),
+        verdict=verdict(ci, anchor, recovery_threshold),
+        n_positives=int((y == 1).sum()),
+        n_matched_negatives=int((y == 0).sum()),
+        match_complete=match.complete,
+        match_selective=match.selective,
+        unmatched_positions=list(match.unmatched_positions),
+    )
+
+
+def run_battery(
+    df: pd.DataFrame,
+    prob_map: Mapping,
+    spec: Mapping[str, Sequence[str]],
+    anchor: float,
+    **kwargs,
+) -> dict[str, ConfoundResult]:
+    """Evaluate every confound in `spec`. Keys name the confound, values its columns."""
+    if not spec:
+        raise ValueError("spec is empty; there is nothing to control for")
+    return {
+        name: evaluate_confound(df, prob_map, cols, anchor, name=name, **kwargs)
+        for name, cols in spec.items()
+    }
+
+
+def battery_passes(results: Mapping[str, ConfoundResult]) -> bool:
+    """True only if every confound came back robust.
+
+    The source computed this over a hardcoded subset of its own confound names
+    (`univariate + ["joint"]`), so a confound added to the spec was scored,
+    printed, and then left out of the pass/fail decision. Here every result in
+    the battery counts -- a confound you bothered to declare cannot be quietly
+    excluded from the verdict it exists to inform.
+    """
+    if not results:
+        raise ValueError("no results; an empty battery cannot pass")
+    from .metrics import ROBUST
+
+    return all(r.verdict == ROBUST for r in results.values())
+
+
+def format_battery(results: Mapping[str, ConfoundResult], anchor: float) -> str:
+    lines = [f"confound battery (anchor AUROC={anchor:.4f})", ""]
+    for name, r in results.items():
+        flag = "" if r.match_complete else "  [INCOMPLETE MATCH]"
+        if not r.match_selective:
+            flag += "  [VACUOUS CONTROL: whole pool used]"
+        lines.append(
+            f"  {name:18s} AUROC={r.ci.point:.4f} "
+            f"CI[{r.ci.lo:.4f},{r.ci.hi:.4f}] "
+            f"recovery={r.recovery:.2f} npos={r.n_positives} "
+            f"-> {r.verdict}{flag}"
+        )
+    lines += ["", f"BATTERY PASS: {battery_passes(results)}"]
+    return "\n".join(lines)

confound_controls/incremental.py

@@ -0,0 +1,175 @@
+"""Does the new feature add anything beyond the confounds?
+
+Extracted from an internal incremental-validity script
+(byte-identical in a second internal project). Fit a confound-only model and a
+confound+feature model, evaluate both on the same held-out rows, and put a
+paired bootstrap interval on the AUROC difference.
+
+Paired is the point. Two independent intervals on two AUROCs overlap far more
+often than the interval on their difference excludes zero, because the pair is
+computed on the SAME resampled rows and the shared variance cancels. Comparing
+two separately-reported AUROCs by eye is the error the paired delta exists to
+prevent.
+
+The source's verdict was two-valued:
+
+    verdict = "FM-adds-signal" if lo > 0 else "no-added-signal"
+
+so an interval straddling zero, an interval too wide to say anything, and an
+interval lying entirely BELOW zero -- the augmented model actively worse --
+all reported as "no-added-signal". The last of those is not an absence of
+signal, it is a finding, and it was being discarded.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+import numpy as np
+from sklearn.linear_model import LogisticRegression
+from sklearn.metrics import roc_auc_score
+from sklearn.preprocessing import StandardScaler
+
+ADDS = "adds-signal"
+HARMS = "harms"
+NO_EFFECT = "no-added-signal"
+UNDERPOWERED = "underpowered"
+
+
+@dataclass(frozen=True)
+class DeltaResult:
+    auroc_base: float
+    auroc_augmented: float
+    delta: float
+    lo: float
+    hi: float
+    p_delta_le_0: float
+    verdict: str
+    n_resamples_used: int
+
+    @property
+    def excludes_zero(self) -> bool:
+        return self.lo > 0 or self.hi < 0
+
+
+def paired_delta_auroc(
+    y,
+    p_base,
+    p_augmented,
+    n: int = 2000,
+    seed: int = 42,
+    max_ci_width: float | None = None,
+) -> DeltaResult:
+    """Bootstrap the AUROC difference on paired resamples.
+
+    `max_ci_width` opts in to reporting UNDERPOWERED rather than NO_EFFECT when
+    the interval is too wide to distinguish the two. Off by default, because
+    turning it on silently would reclassify existing results.
+    """
+    y = np.asarray(y)
+    p_base = np.asarray(p_base)
+    p_augmented = np.asarray(p_augmented)
+    if not (len(y) == len(p_base) == len(p_augmented)):
+        raise ValueError(
+            f"lengths differ: y={len(y)}, base={len(p_base)}, augmented={len(p_augmented)}"
+        )
+    if len(np.unique(y)) < 2:
+        raise ValueError("AUROC needs both classes present in y")
+
+    base = roc_auc_score(y, p_base)
+    aug = roc_auc_score(y, p_augmented)
+
+    rng = np.random.RandomState(seed)
+    idx = np.arange(len(y))
+    deltas = []
+    for _ in range(n):
+        b = rng.choice(idx, len(idx), replace=True)
+        if len(np.unique(y[b])) < 2:
+            continue
+        # Same rows for both models -- that is what makes it paired.
+        deltas.append(roc_auc_score(y[b], p_augmented[b]) - roc_auc_score(y[b], p_base[b]))
+
+    if len(deltas) < n // 2:
+        raise ValueError(
+            f"only {len(deltas)} of {n} resamples contained both classes; the "
+            f"interval would rest on too few replicates to mean anything "
+            f"(n_pos={int((y == 1).sum())}, n_neg={int((y == 0).sum())})"
+        )
+
+    deltas_arr = np.asarray(deltas)
+    lo, hi = np.percentile(deltas_arr, [2.5, 97.5])
+    p_le0 = float(np.mean(deltas_arr <= 0))
+
+    if lo > 0:
+        verdict = ADDS
+    elif hi < 0:
+        # Not "no signal" -- the augmented model is reliably WORSE, which the
+        # source folded into its no-signal branch and never surfaced.
+        verdict = HARMS
+    elif max_ci_width is not None and (hi - lo) > max_ci_width:
+        verdict = UNDERPOWERED
+    else:
+        verdict = NO_EFFECT
+
+    return DeltaResult(
+        auroc_base=float(base),
+        auroc_augmented=float(aug),
+        delta=float(aug - base),
+        lo=float(lo),
+        hi=float(hi),
+        p_delta_le_0=p_le0,
+        verdict=verdict,
+        n_resamples_used=len(deltas),
+    )
+
+
+def _fit_predict(X_train, y_train, X_test, seed: int = 42):
+    scaler = StandardScaler().fit(X_train)
+    model = LogisticRegression(class_weight="balanced", max_iter=2000, random_state=seed)
+    model.fit(scaler.transform(X_train), y_train)
+    return model.predict_proba(scaler.transform(X_test))[:, 1]
+
+
+def incremental_validity(
+    X_train_confounds,
+    X_test_confounds,
+    feature_train,
+    feature_test,
+    y_train,
+    y_test,
+    *,
+    seed: int = 42,
+    n: int = 2000,
+    max_ci_width: float | None = None,
+) -> DeltaResult:
+    """Confound-only vs confound+feature, compared with a paired delta.
+
+    `feature_train` must be out-of-fold. Fitting the feature on the same rows
+    the confound model is evaluated against leaks the label into the augmented
+    model and manufactures the very lift this control is testing for -- the
+    source computed it with GroupKFold for exactly that reason.
+    """
+    X_train_confounds = np.asarray(X_train_confounds, dtype=float)
+    X_test_confounds = np.asarray(X_test_confounds, dtype=float)
+    feature_train = np.asarray(feature_train, dtype=float).reshape(-1, 1)
+    feature_test = np.asarray(feature_test, dtype=float).reshape(-1, 1)
+
+    if X_train_confounds.shape[0] != feature_train.shape[0]:
+        raise ValueError(
+            f"train rows differ: confounds={X_train_confounds.shape[0]}, "
+            f"feature={feature_train.shape[0]}"
+        )
+    if X_test_confounds.shape[0] != feature_test.shape[0]:
+        raise ValueError(
+            f"test rows differ: confounds={X_test_confounds.shape[0]}, "
+            f"feature={feature_test.shape[0]}"
+        )
+
+    p_base = _fit_predict(X_train_confounds, y_train, X_test_confounds, seed)
+    p_aug = _fit_predict(
+        np.column_stack([X_train_confounds, feature_train]),
+        y_train,
+        np.column_stack([X_test_confounds, feature_test]),
+        seed,
+    )
+    return paired_delta_auroc(y_test, p_base, p_aug, n=n, seed=seed, max_ci_width=max_ci_width)