confound-controls 0.3.2__py3-none-any.whl

confound_controls/matching.py

@@ -0,0 +1,154 @@
+"""Match negatives to positives on confounds, and say so when you cannot.
+
+Extracted from an internal matched-negatives script (also present
+byte-identical in a second internal project -- the code was copied between two
+independent repos, which is what marked it as worth extracting).
+
+The matching itself is unchanged. What changed is that it now reports whether
+it achieved the 1:1 match it advertises. The original looped
+
+    while True:
+        ...
+        if pick is not None: ...; break
+        if k >= len(neg_ids): break     # <- silently gives up on this positive
+        k *= 2
+
+so once the negative pool was exhausted it returned FEWER negatives than
+positives and said nothing. A caller comparing AUROC before and after matching
+could not tell a clean 1:1 design from one that quietly dropped a third of its
+positives -- and the drop is not random, it hits the positives in the densest
+region of confound space, exactly where matching matters most.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+import numpy as np
+from scipy.spatial import cKDTree
+
+
+@dataclass(frozen=True)
+class MatchResult:
+    """Outcome of a matching attempt.
+
+    `complete` is the question a caller actually needs answered: did every
+    positive get its own negative? `unmatched_positions` gives the indices of
+    the positives that did not, so the caller can inspect rather than guess.
+    """
+
+    matched_ids: list
+    unmatched_positions: list
+    n_positives: int
+    n_pool: int = 0
+
+    @property
+    def complete(self) -> bool:
+        return not self.unmatched_positions
+
+    @property
+    def n_matched(self) -> int:
+        return len(self.matched_ids)
+
+    @property
+    def selective(self) -> bool:
+        """Did matching actually CHOOSE, or take the whole pool?
+
+        1:1 matching without replacement from a pool the same size as the
+        positive set consumes every negative, whatever the confound values are.
+        The "matched" comparison is then byte-identical to the unmatched one,
+        so the control is vacuous -- and a vacuous control does not report
+        "inconclusive", it reports the original result, which reads as the
+        confound having been ruled out.
+        """
+        return self.n_matched < self.n_pool
+
+    def require_complete(self) -> MatchResult:
+        if not self.complete:
+            raise ValueError(
+                f"matched only {self.n_matched} of {self.n_positives} positives; "
+                f"the negative pool ran out. Unmatched positive positions: "
+                f"{self.unmatched_positions[:10]}"
+                f"{'...' if len(self.unmatched_positions) > 10 else ''}"
+            )
+        return self
+
+    def require_selective(self) -> MatchResult:
+        if not self.selective:
+            raise ValueError(
+                f"matching consumed the entire negative pool "
+                f"({self.n_matched} matched from {self.n_pool} available), so it "
+                f"selected nothing and the control is vacuous -- the matched "
+                f"comparison equals the unmatched one and will report the "
+                f"confound as ruled out. Supply more negatives than positives, "
+                f"or pass require_selective_match=False if you intend this."
+            )
+        return self
+
+
+def match_negatives(pos_vecs, neg_ids, neg_vecs) -> MatchResult:
+    """Greedy 1:1 nearest-neighbour match without replacement, on standardised columns.
+
+    Columns are standardised over positives and negatives jointly so that a
+    confound measured on a wide scale does not dominate the distance purely
+    through its units.
+
+    Args:
+        pos_vecs: (n_pos, n_confounds) confound values for the positives.
+        neg_ids: identifiers for the negatives, parallel to `neg_vecs`.
+        neg_vecs: (n_neg, n_confounds) confound values for the negatives.
+
+    Returns:
+        MatchResult -- inspect `.complete` before trusting a downstream metric,
+        or call `.require_complete()` to make an incomplete match an error.
+    """
+    pos_vecs = np.asarray(pos_vecs, dtype=float)
+    neg_vecs = np.asarray(neg_vecs, dtype=float)
+    neg_ids = list(neg_ids)
+
+    if pos_vecs.ndim != 2 or neg_vecs.ndim != 2:
+        raise ValueError("pos_vecs and neg_vecs must both be 2-D")
+    if pos_vecs.shape[1] != neg_vecs.shape[1]:
+        raise ValueError(
+            f"confound count differs: positives have {pos_vecs.shape[1]}, "
+            f"negatives have {neg_vecs.shape[1]}"
+        )
+    if len(neg_ids) != neg_vecs.shape[0]:
+        raise ValueError(
+            f"neg_ids has {len(neg_ids)} entries but neg_vecs has {neg_vecs.shape[0]} rows"
+        )
+    if not len(neg_ids):
+        return MatchResult([], list(range(len(pos_vecs))), len(pos_vecs), 0)
+
+    allv = np.vstack([pos_vecs, neg_vecs])
+    mu, sd = allv.mean(0), allv.std(0)
+    # A confound that is constant across every sample carries no information;
+    # dividing by its zero spread would produce nan distances and match at
+    # random, which looks exactly like a successful match.
+    sd = np.where(sd == 0, 1.0, sd)
+    pos_s = (pos_vecs - mu) / sd
+    neg_s = (neg_vecs - mu) / sd
+
+    tree = cKDTree(neg_s)
+    used: set[int] = set()
+    matched: list = []
+    unmatched: list[int] = []
+
+    for position, p in enumerate(pos_s):
+        k = 1
+        pick = None
+        while True:
+            k = min(k, len(neg_ids))
+            _, idxs = tree.query(p, k=k)
+            idxs = np.atleast_1d(idxs)
+            pick = next((int(i) for i in idxs if i not in used), None)
+            if pick is not None or k >= len(neg_ids):
+                break
+            k *= 2
+        if pick is None:
+            unmatched.append(position)
+        else:
+            used.add(pick)
+            matched.append(neg_ids[pick])
+
+    return MatchResult(matched, unmatched, len(pos_s), len(neg_ids))

confound_controls/metrics.py

@@ -0,0 +1,126 @@
+"""AUROC with a bootstrap interval, and the anchored verdict built on it.
+
+Extracted from an internal probe-common module (bootstrap)
+and aim3_matched_negatives.py (recovery/verdict). The maths is unchanged; the
+study constants are gone.
+
+The original had
+
+    ANCHOR = 0.629
+
+as a module-level constant -- one study's own baseline AUROC, compiled into the
+library. Every verdict the module produced was relative to it, and importing
+the module anywhere else silently scored against that number. It is now a
+required argument, because there is no defensible default for "what does good
+look like in your problem".
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+import numpy as np
+from sklearn.metrics import roc_auc_score
+
+# Verdict vocabulary. `INCONCLUSIVE` has no counterpart in the source, which
+# had no way to say "this interval is too wide to decide" and folded those
+# cases into `confound-driven` -- reporting an absent measurement as a finding.
+ROBUST = "confound-robust"
+DRIVEN = "confound-driven"
+PARTIAL = "partial"
+INCONCLUSIVE = "inconclusive"
+INVERTED = "inverted"  # the interval sits entirely BELOW chance
+
+
+@dataclass(frozen=True)
+class AurocCI:
+    point: float
+    lo: float
+    hi: float
+
+    @property
+    def excludes_chance(self) -> bool:
+        return self.lo > 0.5 or self.hi < 0.5
+
+
+def bootstrap_auroc(y, p, n: int = 2000, seed: int = 42) -> AurocCI:
+    """Percentile bootstrap CI for AUROC.
+
+    Resamples that end up single-class are skipped -- AUROC is undefined for
+    them. If too many are skipped the interval is not trustworthy, so this
+    raises rather than returning a confident-looking number computed from a
+    handful of replicates.
+    """
+    y = np.asarray(y)
+    p = np.asarray(p)
+    if y.shape[0] != p.shape[0]:
+        raise ValueError(f"y has {y.shape[0]} entries, p has {p.shape[0]}")
+    if len(np.unique(y)) < 2:
+        raise ValueError("AUROC needs both classes present in y")
+
+    point = roc_auc_score(y, p)
+    rng = np.random.RandomState(seed)
+    idx = np.arange(len(y))
+    stats = []
+    for _ in range(n):
+        b = rng.choice(idx, len(idx), replace=True)
+        if len(np.unique(y[b])) < 2:
+            continue
+        stats.append(roc_auc_score(y[b], p[b]))
+
+    if len(stats) < n // 2:
+        raise ValueError(
+            f"only {len(stats)} of {n} bootstrap resamples contained both "
+            f"classes; the interval would be built from too few replicates to "
+            f"mean anything (n_pos={int((y == 1).sum())}, "
+            f"n_neg={int((y == 0).sum())})"
+        )
+    lo, hi = np.percentile(stats, [2.5, 97.5])
+    return AurocCI(float(point), float(lo), float(hi))
+
+
+def recovery(auroc: float, anchor: float) -> float:
+    """How much of the anchor's above-chance signal survived, in [0, 1]-ish.
+
+    `anchor` is the unmatched/uncontrolled AUROC this result is being compared
+    against. An anchor at or below chance makes the ratio meaningless rather
+    than merely large, so it is refused.
+    """
+    if anchor <= 0.5:
+        raise ValueError(
+            f"anchor must be above chance to divide by its signal; got {anchor}. "
+            f"An anchor at or below 0.5 has no above-chance signal to recover, "
+            f"so the ratio is a non-quantity, not a small number."
+        )
+    return (auroc - 0.5) / (anchor - 0.5)
+
+
+def verdict(
+    ci: AurocCI,
+    anchor: float,
+    recovery_threshold: float = 0.70,
+    max_ci_width: float | None = None,
+) -> str:
+    """Classify a controlled result against its anchor.
+
+    - ROBUST: enough of the anchor's signal survived AND the interval clears chance
+    - DRIVEN: the interval straddles chance -- the control removed the signal
+    - PARTIAL: signal is real but diminished
+    - INVERTED: the interval clears chance from BELOW -- the ranking reversed
+    - INCONCLUSIVE: the interval is too wide to support any of the above
+
+    `max_ci_width` is opt-in because the source had no such concept, and adding
+    it silently would reclassify existing results.
+    """
+    if max_ci_width is not None and (ci.hi - ci.lo) > max_ci_width:
+        return INCONCLUSIVE
+    if recovery(ci.point, anchor) >= recovery_threshold and ci.excludes_chance:
+        return ROBUST
+    if not ci.excludes_chance:
+        return DRIVEN
+    if ci.hi < 0.5:
+        # Excludes chance, but from below: the control did not diminish the
+        # signal, it REVERSED it. Calling that "partial" ("signal is real but
+        # diminished") reports a pathology as a weak positive.
+        return INVERTED
+    return PARTIAL

confound_controls/sequence.py

@@ -0,0 +1,323 @@
+"""Sequence-level ablation: knock out a span, keep composition, say whether it worked.
+
+Extracted from an internal project (byte-identical in a second one):
+`aim3_shuffle_promoters.dinucl_shuffle`, `ig_knockout_ism.knockout_instance` /
+`delta_ci`, and `cg_ism._nonoverlap_start`. `cg_ism`'s own docstring says it
+"reuses ig_knockout_ism (knockout_instance, delta_ci)", so the reuse was
+already explicit -- it just had nowhere to live.
+
+## What changed: three silent fallbacks, all in the same direction
+
+Each of these returned a plausible value on failure, and each failure makes an
+ablation weaker or absent -- which reads as the model SURVIVING it, the most
+reassuring answer available.
+
+1. `dinucl_shuffle` ended with
+
+       # deterministic fallback: identity preserves counts (rare; logged by caller)
+       return seq
+
+   An un-shuffleable sequence came back UNCHANGED. Downstream that is a
+   knockout that knocked nothing out, indistinguishable from a real one.
+   "Logged by caller" is a promise, not a mechanism.
+
+2. `_nonoverlap_start` tried 50 times to place a length-matched control span
+   clear of the real motif spans, then returned `rng.randint(0, hi)` regardless
+   -- a control span that may OVERLAP the motifs, i.e. a partial real knockout
+   masquerading as the negative control it is being compared against.
+
+3. `delta_ci` resampled groups with no check that there were enough distinct
+   groups for the interval to mean anything.
+
+The dinucleotide-shuffle algorithm itself (Altschul-Erikson) is ported
+unchanged, including the determinism fix its author found the hard way -- see
+`_vertices`.
+"""
+
+from __future__ import annotations
+
+import random
+from collections import defaultdict
+from dataclasses import dataclass
+
+import numpy as np
+
+
+def dinucleotide_counts(seq: str) -> dict:
+    c: dict = defaultdict(int)
+    for i in range(len(seq) - 1):
+        c[seq[i : i + 2]] += 1
+    return dict(c)
+
+
+def _vertices(graph):
+    """`list(graph)`, never `set(graph)`.
+
+    dict key order is insertion order since 3.7, so it is derived from the
+    sequence. Set iteration order depends on the per-process randomized hash of
+    the string keys, which made `rng.choice`/`rng.shuffle` consume the PRNG in a
+    different order across process invocations -- so the same `(seq, seed)`
+    produced DIFFERENT shuffles run to run despite the PRNG being correctly
+    seeded. Preserved verbatim from the source, which carries the same warning.
+    """
+    return list(graph)
+
+
+def _connected_to_last(last, last_edges, verts) -> bool:
+    """Altschul-Erikson connectivity: every vertex with out-edges reaches `last`."""
+    intree = {v: False for v in verts}
+    intree[last] = True
+    changed = True
+    while changed:
+        changed = False
+        for v in verts:
+            if not intree[v] and v in last_edges and intree.get(last_edges[v], False):
+                intree[v] = True
+                changed = True
+    return all(intree[v] for v in verts)
+
+
+@dataclass(frozen=True)
+class ShuffleResult:
+    """A shuffle attempt, and whether it actually changed anything.
+
+    `changed` is the question the source could not answer. A shuffle that
+    returns its input is not a shuffle, and every ablation built on it is inert.
+    """
+
+    sequence: str
+    changed: bool
+    attempts: int
+
+    def require_changed(self) -> ShuffleResult:
+        if not self.changed:
+            raise ValueError(
+                f"dinucleotide shuffle returned the input unchanged after "
+                f"{self.attempts} attempts, so any ablation built on it is "
+                f"inert and will report the model as surviving an ablation "
+                f"that never happened. Short, low-complexity or homopolymeric "
+                f"sequences often cannot be reshuffled at all."
+            )
+        return self
+
+
+def dinucleotide_shuffle(seq: str, seed: int, attempts: int = 50) -> ShuffleResult:
+    """Altschul-Erikson shuffle preserving exact dinucleotide counts.
+
+    Composition and dinucleotide frequency are held fixed while order is
+    destroyed, which is what makes it a control for "did the model read order,
+    or just composition?".
+
+    Returns a ShuffleResult -- check `.changed` (or call `.require_changed()`)
+    before using the output as an ablation.
+    """
+    if len(seq) < 3:
+        return ShuffleResult(seq, False, 0)
+
+    rng = random.Random(seed)
+    last = seq[-1]
+    graph = defaultdict(list)
+    for i in range(len(seq) - 1):
+        graph[seq[i]].append(seq[i + 1])
+    verts = _vertices(graph)
+
+    for attempt in range(attempts):
+        last_edges = {}
+        ok = True
+        for v in verts:
+            if v == last:
+                continue
+            if not graph[v]:
+                ok = False
+                break
+            last_edges[v] = rng.choice(graph[v])
+        if not ok or not _connected_to_last(last, last_edges, verts):
+            rng = random.Random(seed + 7919 * (attempt + 1))
+            continue
+
+        edges = {v: list(succ) for v, succ in graph.items()}
+        for v, w in last_edges.items():
+            edges[v].remove(w)
+            rng.shuffle(edges[v])
+            edges[v].append(w)  # the chosen last-edge departs last
+        for v, succ_edges in edges.items():
+            if v not in last_edges:
+                rng.shuffle(succ_edges)
+
+        result = [seq[0]]
+        cur = seq[0]
+        for _ in range(len(seq) - 1):
+            if not edges.get(cur):
+                ok = False
+                break
+            nxt = edges[cur].pop(0)
+            result.append(nxt)
+            cur = nxt
+        out = "".join(result)
+        if ok and dinucleotide_counts(out) == dinucleotide_counts(seq):
+            return ShuffleResult(out, out != seq, attempt + 1)
+        rng = random.Random(seed + 7919 * (attempt + 1))
+
+    # The source returned `seq` here silently. It is still the only
+    # count-preserving answer available -- but it is reported, not disguised.
+    #
+    # NOT COVERED BY THE SUITE, and deliberately marked so rather than left
+    # looking tested: reaching this line needs 50 consecutive attempts to fail
+    # connectivity or the count check, and a probe over 8 sequences x 30 seeds
+    # (homopolymers, dinucleotide repeats, 2-4mers included) reached it zero
+    # times. The unshuffleable cases users actually hit -- "AAAAAAAA", or a
+    # 12-mer under an unlucky seed -- exit through the success path above with
+    # `out == seq`, which IS covered. A mutation planted on this line survives
+    # the suite; that is a true coverage gap on an unreachable backstop, not a
+    # missing test.
+    return ShuffleResult(seq, False, attempts)
+
+
+def knockout_span(seq: str, start: int, end: int, seed: int) -> ShuffleResult:
+    """Scramble `seq[start:end]` in place; flanks stay byte-identical."""
+    if not 0 <= start < end <= len(seq):
+        raise ValueError(f"span [{start}, {end}) is not inside a sequence of length {len(seq)}")
+    inner = dinucleotide_shuffle(seq[start:end], seed)
+    return ShuffleResult(seq[:start] + inner.sequence + seq[end:], inner.changed, inner.attempts)
+
+
+@dataclass(frozen=True)
+class ControlSpan:
+    start: int
+    length: int
+    disjoint: bool
+
+    @property
+    def end(self) -> int:
+        return self.start + self.length
+
+    def require_disjoint(self) -> ControlSpan:
+        if not self.disjoint:
+            raise ValueError(
+                f"could not place a length-{self.length} control span clear of "
+                f"the real spans; the returned start {self.start} OVERLAPS them, "
+                f"so the 'random control' is a partial real knockout and will be "
+                f"compared against itself. Use a longer sequence, a shorter "
+                f"total, or accept it explicitly."
+            )
+        return self
+
+
+def _draw_start(rng, lo: int, hi: int) -> int:
+    """Draw an integer in [lo, hi] INCLUSIVE, whichever PRNG flavour was passed.
+
+    The three PRNGs this package already uses disagree about the upper bound:
+    numpy's RandomState.randint and Generator.integers EXCLUDE it, stdlib
+    random.Random.randint INCLUDES it -- and Generator has no `.randint` at all.
+    `rng.randint(0, hi)` therefore meant a different range for different callers
+    and, under numpy, could never draw the last legal start.
+    """
+    if hasattr(rng, "integers"):  # numpy Generator
+        return int(rng.integers(lo, hi + 1))
+    if hasattr(rng, "random_sample"):  # numpy RandomState
+        return int(rng.randint(lo, hi + 1))
+    return int(rng.randint(lo, hi))  # stdlib random.Random
+
+
+def sample_control_span(length: int, total: int, spans, rng, attempts: int = 50) -> ControlSpan:
+    """Place a length-matched control span avoiding every real span.
+
+    The source returned a random start after 50 failures with no indication,
+    so a control that overlapped the motifs it was controlling for looked
+    identical to one that did not.
+    """
+    if total <= 0:
+        raise ValueError(f"control span length must be positive, got {total}")
+    if total > length:
+        raise ValueError(
+            f"control span length {total} is longer than the sequence ({length}), "
+            f"so no placement fits; the source clamped the range and returned a "
+            f"span running off the end, flagged as if it were valid."
+        )
+    spans = list(spans)
+    max_start = length - total  # INCLUSIVE: a span may sit flush at the end
+
+    def _clear(st: int) -> bool:
+        return all(st + total <= s or st >= e for s, e in spans)
+
+    for _ in range(attempts):
+        st = _draw_start(rng, 0, max_start)
+        if _clear(st):
+            return ControlSpan(st, total, True)
+    # Report what the fallback draw ACTUALLY is. The source hardcoded False, so a
+    # span that happened to land clear was reported as overlapping, and callers
+    # discarded a usable control (or trusted require_disjoint's error over the data).
+    st = _draw_start(rng, 0, max_start)
+    return ControlSpan(st, total, _clear(st))
+
+
+@dataclass(frozen=True)
+class GroupedDelta:
+    mean: float
+    lo: float
+    hi: float
+    n_groups: int
+
+    @property
+    def excludes_zero(self) -> bool:
+        return self.lo > 0 or self.hi < 0
+
+
+def grouped_delta_ci(
+    real, ko, groups, n: int = 2000, seed: int = 42, min_groups: int = 5
+) -> GroupedDelta:
+    """Paired CLUSTER bootstrap on mean(real - ko), resampling whole groups.
+
+    Resampling groups rather than rows is the point: promoters from one paralog
+    family are not independent observations, and a row-level bootstrap would
+    report an interval far narrower than the data supports.
+
+    `min_groups` guards the case the source did not check -- with a handful of
+    clusters the percentile interval is a number, not evidence.
+    """
+    real = np.asarray(real, dtype=float)
+    ko = np.asarray(ko, dtype=float)
+    groups = np.asarray(groups)
+    if not (real.shape == ko.shape == groups.shape):
+        raise ValueError(f"shapes differ: real={real.shape}, ko={ko.shape}, groups={groups.shape}")
+    if real.size == 0:
+        raise ValueError("no observations")
+
+    d = real - ko
+    uniq = np.unique(groups)
+    if len(uniq) < min_groups:
+        raise ValueError(
+            f"only {len(uniq)} distinct group(s); a cluster bootstrap resamples "
+            f"groups, so the interval would be drawn from at most {len(uniq)} "
+            f"distinct values. Pass min_groups=1 to override if you know what "
+            f"that interval means."
+        )
+
+    rng = np.random.RandomState(seed)
+    index_of = {g: np.where(groups == g)[0] for g in uniq}
+    stats = []
+    for _ in range(n):
+        gs = rng.choice(uniq, len(uniq), replace=True)
+        idx = np.concatenate([index_of[g] for g in gs])
+        stats.append(float(d[idx].mean()))
+    lo, hi = np.percentile(stats, [2.5, 97.5])
+    return GroupedDelta(float(d.mean()), float(lo), float(hi), len(uniq))
+
+
+CONFIRMED = "confirmed"
+NOT_CONFIRMED = "not-confirmed"
+
+
+def confirm_knockout(pooled: GroupedDelta, control: GroupedDelta) -> str:
+    """The source's confirmation criterion, unchanged.
+
+    Confirmed iff the pooled knockout drops the score, its interval clears
+    zero, AND it drops more than a length-matched control knockout does. The
+    third clause is what stops "any perturbation hurts" being read as "this
+    element matters".
+    """
+    return (
+        CONFIRMED
+        if (pooled.mean > 0 and pooled.lo > 0 and pooled.mean > control.mean)
+        else NOT_CONFIRMED
+    )