detectorproof 0.1.0__py3-none-any.whl

detectorproof/panel.py

@@ -0,0 +1,434 @@
+"""`panel` - measure the detectors before trusting any of them.
+
+This is the gate everything else in the instrument depends on, and it exists
+because of a specific measured failure: on one corpus, three widely benchmarked
+detectors sat at chance while their model cards advertised sub-1% equal error
+rates. A panel that quietly averaged them in would have reported a confident
+number built partly on models that could not do the task at all.
+
+So `panel` scores a **declared** set of frozen detectors over a labelled control
+set, reports the area under the ROC curve for each, and refuses to let an
+incompetent one contribute anything downstream.
+
+Five refusals are enforced here rather than described in a README. Each one was
+bought by a real error:
+
+1. **A detector at or near chance is NOT COMPETENT ON THIS MATERIAL.** It is
+   reported in full - never dropped, never footnoted - and excluded from every
+   aggregate. `competent_detectors()` is the only accessor downstream code
+   should use.
+
+2. **The panel is declared before the run.** A score for a detector nobody
+   declared is a fault. A declared detector with no scores is NOT MEASURED and
+   stays in the report as that. This is the rule that makes the panel a
+   pre-registration rather than a selection.
+
+3. **Overlapping crops of one file are not independent samples.** Two scores
+   for the same detector, item and condition are a fault, not something to
+   average. Collapsing windows to one value per item is the caller's job and
+   must happen before the statistic, because doing it afterwards is exactly the
+   error that inflated one published proportion from 75% to 86%.
+
+4. **NOT MEASURED is a third state.** It is never zero, never an absence of
+   effect, and it propagates to the report as itself.
+
+5. **A near-miss at the band edge is recorded, not re-banded.** When a detector
+   lands just outside the chance band, the report says so. In the study behind
+   this package one detector landed at 0.447 against a band of [0.45, 0.55] -
+   marginally outside, on the inverted side - and the honest response was to
+   record the edge rather than widen the band after seeing the number.
+
+There is no function here that recommends a transformation, and there will not
+be one. Optimising audio against a detector score is measured, in the work this
+came from, to have made the audio *more* separable, not less - one intervention
+tuned to a chosen statistic moved a detector's AUC from 0.576 to 0.763. The
+instrument reports; it does not advise.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Iterable, Mapping, Sequence
+
+from .domain import (
+    BONAFIDE,
+    DETECTOR,
+    HIGHER_SYNTHETIC,
+    ITEM,
+    LABEL,
+    ORIENTATION,
+    ORIENTATION_UNRESOLVED,
+    score_is_measured,
+    validate_rows,
+)
+from .stats import auc
+
+# The pre-declared chance band. A detector whose AUC falls inside it is not
+# competent on the material it was scored against. The default is the band the
+# originating protocol declared before any score existed; a caller may declare a
+# different one, but it is an argument to the run and not something to adjust
+# afterwards.
+CHANCE_BAND = (0.45, 0.55)
+
+# How far outside the band still counts as "at the edge" for reporting purposes.
+# This changes no verdict. It exists so that a marginal result is visible as
+# marginal instead of reading like a clean pass.
+BAND_EDGE_MARGIN = 0.01
+
+COMPETENT = "COMPETENT"
+NOT_COMPETENT = "NOT COMPETENT ON THIS MATERIAL"
+NOT_MEASURED_VERDICT = "NOT MEASURED"
+UNRESOLVED_ORIENTATION = "ORIENTATION UNRESOLVED"
+
+
+class PanelFault(ValueError):
+    """A fault in the panel declaration or the score table.
+
+    Distinct from `DomainError`, which is a single field outside its domain.
+    A `PanelFault` is a table that parses cleanly and still cannot support the
+    measurement asked of it - a duplicate key, an undeclared detector, a control
+    set with one class.
+    """
+
+
+@dataclass(frozen=True)
+class Detector:
+    """One declared panel member.
+
+    `identity` is whatever pins the weights: a commit, a revision, a file hash.
+    It is free text because the sources differ, but it is required, because a
+    panel that cannot say which checkpoint produced a number is not frozen.
+    """
+
+    name: str
+    orientation: str
+    identity: str
+    source: str = ""
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "name", DETECTOR.parse(self.name))
+        object.__setattr__(self, "orientation", ORIENTATION.parse(self.orientation))
+        identity = (self.identity or "").strip()
+        if not identity:
+            raise PanelFault(
+                f"detector {self.name!r} has no identity. A panel member must name the "
+                "checkpoint that produced its scores - a revision, a commit or a file "
+                "hash. A panel that cannot say which weights it ran is not frozen."
+            )
+        object.__setattr__(self, "identity", identity)
+
+
+@dataclass(frozen=True)
+class DetectorResult:
+    """What the panel found for one declared detector."""
+
+    detector: str
+    verdict: str
+    auc: float | None
+    n_bonafide: int
+    n_synthetic: int
+    orientation: str
+    identity: str
+    band_edge: bool = False
+    note: str = ""
+
+    @property
+    def competent(self) -> bool:
+        return self.verdict == COMPETENT
+
+
+@dataclass(frozen=True)
+class PanelReport:
+    """The full panel result. Every declared detector appears exactly once."""
+
+    results: tuple[DetectorResult, ...]
+    chance_band: tuple[float, float]
+    control_items: int
+    faults: tuple[str, ...] = field(default=())
+
+    def competent_detectors(self) -> tuple[DetectorResult, ...]:
+        """The only accessor downstream code may aggregate over.
+
+        Incompetent and unmeasured detectors are still in `results` and still in
+        every printed report. They are absent from here, which is what "excluded
+        from every downstream answer" means in code.
+        """
+        return tuple(r for r in self.results if r.competent)
+
+    def excluded(self) -> tuple[DetectorResult, ...]:
+        return tuple(r for r in self.results if not r.competent)
+
+
+def canonical_labels(labels: Mapping[str, str] | Iterable[tuple[str, str]]) -> dict[str, str]:
+    """Canonicalise the control set, refusing any item labelled two ways.
+
+    **Refusal 6, and it is the reason this function exists rather than a dict
+    comprehension.** The control set is what defines the two classes, so an item
+    carrying two labels does not make the answer slightly less certain - it makes
+    the answer depend on which row happened to be read last.
+
+    Measured, before this refusal existed: three items scored 3, 2, 1, with item
+    `a` labelled both ways and `b` bona fide and `c` synthetic. Reversing only
+    the two conflicting rows for `a` moved the result from **AUC 0.5, exit 1** to
+    **AUC 1.0, exit 0**. Neither input was refused. A verdict that depends on row
+    order is not a measurement.
+
+    The collision is detected **after** canonicalisation, because items are
+    compared case-insensitively after stripping: `A` and `a` are one item here,
+    so labelling them differently is the same fault wearing different case.
+
+    Repeating an item with the *same* label is accepted. It is redundant rather
+    than ambiguous, and no ordering of those rows can change any verdict.
+    """
+    pairs = labels.items() if isinstance(labels, Mapping) else labels
+
+    canonical: dict[str, str] = {}
+    original: dict[str, str] = {}
+    for raw_item, raw_label in pairs:
+        item = ITEM.parse(raw_item)
+        key = item.lower()
+        label = LABEL.parse(raw_label)
+        if key in canonical and canonical[key] != label:
+            raise PanelFault(
+                f"control item {item!r} is labelled both {canonical[key]!r} and "
+                f"{label!r}"
+                + (
+                    f" (as {original[key]!r} and {item!r}, which are one item after "
+                    "canonicalisation)"
+                    if original[key] != item
+                    else ""
+                )
+                + ". A conflicting label is refused rather than resolved: taking "
+                "either one would make the verdict depend on the order the rows "
+                "were read in. Correct the control set."
+            )
+        canonical[key] = label
+        original.setdefault(key, item)
+    return canonical
+
+
+def _label_of(item: str, labels: Mapping[str, str]) -> str:
+    try:
+        raw = labels[item]
+    except KeyError:
+        raise PanelFault(
+            f"control item {item!r} has no label. Every item in the control set must be "
+            f"declared {BONAFIDE!r} or {'synthetic'!r}; an unlabelled item cannot be "
+            "scored and must not be silently dropped."
+        ) from None
+    return LABEL.parse(raw)
+
+
+def run_panel(
+    detectors: Sequence[Detector],
+    scores: Iterable[Mapping[str, object]],
+    labels: Mapping[str, str] | Iterable[tuple[str, str]],
+    *,
+    condition: str,
+    chance_band: tuple[float, float] = CHANCE_BAND,
+) -> PanelReport:
+    """Score a declared panel over a labelled control set.
+
+    `detectors` is the declaration, fixed before the run. `scores` is a table of
+    rows carrying `detector`, `item`, `condition`, `score`. `labels` maps each
+    control item to `bonafide` or `synthetic`. `condition` selects which rows of
+    the table are the control set - the same table usually carries the
+    transformed conditions too, and they are not what this gate measures.
+
+    Every declared detector appears in the result exactly once, whatever
+    happened to it.
+    """
+    low, high = chance_band
+    if not (0.0 <= low < high <= 1.0):
+        raise PanelFault(
+            f"chance band {chance_band} is not an interval inside [0, 1]. "
+            "The band is declared before the run; it is not a result."
+        )
+    if not detectors:
+        raise PanelFault(
+            "the panel is empty. A panel is declared before it is run, so an empty "
+            "declaration is a fault rather than a result with no rows."
+        )
+
+    declared: dict[str, Detector] = {}
+    for det in detectors:
+        if det.name.lower() in declared:
+            raise PanelFault(
+                f"detector {det.name!r} is declared twice. A panel member appears once; "
+                "two declarations of one name make its result ambiguous."
+            )
+        declared[det.name.lower()] = det
+
+    rows = validate_rows(scores)
+    wanted = condition.strip().lower()
+
+    # Refusal 3: one score per (detector, item). A second score for the same
+    # pair is a fault. This is the guard against overlapping windows arriving as
+    # independent observations.
+    by_detector: dict[str, dict[str, float | object]] = {k: {} for k in declared}
+    faults: list[str] = []
+    for row in rows:
+        if row.condition.lower() != wanted:
+            continue
+        key = row.detector.lower()
+        if key not in declared:
+            raise PanelFault(
+                f"scores contain detector {row.detector!r}, which the panel did not "
+                "declare. The panel is fixed before the run: a detector that appears "
+                "only in the results is a selection, not a measurement."
+            )
+        seen = by_detector[key]
+        item_key = row.item.lower()
+        if item_key in seen:
+            raise PanelFault(
+                f"detector {row.detector!r} has more than one score for item "
+                f"{row.item!r} under condition {condition!r}. Overlapping analysis "
+                "windows are not independent samples and must be collapsed to one "
+                "value per item before this gate, not averaged inside it."
+            )
+        seen[item_key] = row.score
+
+    # Canonicalise through the collision-refusing helper. A dict comprehension
+    # here was the blocking defect: it silently kept whichever label was read
+    # last, so reversing two rows could flip a verdict.
+    control = canonical_labels(labels)
+    if not control:
+        raise PanelFault("the control set is empty; there is nothing to score against.")
+
+    results: list[DetectorResult] = []
+    for key, det in declared.items():
+        seen = by_detector[key]
+
+        # Refusal 4: separate the two ways a number can be absent. A detector
+        # never run is NOT MEASURED. A detector run and recorded as the sentinel
+        # is also NOT MEASURED. Neither is zero.
+        bonafide_scores: list[float] = []
+        synthetic_scores: list[float] = []
+        for item, label in control.items():
+            if item not in seen:
+                continue
+            value = seen[item]
+            if not score_is_measured(value):
+                continue
+            if label == BONAFIDE:
+                bonafide_scores.append(float(value))
+            else:
+                synthetic_scores.append(float(value))
+        measured = len(bonafide_scores) + len(synthetic_scores)
+
+        # Orientation decides which class the detector is supposed to score
+        # HIGHER, and therefore which is the positive class for the AUC. Both
+        # branches produce an AUC where above 0.5 means "separates correctly",
+        # so a single chance band applies to the whole panel regardless of which
+        # way each model's native scale runs.
+        if det.orientation == HIGHER_SYNTHETIC:
+            positive, negative = synthetic_scores, bonafide_scores
+        else:
+            positive, negative = bonafide_scores, synthetic_scores
+
+        if measured == 0:
+            results.append(
+                DetectorResult(
+                    detector=det.name,
+                    verdict=NOT_MEASURED_VERDICT,
+                    auc=None,
+                    n_bonafide=0,
+                    n_synthetic=0,
+                    orientation=det.orientation,
+                    identity=det.identity,
+                    note=(
+                        "declared but produced no measurement on this control set. "
+                        "NOT MEASURED is not zero and not an absence of effect."
+                    ),
+                )
+            )
+            continue
+
+        if not bonafide_scores or not synthetic_scores:
+            results.append(
+                DetectorResult(
+                    detector=det.name,
+                    verdict=NOT_MEASURED_VERDICT,
+                    auc=None,
+                    n_bonafide=len(bonafide_scores),
+                    n_synthetic=len(synthetic_scores),
+                    orientation=det.orientation,
+                    identity=det.identity,
+                    note=(
+                        "only one class of the control set was measured, so AUC is "
+                        "undefined. An undefined AUC is not 0.5 and is not at chance."
+                    ),
+                )
+            )
+            continue
+
+        value = auc(positive, negative)
+        n_bona = len(bonafide_scores)
+        n_synth = len(synthetic_scores)
+
+        # Refusal 5: the band is applied by the letter, and a near miss is
+        # recorded as a near miss rather than resolved by moving the band.
+        at_chance = low <= value <= high
+        edge = (not at_chance) and (
+            low - BAND_EDGE_MARGIN <= value < low or high < value <= high + BAND_EDGE_MARGIN
+        )
+
+        if det.orientation == ORIENTATION_UNRESOLVED:
+            verdict = UNRESOLVED_ORIENTATION
+            note = (
+                "orientation is not resolved from documentation, so the DIRECTION of "
+                "this detector's score is not interpreted. The value is reported; its "
+                "sign is not. Excluded from aggregates."
+            )
+        elif at_chance:
+            verdict = NOT_COMPETENT
+            note = (
+                f"AUC {value:.4f} is inside the declared chance band "
+                f"[{low}, {high}]. Reported in full and excluded from every "
+                "downstream answer; never averaged in, never used to break a tie."
+            )
+        else:
+            verdict = COMPETENT
+            note = ""
+            if edge:
+                note = (
+                    f"AUC {value:.4f} is within {BAND_EDGE_MARGIN} of the declared band "
+                    f"[{low}, {high}]. Recorded as a band-edge result. The band was "
+                    "declared before the run and is not widened to absorb it."
+                )
+
+        results.append(
+            DetectorResult(
+                detector=det.name,
+                verdict=verdict,
+                auc=value,
+                n_bonafide=n_bona,
+                n_synthetic=n_synth,
+                orientation=det.orientation,
+                identity=det.identity,
+                band_edge=edge,
+                note=note,
+            )
+        )
+
+    return PanelReport(
+        results=tuple(results),
+        chance_band=(low, high),
+        control_items=len(control),
+        faults=tuple(faults),
+    )
+
+
+__all__ = [
+    "BAND_EDGE_MARGIN",
+    "canonical_labels",
+    "CHANCE_BAND",
+    "COMPETENT",
+    "Detector",
+    "DetectorResult",
+    "NOT_COMPETENT",
+    "NOT_MEASURED_VERDICT",
+    "PanelFault",
+    "PanelReport",
+    "UNRESOLVED_ORIENTATION",
+    "run_panel",
+]

detectorproof/report.py

@@ -0,0 +1,102 @@
+"""Rendering a panel result as text.
+
+The wording here is part of the instrument, not decoration. Three rules govern
+every line this module prints, and each was bought by a real reporting error in
+the study behind the package:
+
+**A null is "no evidence of a difference", never "no difference".** A
+non-significant test at a small n is absence of evidence. The one time this was
+written the other way round it had to be corrected in four documents.
+
+**NOT MEASURED is printed as itself.** Never as 0, never as a blank cell, never
+omitted from the table. A reader scanning a column of numbers must be able to
+see that a cell is missing rather than small.
+
+**An excluded detector is printed in full.** The refusal removes it from the
+aggregates, not from the page. A panel that hid its incompetent members would be
+indistinguishable from a panel chosen to agree.
+"""
+from __future__ import annotations
+
+from .panel import PanelReport
+
+
+def _fmt_auc(value: float | None) -> str:
+    return "NOT MEASURED" if value is None else f"{value:.4f}"
+
+
+def render(report: PanelReport) -> str:
+    """A plain-text panel report. Every declared detector appears exactly once."""
+    low, high = report.chance_band
+    lines: list[str] = []
+    lines.append("PANEL")
+    lines.append("=====")
+    lines.append("")
+    lines.append(f"control items declared : {report.control_items}")
+    lines.append(f"chance band (declared) : [{low}, {high}]")
+    lines.append(f"detectors declared     : {len(report.results)}")
+    lines.append("")
+
+    width = max((len(r.detector) for r in report.results), default=8)
+    width = max(width, 8)
+    header = f"{'detector'.ljust(width)}  {'AUC':>12}  {'bona':>5} {'synth':>5}  verdict"
+    lines.append(header)
+    lines.append("-" * len(header))
+    for r in sorted(report.results, key=lambda x: (x.auc is None, -(x.auc or 0.0))):
+        lines.append(
+            f"{r.detector.ljust(width)}  {_fmt_auc(r.auc):>12}  "
+            f"{r.n_bonafide:>5} {r.n_synthetic:>5}  {r.verdict}"
+        )
+    lines.append("")
+
+    # The declared identity of every panel member, printed rather than merely
+    # promised. A verdict is a statement about a specific checkpoint, so a
+    # report that names the detector but not the weights that produced its
+    # scores cannot be checked by the person reading it. An earlier version of
+    # this renderer said identities were "recorded above" and then printed none
+    # of them.
+    lines.append("DECLARED IDENTITIES")
+    lines.append("-------------------")
+    for r in sorted(report.results, key=lambda x: x.detector.lower()):
+        lines.append(f"  {r.detector.ljust(width)}  {r.orientation:<17}  {r.identity}")
+    lines.append("")
+
+    notes = [r for r in report.results if r.note]
+    if notes:
+        lines.append("NOTES")
+        lines.append("-----")
+        for r in notes:
+            lines.append(f"  {r.detector}: {r.note}")
+        lines.append("")
+
+    competent = report.competent_detectors()
+    excluded = report.excluded()
+    lines.append("GATE")
+    lines.append("----")
+    lines.append(f"  competent on this material : {len(competent)} of {len(report.results)}")
+    if competent:
+        lines.append("    " + ", ".join(r.detector for r in competent))
+    if excluded:
+        lines.append(f"  excluded from every downstream answer : {len(excluded)}")
+        for r in excluded:
+            lines.append(f"    {r.detector} - {r.verdict}")
+    lines.append("")
+
+    if not competent:
+        lines.append(
+            "  No detector in this declared panel is competent on this control set.\n"
+            "  That is a reportable result about the panel and this material, and it is\n"
+            "  NOT evidence that the material is genuine or that detection is impossible.\n"
+            "  It is absence of evidence, not evidence of absence. Nothing downstream may\n"
+            "  run on an empty competent set."
+        )
+    else:
+        lines.append(
+            "  Each verdict above is a statement about the checkpoint named in DECLARED\n"
+            "  IDENTITIES, measured on this control set - 'not competent on this\n"
+            "  material' - and never about a detector's general quality."
+        )
+    return "\n".join(lines)
+
+
+__all__ = ["render"]

detectorproof/stats.py

@@ -0,0 +1,92 @@
+"""Exact, dependency-free statistics.
+
+Everything here is computed from the numbers a study already has. There is no
+numpy and no scipy, so this package installs and runs on a machine that will
+never hold the audio - the same reason spkproof's core has no dependencies.
+
+Two properties are deliberate and both are load-bearing:
+
+**Ties are handled by midranks, everywhere.** A detector whose scores are
+compressed near zero produces long runs of equal values, and a rank statistic
+that breaks ties arbitrarily reports a different answer depending on the order
+its input happened to arrive in. One detector in the study this package came
+out of had real scores of 0.003 against 0.002; its rank statistic is the only
+interpretable thing about it, so the ranks have to be right.
+
+**AUC is computed from the rank sum, not by counting pairs.** They agree, but
+the rank-sum form is O(n log n) rather than O(n^2) and, more importantly, it
+makes the tie correction a property of the ranking rather than a special case
+bolted onto a comparison.
+"""
+from __future__ import annotations
+
+from typing import Sequence
+
+
+def midranks(values: Sequence[float]) -> list[float]:
+    """Ranks, 1-based, with tied values sharing the mean of their positions."""
+    order = sorted(range(len(values)), key=lambda i: values[i])
+    ranks = [0.0] * len(values)
+    i = 0
+    while i < len(order):
+        j = i
+        while j + 1 < len(order) and values[order[j + 1]] == values[order[i]]:
+            j += 1
+        # Positions i..j (0-based) are tied; their 1-based ranks average to this.
+        shared = (i + j + 2) / 2.0
+        for k in range(i, j + 1):
+            ranks[order[k]] = shared
+        i = j + 1
+    return ranks
+
+
+def auc(positive: Sequence[float], negative: Sequence[float]) -> float:
+    """Area under the ROC curve, as P(positive scores above negative).
+
+    `positive` are the scores of the class the detector is supposed to give
+    HIGHER values to; `negative` the other class. Ties contribute 0.5, which is
+    what the midrank form produces without a special case.
+
+    Returns a value in [0, 1]. Both classes must be non-empty: an AUC computed
+    against an empty class is not a weak measurement, it is undefined, and
+    returning 0.5 for it would put a fabricated "at chance" into a report.
+    """
+    n_pos, n_neg = len(positive), len(negative)
+    if n_pos == 0 or n_neg == 0:
+        raise ValueError(
+            "AUC needs at least one item of each class; "
+            f"got {n_pos} positive and {n_neg} negative. "
+            "An undefined AUC is not 0.5."
+        )
+    ranks = midranks(list(positive) + list(negative))
+    rank_sum_pos = sum(ranks[:n_pos])
+    return (rank_sum_pos - n_pos * (n_pos + 1) / 2.0) / (n_pos * n_neg)
+
+
+def holm_adjusted(pvalues: Sequence[float]) -> list[float]:
+    """Holm-Bonferroni adjusted p-values, in the caller's original order.
+
+    Sorted ascending, multiplied by the descending family size, then made
+    monotone by a running maximum and capped at 1. The running maximum is the
+    step people leave out; without it an adjusted p-value can come out below
+    the one before it, which is not a Holm result.
+    """
+    n = len(pvalues)
+    if n == 0:
+        return []
+    order = sorted(range(n), key=lambda i: pvalues[i])
+    adjusted = [0.0] * n
+    running = 0.0
+    for position, index in enumerate(order):
+        scaled = pvalues[index] * (n - position)
+        running = max(running, scaled)
+        adjusted[index] = min(1.0, running)
+    return adjusted
+
+
+def holm(pvalues: Sequence[float], alpha: float = 0.05) -> list[bool]:
+    """Which tests survive Holm-Bonferroni at family-wise `alpha`."""
+    return [p <= alpha for p in holm_adjusted(pvalues)]
+
+
+__all__ = ["auc", "holm", "holm_adjusted", "midranks"]