detectorproof 0.1.0__py3-none-any.whl

detectorproof/__init__.py

@@ -0,0 +1,101 @@
+"""detectorproof - what is a synthetic-speech detector's score reacting to?
+
+This is **not** a synthetic-speech detector, and it must not become one. It is an
+instrument for measuring detectors: what a declared panel of frozen checkpoints
+can and cannot do on your material, and how stable those judgments are under
+ordinary benign processing.
+
+`panel` is what is built. It scores a declared set of frozen detectors over a
+labelled control set, reports the area under the ROC curve for each, and refuses
+to let an incompetent one contribute to anything downstream. It is the gate the
+rest of the instrument depends on, which is why it exists first and alone.
+
+Five refusals are enforced in code rather than described in documentation. Each
+was bought by a specific measured failure:
+
+    a detector at or near chance is NOT COMPETENT ON THIS MATERIAL, reported in
+    full and excluded from every aggregate
+
+    the panel is declared before the run - an undeclared detector in the scores
+    is a fault, and a declared detector with no scores stays in the report as
+    NOT MEASURED
+
+    two scores for one detector and item are a fault, not something to average;
+    overlapping analysis windows are not independent samples
+
+    NOT MEASURED is a third state, never zero and never an absence of effect
+
+    a near miss at the edge of the chance band is recorded as a near miss, not
+    resolved by widening the band after seeing the number
+
+There is deliberately no function that recommends a transformation to move a
+score. In the work this package came out of, an intervention tuned against a
+chosen detector statistic made the audio *more* separable, not less - one
+detector's AUC moved from 0.576 to 0.763. The instrument reports; it does not
+advise, and that refusal is the point rather than an omission.
+
+No audio is read and no model is loaded. Everything here works on a score table
+a study already has, with no dependencies, so it runs on a machine that will
+never hold the corpus.
+"""
+from .domain import (
+    BONAFIDE,
+    HIGHER_BONAFIDE,
+    HIGHER_SYNTHETIC,
+    NOT_MEASURED,
+    ORIENTATION_UNRESOLVED,
+    SYNTHETIC,
+    DomainError,
+    ScoreRow,
+    score_is_measured,
+    validate_row,
+    validate_rows,
+)
+from .panel import (
+    BAND_EDGE_MARGIN,
+    CHANCE_BAND,
+    COMPETENT,
+    NOT_COMPETENT,
+    NOT_MEASURED_VERDICT,
+    UNRESOLVED_ORIENTATION,
+    Detector,
+    DetectorResult,
+    PanelFault,
+    PanelReport,
+    run_panel,
+)
+from .report import render
+from .stats import auc, holm, holm_adjusted, midranks
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "BAND_EDGE_MARGIN",
+    "BONAFIDE",
+    "CHANCE_BAND",
+    "COMPETENT",
+    "Detector",
+    "DetectorResult",
+    "DomainError",
+    "HIGHER_BONAFIDE",
+    "HIGHER_SYNTHETIC",
+    "NOT_COMPETENT",
+    "NOT_MEASURED",
+    "NOT_MEASURED_VERDICT",
+    "ORIENTATION_UNRESOLVED",
+    "PanelFault",
+    "PanelReport",
+    "SYNTHETIC",
+    "ScoreRow",
+    "UNRESOLVED_ORIENTATION",
+    "__version__",
+    "auc",
+    "holm",
+    "holm_adjusted",
+    "midranks",
+    "render",
+    "run_panel",
+    "score_is_measured",
+    "validate_row",
+    "validate_rows",
+]

detectorproof/cli.py

@@ -0,0 +1,153 @@
+"""Command line: `detectorproof panel`.
+
+One subcommand, because one thing is built. The instrument's own build order
+puts `panel` first and says not to build all four at once: `panel` is the gate
+the other three depend on, and it has to refuse correctly on a detector already
+known to be incompetent before anything is built on top of it.
+
+Input is three files a study already has:
+
+    --panel     JSON: the declared detectors, fixed before the run
+    --scores    CSV:  detector,item,condition,score
+    --labels    CSV:  item,label   (label is bonafide or synthetic)
+
+No audio is read and no model is loaded. This runs on a machine that will never
+hold the corpus.
+"""
+from __future__ import annotations
+
+import argparse
+import csv
+import json
+import sys
+from pathlib import Path
+
+from .domain import DomainError
+from .panel import CHANCE_BAND, Detector, PanelFault, run_panel
+from .report import render
+
+
+def _read_csv(path: Path) -> list[dict[str, str]]:
+    with path.open(newline="", encoding="utf-8-sig") as handle:
+        return list(csv.DictReader(handle))
+
+
+def label_rows_of(path: Path) -> list[dict[str, str]]:
+    """Every label row, in file order, with duplicates preserved.
+
+    Deliberately returns rows rather than a mapping. A dict built here loses
+    duplicate items before anything can object to them, which is exactly how a
+    conflicting label became a silent last-write-wins.
+    """
+    return _read_csv(path)
+
+
+def _load_panel(path: Path) -> list[Detector]:
+    text = path.read_text(encoding="utf-8")
+    try:
+        raw = json.loads(text)
+    except json.JSONDecodeError as exc:
+        # A malformed declaration is bad user input, not a bug in this package,
+        # so it takes the documented refusal path and exits 2 rather than
+        # escaping as a traceback and exiting 1. The position is reported
+        # because "invalid JSON" in a hand-written file is unactionable without
+        # it.
+        raise PanelFault(
+            f"the panel declaration at {path} is not valid JSON: {exc.msg} "
+            f"(line {exc.lineno}, column {exc.colno})."
+        ) from None
+    if isinstance(raw, dict):
+        raw = raw.get("detectors", raw)
+    if not isinstance(raw, list):
+        raise PanelFault(
+            "the panel declaration must be a JSON list of detectors, or an object with "
+            "a 'detectors' list. The declaration is the pre-registration of this run."
+        )
+    out = []
+    for entry in raw:
+        if not isinstance(entry, dict):
+            raise PanelFault("each declared detector must be a JSON object.")
+        out.append(
+            Detector(
+                name=entry.get("name", ""),
+                orientation=entry.get("orientation", ""),
+                identity=entry.get("identity", ""),
+                source=entry.get("source", ""),
+            )
+        )
+    return out
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="detectorproof",
+        description=(
+            "Report what a declared panel of frozen detectors can and cannot do on "
+            "your control set. This is not a detector and it never recommends a "
+            "transformation."
+        ),
+    )
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    p = sub.add_parser("panel", help="measure the detectors before trusting any of them")
+    p.add_argument("--panel", required=True, type=Path, help="JSON declaration of the panel")
+    p.add_argument("--scores", required=True, type=Path, help="CSV of detector scores")
+    p.add_argument("--labels", required=True, type=Path, help="CSV of item,label")
+    p.add_argument(
+        "--condition",
+        required=True,
+        help="which condition in the score table is the labelled control set",
+    )
+    p.add_argument(
+        "--chance-band",
+        nargs=2,
+        type=float,
+        metavar=("LOW", "HIGH"),
+        default=list(CHANCE_BAND),
+        help=(
+            "the AUC band inside which a detector is not competent on this material. "
+            "Declared before the run; not adjusted afterwards."
+        ),
+    )
+
+    args = parser.parse_args(argv)
+
+    try:
+        detectors = _load_panel(args.panel)
+        scores = _read_csv(args.scores)
+        # Passed as PAIRS, not collapsed into a dict. Building a dict here was
+        # half of the blocking defect: a duplicate item silently kept whichever
+        # row was read last, so reversing two conflicting rows could flip the
+        # verdict. `run_panel` canonicalises and refuses collisions itself.
+        label_pairs = [(row["item"], row["label"]) for row in label_rows_of(args.labels)]
+        report = run_panel(
+            detectors,
+            scores,
+            label_pairs,
+            condition=args.condition,
+            chance_band=(args.chance_band[0], args.chance_band[1]),
+        )
+    except (DomainError, PanelFault) as exc:
+        print(f"detectorproof: {exc}", file=sys.stderr)
+        return 2
+    except KeyError as exc:
+        print(
+            f"detectorproof: the labels file needs an 'item' and a 'label' column; "
+            f"missing {exc}",
+            file=sys.stderr,
+        )
+        return 2
+    except OSError as exc:
+        # A missing or unreadable input file is user input, not a defect here.
+        print(f"detectorproof: cannot read {exc.filename or 'an input file'}: {exc.strerror}",
+              file=sys.stderr)
+        return 2
+
+    print(render(report))
+    # A run that produced no competent detector is not a crash, but it is not a
+    # success either: nothing downstream may run on an empty competent set.
+    return 0 if report.competent_detectors() else 1
+
+
+if __name__ == "__main__":  # pragma: no cover
+    raise SystemExit(main())

detectorproof/domain.py

@@ -0,0 +1,314 @@
+"""Declared domains for every field this package reads.
+
+R19, and the reason this module exists at all:
+
+A checker that decides whether input is acceptable must **declare what each
+field may be** and test membership of that declaration. It must not carry a
+list of the bad values someone has seen. The two look identical on the day
+they are written and diverge immediately afterwards, because the list of bad
+values is unbounded and the domain is not. A field with no declared domain
+turns "is this present?" into the only question that can be asked of it, and
+then `""`, `0`, `-1`, `[]`, `"NaN"` and `None` all arrive in turn, each one a
+separate repair, none of them the cause.
+
+So every field below names its type, its admissible values, and what it means
+when it is absent. `validate_row` asks one question of each field - is this
+value inside the declared domain - and every rejection carries the domain it
+failed, not a description of the value.
+
+The third state matters as much as the domain. A score is one of:
+
+    a finite real number      the detector produced a value
+    NOT_MEASURED              the detector was not run on this item
+    (absent)                  a fault - the row does not describe what it claims
+
+`NOT_MEASURED` is never zero, never an empty string, and never silently
+dropped. It propagates to the report as itself, because a panel that reports a
+missing measurement as an absence of effect is worse than one that reports
+nothing.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Iterable
+
+
+# The sentinel. It is a singleton object rather than a string so that no CSV
+# cell, however malformed, can ever be mistaken for it, and so that arithmetic
+# on it raises instead of silently producing a number.
+class _NotMeasured:
+    __slots__ = ()
+
+    def __repr__(self) -> str:  # pragma: no cover - trivial
+        return "NOT_MEASURED"
+
+    def __bool__(self) -> bool:
+        raise TypeError(
+            "NOT_MEASURED has no truth value. A missing measurement is not "
+            "False, not zero and not absent - handle it explicitly."
+        )
+
+
+NOT_MEASURED = _NotMeasured()
+
+# The literal spellings accepted in a table for the sentinel. This is a parsing
+# convenience, not a domain: the domain of `score` is "finite real, or the
+# sentinel", and these strings are how the sentinel is written in a CSV.
+NOT_MEASURED_SPELLINGS = frozenset({"not_measured", "not measured", "n/a", "na", "nm"})
+
+# Identifiers are the join keys of the whole package: detector, item, condition
+# and speaker. An identifier that differs only by surrounding whitespace or by
+# case is the classic silent-mismatch fault, so the domain is deliberately
+# narrow and normalisation happens once, here, at the boundary.
+_ID_MAX = 200
+
+
+@dataclass(frozen=True)
+class Identifier:
+    """A non-empty label used as a join key.
+
+    Admissible: a string of 1..200 characters after stripping, containing no
+    control characters. Case and surrounding whitespace are normalised, because
+    `Pellav2 ` and `pellav2` naming the same detector in two rows is a fault
+    that produces two detectors and no error.
+    """
+
+    field: str
+
+    description = (
+        "non-empty text, at most 200 characters, no control characters; "
+        "compared case-insensitively after stripping"
+    )
+
+    def parse(self, value: Any) -> str:
+        if value is None:
+            raise DomainError(self.field, value, self.description, "absent")
+        if not isinstance(value, str):
+            raise DomainError(self.field, value, self.description, "not text")
+        text = value.strip()
+        if not text:
+            raise DomainError(self.field, value, self.description, "empty after stripping")
+        if len(text) > _ID_MAX:
+            raise DomainError(self.field, value, self.description, f"longer than {_ID_MAX}")
+        if any(ord(ch) < 0x20 or ord(ch) == 0x7F for ch in text):
+            raise DomainError(self.field, value, self.description, "contains a control character")
+        return text
+
+
+@dataclass(frozen=True)
+class Score:
+    """A detector's native output for one item, or the explicit sentinel.
+
+    Admissible: any finite real number, on the detector's own scale, or
+    NOT_MEASURED. Deliberately unbounded - these are native units from thirteen
+    models on mutually incomparable scales, and a package that imposed a range
+    would be inventing one. Non-finite values are rejected rather than coerced:
+    an inf or a NaN in a score column is a fault upstream, and replacing it with
+    zero would put a fabricated measurement into an average.
+    """
+
+    field: str = "score"
+
+    description = (
+        "a finite real number in ASCII decimal notation, in the detector's native "
+        "units, or NOT_MEASURED"
+    )
+
+    def parse(self, value: Any) -> float | _NotMeasured:
+        if value is NOT_MEASURED:
+            return NOT_MEASURED
+        if value is None:
+            raise DomainError(self.field, value, self.description, "absent")
+        if isinstance(value, str):
+            text = value.strip()
+            if text.lower() in NOT_MEASURED_SPELLINGS:
+                return NOT_MEASURED
+            if not text:
+                raise DomainError(self.field, value, self.description, "empty")
+            # The domain is narrowed to ASCII decimal deliberately. Python's
+            # float() accepts any Unicode decimal digit, so "٤" (Arabic-Indic
+            # four) becomes 4.0 without complaint. A score column carrying digits
+            # from another numeral system is far more likely to be a corrupted
+            # export than a deliberate measurement, and a package that silently
+            # converts it has invented a number nobody wrote. Found by the
+            # property fuzz in tests/test_fuzz.py, not by an auditor.
+            if not text.isascii():
+                raise DomainError(
+                    self.field, value, self.description, "not ASCII decimal notation"
+                )
+            try:
+                number = float(text)
+            except ValueError:
+                raise DomainError(self.field, value, self.description, "not a number") from None
+        elif isinstance(value, bool):
+            # bool is a subclass of int. A boolean in a score column means the
+            # upstream table has a label where a measurement should be.
+            raise DomainError(self.field, value, self.description, "boolean, not a measurement")
+        elif isinstance(value, (int, float)):
+            number = float(value)
+        else:
+            raise DomainError(self.field, value, self.description, "not a number")
+        if number != number:  # NaN
+            raise DomainError(self.field, value, self.description, "NaN")
+        if number in (float("inf"), float("-inf")):
+            raise DomainError(self.field, value, self.description, "not finite")
+        return number
+
+
+@dataclass(frozen=True)
+class Enum:
+    """One of a declared, closed set of spellings."""
+
+    field: str
+    allowed: frozenset[str]
+
+    @property
+    def description(self) -> str:
+        return "one of: " + ", ".join(sorted(self.allowed))
+
+    def parse(self, value: Any) -> str:
+        if value is None:
+            raise DomainError(self.field, value, self.description, "absent")
+        if not isinstance(value, str):
+            raise DomainError(self.field, value, self.description, "not text")
+        text = value.strip().lower()
+        if text not in self.allowed:
+            raise DomainError(self.field, value, self.description, "outside the declared set")
+        return text
+
+
+class DomainError(ValueError):
+    """A value outside its field's declared domain.
+
+    The message names the domain that was violated, never a catalogue of values
+    that have been seen before. That is the whole point of R19: the next bad
+    value is one nobody has written down yet, and it has to fail for the same
+    reason as the last one.
+    """
+
+    def __init__(self, field: str, value: Any, description: str, why: str) -> None:
+        self.field = field
+        self.value = value
+        self.description = description
+        self.why = why
+        shown = repr(value)
+        if len(shown) > 80:
+            shown = shown[:77] + "..."
+        super().__init__(
+            f"{field}={shown} is outside its declared domain ({why}). "
+            f"{field} must be: {description}."
+        )
+
+
+# ---------------------------------------------------------------------------
+# Orientation: which direction of a detector's native score means "synthetic".
+#
+# This is the field the estate's own protocol is most emphatic about. It is
+# resolved from a model card or a training-label path and never from whether it
+# makes a benchmark look right, so the package treats it as declared input, not
+# as something to infer. "unresolved" is a first-class value: a detector whose
+# documentation does not settle the direction is scored, reported, and its SIGN
+# is not interpreted.
+# ---------------------------------------------------------------------------
+HIGHER_BONAFIDE = "higher_bonafide"
+HIGHER_SYNTHETIC = "higher_synthetic"
+ORIENTATION_UNRESOLVED = "unresolved"
+
+ORIENTATION = Enum(
+    "orientation",
+    frozenset({HIGHER_BONAFIDE, HIGHER_SYNTHETIC, ORIENTATION_UNRESOLVED}),
+)
+
+# Labels for the control set. `bonafide` is genuine human speech; `synthetic`
+# is generated. Anything else in a label column is a fault: a control set with
+# a third class is not the two-class problem AUC describes.
+BONAFIDE = "bonafide"
+SYNTHETIC = "synthetic"
+LABEL = Enum("label", frozenset({BONAFIDE, SYNTHETIC}))
+
+DETECTOR = Identifier("detector")
+ITEM = Identifier("item")
+CONDITION = Identifier("condition")
+SCORE = Score()
+
+
+@dataclass(frozen=True)
+class ScoreRow:
+    """One detector's measurement of one item under one condition."""
+
+    detector: str
+    item: str
+    condition: str
+    score: float | _NotMeasured
+
+    @property
+    def measured(self) -> bool:
+        return score_is_measured(self.score)
+
+
+def score_is_measured(value: float | _NotMeasured) -> bool:
+    """True when a score is a real measurement rather than the sentinel.
+
+    Written as a function rather than a truth test because `NOT_MEASURED`
+    deliberately raises on `bool()`, so `if row.score:` fails loudly instead of
+    quietly treating a missing measurement as zero.
+    """
+    return not isinstance(value, _NotMeasured)
+
+
+def validate_row(row: dict[str, Any]) -> ScoreRow:
+    """Parse one score row against the declared domains.
+
+    Every field is checked against its own domain and the first violation is
+    raised with that domain attached. Unknown extra keys are ignored: a study's
+    table usually carries columns this package has no opinion about, and
+    rejecting them would make the package harder to adopt without making any
+    measurement safer.
+    """
+    return ScoreRow(
+        detector=DETECTOR.parse(row.get("detector")),
+        item=ITEM.parse(row.get("item")),
+        condition=CONDITION.parse(row.get("condition")),
+        score=SCORE.parse(row.get("score", None)),
+    )
+
+
+def validate_rows(rows: Iterable[dict[str, Any]]) -> list[ScoreRow]:
+    """Parse a table, reporting the row index of the first fault.
+
+    Row numbers are 1-based and count the data rows a caller passed, not file
+    lines, because the caller may have read the table from anywhere.
+    """
+    out: list[ScoreRow] = []
+    for index, raw in enumerate(rows, start=1):
+        try:
+            out.append(validate_row(raw))
+        except DomainError as exc:
+            raise DomainError(
+                exc.field, exc.value, exc.description, f"row {index}: {exc.why}"
+            ) from None
+    return out
+
+
+__all__ = [
+    "BONAFIDE",
+    "CONDITION",
+    "DETECTOR",
+    "DomainError",
+    "Enum",
+    "HIGHER_BONAFIDE",
+    "HIGHER_SYNTHETIC",
+    "ITEM",
+    "Identifier",
+    "LABEL",
+    "NOT_MEASURED",
+    "ORIENTATION",
+    "ORIENTATION_UNRESOLVED",
+    "SCORE",
+    "SYNTHETIC",
+    "Score",
+    "ScoreRow",
+    "score_is_measured",
+    "validate_row",
+    "validate_rows",
+]