evalrx 0.1.2__py3-none-any.whl

evalrx/contract/m2.py

@@ -0,0 +1,317 @@
+"""M2 — analysis: join measurements to outcomes, test which signals track failure.
+
+M2 answers *what correlates*, never *why*. That boundary is load-bearing: an
+in-sample association rendered with the same language as a corrected verdict is
+the single most damaging mistake this pipeline can make, so the descriptive and
+confirmatory states are distinguishable here at the type level
+(:attr:`StatsReportWire.descriptive_only`), not by convention.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Literal
+
+from pydantic import Field, field_validator, model_validator
+
+from evalrx.contract.common import (
+    ArtifactRef,
+    CaseBatchRef,
+    ExternalRef,
+    JoinReport,
+    StageEnvelope,
+    WireModel,
+)
+from evalrx.contract.m1 import ProtocolWire
+
+# ---------------------------------------------------------------------------
+# Input
+# ---------------------------------------------------------------------------
+
+class AnalysisInput(WireModel):
+    """M2 needs BOTH halves of the join: M1's numbers and the cases' labels.
+
+    ``data`` is not optional in practice — without labels there is nothing to
+    correlate against and M2 degrades to threshold rules only.
+    """
+
+    results_ref: str = Field(description="Path to the M1 ProbeOutput this analyzes.")
+    data: CaseBatchRef
+    model_name: str
+    protocol: ProtocolWire | None = None
+    confirmatory: bool = Field(
+        default=True,
+        description="False defers the e-BH validity verdict to the confirm phase; "
+                    "the report then carries effect sizes and charts but NO reject decision.",
+    )
+
+
+# ---------------------------------------------------------------------------
+# Output
+# ---------------------------------------------------------------------------
+
+class AnalysisFindingWire(WireModel):
+    """One threshold-rule violation. The no-judge baseline signal."""
+
+    analyzer: str
+    metric: str
+    value: float
+    threshold: float
+    direction: Literal["above", "below"]
+    severity: Literal["high", "medium", "low"]
+    message: str
+
+
+class StatsToolResultWire(WireModel):
+    """One statistical test's verdict.
+
+    ``reject`` is the RAW, uncorrected decision and must never drive a verdict on
+    its own. ``fdr_corrected`` is the family-level decision after multiplicity
+    control and is the only field a downstream verdict may read. They are kept as
+    separate fields precisely because collapsing them once let a p=0.14 result on
+    n=3 "support" a hypothesis.
+    """
+
+    tool: str = Field(description="e.g. signal_label_assoc / mcnemar_evalue / bootstrap_diff / rank_corr")
+    measured: str | None = Field(
+        default=None,
+        description="WHAT this result is about, in words, and unique within the report — "
+                    "the label a chart axis or a table row should carry.\n\n"
+                    "`tool` is the procedure, not the subject, and using it as a label "
+                    "put two rows reading 'Mcnemar evalue' on a chart with no way to "
+                    "tell them apart. `config['signal']` is the subject but is a machine "
+                    "name, and is absent on paired tools entirely. Neither can label a "
+                    "row on its own, so the producer states the label here.",
+    )
+    means: str | None = Field(
+        default=None,
+        description="What the measured quantity IS, in one sentence, taken from the "
+                    "producing analyzer's `signal_docs`. Null means the analyzer did not "
+                    "document that metric — render it as undocumented rather than "
+                    "inventing a gloss, because a plausible wrong explanation of a "
+                    "statistic is worse than an admitted missing one.",
+    )
+    config: dict[str, Any] = Field(default_factory=dict, description="Includes 'signal' — the routing key.")
+    ok: bool
+    error: str | None = None
+
+    effect: float | None = None
+    ci: tuple[float, float] | None = None
+    p_value: float | None = None
+    e_value: float | None = None
+    underpowered: bool = False
+
+    reject: bool = Field(default=False, description="RAW per-test decision. Not a verdict.")
+    fdr_corrected: bool | None = Field(
+        default=None,
+        description="Family-level decision after correction. None = correction not yet run "
+                    "(deferred descriptive phase) — which is NOT the same as False.",
+    )
+    correction_method: str | None = Field(default=None, description="e.g. 'bh' / 'ebh'.")
+    correction_family: str | None = None
+    analysis_key: str | None = Field(
+        default=None, description="Stable per-result key, e.g. 'signal_label_assoc:attention.entropy'. "
+                                  "Required for correction: one tool NAME covers many tests."
+    )
+
+    raw_reject: bool | None = Field(
+        default=None, description="Pre-correction decision, kept for audit alongside `reject`."
+    )
+    figure_path: str | None = None
+
+    n_signal: int | None = Field(default=None, description="Cases in the signal-present group.")
+    n_control: int | None = Field(default=None, description="Cases in the signal-absent group.")
+    n_measured: int | None = Field(
+        default=None,
+        description="Cases the producing analyzer actually measured. Analyzers measure "
+                    "every case unless capped (an analyzer's max_cases or the probe "
+                    "agent's max_cases_per_analyzer), so compare this with the batch size.",
+    )
+    n_imputed_absent: int | None = Field(
+        default=None,
+        description="Labeled cases with no measurement that were counted as signal-ABSENT. "
+                    "Valid for a genuinely sparse flag that only lists where it fired; invalid "
+                    "when the analyzer simply never looked. Nothing in the value distinguishes "
+                    "the two, so the count must travel with the result.",
+    )
+
+    summary: str = ""
+    details: dict[str, Any] = Field(
+        default_factory=dict, description="fail rates / permutation_p / CI internals / ..."
+    )
+
+    @property
+    def imputation_share(self) -> float | None:
+        """Share of the tested sample that was never measured.
+
+        Observed live (bbh_tracking7): ``step_rollout_value.recoverable`` was
+        measured on 8 cases, tested on 125, and survived e-BH — 94% of its
+        control group is imputed absence. A reader cannot see that from the
+        effect size, the CI, or the corrected flag.
+        """
+        total = (self.n_signal or 0) + (self.n_control or 0)
+        if not total or self.n_imputed_absent is None:
+            return None
+        return self.n_imputed_absent / total
+
+    def is_mostly_imputed(self, threshold: float = 0.5) -> bool:
+        share = self.imputation_share
+        return share is not None and share > threshold
+
+    def is_decisive(self, report_survivors: set[str] | None = None) -> bool:
+        """Whether this result may decide a hypothesis's direction.
+
+        Mirrors the M4 gate: raw rejection is necessary but never sufficient.
+        """
+        if not self.reject:
+            return False
+        if self.correction_method:
+            return bool(self.fdr_corrected)
+        if self.e_value is not None:
+            return bool(self.analysis_key) and self.analysis_key in (report_survivors or set())
+        return True
+
+
+#: Legacy spellings this contract shipped before it was checked against the
+#: producer. ``evalrx.stats.multiplicity`` writes the hyphenated names and
+#: has a fourth value the contract had no member for.
+_LEGACY_CORRECTION_METHODS = {"ebh": "e-BH", "bh": "BH"}
+
+
+class CorrectedRejections(WireModel):
+    """Family-level multiplicity control across every tested signal.
+
+    Testing 40 signals at alpha=0.05 yields ~2 "significant" results from luck
+    alone. Without this the screen is not a screen.
+
+    ``mixed-BH/e-BH`` is its own member rather than being folded into either
+    neighbour: it says the family contained both p-values and e-values, which
+    carry different validity guarantees, and a reader weighing the verdict needs
+    to know that the guarantee is the weaker of the two.
+    """
+
+    method: Literal["e-BH", "BH", "mixed-BH/e-BH", "none"] = "none"
+    alpha: float = 0.05
+    deferred: bool = Field(
+        default=False, description="True while the analysis phase withholds the verdict."
+    )
+    n_tested: int = 0
+    rejected_result_keys: list[str] = Field(
+        default_factory=list, description="analysis_key values that survived. The source of truth."
+    )
+
+    @field_validator("method", mode="before")
+    @classmethod
+    def _normalise_method(cls, v: Any) -> Any:
+        if isinstance(v, str):
+            return _LEGACY_CORRECTION_METHODS.get(v.lower(), v)
+        return v
+
+
+class StatsReportWire(StageEnvelope):
+    """M2's serialized report.
+
+    Two field groups with different authority: everything above
+    ``descriptive_only`` is computed by code and is authoritative; ``conclusion``
+    and ``evidence_chain`` are written by the judge and are presentation only.
+    Empty narrative is valid and must never be backfilled with an invented
+    summary. The LLM may not write a number, and keeping the groups visibly
+    apart is what makes that reviewable.
+
+    Absent by design: the judge's prompt and raw response (already
+    ``prompts/c<cycle>_m2_analysis.{prompt,response}.txt``), the tool-selection
+    plan (already ``artifacts/c<cycle>_m2_stats_plan.json``), the model name and
+    protocol (both in the run config and in :class:`AnalysisInput`), and the
+    threshold-rule narrative (a rendering of ``findings``).
+    """
+
+    findings: list[AnalysisFindingWire] = Field(default_factory=list)
+    stats_tool: Literal["threshold_rules", "llm_guided", "selected_tools", "generated"] = "threshold_rules"
+    stats_results: list[StatsToolResultWire] | ExternalRef = Field(default_factory=list)
+    corrected_rejections: CorrectedRejections | ExternalRef = Field(default_factory=CorrectedRejections)
+    figures: list[ArtifactRef] = Field(default_factory=list)
+    descriptive_only: bool = Field(
+        default=False,
+        description="True = effect sizes and charts only, e-BH deferred. A reader MUST NOT "
+                    "render supported/rejected language while this is true.",
+    )
+
+    conclusion: str = Field(default="", description="Judge-written. Presentation only.")
+    evidence_chain: list[str] = Field(default_factory=list, description="Judge-written derivation.")
+    llm_fallback_reason: str = Field(
+        default="",
+        description="Non-empty only when the judge path was attempted AND raised. Not recoverable "
+                    "from the prompt files: a fallback still writes both of them.",
+    )
+
+    raw_results_ref: str | None = Field(
+        default=None,
+        description="Path to the M1 ProbeOutput. The in-memory report carries the Result objects "
+                    "themselves, but to_dict() drops them — M4/M5 still need per-case signals, so "
+                    "the serialized contract carries a reference instead of pretending they survive.",
+    )
+    joins: list[JoinReport] = Field(
+        default_factory=list, description="Signal<->label join health. Zero coverage is a plumbing failure."
+    )
+
+    @property
+    def severity(self) -> Literal["high", "medium", "low", "none"]:
+        """Worst flagged finding. Derived — a stored copy can contradict the list."""
+        for level in ("high", "medium", "low"):
+            if any(f.severity == level for f in self.findings):
+                return level  # type: ignore[return-value]
+        return "none"
+
+    @model_validator(mode="after")
+    def _measured_labels_are_distinct(self) -> "StatsReportWire":
+        """Two rows labelled the same are two rows a reader cannot tell apart.
+
+        Deduplicated rather than rejected: a colliding label is a presentation
+        fault, and dropping the whole M2 payload over one would lose the
+        statistics too. The suffix is the tool, then an ordinal — enough to
+        separate them without inventing meaning.
+        """
+        rows = self.stats_results
+        if not isinstance(rows, list):
+            return self
+        seen: dict[str, int] = {}
+        for r in rows:
+            if not r.measured:
+                continue
+            if r.measured in seen:
+                seen[r.measured] += 1
+                r.measured = f"{r.measured} ({r.tool}, {seen[r.measured]})"
+            else:
+                seen[r.measured] = 1
+        return self
+
+    @model_validator(mode="after")
+    def _descriptive_has_no_verdict(self) -> "StatsReportWire":
+        corr = self.corrected_rejections
+        if self.descriptive_only and isinstance(corr, CorrectedRejections):
+            if corr.rejected_result_keys and not corr.deferred:
+                raise ValueError(
+                    "descriptive_only=True but corrected_rejections carries survivors; "
+                    "a descriptive report must not ship a validity verdict"
+                )
+        return self
+
+
+class ExploreContextWire(WireModel):
+    """Descriptive EDA notes from the optional explore side-path.
+
+    Never authoritative. It enters the M3 prompt and the dashboard and nothing
+    else — not M2's tested family, not M4, not the fix gate. It shapes WHICH
+    hypotheses get proposed, never WHETHER one is true.
+    """
+
+    observations: list[str] = Field(default_factory=list)
+    charts: list[dict[str, Any]] = Field(default_factory=list)
+    caveats: list[str] = Field(default_factory=list)
+    source: str = "lambda_explorer"
+    authoritative: Literal[False] = False
+
+
+__all__ = [
+    "AnalysisInput", "AnalysisFindingWire", "StatsToolResultWire",
+    "CorrectedRejections", "StatsReportWire", "ExploreContextWire",
+]

evalrx/contract/m3.py

@@ -0,0 +1,165 @@
+"""M3 — diagnosis: turn correlations into falsifiable mechanism claims.
+
+This is the one stage where the LLM produces the payload rather than the prose.
+That is exactly why its output must be machine-checkable: a hypothesis carries a
+``test_design`` naming the signal that would confirm or refute it, so M4 can
+route deterministically instead of re-reading the sentence.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+
+from pydantic import Field, field_validator
+
+from evalrx.contract.common import HypothesisStatus, StageEnvelope, WireModel
+from evalrx.contract.m2 import ExploreContextWire
+
+# ---------------------------------------------------------------------------
+# Input
+# ---------------------------------------------------------------------------
+
+class DiagnosisInput(WireModel):
+    """M3 reads M2's report — both the numbers and the narrative."""
+
+    analysis_ref: str = Field(description="Path to the M2 StatsReportWire.")
+    prior_cycles: list[dict[str, Any]] = Field(
+        default_factory=list, description="Summaries of earlier cycles, so the judge avoids re-proposing."
+    )
+    explore_context: ExploreContextWire | None = None
+    failure_modes: list[str] = Field(
+        default_factory=list, description="Descriptive-only clustering tags."
+    )
+
+
+# ---------------------------------------------------------------------------
+# Output
+# ---------------------------------------------------------------------------
+
+#: A ``<analyzer>.<metric>`` reference — the form M4 resolves against M2's signals.
+_SIGNAL_REF = re.compile(r"\b[a-z_][a-z0-9_]*\.[a-z_][a-z0-9_]*\b")
+
+#: Non-signal test designs M4 knows how to route. Extend this together with the
+#: router; a directive listed here and unhandled there is worse than a rejection.
+TEST_DESIGN_DIRECTIVES = frozenset({
+    "prompt_contrast",
+    "analyzer_params",
+    "strategy_contrast",
+    "paired_rerun",
+})
+
+
+class HypothesisWire(WireModel):
+    """One falsifiable claim about *why* the model fails.
+
+    ``statement`` is the mechanism; ``test_design`` is the machine-readable
+    commitment to how it could be wrong. A hypothesis without the latter cannot
+    be tested and is therefore not a hypothesis — M4 has nothing to route on and
+    M1 has nothing to re-probe.
+    """
+
+    id: str = Field(min_length=1, description="Stable id. Never slugify the statement and assume uniqueness.")
+    statement: str = Field(min_length=10, description="Technical mechanism claim.")
+    plain_statement: str = Field(
+        default="",
+        description="The SAME claim in one everyday sentence, for a reader who runs "
+                    "evaluations and does no statistics. Not a second, softer claim — a "
+                    "second rendering of this one, checked host-side against a jargon "
+                    "list before it is accepted (see analysis.plain_language). Empty "
+                    "when the producer wrote none; a consumer should then show "
+                    "`statement` rather than paraphrase it into something the run "
+                    "never said.",
+    )
+    target_model: str
+    predicted_failure_mode: str = Field(min_length=2)
+    test_design: str = Field(
+        default="",
+        description="Signal or contrast that would decide this, e.g. 'attention.image_token_ratio' "
+                    "or 'prompt_contrast describe_first'. M4 routes on it; M1 re-probes on it. "
+                    "EMPTY means the judge proposed none — see `is_routable`.",
+    )
+    status: HypothesisStatus = HypothesisStatus.PROPOSED
+    parent_id: str | None = Field(default=None, description="Set when mutated from an earlier hypothesis.")
+    metadata: dict[str, Any] = Field(default_factory=dict)
+
+    @property
+    def is_routable(self) -> bool:
+        """Whether M4 can resolve this design against measured signals.
+
+        Mirrors what ``hypothesis_tester`` actually extracts — an identifier
+        anywhere in the text, or one of the directives — rather than demanding a
+        shape. The strict version required the string to BE a bare
+        ``<analyzer>.<metric>`` or to START with a directive, and a strong judge
+        does not write like that: on the Music-AVQA run, Opus at high effort
+        produced three designs that named their analyzers and metrics inside a
+        paragraph of interventional protocol. M4 routed all three
+        (``routed_by="test_design"``, two at INTERVENTION grade) and the contract
+        rejected the whole M3 payload for the formatting.
+
+        A validator stricter than the consumer it protects does not prevent
+        anything; it just discards good work. So routability is now REPORTED, and
+        the three states stay distinguishable: empty (no test proposed),
+        non-empty but nothing resolvable (a human can act on it, M4 cannot), and
+        routable.
+        """
+        text = self.test_design.strip()
+        if not text:
+            return False
+        if _SIGNAL_REF.search(text.lower()):
+            return True
+        return any(d in text for d in TEST_DESIGN_DIRECTIVES)
+
+    @property
+    def is_proposed(self) -> bool:
+        """Whether the judge proposed a test at all, routable or not."""
+        return bool(self.test_design.strip())
+
+
+class DiagnosisOutput(StageEnvelope):
+    """M3's serialized output: the hypotheses, and nothing that is already on disk.
+
+    Zero hypotheses is a completed, empty result (state EMPTY or ABSTAINED) — not
+    an error. Rendering it red teaches readers to distrust a correct abstention.
+
+    Judge I/O is deliberately absent. M3 is the one stage an LLM writes the
+    payload for, so its prompt and verbatim response do have to be recoverable —
+    but they are already persisted per cycle as
+    ``prompts/c<cycle>_m3_diagnosis.{prompt,response}.txt``, and the findings the
+    judge was shown are the M2 report named by :attr:`DiagnosisInput.analysis_ref`.
+    Carrying either inline here would be a second copy that can disagree with the
+    first.
+    """
+
+    hypotheses: list[HypothesisWire] = Field(default_factory=list)
+
+    @property
+    def untestable(self) -> list[str]:
+        """Ids of hypotheses carrying no test_design at all.
+
+        These come back INCONCLUSIVE no matter how much evidence the next cycle
+        gathers, because nothing was ever named that could decide them.
+        """
+        return [h.id for h in self.hypotheses if not h.is_proposed]
+
+    @property
+    def unroutable(self) -> list[str]:
+        """Ids whose design a human can act on but M4 cannot resolve.
+
+        Distinct from :attr:`untestable`: a test WAS proposed, it just names no
+        measured signal, so it is work for the next M1 cycle rather than a claim
+        with no falsifier. Collapsing the two would report a real experimental
+        plan as an empty one.
+        """
+        return [h.id for h in self.hypotheses if h.is_proposed and not h.is_routable]
+
+    @field_validator("hypotheses")
+    @classmethod
+    def _ids_unique(cls, v: list[HypothesisWire]) -> list[HypothesisWire]:
+        ids = [h.id for h in v]
+        if len(set(ids)) != len(ids):
+            raise ValueError("duplicate hypothesis ids make M4 verdicts ambiguous")
+        return v
+
+
+__all__ = ["TEST_DESIGN_DIRECTIVES", "DiagnosisInput", "HypothesisWire", "DiagnosisOutput"]

evalrx/contract/m4.py

@@ -0,0 +1,130 @@
+"""M4 — hypothesis testing: adjudicate M3's claims against corrected statistics.
+
+M4 mostly does not compute. It routes a hypothesis's ``test_design`` to the M2
+results that bear on it, admits only results that survived multiplicity
+correction, reads the sign of the effect, and applies a second, independent gate:
+is this claim even about what the user asked to investigate.
+
+SUPPORTED requires BOTH gates. Either alone is a different, weaker statement.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Literal
+
+from pydantic import Field, model_validator
+
+from evalrx.contract.common import (
+    CaseBatchRef,
+    EvidenceGrade,
+    HypothesisStatus,
+    StageEnvelope,
+    WireModel,
+)
+from evalrx.contract.m1 import ProtocolWire
+
+# ---------------------------------------------------------------------------
+# Input
+# ---------------------------------------------------------------------------
+
+class HypothesisTestInput(WireModel):
+    """M4's inputs.
+
+    ``split`` is recorded because it changes what the verdict MEANS: testing on
+    the same cases the hypotheses were generated from is an internal consistency
+    check, not out-of-sample evidence. A reader cannot infer this from the numbers
+    and must not be left to guess.
+    """
+
+    hypotheses_ref: str = Field(description="Path to the M3 DiagnosisOutput.")
+    stats_report_ref: str = Field(description="Path to the M2 StatsReportWire (post-correction).")
+    data: CaseBatchRef
+    protocol: ProtocolWire | None = Field(
+        default=None, description="None => every hypothesis is assumed protocol-consistent."
+    )
+    split: Literal["explore", "confirm", "all"] = Field(
+        default="explore",
+        description="Which partition M4 actually ran on. 'explore' means the hypotheses were "
+                    "generated on these same cases — an in-sample adjudication, not held-out.",
+    )
+    alpha: float = 0.05
+
+
+# ---------------------------------------------------------------------------
+# Output
+# ---------------------------------------------------------------------------
+
+class TestEvidence(WireModel):
+    """The statistics behind one verdict, with its provenance."""
+
+    source: Literal["m2_stats_results", "fallback_per_case", "none"] = "none"
+    chosen_tool: str | None = None
+    routed_by: str = Field(default="", description="How test_design resolved to this result.")
+    consulted_tools: list[str] = Field(default_factory=list)
+    ci: tuple[float, float] | None = None
+    e_value: float | None = None
+    fdr_corrected: bool | None = Field(default=None, description="The field that actually decided this.")
+    underpowered: bool = False
+    n_signal: int | None = None
+    n_control: int | None = None
+    fail_rate_signal: float | None = None
+    fail_rate_control: float | None = None
+    details: dict[str, Any] = Field(default_factory=dict)
+
+
+class HypothesisTestResultWire(WireModel):
+    """One hypothesis's verdict.
+
+    ``effect_size`` is signal-group minus control-group fail rate, so its SIGN is
+    the verdict's direction: positive supports the claim, negative refutes it.
+    A refutation is a real, useful outcome — not a failure to conclude.
+    """
+
+    hypothesis_id: str = Field(min_length=1, description="Points into the M3 DiagnosisOutput.")
+    status: HypothesisStatus
+    test_name: str = Field(description="e.g. 'signal_label_assoc' / 'fail_rate_comparison'.")
+    effect_size: float | None = None
+    confidence: float = Field(ge=0.0, le=1.0, description="Geometric mean of evidence gap, "
+                                                          "sample adequacy, control cleanliness.")
+    evidence_grade: EvidenceGrade = EvidenceGrade.OBSERVATIONAL
+    is_consistent_with_protocol: bool
+    verdict: str = Field(description="One-line natural-language summary.")
+    evidence: TestEvidence = Field(default_factory=TestEvidence)
+
+    @model_validator(mode="after")
+    def _supported_needs_both_gates(self) -> "HypothesisTestResultWire":
+        # The whole point of the second gate: a statistically real claim about
+        # something the user never asked about is not a supported diagnosis.
+        if self.status == HypothesisStatus.SUPPORTED and not self.is_consistent_with_protocol:
+            raise ValueError(
+                "SUPPORTED requires both the statistical test AND protocol consistency; "
+                "mark it INCONCLUSIVE and record the inconsistency in `verdict`"
+            )
+        if self.status == HypothesisStatus.SUPPORTED and self.evidence_grade == EvidenceGrade.NONE:
+            raise ValueError("SUPPORTED with evidence_grade=none has nothing backing it")
+        return self
+
+
+class HypothesisTestOutput(StageEnvelope):
+    """M4's serialized output — one result per hypothesis, plus the loop gate."""
+
+    split: Literal["explore", "confirm", "all"] = "explore"
+    results: list[HypothesisTestResultWire] = Field(default_factory=list)
+    stopping_criteria_met: bool = Field(
+        default=False,
+        description="Whether the loop may stop on this cycle's evidence. Not simply "
+                    "`verified() != []` — the depth-tiered rule also weighs evidence grade.",
+    )
+
+    def verified(self) -> list[str]:
+        """Hypothesis ids M5 may operate on, strongest first.
+
+        Derived, never stored: a persisted copy can disagree with `results`, and
+        then nothing tells a reader which one the loop actually acted on.
+        """
+        ok = [r for r in self.results
+              if r.status == HypothesisStatus.SUPPORTED and r.is_consistent_with_protocol]
+        return [r.hypothesis_id for r in sorted(ok, key=lambda r: -r.confidence)]
+
+
+__all__ = ["HypothesisTestInput", "TestEvidence", "HypothesisTestResultWire", "HypothesisTestOutput"]