lcb-gate 0.1.0__tar.gz

lcb_gate-0.1.0/.gitattributes

@@ -0,0 +1 @@
+* text=auto eol=lf

lcb_gate-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,31 @@
+name: ci
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ["3.9", "3.13"]
+        include:
+          - os: windows-latest
+            python-version: "3.13"
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install
+        run: pip install -e ".[dev]"
+      - name: Test suite
+        run: pytest -q
+      - name: Zero-dependency check (tests run on bare python)
+        run: python tests/test_lcb.py
+      - name: Demo
+        run: python examples/demo.py

lcb_gate-0.1.0/.github/workflows/release.yml

@@ -0,0 +1,44 @@
+name: release
+
+on:
+  push:
+    tags: ["v*"]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+      - name: Test
+        run: |
+          pip install -e ".[dev]"
+          pytest -q
+      - name: Build sdist and wheel
+        run: |
+          pip install build twine
+          python -m build
+          twine check dist/*
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish:
+    needs: build
+    runs-on: ubuntu-latest
+    # Trusted Publishing (OIDC): configure this repo + workflow + environment
+    # as a (pending) publisher for the project on pypi.org — no API token.
+    environment:
+      name: pypi
+      url: https://pypi.org/p/lcb-gate
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+      - uses: pypa/gh-action-pypi-publish@release/v1

lcb_gate-0.1.0/.gitignore

@@ -0,0 +1,10 @@
+__pycache__/
+*.py[cod]
+*.egg-info/
+.pytest_cache/
+dist/
+build/
+.venv/
+venv/
+.idea/
+.vscode/

lcb_gate-0.1.0/CHANGELOG.md

@@ -0,0 +1,12 @@
+# Changelog
+
+## 0.1.0 — 2026-07-10
+
+Initial release.
+
+- `wilson_lcb` / `min_trials` — one-sided Wilson score lower bound and gate sizing.
+- `run_gate` — N-run pass/fail gating with settled-verdict early stopping.
+- `compare` — paired champion/candidate comparison (common random numbers,
+  ties count as half a win), verdict on the win-rate LCB vs 0.5.
+- pytest plugin: `lcb` fixture (`check`, `check_better`) via the `pytest11`
+  entry point.

lcb_gate-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Ciphemon
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

lcb_gate-0.1.0/PKG-INFO

@@ -0,0 +1,160 @@
+Metadata-Version: 2.4
+Name: lcb-gate
+Version: 0.1.0
+Summary: Statistical pass/fail gating for stochastic tests: run N times, pass only when the Wilson lower confidence bound clears the bar. Built for agent evals; stdlib only.
+Project-URL: Homepage, https://github.com/CiphemonJY/lcb-gate
+Project-URL: Repository, https://github.com/CiphemonJY/lcb-gate
+Project-URL: Issues, https://github.com/CiphemonJY/lcb-gate/issues
+Project-URL: Changelog, https://github.com/CiphemonJY/lcb-gate/blob/main/CHANGELOG.md
+Author: Ciphemon
+License-Expression: MIT
+License-File: LICENSE
+Keywords: agent-evals,confidence-interval,flaky-tests,llm,pytest,statistics,testing
+Classifier: Development Status :: 3 - Alpha
+Classifier: Framework :: Pytest
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Quality Assurance
+Requires-Python: >=3.9
+Provides-Extra: dev
+Requires-Dist: pytest>=7; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# lcb-gate
+
+[![ci](https://github.com/CiphemonJY/lcb-gate/actions/workflows/ci.yml/badge.svg)](https://github.com/CiphemonJY/lcb-gate/actions/workflows/ci.yml)
+
+**Statistical pass/fail gating for stochastic tests.** Run the trial N times;
+pass only when the **Wilson lower confidence bound** of the pass rate clears
+your threshold. Built for agent evals — anything where a single green run
+proves almost nothing. Stdlib only, no dependencies.
+
+```
+pip install lcb-gate
+```
+
+The demo ships in the repo (not the wheel):
+
+```
+git clone https://github.com/CiphemonJY/lcb-gate && cd lcb-gate
+python examples/demo.py     # why a single green run lies, in three acts
+```
+
+## The problem
+
+Agent behaviors, LLM evals, and flaky integration tests are *stochastic*. CI
+treats them as deterministic: one green run → merged. Two failure modes follow:
+
+- **The lucky pass.** A 75%-reliable agent task passes a single CI run 75% of
+  the time. You ship a coin flip and call it tested.
+- **The winner's curse.** You try 20 variants, one "beats the baseline" on a
+  single eval run, and you promote it. Most single-run wins are noise: in one
+  production self-improvement pipeline, fresh-seed re-verification killed
+  **85% of apparent improvements** found by a first-pass scan.
+
+Averages don't fix this — a point estimate without n is a vibe. What fixes it
+is a *lower confidence bound*: "with 95% confidence the true pass rate is at
+least X". Gate on X.
+
+## Usage
+
+### As a library
+
+```python
+from lcb_gate import run_gate
+
+result = run_gate(lambda i: run_my_agent(seed=i).succeeded, n=25, threshold=0.9)
+print(result)
+# PASS: 25/25 passed (gate n=25); LCB 0.902 >= 0.9 @ 95% confidence
+assert result.passed
+```
+
+`run_gate` stops early — in either direction — the moment the verdict is
+mathematically settled, so hopeless runs don't burn the full budget. The
+verdict is identical to running all n trials.
+
+### As a pytest fixture
+
+```python
+def test_agent_completes_checkout(lcb):
+    lcb.check(lambda i: run_agent_eval(seed=i).ok, n=25, threshold=0.9)
+```
+
+The plugin registers automatically on install. Failures explain themselves:
+
+```
+Failed: lcb-gate FAIL: 21/30 passed (gate n=30); LCB 0.551 < 0.9 @ 95% confidence
+```
+
+### Candidate vs champion (promotion gates)
+
+"Is the new prompt/model/policy actually better?" is the same statistics with
+a different threshold — the win-rate LCB must exceed 0.5:
+
+```python
+from lcb_gate import compare
+
+result = compare(candidate=eval_new, champion=eval_old, n=400)
+print(result)
+# BETTER: candidate won 243.0/400 paired trials (60.8%); win-rate LCB 0.567 > 0.5 @ 95%
+```
+
+Both callables receive the **same seed per trial** (common random numbers):
+pairing removes the noise both variants share, which is where most of the
+statistical power comes from. Ties count as half a win. The honest default
+verdict is `NOT PROVEN` — an underpowered comparison never certifies.
+
+```python
+def test_new_prompt_beats_production(lcb):
+    lcb.check_better(eval_new, eval_old, n=400)
+```
+
+## Sizing your gate
+
+A perfect record at small n still can't clear a high bar — by design. Minimum
+trials for a *flawless* run to pass, at 95% confidence:
+
+| threshold | min n (all passing) |
+|-----------|---------------------|
+| 0.80      | 11                  |
+| 0.90      | 25                  |
+| 0.95      | 52                  |
+| 0.99      | 268                 |
+
+`min_trials(threshold)` computes this. If your eval budget can't afford the
+n, lower the threshold honestly rather than pretending n=5 certifies 99%.
+
+## CI for agents — the full stack
+
+This gate answers "does the behavior hold up statistically **across runs**?"
+Its sibling project [grounding-gate](https://github.com/CiphemonJY/grounding-gate)
+answers "was each claim structurally grounded **within a run**?" Together:
+
+1. **Per-trace, structural**: grounding-gate rejects terminals whose claims
+   were never observed (zero tokens, zero LLM calls).
+2. **Across-runs, statistical**: lcb-gate runs the eval N times and certifies
+   the pass rate's lower bound.
+3. **On promotion**: `compare()` with paired seeds gates champion swaps.
+
+A GitHub Actions job needs nothing special — the gate lives inside the tests:
+
+```yaml
+- run: pip install lcb-gate
+- run: pytest tests/agent_evals -q     # each test is an N-run certified gate
+```
+
+## Roadmap
+
+- Sequential probability ratio test (SPRT) mode — even fewer trials at the
+  same error rates.
+- `@pytest.mark.lcb(n=..., threshold=...)` marker API.
+- JSON report artifact for tracking LCBs over time.
+
+## License
+
+MIT

lcb_gate-0.1.0/README.md

@@ -0,0 +1,133 @@
+# lcb-gate
+
+[![ci](https://github.com/CiphemonJY/lcb-gate/actions/workflows/ci.yml/badge.svg)](https://github.com/CiphemonJY/lcb-gate/actions/workflows/ci.yml)
+
+**Statistical pass/fail gating for stochastic tests.** Run the trial N times;
+pass only when the **Wilson lower confidence bound** of the pass rate clears
+your threshold. Built for agent evals — anything where a single green run
+proves almost nothing. Stdlib only, no dependencies.
+
+```
+pip install lcb-gate
+```
+
+The demo ships in the repo (not the wheel):
+
+```
+git clone https://github.com/CiphemonJY/lcb-gate && cd lcb-gate
+python examples/demo.py     # why a single green run lies, in three acts
+```
+
+## The problem
+
+Agent behaviors, LLM evals, and flaky integration tests are *stochastic*. CI
+treats them as deterministic: one green run → merged. Two failure modes follow:
+
+- **The lucky pass.** A 75%-reliable agent task passes a single CI run 75% of
+  the time. You ship a coin flip and call it tested.
+- **The winner's curse.** You try 20 variants, one "beats the baseline" on a
+  single eval run, and you promote it. Most single-run wins are noise: in one
+  production self-improvement pipeline, fresh-seed re-verification killed
+  **85% of apparent improvements** found by a first-pass scan.
+
+Averages don't fix this — a point estimate without n is a vibe. What fixes it
+is a *lower confidence bound*: "with 95% confidence the true pass rate is at
+least X". Gate on X.
+
+## Usage
+
+### As a library
+
+```python
+from lcb_gate import run_gate
+
+result = run_gate(lambda i: run_my_agent(seed=i).succeeded, n=25, threshold=0.9)
+print(result)
+# PASS: 25/25 passed (gate n=25); LCB 0.902 >= 0.9 @ 95% confidence
+assert result.passed
+```
+
+`run_gate` stops early — in either direction — the moment the verdict is
+mathematically settled, so hopeless runs don't burn the full budget. The
+verdict is identical to running all n trials.
+
+### As a pytest fixture
+
+```python
+def test_agent_completes_checkout(lcb):
+    lcb.check(lambda i: run_agent_eval(seed=i).ok, n=25, threshold=0.9)
+```
+
+The plugin registers automatically on install. Failures explain themselves:
+
+```
+Failed: lcb-gate FAIL: 21/30 passed (gate n=30); LCB 0.551 < 0.9 @ 95% confidence
+```
+
+### Candidate vs champion (promotion gates)
+
+"Is the new prompt/model/policy actually better?" is the same statistics with
+a different threshold — the win-rate LCB must exceed 0.5:
+
+```python
+from lcb_gate import compare
+
+result = compare(candidate=eval_new, champion=eval_old, n=400)
+print(result)
+# BETTER: candidate won 243.0/400 paired trials (60.8%); win-rate LCB 0.567 > 0.5 @ 95%
+```
+
+Both callables receive the **same seed per trial** (common random numbers):
+pairing removes the noise both variants share, which is where most of the
+statistical power comes from. Ties count as half a win. The honest default
+verdict is `NOT PROVEN` — an underpowered comparison never certifies.
+
+```python
+def test_new_prompt_beats_production(lcb):
+    lcb.check_better(eval_new, eval_old, n=400)
+```
+
+## Sizing your gate
+
+A perfect record at small n still can't clear a high bar — by design. Minimum
+trials for a *flawless* run to pass, at 95% confidence:
+
+| threshold | min n (all passing) |
+|-----------|---------------------|
+| 0.80      | 11                  |
+| 0.90      | 25                  |
+| 0.95      | 52                  |
+| 0.99      | 268                 |
+
+`min_trials(threshold)` computes this. If your eval budget can't afford the
+n, lower the threshold honestly rather than pretending n=5 certifies 99%.
+
+## CI for agents — the full stack
+
+This gate answers "does the behavior hold up statistically **across runs**?"
+Its sibling project [grounding-gate](https://github.com/CiphemonJY/grounding-gate)
+answers "was each claim structurally grounded **within a run**?" Together:
+
+1. **Per-trace, structural**: grounding-gate rejects terminals whose claims
+   were never observed (zero tokens, zero LLM calls).
+2. **Across-runs, statistical**: lcb-gate runs the eval N times and certifies
+   the pass rate's lower bound.
+3. **On promotion**: `compare()` with paired seeds gates champion swaps.
+
+A GitHub Actions job needs nothing special — the gate lives inside the tests:
+
+```yaml
+- run: pip install lcb-gate
+- run: pytest tests/agent_evals -q     # each test is an N-run certified gate
+```
+
+## Roadmap
+
+- Sequential probability ratio test (SPRT) mode — even fewer trials at the
+  same error rates.
+- `@pytest.mark.lcb(n=..., threshold=...)` marker API.
+- JSON report artifact for tracking LCBs over time.
+
+## License
+
+MIT

lcb_gate-0.1.0/examples/demo.py

@@ -0,0 +1,69 @@
+"""Why a single green run lies, in three acts. Deterministic (seeded), stdlib only.
+
+Run: ``python examples/demo.py`` (asserts its own expected outcomes; exit 0).
+"""
+
+import random
+import sys
+from pathlib import Path
+
+try:
+    import lcb_gate  # noqa: F401
+except ImportError:
+    sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
+
+from lcb_gate import compare, min_trials, run_gate
+
+
+def flaky_eval(true_pass_rate, salt):
+    """A stochastic 'agent eval': passes with the given true probability."""
+    def trial(i):
+        # str seed: stable across Python versions AND legal on 3.13+
+        # (tuple seeds became a TypeError in 3.13)
+        return random.Random(f"{salt}:{i}").random() < true_pass_rate
+    return trial
+
+
+print("ACT 1 — the 75% eval that looks fine once")
+trial = flaky_eval(0.75, salt=101)
+singles = [trial(i) for i in range(5)]
+print(f"    five individual CI runs: {['green' if s else 'RED' for s in singles]}")
+res = run_gate(trial, n=30, threshold=0.9)
+print(f"    gate verdict: {res}")
+assert any(singles)          # at least one single run looked green
+assert not res.passed        # the gate is not fooled
+
+print("\nACT 2 — the genuinely good eval, and what n it takes")
+trial = flaky_eval(0.97, salt=202)
+small = run_gate(trial, n=25, threshold=0.9)
+large = run_gate(trial, n=150, threshold=0.9)
+print(f"    n=25:  {small}")
+print(f"    n=150: {large}")
+print(f"    (a PERFECT record needs n >= {min_trials(0.9)} to clear 0.9 at 95%)")
+assert large.passed
+
+print("\nACT 3 — candidate vs champion, paired seeds (common random numbers)")
+
+
+def champion(seed):
+    return random.Random(seed).random()
+
+
+def candidate(seed):
+    # same shared noise as the champion (CRN), plus a real but small edge:
+    # wins the pair 60% of the time
+    r = random.Random(seed)
+    base = r.random()
+    return base + (0.01 if r.random() < 0.6 else -0.01)
+
+
+underpowered = compare(candidate, champion, seeds=range(30))
+powered = compare(candidate, champion, seeds=range(400))
+print(f"    n=30:  {underpowered}")
+print(f"    n=400: {powered}")
+print("    same candidate, same true edge — only the evidence changed")
+print("    (30 pairs is so underpowered the sample can even reverse the sign)")
+assert not underpowered.better   # honest: cannot certify on 30 pairs
+assert powered.better            # certified on 400
+
+print("\nAll demo assertions passed.")

lcb_gate-0.1.0/pyproject.toml

@@ -0,0 +1,52 @@
+[build-system]
+requires = ["hatchling>=1.27"]
+build-backend = "hatchling.build"
+
+[project]
+name = "lcb-gate"
+version = "0.1.0"
+description = "Statistical pass/fail gating for stochastic tests: run N times, pass only when the Wilson lower confidence bound clears the bar. Built for agent evals; stdlib only."
+readme = "README.md"
+license = "MIT"
+license-files = ["LICENSE"]
+requires-python = ">=3.9"
+authors = [{ name = "Ciphemon" }]
+keywords = [
+  "testing",
+  "flaky-tests",
+  "agent-evals",
+  "llm",
+  "statistics",
+  "confidence-interval",
+  "pytest",
+]
+classifiers = [
+  "Development Status :: 3 - Alpha",
+  "Framework :: Pytest",
+  "Intended Audience :: Developers",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.9",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: 3.13",
+  "Topic :: Software Development :: Quality Assurance",
+]
+
+[project.optional-dependencies]
+dev = ["pytest>=7"]
+
+[project.entry-points.pytest11]
+lcb_gate = "lcb_gate.plugin"
+
+[project.urls]
+Homepage = "https://github.com/CiphemonJY/lcb-gate"
+Repository = "https://github.com/CiphemonJY/lcb-gate"
+Issues = "https://github.com/CiphemonJY/lcb-gate/issues"
+Changelog = "https://github.com/CiphemonJY/lcb-gate/blob/main/CHANGELOG.md"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/lcb_gate"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]

lcb_gate-0.1.0/src/lcb_gate/__init__.py

@@ -0,0 +1,22 @@
+"""lcb-gate — statistical pass/fail gating for stochastic tests.
+
+A single green run of a stochastic test proves almost nothing. This package
+runs the trial N times and passes only when the Wilson lower confidence bound
+of the observed pass rate clears your threshold — so "it passed" means
+"it passes at ≥ X% with statistical backing", not "it got lucky once".
+"""
+
+from .stats import min_trials, wilson_lcb
+from .gate import CompareResult, GateResult, compare, run_gate
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "CompareResult",
+    "GateResult",
+    "compare",
+    "min_trials",
+    "run_gate",
+    "wilson_lcb",
+    "__version__",
+]

lcb_gate-0.1.0/src/lcb_gate/gate.py

@@ -0,0 +1,90 @@
+"""The gate: N-run consensus with an LCB verdict, and paired A/B comparison."""
+
+from dataclasses import dataclass
+
+from .stats import wilson_lcb
+
+
+@dataclass
+class GateResult:
+    passes: int
+    n: int              # the gate's denominator — unrun trials count as failures
+    trials_run: int
+    lcb: float
+    threshold: float
+    confidence: float
+    passed: bool
+
+    def __str__(self):
+        verdict = "PASS" if self.passed else "FAIL"
+        return (f"{verdict}: {self.passes}/{self.trials_run} passed "
+                f"(gate n={self.n}); LCB {self.lcb:.3f} "
+                f"{'>=' if self.passed else '<'} {self.threshold} "
+                f"@ {self.confidence:.0%} confidence")
+
+
+def run_gate(trial, n=25, threshold=0.9, confidence=0.95, stop_early=True):
+    """Run ``trial(i) -> bool`` up to ``n`` times; pass iff the Wilson LCB of
+    the pass rate (over the FIXED denominator n) clears ``threshold``.
+
+    Unrun trials count as failures, so the reported bound is always
+    conservative. With ``stop_early`` the loop exits as soon as the verdict is
+    mathematically settled either way — the verdict is identical to running
+    all n trials, only cheaper.
+    """
+    passes = 0
+    ran = 0
+    for i in range(n):
+        # verdict already settled?
+        if stop_early:
+            if wilson_lcb(passes, n, confidence) >= threshold:
+                break                                     # pass even if the rest fail
+            if wilson_lcb(passes + (n - ran), n, confidence) < threshold:
+                break                                     # fail even if the rest pass
+        passes += 1 if trial(i) else 0
+        ran += 1
+    lcb = wilson_lcb(passes, n, confidence)
+    return GateResult(passes=passes, n=n, trials_run=ran, lcb=lcb,
+                      threshold=threshold, confidence=confidence,
+                      passed=lcb >= threshold)
+
+
+@dataclass
+class CompareResult:
+    wins: float         # ties count 0.5
+    n: int
+    win_rate: float
+    lcb: float
+    confidence: float
+    better: bool        # LCB of the win rate > 0.5
+
+    def __str__(self):
+        verdict = "BETTER" if self.better else "NOT PROVEN"
+        return (f"{verdict}: candidate won {self.wins}/{self.n} paired trials "
+                f"({self.win_rate:.1%}); win-rate LCB {self.lcb:.3f} "
+                f"{'>' if self.better else '<='} 0.5 @ {self.confidence:.0%}")
+
+
+def compare(candidate, champion, n=200, confidence=0.95, seeds=None):
+    """Paired comparison: is ``candidate`` better than ``champion``?
+
+    Both callables receive the SAME seed per trial (common random numbers) —
+    pairing removes shared noise, which is where most of the statistical power
+    comes from. Callables return a comparable score (float or bool); ties
+    count as half a win. Verdict ``better`` requires the win-rate LCB to
+    exceed 0.5 — "not proven" is the honest default for underpowered n.
+    """
+    seed_list = list(seeds) if seeds is not None else list(range(n))
+    total = len(seed_list)
+    if total == 0:
+        raise ValueError("compare() needs at least one seed")
+    wins = 0.0
+    for s in seed_list:
+        c, ch = candidate(s), champion(s)
+        if c > ch:
+            wins += 1.0
+        elif c == ch:
+            wins += 0.5
+    lcb = wilson_lcb(wins, total, confidence)
+    return CompareResult(wins=wins, n=total, win_rate=wins / total, lcb=lcb,
+                         confidence=confidence, better=lcb > 0.5)

lcb_gate-0.1.0/src/lcb_gate/plugin.py

@@ -0,0 +1,36 @@
+"""pytest integration: the ``lcb`` fixture.
+
+Usage::
+
+    def test_agent_completes_task(lcb):
+        lcb.check(lambda i: run_my_agent(seed=i).succeeded, n=25, threshold=0.9)
+
+The test fails with a statistical explanation unless the Wilson LCB of the
+pass rate clears the threshold.
+"""
+
+import pytest
+
+from .gate import compare, run_gate
+
+
+class LcbHelper:
+    def check(self, trial, n=25, threshold=0.9, confidence=0.95, stop_early=True):
+        """Fail the surrounding test unless the gate passes. Returns GateResult."""
+        result = run_gate(trial, n=n, threshold=threshold,
+                          confidence=confidence, stop_early=stop_early)
+        if not result.passed:
+            pytest.fail(f"lcb-gate {result}")
+        return result
+
+    def check_better(self, candidate, champion, n=200, confidence=0.95, seeds=None):
+        """Fail the surrounding test unless candidate provably beats champion."""
+        result = compare(candidate, champion, n=n, confidence=confidence, seeds=seeds)
+        if not result.better:
+            pytest.fail(f"lcb-gate {result}")
+        return result
+
+
+@pytest.fixture
+def lcb():
+    return LcbHelper()

lcb_gate-0.1.0/src/lcb_gate/stats.py

@@ -0,0 +1,38 @@
+"""Wilson score interval, lower bound — the workhorse statistic.
+
+The Wilson bound is preferred over the normal (Wald) approximation because it
+behaves at the edges: a perfect record never yields LCB = 1.0, and small n
+yields honestly wide bounds. ``successes`` may be fractional (ties in paired
+comparisons count as half a win).
+"""
+
+import math
+from statistics import NormalDist
+
+
+def wilson_lcb(successes, n, confidence=0.95):
+    """One-sided Wilson score lower bound for a binomial proportion.
+
+    Returns 0.0 for n == 0 (no evidence -> no lower bound).
+    """
+    if n == 0:
+        return 0.0
+    z = NormalDist().inv_cdf(confidence)
+    phat = successes / n
+    denom = 1 + z * z / n
+    center = phat + z * z / (2 * n)
+    margin = z * math.sqrt(phat * (1 - phat) / n + z * z / (4 * n * n))
+    return max(0.0, (center - margin) / denom)
+
+
+def min_trials(threshold, confidence=0.95):
+    """Smallest n such that a PERFECT record clears ``threshold``.
+
+    For phat = 1 the Wilson LCB reduces to 1 / (1 + z²/n), so the requirement
+    is n >= z² · threshold / (1 - threshold). Useful for sizing a gate: below
+    this n, the gate is unpassable even by a flawless run — by design.
+    """
+    if not 0 < threshold < 1:
+        raise ValueError("threshold must be in (0, 1)")
+    z = NormalDist().inv_cdf(confidence)
+    return math.ceil(z * z * threshold / (1 - threshold))

lcb_gate-0.1.0/tests/conftest.py

@@ -0,0 +1,19 @@
+import sys
+from pathlib import Path
+
+# allow running the suite from a raw checkout, before `pip install -e .`
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
+
+try:
+    from importlib.metadata import distribution
+
+    distribution("lcb-gate")
+    _installed = True   # the pytest11 entry point already registers the plugin
+except Exception:
+    _installed = False
+
+if not _installed:
+    # Raw checkout only: no entry point exists, so load the plugin explicitly.
+    # Doing this unconditionally double-registers the module once the package
+    # is installed, and pluggy crashes before collecting a single test.
+    pytest_plugins = ["lcb_gate.plugin"]

lcb_gate-0.1.0/tests/test_lcb.py

@@ -0,0 +1,128 @@
+"""Core suite — deterministic, zero dependencies: ``python tests/test_lcb.py``."""
+
+import sys
+from pathlib import Path
+
+try:
+    import lcb_gate  # noqa: F401
+except ImportError:
+    sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
+
+from lcb_gate import compare, min_trials, run_gate, wilson_lcb
+
+
+# ------------------------------------------------------------------ stats
+
+def test_wilson_known_value_90_of_100():
+    # hand-computed: z=1.6449, LCB ≈ 0.8396
+    assert abs(wilson_lcb(90, 100, 0.95) - 0.8396) < 1e-3
+
+
+def test_wilson_perfect_record_closed_form():
+    # for phat=1 the bound reduces to 1/(1+z²/n)
+    z2 = 1.6448536269514722 ** 2
+    assert abs(wilson_lcb(20, 20, 0.95) - 1 / (1 + z2 / 20)) < 1e-9
+
+
+def test_wilson_edges():
+    assert wilson_lcb(0, 0) == 0.0
+    assert wilson_lcb(0, 10) == 0.0
+
+
+def test_wilson_never_exceeds_point_estimate():
+    for s, n in [(1, 2), (5, 10), (45, 50), (99, 100)]:
+        assert wilson_lcb(s, n) <= s / n
+
+
+def test_wilson_tightens_with_n():
+    assert wilson_lcb(9, 10) < wilson_lcb(90, 100) < wilson_lcb(900, 1000)
+
+
+def test_min_trials_table():
+    assert min_trials(0.8) == 11
+    assert min_trials(0.9) == 25
+    assert min_trials(0.95) == 52
+
+
+def test_min_trials_is_exact_boundary():
+    n = min_trials(0.9)
+    assert wilson_lcb(n, n) >= 0.9          # perfect record at n passes
+    assert wilson_lcb(n - 1, n - 1) < 0.9   # one fewer trial cannot pass
+
+
+# ------------------------------------------------------------------ gate
+
+def test_gate_passes_one_failure_in_twenty():
+    res = run_gate(lambda i: i != 3, n=20, threshold=0.75)
+    assert res.passed and res.passes == 19
+
+
+def test_gate_fails_chronic_flake():
+    res = run_gate(lambda i: i % 2 == 0, n=20, threshold=0.9)  # 50% pass rate
+    assert not res.passed
+
+
+def test_gate_early_stop_on_hopeless_run():
+    res = run_gate(lambda i: False, n=100, threshold=0.9)
+    assert not res.passed
+    assert res.trials_run < 100     # settled long before 100 runs
+
+
+def test_gate_early_stop_verdict_matches_full_run():
+    trial = lambda i: i % 7 != 0    # noqa: E731 — ~86% pass rate, deterministic
+    early = run_gate(trial, n=50, threshold=0.7, stop_early=True)
+    full = run_gate(trial, n=50, threshold=0.7, stop_early=False)
+    assert early.passed == full.passed
+
+
+def test_gate_perfect_record_at_min_trials():
+    n = min_trials(0.9)
+    res = run_gate(lambda i: True, n=n, threshold=0.9)
+    assert res.passed
+
+
+# ------------------------------------------------------------------ compare
+
+def _champ(seed):
+    return (seed % 100) / 100
+
+
+def test_compare_detects_70pct_winner():
+    def cand(seed):
+        return _champ(seed) + (0.01 if seed % 10 < 7 else -0.01)
+    res = compare(cand, _champ, seeds=range(100))
+    assert res.better
+    assert abs(res.win_rate - 0.7) < 1e-9
+
+
+def test_compare_null_candidate_not_proven():
+    res = compare(_champ, _champ, seeds=range(100))   # all ties -> 0.5 win rate
+    assert not res.better
+    assert res.win_rate == 0.5
+
+
+def test_compare_underpowered_n_not_proven():
+    def cand(seed):
+        return _champ(seed) + (0.01 if seed % 10 < 6 else -0.01)   # true 60% winner
+    assert not compare(cand, _champ, seeds=range(10)).better       # n=10: can't know
+    assert compare(cand, _champ, seeds=range(500)).better          # n=500: proven
+
+
+# ------------------------------------------------------- bare-python runner
+
+if __name__ == "__main__":
+    import inspect
+    failures = []
+    cases = [(n, f) for n, f in sorted(globals().items())
+             if n.startswith("test_") and callable(f)
+             and not inspect.signature(f).parameters]
+    for name, fn in cases:
+        try:
+            fn()
+            print(f"  PASS  {name}")
+        except AssertionError:
+            print(f"  FAIL  {name}")
+            failures.append(name)
+    print(f"\n{'ALL PASS' if not failures else f'FAILED: {failures}'}"
+          f" — {len(cases) - len(failures)}/{len(cases)}")
+    sys.exit(1 if failures else 0)

lcb_gate-0.1.0/tests/test_plugin.py

@@ -0,0 +1,26 @@
+"""Fixture integration — pytest only (excluded from the bare-python runner)."""
+
+import pytest
+
+from lcb_gate.plugin import LcbHelper
+
+
+def test_fixture_is_registered(lcb):
+    assert isinstance(lcb, LcbHelper)
+
+
+def test_check_passes_clean_run(lcb):
+    res = lcb.check(lambda i: True, n=25, threshold=0.9)
+    assert res.passed
+
+
+def test_check_fails_flaky_run():
+    helper = LcbHelper()
+    with pytest.raises(pytest.fail.Exception):
+        helper.check(lambda i: i % 2 == 0, n=20, threshold=0.9)
+
+
+def test_check_better_fails_on_tie():
+    helper = LcbHelper()
+    with pytest.raises(pytest.fail.Exception):
+        helper.check_better(lambda s: 1.0, lambda s: 1.0, seeds=range(50))