lcb-gate 0.1.0__py3-none-any.whl

lcb_gate/__init__.py

@@ -0,0 +1,22 @@
+"""lcb-gate — statistical pass/fail gating for stochastic tests.
+
+A single green run of a stochastic test proves almost nothing. This package
+runs the trial N times and passes only when the Wilson lower confidence bound
+of the observed pass rate clears your threshold — so "it passed" means
+"it passes at ≥ X% with statistical backing", not "it got lucky once".
+"""
+
+from .stats import min_trials, wilson_lcb
+from .gate import CompareResult, GateResult, compare, run_gate
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "CompareResult",
+    "GateResult",
+    "compare",
+    "min_trials",
+    "run_gate",
+    "wilson_lcb",
+    "__version__",
+]

lcb_gate/gate.py

@@ -0,0 +1,90 @@
+"""The gate: N-run consensus with an LCB verdict, and paired A/B comparison."""
+
+from dataclasses import dataclass
+
+from .stats import wilson_lcb
+
+
+@dataclass
+class GateResult:
+    passes: int
+    n: int              # the gate's denominator — unrun trials count as failures
+    trials_run: int
+    lcb: float
+    threshold: float
+    confidence: float
+    passed: bool
+
+    def __str__(self):
+        verdict = "PASS" if self.passed else "FAIL"
+        return (f"{verdict}: {self.passes}/{self.trials_run} passed "
+                f"(gate n={self.n}); LCB {self.lcb:.3f} "
+                f"{'>=' if self.passed else '<'} {self.threshold} "
+                f"@ {self.confidence:.0%} confidence")
+
+
+def run_gate(trial, n=25, threshold=0.9, confidence=0.95, stop_early=True):
+    """Run ``trial(i) -> bool`` up to ``n`` times; pass iff the Wilson LCB of
+    the pass rate (over the FIXED denominator n) clears ``threshold``.
+
+    Unrun trials count as failures, so the reported bound is always
+    conservative. With ``stop_early`` the loop exits as soon as the verdict is
+    mathematically settled either way — the verdict is identical to running
+    all n trials, only cheaper.
+    """
+    passes = 0
+    ran = 0
+    for i in range(n):
+        # verdict already settled?
+        if stop_early:
+            if wilson_lcb(passes, n, confidence) >= threshold:
+                break                                     # pass even if the rest fail
+            if wilson_lcb(passes + (n - ran), n, confidence) < threshold:
+                break                                     # fail even if the rest pass
+        passes += 1 if trial(i) else 0
+        ran += 1
+    lcb = wilson_lcb(passes, n, confidence)
+    return GateResult(passes=passes, n=n, trials_run=ran, lcb=lcb,
+                      threshold=threshold, confidence=confidence,
+                      passed=lcb >= threshold)
+
+
+@dataclass
+class CompareResult:
+    wins: float         # ties count 0.5
+    n: int
+    win_rate: float
+    lcb: float
+    confidence: float
+    better: bool        # LCB of the win rate > 0.5
+
+    def __str__(self):
+        verdict = "BETTER" if self.better else "NOT PROVEN"
+        return (f"{verdict}: candidate won {self.wins}/{self.n} paired trials "
+                f"({self.win_rate:.1%}); win-rate LCB {self.lcb:.3f} "
+                f"{'>' if self.better else '<='} 0.5 @ {self.confidence:.0%}")
+
+
+def compare(candidate, champion, n=200, confidence=0.95, seeds=None):
+    """Paired comparison: is ``candidate`` better than ``champion``?
+
+    Both callables receive the SAME seed per trial (common random numbers) —
+    pairing removes shared noise, which is where most of the statistical power
+    comes from. Callables return a comparable score (float or bool); ties
+    count as half a win. Verdict ``better`` requires the win-rate LCB to
+    exceed 0.5 — "not proven" is the honest default for underpowered n.
+    """
+    seed_list = list(seeds) if seeds is not None else list(range(n))
+    total = len(seed_list)
+    if total == 0:
+        raise ValueError("compare() needs at least one seed")
+    wins = 0.0
+    for s in seed_list:
+        c, ch = candidate(s), champion(s)
+        if c > ch:
+            wins += 1.0
+        elif c == ch:
+            wins += 0.5
+    lcb = wilson_lcb(wins, total, confidence)
+    return CompareResult(wins=wins, n=total, win_rate=wins / total, lcb=lcb,
+                         confidence=confidence, better=lcb > 0.5)

lcb_gate/plugin.py

@@ -0,0 +1,36 @@
+"""pytest integration: the ``lcb`` fixture.
+
+Usage::
+
+    def test_agent_completes_task(lcb):
+        lcb.check(lambda i: run_my_agent(seed=i).succeeded, n=25, threshold=0.9)
+
+The test fails with a statistical explanation unless the Wilson LCB of the
+pass rate clears the threshold.
+"""
+
+import pytest
+
+from .gate import compare, run_gate
+
+
+class LcbHelper:
+    def check(self, trial, n=25, threshold=0.9, confidence=0.95, stop_early=True):
+        """Fail the surrounding test unless the gate passes. Returns GateResult."""
+        result = run_gate(trial, n=n, threshold=threshold,
+                          confidence=confidence, stop_early=stop_early)
+        if not result.passed:
+            pytest.fail(f"lcb-gate {result}")
+        return result
+
+    def check_better(self, candidate, champion, n=200, confidence=0.95, seeds=None):
+        """Fail the surrounding test unless candidate provably beats champion."""
+        result = compare(candidate, champion, n=n, confidence=confidence, seeds=seeds)
+        if not result.better:
+            pytest.fail(f"lcb-gate {result}")
+        return result
+
+
+@pytest.fixture
+def lcb():
+    return LcbHelper()

lcb_gate/stats.py

@@ -0,0 +1,38 @@
+"""Wilson score interval, lower bound — the workhorse statistic.
+
+The Wilson bound is preferred over the normal (Wald) approximation because it
+behaves at the edges: a perfect record never yields LCB = 1.0, and small n
+yields honestly wide bounds. ``successes`` may be fractional (ties in paired
+comparisons count as half a win).
+"""
+
+import math
+from statistics import NormalDist
+
+
+def wilson_lcb(successes, n, confidence=0.95):
+    """One-sided Wilson score lower bound for a binomial proportion.
+
+    Returns 0.0 for n == 0 (no evidence -> no lower bound).
+    """
+    if n == 0:
+        return 0.0
+    z = NormalDist().inv_cdf(confidence)
+    phat = successes / n
+    denom = 1 + z * z / n
+    center = phat + z * z / (2 * n)
+    margin = z * math.sqrt(phat * (1 - phat) / n + z * z / (4 * n * n))
+    return max(0.0, (center - margin) / denom)
+
+
+def min_trials(threshold, confidence=0.95):
+    """Smallest n such that a PERFECT record clears ``threshold``.
+
+    For phat = 1 the Wilson LCB reduces to 1 / (1 + z²/n), so the requirement
+    is n >= z² · threshold / (1 - threshold). Useful for sizing a gate: below
+    this n, the gate is unpassable even by a flawless run — by design.
+    """
+    if not 0 < threshold < 1:
+        raise ValueError("threshold must be in (0, 1)")
+    z = NormalDist().inv_cdf(confidence)
+    return math.ceil(z * z * threshold / (1 - threshold))

lcb_gate-0.1.0.dist-info/METADATA

@@ -0,0 +1,160 @@
+Metadata-Version: 2.4
+Name: lcb-gate
+Version: 0.1.0
+Summary: Statistical pass/fail gating for stochastic tests: run N times, pass only when the Wilson lower confidence bound clears the bar. Built for agent evals; stdlib only.
+Project-URL: Homepage, https://github.com/CiphemonJY/lcb-gate
+Project-URL: Repository, https://github.com/CiphemonJY/lcb-gate
+Project-URL: Issues, https://github.com/CiphemonJY/lcb-gate/issues
+Project-URL: Changelog, https://github.com/CiphemonJY/lcb-gate/blob/main/CHANGELOG.md
+Author: Ciphemon
+License-Expression: MIT
+License-File: LICENSE
+Keywords: agent-evals,confidence-interval,flaky-tests,llm,pytest,statistics,testing
+Classifier: Development Status :: 3 - Alpha
+Classifier: Framework :: Pytest
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Quality Assurance
+Requires-Python: >=3.9
+Provides-Extra: dev
+Requires-Dist: pytest>=7; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# lcb-gate
+
+[![ci](https://github.com/CiphemonJY/lcb-gate/actions/workflows/ci.yml/badge.svg)](https://github.com/CiphemonJY/lcb-gate/actions/workflows/ci.yml)
+
+**Statistical pass/fail gating for stochastic tests.** Run the trial N times;
+pass only when the **Wilson lower confidence bound** of the pass rate clears
+your threshold. Built for agent evals — anything where a single green run
+proves almost nothing. Stdlib only, no dependencies.
+
+```
+pip install lcb-gate
+```
+
+The demo ships in the repo (not the wheel):
+
+```
+git clone https://github.com/CiphemonJY/lcb-gate && cd lcb-gate
+python examples/demo.py     # why a single green run lies, in three acts
+```
+
+## The problem
+
+Agent behaviors, LLM evals, and flaky integration tests are *stochastic*. CI
+treats them as deterministic: one green run → merged. Two failure modes follow:
+
+- **The lucky pass.** A 75%-reliable agent task passes a single CI run 75% of
+  the time. You ship a coin flip and call it tested.
+- **The winner's curse.** You try 20 variants, one "beats the baseline" on a
+  single eval run, and you promote it. Most single-run wins are noise: in one
+  production self-improvement pipeline, fresh-seed re-verification killed
+  **85% of apparent improvements** found by a first-pass scan.
+
+Averages don't fix this — a point estimate without n is a vibe. What fixes it
+is a *lower confidence bound*: "with 95% confidence the true pass rate is at
+least X". Gate on X.
+
+## Usage
+
+### As a library
+
+```python
+from lcb_gate import run_gate
+
+result = run_gate(lambda i: run_my_agent(seed=i).succeeded, n=25, threshold=0.9)
+print(result)
+# PASS: 25/25 passed (gate n=25); LCB 0.902 >= 0.9 @ 95% confidence
+assert result.passed
+```
+
+`run_gate` stops early — in either direction — the moment the verdict is
+mathematically settled, so hopeless runs don't burn the full budget. The
+verdict is identical to running all n trials.
+
+### As a pytest fixture
+
+```python
+def test_agent_completes_checkout(lcb):
+    lcb.check(lambda i: run_agent_eval(seed=i).ok, n=25, threshold=0.9)
+```
+
+The plugin registers automatically on install. Failures explain themselves:
+
+```
+Failed: lcb-gate FAIL: 21/30 passed (gate n=30); LCB 0.551 < 0.9 @ 95% confidence
+```
+
+### Candidate vs champion (promotion gates)
+
+"Is the new prompt/model/policy actually better?" is the same statistics with
+a different threshold — the win-rate LCB must exceed 0.5:
+
+```python
+from lcb_gate import compare
+
+result = compare(candidate=eval_new, champion=eval_old, n=400)
+print(result)
+# BETTER: candidate won 243.0/400 paired trials (60.8%); win-rate LCB 0.567 > 0.5 @ 95%
+```
+
+Both callables receive the **same seed per trial** (common random numbers):
+pairing removes the noise both variants share, which is where most of the
+statistical power comes from. Ties count as half a win. The honest default
+verdict is `NOT PROVEN` — an underpowered comparison never certifies.
+
+```python
+def test_new_prompt_beats_production(lcb):
+    lcb.check_better(eval_new, eval_old, n=400)
+```
+
+## Sizing your gate
+
+A perfect record at small n still can't clear a high bar — by design. Minimum
+trials for a *flawless* run to pass, at 95% confidence:
+
+| threshold | min n (all passing) |
+|-----------|---------------------|
+| 0.80      | 11                  |
+| 0.90      | 25                  |
+| 0.95      | 52                  |
+| 0.99      | 268                 |
+
+`min_trials(threshold)` computes this. If your eval budget can't afford the
+n, lower the threshold honestly rather than pretending n=5 certifies 99%.
+
+## CI for agents — the full stack
+
+This gate answers "does the behavior hold up statistically **across runs**?"
+Its sibling project [grounding-gate](https://github.com/CiphemonJY/grounding-gate)
+answers "was each claim structurally grounded **within a run**?" Together:
+
+1. **Per-trace, structural**: grounding-gate rejects terminals whose claims
+   were never observed (zero tokens, zero LLM calls).
+2. **Across-runs, statistical**: lcb-gate runs the eval N times and certifies
+   the pass rate's lower bound.
+3. **On promotion**: `compare()` with paired seeds gates champion swaps.
+
+A GitHub Actions job needs nothing special — the gate lives inside the tests:
+
+```yaml
+- run: pip install lcb-gate
+- run: pytest tests/agent_evals -q     # each test is an N-run certified gate
+```
+
+## Roadmap
+
+- Sequential probability ratio test (SPRT) mode — even fewer trials at the
+  same error rates.
+- `@pytest.mark.lcb(n=..., threshold=...)` marker API.
+- JSON report artifact for tracking LCBs over time.
+
+## License
+
+MIT

lcb_gate-0.1.0.dist-info/RECORD

@@ -0,0 +1,9 @@
+lcb_gate/__init__.py,sha256=a0ffqTt4wVwnfy3PlIFWlA4jpD7J1KVhbtosWfwnC1Y,641
+lcb_gate/gate.py,sha256=b9ee609Fwzi2z1rjVfG48FxL5OWlgs7VaNJuGRRsuqg,3465
+lcb_gate/plugin.py,sha256=e0uQCocuVGkQSMTR-nu8swIPpnciaPsxy-PDEGZ7OF0,1159
+lcb_gate/stats.py,sha256=rGaOd7eb__Spv94yzLtyNrq6DVk_AXOb5KK_PvVlce0,1390
+lcb_gate-0.1.0.dist-info/METADATA,sha256=07NXrv5ppPZkdGHW7o3EoVOcMtTxsjslPQf4l4kW1X4,5827
+lcb_gate-0.1.0.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
+lcb_gate-0.1.0.dist-info/entry_points.txt,sha256=cl6VESqUUyDfDQyeNf1Yz6os8hSWpfs1VbaYrbuL5zk,38
+lcb_gate-0.1.0.dist-info/licenses/LICENSE,sha256=76kJpx8qwE8s8qdKjsnngPtznV3WasZ54ed4WWW2y7o,1065
+lcb_gate-0.1.0.dist-info/RECORD,,

lcb_gate-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.31.0
+Root-Is-Purelib: true
+Tag: py3-none-any

lcb_gate-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[pytest11]
+lcb_gate = lcb_gate.plugin

lcb_gate-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Ciphemon
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.