anytimemonitor 0.9.0__py3-none-any.whl

anytimemonitor/__init__.py

@@ -0,0 +1,21 @@
+"""anytimemonitor — adaptive controller for TTS-MOS listening tests.
+
+Public API:
+    AnytimeMonitor — stateful controller built on an anytime-valid
+                        confidence sequence (per-item paired diffs).
+                        Add ratings, query for per-pair verdicts and
+                        two-lever (+items / +listeners) advice at any
+                        data-dependent time.
+    Decision          — dataclass returned by AnytimeMonitor.decision().
+
+See README.md for use case and architecture.
+"""
+from .controller import Decision, AnytimeMonitor
+from .variance import VarianceEstimate, estimate_components
+
+__all__ = [
+    "AnytimeMonitor",
+    "Decision",
+    "VarianceEstimate",
+    "estimate_components",
+]

anytimemonitor/avci.py

@@ -0,0 +1,387 @@
+"""AnytimeCI — confidence sequence on the mean of a per-item paired diff stream.
+
+V2 per-item revision — supersedes the per-listener version preserved at
+``git tag avci-per-listener-v1.1``. See ``Architecture.md`` for the design
+rationale.
+
+The observation is the per-item paired difference
+``d_j = mean_ℓ [Y_{ℓ,A,j} − Y_{ℓ,B,j}]`` over the listeners who rated
+item ``j`` for both systems A and B. Under the §4.5 random-effects
+model, conditional on the listener pool fixed at ``L_pool`` distinct
+listeners, the d_j are i.i.d. with
+
+    Var(d_j | pool, L_j) = Σ²_int_pair + (Σ²_γαβ + Σ²_eval) / L_j
+
+The pool itself contributes a *constant offset* C across all d_j (the
+realised mean of (γα)_ℓA − (γα)_ℓB over the pool). Marginalising over
+the pool adds an irreducible variance term
+
+    2 σ²_(γα) / L_pool
+
+that does not shrink with N_items. To preserve coverage of the
+marginal effect δ = E[Y_A − Y_B] (and not just the pool-conditional
+effect), the AVCI boundary combines the Howard 2021 sub-Gaussian
+half-width and a deterministic floor in quadrature:
+
+    Ψ_eff(t) = √( Ψ_Howard(σ̂²_t, t)² + plancher² )
+    plancher = √( 2 · c · σ̂²_t / L_pool )
+
+where ``c`` (``gamma_alpha_factor``) is a multiplier expressing the
+upper bound assumed for σ²_(γα) as a fraction of σ²_int (default 1.0,
+i.e. σ²_(γα) ≤ σ²_int — see ``Architecture.md`` for the prior
+discussion). σ̂²_t (Welford or Hoeffding fallback) is used as a
+self-contained proxy for σ²_int: this slightly inflates the floor
+(σ̂²_d_j also absorbs listener-noise / k_ℓ terms) but keeps the
+boundary auto-contained — no MoM coupling required.
+
+The Howard 2021 line-crossing boundary itself is unchanged
+(eq.14 of arXiv 1810.08240): two proxy variance options remain:
+
+- ``hoeffding``   : σ²_proxy = ((b−a)/2)² fixed (worst-case bound).
+- ``ebernstein``  : σ²_proxy = σ̂²_t (Welford running estimator), with
+                    Hoeffding fallback while t < t_min_ebernstein.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Literal, Union
+
+import numpy as np
+from numpy.typing import ArrayLike
+
+
+PROXY = Literal["hoeffding", "ebernstein"]
+VERDICT = Literal["STOP_SEP", "STOP_EQUIV", "CONTINUE"]
+
+
+def eta_from_n_planned(n_planned: int) -> float:
+    """Howard's a-priori tightness rule: η = 1/√(N_planned).
+
+    ``N_planned`` is the PLANNED item budget; the boundary is tightest near
+    t ≈ 1/η² = N_planned items. η is fixed a priori from the plan and is
+    never tuned to the observed data (it sets efficiency, not validity).
+    Single source of truth for this formula — used by the controller and
+    the UI so neither re-implements it.
+    """
+    if n_planned < 1:
+        raise ValueError("n_planned must be >= 1")
+    return 1.0 / float(np.sqrt(n_planned))
+
+
+@dataclass
+class AnytimeCI:
+    """Sub-Gaussian anytime-valid confidence sequence on E[X].
+
+    Streaming API: feed per-item paired differences through ``update``,
+    query the current confidence interval and verdict at any time.
+
+    The boundary is widened by a deterministic floor accounting for the
+    listener-pool contribution that does not shrink with t. See module
+    docstring for the model rationale.
+    """
+
+    alpha: float = 0.05
+    proxy: PROXY = "ebernstein"
+    # Range of the *observation* d_j (per-item paired diff). For MOS
+    # ∈ [1, 5] this is (-4, 4). Override for other rating scales.
+    d_bounds: tuple[float, float] = (-4.0, 4.0)
+    # Howard 2021 line-crossing shape parameter. Boundary is tightest near
+    # t ≈ 1/η²; set it a priori from the PLANNED item budget as
+    # η = 1/√(N_planned) (never tune it to the data — efficiency, not
+    # validity). Default 0.07 ≈ 1/√200 from the §7 survey median budget
+    # (~400 ratings/system at K=2 ⇒ N_planned ≈ 200); PROVISIONAL until
+    # that survey table is validated.
+    eta: float = 0.07
+    # Below this t, ``ebernstein`` proxy falls back to Hoeffding to avoid
+    # exploding bounds when σ̂² is undefined / very noisy.
+    t_min_ebernstein: int = 10
+    # Relative floor on the empirical variance estimate, expressed as a
+    # fraction of the Hoeffding proxy ((b-a)/2)². Prevents anti-conservative
+    # bounds when the running σ̂² collapses on near-constant streams.
+    ebernstein_floor_frac: float = 0.01
+    # Listener-pool size — the number of distinct listeners contributing
+    # to the d_j stream. Used in the marginal-coverage floor only ; the
+    # Howard 2021 boundary still drives the bulk of Ψ at small/medium t.
+    # Set to ``None`` for the rare regime where the listener pool is
+    # already effectively infinite (e.g. fresh-listener-per-item
+    # crowdsourcing) — the floor then collapses to zero.
+    L_pool: int | None = None
+    # Upper-bound multiplier for σ²_(γα) as a fraction of σ²_int.
+    # Default 1.0: assume σ²_(γα) ≤ σ²_int (cf. Architecture.md ;
+    # 1.0 is the conservative baseline, 2.0 the worst-case Blizzard
+    # envelope). This is the fallback upper bound used ONLY when no
+    # estimate is supplied; the controller's own cold-start instead plugs
+    # in a large BTTF-anchored prior (≈2.7·σ²_int) — see
+    # `controller.cold_start_ga_ratio`.
+    # **Only used when `sigma2_ga_estimate` is None** (no data-driven
+    # estimate available, e.g. warm-up).
+    gamma_alpha_factor: float = 1.0
+    # Optional data-driven estimate of σ²_(γα). When set (typically by
+    # the controller from the per-(listener, item) d_ℓj matrix via
+    # MS_across − MS_within/k_ℓ_mean), the plancher uses it directly:
+    # plancher = √(2·σ̂²_(γα)/L_pool). Avoids the surcoût conservatif of
+    # c·σ̂²_proxy on homogeneous palettes (where σ²_(γα) ≪ σ²_int).
+    sigma2_ga_estimate: Optional[float] = None
+    # Optional override on the ITEM-side (iid) standard error of the
+    # mean. When set, half_width() uses
+    # ``Ψ_eff = √((se_iid_override · multiplier)² + plancher²)``
+    # bypassing the Welford σ̂²_proxy → Ψ_Howard path. The multiplier
+    # is the AVCI peeking-penalty factor (cf. _avci_multiplier).
+    # Typically set by the controller from a bootstrap-by-items
+    # resample, which estimates Var(δ̂ | pool) — the iid component
+    # that DOES decay with t and IS subject to peeking penalty.
+    se_iid_override: Optional[float] = None
+    # Optional override on the listener-pool plancher (the constant-
+    # in-N component of the SE). When set, half_width() uses
+    # ``plancher_override`` instead of computing it from
+    # ``sigma2_ga_estimate`` (or the conservative fallback). Typically
+    # set by the controller from a bootstrap-by-listeners resample,
+    # which estimates Var(C(pool)) directly and bypasses the MoM
+    # σ̂²_(γα) estimation. NOT subject to the peeking penalty (it is
+    # a constant-in-t variance component).
+    plancher_override: Optional[float] = None
+
+    _n: int = field(default=0, init=False, repr=False)
+    _mean: float = field(default=0.0, init=False, repr=False)
+    _M2: float = field(default=0.0, init=False, repr=False)
+
+    @property
+    def n(self) -> int:
+        return self._n
+
+    @property
+    def mean(self) -> float:
+        return self._mean
+
+    @property
+    def variance(self) -> float:
+        """Welford sample variance (unbiased, divisor n−1). 0 if n < 2."""
+        if self._n < 2:
+            return 0.0
+        return self._M2 / (self._n - 1)
+
+    def update(self, new_obs: Union[ArrayLike, float]) -> None:
+        """Ingest one or more per-item paired diffs (Welford streaming update)."""
+        arr = np.atleast_1d(np.asarray(new_obs, dtype=float)).ravel()
+        for x in arr:
+            self._n += 1
+            delta = x - self._mean
+            self._mean += delta / self._n
+            delta2 = x - self._mean
+            self._M2 += delta * delta2
+
+    def reset(self) -> None:
+        self._n = 0
+        self._mean = 0.0
+        self._M2 = 0.0
+
+    def _sigma2_proxy(self) -> float:
+        a, b = self.d_bounds
+        sigma2_hoeffding = ((b - a) / 2.0) ** 2
+        if self.proxy == "hoeffding":
+            return sigma2_hoeffding
+        if self.proxy == "ebernstein":
+            if self._n < self.t_min_ebernstein:
+                return sigma2_hoeffding
+            floor = self.ebernstein_floor_frac * sigma2_hoeffding
+            return max(self.variance, floor, 1e-12)
+        raise ValueError(f"unknown proxy: {self.proxy!r}")
+
+    def _psi_howard(self, sigma2: float) -> float:
+        """Howard et al. 2021 line-crossing half-width.
+
+        Ψ = σ · √( 2 (t·η² + 1) / (t²·η²) · log(√(t·η² + 1) / α) )
+
+        Pulled out as a separate method so :meth:`half_width` can compose
+        it with the listener-pool floor in quadrature.
+        """
+        if self._n < 1:
+            return float("inf")
+        t = self._n
+        eta2_t = (self.eta ** 2) * t
+        log_term = float(np.log(np.sqrt(eta2_t + 1.0) / self.alpha))
+        if log_term <= 0:
+            return float("inf")
+        rad = 2.0 * (eta2_t + 1.0) / (t * eta2_t) * log_term
+        return float(np.sqrt(sigma2) * np.sqrt(rad))
+
+    def plancher(self) -> float:
+        """Listener-pool floor √(2·σ²_(γα)/L_pool).
+
+        Two paths:
+        - If ``sigma2_ga_estimate`` is set (data-driven, from the
+          controller's MS_across−MS_within decomposition), use it
+          directly: plancher = √(2·σ̂²_(γα)/L_pool).
+        - Otherwise fall back to the conservative upper bound
+          ``c · σ̂²_proxy`` for σ²_(γα): plancher =
+          √(2·c·σ̂²_proxy/L_pool). This is the original V2 design;
+          used during warm-up or when the per-listener decomposition
+          isn't reliable.
+
+        Returns 0.0 when ``L_pool`` is None (assumed infinite pool).
+        """
+        if self.L_pool is None or self.L_pool <= 0:
+            return 0.0
+        if self.sigma2_ga_estimate is not None:
+            sigma2_ga = max(0.0, float(self.sigma2_ga_estimate))
+        else:
+            sigma2_ga = self.gamma_alpha_factor * self._sigma2_proxy()
+        return float(np.sqrt(2.0 * sigma2_ga / self.L_pool))
+
+    def _avci_multiplier(self) -> float:
+        """AVCI peeking-penalty multiplier such that
+        ``Ψ_Howard(σ², t) = (σ/√t) · _avci_multiplier(t)``.
+
+        Used by the ``se_override`` path: given an SE that already
+        captures the variance structure (bootstrap, lmer, etc.), the
+        AVCI band is ``Ψ = SE · _avci_multiplier(t)``. Asymptotically
+        ~ √(2·log(log t / α)) (loi du logarithme itéré).
+        """
+        if self._n < 1:
+            return float("inf")
+        t = self._n
+        eta2_t = (self.eta ** 2) * t
+        log_term = float(np.log(np.sqrt(eta2_t + 1.0) / self.alpha))
+        if log_term <= 0:
+            return float("inf")
+        return float(np.sqrt(2.0 * (eta2_t + 1.0) / eta2_t * log_term))
+
+    def half_width(self) -> float:
+        """Effective half-width.
+
+        Ψ_eff = √( Ψ_iid²  +  plancher² )
+
+        where the iid term gets the AVCI peeking-penalty multiplier and
+        the plancher does not (it is constant in t). Each component can
+        come from either MoM-Welford or a bootstrap override:
+
+        - ``se_iid_override`` set ⇒ ``Ψ_iid = se_iid · multiplier(t)``.
+          Otherwise ``Ψ_iid = Ψ_Howard(σ̂²_proxy_Welford, t)``.
+        - ``plancher_override`` set ⇒ ``plancher = plancher_override``.
+          Otherwise ``plancher = plancher()`` (MoM σ̂²_(γα) or fallback).
+        """
+        # iid component
+        if self.se_iid_override is not None:
+            multiplier = self._avci_multiplier()
+            if not np.isfinite(multiplier):
+                return multiplier
+            psi_iid = float(self.se_iid_override) * multiplier
+        else:
+            sigma2_proxy = self._sigma2_proxy()
+            psi_iid = self._psi_howard(sigma2_proxy)
+            if not np.isfinite(psi_iid):
+                return psi_iid
+
+        # plancher component
+        if self.plancher_override is not None:
+            plancher = float(self.plancher_override)
+        else:
+            plancher = self.plancher()
+
+        return float(np.sqrt(psi_iid * psi_iid + plancher * plancher))
+
+    def interval(self) -> tuple[float, float]:
+        psi = self.half_width()
+        return (self._mean - psi, self._mean + psi)
+
+    def verdict(self, delta_sep: float = 0.0,
+                delta_equiv: float | None = None,
+                effect_size_guard: bool = False) -> VERDICT:
+        """STOP / CONTINUE verdict on the AVCI confidence sequence.
+
+        Parameters
+        ----------
+        delta_sep : float, default 0.0
+            Used as the default value of ``delta_equiv`` when the latter
+            is None (single-threshold TOST behaviour). It is **NOT** a
+            STOP_SEP gate by default — that would be a minimum-effect
+            (TOST-sup) test, stricter than the classical anytime-valid
+            rejection. Use ``effect_size_guard=True`` to activate the
+            guard explicitly.
+        delta_equiv : float or None, default None
+            Equivalence margin for STOP_EQUIV. If None, falls back to
+            ``delta_sep`` (single-threshold default).
+        effect_size_guard : bool, default False
+            If True, STOP_SEP additionally requires
+            ``|mean| − Ψ_eff ≥ delta_sep`` (TOST-sup). Default off —
+            matches the classical anytime-valid rejection semantics
+            (0 ∉ CI_t).
+
+        Verdicts:
+
+        - STOP_SEP   : 0 ∉ CI_t. With ``effect_size_guard=True``, also
+                       requires ``|mean| − Ψ_eff ≥ delta_sep``.
+        - STOP_EQUIV : CI ⊂ (−delta_equiv, +delta_equiv) AND
+                       Ψ_eff ≤ delta_equiv/2
+                       (pair indistinguishable within tolerance)
+        - CONTINUE   : otherwise.
+        """
+        if delta_equiv is None:
+            delta_equiv = delta_sep
+        if self._n < 1:
+            return "CONTINUE"
+        psi = self.half_width()
+        lo, hi = self._mean - psi, self._mean + psi
+        # STOP_SEP: classical AVCI rejection — 0 ∉ CI_t.
+        # The effect-size guard is opt-in (TOST-sup semantics).
+        if lo > 0 or hi < 0:
+            if not effect_size_guard or abs(self._mean) - psi >= delta_sep:
+                return "STOP_SEP"
+        if (-delta_equiv < lo and hi < delta_equiv
+                and psi <= delta_equiv / 2.0):
+            return "STOP_EQUIV"
+        return "CONTINUE"
+
+
+if __name__ == "__main__":
+    rng = np.random.default_rng(42)
+
+    print("Sanity check — per-item AVCI on simulated d_j streams\n")
+
+    # H1: true δ = 0.3, σ²_d_j = 1 (MOS-ish scale, mid-effect).
+    # No pool floor (L_pool=None) — recovers the pure Howard 2021 boundary.
+    print("[H1, no floor] δ_true = 0.30, σ²_d = 1.00, MDE_threshold = 0.20, α = 0.05")
+    print(f"{'t':>5} {'mean':>8} {'Ψ':>8} {'CI_lo':>8} {'CI_hi':>8} {'verdict':>10}")
+    ci = AnytimeCI(alpha=0.05, proxy="ebernstein", eta=0.1, L_pool=None)
+    delta_true, sigma_true, mde_threshold = 0.30, 1.0, 0.20
+    for t in range(1, 301):
+        ci.update(rng.normal(delta_true, sigma_true))
+        if t in (1, 5, 10, 20, 50, 100, 200, 300):
+            psi = ci.half_width()
+            lo, hi = ci.interval()
+            v = ci.verdict(mde_threshold)
+            print(f"{t:>5d} {ci.mean:>8.3f} {psi:>8.3f} {lo:>8.3f} {hi:>8.3f} {v:>10}")
+
+    # H1: same setup but with L_pool=20 — floor should plateau Ψ at √(2·1·σ̂²/20)
+    print("\n[H1, L_pool=20, c=1.0] same setup but with listener-pool floor")
+    print(f"{'t':>5} {'mean':>8} {'Ψ':>8} {'plancher':>10} {'verdict':>10}")
+    ci = AnytimeCI(alpha=0.05, proxy="ebernstein", eta=0.1,
+                   L_pool=20, gamma_alpha_factor=1.0)
+    for t in range(1, 1001):
+        ci.update(rng.normal(delta_true, sigma_true))
+        if t in (1, 10, 50, 100, 300, 500, 1000):
+            psi = ci.half_width()
+            plancher = ci.plancher()
+            v = ci.verdict(mde_threshold)
+            print(f"{t:>5d} {ci.mean:>8.3f} {psi:>8.3f} "
+                  f"{plancher:>10.3f} {v:>10}")
+
+    # H0 coverage check
+    print("\n[H0 coverage, no floor] δ_true = 0, R = 200 replicates, T = 200")
+    print("       Verify: AVCI brackets 0 at every t in ≥ 95 % of replicates.")
+    R, T = 200, 200
+    n_falses = 0
+    for r in range(R):
+        ci = AnytimeCI(alpha=0.05, proxy="ebernstein", eta=0.1, L_pool=None)
+        crossed = False
+        for _ in range(T):
+            ci.update(rng.normal(0.0, 1.0))
+            lo, hi = ci.interval()
+            if lo > 0 or hi < 0:
+                crossed = True
+                break
+        if crossed:
+            n_falses += 1
+    coverage = 1.0 - n_falses / R
+    print(f"       Empirical coverage = {coverage:.3f}  "
+          f"(target ≥ 0.95)  — {n_falses} crossings out of {R}")