experimentpy 0.1.0__py3-none-any.whl

experimentpy/__init__.py

@@ -0,0 +1,9 @@
+"""Statistical tools for randomized online experiments."""
+
+from .design import SampleSizeResult, sample_size_proportions
+from .quality import SRMResult, check_srm
+
+__version__ = "0.1.0"
+__all__ = [
+    "SampleSizeResult", "SRMResult", "sample_size_proportions", "check_srm",
+]

experimentpy/_validation.py

@@ -0,0 +1,26 @@
+"""Shared input validation."""
+
+from math import isfinite
+from numbers import Integral, Real
+
+
+def finite_real(value: Real, name: str) -> float:
+    if isinstance(value, bool) or not isinstance(value, Real):
+        raise ValueError(f"{name} must be a finite real number.")
+    result = float(value)
+    if not isfinite(result):
+        raise ValueError(f"{name} must be a finite real number.")
+    return result
+
+
+def probability(value: Real, name: str) -> float:
+    result = finite_real(value, name)
+    if not 0 < result < 1:
+        raise ValueError(f"{name} must be strictly between 0 and 1.")
+    return result
+
+
+def count(value: Integral, name: str) -> int:
+    if isinstance(value, bool) or not isinstance(value, Integral) or value < 0:
+        raise ValueError(f"{name} must be a nonnegative integer count.")
+    return int(value)

experimentpy/design.py

@@ -0,0 +1,85 @@
+"""Fixed-horizon planning for two independent binary outcomes."""
+
+from dataclasses import dataclass
+from math import ceil, isfinite, sqrt
+
+from scipy.stats import norm
+
+from ._validation import finite_real, probability
+
+
+@dataclass(frozen=True)
+class SampleSizeResult:
+    """Required independent experimental units, rounded up by arm."""
+
+    n_control: int
+    n_variant: int
+    baseline: float
+    variant_rate: float
+    mde_absolute: float
+    alpha: float
+    power: float
+    ratio: float
+
+    @property
+    def total(self) -> int:
+        return self.n_control + self.n_variant
+
+
+def sample_size_proportions(
+    baseline: float,
+    mde_absolute: float,
+    *,
+    alpha: float = 0.05,
+    power: float = 0.80,
+    ratio: float = 1.0,
+) -> SampleSizeResult:
+    """Plan a two-sided, fixed-horizon, two-proportion z-test.
+
+    ``baseline`` and ``mde_absolute`` use proportions, not percentages:
+    baseline=0.05 and mde_absolute=0.005 plan for 5% versus 5.5%.
+    A negative MDE plans for a decrease. ``ratio`` is n_variant/n_control.
+
+    Uses a normal approximation with pooled null variance and unpooled
+    alternative variance, without continuity correction. Power is approximate.
+    Assumes independent Bernoulli units and a single planned comparison.
+    """
+    baseline = probability(baseline, "baseline")
+    alpha = probability(alpha, "alpha")
+    power = probability(power, "power")
+    if power <= max(0.5, alpha):
+        raise ValueError("power must exceed both 0.5 and alpha.")
+    mde_absolute = finite_real(mde_absolute, "mde_absolute")
+    if mde_absolute == 0:
+        raise ValueError("mde_absolute must be nonzero.")
+    variant = probability(baseline + mde_absolute, "baseline + mde_absolute")
+    ratio = finite_real(ratio, "ratio")
+    if ratio <= 0:
+        raise ValueError("ratio must be positive.")
+
+    # Weighted event rate across the two planned allocation sizes.
+    weight = ratio / (1.0 + ratio)
+    pooled = baseline * (1.0 - weight) + variant * weight
+    variance_null = pooled * (1.0 - pooled) * (1.0 + 1.0 / ratio)
+    variance_alternative = (
+        baseline * (1.0 - baseline) + variant * (1.0 - variant) / ratio
+    )
+    # isf avoids loss of precision from computing 1 - alpha/2.
+    critical = norm.isf(alpha / 2.0)
+    z_power = norm.ppf(power)
+    required = ((critical * sqrt(variance_null)
+                 + z_power * sqrt(variance_alternative)) / abs(mde_absolute)) ** 2
+    if not isfinite(required) or required <= 0:
+        raise ValueError("Inputs exceed the supported numerical range.")
+    n_control = max(1, ceil(required))
+    n_variant = max(1, ceil(ratio * n_control))
+    # Explicitly reject sparse designs instead of silently promising accuracy.
+    if min(n_control * baseline, n_control * (1 - baseline),
+           n_variant * variant, n_variant * (1 - variant)) < 5:
+        raise ValueError(
+            "The normal approximation requires at least 5 expected events "
+            "and non-events per arm; use an exact or simulation-based design."
+        )
+    return SampleSizeResult(
+        n_control, n_variant, baseline, variant, mde_absolute, alpha, power, ratio
+    )

experimentpy/quality.py

@@ -0,0 +1,74 @@
+"""Sample ratio mismatch checks on assignment counts."""
+
+from collections.abc import Mapping
+from dataclasses import dataclass
+from math import fsum, isclose
+
+from scipy.stats import chi2
+
+from ._validation import count, finite_real, probability
+
+
+@dataclass(frozen=True)
+class SRMResult:
+    """Pearson goodness-of-fit result for planned allocation proportions."""
+
+    labels: tuple[str, ...]
+    observed: tuple[int, ...]
+    expected: tuple[float, ...]
+    statistic: float
+    pvalue: float
+    degrees_of_freedom: int
+    alpha: float
+
+    @property
+    def has_srm(self) -> bool:
+        return self.pvalue < self.alpha
+
+
+def check_srm(
+    observed: Mapping[str, int],
+    allocation: Mapping[str, float] | None = None,
+    *,
+    alpha: float = 0.001,
+) -> SRMResult:
+    """Check assignment counts against a prespecified allocation.
+
+    Defaults to equal allocation. Allocation values must be positive
+    proportions summing to 1; keys must match observed exactly. Supports
+    two or more arms. Uses Pearson's chi-square with k-1 degrees of freedom.
+
+    Count each independent randomized unit once, before post-treatment
+    filters. A nonsignificant result does not establish experiment validity.
+    Raises ValueError if any expected arm count is below 5.
+    """
+    alpha = probability(alpha, "alpha")
+    if not isinstance(observed, Mapping) or len(observed) < 2:
+        raise ValueError("observed must map at least two arm labels to counts.")
+    labels = tuple(observed)
+    if any(not isinstance(label, str) or not label for label in labels):
+        raise ValueError("Arm labels must be nonempty strings.")
+    counts = tuple(count(observed[label], label) for label in labels)
+    total = sum(counts)
+    if total == 0:
+        raise ValueError("The total observed count must be positive.")
+    if allocation is None:
+        weights = (1.0 / len(labels),) * len(labels)
+    else:
+        if not isinstance(allocation, Mapping) or set(allocation) != set(labels):
+            raise ValueError("allocation and observed must have identical arm labels.")
+        weights = tuple(finite_real(allocation[label], label) for label in labels)
+        if any(weight <= 0 for weight in weights):
+            raise ValueError("Allocation proportions must be positive.")
+        if not isclose(fsum(weights), 1.0, rel_tol=0, abs_tol=1e-12):
+            raise ValueError("Allocation proportions must sum to 1.")
+        # Remove harmless floating-point summation error only.
+        weights = tuple(weight / fsum(weights) for weight in weights)
+    expected = tuple(total * weight for weight in weights)
+    if min(expected) < 5:
+        raise ValueError("Chi-square approximation requires expected counts >= 5 per arm.")
+    statistic = fsum((actual - target) ** 2 / target
+                     for actual, target in zip(counts, expected))
+    df = len(labels) - 1
+    pvalue = float(chi2.sf(statistic, df))
+    return SRMResult(labels, counts, expected, statistic, pvalue, df, alpha)

experimentpy-0.1.0.dist-info/METADATA

@@ -0,0 +1,136 @@
+Metadata-Version: 2.4
+Name: experimentpy
+Version: 0.1.0
+Summary: Statistical tools for online experimentation: sample size and sample ratio mismatch.
+Author: Ubaldo Hervas
+License-Expression: MIT
+Keywords: ab-testing,experimentation,sample-size,srm,statistics
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Topic :: Scientific/Engineering :: Mathematics
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: scipy>=1.10
+Provides-Extra: dev
+Requires-Dist: build>=1.2; extra == "dev"
+Requires-Dist: twine>=5; extra == "dev"
+Dynamic: license-file
+
+# ExperimentPy
+
+Statistical tools for online experimentation.
+
+**Version 0.1.0** provides two working tools: sample-size planning for binary
+A/B metrics and sample ratio mismatch (SRM) checks. This library concerns
+statistical experiments; it is independent of the `pytest` software-testing framework.
+
+## Install
+
+Python 3.10 or newer is required. SciPy is the only direct runtime dependency.
+
+```bash
+pip install experimentpy
+```
+
+That command becomes available after publication on PyPI. To install the
+included wheel before publication, run this from the extracted project folder:
+
+```bash
+python -m pip install dist/experimentpy-0.1.0-py3-none-any.whl
+```
+
+## Sample size for a binary A/B metric
+
+```python
+from experimentpy import sample_size_proportions
+
+plan = sample_size_proportions(
+    baseline=0.05,        # 5% control conversion rate
+    mde_absolute=0.005,  # +0.5 percentage points: 5% -> 5.5%
+    alpha=0.05,
+    power=0.80,
+    ratio=1.0,           # variant units / control units
+)
+print(plan.n_control, plan.n_variant, plan.total)
+```
+
+An MDE of `0.005` means **0.5 percentage points**, or a **10% relative lift**
+on a 5% baseline. For a relative MDE, calculate `baseline * relative_lift`
+before calling the function. Negative absolute MDEs plan for decreases.
+
+The design uses a two-sided normal approximation for two independent
+proportions, pooled variance under the null and unpooled variance under the
+alternative. With `p0=baseline`, `p1=p0+mde_absolute`, and
+`r=n_variant/n_control`, the unrounded control sample size is:
+
+```text
+p_bar = (p0 + r*p1) / (1+r)
+n_control = (
+    (z_(1-alpha/2)*sqrt(p_bar*(1-p_bar)*(1+1/r))
+     + z_power*sqrt(p0*(1-p0) + p1*(1-p1)/r)) / abs(p1-p0)
+)**2
+```
+
+Arm sizes are rounded up. Power is approximate; there is no continuity
+correction. Designs with fewer than five expected events or non-events in
+either arm are rejected. This is a single-comparison, fixed-horizon design:
+repeated peeking, multiple comparisons and cluster randomization require
+different procedures. Users, sessions or other units must match the unit
+of independent randomization.
+
+## Sample ratio mismatch
+
+```python
+from experimentpy import check_srm
+
+quality = check_srm(
+    observed={"control": 5100, "variant": 4900},
+    allocation={"control": 0.5, "variant": 0.5},
+    alpha=0.001,
+)
+print(f"SRM p-value: {quality.pvalue:.4f}")
+print(f"SRM detected: {quality.has_srm}")
+```
+
+Pearson's goodness-of-fit test compares observed counts with the prespecified
+allocation, with `number_of_arms - 1` degrees of freedom. Omit `allocation`
+for equal allocation. Two or more arms are supported. Allocation proportions
+must be positive and sum to one, and expected counts must be at least five
+per arm. Counts must be nonnegative integers, not event totals or weights.
+
+Use each randomized unit once and avoid post-treatment filters. SRM signals
+an allocation discrepancy; it does not identify its cause. Absence of SRM
+does not establish overall experiment validity. The default SRM alpha is
+0.001 and is separate from the outcome-analysis alpha.
+
+Results are dataclasses and can be exported with `dataclasses.asdict(result)`.
+
+## Scope and next steps
+
+This release implements sample-size planning and SRM only. Outcome p-values,
+effect intervals, practical significance, variance reduction and
+quasi-experimental methods are planned extensions, not implemented APIs.
+
+## Development
+
+```bash
+python -m pip install -e ".[dev]"
+python -m unittest discover -s tests -v
+python -m build
+python -m twine check dist/*
+```
+
+Tests use Python's standard `unittest` framework; no `pytest` dependency is
+needed. See `examples/quickstart.py` for an executable example.
+
+## References
+
+- [SciPy Pearson chi-square documentation](https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.chisquare.html)
+- [statsmodels two-proportion power documentation](https://www.statsmodels.org/stable/generated/statsmodels.stats.proportion.power_proportions_2indep.html)
+
+## License
+
+MIT. Copyright (c) 2026 Ubaldo Hervas.

experimentpy-0.1.0.dist-info/RECORD

@@ -0,0 +1,10 @@
+experimentpy/__init__.py,sha256=uiDkkRA3mDaDUi9kUgha_TvcAlb4-j-pHazOxZxJVQg,278
+experimentpy/_validation.py,sha256=GYQiD3nQQ-M7X9woxMy8rKlORFSaIZjP-tYehdYiMVQ,841
+experimentpy/design.py,sha256=5efQ1lSPU4HZugHMHrUeU9geXHdtJ1K4CmNC9Hml4jU,3156
+experimentpy/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+experimentpy/quality.py,sha256=7zYykXBIxgmQslFxLfAeLtAX9T_B7g_WNrOZFteWb9o,2997
+experimentpy-0.1.0.dist-info/licenses/LICENSE,sha256=aebUfBenioyYLiBCcFPNF4RcQ1hScWz2HeTb37An0cw,1070
+experimentpy-0.1.0.dist-info/METADATA,sha256=zJS_bQEvogFo2zfNvIZcS6tu664hJWHQ7D3Vx8nSz2g,4791
+experimentpy-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
+experimentpy-0.1.0.dist-info/top_level.txt,sha256=p5lQGMpMVbm8_G6zf3s2aZ7qMJazweFA6Wa7Ku9F5gM,13
+experimentpy-0.1.0.dist-info/RECORD,,

experimentpy-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.2.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

experimentpy-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Ubaldo Hervas
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

experimentpy-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+experimentpy