experimentation-toolkit 0.1.0__tar.gz

experimentation_toolkit-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 datenwissenschaften
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

experimentation_toolkit-0.1.0/PKG-INFO

@@ -0,0 +1,213 @@
+Metadata-Version: 2.4
+Name: experimentation-toolkit
+Version: 0.1.0
+Summary: Typed, statistically explicit frequentist analysis for controlled A/B experiments
+License-Expression: MIT
+License-File: LICENSE
+Author: datenwissenschaften
+Author-email: martin.franke@datenwissenschaften.com
+Requires-Python: >=3.12,<4.0
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Mathematics
+Classifier: Typing :: Typed
+Provides-Extra: dev
+Provides-Extra: examples
+Requires-Dist: ipykernel (>=6.29,<8.0) ; extra == "examples"
+Requires-Dist: kaleido (>=1.1,<2.0) ; extra == "examples"
+Requires-Dist: mypy (>=1.11,<2.0) ; extra == "dev"
+Requires-Dist: nbclient (>=0.10,<1.0) ; extra == "examples"
+Requires-Dist: nbformat (>=5.10,<6.0) ; extra == "examples"
+Requires-Dist: numpy (>=2.0,<3.0)
+Requires-Dist: plotly (>=6.0,<7.0) ; extra == "examples"
+Requires-Dist: pydantic (>=2.8,<3.0)
+Requires-Dist: pytest (>=8.3,<9.0) ; extra == "dev"
+Requires-Dist: pytest-cov (>=5.0,<7.0) ; extra == "dev"
+Requires-Dist: ruff (>=0.6,<1.0) ; extra == "dev"
+Requires-Dist: scipy (>=1.13,<2.0)
+Requires-Dist: scipy-stubs (>=1.15,<2.0) ; extra == "dev"
+Project-URL: Repository, https://github.com/datenwissenschaften/experimentation-toolkit
+Description-Content-Type: text/markdown
+
+# experimentation-toolkit
+
+[![CI](https://github.com/datenwissenschaften/experimentation-toolkit/actions/workflows/ci.yml/badge.svg)](https://github.com/datenwissenschaften/experimentation-toolkit/actions/workflows/ci.yml) [![PyPI](https://img.shields.io/pypi/v/experimentation-toolkit.svg)](https://pypi.org/project/experimentation-toolkit/) ![Python 3.12](https://img.shields.io/badge/python-3.12-blue.svg) [![Last commit](https://img.shields.io/github/last-commit/datenwissenschaften/experimentation-toolkit)](https://github.com/datenwissenschaften/experimentation-toolkit/commits/main) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE) ![Ruff](https://img.shields.io/badge/lint-Ruff-D7FF64.svg) ![mypy](https://img.shields.io/badge/types-mypy-2A6DB2.svg) [![codecov](https://codecov.io/gh/datenwissenschaften/experimentation-toolkit/graph/badge.svg)](https://codecov.io/gh/datenwissenschaften/experimentation-toolkit)
+
+`experimentation-toolkit` is a typed Python library for classical frequentist analysis of
+independent, two-arm controlled experiments. It connects assignment diagnostics, metric-level
+inference, multiplicity correction, power planning, and a serializable report without making a
+DataFrame the core abstraction.
+
+The package reports estimates, uncertainty, and statistical evidence. It deliberately does not
+turn a small p-value into a claim that an effect is important, beneficial, or worth shipping.
+
+## Quick example
+
+```python
+from experimentation_toolkit import (
+    Experiment,
+    ExperimentReport,
+    analyze_proportion,
+    check_sample_ratio_mismatch,
+)
+
+experiment = Experiment(control="control", treatment="treatment")
+srm = check_sample_ratio_mismatch(
+    observed_counts={"control": 10_000, "treatment": 10_050},
+    expected_proportions={"control": 0.5, "treatment": 0.5},
+)
+conversion = analyze_proportion(
+    control_successes=1_200,
+    control_total=10_000,
+    treatment_successes=1_280,
+    treatment_total=10_050,
+    confidence_level=0.95,
+    alternative="two-sided",
+    metric_name="conversion",
+)
+report = ExperimentReport(
+    experiment=experiment,
+    analyses=(conversion,),
+    diagnostics=(srm,),
+)
+print(report.model_dump_json(indent=2))
+```
+
+The absolute effect is a risk difference. A move from 10% to 11% is `0.01`, or **one
+percentage point**. Its relative lift is `0.10`, or **10% relative**. Results expose these as
+separate fields with an explicit relative-effect definition.
+
+## Supported analyses
+
+- Independent binary outcomes from success counts and totals
+- Independent continuous outcomes from numeric array-like samples
+- Two-sided, treatment-greater, and treatment-less alternatives
+- Sample-ratio-mismatch checks for two or more assignment groups
+- Proportion sample-size planning and achieved-power approximation
+- Bonferroni, Holm, and Benjamini-Hochberg p-value adjustment
+- Immutable Pydantic models and standards-compliant JSON reports
+
+## Statistical methods
+
+Proportion inference uses a pooled two-sample score z-test for the point null
+`p_treatment - p_control = 0`. The risk-difference confidence interval is Newcombe's hybrid
+score interval formed from Wilson limits without continuity correction. It remains useful at
+zero and one, while an explicit warning flags sparse expected cells where the test's normal
+approximation may be inaccurate.
+
+Continuous inference uses Welch's unequal-variance two-sample t-test and the
+Welch-Satterthwaite degrees of freedom. It does not assume equal group variances. Both analyses
+define effects as treatment minus control and align one-sided confidence bounds with the chosen
+alternative. An unbounded endpoint is serialized as `null`, never infinity.
+
+See [docs/statistical-methods.md](docs/statistical-methods.md) for formulas, assumptions, and
+references.
+
+## Assumptions
+
+Inference assumes randomized or otherwise exchangeable groups, independent experimental units,
+a metric definition fixed independently of observed treatment results, and no interference
+between units. Binary observations must be Bernoulli outcomes at the analysis unit. Welch
+inference assumes finite variance and is exact under normal sampling; with larger samples it
+uses the sampling distribution of the mean. The library cannot establish that these assumptions
+hold from arrays or counts alone.
+
+## Sample ratio mismatch
+
+`check_sample_ratio_mismatch` applies Pearson's chi-square goodness-of-fit test to observed arm
+counts and configured allocation probabilities. `FAIL` means the allocation is statistically
+inconsistent with the plan at the selected threshold. It indicates a potential randomization,
+eligibility, logging, or instrumentation problem; it does not identify the cause. Inspect SRM
+before interpreting treatment effects.
+
+## Power and sample size
+
+```python
+from experimentation_toolkit import proportion_sample_size
+
+plan = proportion_sample_size(
+    baseline_rate=0.10,
+    minimum_detectable_effect=0.01,
+    mde_type="absolute",  # +1 percentage point
+    alpha=0.05,
+    power=0.80,
+    allocation_ratio=1.0,  # treatment / control
+)
+```
+
+Set `mde_type="relative"` to express the MDE as a fraction of baseline: `0.10` at a 10%
+baseline targets an 11% treatment rate. The calculation is a fixed-horizon normal approximation,
+and the returned achieved power reflects the integer group sizes. Planning inputs are assumptions,
+not estimates guaranteed to hold after launch.
+
+## Multiple testing
+
+`adjust_p_values` preserves original hypothesis order and optionally carries labels. Bonferroni
+and Holm control the family-wise error rate. Benjamini-Hochberg controls the false discovery rate
+under independence or positive dependence and must not be interpreted as FWER control. The caller
+defines the hypothesis family before looking at results.
+
+## Interpretation
+
+Statistical significance quantifies incompatibility with a null model under its assumptions.
+Practical significance requires domain context, costs, guardrails, and a pre-specified decision
+rule. Read the absolute effect and its confidence interval first. Relative effects are omitted
+when the control denominator is zero or when continuous means do not support a conservative,
+non-negative and numerically resolved ratio-scale interpretation.
+
+## Installation
+
+Python 3.12 is required.
+
+```bash
+python -m pip install experimentation-toolkit
+```
+
+For a source checkout:
+
+```bash
+python -m pip install -e '.[dev]'
+```
+
+To execute the visual notebook, install the separate example dependencies:
+
+```bash
+python -m pip install -e '.[dev,examples]'
+```
+
+## Testing and development
+
+```bash
+ruff check .
+ruff format --check .
+mypy
+pytest
+pytest --cov=experimentation_toolkit --cov-report=term-missing
+```
+
+The deterministic [game onboarding notebook](examples/game_onboarding.ipynb) demonstrates
+assignment counts, SRM, a primary completion metric, retention and session-duration metrics, Holm
+correction, Plotly-generated uncertainty plots rendered as GitHub-compatible PNGs, and report
+serialization. It is self-contained, uses a fixed seed, and labels all generated observations as
+synthetic.
+
+## Limitations
+
+Version 0.1 supports independent, two-arm, fixed-horizon analyses only. It does not implement
+paired or clustered designs, exact unconditional proportion tests, covariate adjustment, CUPED,
+sequential monitoring, alpha spending, Bayesian analysis, causal identification, heterogeneous
+effects, feature flags, or platform infrastructure. The proportion test and power methods use
+large-sample approximations; warnings do not convert them into exact procedures.
+
+## Roadmap
+
+Statistically useful next steps include stratified analyses with explicit estimands, cluster-aware
+standard errors and design effects, robust or transformed continuous outcomes, exact methods for
+sparse binary data, and simulation-based validation of planning assumptions. Each belongs behind
+a distinct API that states its sampling unit and assumptions.
+
+## License
+
+MIT
+

experimentation_toolkit-0.1.0/README.md

@@ -0,0 +1,180 @@
+# experimentation-toolkit
+
+[![CI](https://github.com/datenwissenschaften/experimentation-toolkit/actions/workflows/ci.yml/badge.svg)](https://github.com/datenwissenschaften/experimentation-toolkit/actions/workflows/ci.yml) [![PyPI](https://img.shields.io/pypi/v/experimentation-toolkit.svg)](https://pypi.org/project/experimentation-toolkit/) ![Python 3.12](https://img.shields.io/badge/python-3.12-blue.svg) [![Last commit](https://img.shields.io/github/last-commit/datenwissenschaften/experimentation-toolkit)](https://github.com/datenwissenschaften/experimentation-toolkit/commits/main) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE) ![Ruff](https://img.shields.io/badge/lint-Ruff-D7FF64.svg) ![mypy](https://img.shields.io/badge/types-mypy-2A6DB2.svg) [![codecov](https://codecov.io/gh/datenwissenschaften/experimentation-toolkit/graph/badge.svg)](https://codecov.io/gh/datenwissenschaften/experimentation-toolkit)
+
+`experimentation-toolkit` is a typed Python library for classical frequentist analysis of
+independent, two-arm controlled experiments. It connects assignment diagnostics, metric-level
+inference, multiplicity correction, power planning, and a serializable report without making a
+DataFrame the core abstraction.
+
+The package reports estimates, uncertainty, and statistical evidence. It deliberately does not
+turn a small p-value into a claim that an effect is important, beneficial, or worth shipping.
+
+## Quick example
+
+```python
+from experimentation_toolkit import (
+    Experiment,
+    ExperimentReport,
+    analyze_proportion,
+    check_sample_ratio_mismatch,
+)
+
+experiment = Experiment(control="control", treatment="treatment")
+srm = check_sample_ratio_mismatch(
+    observed_counts={"control": 10_000, "treatment": 10_050},
+    expected_proportions={"control": 0.5, "treatment": 0.5},
+)
+conversion = analyze_proportion(
+    control_successes=1_200,
+    control_total=10_000,
+    treatment_successes=1_280,
+    treatment_total=10_050,
+    confidence_level=0.95,
+    alternative="two-sided",
+    metric_name="conversion",
+)
+report = ExperimentReport(
+    experiment=experiment,
+    analyses=(conversion,),
+    diagnostics=(srm,),
+)
+print(report.model_dump_json(indent=2))
+```
+
+The absolute effect is a risk difference. A move from 10% to 11% is `0.01`, or **one
+percentage point**. Its relative lift is `0.10`, or **10% relative**. Results expose these as
+separate fields with an explicit relative-effect definition.
+
+## Supported analyses
+
+- Independent binary outcomes from success counts and totals
+- Independent continuous outcomes from numeric array-like samples
+- Two-sided, treatment-greater, and treatment-less alternatives
+- Sample-ratio-mismatch checks for two or more assignment groups
+- Proportion sample-size planning and achieved-power approximation
+- Bonferroni, Holm, and Benjamini-Hochberg p-value adjustment
+- Immutable Pydantic models and standards-compliant JSON reports
+
+## Statistical methods
+
+Proportion inference uses a pooled two-sample score z-test for the point null
+`p_treatment - p_control = 0`. The risk-difference confidence interval is Newcombe's hybrid
+score interval formed from Wilson limits without continuity correction. It remains useful at
+zero and one, while an explicit warning flags sparse expected cells where the test's normal
+approximation may be inaccurate.
+
+Continuous inference uses Welch's unequal-variance two-sample t-test and the
+Welch-Satterthwaite degrees of freedom. It does not assume equal group variances. Both analyses
+define effects as treatment minus control and align one-sided confidence bounds with the chosen
+alternative. An unbounded endpoint is serialized as `null`, never infinity.
+
+See [docs/statistical-methods.md](docs/statistical-methods.md) for formulas, assumptions, and
+references.
+
+## Assumptions
+
+Inference assumes randomized or otherwise exchangeable groups, independent experimental units,
+a metric definition fixed independently of observed treatment results, and no interference
+between units. Binary observations must be Bernoulli outcomes at the analysis unit. Welch
+inference assumes finite variance and is exact under normal sampling; with larger samples it
+uses the sampling distribution of the mean. The library cannot establish that these assumptions
+hold from arrays or counts alone.
+
+## Sample ratio mismatch
+
+`check_sample_ratio_mismatch` applies Pearson's chi-square goodness-of-fit test to observed arm
+counts and configured allocation probabilities. `FAIL` means the allocation is statistically
+inconsistent with the plan at the selected threshold. It indicates a potential randomization,
+eligibility, logging, or instrumentation problem; it does not identify the cause. Inspect SRM
+before interpreting treatment effects.
+
+## Power and sample size
+
+```python
+from experimentation_toolkit import proportion_sample_size
+
+plan = proportion_sample_size(
+    baseline_rate=0.10,
+    minimum_detectable_effect=0.01,
+    mde_type="absolute",  # +1 percentage point
+    alpha=0.05,
+    power=0.80,
+    allocation_ratio=1.0,  # treatment / control
+)
+```
+
+Set `mde_type="relative"` to express the MDE as a fraction of baseline: `0.10` at a 10%
+baseline targets an 11% treatment rate. The calculation is a fixed-horizon normal approximation,
+and the returned achieved power reflects the integer group sizes. Planning inputs are assumptions,
+not estimates guaranteed to hold after launch.
+
+## Multiple testing
+
+`adjust_p_values` preserves original hypothesis order and optionally carries labels. Bonferroni
+and Holm control the family-wise error rate. Benjamini-Hochberg controls the false discovery rate
+under independence or positive dependence and must not be interpreted as FWER control. The caller
+defines the hypothesis family before looking at results.
+
+## Interpretation
+
+Statistical significance quantifies incompatibility with a null model under its assumptions.
+Practical significance requires domain context, costs, guardrails, and a pre-specified decision
+rule. Read the absolute effect and its confidence interval first. Relative effects are omitted
+when the control denominator is zero or when continuous means do not support a conservative,
+non-negative and numerically resolved ratio-scale interpretation.
+
+## Installation
+
+Python 3.12 is required.
+
+```bash
+python -m pip install experimentation-toolkit
+```
+
+For a source checkout:
+
+```bash
+python -m pip install -e '.[dev]'
+```
+
+To execute the visual notebook, install the separate example dependencies:
+
+```bash
+python -m pip install -e '.[dev,examples]'
+```
+
+## Testing and development
+
+```bash
+ruff check .
+ruff format --check .
+mypy
+pytest
+pytest --cov=experimentation_toolkit --cov-report=term-missing
+```
+
+The deterministic [game onboarding notebook](examples/game_onboarding.ipynb) demonstrates
+assignment counts, SRM, a primary completion metric, retention and session-duration metrics, Holm
+correction, Plotly-generated uncertainty plots rendered as GitHub-compatible PNGs, and report
+serialization. It is self-contained, uses a fixed seed, and labels all generated observations as
+synthetic.
+
+## Limitations
+
+Version 0.1 supports independent, two-arm, fixed-horizon analyses only. It does not implement
+paired or clustered designs, exact unconditional proportion tests, covariate adjustment, CUPED,
+sequential monitoring, alpha spending, Bayesian analysis, causal identification, heterogeneous
+effects, feature flags, or platform infrastructure. The proportion test and power methods use
+large-sample approximations; warnings do not convert them into exact procedures.
+
+## Roadmap
+
+Statistically useful next steps include stratified analyses with explicit estimands, cluster-aware
+standard errors and design effects, robust or transformed continuous outcomes, exact methods for
+sparse binary data, and simulation-based validation of planning assumptions. Each belongs behind
+a distinct API that states its sampling unit and assumptions.
+
+## License
+
+MIT

experimentation_toolkit-0.1.0/pyproject.toml

@@ -0,0 +1,79 @@
+[project]
+name = "experimentation-toolkit"
+version = "0.1.0"
+description = "Typed, statistically explicit frequentist analysis for controlled A/B experiments"
+readme = "README.md"
+license = "MIT"
+license-files = ["LICENSE"]
+authors = [
+    { name = "datenwissenschaften", email = "martin.franke@datenwissenschaften.com" },
+]
+requires-python = ">=3.12,<4.0"
+dependencies = [
+    "numpy>=2.0,<3.0",
+    "scipy>=1.13,<2.0",
+    "pydantic>=2.8,<3.0"
+]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Science/Research",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Scientific/Engineering :: Mathematics",
+    "Typing :: Typed",
+]
+
+[project.optional-dependencies]
+dev = [
+    "mypy>=1.11,<2.0",
+    "pytest>=8.3,<9.0",
+    "pytest-cov>=5.0,<7.0",
+    "ruff>=0.6,<1.0",
+    "scipy-stubs>=1.15,<2.0",
+]
+examples = [
+    "ipykernel>=6.29,<8.0",
+    "nbclient>=0.10,<1.0",
+    "kaleido>=1.1,<2.0",
+    "nbformat>=5.10,<6.0",
+    "plotly>=6.0,<7.0",
+]
+
+[project.urls]
+Repository = "https://github.com/datenwissenschaften/experimentation-toolkit"
+
+[build-system]
+requires = ["poetry-core>=2.0.0,<3.0.0"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.poetry]
+packages = [{ include = "experimentation_toolkit", from = "src" }]
+
+[tool.ruff]
+target-version = "py312"
+line-length = 100
+src = ["src", "tests"]
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "B", "UP", "SIM", "RUF"]
+
+[tool.ruff.format]
+quote-style = "double"
+
+[tool.mypy]
+python_version = "3.12"
+strict = true
+files = ["src", "examples"]
+plugins = ["pydantic.mypy"]
+
+[tool.pytest.ini_options]
+addopts = "--strict-markers --strict-config -ra"
+testpaths = ["tests"]
+pythonpath = ["."]
+
+[tool.coverage.run]
+branch = true
+source = ["experimentation_toolkit"]
+
+[tool.coverage.report]
+show_missing = true
+skip_covered = true

experimentation_toolkit-0.1.0/src/experimentation_toolkit/__init__.py

@@ -0,0 +1,51 @@
+"""Statistically explicit tools for classical controlled A/B experiments."""
+
+from experimentation_toolkit.analysis import analyze_continuous, analyze_proportion
+from experimentation_toolkit.diagnostics import check_sample_ratio_mismatch
+from experimentation_toolkit.models import (
+    AchievedPowerResult,
+    AlternativeHypothesis,
+    AnalysisResult,
+    ConfidenceInterval,
+    CorrectionMethod,
+    DiagnosticResult,
+    DiagnosticStatus,
+    Experiment,
+    ExperimentReport,
+    MDEType,
+    Metric,
+    MetricType,
+    MultipleTestingResult,
+    ProportionSampleSizeResult,
+    SRMResult,
+    Variant,
+)
+from experimentation_toolkit.multiple_testing import adjust_p_values
+from experimentation_toolkit.power import proportion_achieved_power, proportion_sample_size
+
+__all__ = [
+    "AchievedPowerResult",
+    "AlternativeHypothesis",
+    "AnalysisResult",
+    "ConfidenceInterval",
+    "CorrectionMethod",
+    "DiagnosticResult",
+    "DiagnosticStatus",
+    "Experiment",
+    "ExperimentReport",
+    "MDEType",
+    "Metric",
+    "MetricType",
+    "MultipleTestingResult",
+    "ProportionSampleSizeResult",
+    "SRMResult",
+    "Variant",
+    "adjust_p_values",
+    "analyze_continuous",
+    "analyze_proportion",
+    "check_sample_ratio_mismatch",
+    "proportion_achieved_power",
+    "proportion_sample_size",
+]
+
+__version__ = "0.1.0"

experimentation_toolkit-0.1.0/src/experimentation_toolkit/analysis/__init__.py

@@ -0,0 +1,6 @@
+"""Metric analysis functions."""
+
+from experimentation_toolkit.analysis.continuous import analyze_continuous
+from experimentation_toolkit.analysis.proportion import analyze_proportion
+
+__all__ = ["analyze_continuous", "analyze_proportion"]