experimentation-toolkit 0.1.0__py3-none-any.whl

experimentation_toolkit/__init__.py

@@ -0,0 +1,51 @@
+"""Statistically explicit tools for classical controlled A/B experiments."""
+
+from experimentation_toolkit.analysis import analyze_continuous, analyze_proportion
+from experimentation_toolkit.diagnostics import check_sample_ratio_mismatch
+from experimentation_toolkit.models import (
+    AchievedPowerResult,
+    AlternativeHypothesis,
+    AnalysisResult,
+    ConfidenceInterval,
+    CorrectionMethod,
+    DiagnosticResult,
+    DiagnosticStatus,
+    Experiment,
+    ExperimentReport,
+    MDEType,
+    Metric,
+    MetricType,
+    MultipleTestingResult,
+    ProportionSampleSizeResult,
+    SRMResult,
+    Variant,
+)
+from experimentation_toolkit.multiple_testing import adjust_p_values
+from experimentation_toolkit.power import proportion_achieved_power, proportion_sample_size
+
+__all__ = [
+    "AchievedPowerResult",
+    "AlternativeHypothesis",
+    "AnalysisResult",
+    "ConfidenceInterval",
+    "CorrectionMethod",
+    "DiagnosticResult",
+    "DiagnosticStatus",
+    "Experiment",
+    "ExperimentReport",
+    "MDEType",
+    "Metric",
+    "MetricType",
+    "MultipleTestingResult",
+    "ProportionSampleSizeResult",
+    "SRMResult",
+    "Variant",
+    "adjust_p_values",
+    "analyze_continuous",
+    "analyze_proportion",
+    "check_sample_ratio_mismatch",
+    "proportion_achieved_power",
+    "proportion_sample_size",
+]
+
+__version__ = "0.1.0"

experimentation_toolkit/analysis/__init__.py

@@ -0,0 +1,6 @@
+"""Metric analysis functions."""
+
+from experimentation_toolkit.analysis.continuous import analyze_continuous
+from experimentation_toolkit.analysis.proportion import analyze_proportion
+
+__all__ = ["analyze_continuous", "analyze_proportion"]

experimentation_toolkit/analysis/continuous.py

@@ -0,0 +1,255 @@
+"""Welch analysis for independent continuous outcomes."""
+
+from math import hypot, sqrt
+from typing import Literal
+
+import numpy as np
+import numpy.typing as npt
+from scipy import stats
+
+from experimentation_toolkit.models.enums import (
+    AlternativeHypothesis,
+    DiagnosticStatus,
+    MetricType,
+)
+from experimentation_toolkit.models.results import (
+    AnalysisResult,
+    ConfidenceInterval,
+    DiagnosticResult,
+)
+from experimentation_toolkit.validation.inputs import (
+    normalize_alternative,
+    normalize_numeric_sample,
+    validate_probability,
+)
+
+
+def _stable_mean_and_standard_deviation(
+    values: npt.NDArray[np.float64],
+) -> tuple[float, float]:
+    """Compute float64 moments after scaling to avoid avoidable overflow/underflow."""
+    scale = float(np.max(np.abs(values)))
+    if scale == 0.0:
+        return 0.0, 0.0
+    scaled = values / scale
+    scaled_mean = float(np.mean(scaled, dtype=np.float64))
+    centered = scaled - scaled_mean
+    scaled_sum_of_squares = float(np.dot(centered, centered))
+    mean = scaled_mean * scale
+    standard_deviation = scale * sqrt(scaled_sum_of_squares / (values.size - 1))
+    if not np.isfinite(mean) or not np.isfinite(standard_deviation):
+        raise ValueError("sample moments are non-finite; rescale extreme observations")
+    return mean, standard_deviation
+
+
+def _welch_degrees_of_freedom(
+    control_standard_error: float,
+    control_size: int,
+    treatment_standard_error: float,
+    treatment_size: int,
+) -> float:
+    scale = max(control_standard_error, treatment_standard_error)
+    control_term = (control_standard_error / scale) ** 2
+    treatment_term = (treatment_standard_error / scale) ** 2
+    return (control_term + treatment_term) ** 2 / (
+        control_term**2 / (control_size - 1) + treatment_term**2 / (treatment_size - 1)
+    )
+
+
+def _welch_interval(
+    effect: float,
+    standard_error: float,
+    degrees_of_freedom: float,
+    confidence_level: float,
+    alternative: AlternativeHypothesis,
+) -> ConfidenceInterval:
+    alpha = 1.0 - confidence_level
+    tail_probability = alpha / 2.0 if alternative is AlternativeHypothesis.TWO_SIDED else alpha
+    critical_value = float(stats.t.ppf(1.0 - tail_probability, degrees_of_freedom))
+    margin = critical_value * standard_error
+    if alternative is AlternativeHypothesis.GREATER:
+        return ConfidenceInterval(
+            lower=effect - margin, upper=None, confidence_level=confidence_level
+        )
+    if alternative is AlternativeHypothesis.LESS:
+        return ConfidenceInterval(
+            lower=None, upper=effect + margin, confidence_level=confidence_level
+        )
+    return ConfidenceInterval(
+        lower=effect - margin,
+        upper=effect + margin,
+        confidence_level=confidence_level,
+    )
+
+
+def analyze_continuous(
+    *,
+    control: npt.ArrayLike,
+    treatment: npt.ArrayLike,
+    confidence_level: float = 0.95,
+    alternative: AlternativeHypothesis | str = AlternativeHypothesis.TWO_SIDED,
+    nonfinite_policy: Literal["raise", "omit"] = "raise",
+    metric_name: str | None = None,
+) -> AnalysisResult:
+    """Analyze independent means using Welch's unequal-variance t procedure.
+
+    Inputs must be one-dimensional and contain at least two finite values per arm.
+    Non-finite observations raise by default; callers may explicitly request omission.
+    A relative effect is emitted only when both group means are non-negative and the
+    control mean is numerically resolved relative to the observed data scale.
+    """
+    control_values, control_omitted = normalize_numeric_sample(
+        control, "control", nonfinite_policy=nonfinite_policy
+    )
+    treatment_values, treatment_omitted = normalize_numeric_sample(
+        treatment, "treatment", nonfinite_policy=nonfinite_policy
+    )
+    confidence_level = validate_probability(
+        confidence_level, "confidence_level", open_interval=True
+    )
+    alternative = normalize_alternative(alternative)
+    control_size = int(control_values.size)
+    treatment_size = int(treatment_values.size)
+    control_mean, control_sd = _stable_mean_and_standard_deviation(control_values)
+    treatment_mean, treatment_sd = _stable_mean_and_standard_deviation(treatment_values)
+    effect = treatment_mean - control_mean
+    if not np.isfinite(effect):
+        raise ValueError("the mean difference is non-finite; rescale extreme observations")
+    diagnostics: list[DiagnosticResult] = []
+
+    omitted = control_omitted + treatment_omitted
+    if omitted:
+        diagnostics.append(
+            DiagnosticResult(
+                code="NONFINITE_OBSERVATIONS_OMITTED",
+                status=DiagnosticStatus.WARN,
+                message="Non-finite observations were omitted by explicit caller policy.",
+                details={
+                    "control_omitted": control_omitted,
+                    "treatment_omitted": treatment_omitted,
+                },
+            )
+        )
+    if control_size < 30 or treatment_size < 30:
+        diagnostics.append(
+            DiagnosticResult(
+                code="SMALL_SAMPLE",
+                status=DiagnosticStatus.WARN,
+                message=(
+                    "At least one arm has fewer than 30 observations; Welch inference relies "
+                    "more strongly on approximate normality of that arm's sample mean."
+                ),
+                details={"control_size": control_size, "treatment_size": treatment_size},
+            )
+        )
+
+    control_standard_error = control_sd / sqrt(control_size)
+    treatment_standard_error = treatment_sd / sqrt(treatment_size)
+    standard_error = hypot(control_standard_error, treatment_standard_error)
+    if standard_error == 0.0:
+        diagnostics.append(
+            DiagnosticResult(
+                code="DEGENERATE_ZERO_VARIANCE",
+                status=DiagnosticStatus.WARN,
+                message=(
+                    "Both samples have zero variance; Welch's reference distribution is "
+                    "undefined, so no test statistic or p-value is reported."
+                ),
+            )
+        )
+        interval = ConfidenceInterval(
+            lower=effect if alternative is not AlternativeHypothesis.LESS else None,
+            upper=effect if alternative is not AlternativeHypothesis.GREATER else None,
+            confidence_level=confidence_level,
+        )
+        statistic = None
+        p_value = None
+        degrees_of_freedom = None
+    else:
+        degrees_of_freedom = _welch_degrees_of_freedom(
+            control_standard_error,
+            control_size,
+            treatment_standard_error,
+            treatment_size,
+        )
+        statistic = effect / standard_error
+        if alternative is AlternativeHypothesis.GREATER:
+            p_value = float(stats.t.sf(statistic, degrees_of_freedom))
+        elif alternative is AlternativeHypothesis.LESS:
+            p_value = float(stats.t.cdf(statistic, degrees_of_freedom))
+        else:
+            p_value = float(2.0 * stats.t.sf(abs(statistic), degrees_of_freedom))
+        interval = _welch_interval(
+            effect,
+            standard_error,
+            degrees_of_freedom,
+            confidence_level,
+            alternative,
+        )
+        if control_sd == 0.0 or treatment_sd == 0.0:
+            diagnostics.append(
+                DiagnosticResult(
+                    code="ONE_ZERO_VARIANCE_SAMPLE",
+                    status=DiagnosticStatus.WARN,
+                    message=(
+                        "One sample has zero variance; Welch inference is computable but the "
+                        "data-generating assumptions deserve review."
+                    ),
+                )
+            )
+
+    observed_scale = max(
+        float(np.max(np.abs(control_values))),
+        float(np.max(np.abs(treatment_values))),
+        np.finfo(np.float64).tiny,
+    )
+    denominator_resolution = sqrt(np.finfo(np.float64).eps) * observed_scale
+    relative_effect = (
+        effect / control_mean
+        if control_mean > denominator_resolution and treatment_mean >= 0.0
+        else None
+    )
+    if relative_effect is None:
+        diagnostics.append(
+            DiagnosticResult(
+                code="RELATIVE_EFFECT_NOT_REPORTED",
+                status=DiagnosticStatus.WARN,
+                message=(
+                    "Relative mean difference is not reported because the observed means do "
+                    "not support a non-negative, numerically resolved ratio-scale interpretation."
+                ),
+                details={"minimum_resolved_control_mean": float(denominator_resolution)},
+            )
+        )
+    direction = {
+        AlternativeHypothesis.TWO_SIDED: "mu_treatment - mu_control != 0",
+        AlternativeHypothesis.GREATER: "mu_treatment - mu_control > 0",
+        AlternativeHypothesis.LESS: "mu_treatment - mu_control < 0",
+    }[alternative]
+    return AnalysisResult(
+        metric_name=metric_name,
+        metric_type=MetricType.CONTINUOUS,
+        control_estimate=control_mean,
+        treatment_estimate=treatment_mean,
+        absolute_effect=effect,
+        relative_effect=relative_effect,
+        relative_effect_definition=(
+            "(treatment mean - control mean) / control mean"
+            if relative_effect is not None
+            else None
+        ),
+        confidence_interval=interval,
+        test_statistic=statistic,
+        p_value=p_value,
+        confidence_level=confidence_level,
+        alternative=alternative,
+        control_sample_size=control_size,
+        treatment_sample_size=treatment_size,
+        control_standard_deviation=control_sd,
+        treatment_standard_deviation=treatment_sd,
+        degrees_of_freedom=degrees_of_freedom,
+        method="Welch's two-sample t-test and Welch-Satterthwaite confidence interval",
+        null_hypothesis="mu_treatment - mu_control = 0",
+        alternative_hypothesis=direction,
+        diagnostics=tuple(diagnostics),
+    )

experimentation_toolkit/analysis/proportion.py

@@ -0,0 +1,204 @@
+"""Two-sample analysis for binary outcomes."""
+
+from math import sqrt
+
+from scipy import stats
+
+from experimentation_toolkit.models.enums import (
+    AlternativeHypothesis,
+    DiagnosticStatus,
+    MetricType,
+)
+from experimentation_toolkit.models.results import (
+    AnalysisResult,
+    ConfidenceInterval,
+    DiagnosticResult,
+)
+from experimentation_toolkit.validation.inputs import (
+    normalize_alternative,
+    validate_nonnegative_integer,
+    validate_positive_integer,
+    validate_probability,
+)
+
+
+def _wilson_bounds(successes: int, total: int, z_value: float) -> tuple[float, float]:
+    proportion = successes / total
+    z_squared = z_value**2
+    denominator = 1.0 + z_squared / total
+    center = (proportion + z_squared / (2.0 * total)) / denominator
+    half_width = (
+        z_value
+        * sqrt(proportion * (1.0 - proportion) / total + z_squared / (4.0 * total) / total)
+        / denominator
+    )
+    return max(0.0, center - half_width), min(1.0, center + half_width)
+
+
+def _newcombe_difference_interval(
+    control_successes: int,
+    control_total: int,
+    treatment_successes: int,
+    treatment_total: int,
+    confidence_level: float,
+    alternative: AlternativeHypothesis,
+) -> ConfidenceInterval:
+    alpha = 1.0 - confidence_level
+    tail_probability = alpha / 2.0 if alternative is AlternativeHypothesis.TWO_SIDED else alpha
+    z_value = float(stats.norm.ppf(1.0 - tail_probability))
+    control_rate = control_successes / control_total
+    treatment_rate = treatment_successes / treatment_total
+    effect = treatment_rate - control_rate
+    control_lower, control_upper = _wilson_bounds(control_successes, control_total, z_value)
+    treatment_lower, treatment_upper = _wilson_bounds(treatment_successes, treatment_total, z_value)
+    lower = effect - sqrt(
+        (treatment_rate - treatment_lower) ** 2 + (control_upper - control_rate) ** 2
+    )
+    upper = effect + sqrt(
+        (treatment_upper - treatment_rate) ** 2 + (control_rate - control_lower) ** 2
+    )
+    if alternative is AlternativeHypothesis.GREATER:
+        return ConfidenceInterval(lower=float(lower), upper=None, confidence_level=confidence_level)
+    if alternative is AlternativeHypothesis.LESS:
+        return ConfidenceInterval(lower=None, upper=float(upper), confidence_level=confidence_level)
+    return ConfidenceInterval(
+        lower=float(lower), upper=float(upper), confidence_level=confidence_level
+    )
+
+
+def _score_test(
+    control_successes: int,
+    control_total: int,
+    treatment_successes: int,
+    treatment_total: int,
+    alternative: AlternativeHypothesis,
+) -> tuple[float, float]:
+    control_rate = control_successes / control_total
+    treatment_rate = treatment_successes / treatment_total
+    pooled_rate = (control_successes + treatment_successes) / (control_total + treatment_total)
+    standard_error = sqrt(
+        pooled_rate * (1.0 - pooled_rate) * (1.0 / control_total + 1.0 / treatment_total)
+    )
+    if standard_error == 0.0:
+        return 0.0, 1.0
+    statistic = (treatment_rate - control_rate) / standard_error
+    if alternative is AlternativeHypothesis.GREATER:
+        p_value = stats.norm.sf(statistic)
+    elif alternative is AlternativeHypothesis.LESS:
+        p_value = stats.norm.cdf(statistic)
+    else:
+        p_value = 2.0 * stats.norm.sf(abs(statistic))
+    return float(statistic), float(p_value)
+
+
+def analyze_proportion(
+    *,
+    control_successes: int,
+    control_total: int,
+    treatment_successes: int,
+    treatment_total: int,
+    confidence_level: float = 0.95,
+    alternative: AlternativeHypothesis | str = AlternativeHypothesis.TWO_SIDED,
+    metric_name: str | None = None,
+) -> AnalysisResult:
+    """Analyze a treatment-minus-control difference in independent proportions.
+
+    The test is the pooled two-sample score z-test for ``H0: p_t - p_c = 0``.
+    The confidence interval is Newcombe's hybrid score interval based on Wilson
+    intervals, without continuity correction. One-sided alternatives return the
+    corresponding one-sided interval with ``None`` for its unbounded endpoint.
+    """
+    control_total = validate_positive_integer(control_total, "control_total")
+    treatment_total = validate_positive_integer(treatment_total, "treatment_total")
+    control_successes = validate_nonnegative_integer(control_successes, "control_successes")
+    treatment_successes = validate_nonnegative_integer(treatment_successes, "treatment_successes")
+    if control_successes > control_total:
+        raise ValueError("control_successes must not exceed control_total")
+    if treatment_successes > treatment_total:
+        raise ValueError("treatment_successes must not exceed treatment_total")
+    confidence_level = validate_probability(
+        confidence_level, "confidence_level", open_interval=True
+    )
+    alternative = normalize_alternative(alternative)
+
+    control_rate = control_successes / control_total
+    treatment_rate = treatment_successes / treatment_total
+    effect = treatment_rate - control_rate
+    relative_effect = effect / control_rate if control_rate > 0.0 else None
+    diagnostics: list[DiagnosticResult] = []
+    if relative_effect is None:
+        diagnostics.append(
+            DiagnosticResult(
+                code="RELATIVE_EFFECT_UNDEFINED",
+                status=DiagnosticStatus.WARN,
+                message="Relative lift is undefined because the control rate is zero.",
+            )
+        )
+
+    pooled_rate = (control_successes + treatment_successes) / (control_total + treatment_total)
+    expected_cells = (
+        control_total * pooled_rate,
+        control_total * (1.0 - pooled_rate),
+        treatment_total * pooled_rate,
+        treatment_total * (1.0 - pooled_rate),
+    )
+    if min(expected_cells) < 5.0:
+        diagnostics.append(
+            DiagnosticResult(
+                code="ASYMPTOTIC_APPROXIMATION_WARNING",
+                status=DiagnosticStatus.WARN,
+                message=(
+                    "At least one expected success/failure count under the null is below 5; "
+                    "the score-test normal approximation may be inaccurate."
+                ),
+                details={"minimum_expected_cell_count": float(min(expected_cells))},
+            )
+        )
+
+    statistic, p_value = _score_test(
+        control_successes,
+        control_total,
+        treatment_successes,
+        treatment_total,
+        alternative,
+    )
+    interval = _newcombe_difference_interval(
+        control_successes,
+        control_total,
+        treatment_successes,
+        treatment_total,
+        confidence_level,
+        alternative,
+    )
+    direction = {
+        AlternativeHypothesis.TWO_SIDED: "p_treatment - p_control != 0",
+        AlternativeHypothesis.GREATER: "p_treatment - p_control > 0",
+        AlternativeHypothesis.LESS: "p_treatment - p_control < 0",
+    }[alternative]
+    return AnalysisResult(
+        metric_name=metric_name,
+        metric_type=MetricType.PROPORTION,
+        control_estimate=float(control_rate),
+        treatment_estimate=float(treatment_rate),
+        absolute_effect=float(effect),
+        relative_effect=float(relative_effect) if relative_effect is not None else None,
+        relative_effect_definition=(
+            "(treatment rate - control rate) / control rate"
+            if relative_effect is not None
+            else None
+        ),
+        confidence_interval=interval,
+        test_statistic=statistic,
+        p_value=p_value,
+        confidence_level=confidence_level,
+        alternative=alternative,
+        control_sample_size=control_total,
+        treatment_sample_size=treatment_total,
+        method=(
+            "Pooled two-sample score z-test; Newcombe hybrid Wilson confidence interval "
+            "for the absolute risk difference (no continuity correction)"
+        ),
+        null_hypothesis="p_treatment - p_control = 0",
+        alternative_hypothesis=direction,
+        diagnostics=tuple(diagnostics),
+    )

experimentation_toolkit/diagnostics/__init__.py

@@ -0,0 +1,5 @@
+"""Experiment diagnostics."""
+
+from experimentation_toolkit.diagnostics.srm import check_sample_ratio_mismatch
+
+__all__ = ["check_sample_ratio_mismatch"]

experimentation_toolkit/diagnostics/srm.py

@@ -0,0 +1,89 @@
+"""Sample-ratio-mismatch diagnostics."""
+
+from collections.abc import Mapping
+
+import numpy as np
+from scipy import stats
+
+from experimentation_toolkit.models.enums import DiagnosticStatus
+from experimentation_toolkit.models.results import DiagnosticResult, SRMResult
+from experimentation_toolkit.validation.inputs import (
+    validate_nonnegative_integer,
+    validate_probability,
+)
+
+
+def check_sample_ratio_mismatch(
+    *,
+    observed_counts: Mapping[str, int],
+    expected_proportions: Mapping[str, float],
+    significance_level: float = 0.01,
+    proportion_tolerance: float = 1e-12,
+) -> SRMResult:
+    """Run Pearson's chi-square goodness-of-fit test for assignment counts.
+
+    A failing result flags a potential randomization or instrumentation problem. It
+    does not identify the cause. The chi-square approximation assumes independent
+    assignments and sufficiently large positive expected counts.
+    """
+    significance_level = validate_probability(
+        significance_level, "significance_level", open_interval=True
+    )
+    if not np.isfinite(proportion_tolerance) or proportion_tolerance < 0.0:
+        raise ValueError("proportion_tolerance must be finite and non-negative")
+    if len(observed_counts) < 2:
+        raise ValueError("at least two variants are required for an SRM check")
+    if set(observed_counts) != set(expected_proportions):
+        raise ValueError("observed_counts and expected_proportions must have identical keys")
+
+    observed = {
+        name: validate_nonnegative_integer(count, f"observed_counts[{name!r}]")
+        for name, count in observed_counts.items()
+    }
+    proportions = {name: float(expected_proportions[name]) for name in observed}
+    if any(not np.isfinite(value) or value <= 0.0 for value in proportions.values()):
+        raise ValueError("all expected allocation proportions must be finite and positive")
+    if not np.isclose(sum(proportions.values()), 1.0, rtol=0.0, atol=proportion_tolerance):
+        raise ValueError("expected allocation proportions must sum to 1")
+    total = sum(observed.values())
+    if total <= 0:
+        raise ValueError("the total observed assignment count must be greater than zero")
+    expected = {name: total * proportion for name, proportion in proportions.items()}
+    diagnostics: tuple[DiagnosticResult, ...] = ()
+    minimum_expected_count = min(expected.values())
+    if minimum_expected_count < 5.0:
+        diagnostics = (
+            DiagnosticResult(
+                code="SMALL_EXPECTED_ASSIGNMENT_COUNT",
+                status=DiagnosticStatus.WARN,
+                message=(
+                    "At least one expected assignment count is below 5; the chi-square "
+                    "reference approximation may be inaccurate."
+                ),
+                details={"minimum_expected_count": float(minimum_expected_count)},
+            ),
+        )
+
+    result = stats.chisquare(
+        f_obs=np.asarray(tuple(observed.values()), dtype=np.float64),
+        f_exp=np.asarray(tuple(expected.values()), dtype=np.float64),
+    )
+    statistic = float(result.statistic)
+    p_value = float(result.pvalue)
+    failed = p_value < significance_level
+    return SRMResult(
+        observed_counts=observed,
+        expected_counts={name: float(value) for name, value in expected.items()},
+        chi_square_statistic=statistic,
+        p_value=p_value,
+        significance_level=significance_level,
+        degrees_of_freedom=len(observed) - 1,
+        status=DiagnosticStatus.FAIL if failed else DiagnosticStatus.PASS,
+        message=(
+            "Assignment counts are inconsistent with the configured allocation at the "
+            "selected threshold; investigate randomization or instrumentation."
+            if failed
+            else "No sample ratio mismatch was detected at the selected threshold."
+        ),
+        diagnostics=diagnostics,
+    )

experimentation_toolkit/models/__init__.py

@@ -0,0 +1,40 @@
+"""Public domain and result models."""
+
+from experimentation_toolkit.models.core import Experiment, Metric, ToolkitModel, Variant
+from experimentation_toolkit.models.enums import (
+    AlternativeHypothesis,
+    CorrectionMethod,
+    DiagnosticStatus,
+    MDEType,
+    MetricType,
+)
+from experimentation_toolkit.models.results import (
+    AchievedPowerResult,
+    AnalysisResult,
+    ConfidenceInterval,
+    DiagnosticResult,
+    ExperimentReport,
+    MultipleTestingResult,
+    ProportionSampleSizeResult,
+    SRMResult,
+)
+
+__all__ = [
+    "AchievedPowerResult",
+    "AlternativeHypothesis",
+    "AnalysisResult",
+    "ConfidenceInterval",
+    "CorrectionMethod",
+    "DiagnosticResult",
+    "DiagnosticStatus",
+    "Experiment",
+    "ExperimentReport",
+    "MDEType",
+    "Metric",
+    "MetricType",
+    "MultipleTestingResult",
+    "ProportionSampleSizeResult",
+    "SRMResult",
+    "ToolkitModel",
+    "Variant",
+]