argonx 0.1.0__py3-none-any.whl

argonx/__init__.py

@@ -0,0 +1,11 @@
+"""
+argonx: Bayesian decision engine for robust A/B testing.
+
+This package provides a comprehensive framework for Bayesian experimentation,
+integrating primary metrics with guardrails, sequential stopping rules, and
+hierarchical partial pooling.
+"""
+
+from .experiment import Experiment
+
+__all__ = ["Experiment"]

argonx/decision_rules/__init__.py

@@ -0,0 +1,99 @@
+"""
+Decision rules and analysis engines for the Bayesian A/B testing framework.
+
+This subpackage implements the full analytical pipeline that transforms raw posterior
+samples into actionable experiment outcomes. It covers five distinct concerns:
+
+- **Metrics** (`metrics.py`): Core Bayesian decision metrics — P(best), expected loss,
+  CVaR, ROPE analysis, and HDI-bounded lift. All metrics are derived from the same
+  posterior draws to ensure coherence.
+
+- **Guardrails** (`guardrails.py`): Safety constraint evaluation. Computes P(degraded)
+  per metric per variant and surfaces conflicts when the primary metric clears its bar
+  but a secondary metric fails.
+
+- **Joint** (`joint.py`): Joint probability of simultaneously satisfying both the primary
+  metric and all selected guardrails, with correlation diagnostics to expose when
+  metric co-movement helps or hurts the compound decision.
+
+- **Composite** (`composite.py`): Weighted scoring of multiple metrics into a single
+  posterior distribution of business value, with asymmetric deterioration weights and
+  optional guardrail penalties.
+
+- **Engine** (`engine.py`): Orchestration layer. Calls all of the above in dependency
+  order and assembles a structured `DecisionResult` with a plain-English recommendation.
+"""
+
+from .composite import (
+    CompositeResult,
+    compute_composite_score,
+)
+
+from .engine import (
+    DecisionResult,
+    run_engine,
+)
+
+from .guardrails import (
+    ConflictResult,
+    GuardrailBundle,
+    GuardrailResult,
+    compute_all_guardrails,
+    compute_guardrail,
+)
+
+from .joint import (
+    JointResult,
+    compute_joint_probability,
+)
+
+from .metrics import (
+    CVaRResult,
+    LiftResult,
+    LossResult,
+    MetricsBundle,
+    PBestResult,
+    ROPEResult,
+    compute_all_metrics,
+    compute_cvar,
+    compute_expected_loss,
+    compute_lift_hdi,
+    compute_prob_best,
+    compute_rope,
+)
+
+__all__ = [
+    # Dataclasses — composite
+    "CompositeResult",
+    # Dataclasses — engine
+    "DecisionResult",
+    # Dataclasses — guardrails
+    "ConflictResult",
+    "GuardrailBundle",
+    "GuardrailResult",
+    # Dataclasses — joint
+    "JointResult",
+    # Dataclasses — metrics
+    "CVaRResult",
+    "LiftResult",
+    "LossResult",
+    "MetricsBundle",
+    "PBestResult",
+    "ROPEResult",
+    # Functions — composite
+    "compute_composite_score",
+    # Functions — engine
+    "run_engine",
+    # Functions — guardrails
+    "compute_all_guardrails",
+    "compute_guardrail",
+    # Functions — joint
+    "compute_joint_probability",
+    # Functions — metrics
+    "compute_all_metrics",
+    "compute_cvar",
+    "compute_expected_loss",
+    "compute_lift_hdi",
+    "compute_prob_best",
+    "compute_rope",
+]

argonx/decision_rules/composite.py

@@ -0,0 +1,203 @@
+from __future__ import annotations
+
+import numpy as np
+import warnings
+from dataclasses import dataclass, field
+
+from argonx.decision_rules.guardrails import GuardrailBundle
+
+
+@dataclass
+class CompositeResult:
+    """
+    Structured outcome of a composite business metric calculation across variants.
+
+    This encapsulates the final scores, the distributions representing Bayesian uncertainty,
+    and the relative contributions of individual metrics. It serves as the primary data
+    payload for rendering plotting results and making ultimate ship decisions.
+    """
+
+    score: dict[str, float]
+    score_distribution: dict[str, np.ndarray]
+    metric_contributions: dict[str, dict[str, float]]
+    prob_exceeds_threshold: dict[str, float]
+    gap_distribution: dict[str, np.ndarray]
+    gap_hdi: dict[str, tuple[float, float]]
+    best_variant: str
+    threshold: float
+    warnings: list[str] = field(default_factory=list)
+
+
+def _compute_hdi(samples: np.ndarray, prob: float = 0.94) -> tuple[float, float]:
+    """
+    Calculate the Highest Density Interval (HDI) for a given set of posterior draws.
+
+    This function finds the shortest contiguous interval that contains a specified
+    proportion of the distribution, which is characteristic of the HDI.
+
+    Parameters
+    ----------
+    samples : np.ndarray
+        A 1-dimensional array of posterior draws.
+    prob : float, optional
+        The probability mass that should be contained within the interval, by default 0.94.
+
+    Returns
+    -------
+    tuple[float, float]
+        The lower and upper bounds of the computed credible interval.
+    """
+    sorted_samples = np.sort(samples)
+    n = len(samples)
+    interval_idx = int(np.floor(prob * n))
+    widths = sorted_samples[interval_idx:] - sorted_samples[: n - interval_idx]
+    min_idx = np.argmin(widths)
+    return (
+        float(sorted_samples[min_idx]),
+        float(sorted_samples[min_idx + interval_idx]),
+    )
+
+
+def compute_composite_score(
+    primary_samples: np.ndarray,
+    guardrail_samples: dict[str, np.ndarray],
+    variant_names: list[str],
+    control: str,
+    weights: dict[str, float],
+    guardrail_bundle: GuardrailBundle,
+    deterioration_weights: dict[str, float] | None = None,
+    guardrail_penalty: float = 0.0,
+    threshold: float = 0.0,
+) -> CompositeResult:
+    """
+    Calculate a unified decision score integrating primary metrics and guardrails.
+
+    This function isolates the lift of each variant over control, applies directional
+    weights to model relative business impact, enforces penalties for failing categorical
+    guardrails, and aggregates them into a singular posterior distribution of scores.
+
+    Parameters
+    ----------
+    primary_samples : np.ndarray
+        Posterior samples for the primary target metric (draws x variants).
+    guardrail_samples : dict[str, np.ndarray]
+        A mapping of guardrail metric names to their corresponding posterior draws.
+    variant_names : list[str]
+        A sequential list of variant names corresponding to the columns in samples arrays.
+    control : str
+        The designated control variant used as the baseline for lift computation.
+    weights : dict[str, float]
+        Linear weights defining the reward (or penalty) for positive shifts per metric.
+    guardrail_bundle : GuardrailBundle
+        A evaluated bundle containing boolean pass/fail states for variants across guardrails.
+    deterioration_weights : dict[str, float] | None, optional
+        Asymmetric weights applied exclusively to negative degradation. Defaults to `weights`.
+    guardrail_penalty : float, optional
+        Fixed deduction applied to a variant's total score if it fails the bundle. Defaults to 0.0.
+    threshold : float, optional
+        The minimal composite score required to consider a variant viably improved. Defaults to 0.0.
+
+    Returns
+    -------
+    CompositeResult
+        The comprehensive scoring breakdown, including expected scores, underlying distributions,
+        and interval estimations.
+    """
+
+    collected: list[str] = []
+
+    if not weights:
+        raise ValueError("weights must be provided")
+
+    valid_metrics = {"primary"} | set(guardrail_samples.keys())
+    matched = [k for k in weights if k in valid_metrics]
+    if not matched:
+        raise ValueError(
+            f"No valid metric keys in weights. Expected one of {valid_metrics}"
+        )
+
+    if deterioration_weights is None:
+        deterioration_weights = weights.copy()
+        warnings.warn(
+            "deterioration_weights not provided, using symmetric weights",
+            UserWarning,
+            stacklevel=2,
+        )
+        collected.append("Using symmetric deterioration weights")
+
+    if guardrail_penalty == 0:
+        warnings.warn(
+            "guardrail_penalty is zero, guardrails have no effect",
+            UserWarning,
+            stacklevel=2,
+        )
+        collected.append("Guardrail penalty is zero")
+
+    c_idx = variant_names.index(control)
+    variants = [v for v in variant_names if v != control]
+
+    deltas = {}
+    control_vals = primary_samples[:, c_idx]
+    deltas["primary"] = primary_samples - control_vals[:, None]
+
+    for m, s in guardrail_samples.items():
+        deltas[m] = s - s[:, c_idx][:, None]
+
+    score_per_draw: dict[str, np.ndarray] = {}
+    contributions: dict[str, dict[str, float]] = {}
+
+    for i, v in enumerate(variant_names):
+        if v == control:
+            continue
+
+        total = np.zeros(primary_samples.shape[0])
+        contrib = {}
+
+        for m, d in deltas.items():
+            w = weights.get(m, 0.0)
+            dw = deterioration_weights.get(m, w)
+
+            val = d[:, i]
+
+            pos = np.maximum(val, 0)
+            neg = np.minimum(val, 0)
+
+            comp = w * pos + dw * neg
+            total += comp
+            contrib[m] = float(np.mean(comp))
+
+        variant_failed = not guardrail_bundle.variant_passed.get(v, True)
+        penalty = guardrail_penalty * (1 if variant_failed else 0)
+        total -= penalty
+
+        score_per_draw[v] = total
+        contributions[v] = contrib
+
+    score = {v: float(np.mean(score_per_draw[v])) for v in variants}
+
+    prob_exceeds = {v: float(np.mean(score_per_draw[v] > threshold)) for v in variants}
+
+    gap_distribution = {v: score_per_draw[v] - threshold for v in variants}
+
+    gap_hdi = {v: _compute_hdi(gap_distribution[v]) for v in variants}
+
+    best = max(score, key=lambda k: score[k])
+
+    if all(s < threshold for s in score.values()):
+        collected.append("No variant exceeds composite threshold")
+
+    for v, s in score.items():
+        if s < 0:
+            collected.append(f"{v} has negative composite score")
+
+    return CompositeResult(
+        score=score,
+        score_distribution=score_per_draw,
+        metric_contributions=contributions,
+        prob_exceeds_threshold=prob_exceeds,
+        gap_distribution=gap_distribution,
+        gap_hdi=gap_hdi,
+        best_variant=best,
+        threshold=threshold,
+        warnings=collected,
+    )

argonx/decision_rules/engine.py

@@ -0,0 +1,335 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+import numpy as np
+
+from .metrics import MetricsBundle, compute_all_metrics
+from .guardrails import GuardrailBundle, compute_all_guardrails
+from .joint import JointResult, compute_joint_probability
+from .composite import CompositeResult, compute_composite_score
+
+
+@dataclass
+class DecisionResult:
+    """Final decision output with structured interpretation."""
+
+    state: str
+    recommendation: str
+    best_variant: str
+
+    primary_strength: str
+    risk_level: str
+    practical_significance: str
+    guardrail_status: str
+    confidence: str
+
+    metrics: MetricsBundle
+    guardrails: GuardrailBundle
+    joint: JointResult | None
+    composite: CompositeResult | None
+
+    reasons: list[str] = field(default_factory=list)
+    notes: list[str] = field(default_factory=list)
+
+
+def _evaluate_primary_strength(metrics: MetricsBundle, config: dict) -> str:
+    """Compute strength of primary signal."""
+    best = metrics.prob_best.best_variant
+    p_best = metrics.prob_best.probabilities[best]
+    loss = metrics.loss.expected_loss[best]
+    practical = metrics.rope.prob_practical.get(best, 0.0)
+
+    if (
+        p_best >= config["prob_best_strong"]
+        and loss <= config["expected_loss_max"]
+        and practical >= config["rope_practical_min"]
+    ):
+        return "strong"
+
+    if p_best >= config["prob_best_moderate"]:
+        return "moderate"
+
+    return "weak"
+
+
+def _evaluate_risk(metrics: MetricsBundle, config: dict) -> str:
+    """Classify risk using expected loss and CVaR."""
+    best = metrics.prob_best.best_variant
+    el = metrics.loss.expected_loss[best]
+    cv = metrics.cvar.cvar[best]
+
+    if el <= config["expected_loss_max"] and (
+        el == 0 or cv / max(el, 1e-12) <= config["cvar_ratio_max"]
+    ):
+        return "low"
+
+    if el <= config["expected_loss_max"] * 5:
+        return "medium"
+
+    return "high"
+
+
+def _evaluate_practical_significance(metrics: MetricsBundle, config: dict) -> str:
+    """Classify practical significance using ROPE."""
+    best = metrics.prob_best.best_variant
+    prob = metrics.rope.prob_practical.get(best, 0.0)
+
+    if prob >= config["rope_practical_min"]:
+        return "yes"
+
+    if prob >= 0.5:
+        return "uncertain"
+
+    return "no"
+
+
+def _evaluate_guardrails(guardrails: GuardrailBundle) -> str:
+    """Summarize guardrail status."""
+    return "pass" if guardrails.all_passed else "fail"
+
+
+def _evaluate_confidence(metrics: MetricsBundle, config: dict) -> str:
+    """Assess confidence using posterior certainty."""
+    best = metrics.prob_best.best_variant
+    p_best = metrics.prob_best.probabilities[best]
+
+    low = metrics.lift.hdi_low[best]
+    high = metrics.lift.hdi_high[best]
+    width = abs(high - low)
+
+    if p_best >= 0.95 and width < 0.1:
+        return "high"
+
+    if p_best >= 0.8:
+        return "medium"
+
+    return "low"
+
+
+def _determine_state(
+    primary_strength: str,
+    risk_level: str,
+    practical_significance: str,
+    guardrails: GuardrailBundle,
+) -> str:
+    """Determine overall decision state."""
+    if guardrails.conflicts:
+        return "guardrail conflicts"
+
+    if (
+        primary_strength == "strong"
+        and risk_level == "low"
+        and practical_significance == "yes"
+        and guardrails.all_passed
+    ):
+        return "strong win"
+
+    if (
+        primary_strength == "moderate"
+        and guardrails.all_passed
+        and risk_level == "low"
+        and practical_significance == "yes"
+    ):
+        return "weak win"
+
+    if risk_level == "high":
+        return "high risk"
+
+    return "inconclusive"
+
+
+def _map_recommendation(state: str) -> str:
+    """Map decision state to recommendation."""
+    return {
+        "strong win": "ship variant",
+        "weak win": "consider shipping",
+        "high risk": "do not ship",
+        "guardrail conflicts": "review required",
+        "inconclusive": "continue experiment",
+    }[state]
+
+
+def _build_reasons(
+    primary_strength: str,
+    risk_level: str,
+    practical_significance: str,
+    guardrail_status: str,
+) -> list[str]:
+    """Generate human readable reasoning signals."""
+    reasons = [
+        f"Primary signal is {primary_strength}",
+        f"Risk level is {risk_level}",
+        f"Practical significance is {practical_significance}",
+    ]
+
+    if guardrail_status == "pass":
+        reasons.append("All guardrails passed")
+    else:
+        reasons.append("One or more guardrails failed")
+
+    return reasons
+
+
+def _collect_notes(
+    metrics: MetricsBundle,
+    guardrails: GuardrailBundle,
+    joint: JointResult,
+    composite: CompositeResult,
+) -> list[str]:
+    """Collect warnings and edge case signals from all components."""
+    notes = []
+
+    notes.extend(metrics.warnings)
+    notes.extend(guardrails.warnings)
+
+    if joint is not None:
+        notes.extend(joint.warnings)
+
+    if composite is not None:
+        notes.extend(composite.warnings)
+
+    for conflict in guardrails.conflicts:
+        if conflict.severity == "high":
+            notes.append(
+                f"High severity guardrail violation on '{conflict.metric}' "
+                f"for variant '{conflict.variant}'"
+            )
+
+    for v in metrics.loss.expected_loss:
+        el = metrics.loss.expected_loss[v]
+        cv = metrics.cvar.cvar[v]
+        if el > 0 and cv / el > 5:
+            notes.append(f"Variant '{v}' has extreme tail risk")
+
+    return notes
+
+
+def run_engine(
+    samples: np.ndarray,
+    variant_names: list[str],
+    control: str,
+    guardrail_samples: dict[str, np.ndarray],
+    config: dict,
+) -> DecisionResult:
+    """
+    Run full decision pipeline from posterior samples.
+
+    Processes primary Bayesian updates, checks safety guardrails, isolates joint constraint
+    probabilities, and coalesces performance metrics into a composite decision score.
+    Returns a structured recommendation detailing business risk, primary strength,
+    and significance.
+
+    Parameters
+    ----------
+    samples : np.ndarray
+        Array containing primary metric posterior draws.
+    variant_names : list[str]
+        Ordered collection of variants identifying the draw columns.
+    control : str
+        The primary baseline variant used for comparisons.
+    guardrail_samples : dict[str, np.ndarray]
+        Mapping of safety metric designations to their posterior distributions.
+    config : dict
+        A master configuration dictionary determining parameters like decision thresholds,
+        cvar bounds, rope regions, and guardrail limits.
+
+    Returns
+    -------
+    DecisionResult
+        A bundled diagnostic report mapping qualitative shipping recommendations to
+        underlying statistical confidence intervals and rule triggers.
+    """
+
+    metrics = compute_all_metrics(
+        samples=samples,
+        variant_names=variant_names,
+        control=control,
+        rope_bounds=config["rope_bounds"],
+        alpha=config.get("alpha", 0.95),
+        hdi_prob=config.get("hdi_prob", 0.95),
+    )
+
+    primary_strength = _evaluate_primary_strength(metrics, config)
+    risk_level = _evaluate_risk(metrics, config)
+    practical_significance = _evaluate_practical_significance(metrics, config)
+
+    primary_passed = (
+        primary_strength == "strong"
+        and risk_level != "high"
+        and practical_significance == "yes"
+    )
+
+    guardrails = compute_all_guardrails(
+        guardrail_samples=guardrail_samples,
+        variant_names=variant_names,
+        control=control,
+        thresholds=config.get("guardrail_thresholds", {}),
+        primary_passed=primary_passed,
+        lower_is_better=config.get("lower_is_better", {}),
+    )
+
+    guardrail_status = _evaluate_guardrails(guardrails)
+    confidence = _evaluate_confidence(metrics, config)
+
+    joint = None
+    if guardrail_samples:
+        joint = compute_joint_probability(
+            primary_samples=samples,
+            guardrail_samples=guardrail_samples,
+            variant_names=variant_names,
+            control=control,
+            primary_lower_is_better=config.get("primary_lower_is_better", False),
+            lower_is_better=config.get("lower_is_better", {}),
+            guardrail_thresholds=config.get("guardrail_thresholds", {}),
+            metrics_to_join=config.get("metrics_to_join", None),
+        )
+
+    composite = None
+    if "composite_weights" in config:
+        composite = compute_composite_score(
+            primary_samples=samples,
+            guardrail_samples=guardrail_samples,
+            variant_names=variant_names,
+            control=control,
+            weights=config["composite_weights"],
+            guardrail_bundle=guardrails,
+            deterioration_weights=config.get("deterioration_weights", None),
+            guardrail_penalty=config.get("guardrail_penalty", 0.0),
+            threshold=config.get("composite_threshold", 0.0),
+        )
+
+    state = _determine_state(
+        primary_strength,
+        risk_level,
+        practical_significance,
+        guardrails,
+    )
+
+    recommendation = _map_recommendation(state)
+
+    reasons = _build_reasons(
+        primary_strength,
+        risk_level,
+        practical_significance,
+        guardrail_status,
+    )
+
+    notes = _collect_notes(metrics, guardrails, joint, composite)
+
+    return DecisionResult(
+        state=state,
+        recommendation=recommendation,
+        best_variant=metrics.prob_best.best_variant,
+        primary_strength=primary_strength,
+        risk_level=risk_level,
+        practical_significance=practical_significance,
+        guardrail_status=guardrail_status,
+        confidence=confidence,
+        metrics=metrics,
+        guardrails=guardrails,
+        joint=joint,
+        composite=composite,
+        reasons=reasons,
+        notes=notes,
+    )