ssbc 0.1.0__py3-none-any.whl

ssbc/hyperparameter.py

@@ -0,0 +1,258 @@
+"""Hyperparameter sweep and optimization for Mondrian conformal prediction."""
+
+import itertools
+from collections.abc import Callable
+from typing import Any, Literal
+
+import numpy as np
+import pandas as pd
+
+from .conformal import mondrian_conformal_calibrate
+from .visualization import plot_parallel_coordinates_plotly, report_prediction_stats
+
+
+def sweep_hyperparams_and_collect(
+    class_data: dict[int, dict[str, Any]],
+    alpha_0: np.ndarray,
+    delta_0: np.ndarray,
+    alpha_1: np.ndarray,
+    delta_1: np.ndarray,
+    mode: Literal["beta", "beta-binomial"] = "beta",
+    extra_metrics: dict[str, Callable] | None = None,
+    quiet: bool = True,
+) -> pd.DataFrame:
+    """Sweep (a0,d0,a1,d1), run mondrian_conformal_calibrate + report_prediction_stats,
+    and return a tidy DataFrame with hyperparams + selected metrics.
+
+    This function performs a grid search over hyperparameter combinations and
+    evaluates the resulting conformal prediction performance.
+
+    Parameters
+    ----------
+    class_data : dict
+        Output from split_by_class()
+    alpha_0 : array-like
+        Grid of alpha values for class 0
+    delta_0 : array-like
+        Grid of delta values for class 0
+    alpha_1 : array-like
+        Grid of alpha values for class 1
+    delta_1 : array-like
+        Grid of delta values for class 1
+    mode : str, default="beta"
+        "beta" or "beta-binomial" mode for SSBC
+    extra_metrics : dict of {name: function}, optional
+        Additional metrics to compute. Each function takes the summary dict
+        and returns a scalar value.
+    quiet : bool, default=True
+        If True, suppress progress output
+
+    Returns
+    -------
+    pd.DataFrame
+        Tidy dataframe with one row per hyperparameter combination.
+        Columns include:
+        - a0, d0, a1, d1: hyperparameters
+        - cov: overall coverage rate
+        - sing_rate: singleton prediction rate
+        - err_all: overall singleton error rate
+        - err_pred0, err_pred1: errors by predicted class
+        - err_y0, err_y1: errors by true class
+        - esc_rate: escalation rate (doublets + abstentions)
+        - n_total, sing_count, m_abst, m_doublets: counts
+        - Any additional metrics from extra_metrics
+
+    Examples
+    --------
+    >>> import numpy as np
+    >>> from ssbc import BinaryClassifierSimulator, split_by_class
+    >>>
+    >>> # Generate data
+    >>> sim = BinaryClassifierSimulator(0.1, (2, 8), (8, 2), seed=42)
+    >>> labels, probs = sim.generate(1000)
+    >>> class_data = split_by_class(labels, probs)
+    >>>
+    >>> # Define grid
+    >>> alpha_grid = np.arange(0.05, 0.20, 0.05)
+    >>> delta_grid = np.arange(0.05, 0.20, 0.05)
+    >>>
+    >>> # Run sweep
+    >>> df = sweep_hyperparams_and_collect(
+    ...     class_data,
+    ...     alpha_0=alpha_grid, delta_0=delta_grid,
+    ...     alpha_1=alpha_grid, delta_1=delta_grid,
+    ... )
+    >>>
+    >>> # Analyze results
+    >>> print(df[['a0', 'a1', 'cov', 'sing_rate', 'err_all']].head())
+
+    Notes
+    -----
+    The function performs a complete grid search, so the total number of
+    evaluations is len(alpha_0) × len(delta_0) × len(alpha_1) × len(delta_1).
+    For large grids, this can be computationally expensive.
+    """
+    rows = []
+    combos = list(itertools.product(alpha_0, delta_0, alpha_1, delta_1))
+
+    for a0, d0, a1, d1 in combos:
+        if not quiet:
+            print(f"a0={a0:.3f}, d0={d0:.3f}, a1={a1:.3f}, d1={d1:.3f}")
+
+        cal_result, pred_stats = mondrian_conformal_calibrate(
+            class_data=class_data,
+            alpha_target={0: float(a0), 1: float(a1)},
+            delta={0: float(d0), 1: float(d1)},
+            mode=mode,
+        )
+        summary = report_prediction_stats(pred_stats, cal_result, verbose=False)
+
+        # Robust getter
+        def g(d, *keys, default=None):
+            """Navigate nested dict safely."""
+            cur = d
+            for k in keys:
+                if not isinstance(cur, dict) or k not in cur:
+                    return default
+                cur = cur[k]
+            return cur
+
+        n_total = int(g(summary, "marginal", "n_total", default=0) or 0)
+        cov = float(g(summary, "marginal", "coverage", "rate", default=0.0) or 0.0)
+        sing_rate = float(g(summary, "marginal", "singletons", "rate", default=0.0) or 0.0)
+        sing_cnt = int(g(summary, "marginal", "singletons", "count", default=0) or 0)
+        abst_cnt = int(g(summary, "marginal", "abstentions", "count", default=0) or 0)
+        doub_cnt = int(g(summary, "marginal", "doublets", "count", default=0) or 0)
+        esc_rate = (abst_cnt + doub_cnt) / float(n_total if n_total else 1)
+
+        err_all = float(g(summary, "marginal", "singletons", "errors", "rate", default=0.0) or 0.0)
+        err_p0 = float(g(summary, "marginal", "singletons", "errors_by_pred", "pred_0", "rate", default=0.0) or 0.0)
+        err_p1 = float(g(summary, "marginal", "singletons", "errors_by_pred", "pred_1", "rate", default=0.0) or 0.0)
+
+        err_y0 = float(g(summary, 0, "singletons", "error_given_singleton", "rate", default=0.0) or 0.0)
+        err_y1 = float(g(summary, 1, "singletons", "error_given_singleton", "rate", default=0.0) or 0.0)
+
+        row = {
+            "a0": float(a0),
+            "d0": float(d0),
+            "a1": float(a1),
+            "d1": float(d1),
+            "cov": cov,
+            "sing_rate": sing_rate,
+            "err_all": err_all,
+            "err_pred0": err_p0,
+            "err_pred1": err_p1,
+            "err_y0": err_y0,
+            "err_y1": err_y1,
+            "esc_rate": esc_rate,
+            "n_total": int(n_total),
+            "sing_count": int(sing_cnt),
+            "m_abst": abst_cnt,
+            "m_doublets": doub_cnt,
+        }
+
+        if extra_metrics:
+            for name, fn in extra_metrics.items():
+                try:
+                    row[name] = fn(summary)
+                except Exception:
+                    row[name] = np.nan
+
+        rows.append(row)
+
+    df = pd.DataFrame(rows)
+    return df.sort_values(["a0", "d0", "a1", "d1"], kind="mergesort").reset_index(drop=True)
+
+
+def sweep_and_plot_parallel_plotly(
+    class_data: dict[int, dict[str, Any]],
+    delta_0: np.ndarray,
+    delta_1: np.ndarray,
+    alpha_0: np.ndarray,
+    alpha_1: np.ndarray,
+    mode: Literal["beta", "beta-binomial"] = "beta",
+    extra_metrics: dict[str, Callable] | None = None,
+    color: str = "err_all",
+    color_continuous_scale=None,
+    title: str | None = None,
+    height: int = 600,
+):
+    """Convenience wrapper: run sweep + show plotly parallel coordinates figure.
+
+    This function combines hyperparameter sweep and visualization in one call.
+
+    Parameters
+    ----------
+    class_data : dict
+        Output from split_by_class()
+    delta_0, delta_1 : array-like
+        Grid of delta values for classes 0 and 1
+    alpha_0, alpha_1 : array-like
+        Grid of alpha values for classes 0 and 1
+    mode : str, default="beta"
+        "beta" or "beta-binomial" mode for SSBC
+    extra_metrics : dict of {name: function}, optional
+        Additional metrics to compute
+    color : str, default='err_all'
+        Column to use for coloring the parallel coordinates
+    color_continuous_scale : plotly colorscale, optional
+        Color scale for the plot
+    title : str, optional
+        Plot title (defaults to auto-generated title)
+    height : int, default=600
+        Plot height in pixels
+
+    Returns
+    -------
+    df : pd.DataFrame
+        Results dataframe
+    fig : plotly.graph_objects.Figure
+        Interactive parallel coordinates plot
+
+    Examples
+    --------
+    >>> import numpy as np
+    >>> from ssbc import BinaryClassifierSimulator, split_by_class
+    >>>
+    >>> # Generate data
+    >>> sim = BinaryClassifierSimulator(0.1, (2, 8), (8, 2), seed=42)
+    >>> labels, probs = sim.generate(1000)
+    >>> class_data = split_by_class(labels, probs)
+    >>>
+    >>> # Run sweep and plot
+    >>> df, fig = sweep_and_plot_parallel_plotly(
+    ...     class_data,
+    ...     delta_0=np.arange(0.05, 0.20, 0.05),
+    ...     delta_1=np.arange(0.05, 0.20, 0.05),
+    ...     alpha_0=np.arange(0.05, 0.20, 0.05),
+    ...     alpha_1=np.arange(0.05, 0.20, 0.05),
+    ...     color='err_all'
+    ... )
+    >>> fig.show()  # Display in notebook
+    >>> # Or save: fig.write_html("sweep_results.html")
+
+    Notes
+    -----
+    The parallel coordinates plot allows interactive exploration of the
+    hyperparameter space. You can brush (select) ranges on any axis to
+    filter configurations and see their impact on other metrics.
+    """
+    df = sweep_hyperparams_and_collect(
+        class_data=class_data,
+        alpha_0=alpha_0,
+        delta_0=delta_0,
+        alpha_1=alpha_1,
+        delta_1=delta_1,
+        mode=mode,
+        extra_metrics=extra_metrics,
+        quiet=True,
+    )
+
+    if title is None:
+        title = f"Mondrian Hyperparameter Sweep (n={len(df)} configs)"
+
+    fig = plot_parallel_coordinates_plotly(
+        df, color=color, color_continuous_scale=color_continuous_scale, title=title, height=height
+    )
+
+    return df, fig

ssbc/simulation.py

@@ -0,0 +1,148 @@
+"""Simulation utilities for testing conformal prediction."""
+
+import numpy as np
+
+
+class BinaryClassifierSimulator:
+    """Simulate binary classification data with probabilities from Beta distributions.
+
+    This simulator generates realistic classification scenarios where the predicted
+    probabilities for each class follow Beta distributions. Useful for testing and
+    benchmarking conformal prediction methods.
+
+    Parameters
+    ----------
+    p_class1 : float
+        Probability of drawing class 1 (class imbalance parameter)
+        Must be in [0, 1]
+    beta_params_class0 : tuple of (a, b)
+        Beta distribution parameters for p(class=1) when true label is 0
+        Typically use parameters that give low probabilities (e.g., (2, 8))
+    beta_params_class1 : tuple of (a, b)
+        Beta distribution parameters for p(class=1) when true label is 1
+        Typically use parameters that give high probabilities (e.g., (8, 2))
+    seed : int, optional
+        Random seed for reproducibility
+
+    Attributes
+    ----------
+    p_class1 : float
+        Probability of class 1
+    p_class0 : float
+        Probability of class 0 (= 1 - p_class1)
+    a0, b0 : float
+        Beta parameters for class 0
+    a1, b1 : float
+        Beta parameters for class 1
+    rng : numpy.random.Generator
+        Random number generator
+
+    Examples
+    --------
+    >>> # Simulate imbalanced data: 10% positive class
+    >>> # Class 0: Beta(2, 8) → mean p(class=1) = 0.2 (low scores, correct)
+    >>> # Class 1: Beta(8, 2) → mean p(class=1) = 0.8 (high scores, correct)
+    >>> sim = BinaryClassifierSimulator(
+    ...     p_class1=0.10,
+    ...     beta_params_class0=(2, 8),
+    ...     beta_params_class1=(8, 2),
+    ...     seed=42
+    ... )
+    >>> labels, probs = sim.generate(n_samples=100)
+    >>> print(labels.shape)
+    (100,)
+    >>> print(probs.shape)
+    (100, 2)
+
+    Notes
+    -----
+    The Beta distribution parameters (a, b) control the shape:
+    - Mean = a / (a + b)
+    - For a classifier that works well:
+      - Class 0 should have low p(class=1): use (a, b) with a < b
+      - Class 1 should have high p(class=1): use (a, b) with a > b
+    """
+
+    def __init__(
+        self,
+        p_class1: float,
+        beta_params_class0: tuple[float, float],
+        beta_params_class1: tuple[float, float],
+        seed: int | None = None,
+    ):
+        """Initialize the binary classifier simulator."""
+        if not 0 <= p_class1 <= 1:
+            raise ValueError("p_class1 must be in [0, 1]")
+
+        self.p_class1 = p_class1
+        self.p_class0 = 1.0 - p_class1
+        self.a0, self.b0 = beta_params_class0
+        self.a1, self.b1 = beta_params_class1
+        self.rng = np.random.default_rng(seed)
+
+        # Validate beta parameters
+        if self.a0 <= 0 or self.b0 <= 0:
+            raise ValueError("Beta parameters for class 0 must be positive")
+        if self.a1 <= 0 or self.b1 <= 0:
+            raise ValueError("Beta parameters for class 1 must be positive")
+
+    def generate(self, n_samples: int) -> tuple[np.ndarray, np.ndarray]:
+        """Generate n_samples of (label, p(class=0), p(class=1)).
+
+        Parameters
+        ----------
+        n_samples : int
+            Number of samples to generate
+
+        Returns
+        -------
+        labels : np.ndarray, shape (n_samples,)
+            True binary labels (0 or 1)
+        probs : np.ndarray, shape (n_samples, 2)
+            Classification probabilities [p(class=0), p(class=1)]
+            Each row sums to 1.0
+
+        Examples
+        --------
+        >>> sim = BinaryClassifierSimulator(
+        ...     p_class1=0.5,
+        ...     beta_params_class0=(2, 8),
+        ...     beta_params_class1=(8, 2),
+        ...     seed=42
+        ... )
+        >>> labels, probs = sim.generate(n_samples=5)
+        >>> print(f"Generated {len(labels)} samples")
+        Generated 5 samples
+        >>> print(f"Class balance: {np.bincount(labels)}")
+        Class balance: [2 3]
+        """
+        if n_samples <= 0:
+            raise ValueError("n_samples must be positive")
+
+        # Draw true labels according to class distribution
+        labels = self.rng.choice([0, 1], size=n_samples, p=[self.p_class0, self.p_class1])
+
+        # Initialize probability array
+        probs = np.zeros((n_samples, 2))
+
+        # For each label, draw classification probability from appropriate Beta
+        for i, label in enumerate(labels):
+            if label == 0:
+                # True label is 0: sample p(class=1) from Beta(a0, b0)
+                p_class1 = self.rng.beta(self.a0, self.b0)
+            else:
+                # True label is 1: sample p(class=1) from Beta(a1, b1)
+                p_class1 = self.rng.beta(self.a1, self.b1)
+
+            probs[i, 1] = p_class1  # p(class=1)
+            probs[i, 0] = 1.0 - p_class1  # p(class=0)
+
+        return labels, probs
+
+    def __repr__(self) -> str:
+        """String representation of the simulator."""
+        return (
+            f"BinaryClassifierSimulator(p_class1={self.p_class1:.3f}, "
+            f"beta_class0=({self.a0}, {self.b0}), "
+            f"beta_class1=({self.a1}, {self.b1}))"
+        )

ssbc/ssbc.py

@@ -0,0 +1 @@
+"""Main module."""

ssbc/statistics.py

@@ -0,0 +1,158 @@
+"""Statistical utility functions for SSBC."""
+
+from typing import Any
+
+import numpy as np
+from scipy import stats
+
+
+def clopper_pearson_intervals(labels: np.ndarray, confidence: float = 0.95) -> dict[int, dict[str, Any]]:
+    """Compute Clopper-Pearson (exact binomial) confidence intervals for class prevalences.
+
+    Parameters
+    ----------
+    labels : np.ndarray
+        Binary labels (0 or 1)
+    confidence : float, default=0.95
+        Confidence level (e.g., 0.95 for 95% CI)
+
+    Returns
+    -------
+    dict
+        Dictionary with keys 0 and 1, each containing:
+        - 'count': number of samples in this class
+        - 'proportion': observed proportion
+        - 'lower': lower bound of CI
+        - 'upper': upper bound of CI
+
+    Examples
+    --------
+    >>> labels = np.array([0, 0, 1, 1, 0])
+    >>> intervals = clopper_pearson_intervals(labels, confidence=0.95)
+    >>> print(intervals[0]['proportion'])
+    0.6
+
+    Notes
+    -----
+    The Clopper-Pearson interval is an exact binomial confidence interval
+    based on Beta distribution quantiles. It provides conservative coverage
+    guarantees.
+    """
+    alpha = 1 - confidence
+    n_total = len(labels)
+
+    intervals = {}
+
+    for label in [0, 1]:
+        count = np.sum(labels == label)
+        proportion = count / n_total
+
+        # Clopper-Pearson uses Beta distribution quantiles
+        # Lower bound: Beta(count, n-count+1) at alpha/2
+        # Upper bound: Beta(count+1, n-count) at 1-alpha/2
+
+        if count == 0:
+            lower = 0.0
+            upper = stats.beta.ppf(1 - alpha / 2, count + 1, n_total - count)
+        elif count == n_total:
+            lower = stats.beta.ppf(alpha / 2, count, n_total - count + 1)
+            upper = 1.0
+        else:
+            lower = stats.beta.ppf(alpha / 2, count, n_total - count + 1)
+            upper = stats.beta.ppf(1 - alpha / 2, count + 1, n_total - count)
+
+        intervals[label] = {"count": count, "proportion": proportion, "lower": lower, "upper": upper}
+
+    return intervals
+
+
+def cp_interval(count: int, total: int, confidence: float = 0.95) -> dict[str, float]:
+    """Compute Clopper-Pearson exact confidence interval.
+
+    Helper function for computing a single CI from count and total.
+
+    Parameters
+    ----------
+    count : int
+        Number of successes
+    total : int
+        Total number of trials
+    confidence : float, default=0.95
+        Confidence level
+
+    Returns
+    -------
+    dict
+        Dictionary with keys:
+        - 'count': original count
+        - 'proportion': count/total
+        - 'lower': lower CI bound
+        - 'upper': upper CI bound
+    """
+    alpha = 1 - confidence
+    count = int(count)
+    total = int(total)
+
+    if total <= 0:
+        return {"count": count, "proportion": 0.0, "lower": 0.0, "upper": 0.0}
+
+    p = count / total
+
+    if count == 0:
+        lower = 0.0
+        upper = stats.beta.ppf(1 - alpha / 2, 1, total)
+    elif count == total:
+        lower = stats.beta.ppf(alpha / 2, total, 1)
+        upper = 1.0
+    else:
+        lower = stats.beta.ppf(alpha / 2, count, total - count + 1)
+        upper = stats.beta.ppf(1 - alpha / 2, count + 1, total - count)
+
+    return {"count": count, "proportion": float(p), "lower": float(lower), "upper": float(upper)}
+
+
+def ensure_ci(d: dict[str, Any], count: int, total: int, confidence: float = 0.95) -> tuple[float, float, float]:
+    """Extract or compute rate and confidence interval from a dictionary.
+
+    If the dictionary already contains rate/CI information, use it.
+    Otherwise, compute Clopper-Pearson CI from count/total.
+
+    Parameters
+    ----------
+    d : dict
+        Dictionary that may contain 'rate'/'proportion' and 'lower'/'upper'
+    count : int
+        Count for CI computation (if needed)
+    total : int
+        Total for CI computation (if needed)
+    confidence : float, default=0.95
+        Confidence level
+
+    Returns
+    -------
+    tuple of (rate, lower, upper)
+        Rate and confidence interval bounds
+    """
+    # Try to get existing rate
+    r = None
+    if isinstance(d, dict):
+        if "rate" in d:
+            r = float(d["rate"])
+        elif "proportion" in d:
+            r = float(d["proportion"])
+
+    # Try to get existing CI
+    lo, hi = 0.0, 0.0
+    if isinstance(d, dict):
+        if "ci_95" in d and isinstance(d["ci_95"], tuple | list) and len(d["ci_95"]) == 2:
+            lo, hi = float(d["ci_95"][0]), float(d["ci_95"][1])
+        else:
+            lo = float(d.get("lower", 0.0))
+            hi = float(d.get("upper", 0.0))
+
+    # If missing or invalid, compute CP interval
+    if r is None or (lo == 0.0 and hi == 0.0 and (count > 0 or total > 0)):
+        ci = cp_interval(count, total, confidence)
+        return ci["proportion"], ci["lower"], ci["upper"]
+
+    return float(r), float(lo), float(hi)

ssbc/utils.py

@@ -0,0 +1,2 @@
+def do_something_useful():
+    print("Replace this with a utility function")