score-select 0.1.1__py3-none-any.whl

SCoRE/SCoRE.py

@@ -0,0 +1,497 @@
+import numpy as np
+from .utility import BH, eBH
+
+# implementation of SCoRE procedures
+
+def _uniform_random(random_state, size=None):
+    if random_state is None:
+        return np.random.uniform(0, 1, size)
+    if isinstance(random_state, np.random.Generator):
+        return random_state.uniform(0, 1, size)
+    return np.random.default_rng(random_state).uniform(0, 1, size)
+
+
+def _as_index_array(sel):
+    return np.asarray(sel, dtype=int)
+
+
+def _as_1d_array(name, values):
+    arr = np.asarray(values)
+    if arr.ndim != 1:
+        raise ValueError(f"{name} must be a one-dimensional array.")
+    return arr
+
+
+def _split_calib(Dcalib):
+    if not isinstance(Dcalib, (tuple, list)) or len(Dcalib) != 2:
+        raise ValueError("Dcalib must be a tuple or list of losses and scores (Lcalib, Scalib).")
+
+    Lcalib = _as_1d_array("Lcalib", Dcalib[0])
+    Scalib = _as_1d_array("Scalib", Dcalib[1])
+    if len(Lcalib) != len(Scalib):
+        raise ValueError("The losses and scores (Lcalib, Scalib) must have the same length.")
+    return Lcalib, Scalib
+
+
+def _is_legacy_dtest(Dtest):
+    if not isinstance(Dtest, (tuple, list)) or len(Dtest) != 2:
+        return False
+    if np.ndim(Dtest[1]) == 0:
+        return False
+    return Dtest[0] is None or np.ndim(Dtest[0]) > 0
+
+
+def _get_stest(Dtest):
+    if _is_legacy_dtest(Dtest):
+        Dtest = Dtest[1]
+    return _as_1d_array("Dtest", Dtest)
+
+
+def _validate_binary_loss(Lcalib):
+    if not np.all(np.isin(Lcalib, [0, 1])):
+        raise ValueError("Conformal selection requires binary calibration losses in {0, 1}.")
+
+
+def _validate_alpha(alpha):
+    alpha = float(alpha)
+    if not np.isfinite(alpha) or alpha <= 0 or alpha > 1:
+        raise ValueError("alpha must be in (0, 1]")
+    return alpha
+
+
+def _validate_gamma(gamma):
+    gamma = float(gamma)
+    if not np.isfinite(gamma) or gamma < 0 or gamma > 1:
+        raise ValueError("gamma must be in [0, 1]")
+    return gamma
+
+
+def _validate_prune(prune):
+    if prune not in (None, "hete", "homo"):
+        raise ValueError("prune must be one of None, 'hete', or 'homo'")
+    return prune
+
+
+def CS(Dcalib, Dtest, alpha, mult_test=True, return_pvals=False):
+    """Conformal Selection (CS) procedure for binary losses that controls the marginal deployment risk (MDR) or selective deployment risk (SDR).
+    Here, MDR reduces to the average type-I error and SDR reduces to the usual false discovery rate (FDR).
+    
+    The function applies only when the loss function evaluates strictly to {0,1}.
+    
+    Args:
+        Dcalib (tuple): A tuple containing losses and scores (Lcalib, Scalib) for the calibration set.
+        Dtest (array-like): Test scores Stest. A legacy tuple/list (ignored, Stest) is also accepted.
+        alpha (float): The target error margin.
+        mult_test (bool): Whether to perform multiple testing correction using the Benjamini-Hochberg (BH) procedure. If False, MDR is controlled; otherwise SDR is controlled.
+        return_pvals (bool): If True, returns the calculated p-values alongside the selected indices.
+        
+    Returns:
+        Union[np.ndarray, tuple]: Selected indices, or (selected indices, p-values) if return_pvals is True.
+    """
+    alpha = _validate_alpha(alpha)
+    Lcalib, Scalib = _split_calib(Dcalib)
+    _validate_binary_loss(Lcalib)
+    Stest = _get_stest(Dtest)
+    Ncalib, Ntest = len(Scalib), len(Stest)
+
+    calib_scores = 1000 * (Lcalib == 0) + Scalib
+    test_scores = Stest
+    
+    pvals = np.zeros(Ntest)
+    for j in range(Ntest):
+        pvals[j] = (1 + np.sum(calib_scores <= test_scores[j])) / (Ncalib + 1)
+
+    if mult_test:
+        sel = BH(pvals, alpha)
+    else:
+        sel = np.flatnonzero(pvals <= alpha)
+
+    if not return_pvals:
+        return _as_index_array(sel)
+    return sel, pvals
+
+def SCoRE_MDR_bf(Dcalib, Dtest, alpha, gamma, return_evals=False):
+    """Brute-force algorithm for SCoRE testing with Marginal Deployment Risk (MDR) control. The algorithm manually search for a suitable cutoff t.
+    Compared to SCoRE_MDR, this brute-force computation enables computing the SCoRE e-values explicitly.
+    
+    Args:
+        Dcalib (tuple): A tuple containing losses and scores (Lcalib, Scalib) for the calibration set.
+        Dtest (array-like): Test scores Stest. A legacy tuple/list (ignored, Stest) is also accepted.
+        alpha (float): The target error margin.
+        gamma (float): A tuning parameter spanning [0, 1]. Recommended value is gamma=alpha.
+        return_evals (bool): Whether to output the computed e-values. Defaults to False.
+        
+    Returns:
+        Union[np.ndarray, tuple]: Selected indices, or (selected indices, e-values) if return_evals is True.
+    """
+    alpha = _validate_alpha(alpha)
+    gamma = _validate_gamma(gamma)
+    Lcalib, Scalib = _split_calib(Dcalib)
+    Stest = _get_stest(Dtest)
+    Ncalib, Ntest = len(Scalib), len(Stest)
+
+    M = list(np.concatenate([Scalib, Stest]))
+
+    def F(j, t, l):
+        return (np.sum(Lcalib * (Scalib <= t)) + l * (Stest[j] <= t)) / (Ncalib + 1)
+    
+    def t_gamma(j, l):
+        max_t = -np.inf
+        for cur_t in M:
+            if F(j, cur_t, l) <= gamma:
+                max_t = max(max_t, cur_t)
+        return max_t
+    
+    sel = []
+    evalues = np.zeros(Ntest)
+
+    for i_itr in range(Ntest):
+        evalue = np.inf
+        for l in [0, 1]:
+            t_l = t_gamma(i_itr, l)
+            num = (Ncalib + 1) * (Stest[i_itr] <= t_l)
+            denom = np.sum(Lcalib * (Scalib <= t_l)) + l * (Stest[i_itr] <= t_l)
+
+            evalue = min(evalue, num / denom)
+        evalues[i_itr] = evalue
+
+        phi = (evalue >= (1 / alpha))
+        if phi == 1:
+            sel.append(i_itr)
+
+    if not return_evals:
+        return _as_index_array(sel)
+    return _as_index_array(sel), evalues
+
+def SCoRE_MDR(Dcalib, Dtest, alpha, gamma):
+    """SCoRE testing procedure with Marginal Deployment Risk (MDR) control, implemented using the computational shortcut. Note the e-values are not directly available with this shortcut.
+    
+    Args:
+        Dcalib (tuple): A tuple containing losses and scores (Lcalib, Scalib) for the calibration set.
+        Dtest (array-like): Test scores Stest. A legacy tuple/list (ignored, Stest) is also accepted.
+        alpha (float): The target error margin. 
+        gamma (float): A tuning parameter spanning [0, 1]. Recommended value is gamma=alpha.
+        
+    Returns:
+        list: A list of selected instances with low risk and deemed safe to deploy.
+    """
+    alpha = _validate_alpha(alpha)
+    gamma = _validate_gamma(gamma)
+    Lcalib, Scalib = _split_calib(Dcalib)
+    Stest = _get_stest(Dtest)
+    Ncalib, Ntest = len(Scalib), len(Stest)
+
+    sel = []
+
+    for i_itr in range(Ntest):
+        phi = (1 + np.sum(Lcalib * (Scalib <= Stest[i_itr]))) / (Ncalib + 1) <= gamma
+
+        if gamma > alpha and phi == 1: # need to check the 2nd condition
+            M = list(np.concatenate([Scalib, Stest]))
+            for t in M:
+                upp = (1 + np.sum(Lcalib * (Scalib <= t))) / (Ncalib + 1)
+                low = upp - 1 / (Ncalib + 1)
+
+                # check whether (alpha, gamma] and [low, upp] overlap
+                if not ((upp <= alpha) or (low > gamma)): # overlap
+                    phi = 0
+                    break
+
+        if phi == 1: # selected
+            sel.append(i_itr)
+
+    return _as_index_array(sel)
+
+def SCoRE_MDR_w(Dcalib, Dtest, wcalib, wtest, alpha, gamma):
+    """SCoRE testing procedure with Marginal Deployment Risk (MDR) control under the covariate shift case, implemented using the computational shortcut.
+    
+    Args:
+        Dcalib (tuple): A tuple containing losses and scores (Lcalib, Scalib) for the calibration set.
+        Dtest (array-like): Test scores Stest. A legacy tuple/list (ignored, Stest) is also accepted.
+        wcalib (np.ndarray): The covariate shift weights for the calibration data.
+        wtest (np.ndarray): The covariate shift weights for the test data.
+        alpha (float): The target error margin.
+        gamma (float): A tuning parameter spanning [0, 1]. Recommended value is gamma=alpha.
+        
+    Returns:
+        list: A list of selected instances with low risk and deemed safe to deploy.
+    """
+    alpha = _validate_alpha(alpha)
+    gamma = _validate_gamma(gamma)
+    Lcalib, Scalib = _split_calib(Dcalib)
+    Stest = _get_stest(Dtest)
+    wcalib = _as_1d_array("wcalib", wcalib)
+    wtest = _as_1d_array("wtest", wtest)
+    Ncalib, Ntest = len(Scalib), len(Stest)
+    if len(wcalib) != Ncalib:
+        raise ValueError("wcalib must have the same length as Lcalib and Scalib.")
+    if len(wtest) != Ntest:
+        raise ValueError("wtest must have the same length as Stest.")
+
+    sel = []
+
+    calib_w_sum = np.sum(wcalib)
+    for i_itr in range(Ntest):
+        phi = (wtest[i_itr] + np.sum(wcalib * Lcalib * (Scalib <= Stest[i_itr]))) / (wtest[i_itr] + calib_w_sum) <= gamma
+
+        if gamma > alpha and phi == 1: # need to check the 2nd condition
+            M = list(np.concatenate([Scalib, Stest]))
+            for t in M:
+                upp = (wtest[i_itr] + np.sum(wcalib * Lcalib * (Scalib <= t))) / (wtest[i_itr] + calib_w_sum)
+                low = upp - wtest[i_itr] / (wtest[i_itr] + calib_w_sum)
+
+                # check whether (alpha, gamma] and [low, upp] overlap
+                if not ((upp <= alpha) or (low > gamma)): # overlap
+                    phi = 0
+                    break
+
+        if phi == 1: # selected
+            sel.append(i_itr)
+
+    return _as_index_array(sel)
+
+######## SDR ########
+
+def SCoRE_SDR(Dcalib, Dtest, alpha, gamma, prune=None, return_evals=False, random_state=None):
+    """SCoRE testing procedure for Selective Deployment Risk (SDR) control. Optimized implementation with time complexity $O(m(n+m) + (n+m)\\log(n+m))$.
+    
+    Args:
+        Dcalib (tuple): losses and scores (Lcalib, Scalib) for the calibration set.
+        Dtest (array-like): Test scores Stest. A legacy tuple/list (ignored, Stest) is also accepted.
+        alpha (float): The target error margin.
+        gamma (float): A tuning parameter spanning [0, 1]. Recommended value is gamma=alpha.
+        prune (str, optional): Optional boosting strategy (either 'hete' or 'homo'). Use of 'homo' is generally recommended.
+        return_evals (bool, optional): Returns computed e-values if True.
+        random_state (int or np.random.Generator, optional): Random seed or generator used when pruning is enabled. Randomization is only needed for the boosting strategies.
+        
+    Returns:
+        Union[list, tuple]: Selection set indices, or combined tuple depending on `return_evals`.
+    """
+    alpha = _validate_alpha(alpha)
+    gamma = _validate_gamma(gamma)
+    prune = _validate_prune(prune)
+    Lcalib, Scalib = _split_calib(Dcalib)
+    Stest = _get_stest(Dtest)
+    Ncalib, Ntest = len(Scalib), len(Stest)
+
+    Scalib_tagged = [(lp, l, 'calib') for lp, l in zip(Scalib, Lcalib)]
+    Stest_tagged = [(lp, 0, 'test') for lp in Stest] # 0 is dummy value
+
+    M_tagged = Scalib_tagged + Stest_tagged
+    M_tagged.sort()
+
+    M = np.array([a[0] for a in M_tagged])
+
+    evalues = np.zeros(Ntest)
+
+    # some intermediate prefix sums
+    NUMER = np.zeros(Ncalib + Ntest) # for np.sum(Lcalib * (Scalib <= t)) with t being the i-th ranked value in M
+    DENOM = np.zeros(Ncalib + Ntest) # for 1 + np.sum(Stest <= t).
+    for i, (t, L, l_type) in enumerate(M_tagged):
+        NUMER[i] = (NUMER[i-1] if i != 0 else 0)
+        DENOM[i] = (DENOM[i-1] if i != 0 else 1)
+        if l_type == 'calib':
+            NUMER[i] += L
+        else:
+            DENOM[i] += 1
+            
+    # above will have a bug when there are ties in M_tagged. 
+    # for example, if M_tagged = [(0.5, 0, 'calib'), (0.5, 1, 'calib')], then NUMER[0] = 0, NUMER[1] = 1.
+    # But for t = 0.5, we should have NUMER = 1. So we need to correct for ties.
+    for i in range(len(M_tagged) - 2, -1, -1):
+        if M_tagged[i][0] == M_tagged[i+1][0]:
+            NUMER[i] = NUMER[i+1]
+            DENOM[i] = DENOM[i+1]
+
+    for j in range(Ntest):
+        # we precompute all FR, t_gamma, and ell
+        FR_0 = np.zeros(Ncalib + Ntest)
+        FR_1 = np.zeros(Ncalib + Ntest)
+
+        ELL = np.zeros(Ncalib + Ntest)
+
+        # pairs of (i, t)
+        t_0, t_1 = (-1, -np.inf), (-1, -np.inf)
+
+        # compute FR and ell
+        for i, (t, _, _) in enumerate(M_tagged):
+            FR_0[i] = NUMER[i] / (DENOM[i] - (Stest[j] <= t)) / (Ncalib + 1) * Ntest
+            FR_1[i] = (NUMER[i] + (Stest[j] <= t)) / (DENOM[i] - (Stest[j] <= t)) / (Ncalib + 1) * Ntest
+
+            ELL[i] = (Ncalib + 1) * gamma / Ntest * (DENOM[i] - (Stest[j] <= t)) - NUMER[i]
+
+        # compute t_gamma. Also store the original ranking i
+        for i, t in enumerate(M):
+            if FR_0[i] <= gamma:
+                t_0 = (i, t)
+            if FR_1[i] <= gamma:
+                t_1 = (i, t)
+
+        if Stest[j] > t_1[1]:
+            continue # e-value is zero
+
+        if t_1[1] == t_0[1]:
+            evalues[j] = (Ncalib + 1) / (1 + NUMER[t_1[0]])
+            continue # same upper/lower bound case
+
+        max_ell = np.zeros(Ntest + Ncalib) # max_ell[rank(t)]: max of l(t') with t' > t, t' in M, and FR(t', 0) <= gamma. 
+                                           # max_ell[0] correspond to the smallest t in M, max_ell[-1] correspond to the largest t in M.
+        last_max = -np.inf
+        for i, t in zip(range(Ntest + Ncalib - 1, -1, -1), reversed(M)): # n+m iterations
+            max_ell[i] = last_max
+
+            if FR_0[i] <= gamma:
+                last_max = max(last_max, ELL[i]) # both O(n+m)
+
+        M_star = [] # store pairs of (i, t)
+        for i, t in enumerate(M):
+            if t < max(Stest[j], t_1[1]):
+                continue # this is to keep the index i
+            if t > t_0[1]:
+                break
+
+            if FR_0[i] <= gamma and ELL[i] > max_ell[i]:
+                M_star.append((i, t))
+
+        evalue = np.inf
+        for i, t in M_star:
+            cur_val = (Ncalib + 1) / (ELL[i] + NUMER[i])
+            evalue = min(evalue, cur_val)
+        
+        evalues[j] = evalue
+    
+    if prune == 'hete':
+        evalues /= _uniform_random(random_state, len(evalues))
+    if prune == 'homo':
+        evalues /= _uniform_random(random_state)
+    sel = eBH(evalues, alpha)
+    
+    if not return_evals:
+        return _as_index_array(sel)
+    return sel, evalues
+
+def SCoRE_SDR_w(Dcalib, Dtest, wcalib, wtest, alpha, gamma, prune=None, return_evals=False, random_state=None):
+    """SCoRE testing procedure for Selective Deployment Risk (SDR) control under the covariate shift case. Optimized implementation with time complexity $O(m(n+m) + (n+m)\\log(n+m))$.
+    
+    Args:
+        Dcalib (tuple): losses and scores (Lcalib, Scalib) for the calibration set.
+        Dtest (array-like): Test scores Stest. A legacy tuple/list (ignored, Stest) is also accepted.
+        wcalib (np.ndarray): The covariate shift weights for the calibration data.
+        wtest (np.ndarray): The covariate shift weights for the test data.
+        alpha (float): The target error margin.
+        gamma (float): A tuning parameter spanning [0, 1]. Recommended value is gamma=alpha.
+        prune (str, optional): Optional boosting strategy (either 'hete' or 'homo'). Use of 'homo' is generally recommended.
+        return_evals (bool, optional): Returns computed e-values if True.
+        random_state (int or np.random.Generator, optional): Random seed or generator used when pruning is enabled.
+        
+    Returns:
+        Union[list, tuple]: Selection set indices, or combined tuple depending on `return_evals`.
+    """
+    alpha = _validate_alpha(alpha)
+    gamma = _validate_gamma(gamma)
+    prune = _validate_prune(prune)
+    Lcalib, Scalib = _split_calib(Dcalib)
+    Stest = _get_stest(Dtest)
+    wcalib = _as_1d_array("wcalib", wcalib)
+    wtest = _as_1d_array("wtest", wtest)
+    Ncalib, Ntest = len(Scalib), len(Stest)
+    if len(wcalib) != Ncalib:
+        raise ValueError("wcalib must have the same length as Lcalib and Scalib.")
+    if len(wtest) != Ntest:
+        raise ValueError("wtest must have the same length as Stest.")
+
+    Scalib_tagged = [(lp, l, w, 'calib') for lp, l, w in zip(Scalib, Lcalib, wcalib)]
+    Stest_tagged = [(lp, 0, w, 'test') for lp, w in zip(Stest, wtest)] # 0 is dummy value
+
+    M_tagged = Scalib_tagged + Stest_tagged
+    M_tagged.sort()
+
+    M = np.array([a[0] for a in M_tagged])
+
+    evalues = np.zeros(Ntest)
+
+    calib_w_sum = np.sum(wcalib)
+
+    # some intermediate prefix sums
+    NUMER = np.zeros(Ncalib + Ntest) # for np.sum(wcalib * Lcalib * (Scalib <= t)) with t being the i-th ranked value in M
+    DENOM = np.zeros(Ncalib + Ntest) # for 1 + np.sum(Stest <= t).
+    for i, (t, L, w, l_type) in enumerate(M_tagged):
+        NUMER[i] = (NUMER[i-1] if i != 0 else 0)
+        DENOM[i] = (DENOM[i-1] if i != 0 else 1)
+        if l_type == 'calib':
+            NUMER[i] += w * L
+        else:
+            DENOM[i] += 1
+
+    # Correction for ties
+    for i in range(len(M_tagged) - 2, -1, -1):
+        if M_tagged[i][0] == M_tagged[i+1][0]:
+            NUMER[i] = NUMER[i+1]
+            DENOM[i] = DENOM[i+1]
+
+    for j in range(Ntest):
+        # we precompute all FR, t_gamma, and ell
+        FR_0 = np.zeros(Ncalib + Ntest)
+        FR_1 = np.zeros(Ncalib + Ntest)
+
+        ELL = np.zeros(Ncalib + Ntest)
+
+        # pairs of (i, t)
+        t_0, t_1 = (-1, -np.inf), (-1, -np.inf)
+
+        # compute FR and ell
+        for i, (t, _, _, _) in enumerate(M_tagged):
+            FR_0[i] = NUMER[i] / (DENOM[i] - (Stest[j] <= t)) / (calib_w_sum + wtest[j]) * Ntest
+            FR_1[i] = (NUMER[i] + wtest[j] * (Stest[j] <= t)) / (DENOM[i] - (Stest[j] <= t)) / (calib_w_sum + wtest[j]) * Ntest
+
+            ELL[i] = (calib_w_sum + wtest[j]) / wtest[j] * gamma / Ntest * (DENOM[i] - (Stest[j] <= t)) - NUMER[i] / wtest[j]
+
+        # compute t_gamma. Also store the original ranking i
+        for i, t in enumerate(M):
+            if FR_0[i] <= gamma:
+                t_0 = (i, t)
+            if FR_1[i] <= gamma:
+                t_1 = (i, t)
+
+        if Stest[j] > t_1[1]:
+            continue # e-value is zero
+
+        if t_1[1] == t_0[1]:
+            evalues[j] = (calib_w_sum + wtest[j]) / (wtest[j] + NUMER[t_1[0]])
+            continue # same upper/lower bound case
+
+        max_ell = np.zeros(Ntest + Ncalib) # max_ell[rank(t)]: max of l(t') with t' > t, t' in M, and FR(t', 0) <= gamma. 
+                                           # max_ell[0] correspond to the smallest t in M, max_ell[-1] correspond to the largest t in M.
+        last_max = -np.inf
+        for i, t in zip(range(Ntest + Ncalib - 1, -1, -1), reversed(M)): # n+m iterations
+            max_ell[i] = last_max
+
+            if FR_0[i] <= gamma:
+                last_max = max(last_max, ELL[i]) # both O(n+m)
+
+        M_star = [] # store pairs of (i, t)
+        for i, t in enumerate(M):
+            if t < max(Stest[j], t_1[1]):
+                continue # this is to keep the index i
+            if t > t_0[1]:
+                break
+
+            if FR_0[i] <= gamma and ELL[i] > max_ell[i]:
+                M_star.append((i, t))
+
+        evalue = np.inf
+        for i, t in M_star:
+            cur_val = (calib_w_sum + wtest[j]) / (wtest[j] * ELL[i] + NUMER[i])
+            evalue = min(evalue, cur_val)
+        
+        evalues[j] = evalue
+    
+    if prune == 'hete':
+        evalues /= _uniform_random(random_state, len(evalues))
+    if prune == 'homo':
+        evalues /= _uniform_random(random_state)
+    sel = eBH(evalues, alpha)
+    
+    if not return_evals:
+        return _as_index_array(sel)
+    return sel, evalues

SCoRE/__init__.py

@@ -0,0 +1,46 @@
+"""Public API for SCoRE."""
+
+from .SCoRE import (
+    CS,
+    SCoRE_MDR,
+    SCoRE_MDR_bf,
+    SCoRE_MDR_w,
+    SCoRE_SDR,
+    SCoRE_SDR_w,
+)
+from .utility import (
+    BH,
+    Lpredictor,
+    eBH,
+    eval_MDR,
+    eval_SDR,
+    gen_data_1,
+    gen_data_2,
+    gen_data_Jin2023,
+    loss_1,
+    loss_2,
+    loss_Jin2023,
+)
+
+__version__ = "0.1.1"
+
+__all__ = [
+    "__version__",
+    "BH",
+    "CS",
+    "Lpredictor",
+    "SCoRE_MDR",
+    "SCoRE_MDR_bf",
+    "SCoRE_MDR_w",
+    "SCoRE_SDR",
+    "SCoRE_SDR_w",
+    "eBH",
+    "eval_MDR",
+    "eval_SDR",
+    "gen_data_1",
+    "gen_data_2",
+    "gen_data_Jin2023",
+    "loss_1",
+    "loss_2",
+    "loss_Jin2023",
+]

SCoRE/utility.py

@@ -0,0 +1,246 @@
+import numpy as np
+
+
+def _expit(x):
+    x = np.asarray(x, dtype=float)
+    out = np.empty_like(x, dtype=float)
+    positive = x >= 0
+    out[positive] = 1.0 / (1.0 + np.exp(-x[positive]))
+    exp_x = np.exp(x[~positive])
+    out[~positive] = exp_x / (1.0 + exp_x)
+    return out
+
+
+def _get_rng(random_state):
+    if random_state is None:
+        return np.random
+    if isinstance(random_state, np.random.Generator):
+        return random_state
+    return np.random.default_rng(random_state)
+
+def loss_Jin2023(Y, tau):
+    """Calculates the smoothened indicator loss function, similar to the data generation process in Jin and Candes (2023).
+    
+    The loss function is of the form sigmoid(-tau * Y).
+    
+    Args:
+        Y (np.ndarray): The target values.
+        tau (float): Hyperparameter for smoothing. Larger tau means closer to 1{Y <= 0}. 
+            If tau = np.inf, it returns strictly 1{Y <= 0}.
+            
+    Returns:
+        np.ndarray: The computed loss values.
+    """
+    if tau != np.inf:
+        return _expit(-Y * tau) # L = smoothened indicator of <= 0
+    else:
+        return (Y <= 0)
+
+def gen_data_Jin2023(setting, n, sig, dim=20, random_state=None):
+    """Generates artificial data using the data generation process in Jin and Candes (2023).
+    
+    Args:
+        setting (int): The data generation setting (1 or 2).
+        n (int): Number of samples to generate.
+        sig (float): Noise scaling factor.
+        dim (int, optional): Dimensionality of the feature space. Defaults to 20.
+        random_state (int or np.random.Generator, optional): Random seed or generator for reproducible samples.
+        
+    Returns:
+        tuple: A tuple (X, mu_x, eps, Y) representing the generated data and components.
+    """
+    rng = _get_rng(random_state)
+
+    if setting == 1:
+        X = rng.uniform(low=-1, high=1, size=n*dim).reshape((n,dim))
+        mu_x = (X[:,0] * X[:,1] > 0) * (X[:,3] > 0.5) * (0.25 + X[:,3]) + (X[:,0] * X[:,1] <= 0) * (X[:,3] < -0.5) * (X[:,3] - 0.25)
+        eps = rng.normal(size=n) * (5.5 - abs(mu_x)) / 2 * sig
+        Y = mu_x + eps
+        return X, mu_x, eps, Y
+
+    if setting == 2:
+        X = rng.uniform(low=-1, high=1, size=n*dim).reshape((n,dim))
+        mu_x = (X[:,0] * X[:,1] + X[:,2] ** 2 + np.exp(X[:,3] - 1) - 1) * 2
+        eps = rng.normal(size=n) * (5.5 - abs(mu_x)) / 2 * sig
+        Y = mu_x + eps
+        return X, mu_x, eps, Y
+
+    raise ValueError("setting must be 1 or 2")
+
+def loss_1(Y):
+    """Calculates the expected shortfall-like loss function.
+    
+    The loss takes the form L(f, x, y) = y * 1{y > c}, evaluated against Y.
+    
+    Args:
+        Y (np.ndarray): The target values.
+        
+    Returns:
+        np.ndarray: The computed expected shortfall loss.
+    """
+    return 1/6 * Y * (Y > 2)
+
+def gen_data_1(setting, n, sig, dim=20, random_state=None):
+    """Generates artificial data for the first case.
+    
+    Args:
+        setting (int): The data generation setting (1 or 2).
+        n (int): Number of samples to generate.
+        sig (float): Noise scaling factor.
+        dim (int, optional): Dimensionality of the features. Defaults to 20.
+        random_state (int or np.random.Generator, optional): Random seed or generator for reproducible samples.
+        
+    Returns:
+        tuple: A tuple (X, mu_x, eps, Y) containing the covariates and responses.
+    """
+    rng = _get_rng(random_state)
+
+    if setting == 1:
+        X = rng.uniform(low=-1, high=1, size=n*dim).reshape((n,dim))
+        mu_x = (X[:,0] * X[:,1] > 0) * (X[:,3] > 0.5) * (0.5 + X[:,3]) + (X[:,0] * X[:,1] <= 0) * (X[:,3] < -0.5) * (X[:,3] - 0.5) + 3 # now in (1.5, 4.5)
+        eps = np.clip(rng.normal(size=n) * sig * (5.5 - mu_x), -1.5, 1.5) # clip the noise to be in (-1.5, 1.5)
+        Y = mu_x + eps # (0, 6)
+        return X, mu_x, eps, Y
+    
+    if setting == 2:
+        X = rng.uniform(low=-1, high=1, size=n*dim).reshape((n,dim))
+        mu_x = X[:,0] * X[:,1] + X[:,2] ** 2 + np.exp(X[:,3] - 1) + 2 # in (1, 5)
+        eps = np.clip(rng.normal(size=n) * sig * (6 - mu_x) * 0.5, -1, 1) # clip the noise to be in (-1, 1)
+        Y = mu_x + eps # (0, 6)
+        return X, mu_x, eps, Y
+
+    raise ValueError("setting must be 1 or 2")
+    
+def loss_2(Y, f, X, clip_const):
+    """Calculates clipped prediction error loss.
+    
+    Evaluates loss of the form L(f, x, y) = (y - f(x))^2, clipped at clip_const.
+    
+    Args:
+        Y (np.ndarray): The true target values.
+        f (object): The regression model to use for prediction (must have `.predict()`).
+        X (np.ndarray): The feature matrix to run predictions against.
+        clip_const (float): The clipping boundary, i.e., loss is in [0, clip_const].
+        
+    Returns:
+        np.ndarray: The computed normalized prediction error loss over X.
+    """
+    return np.clip((Y - f.predict(X)) ** 2, 0, clip_const) / clip_const
+
+def gen_data_2(setting, n, sig, dim=20, random_state=None):
+    """Generates artificial data for the second case.
+    
+    Args:
+        setting (int): The data generation setting (1 or 2).
+        n (int): Number of samples to generate.
+        sig (float): Noise scaling factor.
+        dim (int, optional): Dimensionality. Defaults to 20.
+        random_state (int or np.random.Generator, optional): Random seed or generator for reproducible samples.
+        
+    Returns:
+        tuple: A tuple (X, mu_x, eps, Y) with covariates and label values.
+    """
+    rng = _get_rng(random_state)
+
+    if setting == 1:
+        X = rng.uniform(low=-1, high=1, size=n*dim).reshape((n,dim))
+        mu_x = (X[:,0] * X[:,1] > 0) * (X[:,3] > 0.5) * (0.5 + X[:,3]) + (X[:,0] * X[:,1] <= 0) * (X[:,3] < -0.5) * (X[:,3] - 0.5) + 3 # now in (1.5, 4.5)
+        eps = np.clip(rng.normal(size=n) * sig * (5.5 - mu_x), -1.5, 1.5) # clip the noise to be in (-1.5, 1.5)
+        Y = mu_x + eps # (0, 6)
+        return X, mu_x, eps, Y
+    
+    if setting == 2:
+        X = rng.uniform(low=-1, high=1, size=n*dim).reshape((n,dim))
+        mu_x = X[:,0] * X[:,1] + X[:,2] ** 2 + np.exp(X[:,3] - 1) + 2 # in (1, 5)
+        eps = np.clip(rng.normal(size=n) * sig * (6 - mu_x) * 0.5, -1, 1) # clip the noise to be in (-1, 1)
+        Y = mu_x + eps # (0, 6)
+        return X, mu_x, eps, Y
+
+    raise ValueError("setting must be 1 or 2")
+
+def BH(pvals, q):
+    """Applies the Benjamini-Hochberg (BH) procedure to a list of p-values.
+    
+    Args:
+        pvals (array-like): List or array of p-values.
+        q (float): The nominal False Discovery Rate (FDR) level.
+        
+    Returns:
+        np.ndarray: The indices forming the rejection set.
+    """
+    pvals = np.asarray(pvals, dtype=float)
+    ntest = pvals.size
+
+    if ntest == 0:
+        return np.array([], dtype=int)
+
+    order = np.argsort(pvals, kind="mergesort")
+    sorted_pvals = pvals[order]
+    thresholds = q * np.arange(1, ntest + 1) / ntest
+    selected = np.flatnonzero(sorted_pvals <= thresholds)
+
+    if selected.size == 0:
+        return np.array([], dtype=int)
+
+    return order[: selected[-1] + 1]
+
+def eBH(evals, q):
+    """Applies the base e-BH procedure to a list of e-values.
+    
+    Args:
+        evals (array-like): List or array of e-values.
+        q (float): The nominal False Discovery Rate (FDR) level.
+        
+    Returns:
+        np.ndarray: The indices forming the rejection set.
+    """
+    return BH(np.divide(1.0, evals, np.full_like(evals, np.inf), where=(evals != 0)), q)
+
+def eval_MDR(L, R, sel):
+    """Evaluates selection performance for risk and power in the MDR sense.
+    
+    Args:
+        L (np.ndarray): The true loss corresponding to every instance.
+        R (np.ndarray): The true rewards for each instance.
+        sel (array-like): The selection set generated by the test procedure.
+        
+    Returns:
+        tuple: (risk_acc, reward_acc) indicating the MDR risk and cumulative reward.
+    """
+    if len(sel) == 0:
+        return 0, 0
+    risk_acc = np.sum(L[sel]) / len(L)
+    reward_acc = np.sum(R[sel])
+    return risk_acc, reward_acc
+
+def eval_SDR(L, R, sel):
+    """Evaluates selection performance for risk and power in the SDR sense.
+    
+    Args:
+        L (np.ndarray): The true loss corresponding to every instance.
+        R (np.ndarray): The true rewards for each instance.
+        sel (array-like): The selection set generated by the test procedure.
+        
+    Returns:
+        tuple: (sdr, bin_power, reward) corresponding to SDR, binary power (equivalent to power in the binary loss case), and reward metrics.
+    """
+    if len(sel) == 0:
+        return 0, 0, 0
+    true_rej = len(L) - np.sum(L) # number of zeros in L
+    sdr = np.sum(L[sel]) / len(sel)
+    bin_power = (len(sel) - np.sum(L[sel])) / true_rej if true_rej != 0 else 0 # defined only for 0-1 loss
+    reward = np.sum(R[sel])
+    return sdr, bin_power, reward
+
+class Lpredictor:
+    """Encapsulates a target predictor and a loss mapping into an expected loss predictor.
+    
+    Acts as a wrapper returning the loss via its `.predict()` method directly.
+    """
+    def __init__(self, Ypred, loss_fn):
+        self.Ypred = Ypred
+        self.loss_fn = loss_fn
+
+    def predict(self, X):
+        Y_hat = self.Ypred.predict(X)
+        return self.loss_fn(Y_hat, X)

score_select-0.1.1.dist-info/METADATA

@@ -0,0 +1,125 @@
+Metadata-Version: 2.4
+Name: score-select
+Version: 0.1.1
+Summary: Conformal selective prediction with general risk control.
+Author-email: Tian Bai <20.t.bai.05@gmail.com>, Ying Jin <yjinstat@wharton.upenn.edu>
+License-Expression: MIT
+Project-URL: Repository, https://github.com/Tian-Bai/SCoRE
+Project-URL: Paper, https://arxiv.org/abs/2603.24704
+Keywords: conformal prediction,selective prediction,risk control
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Scientific/Engineering :: Mathematics
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy>=1.23
+Provides-Extra: experiments
+Requires-Dist: pandas>=1.5; extra == "experiments"
+Requires-Dist: scikit-learn>=1.1; extra == "experiments"
+Requires-Dist: scipy>=1.9; extra == "experiments"
+Requires-Dist: tqdm>=4.64; extra == "experiments"
+Provides-Extra: dev
+Requires-Dist: build>=1.2; extra == "dev"
+Requires-Dist: pytest>=7; extra == "dev"
+Requires-Dist: twine>=5; extra == "dev"
+Dynamic: license-file
+
+# SCoRE
+
+SCoRE implements conformal selective prediction procedures for marginal
+deployment risk (MDR) and selective deployment risk (SDR) control.
+
+This repository also contains the simulation and application code used for the
+paper [Conformal Selective Prediction with General Risk Control](https://arxiv.org/abs/2603.24704).
+
+## Installation
+
+Install the package from a local checkout:
+
+```bash
+python -m pip install -e .
+```
+
+Install optional dependencies for the research scripts:
+
+```bash
+python -m pip install -e ".[experiments]"
+```
+
+After the package is published, install it with:
+
+```bash
+python -m pip install score-select
+```
+
+## Quickstart
+
+```python
+import numpy as np
+from SCoRE import SCoRE_MDR, SCoRE_SDR
+
+lcalib = np.array([0, 1, 0, 1])
+scalib = np.array([0.1, 0.4, 0.2, 0.8])
+stest = np.array([0.15, 0.5, 0.9])
+
+dcalib = (lcalib, scalib)
+dtest = stest
+
+mdr_selected = SCoRE_MDR(dcalib, dtest, alpha=0.5, gamma=0.5)
+sdr_selected = SCoRE_SDR(dcalib, dtest, alpha=0.5, gamma=0.5)
+```
+
+Functions return NumPy integer index arrays, so selections can be used directly
+to index NumPy arrays.
+
+When using randomized pruning, pass `random_state` for reproducible results:
+
+```python
+selected = SCoRE_SDR(
+    dcalib,
+    dtest,
+    alpha=0.5,
+    gamma=1.0,
+    prune="hete",
+    random_state=123,
+)
+```
+
+## Public API
+
+The top-level package exports the main procedures and utilities:
+
+Recommended package entry points:
+
+- `SCoRE_MDR`
+- `SCoRE_SDR`
+
+Additional utilities:
+
+- `CS`
+- `SCoRE_MDR_bf`, `SCoRE_MDR_w`, `SCoRE_SDR_w`
+- `BH`, `eBH`
+- `eval_MDR`, `eval_SDR`
+- `loss_Jin2023`, `loss_1`, `loss_2`
+- `gen_data_Jin2023`, `gen_data_1`, `gen_data_2`
+- `Lpredictor`
+
+## Repository Layout
+
+- `SCoRE/`: installable Python package
+- `tests/`: package tests
+- `applications/`: real-data applications
+  - `applications/drug/`: efficient, cost-aware drug discovery
+  - `applications/icu/`: clinical prediction error management
+  - `applications/llm/`: flexible LLM abstention
+- `simulation/`: simulation experiments
+- `simulation_w/`: simulation experiments with covariate shifts

score_select-0.1.1.dist-info/RECORD

@@ -0,0 +1,8 @@
+SCoRE/SCoRE.py,sha256=k5fXzFSJc1Z2gDJtLk9wI4l8RpRPxoVfH_2n0NHaBo4,20148
+SCoRE/__init__.py,sha256=mmKP3rku_aJiinVXYZZkGDb59U78d0djtmhjyMuN3rI,662
+SCoRE/utility.py,sha256=m3BoIqcJu8FFvXyYJpPFzlMt7dSMKM8_iwQnfmFXn5o,9466
+score_select-0.1.1.dist-info/licenses/LICENSE,sha256=SYINFQb7BU86dVJs0T25JhoG-4wH7LmMU5I5tP5XEAE,1107
+score_select-0.1.1.dist-info/METADATA,sha256=iKFB6_mRYi6RzmlUZzgW90RuwN-gbSFwGSgn6B3a_gc,3717
+score_select-0.1.1.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+score_select-0.1.1.dist-info/top_level.txt,sha256=tQ8qYOEVtrqK2raAkVMXWvJs75exBWDMZ8NiN1DTCS0,6
+score_select-0.1.1.dist-info/RECORD,,

score_select-0.1.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

score_select-0.1.1.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2026 Tian Bai and Ying Jin
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

score_select-0.1.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+SCoRE