pauc 0.1.0__tar.gz

pauc-0.1.0/PKG-INFO

@@ -0,0 +1,41 @@
+Metadata-Version: 2.4
+Name: pauc
+Version: 0.1.0
+Summary: Compute ROC AUC and confidence intervals using DeLong’s method
+Home-page: https://github.com/srijitseal/pauc
+Author: Srijit Seal
+Author-email: srijit@understanding.bio
+License: MIT
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.6
+Description-Content-Type: text/markdown
+Requires-Dist: numpy
+Requires-Dist: scipy
+Requires-Dist: pandas
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# pAUC
+
+Simple Python package to calculate ROC AUC with confidence intervals using DeLong’s method.
+
+## Installation
+
+pip install pauc
+
+## Usage
+
+from pauc import roc_auc_ci_score
+
+auc, (lb, ub) = roc_auc_ci_score(y_true, y_pred)
+print(f'AUC: {auc}, 95% CI: ({lb}, {ub})')

pauc-0.1.0/README.md

@@ -0,0 +1,14 @@
+# pAUC
+
+Simple Python package to calculate ROC AUC with confidence intervals using DeLong’s method.
+
+## Installation
+
+pip install pauc
+
+## Usage
+
+from pauc import roc_auc_ci_score
+
+auc, (lb, ub) = roc_auc_ci_score(y_true, y_pred)
+print(f'AUC: {auc}, 95% CI: ({lb}, {ub})')

pauc-0.1.0/pauc/__init__.py

@@ -0,0 +1,2 @@
+# __init__.py
+from .roc_auc_ci import roc_auc_ci_score

pauc-0.1.0/pauc/roc_auc_ci.py

@@ -0,0 +1,186 @@
+import pandas as pd
+import numpy as np
+import scipy.stats as stats
+
+# from https://github.com/PatWalters/comparing_classifiers/blob/master/delong_ci.py
+# from https://github.com/yandexdataschool/roc_comparison/blob/master/compare_auc_delong_xu.py
+
+# AUC comparison adapted from
+# https://github.com/Netflix/vmaf/
+def compute_midrank(x):
+    """Computes midranks.
+    Args:
+       x - a 1D numpy array
+    Returns:
+       array of midranks
+    """
+    J = np.argsort(x)
+    Z = x[J]
+    N = len(x)
+    T = np.zeros(N, dtype=float)
+    i = 0
+    while i < N:
+        j = i
+        while j < N and Z[j] == Z[i]:
+            j += 1
+        T[i:j] = 0.5*(i + j - 1)
+        i = j
+    T2 = np.empty(N, dtype=float)
+    # Note(kazeevn) +1 is due to Python using 0-based indexing
+    # instead of 1-based in the AUC formula in the paper
+    T2[J] = T + 1
+    return T2
+
+
+def compute_midrank_weight(x, sample_weight):
+    """Computes midranks.
+    Args:
+       x - a 1D numpy array
+    Returns:
+       array of midranks
+    """
+    J = np.argsort(x)
+    Z = x[J]
+    cumulative_weight = np.cumsum(sample_weight[J])
+    N = len(x)
+    T = np.zeros(N, dtype=float)
+    i = 0
+    while i < N:
+        j = i
+        while j < N and Z[j] == Z[i]:
+            j += 1
+        T[i:j] = cumulative_weight[i:j].mean()
+        i = j
+    T2 = np.empty(N, dtype=float)
+    T2[J] = T
+    return T2
+
+
+def fastDeLong(predictions_sorted_transposed, label_1_count):
+    """
+    The fast version of DeLong's method for computing the covariance of
+    unadjusted AUC.
+    Args:
+       predictions_sorted_transposed: a 2D numpy.array[n_classifiers, n_examples]
+          sorted such as the examples with label "1" are first
+    Returns:
+       (AUC value, DeLong covariance)
+    Reference:
+     @article{sun2014fast,
+       title={Fast Implementation of DeLong's Algorithm for
+              Comparing the Areas Under Correlated Receiver Oerating Characteristic Curves},
+       author={Xu Sun and Weichao Xu},
+       journal={IEEE Signal Processing Letters},
+       volume={21},
+       number={11},
+       pages={1389--1393},
+       year={2014},
+       publisher={IEEE}
+     }
+    """
+    # Short variables are named as they are in the paper
+    m = label_1_count
+    n = predictions_sorted_transposed.shape[1] - m
+    positive_examples = predictions_sorted_transposed[:, :m]
+    negative_examples = predictions_sorted_transposed[:, m:]
+    k = predictions_sorted_transposed.shape[0]
+
+    tx = np.empty([k, m], dtype=float)
+    ty = np.empty([k, n], dtype=float)
+    tz = np.empty([k, m + n], dtype=float)
+    for r in range(k):
+        tx[r, :] = compute_midrank(positive_examples[r, :])
+        ty[r, :] = compute_midrank(negative_examples[r, :])
+        tz[r, :] = compute_midrank(predictions_sorted_transposed[r, :])
+    aucs = tz[:, :m].sum(axis=1) / m / n - float(m + 1.0) / 2.0 / n
+    v01 = (tz[:, :m] - tx[:, :]) / n
+    v10 = 1.0 - (tz[:, m:] - ty[:, :]) / m
+    sx = np.cov(v01)
+    sy = np.cov(v10)
+    delongcov = sx / m + sy / n
+    return aucs, delongcov
+
+
+def calc_pvalue(aucs, sigma):
+    """Computes log(10) of p-values.
+    Args:
+       aucs: 1D array of AUCs
+       sigma: AUC DeLong covariances
+    Returns:
+       log10(pvalue)
+    """
+    l = np.array([[1, -1]])
+    z = np.abs(np.diff(aucs)) / np.sqrt(np.dot(np.dot(l, sigma), l.T))
+    return np.log10(2) + scipy.stats.norm.logsf(z, loc=0, scale=1) / np.log(10)
+
+
+def compute_ground_truth_statistics(ground_truth, sample_weight=None):
+    assert np.array_equal(np.unique(ground_truth), [0, 1])
+    order = (-ground_truth).argsort()
+    label_1_count = int(ground_truth.sum())
+    if sample_weight is None:
+        ordered_sample_weight = None
+    else:
+        ordered_sample_weight = sample_weight[order]
+
+    return order, label_1_count, ordered_sample_weight
+
+
+def delong_roc_variance(ground_truth, predictions):
+    """
+    Computes ROC AUC variance for a single set of predictions
+    Args:
+       ground_truth: np.array of 0 and 1
+       predictions: np.array of floats of the probability of being class 1
+    """
+    sample_weight = None
+    order, label_1_count, ordered_sample_weight = compute_ground_truth_statistics(
+        ground_truth, sample_weight)
+    predictions_sorted_transposed = predictions[np.newaxis, order]
+    aucs, delongcov = fastDeLong(predictions_sorted_transposed, label_1_count)
+    assert len(aucs) == 1, "There is a bug in the code, please forward this to the developers"
+    return aucs[0], delongcov
+
+
+def delong_roc_test(ground_truth, predictions_one, predictions_two):
+    """
+    Computes log(p-value) for hypothesis that two ROC AUCs are different
+    Args:
+       ground_truth: np.array of 0 and 1
+       predictions_one: predictions of the first model,
+          np.array of floats of the probability of being class 1
+       predictions_two: predictions of the second model,
+          np.array of floats of the probability of being class 1
+    """
+    sample_weight = None
+    order, label_1_count = compute_ground_truth_statistics(ground_truth)
+    predictions_sorted_transposed = np.vstack((predictions_one, predictions_two))[:, order]
+    aucs, delongcov = fastDeLong(predictions_sorted_transposed, label_1_count)
+    return calc_pvalue(aucs, delongcov)
+
+
+def roc_auc_ci_score(y_true, y_pred, alpha=0.95):
+    auc, auc_cov = delong_roc_variance(y_true, y_pred)
+    auc_std = np.sqrt(auc_cov)
+
+    # Handle edge cases when auc_std is zero or very small
+    if auc_std < 1e-10:
+        if auc == 1.0:
+            ci = np.array([1.0, 1.0])
+        elif auc == 0.0:
+            ci = np.array([0.0, 0.0])
+        else:
+            # If std dev is extremely low but AUC is not exactly 0 or 1
+            ci = np.array([auc, auc])
+    else:
+        lower_upper_q = np.abs(np.array([0, 1]) - (1 - alpha) / 2)
+        ci = stats.norm.ppf(
+            lower_upper_q,
+            loc=auc,
+            scale=auc_std)
+
+        # Ensure confidence intervals within [0,1]
+        ci[ci > 1] = 1
+        ci[ci < 0] = 0
+
+    return auc, ci

pauc-0.1.0/pauc.egg-info/PKG-INFO

@@ -0,0 +1,41 @@
+Metadata-Version: 2.4
+Name: pauc
+Version: 0.1.0
+Summary: Compute ROC AUC and confidence intervals using DeLong’s method
+Home-page: https://github.com/srijitseal/pauc
+Author: Srijit Seal
+Author-email: srijit@understanding.bio
+License: MIT
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.6
+Description-Content-Type: text/markdown
+Requires-Dist: numpy
+Requires-Dist: scipy
+Requires-Dist: pandas
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# pAUC
+
+Simple Python package to calculate ROC AUC with confidence intervals using DeLong’s method.
+
+## Installation
+
+pip install pauc
+
+## Usage
+
+from pauc import roc_auc_ci_score
+
+auc, (lb, ub) = roc_auc_ci_score(y_true, y_pred)
+print(f'AUC: {auc}, 95% CI: ({lb}, {ub})')

pauc-0.1.0/pauc.egg-info/SOURCES.txt

@@ -0,0 +1,11 @@
+README.md
+setup.py
+pauc/__init__.py
+pauc/roc_auc_ci.py
+pauc.egg-info/PKG-INFO
+pauc.egg-info/SOURCES.txt
+pauc.egg-info/dependency_links.txt
+pauc.egg-info/requires.txt
+pauc.egg-info/top_level.txt
+tests/__init__.py
+tests/test_roc_auc_ci.py

pauc-0.1.0/pauc.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

pauc-0.1.0/pauc.egg-info/requires.txt

@@ -0,0 +1,3 @@
+numpy
+scipy
+pandas

pauc-0.1.0/pauc.egg-info/top_level.txt

@@ -0,0 +1,2 @@
+pauc
+tests

pauc-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

pauc-0.1.0/setup.py

@@ -0,0 +1,28 @@
+from setuptools import setup, find_packages
+
+setup(
+    name='pauc',
+    version='0.1.0',
+    packages=find_packages(),
+    description='Compute ROC AUC and confidence intervals using DeLong’s method',
+    long_description=open('README.md').read(),
+    long_description_content_type='text/markdown',
+    install_requires=[
+        'numpy',
+        'scipy',
+        'pandas'
+    ],
+    tests_require=[
+        'pytest',
+    ],
+    python_requires='>=3.6',
+    author='Srijit Seal',
+    author_email='srijit@understanding.bio',
+    url='https://github.com/srijitseal/pauc',
+    license='MIT',
+    classifiers=[
+        'Programming Language :: Python :: 3',
+        'License :: OSI Approved :: MIT License',
+        'Operating System :: OS Independent',
+    ],
+)

pauc-0.1.0/tests/test_roc_auc_ci.py

@@ -0,0 +1,30 @@
+import numpy as np
+from pauc import roc_auc_ci_score
+
+def test_roc_auc_ci_score_basic():
+    y_true = np.array([0, 0, 1, 1])
+    y_pred = np.array([0.1, 0.4, 0.35, 0.8])
+
+    auc, (lb, ub) = roc_auc_ci_score(y_true, y_pred)
+
+    assert 0 <= auc <= 1
+    assert lb <= auc <= ub
+
+def test_roc_auc_ci_perfect_classifier():
+    y_true = np.array([0, 0, 1, 1])
+    y_pred = np.array([0.1, 0.2, 0.9, 0.95])
+
+    auc, (lb, ub) = roc_auc_ci_score(y_true, y_pred)
+
+    assert auc == 1.0
+    assert lb <= auc <= ub
+    assert ub <= 1.0
+
+def test_roc_auc_ci_worst_classifier():
+    y_true = np.array([0, 0, 1, 1])
+    y_pred = np.array([0.9, 0.95, 0.1, 0.2])
+
+    auc, (lb, ub) = roc_auc_ci_score(y_true, y_pred)
+
+    assert auc == 0.0
+    assert lb <= auc <= ub