sonusai 1.0.16__cp311-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl

sonusai/metrics/one_hot.py

@@ -0,0 +1,283 @@
+import numpy as np
+
+from ..datatypes import Predict
+from ..datatypes import Truth
+
+
+def one_hot(
+    truth: Truth,
+    predict: Predict,
+    predict_thr: float | np.ndarray = 0,
+    truth_thr: float = 0.5,
+    timesteps: int = -1,
+) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
+    """Calculates metrics from one-hot prediction and truth data (numpy float arrays) where
+    both are one-hot probabilities (or quantized decisions) for each class
+    with size [frames, num_classes] or [frames, timesteps, num_classes].
+    For metrics that require it, truth and pred decisions will be made using threshold >= predict_thr.
+    Some metrics like AP and AUC do not depend on predict_thr for predict, but still use truth >= predict_thr
+
+    predict_thr sets the decision threshold(s) applied to predict data for some metrics, thus allowing
+    the input to be continuous probabilities, for AUC-type metrics and root-mean-square error (rmse).
+    1. Default = 0 (multiclass or binary) which infers:
+       binary (num_classes = 1)         use >= 0.5 for truth and pred (same as argmax() for binary)
+       multi-class/single-label    if truth_mutex= = true, use argmax() used on both truth and pred
+       note multilabel metrics are disabled for predict_thr = 0, must set predict_thr > 0
+
+    2. predict_thr > 0 (multilabel or binary) scalar or a vector [num_classes, 1] then use
+       predict_thr as a binary decision threshold in each class:
+       binary (num_classes = 1)         use >= predict_thr[0] for pred and predict_thr[num_classes+1] for truth
+                                   if it exists, else use >= 0.5 for truth
+       multilabel                  use >= predict_thr for pred if scalar, or predict_thr[class_idx] if vector
+                                   use >= predict_thr[num_classes+1] for truth if exists, else 0.5
+       note multi-class/single-label inputs are meaningless in this mode, use predict_thr = 0 argmax mode
+
+    num_classes is inferred from 1D, 2D, or 3D truth inputs by default (default timesteps = -1 which implies None).
+    Only set timesteps > 0 in case of ambiguous binary 2D case where input [frames, timesteps],
+    then it must set to the number of timesteps (which will be > 0).
+    It is safe to always set timesteps <= 0 for binary inputs, and if truth.shape[2] exists
+
+    returns metrics over all frames + timesteps:
+    mcm     [num_classes, 2, 2]            multiclass confusion matrix count ove
+    metrics [num_classes, 14]              [ACC, TPR, PPV, TNR, FPR, HITFA, F1, MCC, NT, PT, TP, FP, AP, AUC]
+    cm      [num_classes, num_classes]     confusion matrix
+    cmn     [num_classes, num_classes]     normalized confusion matrix
+    rmse    [num_classes, 1]               RMS error over all frames + timesteps, before threshold decision
+    mavg    [3, 8] averages                macro, micro, weighted [PPV, TPR, F1, FPR, ACC, mAP, mAUC, TPSUM]
+    """
+    import warnings
+
+    from sklearn.metrics import average_precision_score
+    from sklearn.metrics import confusion_matrix
+    from sklearn.metrics import multilabel_confusion_matrix
+    from sklearn.metrics import precision_recall_fscore_support
+    from sklearn.metrics import roc_auc_score
+
+    from ..utils.reshape import get_num_classes_from_predict
+    from ..utils.reshape import reshape_outputs
+
+    if truth.shape != predict.shape:
+        raise ValueError("truth and predict are not the same shape")
+
+    predict, truth = reshape_outputs(predict=predict, truth=truth, timesteps=timesteps)  # type: ignore[assignment]
+    num_classes = get_num_classes_from_predict(predict=predict, timesteps=timesteps)
+
+    # Regression metric root-mean-square-error always works
+    rmse = np.sqrt(np.mean(np.square(truth - predict), axis=0))
+
+    # Calculate default predict decision thresholds based on mode
+    if not isinstance(predict_thr, np.ndarray):
+        if predict_thr == 0:
+            # if scalar and 0, set defaults
+            if num_classes == 1:
+                # binary case default >= 0.5 which is equiv to argmax()
+                predict_thr = np.atleast_1d(0.5)
+            else:
+                # multiclass, single-label (argmax mode)
+                predict_thr = np.atleast_1d(0)
+        else:
+            predict_thr = np.atleast_1d(predict_thr)
+    else:
+        if predict_thr.ndim > 1:
+            # multilabel with custom thr vector
+            if predict_thr.shape[0] != num_classes:
+                raise ValueError("predict_thr has wrong shape")
+        else:
+            if predict_thr == 0:
+                # binary or multilabel scalar default
+                predict_thr = np.atleast_1d(0.5)
+            else:
+                # user specified binary or multilabel scalar
+                predict_thr = np.atleast_1d(predict_thr)
+
+    if not isinstance(predict_thr, np.ndarray):
+        raise TypeError(f"predict_thr is invalid type: {type(predict_thr)}")
+
+    # Convert continuous probabilities to binary via argmax() or threshold comparison
+    # and create labels of int encoded (0:num_classes-1), and then equivalent one-hot
+    if num_classes == 1:  # If binary
+        labels = list(range(0, 2))  # int encoded 0,1
+        plabel = np.int8(predict >= predict_thr)  # [frames, 1], default 0.5 is equiv. to argmax()
+        tlabel = np.int8(truth >= truth_thr)  # [frames, 1]
+        predb = np.array(plabel)
+        truthb = np.array(tlabel)
+    else:
+        labels = list(range(0, num_classes))  # int encoded 0,...,num_classes-1
+        if predict_thr[0] == 0:  # multiclass single-label (mutex), use argmax
+            plabel = np.argmax(predict, axis=-1)  # [frames, 1] labels
+            tlabel = np.argmax(truth, axis=-1)  # [frames, 1] labels
+            # one-hot binary
+            predb = np.zeros(predict.shape, dtype=np.int8)  # [frames, num_classes]
+            truthb = np.zeros(truth.shape, dtype=np.int8)  # [frames, num_classes]
+            predb[np.arange(predb.shape[0]), plabel] = 1  # single-label [frames, num_classes]
+            if np.sum(truth):  # special case all zero truth leave tlabel all zeros
+                truthb[np.arange(truthb.shape[0]), tlabel] = 1  # single-label [frames, num_classes]
+        else:  # multilabel prob threshold comparison (multiple classes)
+            # multilabel one-hot decision
+            predb = np.array(predict >= predict_thr.transpose()).astype(np.int8)  # [frames, num_classes]
+            truthb = np.array(truth >= truth_thr).astype(np.int8)  # [frames, num_classes]
+            # Return argmax() for optional single-label confusion matrix metrics
+            plabel = np.argmax(predict, axis=-1)  # [frames, 1] labels
+            tlabel = np.argmax(truth, axis=-1)  # [frames, 1] labels
+
+    # debug checks to understand ap, auc:
+    # from sklearn.metrics import roc_curve
+    # fpr, tpr, thr = roc_curve(truthb[:,0],predict[:,0],drop_intermediate=False)
+    # from sklearn.metrics import precision_recall_curve
+    # precision, recall, thr  = precision_recall_curve(truthb[:,0], predict[:,0])
+    # from sklearn.metrics import RocCurveDisplay
+    # RocCurveDisplay.from_predictions(truthb[:,0],predict[:,0])  # Plot ROC class0
+
+    # Create [num_classes, 2, 2] multilabel confusion matrix (mcm)
+    # Note - must include labels or sklearn func. will omit non-exiting classes
+    mcm = multilabel_confusion_matrix(truthb, predb, labels=labels)
+
+    if num_classes == 1:
+        mcm = mcm[1:]  # remove dim 0 if binary
+
+    # Create [num_classes, num_classes] normalized confusion matrix
+    cmn = confusion_matrix(tlabel, plabel, labels=labels, normalize="true")
+
+    # Create [num_classes, num_classes] confusion matrix
+    cm = confusion_matrix(tlabel, plabel, labels=labels)
+
+    # Combine all per-class metrics into a single array
+    # [ACC, TPR, PPV, TNR, FPR, HITFA, F1, MCC, NT, PT, TP, FP, AP, AUC]
+    metrics = np.zeros((num_classes, 14))
+    # threshold_optpr = np.zeros((num_classes, 1))
+    eps = np.finfo(float).eps
+    for nci in range(num_classes):
+        # True negative
+        TN = mcm[nci, 0, 0]
+        # False positive
+        FP = mcm[nci, 0, 1]
+        # False negative
+        FN = mcm[nci, 1, 0]
+        # True positive
+        TP = mcm[nci, 1, 1]
+        # Accuracy
+        ACC = (TP + TN) / (TP + TN + FP + FN + eps)
+        # True positive rate, sensitivity, recall, hit rate (note eps in numerator)
+        # When ``true positive + false negative == 0``, recall is undefined, set to 0
+        TPR = TP / (TP + FN + eps)
+        # Precision, positive predictive value
+        # When ``true positive + false positive == 0``, precision is undefined, set to 0
+        PPV = TP / (TP + FP + eps)
+        # Specificity i.e., selectivity, or true negative rate
+        TNR = TN / (TN + FP + eps)
+        # False positive rate = 1-specificity, roc x-axis
+        FPR = FP / (TN + FP + eps)
+        # HitFA used by some separation research, close match to MCC
+        HITFA = TPR - FPR
+        # F1 harmonic mean of precision, recall = 2*PPV*TPR / (PPV + TPR)
+        F1 = 2 * TP / (2 * TP + FP + FN + eps)
+        # Matthew correlation coefficient
+        MCC = (TP * TN - FP * FN) / (np.sqrt((TP + FP) * (TP + FN) * (TN + FP) * (TN + FN)) + eps)
+        # Num. negatives total (truth), also = TN+FP denom of FPR
+        NT = sum(mcm[nci, 0, :])
+        # Num. positives total (truth), also = FN+TP denom of TPR, precision
+        PT = sum(mcm[nci, 1, :])
+        # Average Precision also called area under the PR curve AUCPR and
+        # AUC ROC curve using binary-ized truth and continuous prediction probabilities
+        # sklearn returns nan if no active truth in a class but w/un-suppressible div-by-zero warning
+        if np.sum(truthb[:, nci]) == 0:  # if no active classes both sklearn will fail, set to NaN
+            AUC = np.NaN
+            AP = np.NaN
+            # threshold_optpr[nci] = np.NaN
+        else:
+            AP = average_precision_score(truthb[:, nci], predict[:, nci], average=None)  # pyright: ignore [reportArgumentType]
+            if len(np.unique(truthb[:, nci])) < 2:  # if active classes not > 1 AUC must be NaN
+                AUC = np.NaN  # i.e. all ones sklearn auc will fail
+            else:
+                AUC = roc_auc_score(truthb[:, nci], predict[:, nci], average=None)  # pyright: ignore [reportArgumentType]
+            # # Optimal threshold from PR curve, optimizes f-score
+            # precision, recall, thresholds = precision_recall_curve(truthb[:, nci], predict[:, nci])
+            # fscore = (2 * precision * recall) / (precision + recall)
+            # ix = np.argmax(fscore)  # index of largest f1 score
+            # threshold_optpr[nci] = thresholds[ix]
+
+        metrics[nci, :] = [
+            ACC,
+            TPR,
+            PPV,
+            TNR,
+            FPR,
+            HITFA,
+            F1,
+            MCC,
+            NT,
+            PT,
+            TP,
+            FP,
+            AP,
+            AUC,
+        ]
+
+    # Calculate averages into single array, 3 types for now Macro, Micro, Weighted
+    mavg = np.zeros((3, 8), dtype=np.float32)
+    s = np.sum(metrics[:, 9].astype(int))  # support = sum (true pos total = FN+TP ) over classes
+
+    # macro average [PPV, TPR, F1, FPR, ACC, mAP, mAUC, TPSUM]
+    with warnings.catch_warnings():
+        warnings.filterwarnings(action="ignore", message="Mean of empty slice")
+        mavg[0, :] = [
+            np.mean(metrics[:, 2]),
+            np.mean(metrics[:, 1]),
+            np.mean(metrics[:, 6]),
+            np.mean(metrics[:, 4]),
+            np.mean(metrics[:, 0]),
+            np.nanmean(metrics[:, 12]),
+            np.nanmean(metrics[:, 13]),
+            s,
+        ]
+
+    # micro average, micro-F1 = micro-precision = micro-recall = accuracy
+    if num_classes > 1:
+        tp_sum = np.sum(metrics[:, 10])  # TP all classes
+        rm = tp_sum / (np.sum(metrics[:, 9]) + eps)  # micro mean PPV = TP / (PT=FN+TP)
+        fp_sum = np.sum(metrics[:, 11])  # FP false-positives all classes
+        fpm = fp_sum / (np.sum(metrics[:, 8]) + eps)  # micro mean FPR = FP / (NT=TN+FP)
+        pm = tp_sum / (tp_sum + fp_sum + eps)  # micro mean TPR = TP / (TP+FP) (note: same as rm for micro-avg)
+        fn_sum = sum(mcm[:, 1, 0])
+        f1m = 2 * tp_sum / (2 * tp_sum + fp_sum + fn_sum + eps)
+        tn_sum = sum(mcm[:, 0, 0])
+        accm = (tp_sum + tn_sum) / (tp_sum + tn_sum + fp_sum + fn_sum + eps)
+        with warnings.catch_warnings():
+            warnings.filterwarnings(action="ignore", message="invalid value encountered in true_divide")
+            miap = average_precision_score(truthb, predict, average="micro")
+        if np.sum(truthb):  # no activity over all classes
+            miauc = roc_auc_score(truthb, predict, average="micro")
+        else:
+            miauc = np.NaN
+
+        # [miPPV, miTPR, miF1, miFPR, miACC, miAP, miAUC, TPSUM]
+        mavg[1, :] = [
+            pm,
+            rm,
+            f1m,
+            fpm,
+            accm,
+            miap,
+            miauc,
+            s,
+        ]  # specific format, last 3 are unique
+
+        # weighted average TBD
+        wp, wr, wf1, _ = precision_recall_fscore_support(truthb, predb, average="weighted", zero_division=0)  # pyright: ignore [reportArgumentType]
+        if np.sum(truthb):
+            taidx = np.sum(truthb, axis=0) > 0
+            wap = average_precision_score(truthb[:, taidx], predict[:, taidx], average="weighted")
+            if len(np.unique(truthb[:, taidx])) < 2:
+                wauc = np.NaN
+            else:
+                wauc = roc_auc_score(truthb[:, taidx], predict[:, taidx], average="weighted")
+        else:
+            wap = np.NaN
+            wauc = np.NaN
+
+        mavg[2, :] = [wp, wr, wf1, 0, 0, wap, wauc, s]
+    else:  # binary case, all are same
+        mavg[1, :] = mavg[0, :]
+        mavg[2, :] = mavg[0, :]
+
+    return mcm, metrics, cm, cmn, rmse, mavg

sonusai/metrics/snr_summary.py

@@ -0,0 +1,128 @@
+# ruff: noqa: F821
+import numpy as np
+import pandas as pd
+
+from ..datatypes import GeneralizedIDs
+from ..datatypes import Predict
+from ..datatypes import Segsnr
+from ..datatypes import Truth
+from ..mixture.mixdb import MixtureDatabase
+
+
+def snr_summary(
+    mixdb: MixtureDatabase,
+    mixid: GeneralizedIDs,
+    truth_f: Truth,
+    predict: Predict,
+    segsnr: Segsnr | None = None,
+    predict_thr: float | np.ndarray = 0,
+    truth_thr: float = 0.5,
+    timesteps: int = 0,
+) -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame, dict]:
+    """Calculate average-over-class metrics per SNR over specified mixture list.
+    Inputs:
+      mixdb        Mixture database
+      mixid
+      truth_f      Truth/labels [features, num_classes]
+      predict      Prediction data / neural net model one-hot out [features, num_classes]
+      segsnr       Segmental SNR from SonusAI genft  [transform_frames, 1]
+      predict_thr  Decision threshold(s) applied to predict data, allowing predict to be
+                   continuous probabilities or decisions
+      truth_thr    Decision threshold(s) applied to truth data, allowing truth to be
+                   continuous probabilities or decisions
+      timesteps
+
+    Default predict_thr=0 will infer 0.5 for multi-label mode (truth_mutex = False), or
+    if single-label mode (truth_mutex == True) then ignore and use argmax mode, and
+    the confusion matrix is calculated for all classes.
+
+    Returns pandas dataframe (snrdf) of metrics per SNR.
+    """
+    import warnings
+
+    from ..metrics.one_hot import one_hot
+    from ..queries.queries import get_mixids_from_snr
+
+    num_classes = truth_f.shape[1]
+
+    snr_mixids = get_mixids_from_snr(mixdb=mixdb, mixids=mixid)
+
+    # Check predict_thr array or scalar and return final scalar predict_thr value
+    if num_classes > 1:
+        if not isinstance(predict_thr, np.ndarray):
+            if predict_thr == 0:
+                # multi-label predict_thr scalar 0 force to 0.5 default
+                predict_thr = np.atleast_1d(0.5)
+            else:
+                predict_thr = np.atleast_1d(predict_thr)
+        else:
+            if predict_thr.ndim == 1 and len(predict_thr) == 1:
+                if predict_thr[0] == 0:
+                    # multi-label predict_thr array scalar 0 force to 0.5 default
+                    predict_thr = np.atleast_1d(0.5)
+                else:
+                    # multi-label predict_thr array set to scalar = array[0]
+                    predict_thr = predict_thr[0]
+
+    macro_avg = np.zeros((len(snr_mixids), 7), dtype=np.float32)
+    micro_avg = np.zeros((len(snr_mixids), 7), dtype=np.float32)
+    wghtd_avg = np.zeros((len(snr_mixids), 7), dtype=np.float32)
+    ssnr_stats = None
+    segsnr_f = None
+
+    if segsnr is not None:
+        # prep segsnr if provided, transform frames to feature frames via mean()
+        # expected to always be an integer
+        feature_frames = int(segsnr.shape[0] / truth_f.shape[0])
+        segsnr_f = np.mean(
+            np.reshape(segsnr, (truth_f.shape[0], feature_frames)),
+            axis=1,
+            keepdims=True,
+        )
+        ssnr_stats = np.zeros((len(snr_mixids), 3), dtype=np.float32)
+
+    for ii, snr in enumerate(snr_mixids):
+        # TODO: re-work for modern mixdb API
+        y_truth, y_predict = get_mixids_data(mixdb, snr_mixids[snr], truth_f, predict)  # type: ignore[name-defined]
+        _, _, _, _, _, mavg = one_hot(y_truth, y_predict, predict_thr, truth_thr, timesteps)
+
+        # mavg macro, micro, weighted: [PPV, TPR, F1, FPR, ACC, mAP, mAUC, TPSUM]
+        macro_avg[ii, :] = mavg[0, 0:7]
+        micro_avg[ii, :] = mavg[1, 0:7]
+        wghtd_avg[ii, :] = mavg[2, 0:7]
+        if segsnr is not None:
+            # TODO: re-work for modern mixdb API
+            y_truth, y_segsnr = get_mixids_data(mixdb, snr_mixids[snr], truth_f, segsnr_f)  # type: ignore[name-defined]
+            with warnings.catch_warnings():
+                warnings.filterwarnings(action="ignore", message="divide by zero encountered in log10")
+                # segmental SNR mean = mixture_snr and target_snr
+                ssnr_stats[ii, 0] = 10 * np.log10(np.mean(y_segsnr))  # type: ignore[index]
+                # segmental SNR 80% percentile
+                ssnr_stats[ii, 1] = 10 * np.log10(np.percentile(y_segsnr, 80, method="midpoint"))  # type: ignore[index]
+                # segmental SNR max
+                ssnr_stats[ii, 2] = 10 * np.log10(max(y_segsnr))  # type: ignore[index]
+
+    # SNR format: PPV, TPR, F1, FPR, ACC, AP, AUC
+    col_n = ["PPV", "TPR", "F1", "FPR", "ACC", "AP", "AUC"]
+    snr_macrodf = pd.DataFrame(macro_avg, index=list(snr_mixids.keys()), columns=col_n)  # pyright: ignore [reportArgumentType]
+    snr_macrodf.sort_index(ascending=False, inplace=True)
+
+    snr_microdf = pd.DataFrame(micro_avg, index=list(snr_mixids.keys()), columns=col_n)  # pyright: ignore [reportArgumentType]
+    snr_microdf.sort_index(ascending=False, inplace=True)
+
+    snr_wghtdf = pd.DataFrame(wghtd_avg, index=list(snr_mixids.keys()), columns=col_n)  # pyright: ignore [reportArgumentType]
+    snr_wghtdf.sort_index(ascending=False, inplace=True)
+
+    # Add segmental SNR columns if provided
+    if segsnr is not None:
+        ssnrdf = pd.DataFrame(
+            ssnr_stats,
+            index=list(snr_mixids.keys()),  # pyright: ignore [reportArgumentType]
+            columns=["SSNRavg", "SSNR80p", "SSNRmax"],  # pyright: ignore [reportArgumentType]
+        )
+        ssnrdf.sort_index(ascending=False, inplace=True)
+        snr_macrodf = pd.concat([snr_macrodf, ssnrdf], axis=1)
+        snr_microdf = pd.concat([snr_microdf, ssnrdf], axis=1)
+        snr_wghtdf = pd.concat([snr_wghtdf, ssnrdf], axis=1)
+
+    return snr_macrodf, snr_microdf, snr_wghtdf, snr_mixids