sonusai 1.0.16__cp311-abi3-macosx_10_12_x86_64.whl

sonusai/metrics/calculate_metrics.py

@@ -0,0 +1,395 @@
+import functools
+from typing import Any
+
+import numpy as np
+from pystoi import stoi
+
+from ..constants import SAMPLE_RATE
+from ..datatypes import AudioF
+from ..datatypes import AudioStatsMetrics
+from ..datatypes import AudioT
+from ..datatypes import Segsnr
+from ..datatypes import SpeechMetrics
+from ..mixture.mixdb import MixtureDatabase
+from ..utils.asr import calc_asr
+from ..utils.db import linear_to_db
+from .calc_audio_stats import calc_audio_stats
+from .calc_pesq import calc_pesq
+from .calc_phase_distance import calc_phase_distance
+from .calc_segsnr_f import calc_segsnr_f
+from .calc_segsnr_f import calc_segsnr_f_bin
+from .calc_speech import calc_speech
+from .calc_wer import calc_wer
+from .calc_wsdr import calc_wsdr
+
+
+def calculate_metrics(mixdb: MixtureDatabase, m_id: int, metrics: list[str], force: bool = False) -> dict[str, Any]:
+    """Get metrics data for the given mixture ID
+
+    :param mixdb: Mixture database object
+    :param m_id: Zero-based mixture ID
+    :param metrics: List of metrics to get
+    :param force: Force computing data from original sources regardless of whether cached data exists
+    :return: Dictionary of metric data
+    """
+
+    # Define cached functions for expensive operations
+    @functools.lru_cache(maxsize=1)
+    def mixture_sources() -> dict[str, AudioT]:
+        return mixdb.mixture_sources(m_id)
+
+    @functools.lru_cache(maxsize=1)
+    def mixture_source() -> AudioT:
+        return mixdb.mixture_source(m_id)
+
+    @functools.lru_cache(maxsize=1)
+    def mixture_source_f() -> AudioF:
+        return mixdb.mixture_source_f(m_id)
+
+    @functools.lru_cache(maxsize=1)
+    def mixture_noise() -> AudioT:
+        return mixdb.mixture_noise(m_id)
+
+    @functools.lru_cache(maxsize=1)
+    def mixture_noise_f() -> AudioF:
+        return mixdb.mixture_noise_f(m_id)
+
+    @functools.lru_cache(maxsize=1)
+    def mixture_mixture() -> AudioT:
+        return mixdb.mixture_mixture(m_id)
+
+    @functools.lru_cache(maxsize=1)
+    def mixture_mixture_f() -> AudioF:
+        return mixdb.mixture_mixture_f(m_id)
+
+    @functools.lru_cache(maxsize=1)
+    def mixture_segsnr() -> Segsnr:
+        return mixdb.mixture_segsnr(m_id)
+
+    @functools.lru_cache(maxsize=1)
+    def calculate_pesq() -> dict[str, float]:
+        return {category: calc_pesq(mixture_mixture(), audio) for category, audio in mixture_sources().items()}
+
+    @functools.lru_cache(maxsize=1)
+    def calculate_speech() -> dict[str, SpeechMetrics]:
+        return {
+            category: calc_speech(mixture_mixture(), audio, calculate_pesq()[category])
+            for category, audio in mixture_sources().items()
+        }
+
+    @functools.lru_cache(maxsize=1)
+    def mixture_stats() -> AudioStatsMetrics:
+        return calc_audio_stats(mixture_mixture(), mixdb.fg_info.ft_config.length / SAMPLE_RATE)
+
+    @functools.lru_cache(maxsize=1)
+    def sources_stats() -> dict[str, AudioStatsMetrics]:
+        return {
+            category: calc_audio_stats(audio, mixdb.fg_info.ft_config.length / SAMPLE_RATE)
+            for category, audio in mixture_sources().items()
+        }
+
+    @functools.lru_cache(maxsize=1)
+    def source_stats() -> AudioStatsMetrics:
+        return calc_audio_stats(mixture_source(), mixdb.fg_info.ft_config.length / SAMPLE_RATE)
+
+    @functools.lru_cache(maxsize=1)
+    def noise_stats() -> AudioStatsMetrics:
+        return calc_audio_stats(mixture_noise(), mixdb.fg_info.ft_config.length / SAMPLE_RATE)
+
+    # Cache ASR configurations
+    @functools.lru_cache(maxsize=32)
+    def get_asr_config(asr_name: str) -> dict:
+        value = mixdb.asr_configs.get(asr_name, None)
+        if value is None:
+            raise ValueError(f"Unrecognized ASR name: '{asr_name}'")
+        return value
+
+    # Cache ASR results for sources, source and mixture
+    @functools.lru_cache(maxsize=16)
+    def sources_asr(asr_name: str) -> dict[str, str]:
+        return {
+            category: calc_asr(audio, **get_asr_config(asr_name)).text for category, audio in mixture_sources().items()
+        }
+
+    @functools.lru_cache(maxsize=16)
+    def source_asr(asr_name: str) -> str:
+        return calc_asr(mixture_source(), **get_asr_config(asr_name)).text
+
+    @functools.lru_cache(maxsize=16)
+    def mixture_asr(asr_name: str) -> str:
+        return calc_asr(mixture_mixture(), **get_asr_config(asr_name)).text
+
+    def get_asr_name(m: str) -> str:
+        parts = m.split(".")
+        if len(parts) != 2:
+            raise ValueError(f"Unrecognized format: '{m}'; must be of the form: '<metric>.<name>'")
+        asr_name = parts[1]
+        return asr_name
+
+    def calc(m: str) -> Any:
+        if m == "mxsnr":
+            return {category: source.snr for category, source in mixdb.mixture(m_id).all_sources.items()}
+
+        # Get cached data first, if exists
+        if not force:
+            value = mixdb.read_mixture_data(m_id, m)[m]
+            if value is not None:
+                return value
+
+        # Otherwise, generate data as needed
+        if m.startswith("mxwer"):
+            asr_name = get_asr_name(m)
+
+            if mixdb.mixture(m_id).is_noise_only:
+                # noise only, ignore/reset target asr
+                return float("nan")
+
+            if source_asr(asr_name):
+                return calc_wer(mixture_asr(asr_name), source_asr(asr_name)).wer * 100
+
+            # TODO: should this be NaN like above?
+            return float(0)
+
+        if m.startswith("basewer"):
+            asr_name = get_asr_name(m)
+
+            text = mixdb.mixture_speech_metadata(m_id, "text")
+            return {
+                category: calc_wer(source, str(text[category])).wer * 100 if isinstance(text[category], str) else 0
+                for category, source in sources_asr(asr_name).items()
+            }
+
+        if m.startswith("mxasr"):
+            return mixture_asr(get_asr_name(m))
+
+        if m == "mxssnr_avg":
+            return calc_segsnr_f(mixture_segsnr()).avg
+
+        if m == "mxssnr_std":
+            return calc_segsnr_f(mixture_segsnr()).std
+
+        if m == "mxssnr_avg_db":
+            val = calc_segsnr_f(mixture_segsnr()).avg
+            if val is not None:
+                return linear_to_db(val)
+            return None
+
+        if m == "mxssnr_std_db":
+            val = calc_segsnr_f(mixture_segsnr()).std
+            if val is not None:
+                return linear_to_db(val)
+            return None
+
+        if m == "mxssnrdb_avg":
+            return calc_segsnr_f(mixture_segsnr()).db_avg
+
+        if m == "mxssnrdb_std":
+            return calc_segsnr_f(mixture_segsnr()).db_std
+
+        if m == "mxssnrf_avg":
+            return calc_segsnr_f_bin(mixture_source_f(), mixture_noise_f()).avg
+
+        if m == "mxssnrf_std":
+            return calc_segsnr_f_bin(mixture_source_f(), mixture_noise_f()).std
+
+        if m == "mxssnrdbf_avg":
+            return calc_segsnr_f_bin(mixture_source_f(), mixture_noise_f()).db_avg
+
+        if m == "mxssnrdbf_std":
+            return calc_segsnr_f_bin(mixture_source_f(), mixture_noise_f()).db_std
+
+        if m == "mxpesq":
+            if mixdb.mixture(m_id).is_noise_only:
+                return dict.fromkeys(calculate_pesq(), 0)
+            return calculate_pesq()
+
+        if m == "mxcsig":
+            if mixdb.mixture(m_id).is_noise_only:
+                return dict.fromkeys(calculate_speech(), 0)
+            return {category: s.csig for category, s in calculate_speech().items()}
+
+        if m == "mxcbak":
+            if mixdb.mixture(m_id).is_noise_only:
+                return dict.fromkeys(calculate_speech(), 0)
+            return {category: s.cbak for category, s in calculate_speech().items()}
+
+        if m == "mxcovl":
+            if mixdb.mixture(m_id).is_noise_only:
+                return dict.fromkeys(calculate_speech(), 0)
+            return {category: s.covl for category, s in calculate_speech().items()}
+
+        if m == "mxwsdr":
+            mixture = mixture_mixture()[:, np.newaxis]
+            target = mixture_source()[:, np.newaxis]
+            noise = mixture_noise()[:, np.newaxis]
+            return calc_wsdr(
+                hypothesis=np.concatenate((mixture, noise), axis=1),
+                reference=np.concatenate((target, noise), axis=1),
+                with_log=True,
+            )[0]
+
+        if m == "mxpd":
+            return calc_phase_distance(hypothesis=mixture_mixture_f(), reference=mixture_source_f())[0]
+
+        if m == "mxstoi":
+            return stoi(
+                x=mixture_source(),
+                y=mixture_mixture(),
+                fs_sig=SAMPLE_RATE,
+                extended=False,
+            )
+
+        if m == "mxdco":
+            return mixture_stats().dco
+
+        if m == "mxmin":
+            return mixture_stats().min
+
+        if m == "mxmax":
+            return mixture_stats().max
+
+        if m == "mxpkdb":
+            return mixture_stats().pkdb
+
+        if m == "mxlrms":
+            return mixture_stats().lrms
+
+        if m == "mxpkr":
+            return mixture_stats().pkr
+
+        if m == "mxtr":
+            return mixture_stats().tr
+
+        if m == "mxcr":
+            return mixture_stats().cr
+
+        if m == "mxfl":
+            return mixture_stats().fl
+
+        if m == "mxpkc":
+            return mixture_stats().pkc
+
+        if m == "sdco":
+            return {category: s.dco for category, s in sources_stats().items()}
+
+        if m == "smin":
+            return {category: s.min for category, s in sources_stats().items()}
+
+        if m == "smax":
+            return {category: s.max for category, s in sources_stats().items()}
+
+        if m == "spkdb":
+            return {category: s.pkdb for category, s in sources_stats().items()}
+
+        if m == "slrms":
+            return {category: s.lrms for category, s in sources_stats().items()}
+
+        if m == "spkr":
+            return {category: s.pkr for category, s in sources_stats().items()}
+
+        if m == "str":
+            return {category: s.tr for category, s in sources_stats().items()}
+
+        if m == "scr":
+            return {category: s.cr for category, s in sources_stats().items()}
+
+        if m == "sfl":
+            return {category: s.fl for category, s in sources_stats().items()}
+
+        if m == "spkc":
+            return {category: s.pkc for category, s in sources_stats().items()}
+
+        if m == "mxsdco":
+            return source_stats().dco
+
+        if m == "mxsmin":
+            return source_stats().min
+
+        if m == "mxsmax":
+            return source_stats().max
+
+        if m == "mxspkdb":
+            return source_stats().pkdb
+
+        if m == "mxslrms":
+            return source_stats().lrms
+
+        if m == "mxspkr":
+            return source_stats().pkr
+
+        if m == "mxstr":
+            return source_stats().tr
+
+        if m == "mxscr":
+            return source_stats().cr
+
+        if m == "mxsfl":
+            return source_stats().fl
+
+        if m == "mxspkc":
+            return source_stats().pkc
+
+        if m.startswith("sasr"):
+            return sources_asr(get_asr_name(m))
+
+        if m.startswith("mxsasr"):
+            return source_asr(get_asr_name(m))
+
+        if m == "ndco":
+            return noise_stats().dco
+
+        if m == "nmin":
+            return noise_stats().min
+
+        if m == "nmax":
+            return noise_stats().max
+
+        if m == "npkdb":
+            return noise_stats().pkdb
+
+        if m == "nlrms":
+            return noise_stats().lrms
+
+        if m == "npkr":
+            return noise_stats().pkr
+
+        if m == "ntr":
+            return noise_stats().tr
+
+        if m == "ncr":
+            return noise_stats().cr
+
+        if m == "nfl":
+            return noise_stats().fl
+
+        if m == "npkc":
+            return noise_stats().pkc
+
+        if m == "sedavg":
+            return 0
+
+        if m == "sedcnt":
+            return 0
+
+        if m == "sedtop3":
+            return np.zeros(3, dtype=np.float32)
+
+        if m == "sedtopn":
+            return 0
+
+        if m == "ssnr":
+            return mixture_segsnr()
+
+        raise AttributeError(f"Unrecognized metric: '{m}'")
+
+    result: dict[str, Any] = {}
+    for metric in metrics:
+        result[metric] = calc(metric)
+
+        # Check for metrics dependencies and add them even if not explicitly requested.
+        if metric.startswith("mxwer"):
+            dependencies = ("mxasr." + metric[6:], "sasr." + metric[6:])
+            for dependency in dependencies:
+                result[dependency] = calc(dependency)
+
+    return result

sonusai/metrics/class_summary.py

@@ -0,0 +1,74 @@
+# ruff: noqa: F821
+import numpy as np
+import pandas as pd
+
+from ..datatypes import GeneralizedIDs
+from ..datatypes import Predict
+from ..datatypes import Truth
+from ..mixture.mixdb import MixtureDatabase
+
+
+def class_summary(
+    mixdb: MixtureDatabase,
+    mixids: GeneralizedIDs,
+    truth_f: Truth,
+    predict: Predict,
+    predict_thr: float | np.ndarray = 0,
+    truth_thr: float = 0.5,
+    timesteps: int = 0,
+) -> pd.DataFrame:
+    """Calculate table of metrics per class, and averages for a list
+    of mixtures using truth and prediction data [features, num_classes]
+    Example:
+    Generate multi-class metric summary into table, for example:
+                    PPV     TPR      F1     FPR     ACC   AP  AUC  Support
+        Class 1     0.71    0.80    0.75    0.00    0.99            44
+        Class 2     0.90    0.76    0.82    0.00    0.99            128
+        Class 3     0.86    0.82    0.84    0.04    0.93            789
+        Other       0.94    0.96    0.95    0.18    0.92            2807
+
+      micro-avg                     0.92    0.027                   3768
+      macro avg     0.85    0.83    0.84    0.05    0.96            3768
+      micro-avgwo
+    """
+    from ..metrics.one_hot import one_hot
+
+    num_classes = truth_f.shape[1]
+
+    # TODO: re-work for modern mixdb API
+    y_truth_f, y_predict = get_mixids_data(mixdb, mixids, truth_f, predict)  # type: ignore[name-defined]
+
+    if num_classes > 1:
+        if not isinstance(predict_thr, np.ndarray):
+            if predict_thr == 0:
+                predict_thr = np.atleast_1d(0.5)
+            else:
+                predict_thr = np.atleast_1d(predict_thr)
+        else:
+            if predict_thr.ndim == 1 and predict_thr[0] == 0:
+                predict_thr = np.atleast_1d(0.5)
+
+    _, metrics, _, _, _, metavg = one_hot(y_truth_f, y_predict, predict_thr, truth_thr, timesteps)
+
+    # [ACC, TPR, PPV, TNR, FPR, HITFA, F1, MCC, NT, PT, TP, FP, AP, AUC]
+    table_idx = np.array([2, 1, 6, 4, 0, 12, 13, 9])
+    col_n = ["PPV", "TPR", "F1", "FPR", "ACC", "AP", "AUC", "Support"]
+    if len(mixdb.class_labels) == num_classes:
+        row_n = mixdb.class_labels
+    else:
+        row_n = [f"Class {i}" for i in range(1, num_classes + 1)]
+
+    df = pd.DataFrame(metrics[:, table_idx], columns=col_n, index=row_n)  # pyright: ignore [reportArgumentType]
+
+    # [miPPV, miTPR, miF1, miFPR, miACC, miAP, miAUC, TPSUM]
+    avg_row_n = ["Macro-avg", "Micro-avg", "Weighted-avg"]
+    dfavg = pd.DataFrame(metavg, columns=col_n, index=avg_row_n)  # pyright: ignore [reportArgumentType]
+
+    # dfblank = pd.DataFrame([''])
+    # pd.concat([df, dfblank, dfblank, dfavg])
+
+    classdf = pd.concat([df, dfavg])
+    # classdf = classdf.round(2)
+    classdf["Support"] = classdf["Support"].astype(int)
+
+    return classdf

sonusai/metrics/confusion_matrix_summary.py

@@ -0,0 +1,75 @@
+# ruff: noqa: F821
+import numpy as np
+import pandas as pd
+
+from ..datatypes import GeneralizedIDs
+from ..datatypes import Predict
+from ..datatypes import Truth
+from ..mixture.mixdb import MixtureDatabase
+
+
+def confusion_matrix_summary(
+    mixdb: MixtureDatabase,
+    mixids: GeneralizedIDs,
+    truth_f: Truth,
+    predict: Predict,
+    class_idx: int,
+    predict_thr: float | np.ndarray = 0,
+    truth_thr: float = 0.5,
+    timesteps: int = 0,
+) -> tuple[pd.DataFrame, pd.DataFrame]:
+    """Calculate confusion matrix for specified class, using truth and prediction
+    data [features, num_classes].
+
+    predict_thr sets the decision threshold(s) applied to predict data, thus allowing
+    predict to be continuous probabilities.
+
+    Default predict_thr=0 will infer 0.5 for multi-label mode (truth_mutex = False), or
+    if single-label mode (truth_mutex == True) then ignore and use argmax mode, and
+    the confusion matrix is calculated for all classes.
+
+    Returns pandas dataframes of confusion matrix cmdf and normalized confusion matrix cmndf.
+    """
+    from ..metrics.one_hot import one_hot
+
+    num_classes = truth_f.shape[1]
+    # TODO: re-work for modern mixdb API
+    ytrue, ypred = get_mixids_data(mixdb=mixdb, mixids=mixids, truth_f=truth_f, predict=predict)  # type: ignore[name-defined]
+
+    # Check predict_thr array or scalar and return final scalar predict_thr value
+    if num_classes > 1:
+        if not isinstance(predict_thr, np.ndarray):
+            if predict_thr == 0:
+                # multi-label predict_thr scalar 0 force to 0.5 default
+                predict_thr = np.atleast_1d(0.5)
+            else:
+                predict_thr = np.atleast_1d(predict_thr)
+        else:
+            if predict_thr.ndim == 1:
+                if predict_thr[0] == 0:
+                    # multi-label predict_thr array scalar 0 force to 0.5 default
+                    predict_thr = np.atleast_1d(0.5)
+                else:
+                    # multi-label predict_thr array set to scalar = array[0]
+                    predict_thr = predict_thr[0]
+            else:
+                # multi-label predict_thr array scalar set = array[class_idx]
+                predict_thr = predict_thr[class_idx]
+
+    if len(mixdb.class_labels) == num_classes:
+        class_names = mixdb.class_labels
+    else:
+        class_names = [f"Class {i}" for i in range(1, num_classes + 1)]
+
+    _, _, cm, cmn, _, _ = one_hot(ytrue[:, class_idx], ypred[:, class_idx], predict_thr, truth_thr, timesteps)
+    cname = class_names[class_idx]
+    row_n = ["TrueN", "TrueP"]
+    col_n = ["N-" + cname, "P-" + cname]
+    cmdf = pd.DataFrame(cm, index=row_n, columns=col_n, dtype=np.int32)  # pyright: ignore [reportArgumentType]
+    cmndf = pd.DataFrame(cmn, index=row_n, columns=col_n, dtype=np.float32)  # pyright: ignore [reportArgumentType]
+    # add thresholds in 3rd row
+    pdnote = pd.DataFrame(np.atleast_2d([predict_thr, truth_thr]), index=["p/t thr:"], columns=col_n)  # pyright: ignore [reportArgumentType, reportCallIssue]
+    cmdf = pd.concat([cmdf, pdnote])
+    cmndf = pd.concat([cmndf, pdnote])
+
+    return cmdf, cmndf