tsadmetrics 0.1.16__py3-none-any.whl → 1.0.0__py3-none-any.whl

tsadmetrics/utils/functions_auc.py

@@ -0,0 +1,393 @@
+import numpy as np
+import warnings
+from functools import partial
+
+
+def average_binary_score(binary_metric, y_true, y_score):
+    """Average a binary metric for multilabel classification.
+
+    Parameters
+    ----------
+    y_true : array, shape = [n_samples] or [n_samples, n_classes]
+        True binary labels in binary label indicators.
+
+    y_score : array, shape = [n_samples] or [n_samples, n_classes]
+        Target scores, can either be probability estimates of the positive
+        class, confidence values, or binary decisions.
+
+
+    Returns
+    -------
+    score : float or array of shape [n_classes]
+        If not ``None``, average the score, else return the score for each
+        classes.
+
+    """
+    
+
+    not_average_axis = 1
+    average_weight = None
+
+
+
+    if y_true.ndim == 1:
+        y_true = y_true.reshape((-1, 1))
+
+    if y_score.ndim == 1:
+        y_score = y_score.reshape((-1, 1))
+
+    n_classes = y_score.shape[not_average_axis]
+    score = np.zeros((n_classes,))
+    for c in range(n_classes):
+        y_true_c = y_true.take([c], axis=not_average_axis).ravel()
+        y_score_c = y_score.take([c], axis=not_average_axis).ravel()
+        score[c] = binary_metric(y_true_c, y_score_c)
+
+    # Average the results
+    return score
+    
+
+def auc(x, y):
+    """Compute Area Under the Curve (AUC) using the trapezoidal rule.
+
+    This is a general function, given points on a curve.  For computing the
+    area under the ROC-curve, see :func:`roc_auc_score`.  For an alternative
+    way to summarize a precision-recall curve, see
+    :func:`average_precision_score`.
+
+    Parameters
+    ----------
+    x : array-like of shape (n,)
+        X coordinates. These must be either monotonic increasing or monotonic
+        decreasing.
+    y : array-like of shape (n,)
+        Y coordinates.
+
+    Returns
+    -------
+    auc : float
+        Area Under the Curve.
+
+    """
+   
+
+    if len(x) < 2:
+        raise ValueError(
+            "At least 2 points are needed to compute area under curve, but x.shape = %s"
+            % x.shape
+        )
+
+    direction = 1
+    dx = np.diff(x)
+    if np.any(dx < 0):
+        if np.all(dx <= 0):
+            direction = -1
+        else:
+            raise ValueError("x is neither increasing nor decreasing : {}.".format(x))
+
+    area = direction * np.trapz(y, x)
+    if isinstance(area, np.memmap):
+        # Reductions such as .sum used internally in trapezoid do not return a
+        # scalar by default for numpy.memmap instances contrary to
+        # regular numpy.ndarray instances.
+        area = area.dtype.type(area)
+    return area
+
+def stable_cumsum(arr, axis=None):
+    """Use high precision for cumsum and check that final value matches sum.
+
+    Warns if the final cumulative sum does not match the sum (up to the chosen
+    tolerance).
+
+    Parameters
+    ----------
+    arr : array-like
+        To be cumulatively summed as flat.
+    axis : int, default=None
+        Axis along which the cumulative sum is computed.
+        The default (None) is to compute the cumsum over the flattened array.
+
+    Returns
+    -------
+    out : ndarray
+        Array with the cumulative sums along the chosen axis.
+    """
+    out = np.cumsum(arr, axis=axis, dtype=np.float64)
+
+    return out
+
+def binary_clf_curve(y_true, y_score):
+    """Calculate true and false positives per binary classification threshold.
+
+    Parameters
+    ----------
+    y_true : ndarray of shape (n_samples,)
+        True targets of binary classification.
+
+    y_score : ndarray of shape (n_samples,)
+        Estimated probabilities or output of a decision function.
+
+    pos_label : int, float, bool or str, default=None
+        The label of the positive class.
+
+    sample_weight : array-like of shape (n_samples,), default=None
+        Sample weights.
+
+    Returns
+    -------
+    fps : ndarray of shape (n_thresholds,)
+        A count of false positives, at index i being the number of negative
+        samples assigned a score >= thresholds[i]. The total number of
+        negative samples is equal to fps[-1] (thus true negatives are given by
+        fps[-1] - fps).
+
+    tps : ndarray of shape (n_thresholds,)
+        An increasing count of true positives, at index i being the number
+        of positive samples assigned a score >= thresholds[i]. The total
+        number of positive samples is equal to tps[-1] (thus false negatives
+        are given by tps[-1] - tps).
+
+    thresholds : ndarray of shape (n_thresholds,)
+        Decreasing score values.
+    """
+
+
+
+
+    # make y_true a boolean vector
+    y_true = y_true == 1
+
+    # sort scores and corresponding truth values
+    desc_score_indices = np.argsort(y_score, kind="mergesort")[::-1]
+    y_score = y_score[desc_score_indices]
+    y_true = y_true[desc_score_indices]
+    weight = 1.0
+
+    # y_score typically has many tied values. Here we extract
+    # the indices associated with the distinct values. We also
+    # concatenate a value for the end of the curve.
+    distinct_value_indices = np.where(np.diff(y_score))[0]
+    threshold_idxs = np.r_[distinct_value_indices, len(y_true) - 1]
+    
+    # accumulate the true positives with decreasing threshold
+    tps = stable_cumsum(y_true * weight)[threshold_idxs]
+    fps = 1 + threshold_idxs - tps
+    return fps, tps, y_score[threshold_idxs]
+
+
+def roc_curve(
+    y_true, y_score
+):
+    """Compute Receiver operating characteristic (ROC).
+
+    Note: this implementation is restricted to the binary classification task.
+
+    Read more in the :ref:`User Guide <roc_metrics>`.
+
+    Parameters
+    ----------
+    y_true : array-like of shape (n_samples,)
+        True binary labels. If labels are not either {-1, 1} or {0, 1}, then
+        pos_label should be explicitly given.
+
+    y_score : array-like of shape (n_samples,)
+        Target scores, can either be probability estimates of the positive
+        class, confidence values, or non-thresholded measure of decisions
+        (as returned by "decision_function" on some classifiers).
+
+    Returns
+    -------
+    fpr : ndarray of shape (>2,)
+        Increasing false positive rates such that element i is the false
+        positive rate of predictions with score >= `thresholds[i]`.
+
+    tpr : ndarray of shape (>2,)
+        Increasing true positive rates such that element `i` is the true
+        positive rate of predictions with score >= `thresholds[i]`.
+
+    thresholds : ndarray of shape (n_thresholds,)
+        Decreasing thresholds on the decision function used to compute
+        fpr and tpr. `thresholds[0]` represents no instances being predicted
+        and is arbitrarily set to `np.inf`.
+
+    """
+  
+    fps, tps, thresholds = binary_clf_curve(
+        y_true, y_score
+    )
+
+    if  len(fps) > 2:
+        optimal_idxs = np.where(
+            np.r_[True, np.logical_or(np.diff(fps, 2), np.diff(tps, 2)), True]
+        )[0]
+        fps = fps[optimal_idxs]
+        tps = tps[optimal_idxs]
+        thresholds = thresholds[optimal_idxs]
+
+    # Add an extra threshold position
+    # to make sure that the curve starts at (0, 0)
+    tps = np.r_[0, tps]
+    fps = np.r_[0, fps]
+    # get dtype of `y_score` even if it is an array-like
+    thresholds = np.r_[np.inf, thresholds]
+    if fps[-1] <= 0:
+        warnings.warn(
+            "No negative samples in y_true, false positive value should be meaningless."
+        )
+        fpr = np.repeat(np.nan, fps.shape)
+    else:
+        fpr = fps / fps[-1]
+
+    if tps[-1] <= 0:
+        warnings.warn(
+            "No positive samples in y_true, true positive value should be meaningless"
+        )
+        tpr = np.repeat(np.nan, tps.shape)
+    else:
+        tpr = tps / tps[-1]
+
+    return fpr, tpr, thresholds
+
+
+def precision_recall_curve(
+    y_true, probas_pred, 
+):
+    """Compute precision-recall pairs for different probability thresholds.
+
+    Note: this implementation is restricted to the binary classification task.
+
+    The precision is the ratio ``tp / (tp + fp)`` where ``tp`` is the number of
+    true positives and ``fp`` the number of false positives. The precision is
+    intuitively the ability of the classifier not to label as positive a sample
+    that is negative.
+
+    The recall is the ratio ``tp / (tp + fn)`` where ``tp`` is the number of
+    true positives and ``fn`` the number of false negatives. The recall is
+    intuitively the ability of the classifier to find all the positive samples.
+
+    The last precision and recall values are 1. and 0. respectively and do not
+    have a corresponding threshold. This ensures that the graph starts on the
+    y axis.
+
+    The first precision and recall values are precision=class balance and recall=1.0
+    which corresponds to a classifier that always predicts the positive class.
+
+    Parameters
+    ----------
+    y_true : array-like of shape (n_samples,)
+        True binary labels. If labels are not either {-1, 1} or {0, 1}, then
+        pos_label should be explicitly given.
+
+    probas_pred : array-like of shape (n_samples,)
+        Target scores, can either be probability estimates of the positive
+        class, or non-thresholded measure of decisions (as returned by
+        `decision_function` on some classifiers).
+
+
+    Returns
+    -------
+    precision : ndarray of shape (n_thresholds + 1,)
+        Precision values such that element i is the precision of
+        predictions with score >= thresholds[i] and the last element is 1.
+
+    recall : ndarray of shape (n_thresholds + 1,)
+        Decreasing recall values such that element i is the recall of
+        predictions with score >= thresholds[i] and the last element is 0.
+
+    thresholds : ndarray of shape (n_thresholds,)
+        Increasing thresholds on the decision function used to compute
+        precision and recall where `n_thresholds = len(np.unique(probas_pred))`.
+
+    """
+    fps, tps, thresholds = binary_clf_curve(
+        y_true, probas_pred
+    )
+
+    ps = tps + fps
+    # Initialize the result array with zeros to make sure that precision[ps == 0]
+    # does not contain uninitialized values.
+    precision = np.zeros_like(tps)
+    np.divide(tps, ps, out=precision, where=(ps != 0))
+
+    # When no positive label in y_true, recall is set to 1 for all thresholds
+    # tps[-1] == 0 <=> y_true == all negative labels
+    if tps[-1] == 0:
+        warnings.warn(
+            "No positive class found in y_true, "
+            "recall is set to one for all thresholds."
+        )
+        recall = np.ones_like(tps)
+    else:
+        recall = tps / tps[-1]
+
+    # reverse the outputs so recall is decreasing
+    sl = slice(None, None, -1)
+    return np.hstack((precision[sl], 1)), np.hstack((recall[sl], 0)), thresholds[sl]
+
+
+def average_precision_score(
+    y_true, y_score,
+):
+    """Compute average precision (AP) from prediction scores.
+
+    AP summarizes a precision-recall curve as the weighted mean of precisions
+    achieved at each threshold, with the increase in recall from the previous
+    threshold used as the weight:
+
+    .. math::
+        \\text{AP} = \\sum_n (R_n - R_{n-1}) P_n
+
+    where :math:`P_n` and :math:`R_n` are the precision and recall at the nth
+    threshold [1]_. This implementation is not interpolated and is different
+    from computing the area under the precision-recall curve with the
+    trapezoidal rule, which uses linear interpolation and can be too
+    optimistic.
+
+    Read more in the :ref:`User Guide <precision_recall_f_measure_metrics>`.
+
+    Parameters
+    ----------
+    y_true : array-like of shape (n_samples,) or (n_samples, n_classes)
+        True binary labels or binary label indicators.
+
+    y_score : array-like of shape (n_samples,) or (n_samples, n_classes)
+        Target scores, can either be probability estimates of the positive
+        class, confidence values, or non-thresholded measure of decisions
+        (as returned by :term:`decision_function` on some classifiers).
+
+
+
+    Returns
+    -------
+    average_precision : float
+        Average precision score.
+
+
+
+    
+    """
+
+    def _binary_uninterpolated_average_precision(
+        y_true, y_score, pos_label=1, sample_weight=None
+    ):
+        precision, recall, _ = precision_recall_curve(
+            y_true, y_score, pos_label=pos_label, sample_weight=sample_weight
+        )
+        # Return the step function integral
+        # The following works because the last entry of precision is
+        # guaranteed to be 1, as returned by precision_recall_curve
+        return -np.sum(np.diff(recall) * np.array(precision)[:-1])
+
+
+    # Convert to Python primitive type to avoid NumPy type / Python str
+    # comparison. See https://github.com/numpy/numpy/issues/6784
+    present_labels = np.unique(y_true).tolist()
+
+
+
+    average_precision = partial(
+        _binary_uninterpolated_average_precision, pos_label=1
+    )
+    return average_binary_score(
+        average_precision, y_true, y_score
+    )

tsadmetrics/utils/functions_conversion.py

@@ -0,0 +1,63 @@
+import numpy as np
+
+def pointwise_to_segmentwise(pointwise):
+    """Reformat anomaly time series from pointwise to segmentwise"""
+    segmentwise = []
+
+    prev = -10
+    for point in pointwise:
+        if point > prev + 1:
+            segmentwise.append([point, point])
+        else:
+            segmentwise[-1][-1] += 1
+        prev = point
+    return np.array(segmentwise)
+
+
+def segmentwise_to_pointwise(segmentwise):
+    """Reformat anomaly time series from segmentwise to pointwise"""
+    pointwise = []
+
+    for start, end in segmentwise:
+        for point in range(start, end + 1):
+            pointwise.append(point)
+
+    return np.array(pointwise)
+
+
+def segmentwise_to_full_series(segmentwise, length):
+    """Reformat anomaly time series from segmentwise to full_series"""
+    pw = segmentwise_to_pointwise(segmentwise)
+
+    return  pointwise_to_full_series(pw, length)
+
+def pointwise_to_full_series(pointwise, length):
+    """Reformat anomaly time series from pointwise to full_series"""
+    anomalies_full_series = np.zeros(length)
+    if len(pointwise) > 0:
+        assert pointwise[-1] < length
+        anomalies_full_series[pointwise] = 1
+    return np.array(anomalies_full_series)
+
+def full_series_to_pointwise(full_series):
+    """Reformat anomaly time series from full_series to pointwise"""
+    anomalies_pointwise = []
+    for i in range(len(full_series)):
+        if full_series[i]==1:
+            anomalies_pointwise.append(i)
+    return np.array(anomalies_pointwise)
+
+def full_series_to_segmentwise(full_series):
+    """Reformat anomaly time series from full_series to segmentwise"""
+    anomalies_segmentwise = []
+    i=0
+    while i < len(full_series):
+        if full_series[i] == 1:
+            start = i
+            while i < len(full_series) and full_series[i] == 1:
+                i += 1
+            end = i - 1
+            anomalies_segmentwise.append([start, end])
+        else:
+            i += 1
+    return np.array(anomalies_segmentwise)

tsadmetrics/utils/functions_counting_metrics.py

@@ -0,0 +1,26 @@
+import numpy as np
+
+def counting_method(y_true: np.array, y_pred: np.array, k: int):
+    em,da,ma,fa = 0,0,0,0
+    for i_gt in range(len(y_true)):
+        i_pa = i_gt
+        gt = y_true[i_gt]
+        pa = y_pred[i_pa]
+        if gt==1 and pa==1:
+            em+=1
+        elif gt==0 and pa==1:
+            fa+=1
+        elif gt==1 and pa==0:
+            anom_range = y_pred[i_gt-k:i_pa+k+1]
+            detected = False
+            for r in anom_range:
+                if r==1:
+                    em+=1
+                    detected=True
+                    break
+            if not detected:
+                ma+=1
+        elif gt==0 and pa==0:
+            pass
+
+    return em,da,ma,fa

tsadmetrics/{_tsadeval/latency_sparsity_aware.py → utils/functions_latency_sparsity_aware.py}

@@ -4,7 +4,7 @@
 # All rights reserved.
 # 
 # Redistribution and use in source and binary forms, with or without modification,
-# are permitted provided that the following conditions are met:
+# are permitted provided that the following conditions are tem:
 # 
 # * Redistributions of source code must retain the above copyright notice, this
 #   list of conditions and the following disclaimer.

tsadmetrics/{_tsadeval/nabscore.py → utils/functions_nabscore.py}

@@ -20,7 +20,8 @@
 from collections import namedtuple
 import logging
 import math
-
+import numpy as np
+from .functions_conversion import pointwise_to_full_series, pointwise_to_segmentwise
 logger = logging.getLogger(__name__)
 AnomalyPoint = namedtuple(
   "AnomalyPoint",
@@ -311,3 +312,16 @@ class Sweeper(object):
       [x.sweepScore for x in anomalyList],
       matchingRow
     )
+
+def calculate_scores(sweeper,y_true, y_pred,length):
+    anomaly_scores = pointwise_to_full_series(y_pred, length)
+    timestamps = np.arange(length)
+    windowLimits = pointwise_to_segmentwise(y_true)
+    dataSetName = "dummyname"
+    anomalyList = sweeper.calcSweepScore(timestamps, anomaly_scores, windowLimits, dataSetName)
+    scoresByThreshold = sweeper.calcScoreByThreshold(anomalyList)
+
+    assert scoresByThreshold[0].threshold == 1.1  # all points regarded normal
+    assert scoresByThreshold[1].threshold == 1.0  # anomal points regarded anomal
+
+    return sweeper, scoresByThreshold[0].score, scoresByThreshold[1].score

tsadmetrics-1.0.0.dist-info/METADATA

@@ -0,0 +1,69 @@
+Metadata-Version: 2.1
+Name: tsadmetrics
+Version: 1.0.0
+Summary: Librería para evaluación de detección de anomalías en series temporales
+Home-page: https://github.com/pathsko/TSADmetrics
+Author: Pedro Rafael Velasco Priego
+Author-email: Pedro Rafael Velasco Priego <i12veprp@uco.es>
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+Requires-Dist: joblib==1.4.2
+Requires-Dist: numpy==1.24.4
+Requires-Dist: pandas==2.0.3
+Requires-Dist: PATE==0.1.1
+Requires-Dist: patsy==0.5.6
+Requires-Dist: python-dateutil==2.9.0.post0
+Requires-Dist: pytz==2024.1
+Requires-Dist: scikit-learn==1.3.2
+Requires-Dist: scipy==1.10.1
+Requires-Dist: six==1.16.0
+Requires-Dist: statsmodels==0.14.1
+Requires-Dist: threadpoolctl==3.5.0
+Requires-Dist: tzdata==2024.1
+
+# TSADmetrics - Time Series Anomaly Detection Metrics
+
+**TSADmetrics** is a Python library for evaluating anomaly detection algorithms in time series data.  
+It provides a comprehensive set of metrics specifically designed to handle the temporal nature of anomalies.
+
+---
+
+## Features
+
+- **Metric Taxonomy**: Metrics are categorized into types based on how they handle temporal context:
+
+  - **MPI Metrics**: Evaluate predictions at each point independently, ignoring temporal continuity.
+  - **MET Metrics**: Consider temporal context, analyzing when and how anomalies occur.
+    - **MDPT**: Partial detection within a real anomaly event counts as correct.
+    - **MDTP**: Requires detection to cover a significant fraction of the real anomaly.
+    - **MECT**: Measures alignment of real vs predicted anomaly events.
+    - **MPR**: Penalizes late detections.
+    - **MTDT**: Allows temporal tolerance for early or late detections.
+
+- **Direct Metric Usage**: Instantiate any metric class and call `compute()` for individual evaluation.
+
+- **Batch Evaluation**: Use `Runner` to evaluate multiple datasets and metrics at once, with support for both direct data and CSV/JSON input.
+
+- **Flexible Configuration**: Load metrics from YAML configuration files or global evaluation config files.
+
+- **CLI Tool**: Compute metrics directly from files without writing Python code.
+
+---
+
+## Installation
+
+Install TSADmetrics via pip:
+
+```bash
+pip install tsadmetrics
+```
+
+## Documentation
+
+The complete documentation for TSADmetrics is available at:  
+📚 [https://tsadmetrics.readthedocs.io/](https://tsadmetrics.readthedocs.io/)
+
+## Acknowledgements
+
+This library is based on the concepts and implementations from:  
+Sørbø, S., & Ruocco, M. (2023). *Navigating the metric maze: a taxonomy of evaluation metrics for anomaly detection in time series*. https://doi.org/10.1007/s10618-023-00988-8