tsadmetrics 0.1.16__py3-none-any.whl → 1.0.0__py3-none-any.whl

tsadmetrics/metrics/tem/tstm/TimeTolerantFScore.py

@@ -0,0 +1,85 @@
+from ....base.Metric import Metric
+import numpy as np
+
+class TimeTolerantFScore(Metric):
+    """
+    Calculate time tolerant F-score for anomaly detection in time series.
+    This metric is based on the standard F-score, but applies a temporal adjustment 
+    to the predictions before computing it. Specifically, a predicted anomalous point is considered 
+    a true positive if it lies within a temporal window of size :math:`{\\tau}` around any ground-truth anomalous point. 
+    This allows for small temporal deviations in the predictions to be tolerated. The adjusted predictions are then used 
+    to _compute the standard point-wise F-Score.
+
+    Implementation of https://link.springer.com/article/10.1007/s10618-023-00988-8
+
+    For more information, see the original paper:
+    https://arxiv.org/abs/2008.05788
+
+    Parameters:
+        t (int):
+            The time tolerance parameter.
+        beta (float):
+            The beta value, which determines the weight of precision in the combined score.
+            Default is 1, which gives equal weight to precision and recall.
+    """
+    name = "ttf"
+    binary_prediction = True
+    param_schema = {
+        "t": {
+            "default": 5,
+            "type": int
+        },
+        "beta": {
+            "default": 1.0,
+            "type": float
+        }
+    }
+
+    def __init__(self, **kwargs):
+        super().__init__(name="ttf", **kwargs)
+
+    def _compute(self, y_true, y_pred):
+        """
+        Calculate the time tolerant F-score (optimized version).
+        """
+        t = self.params['t']
+        beta = self.params['beta']
+        
+        # Precompute masks for efficiency
+        true_anomalies = y_true == 1
+        predictions = y_pred == 1
+        
+        # Create P′1 for recall: for each true anomaly, check if any prediction within ±t
+        p_prime1 = np.zeros_like(y_true, dtype=bool)
+        
+        for i in np.where(true_anomalies)[0]:
+            start = max(0, i - t)
+            end = min(len(y_pred), i + t + 1)
+            if np.any(predictions[start:end]):
+                p_prime1[i] = True
+        
+        # Create P′2 for precision: for each prediction, check if any true anomaly within ±t
+        p_prime2 = np.zeros_like(y_pred, dtype=bool)
+        
+        for j in np.where(predictions)[0]:
+            start = max(0, j - t)
+            end = min(len(y_true), j + t + 1)
+            if np.any(true_anomalies[start:end]):
+                p_prime2[j] = True
+        
+        # Calculate recall using P′1
+        tp_recall = np.sum(true_anomalies & p_prime1)
+        fn_recall = np.sum(true_anomalies & ~p_prime1)
+        recall = tp_recall / (tp_recall + fn_recall) if (tp_recall + fn_recall) > 0 else 0.0
+        
+        # Calculate precision using P′2  
+        tp_precision = np.sum(predictions & p_prime2)
+        fp_precision = np.sum(predictions & ~p_prime2)
+        precision = tp_precision / (tp_precision + fp_precision) if (tp_precision + fp_precision) > 0 else 0.0
+        
+        # Calculate F-score
+        if precision == 0 and recall == 0:
+            return 0.0
+        
+        f_score = ((1 + beta**2) * precision * recall) / (beta**2 * precision + recall)
+        return f_score

tsadmetrics/metrics/tem/tstm/VusPr.py

@@ -0,0 +1,51 @@
+from ....base.Metric import Metric
+from ....utils.functions_vus import generate_curve
+import numpy as np
+class VusPr(Metric):
+    """
+    Calculate the VUS-PR (Volume Under the PR Surface) score for anomaly detection in time series.
+
+    This metric is an extension of the classical AUC-PR, incorporating a temporal tolerance parameter `window`
+    that smooths the binary ground-truth labels. It allows for some flexibility in the detection of 
+    anomalies that are temporally close to the true events. The final metric integrates the PR-AUC
+    over several levels of temporal tolerance (from 0 to `window`), yielding a volume under the PR surface.
+
+    Implementation of https://link.springer.com/article/10.1007/s10618-023-00988-8
+    
+    For more information, see the original paper:
+    https://dl.acm.org/doi/10.14778/3551793.3551830
+
+    Parameters:
+        window (int):
+            Maximum temporal tolerance used to smooth the evaluation.
+            Default is 4.
+    """
+    name = "vus_pr"
+    binary_prediction = False
+    param_schema = {
+        "window": {
+            "default": 4,
+            "type": int
+        }
+    }
+
+    def __init__(self, **kwargs):
+        super().__init__(name="vus_pr", **kwargs)
+
+    def _compute(self, y_true, y_anomaly_scores):
+        """
+        Calculate the VUS-PR score.
+
+        Parameters:
+            y_true (np.array):
+                Ground-truth binary labels (0 = normal, 1 = anomaly).
+            y_anomaly_scores (np.array):
+                Anomaly scores for each time point.
+
+        Returns:
+            float: VUS-PR score.
+        """
+        window = self.params["window"]
+        _, _, _, _, _, _, _, pr = generate_curve(y_true, y_anomaly_scores, slidingWindow=window)
+
+        return pr

tsadmetrics/metrics/tem/tstm/VusRoc.py

@@ -0,0 +1,55 @@
+from ....base.Metric import Metric
+from ....utils.functions_vus import generate_curve
+import numpy as np
+
+class VusRoc(Metric):
+    """
+    Calculate the VUS-ROC (Volume Under the ROC Surface) score for anomaly detection in time series.
+
+    This metric extends the classical AUC-ROC by introducing a temporal tolerance parameter `l`, which
+    smooths the binary ground-truth labels. The idea is to allow a flexible evaluation that tolerates
+    small misalignments in the detection of anomalies. The final score is computed by integrating 
+    the ROC-AUC over different values of the tolerance parameter, from 0 to `window`, thus producing
+    a volume under the ROC surface.
+
+    Implementation of https://link.springer.com/article/10.1007/s10618-023-00988-8
+
+    For more information, see the original paper:
+    https://dl.acm.org/doi/10.14778/3551793.3551830
+
+    Parameters:
+        window (int):
+            Maximum temporal tolerance `l` used to smooth the evaluation.
+            Default is 4.
+    """
+    name = "vus_roc"
+    binary_prediction = False
+    param_schema = {
+        "window": {
+            "default": 4,
+            "type": int
+        }
+    }
+
+    def __init__(self, **kwargs):
+        super().__init__(name="vus_roc", **kwargs)
+
+    def _compute(self, y_true, y_anomaly_scores):
+        """
+        Calculate the VUS-ROC score.
+
+        Parameters:
+            y_true (np.array):
+                Ground-truth binary labels (0 = normal, 1 = anomaly).
+            y_anomaly_scores (np.array):
+                Anomaly scores for each time point.
+
+        Returns:
+            float: VUS-ROC score.
+        """
+
+        _, _, _, _, _, _, roc, _ = generate_curve(
+            y_true, y_anomaly_scores, self.params["window"]
+        )
+
+        return roc

tsadmetrics/metrics/tem/tstm/__init__.py

@@ -0,0 +1,15 @@
+from .AffiliationbasedFScore import AffiliationbasedFScore
+from .TimeTolerantFScore import TimeTolerantFScore
+from .VusPr import VusPr
+from .VusRoc import VusRoc
+from .PateFScore import PateFScore
+from .Pate import Pate
+
+__all__ = [
+    "AffiliationbasedFScore",
+    "TimeTolerantFScore",
+    "VusPr",
+    "VusRoc",
+    "PateFScore",
+    "Pate"
+]

tsadmetrics/{_tsadeval/affiliation/_integral_interval.py → utils/functions_affiliation.py}

@@ -1,15 +1,244 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 import math
-from .generics import _sum_wo_nan
-"""
-In order to shorten the length of the variables,
-the general convention in this file is to let:
-    - I for a predicted event (start, stop),
-    - Is for a list of predicted events,
-    - J for a ground truth event,
-    - Js for a list of ground truth events.
-"""
+import glob
+import os
+import gzip
+from itertools import groupby
+from operator import itemgetter
+
+#Generics
+def convert_vector_to_events(vector = [0, 1, 1, 0, 0, 1, 0]):
+    """
+    Convert a binary vector (indicating 1 for the anomalous instances)
+    to a list of events. The events are considered as durations,
+    i.e. setting 1 at index i corresponds to an anomalous interval [i, i+1).
+    
+    :param vector: a list of elements belonging to {0, 1}
+    :return: a list of couples, each couple representing the start and stop of
+    each event
+    """
+    positive_indexes = [idx for idx, val in enumerate(vector) if val > 0]
+    events = []
+    for k, g in groupby(enumerate(positive_indexes), lambda ix : ix[0] - ix[1]):
+        cur_cut = list(map(itemgetter(1), g))
+        events.append((cur_cut[0], cur_cut[-1]))
+    
+    # Consistent conversion in case of range anomalies (for indexes):
+    # A positive index i is considered as the interval [i, i+1),
+    # so the last index should be moved by 1
+    events = [(x, y+1) for (x,y) in events]
+        
+    return(events)
+
+
+def infer_Trange(events_pred, events_gt):
+    """
+    Given the list of events events_pred and events_gt, get the
+    smallest possible Trange corresponding to the start and stop indexes 
+    of the whole series.
+    Trange will not influence the measure of distances, but will impact the
+    measures of probabilities.
+    
+    :param events_pred: a list of couples corresponding to predicted events
+    :param events_gt: a list of couples corresponding to ground truth events
+    :return: a couple corresponding to the smallest range containing the events
+    """
+    if len(events_gt) == 0:
+        raise ValueError('The gt events should contain at least one event')
+    if len(events_pred) == 0:
+        # empty prediction, base Trange only on events_gt (which is non empty)
+        return(infer_Trange(events_gt, events_gt))
+        
+    min_pred = min([x[0] for x in events_pred])
+    min_gt = min([x[0] for x in events_gt])
+    max_pred = max([x[1] for x in events_pred])
+    max_gt = max([x[1] for x in events_gt])
+    Trange = (min(min_pred, min_gt), max(max_pred, max_gt))
+    return(Trange)
+
+def has_point_anomalies(events):
+    """
+    Checking whether events contain point anomalies, i.e.
+    events starting and stopping at the same time.
+    
+    :param events: a list of couples corresponding to predicted events
+    :return: True is the events have any point anomalies, False otherwise
+    """
+    if len(events) == 0:
+        return(False)
+    return(min([x[1] - x[0] for x in events]) == 0)
+
+def _sum_wo_nan(vec):
+    """
+    Sum of elements, ignoring math.isnan ones
+    
+    :param vec: vector of floating numbers
+    :return: sum of the elements, ignoring math.isnan ones
+    """
+    vec_wo_nan = [e for e in vec if not math.isnan(e)]
+    return(sum(vec_wo_nan))
+    
+def _len_wo_nan(vec):
+    """
+    Count of elements, ignoring math.isnan ones
+    
+    :param vec: vector of floating numbers
+    :return: count of the elements, ignoring math.isnan ones
+    """
+    vec_wo_nan = [e for e in vec if not math.isnan(e)]
+    return(len(vec_wo_nan))
+
+def read_gz_data(filename = 'data/machinetemp_groundtruth.gz'):
+    """
+    Load a file compressed with gz, such that each line of the
+    file is either 0 (representing a normal instance) or 1 (representing)
+    an anomalous instance.
+    :param filename: file path to the gz compressed file
+    :return: list of integers with either 0 or 1
+    """
+    with gzip.open(filename, 'rb') as f:
+        content = f.read().splitlines()
+    content = [int(x) for x in content]
+    return(content)
+
+def read_all_as_events():
+    """
+    Load the files contained in the folder `data/` and convert
+    to events. The length of the series is kept.
+    The convention for the file name is: `dataset_algorithm.gz`
+    :return: two dictionaries:
+        - the first containing the list of events for each dataset and algorithm,
+        - the second containing the range of the series for each dataset
+    """
+    filepaths = glob.glob('data/*.gz')
+    datasets = dict()
+    Tranges = dict()
+    for filepath in filepaths:
+        vector = read_gz_data(filepath)
+        events = convert_vector_to_events(vector)
+        # ad hoc cut for those files
+        cut_filepath = (os.path.split(filepath)[1]).split('_')
+        data_name = cut_filepath[0]
+        algo_name = (cut_filepath[1]).split('.')[0]
+        if not data_name in datasets:
+            datasets[data_name] = dict()
+            Tranges[data_name] = (0, len(vector))
+        datasets[data_name][algo_name] = events
+    return(datasets, Tranges)
+
+
+
+#Affiliation zone
+def t_start(j, Js = [(1,2),(3,4),(5,6)], Trange = (1,10)):
+    """
+    Helper for `E_gt_func`
+    
+    :param j: index from 0 to len(Js) (included) on which to get the start
+    :param Js: ground truth events, as a list of couples
+    :param Trange: range of the series where Js is included
+    :return: generalized start such that the middle of t_start and t_stop 
+    always gives the affiliation zone
+    """
+    b = max(Trange)
+    n = len(Js)
+    if j == n:
+        return(2*b - t_stop(n-1, Js, Trange))
+    else:
+        return(Js[j][0])
+    
+
+def t_stop(j, Js = [(1,2),(3,4),(5,6)], Trange = (1,10)):
+    """
+    Helper for `E_gt_func`
+    
+    :param j: index from 0 to len(Js) (included) on which to get the stop
+    :param Js: ground truth events, as a list of couples
+    :param Trange: range of the series where Js is included
+    :return: generalized stop such that the middle of t_start and t_stop 
+    always gives the affiliation zone
+    """
+    if j == -1:
+        a = min(Trange)
+        return(2*a - t_start(0, Js, Trange))
+    else:
+        return(Js[j][1])
+    
+
+def E_gt_func(j, Js, Trange):
+    """
+    Get the affiliation zone of element j of the ground truth
+    
+    :param j: index from 0 to len(Js) (excluded) on which to get the zone
+    :param Js: ground truth events, as a list of couples
+    :param Trange: range of the series where Js is included, can 
+    be (-math.inf, math.inf) for distance measures
+    :return: affiliation zone of element j of the ground truth represented
+    as a couple
+    """
+    range_left = (t_stop(j-1, Js, Trange) + t_start(j, Js, Trange))/2
+    range_right = (t_stop(j, Js, Trange) + t_start(j+1, Js, Trange))/2
+    return((range_left, range_right))
+
+def get_all_E_gt_func(Js, Trange):
+    """
+    Get the affiliation partition from the ground truth point of view
+    
+    :param Js: ground truth events, as a list of couples
+    :param Trange: range of the series where Js is included, can 
+    be (-math.inf, math.inf) for distance measures
+    :return: affiliation partition of the events
+    """
+    # E_gt is the limit of affiliation/attraction for each ground truth event
+    E_gt = [E_gt_func(j, Js, Trange) for j in range(len(Js))]
+    return(E_gt)
+
+
+def interval_intersection(I = (1, 3), J = (2, 4)): 
+    """
+    Intersection between two intervals I and J
+    I and J should be either empty or represent a positive interval (no point)
+    
+    :param I: an interval represented by start and stop
+    :param J: a second interval of the same form
+    :return: an interval representing the start and stop of the intersection (or None if empty)
+    """
+    if I is None:
+        return(None)
+    if J is None:
+        return(None)
+        
+    I_inter_J = (max(I[0], J[0]), min(I[1], J[1]))
+    if I_inter_J[0] >= I_inter_J[1]:
+        return(None)
+    else:
+        return(I_inter_J)
+    
+
+def affiliation_partition(Is = [(1,1.5),(2,5),(5,6),(8,9)], E_gt = [(1,2.5),(2.5,4.5),(4.5,10)]):
+    """
+    Cut the events into the affiliation zones
+    The presentation given here is from the ground truth point of view,
+    but it is also used in the reversed direction in the main function.
+    
+    :param Is: events as a list of couples
+    :param E_gt: range of the affiliation zones
+    :return: a list of list of intervals (each interval represented by either 
+    a couple or None for empty interval). The outer list is indexed by each
+    affiliation zone of `E_gt`. The inner list is indexed by the events of `Is`.
+    """
+    out = [None] * len(E_gt)
+    for j in range(len(E_gt)):
+        E_gt_j = E_gt[j]
+        discarded_idx_before = [I[1] < E_gt_j[0] for I in Is]  # end point of predicted I is before the begin of E
+        discarded_idx_after = [I[0] > E_gt_j[1] for I in Is] # start of predicted I is after the end of E
+        kept_index = [not(a or b) for a, b in zip(discarded_idx_before, discarded_idx_after)]
+        Is_j = [x for x, y in zip(Is, kept_index)]
+        out[j] = [interval_intersection(I, E_gt[j]) for I in Is_j]
+    return(out)
+
+# Single ground truth event
+
 
 def interval_length(J = (1,2)):
     """
@@ -462,3 +691,142 @@ def integral_interval_probaCDF_recall(I, J, E):
     d_right = f(cut_into_three[2])
     # It's an integral so summable
     return(d_left + d_middle + d_right)
+
+
+def affiliation_precision_proba(Is = [(1,2),(3,4),(5,6)], J = (2,5.5), E = (0,8)):
+    """
+    Compute the individual precision probability from Is to a single ground truth J
+    
+    :param Is: list of predicted events within the affiliation zone of J
+    :param J: couple representating the start and stop of a ground truth interval
+    :param E: couple representing the start and stop of the zone of affiliation of J
+    :return: individual precision probability in [0, 1], or math.nan if undefined
+    """
+    if all([I is None for I in Is]): # no prediction in the current area
+        return(math.nan) # undefined
+    return(sum([integral_interval_probaCDF_precision(I, J, E) for I in Is]) / sum_interval_lengths(Is))
+
+def affiliation_recall_proba(Is = [(1,2),(3,4),(5,6)], J = (2,5.5), E = (0,8)):
+    """
+    Compute the individual recall probability from a single ground truth J to Is
+    
+    :param Is: list of predicted events within the affiliation zone of J
+    :param J: couple representating the start and stop of a ground truth interval
+    :param E: couple representing the start and stop of the zone of affiliation of J
+    :return: individual recall probability in [0, 1]
+    """
+    Is = [I for I in Is if I is not None] # filter possible None in Is
+    if len(Is) == 0: # there is no prediction in the current area
+        return(0)
+    E_gt_recall = get_all_E_gt_func(Is, E) # here from the point of view of the predictions
+    Js = affiliation_partition([J], E_gt_recall) # partition of J depending of proximity with Is
+    return(sum([integral_interval_probaCDF_recall(I, J[0], E) for I, J in zip(Is, Js)]) / interval_length(J))
+def test_events(events):
+    """
+    Verify the validity of the input events
+    :param events: list of events, each represented by a couple (start, stop)
+    :return: None. Raise an error for incorrect formed or non ordered events
+    """
+    if type(events) is not list:
+        raise TypeError('Input `events` should be a list of couples')
+    if not all([type(x) is tuple for x in events]):
+        raise TypeError('Input `events` should be a list of tuples')
+    if not all([len(x) == 2 for x in events]):
+        raise ValueError('Input `events` should be a list of couples (start, stop)')
+    if not all([x[0] <= x[1] for x in events]):
+        raise ValueError('Input `events` should be a list of couples (start, stop) with start <= stop')
+    if not all([events[i][1] < events[i+1][0] for i in range(len(events) - 1)]):
+        raise ValueError('Couples of input `events` should be disjoint and ordered')
+
+def pr_from_events(events_pred, events_gt, Trange):
+    """
+    Compute the affiliation metrics including the precision/recall in [0,1],
+    along with the individual precision/recall distances and probabilities
+    
+    :param events_pred: list of predicted events, each represented by a couple
+    indicating the start and the stop of the event
+    :param events_gt: list of ground truth events, each represented by a couple
+    indicating the start and the stop of the event
+    :param Trange: range of the series where events_pred and events_gt are included,
+    represented as a couple (start, stop)
+    :return: dictionary with precision, recall, and the individual metrics
+    """
+    # testing the inputs
+    # test_events(events_pred)
+    # test_events(events_gt)
+    
+    # other tests
+    minimal_Trange = infer_Trange(events_pred, events_gt)
+    if not Trange[0] <= minimal_Trange[0]:
+        raise ValueError('`Trange` should include all the events')
+    if not minimal_Trange[1] <= Trange[1]:
+        raise ValueError('`Trange` should include all the events')
+    
+    if len(events_gt) == 0:
+        raise ValueError('Input `events_gt` should have at least one event')
+
+    if has_point_anomalies(events_pred) or has_point_anomalies(events_gt):
+        raise ValueError('Cannot manage point anomalies currently')
+
+    if Trange is None:
+        # Set as default, but Trange should be indicated if probabilities are used
+        raise ValueError('Trange should be indicated (or inferred with the `infer_Trange` function')
+
+    E_gt = get_all_E_gt_func(events_gt, Trange)
+    aff_partition = affiliation_partition(events_pred, E_gt)
+
+    # # Computing precision distance
+    # d_precision = [affiliation_precision_distance(Is, J) for Is, J in zip(aff_partition, events_gt)]
+    
+    # # Computing recall distance
+    # d_recall = [affiliation_recall_distance(Is, J) for Is, J in zip(aff_partition, events_gt)]
+
+    # Computing precision
+    p_precision = [affiliation_precision_proba(Is, J, E) for Is, J, E in zip(aff_partition, events_gt, E_gt)]
+
+    # Computing recall
+    p_recall = [affiliation_recall_proba(Is, J, E) for Is, J, E in zip(aff_partition, events_gt, E_gt)]
+
+    if _len_wo_nan(p_precision) > 0:
+        p_precision_average = _sum_wo_nan(p_precision) / _len_wo_nan(p_precision)
+    else:
+        p_precision_average = p_precision[0] # math.nan
+    p_recall_average = sum(p_recall) / len(p_recall)
+
+    dict_out = dict({'precision': p_precision_average,
+                     'recall': p_recall_average,
+                     'individual_precision_probabilities': p_precision,
+                     'individual_recall_probabilities': p_recall})
+    return(dict_out)
+
+def produce_all_results():
+    """
+    Produce the affiliation precision/recall for all files
+    contained in the `data` repository
+    :return: a dictionary indexed by data names, each containing a dictionary
+    indexed by algorithm names, each containing the results of the affiliation
+    metrics (precision, recall, individual probabilities and distances)
+    """
+    datasets, Tranges = read_all_as_events() # read all the events in folder `data`
+    results = dict()
+    for data_name in datasets.keys():
+        results_data = dict()
+        for algo_name in datasets[data_name].keys():
+            if algo_name != 'groundtruth':
+                results_data[algo_name] = pr_from_events(datasets[data_name][algo_name],
+                                                         datasets[data_name]['groundtruth'],
+                                                         Tranges[data_name])
+        results[data_name] = results_data
+    return(results)
+
+
+def reformat_segments(segments):
+    segments = include_end_of_segments(segments)
+    segments = tuplify_segments(segments)
+    return segments
+
+def include_end_of_segments(segments):
+    return [[start, end + 1] for start, end in segments]
+
+def tuplify_segments(segments):
+    return [tuple(segment) for segment in segments]