tsadmetrics 0.1.15__py3-none-any.whl → 0.1.17__py3-none-any.whl

tsadmetrics/metric_utils.py

@@ -1,241 +1,5 @@
 import numpy as np
-from ._tsadeval.metrics import Binary_anomalies, pointwise_to_full_series, segmentwise_to_full_series, DelayThresholdedPointAdjust
-def get_tp_tn_fp_fn_point_wise(y_true: np.array,y_pred: np.array):
-    TP,TN,FP,FN=0,0,0,0
-    for true,pred in zip(y_true,y_pred):
-        if true==pred:
-            if true==1:
-                TP+=1
-            else:
-                TN+=1
-        else:
-            if true==1:
-                FN+=1
-            else:
-                FP+=1
-    return TP,TN,FP,FN
-
-
-def get_events(y_true,anomaly=True):
-    events = []
-    start_idx = None
-    v = 0
-    if anomaly:
-        v = 1
-    else:
-        v = 0
-
-    for i, val in enumerate(y_true):
-        if val == v:  # Si encontramos el inicio de un evento
-            if start_idx is None:
-                start_idx = i  # Establecemos el inicio del evento
-        elif start_idx is not None:  # Si encontramos el final de un evento
-            events.append((start_idx, i - 1))  # Agregamos el evento a la lista de eventos
-            start_idx = None  # Restablecemos el inicio del evento
-
-    if start_idx is not None:  # Si al final de la secuencia aún estamos dentro de un evento
-        events.append((start_idx, len(y_true) - 1))  # Agregamos el evento final a la lista de eventos
-
-
-    return events
-
-def calculate_intersection(event1, event2):
-    start_intersection = max(event1[0], event2[0])
-    end_intersection = min(event1[1], event2[1])
-
-    # If there is an intersection, return the range of the intersection, otherwise return None
-    if start_intersection <= end_intersection:
-        return [start_intersection, end_intersection]
-    else:
-        return None
-
-def get_tp_tn_fp_fn_point_adjusted(y_true: np.array,y_pred: np.array):
-    TP, TN, FP, FN = get_tp_tn_fp_fn_point_wise(y_true, y_pred)
-    TP=0
-    FN=0
-    y_true_events = get_events(y_true,anomaly=True)
-    y_pred_events = get_events(y_pred,anomaly=True)
-
-    i_true = 0
-    i_pred = 0
-    while i_true<len(y_true_events):
-        detected = False
-        while i_pred<len(y_pred_events) and y_true_events[i_true][1]>y_pred_events[i_pred][0]:
-            if calculate_intersection(y_true_events[i_true],y_pred_events[i_pred]) is not None:
-                TP+= y_true_events[i_true][1]-y_true_events[i_true][0]+1
-                detected=True
-                break
-            elif y_true_events[i_true][0]>y_pred_events[i_pred][1]:
-                i_pred+=1
-        
-        if not detected:
-            FN+= y_true_events[i_true][1]-y_true_events[i_true][0]+1
-        i_true+=1
-    
-    return TP, TN, FP, FN
-
-def get_tp_tn_fp_fn_delay_th_point_adjusted(y_true: np.array,y_pred: np.array,k: int):
-    TP, TN, FP, FN = get_tp_tn_fp_fn_point_wise(y_true, y_pred)
-    TP=0
-    FN=0
-    y_true_events = get_events(y_true,anomaly=True)
-    y_pred_events = get_events(y_pred,anomaly=True)
-
-    i_true = 0
-    i_pred = 0
-    while i_true<len(y_true_events):
-        detected = False
-        while i_pred<len(y_pred_events) and y_true_events[i_true][1]>y_pred_events[i_pred][0]:
-            intersec = calculate_intersection(y_true_events[i_true],y_pred_events[i_pred])
-            if intersec is not None and intersec[0]-y_true_events[i_true][0]<k:
-                TP+= y_true_events[i_true][1]-y_true_events[i_true][0]+1
-                detected=True
-                break
-            else:
-                i_pred+=1
-        
-        if not detected:
-            FN+= y_true_events[i_true][1]-y_true_events[i_true][0]+1
-        i_true+=1
-    
-    return TP, TN, FP, FN
-
-def get_tp_tn_fp_fn_point_adjusted_at_k(y_true: np.array,y_pred: np.array, k: float):
-    TP, TN, FP, FN = get_tp_tn_fp_fn_point_wise(y_true, y_pred)
-    TP=0
-    FN=0
-    y_true_events = get_events(y_true,anomaly=True)
-    y_pred_events = get_events(y_pred,anomaly=True)
-
-    i_true = 0
-    i_pred = 0
-    while i_true<len(y_true_events):
-        detected = False
-        while i_pred<len(y_pred_events) and y_true_events[i_true][1]>y_pred_events[i_pred][0]:
-            intersec = calculate_intersection(y_true_events[i_true],y_pred_events[i_pred])
-            if intersec is not None:
-                event_size = y_true_events[i_true][1]-y_true_events[i_true][0]+1
-                intersec_size = intersec[1]-intersec[0]+1
-            if intersec is not None and intersec_size/event_size>=k:
-                
-                TP+= y_true_events[i_true][1]-y_true_events[i_true][0]+1
-                detected=True
-                break
-            else:
-                i_pred+=1
-        
-        if not detected:
-            FN+= y_true_events[i_true][1]-y_true_events[i_true][0]+1
-        i_true+=1
-    
-    return TP, TN, FP, FN
-
-
-def get_tp_tn_fp_fn_latency_sparsity_aw(y_true: np.array, y_pred: np.array, ni: int):
-    batched_shape = (int(np.ceil(y_pred.shape[0] / ni)), 1)
-    label_batch = np.zeros(batched_shape)
-    pred_batch = np.zeros(batched_shape)
-    actual = np.copy(y_true)
-    predict = np.copy(y_pred)
-    detect_state = False  # triggered when a True anomaly is detected by model
-    anomaly_batch_count = 0
-    i, i_ni = 0, 0
-    step = ni
-
-    while i < len(y_true) and step > 1:
-        j = min(i + step, len(y_true))  # end of ni (batch) starting at i
-
-        # Adjust step size if needed
-        if step > 2 and actual[i:j].sum() > 1:
-            if np.diff(np.where(actual[i:j])).max() > 1:  # if it finds an interruption in the true label continuity
-                step = min(int((j - i) / 2), 2)  # reduce step size
-                label_batch = np.append(label_batch, [[0]], axis=0)
-                pred_batch = np.append(pred_batch, [[0]], axis=0)  # increase size
-                j = i + step
-            else:
-                step = ni
-        else:
-            step = ni
-
-        # Start rolling window scoring
-        if actual[i:j].max():  # If label = T
-            if not actual[i]:  # if first value is normal
-                detect_state = False
-            s = actual[i:j].argmax()  # this is the index of the first occurrence
-            if detect_state:  # if anomaly was previously detected by model
-                anomaly_batch_count += 1
-                pred_batch[i_ni], label_batch[i_ni], predict[i + s:j] = 1, 1, 1
-            elif predict[i:j].max():  # if alert was detected with T
-                detect_state = True  # turn on detection state
-                anomaly_batch_count += 1
-                pred_batch[i_ni], label_batch[i_ni], predict[i + s:j] = 1, 1, 1
-            else:
-                detect_state = False
-                label_batch[i_ni] = 1
-        else:
-            detect_state = False
-            if predict[i:j].max():  # if False positive
-                pred_batch[i_ni] = 1
-        i += step
-        i_ni += 1
-
-    if ni == 1:
-        return get_tp_tn_fp_fn_point_wise(actual, predict)
-
-    return get_tp_tn_fp_fn_point_wise(label_batch.flatten().astype(int), pred_batch.flatten().astype(int))
-
-def get_tp_fp_fn_segment_wise(y_true: np.array,y_pred: np.array):
-    
-
-    y_true_anomaly_events = get_events(y_true)
-    pred_anomaly_events = get_events(y_pred)
-    y_true_normal_events = get_events(y_true,False)
-    pred_normal_events = get_events(y_pred,False)
-
-    TP = 0
-    FN = 0
-    FP = 0
-    #TP
-    i = 0
-    for e_p in pred_anomaly_events:
-        
-        c, d = e_p
-        while i<len(y_true_anomaly_events):
-            e_g = y_true_anomaly_events[i]
-            a, b = e_g
-            if a>d:
-              break
-
-            if b<c:
-                i+=1
-                continue
-
-            else:
-              if max(a, c) <= min(b, d):
-                  TP+=1
-            
-
-            i+=1
-
-    #FN
-    FN = len(y_true_anomaly_events) - TP
-    #FP
-    i = 0
-    for e_p in y_true_normal_events:
-        
-        c, d = e_p
-        while i<len(pred_anomaly_events):
-            e_g = pred_anomaly_events[i]
-            a, b = e_g
-            if a>d:
-                break
-            if b<c:
-                i+=1
-                continue
-            if calculate_intersection(e_g, e_p) is not None:
-                FP+=1
-            i+=1
-    return TP, FP, FN
+from ._tsadeval.metrics import pointwise_to_full_series, segmentwise_to_full_series
 
 def is_full_series(length: int, anomalies: np.array):
     # [1 0 1 1 0]
@@ -330,4 +94,5 @@ def position(i, anomaly_length,bias):
         else:
             return anomaly_length - i + 1
     else:
-        raise Exception("Error, wrong bias value.")
+        raise Exception("Error, wrong bias value.")
+    

tsadmetrics/non_binary_metrics.py

@@ -1,7 +1,8 @@
 import numpy as np
 from ._tsadeval.metrics import *
+from .validation import validate_non_binary_inputs
 from sklearn.metrics import auc
-from pate.PATE_metric import PATE
+from pate.PATE_metric import PATE 
 def precision_at_k(y_true : np.array, y_anomaly_scores:  np.array):
     """
     Calculate the precision at k score for anomaly detection in time series.
@@ -14,6 +15,8 @@ def precision_at_k(y_true : np.array, y_anomaly_scores:  np.array):
     The value of k is automatically set to the number of true anomalies present in 
     y_true. That is, k = sum(y_true).
 
+    Implementation of https://link.springer.com/article/10.1007/s10618-023-00988-8
+
     Parameters:
         y_true (np.array):
             The ground truth binary labels for the time series data.
@@ -23,6 +26,8 @@ def precision_at_k(y_true : np.array, y_anomaly_scores:  np.array):
     Returns:
         float: The precision at k score.
     """
+    validate_non_binary_inputs(y_true, y_anomaly_scores)
+
     m = PatK_pw(y_true,y_anomaly_scores)
 
     return m.get_score()
@@ -35,6 +40,8 @@ def auc_roc_pw(y_true : np.array, y_anomaly_scores:  np.array):
     computed in a point-wise manner. That is, each point in the time series is treated
     independently when calculating true positives, false positives, and false negatives.
 
+    Implementation of https://link.springer.com/article/10.1007/s10618-023-00988-8
+
     Parameters:
         y_true (np.array):
             Ground-truth binary labels for the time series (0 = normal, 1 = anomaly).
@@ -44,6 +51,7 @@ def auc_roc_pw(y_true : np.array, y_anomaly_scores:  np.array):
     Returns:
         float: AUC-ROC score.
     """
+    validate_non_binary_inputs(y_true, y_anomaly_scores)
     
     m = AUC_ROC(y_true,y_anomaly_scores)
 
@@ -58,6 +66,8 @@ def auc_pr_pw(y_true : np.array ,y_anomaly_scores:  np.array):
     computed in a point-wise manner. That is, each point in the time series is treated
     independently when calculating precision and recall.
 
+    Implementation of https://link.springer.com/article/10.1007/s10618-023-00988-8
+
     Parameters:
         y_true (np.array):
             Ground-truth binary labels for the time series (0 = normal, 1 = anomaly).
@@ -67,11 +77,98 @@ def auc_pr_pw(y_true : np.array ,y_anomaly_scores:  np.array):
     Returns:
         float: AUC-PR score.
     """
+    validate_non_binary_inputs(y_true, y_anomaly_scores)
+
     m = AUC_PR_pw(y_true,y_anomaly_scores)
 
     return m.get_score()
 
 
+def auc_roc_pa(y_true: np.array, y_anomaly_scores: np.array):
+    """
+    Calculate the AUC-ROC score using point-adjusted evaluation for anomaly detection in time series.
+
+    This is the standard Area Under the Receiver Operating Characteristic Curve (AUC-ROC), but instead 
+    of computing true positive rate (TPR) and false positive rate (FPR) point-wise, it uses a point-adjusted 
+    approach. Specifically, for each ground-truth anomalous segment, if at least one point within that 
+    segment is predicted as anomalous, the entire segment is considered correctly detected. The adjusted 
+    predictions are then compared to the ground-truth labels to compute true positives, false positives, 
+    and false negatives, which are used to construct the ROC curve.
+
+    Implementation of https://link.springer.com/article/10.1007/s10618-023-00988-8
+
+    Parameters:
+        y_true (np.array):
+            Ground-truth binary labels for the time series (0 = normal, 1 = anomaly).
+        y_anomaly_scores (np.array):
+            Continuous anomaly scores assigned to each point in the series.
+
+    Returns:
+        float: AUC-ROC score (with point-adjusted evaluation).
+    """
+    validate_non_binary_inputs(y_true, y_anomaly_scores)
+    
+    tprs = [0]
+    fprs = [0]
+    tps, fps, fns = [], [], []
+
+    p_adj = PointAdjust(len(y_true), y_true, (np.array(y_anomaly_scores) >= 0.5).astype(int))
+    segments = p_adj.get_gt_anomalies_segmentwise()
+    idx = np.argsort(y_anomaly_scores)[::-1].astype(int)
+    y_true_sorted = np.array(y_true)[idx]
+    y_anomaly_scores_sorted = np.array(y_anomaly_scores)[idx]
+    
+    segment_mins = []
+    for start, end in segments:
+        anoms_scores = y_anomaly_scores[start:end+1]
+        segment_mins.append([np.max(anoms_scores), end-start+1])
+
+    for i_t in range(len(y_anomaly_scores_sorted)):
+        fp, tp, fn = 0, 0, 0
+        if i_t > 0 and y_anomaly_scores_sorted[i_t] == y_anomaly_scores_sorted[i_t-1]:
+            tp = tps[-1]
+            fp = fps[-1]
+            fn = fns[-1]
+        else:
+            if y_true_sorted[i_t] == 0:
+                # FP
+                if len(fps) == 0:
+                    aux_y_pred = (y_anomaly_scores >= y_anomaly_scores_sorted[i_t]).astype(int)
+                    for i in range(len(aux_y_pred)):
+                        if aux_y_pred[i] == 1 and y_true[i] == 0:
+                            fp += 1
+                else:
+                    fp = fps[i_t-1] + 1
+            else:
+                if len(fps) == 0:
+                    aux_y_pred = (y_anomaly_scores >= y_anomaly_scores_sorted[i_t]).astype(int)
+                    for i in range(len(aux_y_pred)):
+                        if aux_y_pred[i] == 1 and y_true[i] == 0:
+                            fp += 1
+                else:
+                    fp = fps[i_t-1]
+            for score, length in segment_mins:
+                if score >= y_anomaly_scores_sorted[i_t]:
+                    # TP
+                    tp += length
+                else:
+                    # FN
+                    fn += length
+        tps.append(tp)
+        fns.append(fn)
+        fps.append(fp)
+    for tp, fp, fn in zip(tps, fps, fns):
+        if tp + fn > 0:
+            tprs.append(tp / (tp + fn))
+        else:
+            tprs.append(0)
+        if fp + (len(y_true) - np.sum(y_true)) > 0:
+            fprs.append(fp / (fp + (len(y_true) - np.sum(y_true))))
+        else:
+            fprs.append(0)
+    
+    auc_value = auc(fprs, tprs)
+    return auc_value
 
 def auc_pr_pa(y_true: np.array, y_anomaly_scores: np.array):
     """
@@ -84,6 +181,8 @@ def auc_pr_pa(y_true: np.array, y_anomaly_scores: np.array):
     to the ground-truth labels to compute true positives, false positives, and false negatives,
     which are used to construct the PR curve.
 
+    Implementation of https://link.springer.com/article/10.1007/s10618-023-00988-8
+
     Parameters:
         y_true (np.array):
             Ground-truth binary labels for the time series (0 = normal, 1 = anomaly).
@@ -93,7 +192,7 @@ def auc_pr_pa(y_true: np.array, y_anomaly_scores: np.array):
     Returns:
         float: AUC-PR score (with point-adjusted evaluation).
     """
-        
+    validate_non_binary_inputs(y_true, y_anomaly_scores)
     
     precisions = [1]
     recalls = [0]
@@ -164,114 +263,6 @@ def auc_pr_pa(y_true: np.array, y_anomaly_scores: np.array):
 
 
 
-def auc_pr_sw(y_true: np.array, y_anomaly_scores: np.array):
-    """
-    Calculate the AUC-PR score using segment-wise evaluation for anomaly detection in time series.
-
-    This is the standard Area Under the Precision-Recall Curve (AUC-PR), but it uses a segment-wise 
-    adjustment when computing precision and recall. In this evaluation, each contiguous segment of 
-    anomalous ground-truth points is treated as a single unit. A true positive is counted if at least 
-    one predicted anomaly overlaps with the segment. A false negative occurs when a segment is 
-    completely missed, and a false positive is recorded for each predicted anomalous segment 
-    that does not overlap with any ground-truth anomaly. These adjusted counts are then used 
-    to compute precision and recall for constructing the PR curve.
-
-    Parameters:
-        y_true (np.array):
-            Ground-truth binary labels for the time series (0 = normal, 1 = anomaly).
-        y_anomaly_scores (np.array):
-            Continuous anomaly scores assigned to each point in the series.
-
-    Returns:
-        float: AUC-PR score (with segment-wise evaluation).
-    """
-
-    precisions = [1]
-    recalls = [0]
-    tps,fps,fns = [],[],[]
-
-
-    segments = []
-    i=0
-    while i < len(y_true):
-        if y_true[i] == 1:
-            start = i
-            end = i
-            while i < len(y_true) and y_true[i] == 1:
-                end = i
-                i += 1
-            segments.append([start,end])
-        i+=1
-    idx = np.argsort(y_anomaly_scores)[::-1].astype(int)
-    y_anomaly_scores_sorted = np.array(y_anomaly_scores)[idx]
-    
-    segment_mins = []
-    for start,end in segments:
-        anoms_scores = y_anomaly_scores[start:end+1]
-        segment_mins.append([np.max(anoms_scores),[start,end]])
-
-    for i_t in range(len(y_anomaly_scores_sorted)):
-        fp,tp,fn = 0,0,0
-        
-            
-        aux_y_pred = (y_anomaly_scores >= y_anomaly_scores_sorted[i_t]).astype(int) 
-        for score,seg in segment_mins:
-            start,end = seg
-            if score >= y_anomaly_scores_sorted[i_t]:
-                #TP
-                tp+= 1
-                if aux_y_pred[start]== 1:
-                    # Extender hacia la izquierda
-                    i = start - 1
-                    while i >= 0 and aux_y_pred[i] == 1:
-                        aux_y_pred[i] = 0
-                        i -= 1
-                
-                if aux_y_pred[end] == 1:
-                    # Extender hacia la derecha
-                    i = end + 1
-                    while i < len(aux_y_pred) and aux_y_pred[i] == 1:
-                        aux_y_pred[i] = 0
-                        i += 1
-                aux_y_pred[start:end+1] = 0
-
-            else:
-                #FN
-                fn+= 1
-        
-        if np.sum(aux_y_pred)>0:
-            fpsegments = []
-            i=0
-            while i < len(aux_y_pred):
-                if aux_y_pred[i] == 1:
-                    start = i
-                    end = i
-                    while i < len(aux_y_pred) and aux_y_pred[i] == 1:
-                        end = i
-                        i += 1
-                    fpsegments.append([start,end])
-                i+=1
-            fp = len(fpsegments)
-        else:
-            fp = 0
-            
-            
-        tps.append(tp)
-        fns.append(fn)
-        fps.append(fp)
-    for tp,fp,fn in zip(tps,fps,fns):
-        if tp>0:
-            precisions.append(tp/(tp+fp))
-            recalls.append(tp/(tp+fn))
-        else:
-            precisions.append(0)
-            recalls.append(0)    
-      
-    
-
-    auc_value = auc(recalls, precisions)
-
-    return auc_value
 
 
 def vus_roc(y_true : np.array ,y_anomaly_scores:  np.array, window=4):
@@ -284,6 +275,8 @@ def vus_roc(y_true : np.array ,y_anomaly_scores:  np.array, window=4):
     the ROC-AUC over different values of the tolerance parameter, from 0 to `window`, thus producing
     a volume under the ROC surface.
 
+    Implementation of https://link.springer.com/article/10.1007/s10618-023-00988-8
+
     For more information, see the original paper:
     https://dl.acm.org/doi/10.14778/3551793.3551830
 
@@ -300,6 +293,8 @@ def vus_roc(y_true : np.array ,y_anomaly_scores:  np.array, window=4):
 
     
     """
+    validate_non_binary_inputs(y_true, y_anomaly_scores)
+
     m = VUS_ROC(y_true,y_anomaly_scores,max_window=window)
 
     return m.get_score()
@@ -314,6 +309,8 @@ def vus_pr(y_true : np.array ,y_anomaly_scores:  np.array,  window=4):
     anomalies that are temporally close to the true events. The final metric integrates the PR-AUC
     over several levels of temporal tolerance (from 0 to `window`), yielding a volume under the PR surface.
 
+    Implementation of https://link.springer.com/article/10.1007/s10618-023-00988-8
+    
     For more information, see the original paper:
     https://dl.acm.org/doi/10.14778/3551793.3551830
 
@@ -330,6 +327,8 @@ def vus_pr(y_true : np.array ,y_anomaly_scores:  np.array,  window=4):
 
     
     """
+    validate_non_binary_inputs(y_true, y_anomaly_scores)
+
     m = VUS_PR(y_true,y_anomaly_scores,max_window=window)
 
     return m.get_score()
@@ -350,6 +349,8 @@ def real_pate(y_true: np.array, y_anomaly_scores: np.array, early: int, delay: i
     The final PATE score aggregates these weighted contributions across all time steps, yielding 
     a smooth performance measure that is sensitive to both the timing and confidence of the predictions.
 
+    Implementation of https://arxiv.org/abs/2405.12096
+    
     For more information, see the original paper:
     https://arxiv.org/abs/2405.12096
 
@@ -366,4 +367,6 @@ def real_pate(y_true: np.array, y_anomaly_scores: np.array, early: int, delay: i
     Returns:
         float: The real-valued PATE score.
     """
+    validate_non_binary_inputs(y_true, y_anomaly_scores)
+
     return PATE(y_true, y_anomaly_scores, early, delay, binary_scores=False)

tsadmetrics/scripts/compute_metrics.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+import argparse
+from tsadmetrics.utils import compute_metrics_from_file
+
+
+def main():
+    
+    parser = argparse.ArgumentParser(
+        description='Compute metrics from anomaly detection results and configuration files.'
+    )
+    
+    
+    parser.add_argument(
+        '--res_file', 
+        type=str, 
+        required=True,
+        help='Path to the results CSV file (e.g., results.csv)'
+    )
+    parser.add_argument(
+        '--conf_file', 
+        type=str, 
+        required=True,
+        help='Path to the configuration JSON file (e.g., conf.json)'
+    )
+    parser.add_argument(
+        '--output_dir', 
+        type=str, 
+        required=True,
+        help='Directory where output files will be saved (e.g., ./output_dir)'
+    )
+    
+    args = parser.parse_args()
+    
+    
+    compute_metrics_from_file(
+        results_file=args.res_file,
+        conf_file=args.conf_file,
+        output_dir=args.output_dir
+    )
+
+if __name__ == '__main__':
+    main()