tsadmetrics 0.1.17__py3-none-any.whl → 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {docs_manual → docs/api_doc}/conf.py +3 -26
- docs/{conf.py → full_doc/conf.py} +1 -1
- {docs_api → docs/manual_doc}/conf.py +3 -26
- examples/example_direct_data.py +28 -0
- examples/example_direct_single_data.py +25 -0
- examples/example_file_reference.py +24 -0
- examples/example_global_config_file.py +13 -0
- examples/example_metric_config_file.py +19 -0
- examples/example_simple_metric.py +8 -0
- examples/specific_examples/AbsoluteDetectionDistance_example.py +24 -0
- examples/specific_examples/AffiliationbasedFScore_example.py +24 -0
- examples/specific_examples/AverageDetectionCount_example.py +24 -0
- examples/specific_examples/CompositeFScore_example.py +24 -0
- examples/specific_examples/DelayThresholdedPointadjustedFScore_example.py +24 -0
- examples/specific_examples/DetectionAccuracyInRange_example.py +24 -0
- examples/specific_examples/EnhancedTimeseriesAwareFScore_example.py +24 -0
- examples/specific_examples/LatencySparsityawareFScore_example.py +24 -0
- examples/specific_examples/MeanTimeToDetect_example.py +24 -0
- examples/specific_examples/NabScore_example.py +24 -0
- examples/specific_examples/PateFScore_example.py +24 -0
- examples/specific_examples/Pate_example.py +24 -0
- examples/specific_examples/PointadjustedAtKFScore_example.py +24 -0
- examples/specific_examples/PointadjustedAucPr_example.py +24 -0
- examples/specific_examples/PointadjustedAucRoc_example.py +24 -0
- examples/specific_examples/PointadjustedFScore_example.py +24 -0
- examples/specific_examples/RangebasedFScore_example.py +24 -0
- examples/specific_examples/SegmentwiseFScore_example.py +24 -0
- examples/specific_examples/TemporalDistance_example.py +24 -0
- examples/specific_examples/TimeTolerantFScore_example.py +24 -0
- examples/specific_examples/TimeseriesAwareFScore_example.py +24 -0
- examples/specific_examples/TotalDetectedInRange_example.py +24 -0
- examples/specific_examples/VusPr_example.py +24 -0
- examples/specific_examples/VusRoc_example.py +24 -0
- examples/specific_examples/WeightedDetectionDifference_example.py +24 -0
- tests/test_dpm.py +212 -0
- tests/test_ptdm.py +366 -0
- tests/test_registry.py +58 -0
- tests/test_runner.py +185 -0
- tests/test_spm.py +213 -0
- tests/test_tmem.py +198 -0
- tests/test_tpdm.py +369 -0
- tests/test_tstm.py +338 -0
- tsadmetrics/__init__.py +0 -21
- tsadmetrics/base/Metric.py +188 -0
- tsadmetrics/evaluation/Report.py +25 -0
- tsadmetrics/evaluation/Runner.py +253 -0
- tsadmetrics/metrics/Registry.py +141 -0
- tsadmetrics/metrics/__init__.py +2 -0
- tsadmetrics/metrics/spm/PointwiseAucPr.py +62 -0
- tsadmetrics/metrics/spm/PointwiseAucRoc.py +63 -0
- tsadmetrics/metrics/spm/PointwiseFScore.py +86 -0
- tsadmetrics/metrics/spm/PrecisionAtK.py +81 -0
- tsadmetrics/metrics/spm/__init__.py +9 -0
- tsadmetrics/metrics/tem/dpm/DelayThresholdedPointadjustedFScore.py +83 -0
- tsadmetrics/metrics/tem/dpm/LatencySparsityawareFScore.py +76 -0
- tsadmetrics/metrics/tem/dpm/MeanTimeToDetect.py +47 -0
- tsadmetrics/metrics/tem/dpm/NabScore.py +60 -0
- tsadmetrics/metrics/tem/dpm/__init__.py +11 -0
- tsadmetrics/metrics/tem/ptdm/AverageDetectionCount.py +53 -0
- tsadmetrics/metrics/tem/ptdm/DetectionAccuracyInRange.py +66 -0
- tsadmetrics/metrics/tem/ptdm/PointadjustedAtKFScore.py +80 -0
- tsadmetrics/metrics/tem/ptdm/TimeseriesAwareFScore.py +248 -0
- tsadmetrics/metrics/tem/ptdm/TotalDetectedInRange.py +65 -0
- tsadmetrics/metrics/tem/ptdm/WeightedDetectionDifference.py +97 -0
- tsadmetrics/metrics/tem/ptdm/__init__.py +12 -0
- tsadmetrics/metrics/tem/tmem/AbsoluteDetectionDistance.py +48 -0
- tsadmetrics/metrics/tem/tmem/EnhancedTimeseriesAwareFScore.py +252 -0
- tsadmetrics/metrics/tem/tmem/TemporalDistance.py +68 -0
- tsadmetrics/metrics/tem/tmem/__init__.py +9 -0
- tsadmetrics/metrics/tem/tpdm/CompositeFScore.py +104 -0
- tsadmetrics/metrics/tem/tpdm/PointadjustedAucPr.py +123 -0
- tsadmetrics/metrics/tem/tpdm/PointadjustedAucRoc.py +119 -0
- tsadmetrics/metrics/tem/tpdm/PointadjustedFScore.py +96 -0
- tsadmetrics/metrics/tem/tpdm/RangebasedFScore.py +236 -0
- tsadmetrics/metrics/tem/tpdm/SegmentwiseFScore.py +73 -0
- tsadmetrics/metrics/tem/tpdm/__init__.py +12 -0
- tsadmetrics/metrics/tem/tstm/AffiliationbasedFScore.py +68 -0
- tsadmetrics/metrics/tem/tstm/Pate.py +62 -0
- tsadmetrics/metrics/tem/tstm/PateFScore.py +61 -0
- tsadmetrics/metrics/tem/tstm/TimeTolerantFScore.py +85 -0
- tsadmetrics/metrics/tem/tstm/VusPr.py +51 -0
- tsadmetrics/metrics/tem/tstm/VusRoc.py +55 -0
- tsadmetrics/metrics/tem/tstm/__init__.py +15 -0
- tsadmetrics/{_tsadeval/affiliation/_integral_interval.py → utils/functions_affiliation.py} +377 -9
- tsadmetrics/utils/functions_auc.py +393 -0
- tsadmetrics/utils/functions_conversion.py +63 -0
- tsadmetrics/utils/functions_counting_metrics.py +26 -0
- tsadmetrics/{_tsadeval/latency_sparsity_aware.py → utils/functions_latency_sparsity_aware.py} +1 -1
- tsadmetrics/{_tsadeval/nabscore.py → utils/functions_nabscore.py} +15 -1
- tsadmetrics-1.0.0.dist-info/METADATA +69 -0
- tsadmetrics-1.0.0.dist-info/RECORD +99 -0
- tsadmetrics-1.0.0.dist-info/top_level.txt +4 -0
- entorno/bin/activate_this.py +0 -32
- entorno/bin/rst2html.py +0 -23
- entorno/bin/rst2html4.py +0 -26
- entorno/bin/rst2html5.py +0 -33
- entorno/bin/rst2latex.py +0 -26
- entorno/bin/rst2man.py +0 -27
- entorno/bin/rst2odt.py +0 -28
- entorno/bin/rst2odt_prepstyles.py +0 -20
- entorno/bin/rst2pseudoxml.py +0 -23
- entorno/bin/rst2s5.py +0 -24
- entorno/bin/rst2xetex.py +0 -27
- entorno/bin/rst2xml.py +0 -23
- entorno/bin/rstpep2html.py +0 -25
- tests/test_binary.py +0 -946
- tests/test_non_binary.py +0 -450
- tests/test_utils.py +0 -49
- tsadmetrics/_tsadeval/affiliation/_affiliation_zone.py +0 -86
- tsadmetrics/_tsadeval/affiliation/_single_ground_truth_event.py +0 -68
- tsadmetrics/_tsadeval/affiliation/generics.py +0 -135
- tsadmetrics/_tsadeval/affiliation/metrics.py +0 -114
- tsadmetrics/_tsadeval/auc_roc_pr_plot.py +0 -295
- tsadmetrics/_tsadeval/discontinuity_graph.py +0 -109
- tsadmetrics/_tsadeval/eTaPR_pkg/DataManage/File_IO.py +0 -175
- tsadmetrics/_tsadeval/eTaPR_pkg/DataManage/Range.py +0 -50
- tsadmetrics/_tsadeval/eTaPR_pkg/DataManage/Time_Plot.py +0 -184
- tsadmetrics/_tsadeval/eTaPR_pkg/__init__.py +0 -0
- tsadmetrics/_tsadeval/eTaPR_pkg/etapr.py +0 -386
- tsadmetrics/_tsadeval/eTaPR_pkg/tapr.py +0 -362
- tsadmetrics/_tsadeval/metrics.py +0 -698
- tsadmetrics/_tsadeval/prts/__init__.py +0 -0
- tsadmetrics/_tsadeval/prts/base/__init__.py +0 -0
- tsadmetrics/_tsadeval/prts/base/time_series_metrics.py +0 -165
- tsadmetrics/_tsadeval/prts/basic_metrics_ts.py +0 -121
- tsadmetrics/_tsadeval/prts/time_series_metrics/__init__.py +0 -0
- tsadmetrics/_tsadeval/prts/time_series_metrics/fscore.py +0 -61
- tsadmetrics/_tsadeval/prts/time_series_metrics/precision.py +0 -86
- tsadmetrics/_tsadeval/prts/time_series_metrics/precision_recall.py +0 -21
- tsadmetrics/_tsadeval/prts/time_series_metrics/recall.py +0 -85
- tsadmetrics/_tsadeval/tests.py +0 -376
- tsadmetrics/_tsadeval/threshold_plt.py +0 -30
- tsadmetrics/_tsadeval/time_tolerant.py +0 -33
- tsadmetrics/binary_metrics.py +0 -1652
- tsadmetrics/metric_utils.py +0 -98
- tsadmetrics/non_binary_metrics.py +0 -372
- tsadmetrics/scripts/__init__.py +0 -0
- tsadmetrics/scripts/compute_metrics.py +0 -42
- tsadmetrics/utils.py +0 -124
- tsadmetrics/validation.py +0 -35
- tsadmetrics-0.1.17.dist-info/METADATA +0 -54
- tsadmetrics-0.1.17.dist-info/RECORD +0 -66
- tsadmetrics-0.1.17.dist-info/entry_points.txt +0 -2
- tsadmetrics-0.1.17.dist-info/top_level.txt +0 -6
- /tsadmetrics/{_tsadeval → base}/__init__.py +0 -0
- /tsadmetrics/{_tsadeval/affiliation → evaluation}/__init__.py +0 -0
- /tsadmetrics/{_tsadeval/eTaPR_pkg/DataManage → metrics/tem}/__init__.py +0 -0
- /tsadmetrics/{_tsadeval/vus_utils.py → utils/functions_vus.py} +0 -0
- {tsadmetrics-0.1.17.dist-info → tsadmetrics-1.0.0.dist-info}/WHEEL +0 -0
tsadmetrics/metric_utils.py
DELETED
@@ -1,98 +0,0 @@
|
|
1
|
-
import numpy as np
|
2
|
-
from ._tsadeval.metrics import pointwise_to_full_series, segmentwise_to_full_series
|
3
|
-
|
4
|
-
def is_full_series(length: int, anomalies: np.array):
|
5
|
-
# [1 0 1 1 0]
|
6
|
-
return len(anomalies.shape) == 1 and len(anomalies) == length
|
7
|
-
|
8
|
-
def is_pointwise(length: int, anomalies: np.array):
|
9
|
-
# [0 2 3]
|
10
|
-
return len(anomalies.shape) == 1 and len(anomalies) < length
|
11
|
-
|
12
|
-
def is_segmentwise(length: int, anomalies: np.array):
|
13
|
-
# [[0 0] [2 3]]
|
14
|
-
return len(anomalies.shape) == 2
|
15
|
-
|
16
|
-
|
17
|
-
def transform_to_full_series(length: int, anomalies: np.array):
|
18
|
-
if is_full_series(length, anomalies):
|
19
|
-
return anomalies
|
20
|
-
elif is_pointwise(anomalies):
|
21
|
-
return pointwise_to_full_series(anomalies, length)
|
22
|
-
elif is_segmentwise(length, anomalies):
|
23
|
-
return segmentwise_to_full_series(anomalies, length)
|
24
|
-
else:
|
25
|
-
raise ValueError(f"Illegal shape of anomalies:\n{anomalies}")
|
26
|
-
|
27
|
-
def counting_method(y_true: np.array, y_pred: np.array, k: int):
|
28
|
-
em,da,ma,fa = 0,0,0,0
|
29
|
-
for i_gt in range(len(y_true)):
|
30
|
-
i_pa = i_gt
|
31
|
-
gt = y_true[i_gt]
|
32
|
-
pa = y_pred[i_pa]
|
33
|
-
if gt==1 and pa==1:
|
34
|
-
em+=1
|
35
|
-
elif gt==0 and pa==1:
|
36
|
-
fa+=1
|
37
|
-
elif gt==1 and pa==0:
|
38
|
-
anom_range = y_pred[i_gt-k:i_pa+k+1]
|
39
|
-
detected = False
|
40
|
-
for r in anom_range:
|
41
|
-
if r==1:
|
42
|
-
em+=1
|
43
|
-
detected=True
|
44
|
-
break
|
45
|
-
if not detected:
|
46
|
-
ma+=1
|
47
|
-
elif gt==0 and pa==0:
|
48
|
-
pass
|
49
|
-
# b = DelayThresholdedPointAdjust(len(y_true),y_true,y_pred,k=k)
|
50
|
-
# da = b.tp-em
|
51
|
-
# ma = b.fn
|
52
|
-
|
53
|
-
return em,da,ma,fa
|
54
|
-
|
55
|
-
|
56
|
-
#Range Based utils
|
57
|
-
|
58
|
-
def cardinality(n_intersections,mode):
|
59
|
-
if mode == 'one':
|
60
|
-
return 1
|
61
|
-
elif mode == 'reciprocal':
|
62
|
-
if n_intersections==0:
|
63
|
-
return 1
|
64
|
-
else:
|
65
|
-
return float(1/n_intersections)
|
66
|
-
else:
|
67
|
-
raise Exception("Error, wrong cardinality mode.")
|
68
|
-
|
69
|
-
|
70
|
-
def size(anomaly_range, overlap_set, position, bias):
|
71
|
-
if overlap_set == None:
|
72
|
-
return 0
|
73
|
-
|
74
|
-
my_value = 0
|
75
|
-
max_value = 0
|
76
|
-
anomaly_length = anomaly_range[1] - anomaly_range[0] + 1
|
77
|
-
for i in range(1,anomaly_length+1):
|
78
|
-
bias_value = position(i, anomaly_length,bias)
|
79
|
-
max_value += bias_value
|
80
|
-
if anomaly_range[0]+i-1 >= overlap_set[0] and anomaly_range[0]+i-1 <= overlap_set[1]:
|
81
|
-
my_value += bias_value
|
82
|
-
return my_value / max_value
|
83
|
-
|
84
|
-
def position(i, anomaly_length,bias):
|
85
|
-
if bias == 'flat':
|
86
|
-
return 1
|
87
|
-
elif bias == 'front-end':
|
88
|
-
return anomaly_length - i + 1
|
89
|
-
elif bias == 'back-end':
|
90
|
-
return i
|
91
|
-
elif bias == 'middle':
|
92
|
-
if i <= anomaly_length / 2:
|
93
|
-
return i
|
94
|
-
else:
|
95
|
-
return anomaly_length - i + 1
|
96
|
-
else:
|
97
|
-
raise Exception("Error, wrong bias value.")
|
98
|
-
|
@@ -1,372 +0,0 @@
|
|
1
|
-
import numpy as np
|
2
|
-
from ._tsadeval.metrics import *
|
3
|
-
from .validation import validate_non_binary_inputs
|
4
|
-
from sklearn.metrics import auc
|
5
|
-
from pate.PATE_metric import PATE
|
6
|
-
def precision_at_k(y_true : np.array, y_anomaly_scores: np.array):
|
7
|
-
"""
|
8
|
-
Calculate the precision at k score for anomaly detection in time series.
|
9
|
-
|
10
|
-
This metric evaluates how many of the top-k points (with highest anomaly scores)
|
11
|
-
actually correspond to true anomalies. It is particularly useful when we are
|
12
|
-
interested in identifying the most anomalous points, without needing to set a
|
13
|
-
fixed threshold.
|
14
|
-
|
15
|
-
The value of k is automatically set to the number of true anomalies present in
|
16
|
-
y_true. That is, k = sum(y_true).
|
17
|
-
|
18
|
-
Implementation of https://link.springer.com/article/10.1007/s10618-023-00988-8
|
19
|
-
|
20
|
-
Parameters:
|
21
|
-
y_true (np.array):
|
22
|
-
The ground truth binary labels for the time series data.
|
23
|
-
y_anomaly_scores (np.array):
|
24
|
-
The predicted anomaly scores for the time series data.
|
25
|
-
|
26
|
-
Returns:
|
27
|
-
float: The precision at k score.
|
28
|
-
"""
|
29
|
-
validate_non_binary_inputs(y_true, y_anomaly_scores)
|
30
|
-
|
31
|
-
m = PatK_pw(y_true,y_anomaly_scores)
|
32
|
-
|
33
|
-
return m.get_score()
|
34
|
-
|
35
|
-
def auc_roc_pw(y_true : np.array, y_anomaly_scores: np.array):
|
36
|
-
"""
|
37
|
-
Calculate the AUC-ROC score for anomaly detection in time series.
|
38
|
-
|
39
|
-
This is the standard Area Under the Receiver Operating Characteristic Curve (AUC-ROC),
|
40
|
-
computed in a point-wise manner. That is, each point in the time series is treated
|
41
|
-
independently when calculating true positives, false positives, and false negatives.
|
42
|
-
|
43
|
-
Implementation of https://link.springer.com/article/10.1007/s10618-023-00988-8
|
44
|
-
|
45
|
-
Parameters:
|
46
|
-
y_true (np.array):
|
47
|
-
Ground-truth binary labels for the time series (0 = normal, 1 = anomaly).
|
48
|
-
y_anomaly_scores (np.array):
|
49
|
-
Continuous anomaly scores assigned to each point in the series.
|
50
|
-
|
51
|
-
Returns:
|
52
|
-
float: AUC-ROC score.
|
53
|
-
"""
|
54
|
-
validate_non_binary_inputs(y_true, y_anomaly_scores)
|
55
|
-
|
56
|
-
m = AUC_ROC(y_true,y_anomaly_scores)
|
57
|
-
|
58
|
-
return m.get_score()
|
59
|
-
|
60
|
-
|
61
|
-
def auc_pr_pw(y_true : np.array ,y_anomaly_scores: np.array):
|
62
|
-
"""
|
63
|
-
Calculate the AUC-PR score for anomaly detection in time series.
|
64
|
-
|
65
|
-
This is the standard Area Under the Precision-Recall Curve (AUC-PR),
|
66
|
-
computed in a point-wise manner. That is, each point in the time series is treated
|
67
|
-
independently when calculating precision and recall.
|
68
|
-
|
69
|
-
Implementation of https://link.springer.com/article/10.1007/s10618-023-00988-8
|
70
|
-
|
71
|
-
Parameters:
|
72
|
-
y_true (np.array):
|
73
|
-
Ground-truth binary labels for the time series (0 = normal, 1 = anomaly).
|
74
|
-
y_anomaly_scores (np.array):
|
75
|
-
Continuous anomaly scores assigned to each point in the series.
|
76
|
-
|
77
|
-
Returns:
|
78
|
-
float: AUC-PR score.
|
79
|
-
"""
|
80
|
-
validate_non_binary_inputs(y_true, y_anomaly_scores)
|
81
|
-
|
82
|
-
m = AUC_PR_pw(y_true,y_anomaly_scores)
|
83
|
-
|
84
|
-
return m.get_score()
|
85
|
-
|
86
|
-
|
87
|
-
def auc_roc_pa(y_true: np.array, y_anomaly_scores: np.array):
|
88
|
-
"""
|
89
|
-
Calculate the AUC-ROC score using point-adjusted evaluation for anomaly detection in time series.
|
90
|
-
|
91
|
-
This is the standard Area Under the Receiver Operating Characteristic Curve (AUC-ROC), but instead
|
92
|
-
of computing true positive rate (TPR) and false positive rate (FPR) point-wise, it uses a point-adjusted
|
93
|
-
approach. Specifically, for each ground-truth anomalous segment, if at least one point within that
|
94
|
-
segment is predicted as anomalous, the entire segment is considered correctly detected. The adjusted
|
95
|
-
predictions are then compared to the ground-truth labels to compute true positives, false positives,
|
96
|
-
and false negatives, which are used to construct the ROC curve.
|
97
|
-
|
98
|
-
Implementation of https://link.springer.com/article/10.1007/s10618-023-00988-8
|
99
|
-
|
100
|
-
Parameters:
|
101
|
-
y_true (np.array):
|
102
|
-
Ground-truth binary labels for the time series (0 = normal, 1 = anomaly).
|
103
|
-
y_anomaly_scores (np.array):
|
104
|
-
Continuous anomaly scores assigned to each point in the series.
|
105
|
-
|
106
|
-
Returns:
|
107
|
-
float: AUC-ROC score (with point-adjusted evaluation).
|
108
|
-
"""
|
109
|
-
validate_non_binary_inputs(y_true, y_anomaly_scores)
|
110
|
-
|
111
|
-
tprs = [0]
|
112
|
-
fprs = [0]
|
113
|
-
tps, fps, fns = [], [], []
|
114
|
-
|
115
|
-
p_adj = PointAdjust(len(y_true), y_true, (np.array(y_anomaly_scores) >= 0.5).astype(int))
|
116
|
-
segments = p_adj.get_gt_anomalies_segmentwise()
|
117
|
-
idx = np.argsort(y_anomaly_scores)[::-1].astype(int)
|
118
|
-
y_true_sorted = np.array(y_true)[idx]
|
119
|
-
y_anomaly_scores_sorted = np.array(y_anomaly_scores)[idx]
|
120
|
-
|
121
|
-
segment_mins = []
|
122
|
-
for start, end in segments:
|
123
|
-
anoms_scores = y_anomaly_scores[start:end+1]
|
124
|
-
segment_mins.append([np.max(anoms_scores), end-start+1])
|
125
|
-
|
126
|
-
for i_t in range(len(y_anomaly_scores_sorted)):
|
127
|
-
fp, tp, fn = 0, 0, 0
|
128
|
-
if i_t > 0 and y_anomaly_scores_sorted[i_t] == y_anomaly_scores_sorted[i_t-1]:
|
129
|
-
tp = tps[-1]
|
130
|
-
fp = fps[-1]
|
131
|
-
fn = fns[-1]
|
132
|
-
else:
|
133
|
-
if y_true_sorted[i_t] == 0:
|
134
|
-
# FP
|
135
|
-
if len(fps) == 0:
|
136
|
-
aux_y_pred = (y_anomaly_scores >= y_anomaly_scores_sorted[i_t]).astype(int)
|
137
|
-
for i in range(len(aux_y_pred)):
|
138
|
-
if aux_y_pred[i] == 1 and y_true[i] == 0:
|
139
|
-
fp += 1
|
140
|
-
else:
|
141
|
-
fp = fps[i_t-1] + 1
|
142
|
-
else:
|
143
|
-
if len(fps) == 0:
|
144
|
-
aux_y_pred = (y_anomaly_scores >= y_anomaly_scores_sorted[i_t]).astype(int)
|
145
|
-
for i in range(len(aux_y_pred)):
|
146
|
-
if aux_y_pred[i] == 1 and y_true[i] == 0:
|
147
|
-
fp += 1
|
148
|
-
else:
|
149
|
-
fp = fps[i_t-1]
|
150
|
-
for score, length in segment_mins:
|
151
|
-
if score >= y_anomaly_scores_sorted[i_t]:
|
152
|
-
# TP
|
153
|
-
tp += length
|
154
|
-
else:
|
155
|
-
# FN
|
156
|
-
fn += length
|
157
|
-
tps.append(tp)
|
158
|
-
fns.append(fn)
|
159
|
-
fps.append(fp)
|
160
|
-
for tp, fp, fn in zip(tps, fps, fns):
|
161
|
-
if tp + fn > 0:
|
162
|
-
tprs.append(tp / (tp + fn))
|
163
|
-
else:
|
164
|
-
tprs.append(0)
|
165
|
-
if fp + (len(y_true) - np.sum(y_true)) > 0:
|
166
|
-
fprs.append(fp / (fp + (len(y_true) - np.sum(y_true))))
|
167
|
-
else:
|
168
|
-
fprs.append(0)
|
169
|
-
|
170
|
-
auc_value = auc(fprs, tprs)
|
171
|
-
return auc_value
|
172
|
-
|
173
|
-
def auc_pr_pa(y_true: np.array, y_anomaly_scores: np.array):
|
174
|
-
"""
|
175
|
-
Calculate the AUC-PR score using point-adjusted evaluation for anomaly detection in time series.
|
176
|
-
|
177
|
-
This is the standard Area Under the Precision-Recall Curve (AUC-PR), but instead of computing
|
178
|
-
precision and recall point-wise, it uses a point-adjusted approach. Specifically, for each
|
179
|
-
ground-truth anomalous segment, if at least one point within that segment is predicted as anomalous,
|
180
|
-
the entire segment is considered correctly detected. The adjusted predictions are then compared
|
181
|
-
to the ground-truth labels to compute true positives, false positives, and false negatives,
|
182
|
-
which are used to construct the PR curve.
|
183
|
-
|
184
|
-
Implementation of https://link.springer.com/article/10.1007/s10618-023-00988-8
|
185
|
-
|
186
|
-
Parameters:
|
187
|
-
y_true (np.array):
|
188
|
-
Ground-truth binary labels for the time series (0 = normal, 1 = anomaly).
|
189
|
-
y_anomaly_scores (np.array):
|
190
|
-
Continuous anomaly scores assigned to each point in the series.
|
191
|
-
|
192
|
-
Returns:
|
193
|
-
float: AUC-PR score (with point-adjusted evaluation).
|
194
|
-
"""
|
195
|
-
validate_non_binary_inputs(y_true, y_anomaly_scores)
|
196
|
-
|
197
|
-
precisions = [1]
|
198
|
-
recalls = [0]
|
199
|
-
tps,fps,fns = [],[],[]
|
200
|
-
|
201
|
-
p_adj = PointAdjust(len(y_true),y_true,(np.array(y_anomaly_scores) >= 0.5).astype(int))
|
202
|
-
segments= p_adj.get_gt_anomalies_segmentwise()
|
203
|
-
idx = np.argsort(y_anomaly_scores)[::-1].astype(int)
|
204
|
-
y_true_sorted = np.array(y_true)[idx]
|
205
|
-
y_anomaly_scores_sorted = np.array(y_anomaly_scores)[idx]
|
206
|
-
|
207
|
-
segment_mins = []
|
208
|
-
for start,end in segments:
|
209
|
-
anoms_scores = y_anomaly_scores[start:end+1]
|
210
|
-
segment_mins.append([np.max(anoms_scores),end-start+1])
|
211
|
-
|
212
|
-
for i_t in range(len(y_anomaly_scores_sorted)):
|
213
|
-
fp,tp,fn = 0,0,0
|
214
|
-
if i_t > 0 and y_anomaly_scores_sorted[i_t] == y_anomaly_scores_sorted[i_t-1] :
|
215
|
-
tp = tps[-1]
|
216
|
-
fp = fps[-1]
|
217
|
-
fn = fns[-1]
|
218
|
-
else:
|
219
|
-
if y_true_sorted[i_t] == 0:
|
220
|
-
#FP
|
221
|
-
if len(fps)==0:
|
222
|
-
aux_y_pred = (y_anomaly_scores >= y_anomaly_scores_sorted[i_t]).astype(int)
|
223
|
-
for i in range(len(aux_y_pred)):
|
224
|
-
if aux_y_pred[i] == 1 and y_true[i] == 0:
|
225
|
-
fp+=1
|
226
|
-
|
227
|
-
|
228
|
-
else:
|
229
|
-
fp=fps[i_t-1]+1
|
230
|
-
else:
|
231
|
-
if len(fps)==0:
|
232
|
-
aux_y_pred = (y_anomaly_scores >= y_anomaly_scores_sorted[i_t]).astype(int)
|
233
|
-
for i in range(len(aux_y_pred)):
|
234
|
-
if aux_y_pred[i] == 1 and y_true[i] == 0:
|
235
|
-
fp+=1
|
236
|
-
else:
|
237
|
-
fp=fps[i_t-1]
|
238
|
-
for score, length in segment_mins:
|
239
|
-
if score >= y_anomaly_scores_sorted[i_t]:
|
240
|
-
#TP
|
241
|
-
tp+= length
|
242
|
-
else:
|
243
|
-
#FN
|
244
|
-
fn+= length
|
245
|
-
tps.append(tp)
|
246
|
-
fns.append(fn)
|
247
|
-
fps.append(fp)
|
248
|
-
for tp,fp,fn in zip(tps,fps,fns):
|
249
|
-
if tp>0:
|
250
|
-
precisions.append(tp/(tp+fp))
|
251
|
-
recalls.append(tp/(tp+fn))
|
252
|
-
else:
|
253
|
-
precisions.append(0)
|
254
|
-
recalls.append(0)
|
255
|
-
|
256
|
-
|
257
|
-
recalls.append(1)
|
258
|
-
precisions.append(0)
|
259
|
-
|
260
|
-
auc_value = auc(recalls, precisions)
|
261
|
-
return auc_value
|
262
|
-
|
263
|
-
|
264
|
-
|
265
|
-
|
266
|
-
|
267
|
-
|
268
|
-
def vus_roc(y_true : np.array ,y_anomaly_scores: np.array, window=4):
|
269
|
-
"""
|
270
|
-
Calculate the VUS-ROC (Volume Under the ROC Surface) score for anomaly detection in time series.
|
271
|
-
|
272
|
-
This metric extends the classical AUC-ROC by introducing a temporal tolerance parameter `l`, which
|
273
|
-
smooths the binary ground-truth labels. The idea is to allow a flexible evaluation that tolerates
|
274
|
-
small misalignments in the detection of anomalies. The final score is computed by integrating
|
275
|
-
the ROC-AUC over different values of the tolerance parameter, from 0 to `window`, thus producing
|
276
|
-
a volume under the ROC surface.
|
277
|
-
|
278
|
-
Implementation of https://link.springer.com/article/10.1007/s10618-023-00988-8
|
279
|
-
|
280
|
-
For more information, see the original paper:
|
281
|
-
https://dl.acm.org/doi/10.14778/3551793.3551830
|
282
|
-
|
283
|
-
Parameters:
|
284
|
-
y_true (np.array):
|
285
|
-
Ground-truth binary labels (0 = normal, 1 = anomaly).
|
286
|
-
y_anomaly_scores (np.array):
|
287
|
-
Anomaly scores for each time point.
|
288
|
-
window (int):
|
289
|
-
Maximum temporal tolerance `l` used to smooth the evaluation.
|
290
|
-
|
291
|
-
Returns:
|
292
|
-
float: VUS-ROC score.
|
293
|
-
|
294
|
-
|
295
|
-
"""
|
296
|
-
validate_non_binary_inputs(y_true, y_anomaly_scores)
|
297
|
-
|
298
|
-
m = VUS_ROC(y_true,y_anomaly_scores,max_window=window)
|
299
|
-
|
300
|
-
return m.get_score()
|
301
|
-
|
302
|
-
|
303
|
-
def vus_pr(y_true : np.array ,y_anomaly_scores: np.array, window=4):
|
304
|
-
"""
|
305
|
-
Calculate the VUS-PR (Volume Under the PR Surface) score for anomaly detection in time series.
|
306
|
-
|
307
|
-
This metric is an extension of the classical AUC-PR, incorporating a temporal tolerance parameter `l`
|
308
|
-
that smooths the binary ground-truth labels. It allows for some flexibility in the detection of
|
309
|
-
anomalies that are temporally close to the true events. The final metric integrates the PR-AUC
|
310
|
-
over several levels of temporal tolerance (from 0 to `window`), yielding a volume under the PR surface.
|
311
|
-
|
312
|
-
Implementation of https://link.springer.com/article/10.1007/s10618-023-00988-8
|
313
|
-
|
314
|
-
For more information, see the original paper:
|
315
|
-
https://dl.acm.org/doi/10.14778/3551793.3551830
|
316
|
-
|
317
|
-
Parameters:
|
318
|
-
y_true (np.array):
|
319
|
-
Ground-truth binary labels (0 = normal, 1 = anomaly).
|
320
|
-
y_anomaly_scores (np.array):
|
321
|
-
Anomaly scores for each time point.
|
322
|
-
window (int):
|
323
|
-
Maximum temporal tolerance `l` used to smooth the evaluation.
|
324
|
-
|
325
|
-
Returns:
|
326
|
-
float: VUS-PR score.
|
327
|
-
|
328
|
-
|
329
|
-
"""
|
330
|
-
validate_non_binary_inputs(y_true, y_anomaly_scores)
|
331
|
-
|
332
|
-
m = VUS_PR(y_true,y_anomaly_scores,max_window=window)
|
333
|
-
|
334
|
-
return m.get_score()
|
335
|
-
|
336
|
-
|
337
|
-
def real_pate(y_true: np.array, y_anomaly_scores: np.array, early: int, delay: int):
|
338
|
-
"""
|
339
|
-
Calculate PATE score for anomaly detection in time series using real-valued anomaly scores.
|
340
|
-
|
341
|
-
This version of PATE evaluates real-valued anomaly scores by assigning weights to predictions
|
342
|
-
based on their temporal proximity to the true anomaly intervals. It defines an early buffer of
|
343
|
-
length `early` before each anomaly and a delay buffer of length `delay` after it. Detections with
|
344
|
-
high scores within the anomaly interval receive full weight, while those in the buffer zones are
|
345
|
-
assigned linearly decaying weights depending on their distance from the interval. Scores outside
|
346
|
-
these zones contribute to false positives, and intervals with insufficient detection are penalized
|
347
|
-
as false negatives.
|
348
|
-
|
349
|
-
The final PATE score aggregates these weighted contributions across all time steps, yielding
|
350
|
-
a smooth performance measure that is sensitive to both the timing and confidence of the predictions.
|
351
|
-
|
352
|
-
Implementation of https://arxiv.org/abs/2405.12096
|
353
|
-
|
354
|
-
For more information, see the original paper:
|
355
|
-
https://arxiv.org/abs/2405.12096
|
356
|
-
|
357
|
-
Parameters:
|
358
|
-
y_true (np.array):
|
359
|
-
Ground truth binary labels (0 = normal, 1 = anomaly).
|
360
|
-
y_anomaly_scores (np.array):
|
361
|
-
Real-valued anomaly scores for each time point.
|
362
|
-
early (int):
|
363
|
-
Length of the early buffer zone before each anomaly interval.
|
364
|
-
delay (int):
|
365
|
-
Length of the delay buffer zone after each anomaly interval.
|
366
|
-
|
367
|
-
Returns:
|
368
|
-
float: The real-valued PATE score.
|
369
|
-
"""
|
370
|
-
validate_non_binary_inputs(y_true, y_anomaly_scores)
|
371
|
-
|
372
|
-
return PATE(y_true, y_anomaly_scores, early, delay, binary_scores=False)
|
tsadmetrics/scripts/__init__.py
DELETED
File without changes
|
@@ -1,42 +0,0 @@
|
|
1
|
-
#!/usr/bin/env python3
|
2
|
-
import argparse
|
3
|
-
from tsadmetrics.utils import compute_metrics_from_file
|
4
|
-
|
5
|
-
|
6
|
-
def main():
|
7
|
-
|
8
|
-
parser = argparse.ArgumentParser(
|
9
|
-
description='Compute metrics from anomaly detection results and configuration files.'
|
10
|
-
)
|
11
|
-
|
12
|
-
|
13
|
-
parser.add_argument(
|
14
|
-
'--res_file',
|
15
|
-
type=str,
|
16
|
-
required=True,
|
17
|
-
help='Path to the results CSV file (e.g., results.csv)'
|
18
|
-
)
|
19
|
-
parser.add_argument(
|
20
|
-
'--conf_file',
|
21
|
-
type=str,
|
22
|
-
required=True,
|
23
|
-
help='Path to the configuration JSON file (e.g., conf.json)'
|
24
|
-
)
|
25
|
-
parser.add_argument(
|
26
|
-
'--output_dir',
|
27
|
-
type=str,
|
28
|
-
required=True,
|
29
|
-
help='Directory where output files will be saved (e.g., ./output_dir)'
|
30
|
-
)
|
31
|
-
|
32
|
-
args = parser.parse_args()
|
33
|
-
|
34
|
-
|
35
|
-
compute_metrics_from_file(
|
36
|
-
results_file=args.res_file,
|
37
|
-
conf_file=args.conf_file,
|
38
|
-
output_dir=args.output_dir
|
39
|
-
)
|
40
|
-
|
41
|
-
if __name__ == '__main__':
|
42
|
-
main()
|
tsadmetrics/utils.py
DELETED
@@ -1,124 +0,0 @@
|
|
1
|
-
import numpy as np
|
2
|
-
import pandas as pd
|
3
|
-
import time
|
4
|
-
import sys
|
5
|
-
import tsadmetrics
|
6
|
-
def compute_metrics(y_true: np.array,y_pred: np.array, metrics: list, metrics_params: dict, is_anomaly_score = False, verbose = False):
|
7
|
-
"""
|
8
|
-
Computes the specified metrics for the given true and predicted values.
|
9
|
-
|
10
|
-
Parameters:
|
11
|
-
y_true (np.array):
|
12
|
-
True labels.
|
13
|
-
y_pred (np.array):
|
14
|
-
Predicted labels or scores.
|
15
|
-
metrics (list):
|
16
|
-
List of metric names to compute.
|
17
|
-
metrics_params (dict):
|
18
|
-
Dictionary of parameters for each metric.
|
19
|
-
is_anomaly_score (bool):
|
20
|
-
Flag indicating if y_true and y_pred are anomaly scores. Otherwise, they are treated as binary labels.
|
21
|
-
verbose (bool):
|
22
|
-
Flag to print additional information.
|
23
|
-
Returns:
|
24
|
-
results (DataFrame): DataFrame containing the computed metrics and their values.
|
25
|
-
"""
|
26
|
-
if not (np.array_equal(np.unique(y_true), [0, 1]) or np.array_equal(np.unique(y_true), [0]) or np.array_equal(np.unique(y_true), [1])):
|
27
|
-
raise ValueError("y_true must be binary labels (0 or 1).")
|
28
|
-
if not is_anomaly_score:
|
29
|
-
#Chech if y_true and y_pred are binary labels
|
30
|
-
if not ( np.array_equal(np.unique(y_pred), [0, 1])):
|
31
|
-
raise ValueError("y_true and y_pred must be binary labels (0 or 1) when is_anomaly_score is False. Which is the default.")
|
32
|
-
else:
|
33
|
-
# Check if y_true and y_pred are anomaly scores
|
34
|
-
if not (np.all((y_pred >= 0) & (y_pred <= 1))):
|
35
|
-
raise ValueError("y_true must be binary labels (0 or 1), and y_pred must be anomaly scores in the range [0, 1] when is_anomaly_score is True.")
|
36
|
-
results = {}
|
37
|
-
|
38
|
-
for metric in metrics:
|
39
|
-
metric_name = metric[0]
|
40
|
-
metric_func = metric[1]
|
41
|
-
if verbose:
|
42
|
-
print(f"Calculating metric: {metric_name}")
|
43
|
-
t0 = time.time()
|
44
|
-
metric_value = metric_func(y_true, y_pred, **metrics_params.get(metric_name, {}))
|
45
|
-
if verbose:
|
46
|
-
t1 = time.time()
|
47
|
-
print(f"Metric {metric_name} calculated in {t1 - t0:.4f} seconds")
|
48
|
-
print(f"Metric {metric_name} value: {metric_value}")
|
49
|
-
# Store the result in the DataFrame
|
50
|
-
results[metric_name] = metric_value
|
51
|
-
|
52
|
-
metrics_df = pd.DataFrame(columns=['metric_name', 'metric_value'])
|
53
|
-
metrics_df['metric_name'] = results.keys()
|
54
|
-
metrics_df['metric_value'] = results.values()
|
55
|
-
|
56
|
-
return metrics_df
|
57
|
-
|
58
|
-
|
59
|
-
def compute_metrics_from_file(results_file: str, conf_file: str, output_dir: str = '.'):
|
60
|
-
"""
|
61
|
-
Computes metrics based on prediction results from a CSV file and configuration from a JSON file.
|
62
|
-
|
63
|
-
Parameters:
|
64
|
-
results_file (str):
|
65
|
-
Path to CSV file containing y_true and y_pred columns.
|
66
|
-
conf_file (str):
|
67
|
-
Path to JSON configuration file with metrics and parameters.
|
68
|
-
|
69
|
-
Returns:
|
70
|
-
pd.DataFrame: DataFrame with computed metrics.
|
71
|
-
"""
|
72
|
-
# Read results data
|
73
|
-
res = pd.read_csv(results_file)
|
74
|
-
y_true = res['y_true'].values
|
75
|
-
y_pred = res['y_pred'].values
|
76
|
-
|
77
|
-
# Determine if predictions are binary or scores
|
78
|
-
is_anomaly_score = False
|
79
|
-
unique_values = np.unique(y_pred)
|
80
|
-
if not (np.array_equal(unique_values, [0, 1]) or
|
81
|
-
np.array_equal(unique_values, [0]) or
|
82
|
-
np.array_equal(unique_values, [1])):
|
83
|
-
is_anomaly_score = True
|
84
|
-
if not np.all((y_pred >= 0) & (y_pred <= 1)):
|
85
|
-
raise ValueError("y_pred must be either binary (0/1) or anomaly scores in range [0, 1]")
|
86
|
-
|
87
|
-
# Read configuration from JSON using pandas
|
88
|
-
try:
|
89
|
-
config_df = pd.read_json(conf_file, orient='records')
|
90
|
-
except ValueError as e:
|
91
|
-
raise ValueError(f"Invalid JSON format in configuration file: {str(e)}")
|
92
|
-
|
93
|
-
# Convert pandas DataFrame to format expected by compute_metrics
|
94
|
-
metrics = []
|
95
|
-
metrics_params = {}
|
96
|
-
|
97
|
-
for _, row in config_df.iterrows():
|
98
|
-
metric_name = row['name']
|
99
|
-
try:
|
100
|
-
metric_func = getattr(tsadmetrics, metric_name)
|
101
|
-
except AttributeError:
|
102
|
-
raise ValueError(f"Metric function '{metric_name}' not found in tsadmetrics module")
|
103
|
-
|
104
|
-
metrics.append((metric_name, metric_func))
|
105
|
-
|
106
|
-
# Handle params (convert from pandas Series to dict if needed)
|
107
|
-
params = row.get('params', {})
|
108
|
-
if pd.notna(params) and params: # Check for non-empty params
|
109
|
-
if isinstance(params, pd.Series):
|
110
|
-
metrics_params[metric_name] = params.to_dict()
|
111
|
-
else:
|
112
|
-
metrics_params[metric_name] = params
|
113
|
-
|
114
|
-
# Compute metrics
|
115
|
-
metrics_df = compute_metrics(
|
116
|
-
y_true=y_true,
|
117
|
-
y_pred=y_pred,
|
118
|
-
metrics=metrics,
|
119
|
-
metrics_params=metrics_params,
|
120
|
-
is_anomaly_score=is_anomaly_score,
|
121
|
-
verbose=False
|
122
|
-
)
|
123
|
-
metrics_df.to_csv(output_dir+'/computed_metrics.csv', index=False)
|
124
|
-
|
tsadmetrics/validation.py
DELETED
@@ -1,35 +0,0 @@
|
|
1
|
-
import numpy as np
|
2
|
-
|
3
|
-
def check_gt_binary_array(arr):
|
4
|
-
if len(arr.shape) != 1:
|
5
|
-
raise ValueError("Ground truth input must be a 1D binary array.")
|
6
|
-
if not np.all(np.isin(arr, [0, 1])):
|
7
|
-
raise ValueError("Ground truth input array must contain only binary values (0 or 1).")
|
8
|
-
return True
|
9
|
-
|
10
|
-
def check_pred_binary_array(arr):
|
11
|
-
if len(arr.shape) != 1:
|
12
|
-
raise ValueError("Prediction input must be a 1D binary array.")
|
13
|
-
if not np.all(np.isin(arr, [0, 1])):
|
14
|
-
raise ValueError("Prediction input array must contain only binary values (0 or 1).")
|
15
|
-
return True
|
16
|
-
def check_same_length(arr1, arr2):
|
17
|
-
if len(arr1) != len(arr2):
|
18
|
-
raise ValueError("Ground truth and prediction arrays must have the same length.")
|
19
|
-
return True
|
20
|
-
|
21
|
-
def check_pred_continuous_array(arr):
|
22
|
-
if len(arr.shape) != 1:
|
23
|
-
raise ValueError("Prediction input must be a 1D continuous array.")
|
24
|
-
if not np.all((arr >= 0) & (arr <= 1)):
|
25
|
-
raise ValueError("All values in the array must be in the range [0, 1].")
|
26
|
-
return True
|
27
|
-
|
28
|
-
def validate_binary_inputs(y_true, y_pred):
|
29
|
-
check_gt_binary_array(y_true)
|
30
|
-
check_pred_binary_array(y_pred)
|
31
|
-
check_same_length(y_true, y_pred)
|
32
|
-
|
33
|
-
def validate_non_binary_inputs(y_true, y_anomaly_scores):
|
34
|
-
check_gt_binary_array(y_true)
|
35
|
-
check_same_length(y_true, y_anomaly_scores)
|