tsadmetrics 0.1.17__py3-none-any.whl → 1.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (143) hide show
  1. {docs_api → docs/add_docs/api_doc}/conf.py +3 -26
  2. {docs_manual → docs/add_docs/full_doc}/conf.py +2 -25
  3. docs/add_docs/manual_doc/conf.py +67 -0
  4. docs/conf.py +1 -1
  5. examples/example_direct_data.py +28 -0
  6. examples/example_direct_single_data.py +25 -0
  7. examples/example_file_reference.py +24 -0
  8. examples/example_global_config_file.py +13 -0
  9. examples/example_metric_config_file.py +19 -0
  10. examples/example_simple_metric.py +8 -0
  11. examples/specific_examples/AbsoluteDetectionDistance_example.py +24 -0
  12. examples/specific_examples/AffiliationbasedFScore_example.py +24 -0
  13. examples/specific_examples/AverageDetectionCount_example.py +24 -0
  14. examples/specific_examples/CompositeFScore_example.py +24 -0
  15. examples/specific_examples/DelayThresholdedPointadjustedFScore_example.py +24 -0
  16. examples/specific_examples/DetectionAccuracyInRange_example.py +24 -0
  17. examples/specific_examples/EnhancedTimeseriesAwareFScore_example.py +24 -0
  18. examples/specific_examples/LatencySparsityawareFScore_example.py +24 -0
  19. examples/specific_examples/MeanTimeToDetect_example.py +24 -0
  20. examples/specific_examples/NabScore_example.py +24 -0
  21. examples/specific_examples/PateFScore_example.py +24 -0
  22. examples/specific_examples/Pate_example.py +24 -0
  23. examples/specific_examples/PointadjustedAtKFScore_example.py +24 -0
  24. examples/specific_examples/PointadjustedAucPr_example.py +24 -0
  25. examples/specific_examples/PointadjustedAucRoc_example.py +24 -0
  26. examples/specific_examples/PointadjustedFScore_example.py +24 -0
  27. examples/specific_examples/RangebasedFScore_example.py +24 -0
  28. examples/specific_examples/SegmentwiseFScore_example.py +24 -0
  29. examples/specific_examples/TemporalDistance_example.py +24 -0
  30. examples/specific_examples/TimeTolerantFScore_example.py +24 -0
  31. examples/specific_examples/TimeseriesAwareFScore_example.py +24 -0
  32. examples/specific_examples/TotalDetectedInRange_example.py +24 -0
  33. examples/specific_examples/VusPr_example.py +24 -0
  34. examples/specific_examples/VusRoc_example.py +24 -0
  35. examples/specific_examples/WeightedDetectionDifference_example.py +24 -0
  36. tsadmetrics/__init__.py +0 -21
  37. tsadmetrics/base/Metric.py +188 -0
  38. tsadmetrics/evaluation/Report.py +25 -0
  39. tsadmetrics/evaluation/Runner.py +253 -0
  40. tsadmetrics/metrics/Registry.py +141 -0
  41. tsadmetrics/metrics/__init__.py +2 -0
  42. tsadmetrics/metrics/spm/PointwiseAucPr.py +62 -0
  43. tsadmetrics/metrics/spm/PointwiseAucRoc.py +63 -0
  44. tsadmetrics/metrics/spm/PointwiseFScore.py +86 -0
  45. tsadmetrics/metrics/spm/PrecisionAtK.py +81 -0
  46. tsadmetrics/metrics/spm/__init__.py +9 -0
  47. tsadmetrics/metrics/tem/dpm/DelayThresholdedPointadjustedFScore.py +83 -0
  48. tsadmetrics/metrics/tem/dpm/LatencySparsityawareFScore.py +76 -0
  49. tsadmetrics/metrics/tem/dpm/MeanTimeToDetect.py +47 -0
  50. tsadmetrics/metrics/tem/dpm/NabScore.py +60 -0
  51. tsadmetrics/metrics/tem/dpm/__init__.py +11 -0
  52. tsadmetrics/metrics/tem/ptdm/AverageDetectionCount.py +53 -0
  53. tsadmetrics/metrics/tem/ptdm/DetectionAccuracyInRange.py +66 -0
  54. tsadmetrics/metrics/tem/ptdm/PointadjustedAtKFScore.py +80 -0
  55. tsadmetrics/metrics/tem/ptdm/TimeseriesAwareFScore.py +248 -0
  56. tsadmetrics/metrics/tem/ptdm/TotalDetectedInRange.py +65 -0
  57. tsadmetrics/metrics/tem/ptdm/WeightedDetectionDifference.py +97 -0
  58. tsadmetrics/metrics/tem/ptdm/__init__.py +12 -0
  59. tsadmetrics/metrics/tem/tmem/AbsoluteDetectionDistance.py +48 -0
  60. tsadmetrics/metrics/tem/tmem/EnhancedTimeseriesAwareFScore.py +252 -0
  61. tsadmetrics/metrics/tem/tmem/TemporalDistance.py +68 -0
  62. tsadmetrics/metrics/tem/tmem/__init__.py +9 -0
  63. tsadmetrics/metrics/tem/tpdm/CompositeFScore.py +104 -0
  64. tsadmetrics/metrics/tem/tpdm/PointadjustedAucPr.py +123 -0
  65. tsadmetrics/metrics/tem/tpdm/PointadjustedAucRoc.py +119 -0
  66. tsadmetrics/metrics/tem/tpdm/PointadjustedFScore.py +96 -0
  67. tsadmetrics/metrics/tem/tpdm/RangebasedFScore.py +236 -0
  68. tsadmetrics/metrics/tem/tpdm/SegmentwiseFScore.py +73 -0
  69. tsadmetrics/metrics/tem/tpdm/__init__.py +12 -0
  70. tsadmetrics/metrics/tem/tstm/AffiliationbasedFScore.py +68 -0
  71. tsadmetrics/metrics/tem/tstm/Pate.py +62 -0
  72. tsadmetrics/metrics/tem/tstm/PateFScore.py +61 -0
  73. tsadmetrics/metrics/tem/tstm/TimeTolerantFScore.py +85 -0
  74. tsadmetrics/metrics/tem/tstm/VusPr.py +51 -0
  75. tsadmetrics/metrics/tem/tstm/VusRoc.py +55 -0
  76. tsadmetrics/metrics/tem/tstm/__init__.py +15 -0
  77. tsadmetrics/{_tsadeval/affiliation/_integral_interval.py → utils/functions_affiliation.py} +377 -9
  78. tsadmetrics/utils/functions_auc.py +393 -0
  79. tsadmetrics/utils/functions_conversion.py +63 -0
  80. tsadmetrics/utils/functions_counting_metrics.py +26 -0
  81. tsadmetrics/{_tsadeval/latency_sparsity_aware.py → utils/functions_latency_sparsity_aware.py} +1 -1
  82. tsadmetrics/{_tsadeval/nabscore.py → utils/functions_nabscore.py} +15 -1
  83. tsadmetrics-1.0.1.dist-info/METADATA +83 -0
  84. tsadmetrics-1.0.1.dist-info/RECORD +91 -0
  85. tsadmetrics-1.0.1.dist-info/top_level.txt +3 -0
  86. entorno/bin/activate_this.py +0 -32
  87. entorno/bin/rst2html.py +0 -23
  88. entorno/bin/rst2html4.py +0 -26
  89. entorno/bin/rst2html5.py +0 -33
  90. entorno/bin/rst2latex.py +0 -26
  91. entorno/bin/rst2man.py +0 -27
  92. entorno/bin/rst2odt.py +0 -28
  93. entorno/bin/rst2odt_prepstyles.py +0 -20
  94. entorno/bin/rst2pseudoxml.py +0 -23
  95. entorno/bin/rst2s5.py +0 -24
  96. entorno/bin/rst2xetex.py +0 -27
  97. entorno/bin/rst2xml.py +0 -23
  98. entorno/bin/rstpep2html.py +0 -25
  99. tests/test_binary.py +0 -946
  100. tests/test_non_binary.py +0 -450
  101. tests/test_utils.py +0 -49
  102. tsadmetrics/_tsadeval/affiliation/_affiliation_zone.py +0 -86
  103. tsadmetrics/_tsadeval/affiliation/_single_ground_truth_event.py +0 -68
  104. tsadmetrics/_tsadeval/affiliation/generics.py +0 -135
  105. tsadmetrics/_tsadeval/affiliation/metrics.py +0 -114
  106. tsadmetrics/_tsadeval/auc_roc_pr_plot.py +0 -295
  107. tsadmetrics/_tsadeval/discontinuity_graph.py +0 -109
  108. tsadmetrics/_tsadeval/eTaPR_pkg/DataManage/File_IO.py +0 -175
  109. tsadmetrics/_tsadeval/eTaPR_pkg/DataManage/Range.py +0 -50
  110. tsadmetrics/_tsadeval/eTaPR_pkg/DataManage/Time_Plot.py +0 -184
  111. tsadmetrics/_tsadeval/eTaPR_pkg/DataManage/__init__.py +0 -0
  112. tsadmetrics/_tsadeval/eTaPR_pkg/__init__.py +0 -0
  113. tsadmetrics/_tsadeval/eTaPR_pkg/etapr.py +0 -386
  114. tsadmetrics/_tsadeval/eTaPR_pkg/tapr.py +0 -362
  115. tsadmetrics/_tsadeval/metrics.py +0 -698
  116. tsadmetrics/_tsadeval/prts/__init__.py +0 -0
  117. tsadmetrics/_tsadeval/prts/base/__init__.py +0 -0
  118. tsadmetrics/_tsadeval/prts/base/time_series_metrics.py +0 -165
  119. tsadmetrics/_tsadeval/prts/basic_metrics_ts.py +0 -121
  120. tsadmetrics/_tsadeval/prts/time_series_metrics/__init__.py +0 -0
  121. tsadmetrics/_tsadeval/prts/time_series_metrics/fscore.py +0 -61
  122. tsadmetrics/_tsadeval/prts/time_series_metrics/precision.py +0 -86
  123. tsadmetrics/_tsadeval/prts/time_series_metrics/precision_recall.py +0 -21
  124. tsadmetrics/_tsadeval/prts/time_series_metrics/recall.py +0 -85
  125. tsadmetrics/_tsadeval/tests.py +0 -376
  126. tsadmetrics/_tsadeval/threshold_plt.py +0 -30
  127. tsadmetrics/_tsadeval/time_tolerant.py +0 -33
  128. tsadmetrics/binary_metrics.py +0 -1652
  129. tsadmetrics/metric_utils.py +0 -98
  130. tsadmetrics/non_binary_metrics.py +0 -372
  131. tsadmetrics/scripts/__init__.py +0 -0
  132. tsadmetrics/scripts/compute_metrics.py +0 -42
  133. tsadmetrics/utils.py +0 -124
  134. tsadmetrics/validation.py +0 -35
  135. tsadmetrics-0.1.17.dist-info/METADATA +0 -54
  136. tsadmetrics-0.1.17.dist-info/RECORD +0 -66
  137. tsadmetrics-0.1.17.dist-info/entry_points.txt +0 -2
  138. tsadmetrics-0.1.17.dist-info/top_level.txt +0 -6
  139. {tests → tsadmetrics/base}/__init__.py +0 -0
  140. /tsadmetrics/{_tsadeval → evaluation}/__init__.py +0 -0
  141. /tsadmetrics/{_tsadeval/affiliation → metrics/tem}/__init__.py +0 -0
  142. /tsadmetrics/{_tsadeval/vus_utils.py → utils/functions_vus.py} +0 -0
  143. {tsadmetrics-0.1.17.dist-info → tsadmetrics-1.0.1.dist-info}/WHEEL +0 -0
@@ -1,98 +0,0 @@
1
- import numpy as np
2
- from ._tsadeval.metrics import pointwise_to_full_series, segmentwise_to_full_series
3
-
4
- def is_full_series(length: int, anomalies: np.array):
5
- # [1 0 1 1 0]
6
- return len(anomalies.shape) == 1 and len(anomalies) == length
7
-
8
- def is_pointwise(length: int, anomalies: np.array):
9
- # [0 2 3]
10
- return len(anomalies.shape) == 1 and len(anomalies) < length
11
-
12
- def is_segmentwise(length: int, anomalies: np.array):
13
- # [[0 0] [2 3]]
14
- return len(anomalies.shape) == 2
15
-
16
-
17
- def transform_to_full_series(length: int, anomalies: np.array):
18
- if is_full_series(length, anomalies):
19
- return anomalies
20
- elif is_pointwise(anomalies):
21
- return pointwise_to_full_series(anomalies, length)
22
- elif is_segmentwise(length, anomalies):
23
- return segmentwise_to_full_series(anomalies, length)
24
- else:
25
- raise ValueError(f"Illegal shape of anomalies:\n{anomalies}")
26
-
27
- def counting_method(y_true: np.array, y_pred: np.array, k: int):
28
- em,da,ma,fa = 0,0,0,0
29
- for i_gt in range(len(y_true)):
30
- i_pa = i_gt
31
- gt = y_true[i_gt]
32
- pa = y_pred[i_pa]
33
- if gt==1 and pa==1:
34
- em+=1
35
- elif gt==0 and pa==1:
36
- fa+=1
37
- elif gt==1 and pa==0:
38
- anom_range = y_pred[i_gt-k:i_pa+k+1]
39
- detected = False
40
- for r in anom_range:
41
- if r==1:
42
- em+=1
43
- detected=True
44
- break
45
- if not detected:
46
- ma+=1
47
- elif gt==0 and pa==0:
48
- pass
49
- # b = DelayThresholdedPointAdjust(len(y_true),y_true,y_pred,k=k)
50
- # da = b.tp-em
51
- # ma = b.fn
52
-
53
- return em,da,ma,fa
54
-
55
-
56
- #Range Based utils
57
-
58
- def cardinality(n_intersections,mode):
59
- if mode == 'one':
60
- return 1
61
- elif mode == 'reciprocal':
62
- if n_intersections==0:
63
- return 1
64
- else:
65
- return float(1/n_intersections)
66
- else:
67
- raise Exception("Error, wrong cardinality mode.")
68
-
69
-
70
- def size(anomaly_range, overlap_set, position, bias):
71
- if overlap_set == None:
72
- return 0
73
-
74
- my_value = 0
75
- max_value = 0
76
- anomaly_length = anomaly_range[1] - anomaly_range[0] + 1
77
- for i in range(1,anomaly_length+1):
78
- bias_value = position(i, anomaly_length,bias)
79
- max_value += bias_value
80
- if anomaly_range[0]+i-1 >= overlap_set[0] and anomaly_range[0]+i-1 <= overlap_set[1]:
81
- my_value += bias_value
82
- return my_value / max_value
83
-
84
- def position(i, anomaly_length,bias):
85
- if bias == 'flat':
86
- return 1
87
- elif bias == 'front-end':
88
- return anomaly_length - i + 1
89
- elif bias == 'back-end':
90
- return i
91
- elif bias == 'middle':
92
- if i <= anomaly_length / 2:
93
- return i
94
- else:
95
- return anomaly_length - i + 1
96
- else:
97
- raise Exception("Error, wrong bias value.")
98
-
@@ -1,372 +0,0 @@
1
- import numpy as np
2
- from ._tsadeval.metrics import *
3
- from .validation import validate_non_binary_inputs
4
- from sklearn.metrics import auc
5
- from pate.PATE_metric import PATE
6
- def precision_at_k(y_true : np.array, y_anomaly_scores: np.array):
7
- """
8
- Calculate the precision at k score for anomaly detection in time series.
9
-
10
- This metric evaluates how many of the top-k points (with highest anomaly scores)
11
- actually correspond to true anomalies. It is particularly useful when we are
12
- interested in identifying the most anomalous points, without needing to set a
13
- fixed threshold.
14
-
15
- The value of k is automatically set to the number of true anomalies present in
16
- y_true. That is, k = sum(y_true).
17
-
18
- Implementation of https://link.springer.com/article/10.1007/s10618-023-00988-8
19
-
20
- Parameters:
21
- y_true (np.array):
22
- The ground truth binary labels for the time series data.
23
- y_anomaly_scores (np.array):
24
- The predicted anomaly scores for the time series data.
25
-
26
- Returns:
27
- float: The precision at k score.
28
- """
29
- validate_non_binary_inputs(y_true, y_anomaly_scores)
30
-
31
- m = PatK_pw(y_true,y_anomaly_scores)
32
-
33
- return m.get_score()
34
-
35
- def auc_roc_pw(y_true : np.array, y_anomaly_scores: np.array):
36
- """
37
- Calculate the AUC-ROC score for anomaly detection in time series.
38
-
39
- This is the standard Area Under the Receiver Operating Characteristic Curve (AUC-ROC),
40
- computed in a point-wise manner. That is, each point in the time series is treated
41
- independently when calculating true positives, false positives, and false negatives.
42
-
43
- Implementation of https://link.springer.com/article/10.1007/s10618-023-00988-8
44
-
45
- Parameters:
46
- y_true (np.array):
47
- Ground-truth binary labels for the time series (0 = normal, 1 = anomaly).
48
- y_anomaly_scores (np.array):
49
- Continuous anomaly scores assigned to each point in the series.
50
-
51
- Returns:
52
- float: AUC-ROC score.
53
- """
54
- validate_non_binary_inputs(y_true, y_anomaly_scores)
55
-
56
- m = AUC_ROC(y_true,y_anomaly_scores)
57
-
58
- return m.get_score()
59
-
60
-
61
- def auc_pr_pw(y_true : np.array ,y_anomaly_scores: np.array):
62
- """
63
- Calculate the AUC-PR score for anomaly detection in time series.
64
-
65
- This is the standard Area Under the Precision-Recall Curve (AUC-PR),
66
- computed in a point-wise manner. That is, each point in the time series is treated
67
- independently when calculating precision and recall.
68
-
69
- Implementation of https://link.springer.com/article/10.1007/s10618-023-00988-8
70
-
71
- Parameters:
72
- y_true (np.array):
73
- Ground-truth binary labels for the time series (0 = normal, 1 = anomaly).
74
- y_anomaly_scores (np.array):
75
- Continuous anomaly scores assigned to each point in the series.
76
-
77
- Returns:
78
- float: AUC-PR score.
79
- """
80
- validate_non_binary_inputs(y_true, y_anomaly_scores)
81
-
82
- m = AUC_PR_pw(y_true,y_anomaly_scores)
83
-
84
- return m.get_score()
85
-
86
-
87
- def auc_roc_pa(y_true: np.array, y_anomaly_scores: np.array):
88
- """
89
- Calculate the AUC-ROC score using point-adjusted evaluation for anomaly detection in time series.
90
-
91
- This is the standard Area Under the Receiver Operating Characteristic Curve (AUC-ROC), but instead
92
- of computing true positive rate (TPR) and false positive rate (FPR) point-wise, it uses a point-adjusted
93
- approach. Specifically, for each ground-truth anomalous segment, if at least one point within that
94
- segment is predicted as anomalous, the entire segment is considered correctly detected. The adjusted
95
- predictions are then compared to the ground-truth labels to compute true positives, false positives,
96
- and false negatives, which are used to construct the ROC curve.
97
-
98
- Implementation of https://link.springer.com/article/10.1007/s10618-023-00988-8
99
-
100
- Parameters:
101
- y_true (np.array):
102
- Ground-truth binary labels for the time series (0 = normal, 1 = anomaly).
103
- y_anomaly_scores (np.array):
104
- Continuous anomaly scores assigned to each point in the series.
105
-
106
- Returns:
107
- float: AUC-ROC score (with point-adjusted evaluation).
108
- """
109
- validate_non_binary_inputs(y_true, y_anomaly_scores)
110
-
111
- tprs = [0]
112
- fprs = [0]
113
- tps, fps, fns = [], [], []
114
-
115
- p_adj = PointAdjust(len(y_true), y_true, (np.array(y_anomaly_scores) >= 0.5).astype(int))
116
- segments = p_adj.get_gt_anomalies_segmentwise()
117
- idx = np.argsort(y_anomaly_scores)[::-1].astype(int)
118
- y_true_sorted = np.array(y_true)[idx]
119
- y_anomaly_scores_sorted = np.array(y_anomaly_scores)[idx]
120
-
121
- segment_mins = []
122
- for start, end in segments:
123
- anoms_scores = y_anomaly_scores[start:end+1]
124
- segment_mins.append([np.max(anoms_scores), end-start+1])
125
-
126
- for i_t in range(len(y_anomaly_scores_sorted)):
127
- fp, tp, fn = 0, 0, 0
128
- if i_t > 0 and y_anomaly_scores_sorted[i_t] == y_anomaly_scores_sorted[i_t-1]:
129
- tp = tps[-1]
130
- fp = fps[-1]
131
- fn = fns[-1]
132
- else:
133
- if y_true_sorted[i_t] == 0:
134
- # FP
135
- if len(fps) == 0:
136
- aux_y_pred = (y_anomaly_scores >= y_anomaly_scores_sorted[i_t]).astype(int)
137
- for i in range(len(aux_y_pred)):
138
- if aux_y_pred[i] == 1 and y_true[i] == 0:
139
- fp += 1
140
- else:
141
- fp = fps[i_t-1] + 1
142
- else:
143
- if len(fps) == 0:
144
- aux_y_pred = (y_anomaly_scores >= y_anomaly_scores_sorted[i_t]).astype(int)
145
- for i in range(len(aux_y_pred)):
146
- if aux_y_pred[i] == 1 and y_true[i] == 0:
147
- fp += 1
148
- else:
149
- fp = fps[i_t-1]
150
- for score, length in segment_mins:
151
- if score >= y_anomaly_scores_sorted[i_t]:
152
- # TP
153
- tp += length
154
- else:
155
- # FN
156
- fn += length
157
- tps.append(tp)
158
- fns.append(fn)
159
- fps.append(fp)
160
- for tp, fp, fn in zip(tps, fps, fns):
161
- if tp + fn > 0:
162
- tprs.append(tp / (tp + fn))
163
- else:
164
- tprs.append(0)
165
- if fp + (len(y_true) - np.sum(y_true)) > 0:
166
- fprs.append(fp / (fp + (len(y_true) - np.sum(y_true))))
167
- else:
168
- fprs.append(0)
169
-
170
- auc_value = auc(fprs, tprs)
171
- return auc_value
172
-
173
- def auc_pr_pa(y_true: np.array, y_anomaly_scores: np.array):
174
- """
175
- Calculate the AUC-PR score using point-adjusted evaluation for anomaly detection in time series.
176
-
177
- This is the standard Area Under the Precision-Recall Curve (AUC-PR), but instead of computing
178
- precision and recall point-wise, it uses a point-adjusted approach. Specifically, for each
179
- ground-truth anomalous segment, if at least one point within that segment is predicted as anomalous,
180
- the entire segment is considered correctly detected. The adjusted predictions are then compared
181
- to the ground-truth labels to compute true positives, false positives, and false negatives,
182
- which are used to construct the PR curve.
183
-
184
- Implementation of https://link.springer.com/article/10.1007/s10618-023-00988-8
185
-
186
- Parameters:
187
- y_true (np.array):
188
- Ground-truth binary labels for the time series (0 = normal, 1 = anomaly).
189
- y_anomaly_scores (np.array):
190
- Continuous anomaly scores assigned to each point in the series.
191
-
192
- Returns:
193
- float: AUC-PR score (with point-adjusted evaluation).
194
- """
195
- validate_non_binary_inputs(y_true, y_anomaly_scores)
196
-
197
- precisions = [1]
198
- recalls = [0]
199
- tps,fps,fns = [],[],[]
200
-
201
- p_adj = PointAdjust(len(y_true),y_true,(np.array(y_anomaly_scores) >= 0.5).astype(int))
202
- segments= p_adj.get_gt_anomalies_segmentwise()
203
- idx = np.argsort(y_anomaly_scores)[::-1].astype(int)
204
- y_true_sorted = np.array(y_true)[idx]
205
- y_anomaly_scores_sorted = np.array(y_anomaly_scores)[idx]
206
-
207
- segment_mins = []
208
- for start,end in segments:
209
- anoms_scores = y_anomaly_scores[start:end+1]
210
- segment_mins.append([np.max(anoms_scores),end-start+1])
211
-
212
- for i_t in range(len(y_anomaly_scores_sorted)):
213
- fp,tp,fn = 0,0,0
214
- if i_t > 0 and y_anomaly_scores_sorted[i_t] == y_anomaly_scores_sorted[i_t-1] :
215
- tp = tps[-1]
216
- fp = fps[-1]
217
- fn = fns[-1]
218
- else:
219
- if y_true_sorted[i_t] == 0:
220
- #FP
221
- if len(fps)==0:
222
- aux_y_pred = (y_anomaly_scores >= y_anomaly_scores_sorted[i_t]).astype(int)
223
- for i in range(len(aux_y_pred)):
224
- if aux_y_pred[i] == 1 and y_true[i] == 0:
225
- fp+=1
226
-
227
-
228
- else:
229
- fp=fps[i_t-1]+1
230
- else:
231
- if len(fps)==0:
232
- aux_y_pred = (y_anomaly_scores >= y_anomaly_scores_sorted[i_t]).astype(int)
233
- for i in range(len(aux_y_pred)):
234
- if aux_y_pred[i] == 1 and y_true[i] == 0:
235
- fp+=1
236
- else:
237
- fp=fps[i_t-1]
238
- for score, length in segment_mins:
239
- if score >= y_anomaly_scores_sorted[i_t]:
240
- #TP
241
- tp+= length
242
- else:
243
- #FN
244
- fn+= length
245
- tps.append(tp)
246
- fns.append(fn)
247
- fps.append(fp)
248
- for tp,fp,fn in zip(tps,fps,fns):
249
- if tp>0:
250
- precisions.append(tp/(tp+fp))
251
- recalls.append(tp/(tp+fn))
252
- else:
253
- precisions.append(0)
254
- recalls.append(0)
255
-
256
-
257
- recalls.append(1)
258
- precisions.append(0)
259
-
260
- auc_value = auc(recalls, precisions)
261
- return auc_value
262
-
263
-
264
-
265
-
266
-
267
-
268
- def vus_roc(y_true : np.array ,y_anomaly_scores: np.array, window=4):
269
- """
270
- Calculate the VUS-ROC (Volume Under the ROC Surface) score for anomaly detection in time series.
271
-
272
- This metric extends the classical AUC-ROC by introducing a temporal tolerance parameter `l`, which
273
- smooths the binary ground-truth labels. The idea is to allow a flexible evaluation that tolerates
274
- small misalignments in the detection of anomalies. The final score is computed by integrating
275
- the ROC-AUC over different values of the tolerance parameter, from 0 to `window`, thus producing
276
- a volume under the ROC surface.
277
-
278
- Implementation of https://link.springer.com/article/10.1007/s10618-023-00988-8
279
-
280
- For more information, see the original paper:
281
- https://dl.acm.org/doi/10.14778/3551793.3551830
282
-
283
- Parameters:
284
- y_true (np.array):
285
- Ground-truth binary labels (0 = normal, 1 = anomaly).
286
- y_anomaly_scores (np.array):
287
- Anomaly scores for each time point.
288
- window (int):
289
- Maximum temporal tolerance `l` used to smooth the evaluation.
290
-
291
- Returns:
292
- float: VUS-ROC score.
293
-
294
-
295
- """
296
- validate_non_binary_inputs(y_true, y_anomaly_scores)
297
-
298
- m = VUS_ROC(y_true,y_anomaly_scores,max_window=window)
299
-
300
- return m.get_score()
301
-
302
-
303
- def vus_pr(y_true : np.array ,y_anomaly_scores: np.array, window=4):
304
- """
305
- Calculate the VUS-PR (Volume Under the PR Surface) score for anomaly detection in time series.
306
-
307
- This metric is an extension of the classical AUC-PR, incorporating a temporal tolerance parameter `l`
308
- that smooths the binary ground-truth labels. It allows for some flexibility in the detection of
309
- anomalies that are temporally close to the true events. The final metric integrates the PR-AUC
310
- over several levels of temporal tolerance (from 0 to `window`), yielding a volume under the PR surface.
311
-
312
- Implementation of https://link.springer.com/article/10.1007/s10618-023-00988-8
313
-
314
- For more information, see the original paper:
315
- https://dl.acm.org/doi/10.14778/3551793.3551830
316
-
317
- Parameters:
318
- y_true (np.array):
319
- Ground-truth binary labels (0 = normal, 1 = anomaly).
320
- y_anomaly_scores (np.array):
321
- Anomaly scores for each time point.
322
- window (int):
323
- Maximum temporal tolerance `l` used to smooth the evaluation.
324
-
325
- Returns:
326
- float: VUS-PR score.
327
-
328
-
329
- """
330
- validate_non_binary_inputs(y_true, y_anomaly_scores)
331
-
332
- m = VUS_PR(y_true,y_anomaly_scores,max_window=window)
333
-
334
- return m.get_score()
335
-
336
-
337
- def real_pate(y_true: np.array, y_anomaly_scores: np.array, early: int, delay: int):
338
- """
339
- Calculate PATE score for anomaly detection in time series using real-valued anomaly scores.
340
-
341
- This version of PATE evaluates real-valued anomaly scores by assigning weights to predictions
342
- based on their temporal proximity to the true anomaly intervals. It defines an early buffer of
343
- length `early` before each anomaly and a delay buffer of length `delay` after it. Detections with
344
- high scores within the anomaly interval receive full weight, while those in the buffer zones are
345
- assigned linearly decaying weights depending on their distance from the interval. Scores outside
346
- these zones contribute to false positives, and intervals with insufficient detection are penalized
347
- as false negatives.
348
-
349
- The final PATE score aggregates these weighted contributions across all time steps, yielding
350
- a smooth performance measure that is sensitive to both the timing and confidence of the predictions.
351
-
352
- Implementation of https://arxiv.org/abs/2405.12096
353
-
354
- For more information, see the original paper:
355
- https://arxiv.org/abs/2405.12096
356
-
357
- Parameters:
358
- y_true (np.array):
359
- Ground truth binary labels (0 = normal, 1 = anomaly).
360
- y_anomaly_scores (np.array):
361
- Real-valued anomaly scores for each time point.
362
- early (int):
363
- Length of the early buffer zone before each anomaly interval.
364
- delay (int):
365
- Length of the delay buffer zone after each anomaly interval.
366
-
367
- Returns:
368
- float: The real-valued PATE score.
369
- """
370
- validate_non_binary_inputs(y_true, y_anomaly_scores)
371
-
372
- return PATE(y_true, y_anomaly_scores, early, delay, binary_scores=False)
File without changes
@@ -1,42 +0,0 @@
1
- #!/usr/bin/env python3
2
- import argparse
3
- from tsadmetrics.utils import compute_metrics_from_file
4
-
5
-
6
- def main():
7
-
8
- parser = argparse.ArgumentParser(
9
- description='Compute metrics from anomaly detection results and configuration files.'
10
- )
11
-
12
-
13
- parser.add_argument(
14
- '--res_file',
15
- type=str,
16
- required=True,
17
- help='Path to the results CSV file (e.g., results.csv)'
18
- )
19
- parser.add_argument(
20
- '--conf_file',
21
- type=str,
22
- required=True,
23
- help='Path to the configuration JSON file (e.g., conf.json)'
24
- )
25
- parser.add_argument(
26
- '--output_dir',
27
- type=str,
28
- required=True,
29
- help='Directory where output files will be saved (e.g., ./output_dir)'
30
- )
31
-
32
- args = parser.parse_args()
33
-
34
-
35
- compute_metrics_from_file(
36
- results_file=args.res_file,
37
- conf_file=args.conf_file,
38
- output_dir=args.output_dir
39
- )
40
-
41
- if __name__ == '__main__':
42
- main()
tsadmetrics/utils.py DELETED
@@ -1,124 +0,0 @@
1
- import numpy as np
2
- import pandas as pd
3
- import time
4
- import sys
5
- import tsadmetrics
6
- def compute_metrics(y_true: np.array,y_pred: np.array, metrics: list, metrics_params: dict, is_anomaly_score = False, verbose = False):
7
- """
8
- Computes the specified metrics for the given true and predicted values.
9
-
10
- Parameters:
11
- y_true (np.array):
12
- True labels.
13
- y_pred (np.array):
14
- Predicted labels or scores.
15
- metrics (list):
16
- List of metric names to compute.
17
- metrics_params (dict):
18
- Dictionary of parameters for each metric.
19
- is_anomaly_score (bool):
20
- Flag indicating if y_true and y_pred are anomaly scores. Otherwise, they are treated as binary labels.
21
- verbose (bool):
22
- Flag to print additional information.
23
- Returns:
24
- results (DataFrame): DataFrame containing the computed metrics and their values.
25
- """
26
- if not (np.array_equal(np.unique(y_true), [0, 1]) or np.array_equal(np.unique(y_true), [0]) or np.array_equal(np.unique(y_true), [1])):
27
- raise ValueError("y_true must be binary labels (0 or 1).")
28
- if not is_anomaly_score:
29
- #Chech if y_true and y_pred are binary labels
30
- if not ( np.array_equal(np.unique(y_pred), [0, 1])):
31
- raise ValueError("y_true and y_pred must be binary labels (0 or 1) when is_anomaly_score is False. Which is the default.")
32
- else:
33
- # Check if y_true and y_pred are anomaly scores
34
- if not (np.all((y_pred >= 0) & (y_pred <= 1))):
35
- raise ValueError("y_true must be binary labels (0 or 1), and y_pred must be anomaly scores in the range [0, 1] when is_anomaly_score is True.")
36
- results = {}
37
-
38
- for metric in metrics:
39
- metric_name = metric[0]
40
- metric_func = metric[1]
41
- if verbose:
42
- print(f"Calculating metric: {metric_name}")
43
- t0 = time.time()
44
- metric_value = metric_func(y_true, y_pred, **metrics_params.get(metric_name, {}))
45
- if verbose:
46
- t1 = time.time()
47
- print(f"Metric {metric_name} calculated in {t1 - t0:.4f} seconds")
48
- print(f"Metric {metric_name} value: {metric_value}")
49
- # Store the result in the DataFrame
50
- results[metric_name] = metric_value
51
-
52
- metrics_df = pd.DataFrame(columns=['metric_name', 'metric_value'])
53
- metrics_df['metric_name'] = results.keys()
54
- metrics_df['metric_value'] = results.values()
55
-
56
- return metrics_df
57
-
58
-
59
- def compute_metrics_from_file(results_file: str, conf_file: str, output_dir: str = '.'):
60
- """
61
- Computes metrics based on prediction results from a CSV file and configuration from a JSON file.
62
-
63
- Parameters:
64
- results_file (str):
65
- Path to CSV file containing y_true and y_pred columns.
66
- conf_file (str):
67
- Path to JSON configuration file with metrics and parameters.
68
-
69
- Returns:
70
- pd.DataFrame: DataFrame with computed metrics.
71
- """
72
- # Read results data
73
- res = pd.read_csv(results_file)
74
- y_true = res['y_true'].values
75
- y_pred = res['y_pred'].values
76
-
77
- # Determine if predictions are binary or scores
78
- is_anomaly_score = False
79
- unique_values = np.unique(y_pred)
80
- if not (np.array_equal(unique_values, [0, 1]) or
81
- np.array_equal(unique_values, [0]) or
82
- np.array_equal(unique_values, [1])):
83
- is_anomaly_score = True
84
- if not np.all((y_pred >= 0) & (y_pred <= 1)):
85
- raise ValueError("y_pred must be either binary (0/1) or anomaly scores in range [0, 1]")
86
-
87
- # Read configuration from JSON using pandas
88
- try:
89
- config_df = pd.read_json(conf_file, orient='records')
90
- except ValueError as e:
91
- raise ValueError(f"Invalid JSON format in configuration file: {str(e)}")
92
-
93
- # Convert pandas DataFrame to format expected by compute_metrics
94
- metrics = []
95
- metrics_params = {}
96
-
97
- for _, row in config_df.iterrows():
98
- metric_name = row['name']
99
- try:
100
- metric_func = getattr(tsadmetrics, metric_name)
101
- except AttributeError:
102
- raise ValueError(f"Metric function '{metric_name}' not found in tsadmetrics module")
103
-
104
- metrics.append((metric_name, metric_func))
105
-
106
- # Handle params (convert from pandas Series to dict if needed)
107
- params = row.get('params', {})
108
- if pd.notna(params) and params: # Check for non-empty params
109
- if isinstance(params, pd.Series):
110
- metrics_params[metric_name] = params.to_dict()
111
- else:
112
- metrics_params[metric_name] = params
113
-
114
- # Compute metrics
115
- metrics_df = compute_metrics(
116
- y_true=y_true,
117
- y_pred=y_pred,
118
- metrics=metrics,
119
- metrics_params=metrics_params,
120
- is_anomaly_score=is_anomaly_score,
121
- verbose=False
122
- )
123
- metrics_df.to_csv(output_dir+'/computed_metrics.csv', index=False)
124
-
tsadmetrics/validation.py DELETED
@@ -1,35 +0,0 @@
1
- import numpy as np
2
-
3
- def check_gt_binary_array(arr):
4
- if len(arr.shape) != 1:
5
- raise ValueError("Ground truth input must be a 1D binary array.")
6
- if not np.all(np.isin(arr, [0, 1])):
7
- raise ValueError("Ground truth input array must contain only binary values (0 or 1).")
8
- return True
9
-
10
- def check_pred_binary_array(arr):
11
- if len(arr.shape) != 1:
12
- raise ValueError("Prediction input must be a 1D binary array.")
13
- if not np.all(np.isin(arr, [0, 1])):
14
- raise ValueError("Prediction input array must contain only binary values (0 or 1).")
15
- return True
16
- def check_same_length(arr1, arr2):
17
- if len(arr1) != len(arr2):
18
- raise ValueError("Ground truth and prediction arrays must have the same length.")
19
- return True
20
-
21
- def check_pred_continuous_array(arr):
22
- if len(arr.shape) != 1:
23
- raise ValueError("Prediction input must be a 1D continuous array.")
24
- if not np.all((arr >= 0) & (arr <= 1)):
25
- raise ValueError("All values in the array must be in the range [0, 1].")
26
- return True
27
-
28
- def validate_binary_inputs(y_true, y_pred):
29
- check_gt_binary_array(y_true)
30
- check_pred_binary_array(y_pred)
31
- check_same_length(y_true, y_pred)
32
-
33
- def validate_non_binary_inputs(y_true, y_anomaly_scores):
34
- check_gt_binary_array(y_true)
35
- check_same_length(y_true, y_anomaly_scores)