tsadmetrics 0.1.16__py3-none-any.whl → 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (148) hide show
  1. docs/api_doc/conf.py +67 -0
  2. docs/{conf.py → full_doc/conf.py} +1 -1
  3. docs/manual_doc/conf.py +67 -0
  4. examples/example_direct_data.py +28 -0
  5. examples/example_direct_single_data.py +25 -0
  6. examples/example_file_reference.py +24 -0
  7. examples/example_global_config_file.py +13 -0
  8. examples/example_metric_config_file.py +19 -0
  9. examples/example_simple_metric.py +8 -0
  10. examples/specific_examples/AbsoluteDetectionDistance_example.py +24 -0
  11. examples/specific_examples/AffiliationbasedFScore_example.py +24 -0
  12. examples/specific_examples/AverageDetectionCount_example.py +24 -0
  13. examples/specific_examples/CompositeFScore_example.py +24 -0
  14. examples/specific_examples/DelayThresholdedPointadjustedFScore_example.py +24 -0
  15. examples/specific_examples/DetectionAccuracyInRange_example.py +24 -0
  16. examples/specific_examples/EnhancedTimeseriesAwareFScore_example.py +24 -0
  17. examples/specific_examples/LatencySparsityawareFScore_example.py +24 -0
  18. examples/specific_examples/MeanTimeToDetect_example.py +24 -0
  19. examples/specific_examples/NabScore_example.py +24 -0
  20. examples/specific_examples/PateFScore_example.py +24 -0
  21. examples/specific_examples/Pate_example.py +24 -0
  22. examples/specific_examples/PointadjustedAtKFScore_example.py +24 -0
  23. examples/specific_examples/PointadjustedAucPr_example.py +24 -0
  24. examples/specific_examples/PointadjustedAucRoc_example.py +24 -0
  25. examples/specific_examples/PointadjustedFScore_example.py +24 -0
  26. examples/specific_examples/RangebasedFScore_example.py +24 -0
  27. examples/specific_examples/SegmentwiseFScore_example.py +24 -0
  28. examples/specific_examples/TemporalDistance_example.py +24 -0
  29. examples/specific_examples/TimeTolerantFScore_example.py +24 -0
  30. examples/specific_examples/TimeseriesAwareFScore_example.py +24 -0
  31. examples/specific_examples/TotalDetectedInRange_example.py +24 -0
  32. examples/specific_examples/VusPr_example.py +24 -0
  33. examples/specific_examples/VusRoc_example.py +24 -0
  34. examples/specific_examples/WeightedDetectionDifference_example.py +24 -0
  35. tests/test_dpm.py +212 -0
  36. tests/test_ptdm.py +366 -0
  37. tests/test_registry.py +58 -0
  38. tests/test_runner.py +185 -0
  39. tests/test_spm.py +213 -0
  40. tests/test_tmem.py +198 -0
  41. tests/test_tpdm.py +369 -0
  42. tests/test_tstm.py +338 -0
  43. tsadmetrics/__init__.py +0 -21
  44. tsadmetrics/base/Metric.py +188 -0
  45. tsadmetrics/evaluation/Report.py +25 -0
  46. tsadmetrics/evaluation/Runner.py +253 -0
  47. tsadmetrics/metrics/Registry.py +141 -0
  48. tsadmetrics/metrics/__init__.py +2 -0
  49. tsadmetrics/metrics/spm/PointwiseAucPr.py +62 -0
  50. tsadmetrics/metrics/spm/PointwiseAucRoc.py +63 -0
  51. tsadmetrics/metrics/spm/PointwiseFScore.py +86 -0
  52. tsadmetrics/metrics/spm/PrecisionAtK.py +81 -0
  53. tsadmetrics/metrics/spm/__init__.py +9 -0
  54. tsadmetrics/metrics/tem/dpm/DelayThresholdedPointadjustedFScore.py +83 -0
  55. tsadmetrics/metrics/tem/dpm/LatencySparsityawareFScore.py +76 -0
  56. tsadmetrics/metrics/tem/dpm/MeanTimeToDetect.py +47 -0
  57. tsadmetrics/metrics/tem/dpm/NabScore.py +60 -0
  58. tsadmetrics/metrics/tem/dpm/__init__.py +11 -0
  59. tsadmetrics/metrics/tem/ptdm/AverageDetectionCount.py +53 -0
  60. tsadmetrics/metrics/tem/ptdm/DetectionAccuracyInRange.py +66 -0
  61. tsadmetrics/metrics/tem/ptdm/PointadjustedAtKFScore.py +80 -0
  62. tsadmetrics/metrics/tem/ptdm/TimeseriesAwareFScore.py +248 -0
  63. tsadmetrics/metrics/tem/ptdm/TotalDetectedInRange.py +65 -0
  64. tsadmetrics/metrics/tem/ptdm/WeightedDetectionDifference.py +97 -0
  65. tsadmetrics/metrics/tem/ptdm/__init__.py +12 -0
  66. tsadmetrics/metrics/tem/tmem/AbsoluteDetectionDistance.py +48 -0
  67. tsadmetrics/metrics/tem/tmem/EnhancedTimeseriesAwareFScore.py +252 -0
  68. tsadmetrics/metrics/tem/tmem/TemporalDistance.py +68 -0
  69. tsadmetrics/metrics/tem/tmem/__init__.py +9 -0
  70. tsadmetrics/metrics/tem/tpdm/CompositeFScore.py +104 -0
  71. tsadmetrics/metrics/tem/tpdm/PointadjustedAucPr.py +123 -0
  72. tsadmetrics/metrics/tem/tpdm/PointadjustedAucRoc.py +119 -0
  73. tsadmetrics/metrics/tem/tpdm/PointadjustedFScore.py +96 -0
  74. tsadmetrics/metrics/tem/tpdm/RangebasedFScore.py +236 -0
  75. tsadmetrics/metrics/tem/tpdm/SegmentwiseFScore.py +73 -0
  76. tsadmetrics/metrics/tem/tpdm/__init__.py +12 -0
  77. tsadmetrics/metrics/tem/tstm/AffiliationbasedFScore.py +68 -0
  78. tsadmetrics/metrics/tem/tstm/Pate.py +62 -0
  79. tsadmetrics/metrics/tem/tstm/PateFScore.py +61 -0
  80. tsadmetrics/metrics/tem/tstm/TimeTolerantFScore.py +85 -0
  81. tsadmetrics/metrics/tem/tstm/VusPr.py +51 -0
  82. tsadmetrics/metrics/tem/tstm/VusRoc.py +55 -0
  83. tsadmetrics/metrics/tem/tstm/__init__.py +15 -0
  84. tsadmetrics/{_tsadeval/affiliation/_integral_interval.py → utils/functions_affiliation.py} +377 -9
  85. tsadmetrics/utils/functions_auc.py +393 -0
  86. tsadmetrics/utils/functions_conversion.py +63 -0
  87. tsadmetrics/utils/functions_counting_metrics.py +26 -0
  88. tsadmetrics/{_tsadeval/latency_sparsity_aware.py → utils/functions_latency_sparsity_aware.py} +1 -1
  89. tsadmetrics/{_tsadeval/nabscore.py → utils/functions_nabscore.py} +15 -1
  90. tsadmetrics-1.0.0.dist-info/METADATA +69 -0
  91. tsadmetrics-1.0.0.dist-info/RECORD +99 -0
  92. {tsadmetrics-0.1.16.dist-info → tsadmetrics-1.0.0.dist-info}/top_level.txt +1 -1
  93. entorno/bin/activate_this.py +0 -32
  94. entorno/bin/rst2html.py +0 -23
  95. entorno/bin/rst2html4.py +0 -26
  96. entorno/bin/rst2html5.py +0 -33
  97. entorno/bin/rst2latex.py +0 -26
  98. entorno/bin/rst2man.py +0 -27
  99. entorno/bin/rst2odt.py +0 -28
  100. entorno/bin/rst2odt_prepstyles.py +0 -20
  101. entorno/bin/rst2pseudoxml.py +0 -23
  102. entorno/bin/rst2s5.py +0 -24
  103. entorno/bin/rst2xetex.py +0 -27
  104. entorno/bin/rst2xml.py +0 -23
  105. entorno/bin/rstpep2html.py +0 -25
  106. tests/test_binary.py +0 -946
  107. tests/test_non_binary.py +0 -420
  108. tests/test_utils.py +0 -49
  109. tsadmetrics/_tsadeval/affiliation/_affiliation_zone.py +0 -86
  110. tsadmetrics/_tsadeval/affiliation/_single_ground_truth_event.py +0 -68
  111. tsadmetrics/_tsadeval/affiliation/generics.py +0 -135
  112. tsadmetrics/_tsadeval/affiliation/metrics.py +0 -114
  113. tsadmetrics/_tsadeval/auc_roc_pr_plot.py +0 -295
  114. tsadmetrics/_tsadeval/discontinuity_graph.py +0 -109
  115. tsadmetrics/_tsadeval/eTaPR_pkg/DataManage/File_IO.py +0 -175
  116. tsadmetrics/_tsadeval/eTaPR_pkg/DataManage/Range.py +0 -50
  117. tsadmetrics/_tsadeval/eTaPR_pkg/DataManage/Time_Plot.py +0 -184
  118. tsadmetrics/_tsadeval/eTaPR_pkg/__init__.py +0 -0
  119. tsadmetrics/_tsadeval/eTaPR_pkg/etapr.py +0 -386
  120. tsadmetrics/_tsadeval/eTaPR_pkg/tapr.py +0 -362
  121. tsadmetrics/_tsadeval/metrics.py +0 -698
  122. tsadmetrics/_tsadeval/prts/__init__.py +0 -0
  123. tsadmetrics/_tsadeval/prts/base/__init__.py +0 -0
  124. tsadmetrics/_tsadeval/prts/base/time_series_metrics.py +0 -165
  125. tsadmetrics/_tsadeval/prts/basic_metrics_ts.py +0 -121
  126. tsadmetrics/_tsadeval/prts/time_series_metrics/__init__.py +0 -0
  127. tsadmetrics/_tsadeval/prts/time_series_metrics/fscore.py +0 -61
  128. tsadmetrics/_tsadeval/prts/time_series_metrics/precision.py +0 -86
  129. tsadmetrics/_tsadeval/prts/time_series_metrics/precision_recall.py +0 -21
  130. tsadmetrics/_tsadeval/prts/time_series_metrics/recall.py +0 -85
  131. tsadmetrics/_tsadeval/tests.py +0 -376
  132. tsadmetrics/_tsadeval/threshold_plt.py +0 -30
  133. tsadmetrics/_tsadeval/time_tolerant.py +0 -33
  134. tsadmetrics/binary_metrics.py +0 -1652
  135. tsadmetrics/metric_utils.py +0 -98
  136. tsadmetrics/non_binary_metrics.py +0 -398
  137. tsadmetrics/scripts/__init__.py +0 -0
  138. tsadmetrics/scripts/compute_metrics.py +0 -42
  139. tsadmetrics/utils.py +0 -122
  140. tsadmetrics/validation.py +0 -35
  141. tsadmetrics-0.1.16.dist-info/METADATA +0 -23
  142. tsadmetrics-0.1.16.dist-info/RECORD +0 -64
  143. tsadmetrics-0.1.16.dist-info/entry_points.txt +0 -2
  144. /tsadmetrics/{_tsadeval → base}/__init__.py +0 -0
  145. /tsadmetrics/{_tsadeval/affiliation → evaluation}/__init__.py +0 -0
  146. /tsadmetrics/{_tsadeval/eTaPR_pkg/DataManage → metrics/tem}/__init__.py +0 -0
  147. /tsadmetrics/{_tsadeval/vus_utils.py → utils/functions_vus.py} +0 -0
  148. {tsadmetrics-0.1.16.dist-info → tsadmetrics-1.0.0.dist-info}/WHEEL +0 -0
@@ -1,98 +0,0 @@
1
- import numpy as np
2
- from ._tsadeval.metrics import pointwise_to_full_series, segmentwise_to_full_series
3
-
4
- def is_full_series(length: int, anomalies: np.array):
5
- # [1 0 1 1 0]
6
- return len(anomalies.shape) == 1 and len(anomalies) == length
7
-
8
- def is_pointwise(length: int, anomalies: np.array):
9
- # [0 2 3]
10
- return len(anomalies.shape) == 1 and len(anomalies) < length
11
-
12
- def is_segmentwise(length: int, anomalies: np.array):
13
- # [[0 0] [2 3]]
14
- return len(anomalies.shape) == 2
15
-
16
-
17
- def transform_to_full_series(length: int, anomalies: np.array):
18
- if is_full_series(length, anomalies):
19
- return anomalies
20
- elif is_pointwise(anomalies):
21
- return pointwise_to_full_series(anomalies, length)
22
- elif is_segmentwise(length, anomalies):
23
- return segmentwise_to_full_series(anomalies, length)
24
- else:
25
- raise ValueError(f"Illegal shape of anomalies:\n{anomalies}")
26
-
27
- def counting_method(y_true: np.array, y_pred: np.array, k: int):
28
- em,da,ma,fa = 0,0,0,0
29
- for i_gt in range(len(y_true)):
30
- i_pa = i_gt
31
- gt = y_true[i_gt]
32
- pa = y_pred[i_pa]
33
- if gt==1 and pa==1:
34
- em+=1
35
- elif gt==0 and pa==1:
36
- fa+=1
37
- elif gt==1 and pa==0:
38
- anom_range = y_pred[i_gt-k:i_pa+k+1]
39
- detected = False
40
- for r in anom_range:
41
- if r==1:
42
- em+=1
43
- detected=True
44
- break
45
- if not detected:
46
- ma+=1
47
- elif gt==0 and pa==0:
48
- pass
49
- # b = DelayThresholdedPointAdjust(len(y_true),y_true,y_pred,k=k)
50
- # da = b.tp-em
51
- # ma = b.fn
52
-
53
- return em,da,ma,fa
54
-
55
-
56
- #Range Based utils
57
-
58
- def cardinality(n_intersections,mode):
59
- if mode == 'one':
60
- return 1
61
- elif mode == 'reciprocal':
62
- if n_intersections==0:
63
- return 1
64
- else:
65
- return float(1/n_intersections)
66
- else:
67
- raise Exception("Error, wrong cardinality mode.")
68
-
69
-
70
- def size(anomaly_range, overlap_set, position, bias):
71
- if overlap_set == None:
72
- return 0
73
-
74
- my_value = 0
75
- max_value = 0
76
- anomaly_length = anomaly_range[1] - anomaly_range[0] + 1
77
- for i in range(1,anomaly_length+1):
78
- bias_value = position(i, anomaly_length,bias)
79
- max_value += bias_value
80
- if anomaly_range[0]+i-1 >= overlap_set[0] and anomaly_range[0]+i-1 <= overlap_set[1]:
81
- my_value += bias_value
82
- return my_value / max_value
83
-
84
- def position(i, anomaly_length,bias):
85
- if bias == 'flat':
86
- return 1
87
- elif bias == 'front-end':
88
- return anomaly_length - i + 1
89
- elif bias == 'back-end':
90
- return i
91
- elif bias == 'middle':
92
- if i <= anomaly_length / 2:
93
- return i
94
- else:
95
- return anomaly_length - i + 1
96
- else:
97
- raise Exception("Error, wrong bias value.")
98
-
@@ -1,398 +0,0 @@
1
- import numpy as np
2
- from ._tsadeval.metrics import *
3
- from .validation import validate_non_binary_inputs
4
- from sklearn.metrics import auc
5
- from pate.PATE_metric import PATE
6
- def precision_at_k(y_true : np.array, y_anomaly_scores: np.array):
7
- """
8
- Calculate the precision at k score for anomaly detection in time series.
9
-
10
- This metric evaluates how many of the top-k points (with highest anomaly scores)
11
- actually correspond to true anomalies. It is particularly useful when we are
12
- interested in identifying the most anomalous points, without needing to set a
13
- fixed threshold.
14
-
15
- The value of k is automatically set to the number of true anomalies present in
16
- y_true. That is, k = sum(y_true).
17
-
18
- Implementation of https://link.springer.com/article/10.1007/s10618-023-00988-8
19
-
20
- Parameters:
21
- y_true (np.array):
22
- The ground truth binary labels for the time series data.
23
- y_anomaly_scores (np.array):
24
- The predicted anomaly scores for the time series data.
25
-
26
- Returns:
27
- float: The precision at k score.
28
- """
29
- validate_non_binary_inputs(y_true, y_anomaly_scores)
30
-
31
- m = PatK_pw(y_true,y_anomaly_scores)
32
-
33
- return m.get_score()
34
-
35
- def auc_roc_pw(y_true : np.array, y_anomaly_scores: np.array):
36
- """
37
- Calculate the AUC-ROC score for anomaly detection in time series.
38
-
39
- This is the standard Area Under the Receiver Operating Characteristic Curve (AUC-ROC),
40
- computed in a point-wise manner. That is, each point in the time series is treated
41
- independently when calculating true positives, false positives, and false negatives.
42
-
43
- Implementation of https://link.springer.com/article/10.1007/s10618-023-00988-8
44
-
45
- Parameters:
46
- y_true (np.array):
47
- Ground-truth binary labels for the time series (0 = normal, 1 = anomaly).
48
- y_anomaly_scores (np.array):
49
- Continuous anomaly scores assigned to each point in the series.
50
-
51
- Returns:
52
- float: AUC-ROC score.
53
- """
54
- validate_non_binary_inputs(y_true, y_anomaly_scores)
55
-
56
- m = AUC_ROC(y_true,y_anomaly_scores)
57
-
58
- return m.get_score()
59
-
60
-
61
- def auc_pr_pw(y_true : np.array ,y_anomaly_scores: np.array):
62
- """
63
- Calculate the AUC-PR score for anomaly detection in time series.
64
-
65
- This is the standard Area Under the Precision-Recall Curve (AUC-PR),
66
- computed in a point-wise manner. That is, each point in the time series is treated
67
- independently when calculating precision and recall.
68
-
69
- Implementation of https://link.springer.com/article/10.1007/s10618-023-00988-8
70
-
71
- Parameters:
72
- y_true (np.array):
73
- Ground-truth binary labels for the time series (0 = normal, 1 = anomaly).
74
- y_anomaly_scores (np.array):
75
- Continuous anomaly scores assigned to each point in the series.
76
-
77
- Returns:
78
- float: AUC-PR score.
79
- """
80
- validate_non_binary_inputs(y_true, y_anomaly_scores)
81
-
82
- m = AUC_PR_pw(y_true,y_anomaly_scores)
83
-
84
- return m.get_score()
85
-
86
-
87
-
88
- def auc_pr_pa(y_true: np.array, y_anomaly_scores: np.array):
89
- """
90
- Calculate the AUC-PR score using point-adjusted evaluation for anomaly detection in time series.
91
-
92
- This is the standard Area Under the Precision-Recall Curve (AUC-PR), but instead of computing
93
- precision and recall point-wise, it uses a point-adjusted approach. Specifically, for each
94
- ground-truth anomalous segment, if at least one point within that segment is predicted as anomalous,
95
- the entire segment is considered correctly detected. The adjusted predictions are then compared
96
- to the ground-truth labels to compute true positives, false positives, and false negatives,
97
- which are used to construct the PR curve.
98
-
99
- Implementation of https://link.springer.com/article/10.1007/s10618-023-00988-8
100
-
101
- Parameters:
102
- y_true (np.array):
103
- Ground-truth binary labels for the time series (0 = normal, 1 = anomaly).
104
- y_anomaly_scores (np.array):
105
- Continuous anomaly scores assigned to each point in the series.
106
-
107
- Returns:
108
- float: AUC-PR score (with point-adjusted evaluation).
109
- """
110
- validate_non_binary_inputs(y_true, y_anomaly_scores)
111
-
112
- precisions = [1]
113
- recalls = [0]
114
- tps,fps,fns = [],[],[]
115
-
116
- p_adj = PointAdjust(len(y_true),y_true,(np.array(y_anomaly_scores) >= 0.5).astype(int))
117
- segments= p_adj.get_gt_anomalies_segmentwise()
118
- idx = np.argsort(y_anomaly_scores)[::-1].astype(int)
119
- y_true_sorted = np.array(y_true)[idx]
120
- y_anomaly_scores_sorted = np.array(y_anomaly_scores)[idx]
121
-
122
- segment_mins = []
123
- for start,end in segments:
124
- anoms_scores = y_anomaly_scores[start:end+1]
125
- segment_mins.append([np.max(anoms_scores),end-start+1])
126
-
127
- for i_t in range(len(y_anomaly_scores_sorted)):
128
- fp,tp,fn = 0,0,0
129
- if i_t > 0 and y_anomaly_scores_sorted[i_t] == y_anomaly_scores_sorted[i_t-1] :
130
- tp = tps[-1]
131
- fp = fps[-1]
132
- fn = fns[-1]
133
- else:
134
- if y_true_sorted[i_t] == 0:
135
- #FP
136
- if len(fps)==0:
137
- aux_y_pred = (y_anomaly_scores >= y_anomaly_scores_sorted[i_t]).astype(int)
138
- for i in range(len(aux_y_pred)):
139
- if aux_y_pred[i] == 1 and y_true[i] == 0:
140
- fp+=1
141
-
142
-
143
- else:
144
- fp=fps[i_t-1]+1
145
- else:
146
- if len(fps)==0:
147
- aux_y_pred = (y_anomaly_scores >= y_anomaly_scores_sorted[i_t]).astype(int)
148
- for i in range(len(aux_y_pred)):
149
- if aux_y_pred[i] == 1 and y_true[i] == 0:
150
- fp+=1
151
- else:
152
- fp=fps[i_t-1]
153
- for score, length in segment_mins:
154
- if score >= y_anomaly_scores_sorted[i_t]:
155
- #TP
156
- tp+= length
157
- else:
158
- #FN
159
- fn+= length
160
- tps.append(tp)
161
- fns.append(fn)
162
- fps.append(fp)
163
- for tp,fp,fn in zip(tps,fps,fns):
164
- if tp>0:
165
- precisions.append(tp/(tp+fp))
166
- recalls.append(tp/(tp+fn))
167
- else:
168
- precisions.append(0)
169
- recalls.append(0)
170
-
171
-
172
- recalls.append(1)
173
- precisions.append(0)
174
-
175
- auc_value = auc(recalls, precisions)
176
- return auc_value
177
-
178
-
179
-
180
-
181
- def auc_pr_sw(y_true: np.array, y_anomaly_scores: np.array):
182
- """
183
- Calculate the AUC-PR score using segment-wise evaluation for anomaly detection in time series.
184
-
185
- This is the standard Area Under the Precision-Recall Curve (AUC-PR), but it uses a segment-wise
186
- adjustment when computing precision and recall. In this evaluation, each contiguous segment of
187
- anomalous ground-truth points is treated as a single unit. A true positive is counted if at least
188
- one predicted anomaly overlaps with the segment. A false negative occurs when a segment is
189
- completely missed, and a false positive is recorded for each predicted anomalous segment
190
- that does not overlap with any ground-truth anomaly. These adjusted counts are then used
191
- to compute precision and recall for constructing the PR curve.
192
-
193
- Implementation of https://link.springer.com/article/10.1007/s10618-023-00988-8
194
-
195
- Parameters:
196
- y_true (np.array):
197
- Ground-truth binary labels for the time series (0 = normal, 1 = anomaly).
198
- y_anomaly_scores (np.array):
199
- Continuous anomaly scores assigned to each point in the series.
200
-
201
- Returns:
202
- float: AUC-PR score (with segment-wise evaluation).
203
- """
204
- validate_non_binary_inputs(y_true, y_anomaly_scores)
205
-
206
- precisions = [1]
207
- recalls = [0]
208
- tps,fps,fns = [],[],[]
209
-
210
-
211
- segments = []
212
- i=0
213
- while i < len(y_true):
214
- if y_true[i] == 1:
215
- start = i
216
- end = i
217
- while i < len(y_true) and y_true[i] == 1:
218
- end = i
219
- i += 1
220
- segments.append([start,end])
221
- i+=1
222
- idx = np.argsort(y_anomaly_scores)[::-1].astype(int)
223
- y_anomaly_scores_sorted = np.array(y_anomaly_scores)[idx]
224
-
225
- segment_mins = []
226
- for start,end in segments:
227
- anoms_scores = y_anomaly_scores[start:end+1]
228
- segment_mins.append([np.max(anoms_scores),[start,end]])
229
-
230
- for i_t in range(len(y_anomaly_scores_sorted)):
231
- fp,tp,fn = 0,0,0
232
-
233
-
234
- aux_y_pred = (y_anomaly_scores >= y_anomaly_scores_sorted[i_t]).astype(int)
235
- for score,seg in segment_mins:
236
- start,end = seg
237
- if score >= y_anomaly_scores_sorted[i_t]:
238
- #TP
239
- tp+= 1
240
- if aux_y_pred[start]== 1:
241
- # Extender hacia la izquierda
242
- i = start - 1
243
- while i >= 0 and aux_y_pred[i] == 1:
244
- aux_y_pred[i] = 0
245
- i -= 1
246
-
247
- if aux_y_pred[end] == 1:
248
- # Extender hacia la derecha
249
- i = end + 1
250
- while i < len(aux_y_pred) and aux_y_pred[i] == 1:
251
- aux_y_pred[i] = 0
252
- i += 1
253
- aux_y_pred[start:end+1] = 0
254
-
255
- else:
256
- #FN
257
- fn+= 1
258
-
259
- if np.sum(aux_y_pred)>0:
260
- fpsegments = []
261
- i=0
262
- while i < len(aux_y_pred):
263
- if aux_y_pred[i] == 1:
264
- start = i
265
- end = i
266
- while i < len(aux_y_pred) and aux_y_pred[i] == 1:
267
- end = i
268
- i += 1
269
- fpsegments.append([start,end])
270
- i+=1
271
- fp = len(fpsegments)
272
- else:
273
- fp = 0
274
-
275
-
276
- tps.append(tp)
277
- fns.append(fn)
278
- fps.append(fp)
279
- for tp,fp,fn in zip(tps,fps,fns):
280
- if tp>0:
281
- precisions.append(tp/(tp+fp))
282
- recalls.append(tp/(tp+fn))
283
- else:
284
- precisions.append(0)
285
- recalls.append(0)
286
-
287
-
288
-
289
- auc_value = auc(recalls, precisions)
290
-
291
- return auc_value
292
-
293
-
294
- def vus_roc(y_true : np.array ,y_anomaly_scores: np.array, window=4):
295
- """
296
- Calculate the VUS-ROC (Volume Under the ROC Surface) score for anomaly detection in time series.
297
-
298
- This metric extends the classical AUC-ROC by introducing a temporal tolerance parameter `l`, which
299
- smooths the binary ground-truth labels. The idea is to allow a flexible evaluation that tolerates
300
- small misalignments in the detection of anomalies. The final score is computed by integrating
301
- the ROC-AUC over different values of the tolerance parameter, from 0 to `window`, thus producing
302
- a volume under the ROC surface.
303
-
304
- Implementation of https://link.springer.com/article/10.1007/s10618-023-00988-8
305
-
306
- For more information, see the original paper:
307
- https://dl.acm.org/doi/10.14778/3551793.3551830
308
-
309
- Parameters:
310
- y_true (np.array):
311
- Ground-truth binary labels (0 = normal, 1 = anomaly).
312
- y_anomaly_scores (np.array):
313
- Anomaly scores for each time point.
314
- window (int):
315
- Maximum temporal tolerance `l` used to smooth the evaluation.
316
-
317
- Returns:
318
- float: VUS-ROC score.
319
-
320
-
321
- """
322
- validate_non_binary_inputs(y_true, y_anomaly_scores)
323
-
324
- m = VUS_ROC(y_true,y_anomaly_scores,max_window=window)
325
-
326
- return m.get_score()
327
-
328
-
329
- def vus_pr(y_true : np.array ,y_anomaly_scores: np.array, window=4):
330
- """
331
- Calculate the VUS-PR (Volume Under the PR Surface) score for anomaly detection in time series.
332
-
333
- This metric is an extension of the classical AUC-PR, incorporating a temporal tolerance parameter `l`
334
- that smooths the binary ground-truth labels. It allows for some flexibility in the detection of
335
- anomalies that are temporally close to the true events. The final metric integrates the PR-AUC
336
- over several levels of temporal tolerance (from 0 to `window`), yielding a volume under the PR surface.
337
-
338
- Implementation of https://link.springer.com/article/10.1007/s10618-023-00988-8
339
-
340
- For more information, see the original paper:
341
- https://dl.acm.org/doi/10.14778/3551793.3551830
342
-
343
- Parameters:
344
- y_true (np.array):
345
- Ground-truth binary labels (0 = normal, 1 = anomaly).
346
- y_anomaly_scores (np.array):
347
- Anomaly scores for each time point.
348
- window (int):
349
- Maximum temporal tolerance `l` used to smooth the evaluation.
350
-
351
- Returns:
352
- float: VUS-PR score.
353
-
354
-
355
- """
356
- validate_non_binary_inputs(y_true, y_anomaly_scores)
357
-
358
- m = VUS_PR(y_true,y_anomaly_scores,max_window=window)
359
-
360
- return m.get_score()
361
-
362
-
363
- def real_pate(y_true: np.array, y_anomaly_scores: np.array, early: int, delay: int):
364
- """
365
- Calculate PATE score for anomaly detection in time series using real-valued anomaly scores.
366
-
367
- This version of PATE evaluates real-valued anomaly scores by assigning weights to predictions
368
- based on their temporal proximity to the true anomaly intervals. It defines an early buffer of
369
- length `early` before each anomaly and a delay buffer of length `delay` after it. Detections with
370
- high scores within the anomaly interval receive full weight, while those in the buffer zones are
371
- assigned linearly decaying weights depending on their distance from the interval. Scores outside
372
- these zones contribute to false positives, and intervals with insufficient detection are penalized
373
- as false negatives.
374
-
375
- The final PATE score aggregates these weighted contributions across all time steps, yielding
376
- a smooth performance measure that is sensitive to both the timing and confidence of the predictions.
377
-
378
- Implementation of https://arxiv.org/abs/2405.12096
379
-
380
- For more information, see the original paper:
381
- https://arxiv.org/abs/2405.12096
382
-
383
- Parameters:
384
- y_true (np.array):
385
- Ground truth binary labels (0 = normal, 1 = anomaly).
386
- y_anomaly_scores (np.array):
387
- Real-valued anomaly scores for each time point.
388
- early (int):
389
- Length of the early buffer zone before each anomaly interval.
390
- delay (int):
391
- Length of the delay buffer zone after each anomaly interval.
392
-
393
- Returns:
394
- float: The real-valued PATE score.
395
- """
396
- validate_non_binary_inputs(y_true, y_anomaly_scores)
397
-
398
- return PATE(y_true, y_anomaly_scores, early, delay, binary_scores=False)
File without changes
@@ -1,42 +0,0 @@
1
- #!/usr/bin/env python3
2
- import argparse
3
- from tsadmetrics.utils import compute_metrics_from_file
4
-
5
-
6
- def main():
7
-
8
- parser = argparse.ArgumentParser(
9
- description='Compute metrics from anomaly detection results and configuration files.'
10
- )
11
-
12
-
13
- parser.add_argument(
14
- '--res_file',
15
- type=str,
16
- required=True,
17
- help='Path to the results CSV file (e.g., results.csv)'
18
- )
19
- parser.add_argument(
20
- '--conf_file',
21
- type=str,
22
- required=True,
23
- help='Path to the configuration JSON file (e.g., conf.json)'
24
- )
25
- parser.add_argument(
26
- '--output_dir',
27
- type=str,
28
- required=True,
29
- help='Directory where output files will be saved (e.g., ./output_dir)'
30
- )
31
-
32
- args = parser.parse_args()
33
-
34
-
35
- compute_metrics_from_file(
36
- results_file=args.res_file,
37
- conf_file=args.conf_file,
38
- output_dir=args.output_dir
39
- )
40
-
41
- if __name__ == '__main__':
42
- main()
tsadmetrics/utils.py DELETED
@@ -1,122 +0,0 @@
1
- import numpy as np
2
- import pandas as pd
3
- import time
4
- import sys
5
- import tsadmetrics
6
- def compute_metrics(y_true: np.array,y_pred: np.array, metrics: list, metrics_params: dict, is_anomaly_score = False, verbose = False):
7
- """
8
- Computes the specified metrics for the given true and predicted values.
9
-
10
- Parameters:
11
- y_true (np.array):
12
- True labels.
13
- y_pred (np.array):
14
- Predicted labels or scores.
15
- metrics (list):
16
- List of metric names to compute.
17
- metrics_params (dict):
18
- Dictionary of parameters for each metric.
19
- is_anomaly_score (bool):
20
- Flag indicating if y_true and y_pred are anomaly scores. Otherwise, they are treated as binary labels.
21
- verbose (bool):
22
- Flag to print additional information.
23
- Returns:
24
- results (DataFrame): DataFrame containing the computed metrics and their values.
25
- """
26
- if not (np.array_equal(np.unique(y_true), [0, 1]) or np.array_equal(np.unique(y_true), [0]) or np.array_equal(np.unique(y_true), [1])):
27
- raise ValueError("y_true must be binary labels (0 or 1).")
28
- if not is_anomaly_score:
29
- #Chech if y_true and y_pred are binary labels
30
- if not ( np.array_equal(np.unique(y_pred), [0, 1])):
31
- raise ValueError("y_true and y_pred must be binary labels (0 or 1) when is_anomaly_score is False. Which is the default.")
32
- else:
33
- # Check if y_true and y_pred are anomaly scores
34
- if not (np.all((y_pred >= 0) & (y_pred <= 1))):
35
- raise ValueError("y_true must be binary labels (0 or 1), and y_pred must be anomaly scores in the range [0, 1] when is_anomaly_score is True.")
36
- results = {}
37
-
38
- for metric in metrics:
39
- metric_name = metric[0]
40
- metric_func = metric[1]
41
- if verbose:
42
- print(f"Calculating metric: {metric_name}")
43
- t0 = time.time()
44
- metric_value = metric_func(y_true, y_pred, **metrics_params.get(metric_name, {}))
45
- if verbose:
46
- t1 = time.time()
47
- print(f"Metric {metric_name} calculated in {t1 - t0:.4f} seconds")
48
- print(f"Metric {metric_name} value: {metric_value}")
49
- # Store the result in the DataFrame
50
- results[metric_name] = metric_value
51
-
52
- metrics_df = pd.DataFrame(columns=['metric_name', 'metric_value'])
53
- metrics_df['metric_name'] = results.keys()
54
- metrics_df['metric_value'] = results.values()
55
-
56
- return metrics_df
57
-
58
-
59
- def compute_metrics_from_file(results_file: str, conf_file: str, output_dir: str = '.'):
60
- """
61
- Computes metrics based on prediction results from a CSV file and configuration from a JSON file.
62
-
63
- Parameters:
64
- results_file (str): Path to CSV file containing y_true and y_pred columns.
65
- conf_file (str): Path to JSON configuration file with metrics and parameters.
66
-
67
- Returns:
68
- pd.DataFrame: DataFrame with computed metrics.
69
- """
70
- # Read results data
71
- res = pd.read_csv(results_file)
72
- y_true = res['y_true'].values
73
- y_pred = res['y_pred'].values
74
-
75
- # Determine if predictions are binary or scores
76
- is_anomaly_score = False
77
- unique_values = np.unique(y_pred)
78
- if not (np.array_equal(unique_values, [0, 1]) or
79
- np.array_equal(unique_values, [0]) or
80
- np.array_equal(unique_values, [1])):
81
- is_anomaly_score = True
82
- if not np.all((y_pred >= 0) & (y_pred <= 1)):
83
- raise ValueError("y_pred must be either binary (0/1) or anomaly scores in range [0, 1]")
84
-
85
- # Read configuration from JSON using pandas
86
- try:
87
- config_df = pd.read_json(conf_file, orient='records')
88
- except ValueError as e:
89
- raise ValueError(f"Invalid JSON format in configuration file: {str(e)}")
90
-
91
- # Convert pandas DataFrame to format expected by compute_metrics
92
- metrics = []
93
- metrics_params = {}
94
-
95
- for _, row in config_df.iterrows():
96
- metric_name = row['name']
97
- try:
98
- metric_func = getattr(tsadmetrics, metric_name)
99
- except AttributeError:
100
- raise ValueError(f"Metric function '{metric_name}' not found in tsadmetrics module")
101
-
102
- metrics.append((metric_name, metric_func))
103
-
104
- # Handle params (convert from pandas Series to dict if needed)
105
- params = row.get('params', {})
106
- if pd.notna(params) and params: # Check for non-empty params
107
- if isinstance(params, pd.Series):
108
- metrics_params[metric_name] = params.to_dict()
109
- else:
110
- metrics_params[metric_name] = params
111
-
112
- # Compute metrics
113
- metrics_df = compute_metrics(
114
- y_true=y_true,
115
- y_pred=y_pred,
116
- metrics=metrics,
117
- metrics_params=metrics_params,
118
- is_anomaly_score=is_anomaly_score,
119
- verbose=False
120
- )
121
- metrics_df.to_csv(output_dir+'/computed_metrics.csv', index=False)
122
-
tsadmetrics/validation.py DELETED
@@ -1,35 +0,0 @@
1
- import numpy as np
2
-
3
- def check_gt_binary_array(arr):
4
- if len(arr.shape) != 1:
5
- raise ValueError("Ground truth input must be a 1D binary array.")
6
- if not np.all(np.isin(arr, [0, 1])):
7
- raise ValueError("Ground truth input array must contain only binary values (0 or 1).")
8
- return True
9
-
10
- def check_pred_binary_array(arr):
11
- if len(arr.shape) != 1:
12
- raise ValueError("Prediction input must be a 1D binary array.")
13
- if not np.all(np.isin(arr, [0, 1])):
14
- raise ValueError("Prediction input array must contain only binary values (0 or 1).")
15
- return True
16
- def check_same_length(arr1, arr2):
17
- if len(arr1) != len(arr2):
18
- raise ValueError("Ground truth and prediction arrays must have the same length.")
19
- return True
20
-
21
- def check_pred_continuous_array(arr):
22
- if len(arr.shape) != 1:
23
- raise ValueError("Prediction input must be a 1D continuous array.")
24
- if not np.all((arr >= 0) & (arr <= 1)):
25
- raise ValueError("All values in the array must be in the range [0, 1].")
26
- return True
27
-
28
- def validate_binary_inputs(y_true, y_pred):
29
- check_gt_binary_array(y_true)
30
- check_pred_binary_array(y_pred)
31
- check_same_length(y_true, y_pred)
32
-
33
- def validate_non_binary_inputs(y_true, y_anomaly_scores):
34
- check_gt_binary_array(y_true)
35
- check_same_length(y_true, y_anomaly_scores)