tsadmetrics 0.1.16__py3-none-any.whl → 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- docs/api_doc/conf.py +67 -0
- docs/{conf.py → full_doc/conf.py} +1 -1
- docs/manual_doc/conf.py +67 -0
- examples/example_direct_data.py +28 -0
- examples/example_direct_single_data.py +25 -0
- examples/example_file_reference.py +24 -0
- examples/example_global_config_file.py +13 -0
- examples/example_metric_config_file.py +19 -0
- examples/example_simple_metric.py +8 -0
- examples/specific_examples/AbsoluteDetectionDistance_example.py +24 -0
- examples/specific_examples/AffiliationbasedFScore_example.py +24 -0
- examples/specific_examples/AverageDetectionCount_example.py +24 -0
- examples/specific_examples/CompositeFScore_example.py +24 -0
- examples/specific_examples/DelayThresholdedPointadjustedFScore_example.py +24 -0
- examples/specific_examples/DetectionAccuracyInRange_example.py +24 -0
- examples/specific_examples/EnhancedTimeseriesAwareFScore_example.py +24 -0
- examples/specific_examples/LatencySparsityawareFScore_example.py +24 -0
- examples/specific_examples/MeanTimeToDetect_example.py +24 -0
- examples/specific_examples/NabScore_example.py +24 -0
- examples/specific_examples/PateFScore_example.py +24 -0
- examples/specific_examples/Pate_example.py +24 -0
- examples/specific_examples/PointadjustedAtKFScore_example.py +24 -0
- examples/specific_examples/PointadjustedAucPr_example.py +24 -0
- examples/specific_examples/PointadjustedAucRoc_example.py +24 -0
- examples/specific_examples/PointadjustedFScore_example.py +24 -0
- examples/specific_examples/RangebasedFScore_example.py +24 -0
- examples/specific_examples/SegmentwiseFScore_example.py +24 -0
- examples/specific_examples/TemporalDistance_example.py +24 -0
- examples/specific_examples/TimeTolerantFScore_example.py +24 -0
- examples/specific_examples/TimeseriesAwareFScore_example.py +24 -0
- examples/specific_examples/TotalDetectedInRange_example.py +24 -0
- examples/specific_examples/VusPr_example.py +24 -0
- examples/specific_examples/VusRoc_example.py +24 -0
- examples/specific_examples/WeightedDetectionDifference_example.py +24 -0
- tests/test_dpm.py +212 -0
- tests/test_ptdm.py +366 -0
- tests/test_registry.py +58 -0
- tests/test_runner.py +185 -0
- tests/test_spm.py +213 -0
- tests/test_tmem.py +198 -0
- tests/test_tpdm.py +369 -0
- tests/test_tstm.py +338 -0
- tsadmetrics/__init__.py +0 -21
- tsadmetrics/base/Metric.py +188 -0
- tsadmetrics/evaluation/Report.py +25 -0
- tsadmetrics/evaluation/Runner.py +253 -0
- tsadmetrics/metrics/Registry.py +141 -0
- tsadmetrics/metrics/__init__.py +2 -0
- tsadmetrics/metrics/spm/PointwiseAucPr.py +62 -0
- tsadmetrics/metrics/spm/PointwiseAucRoc.py +63 -0
- tsadmetrics/metrics/spm/PointwiseFScore.py +86 -0
- tsadmetrics/metrics/spm/PrecisionAtK.py +81 -0
- tsadmetrics/metrics/spm/__init__.py +9 -0
- tsadmetrics/metrics/tem/dpm/DelayThresholdedPointadjustedFScore.py +83 -0
- tsadmetrics/metrics/tem/dpm/LatencySparsityawareFScore.py +76 -0
- tsadmetrics/metrics/tem/dpm/MeanTimeToDetect.py +47 -0
- tsadmetrics/metrics/tem/dpm/NabScore.py +60 -0
- tsadmetrics/metrics/tem/dpm/__init__.py +11 -0
- tsadmetrics/metrics/tem/ptdm/AverageDetectionCount.py +53 -0
- tsadmetrics/metrics/tem/ptdm/DetectionAccuracyInRange.py +66 -0
- tsadmetrics/metrics/tem/ptdm/PointadjustedAtKFScore.py +80 -0
- tsadmetrics/metrics/tem/ptdm/TimeseriesAwareFScore.py +248 -0
- tsadmetrics/metrics/tem/ptdm/TotalDetectedInRange.py +65 -0
- tsadmetrics/metrics/tem/ptdm/WeightedDetectionDifference.py +97 -0
- tsadmetrics/metrics/tem/ptdm/__init__.py +12 -0
- tsadmetrics/metrics/tem/tmem/AbsoluteDetectionDistance.py +48 -0
- tsadmetrics/metrics/tem/tmem/EnhancedTimeseriesAwareFScore.py +252 -0
- tsadmetrics/metrics/tem/tmem/TemporalDistance.py +68 -0
- tsadmetrics/metrics/tem/tmem/__init__.py +9 -0
- tsadmetrics/metrics/tem/tpdm/CompositeFScore.py +104 -0
- tsadmetrics/metrics/tem/tpdm/PointadjustedAucPr.py +123 -0
- tsadmetrics/metrics/tem/tpdm/PointadjustedAucRoc.py +119 -0
- tsadmetrics/metrics/tem/tpdm/PointadjustedFScore.py +96 -0
- tsadmetrics/metrics/tem/tpdm/RangebasedFScore.py +236 -0
- tsadmetrics/metrics/tem/tpdm/SegmentwiseFScore.py +73 -0
- tsadmetrics/metrics/tem/tpdm/__init__.py +12 -0
- tsadmetrics/metrics/tem/tstm/AffiliationbasedFScore.py +68 -0
- tsadmetrics/metrics/tem/tstm/Pate.py +62 -0
- tsadmetrics/metrics/tem/tstm/PateFScore.py +61 -0
- tsadmetrics/metrics/tem/tstm/TimeTolerantFScore.py +85 -0
- tsadmetrics/metrics/tem/tstm/VusPr.py +51 -0
- tsadmetrics/metrics/tem/tstm/VusRoc.py +55 -0
- tsadmetrics/metrics/tem/tstm/__init__.py +15 -0
- tsadmetrics/{_tsadeval/affiliation/_integral_interval.py → utils/functions_affiliation.py} +377 -9
- tsadmetrics/utils/functions_auc.py +393 -0
- tsadmetrics/utils/functions_conversion.py +63 -0
- tsadmetrics/utils/functions_counting_metrics.py +26 -0
- tsadmetrics/{_tsadeval/latency_sparsity_aware.py → utils/functions_latency_sparsity_aware.py} +1 -1
- tsadmetrics/{_tsadeval/nabscore.py → utils/functions_nabscore.py} +15 -1
- tsadmetrics-1.0.0.dist-info/METADATA +69 -0
- tsadmetrics-1.0.0.dist-info/RECORD +99 -0
- {tsadmetrics-0.1.16.dist-info → tsadmetrics-1.0.0.dist-info}/top_level.txt +1 -1
- entorno/bin/activate_this.py +0 -32
- entorno/bin/rst2html.py +0 -23
- entorno/bin/rst2html4.py +0 -26
- entorno/bin/rst2html5.py +0 -33
- entorno/bin/rst2latex.py +0 -26
- entorno/bin/rst2man.py +0 -27
- entorno/bin/rst2odt.py +0 -28
- entorno/bin/rst2odt_prepstyles.py +0 -20
- entorno/bin/rst2pseudoxml.py +0 -23
- entorno/bin/rst2s5.py +0 -24
- entorno/bin/rst2xetex.py +0 -27
- entorno/bin/rst2xml.py +0 -23
- entorno/bin/rstpep2html.py +0 -25
- tests/test_binary.py +0 -946
- tests/test_non_binary.py +0 -420
- tests/test_utils.py +0 -49
- tsadmetrics/_tsadeval/affiliation/_affiliation_zone.py +0 -86
- tsadmetrics/_tsadeval/affiliation/_single_ground_truth_event.py +0 -68
- tsadmetrics/_tsadeval/affiliation/generics.py +0 -135
- tsadmetrics/_tsadeval/affiliation/metrics.py +0 -114
- tsadmetrics/_tsadeval/auc_roc_pr_plot.py +0 -295
- tsadmetrics/_tsadeval/discontinuity_graph.py +0 -109
- tsadmetrics/_tsadeval/eTaPR_pkg/DataManage/File_IO.py +0 -175
- tsadmetrics/_tsadeval/eTaPR_pkg/DataManage/Range.py +0 -50
- tsadmetrics/_tsadeval/eTaPR_pkg/DataManage/Time_Plot.py +0 -184
- tsadmetrics/_tsadeval/eTaPR_pkg/__init__.py +0 -0
- tsadmetrics/_tsadeval/eTaPR_pkg/etapr.py +0 -386
- tsadmetrics/_tsadeval/eTaPR_pkg/tapr.py +0 -362
- tsadmetrics/_tsadeval/metrics.py +0 -698
- tsadmetrics/_tsadeval/prts/__init__.py +0 -0
- tsadmetrics/_tsadeval/prts/base/__init__.py +0 -0
- tsadmetrics/_tsadeval/prts/base/time_series_metrics.py +0 -165
- tsadmetrics/_tsadeval/prts/basic_metrics_ts.py +0 -121
- tsadmetrics/_tsadeval/prts/time_series_metrics/__init__.py +0 -0
- tsadmetrics/_tsadeval/prts/time_series_metrics/fscore.py +0 -61
- tsadmetrics/_tsadeval/prts/time_series_metrics/precision.py +0 -86
- tsadmetrics/_tsadeval/prts/time_series_metrics/precision_recall.py +0 -21
- tsadmetrics/_tsadeval/prts/time_series_metrics/recall.py +0 -85
- tsadmetrics/_tsadeval/tests.py +0 -376
- tsadmetrics/_tsadeval/threshold_plt.py +0 -30
- tsadmetrics/_tsadeval/time_tolerant.py +0 -33
- tsadmetrics/binary_metrics.py +0 -1652
- tsadmetrics/metric_utils.py +0 -98
- tsadmetrics/non_binary_metrics.py +0 -398
- tsadmetrics/scripts/__init__.py +0 -0
- tsadmetrics/scripts/compute_metrics.py +0 -42
- tsadmetrics/utils.py +0 -122
- tsadmetrics/validation.py +0 -35
- tsadmetrics-0.1.16.dist-info/METADATA +0 -23
- tsadmetrics-0.1.16.dist-info/RECORD +0 -64
- tsadmetrics-0.1.16.dist-info/entry_points.txt +0 -2
- /tsadmetrics/{_tsadeval → base}/__init__.py +0 -0
- /tsadmetrics/{_tsadeval/affiliation → evaluation}/__init__.py +0 -0
- /tsadmetrics/{_tsadeval/eTaPR_pkg/DataManage → metrics/tem}/__init__.py +0 -0
- /tsadmetrics/{_tsadeval/vus_utils.py → utils/functions_vus.py} +0 -0
- {tsadmetrics-0.1.16.dist-info → tsadmetrics-1.0.0.dist-info}/WHEEL +0 -0
@@ -0,0 +1,252 @@
|
|
1
|
+
from ....base.Metric import Metric
|
2
|
+
import numpy as np
|
3
|
+
import math
|
4
|
+
from ....utils.functions_conversion import full_series_to_segmentwise, full_series_to_pointwise
|
5
|
+
class EnhancedTimeseriesAwareFScore(Metric):
|
6
|
+
"""
|
7
|
+
Calculate enhanced time series aware F-score for anomaly detection in time series.
|
8
|
+
|
9
|
+
This metric is similar to the range-based F-score in that it accounts for both detection existence
|
10
|
+
and overlap proportion. Additionally, it requires that a significant fraction :math:`{\\theta_r}` of each true anomaly
|
11
|
+
segment be detected, and that a significant fraction :math:`{\\theta_p}` of each predicted segment overlaps with the
|
12
|
+
ground truth. Finally, F-score contributions from each event are weighted by the square root of the
|
13
|
+
true segment’s length, providing a compromise between point-wise and segment-wise approaches.
|
14
|
+
|
15
|
+
Implementation of https://link.springer.com/article/10.1007/s10618-023-00988-8
|
16
|
+
|
17
|
+
For more information, see the original paper:
|
18
|
+
https://doi.org/10.1145/3477314.3507024
|
19
|
+
|
20
|
+
Parameters:
|
21
|
+
theta_p (float):
|
22
|
+
Minimum fraction (:math:`{0 \\leq \\theta_p \\leq 1}`) of a predicted segment that must be overlapped
|
23
|
+
by ground truth to count as detected.
|
24
|
+
theta_r (float):
|
25
|
+
Minimum fraction (:math:`{0 \\leq \\theta_r \\leq 1}`) of a true segment that must be overlapped
|
26
|
+
by predictions to count as detected.
|
27
|
+
"""
|
28
|
+
name = "etaf"
|
29
|
+
binary_prediction = True
|
30
|
+
param_schema = {
|
31
|
+
"theta_p": {
|
32
|
+
"default": 0.5,
|
33
|
+
"type": float
|
34
|
+
},
|
35
|
+
"theta_r": {
|
36
|
+
"default": 0.5,
|
37
|
+
"type": float
|
38
|
+
}
|
39
|
+
}
|
40
|
+
|
41
|
+
def __init__(self, **kwargs):
|
42
|
+
super().__init__(name="etaf", **kwargs)
|
43
|
+
|
44
|
+
def _min_max_norm(self, value, org_min, org_max, new_min, new_max) -> float:
|
45
|
+
if org_min == org_max:
|
46
|
+
return new_min
|
47
|
+
else:
|
48
|
+
return (float)(new_min) + (float)(value - org_min) * (new_max - new_min) / (org_max - org_min)
|
49
|
+
|
50
|
+
def _decaying_func(self, val: float) -> float:
|
51
|
+
assert (-6 <= val <= 6)
|
52
|
+
return 1 / (1 + math.exp(val))
|
53
|
+
|
54
|
+
def _uniform_func(self, val: float) -> float:
|
55
|
+
return 1.0
|
56
|
+
|
57
|
+
def _sum_of_func(self, start_time, end_time, org_start, org_end,
|
58
|
+
func) -> float:
|
59
|
+
val = 0.0
|
60
|
+
for timestamp in range(start_time, end_time + 1):
|
61
|
+
val += func(self._min_max_norm(timestamp, org_start, org_end, -6, 6))
|
62
|
+
return val
|
63
|
+
|
64
|
+
def _overlap_and_subsequent_score(self, anomaly, ambiguous, prediction) -> float:
|
65
|
+
score = 0.0
|
66
|
+
|
67
|
+
detected_start = max(anomaly[0], prediction[0])
|
68
|
+
detected_end = min(anomaly[1], prediction[1])
|
69
|
+
|
70
|
+
score += self._sum_of_func(detected_start, detected_end,
|
71
|
+
anomaly[0], anomaly[1], self._uniform_func)
|
72
|
+
|
73
|
+
if ambiguous[0] < ambiguous[1]:
|
74
|
+
detected_start = max(ambiguous[0], prediction[0])
|
75
|
+
detected_end = min(ambiguous[1], prediction[1])
|
76
|
+
|
77
|
+
score += self._sum_of_func(detected_start, detected_end,
|
78
|
+
ambiguous[0], ambiguous[1], self._decaying_func)
|
79
|
+
return score
|
80
|
+
|
81
|
+
def _gen_ambiguous(self,y_true_sw, y_pred_sw):
|
82
|
+
ambiguous_inst = []
|
83
|
+
for i in range(len(y_true_sw)):
|
84
|
+
start_id = y_true_sw[i][1] + 1
|
85
|
+
end_id = start_id
|
86
|
+
|
87
|
+
if i + 1 < len(y_true_sw) and end_id > y_true_sw[i + 1][0]:
|
88
|
+
end_id = y_true_sw[i + 1][0] - 1
|
89
|
+
|
90
|
+
if start_id > end_id:
|
91
|
+
start_id = -2
|
92
|
+
end_id = -1
|
93
|
+
|
94
|
+
ambiguous_inst.append([start_id, end_id])
|
95
|
+
return ambiguous_inst
|
96
|
+
|
97
|
+
def _compute_overlap_scores_and_weights(self, y_true_sw, y_pred_sw):
|
98
|
+
|
99
|
+
predictions_weight = []
|
100
|
+
predictions_total_weight = 0.0
|
101
|
+
#computing weights
|
102
|
+
for a_prediction in y_pred_sw:
|
103
|
+
first, last = a_prediction
|
104
|
+
temp_weight = math.sqrt(last-first+1)
|
105
|
+
predictions_weight.append(temp_weight)
|
106
|
+
predictions_total_weight += temp_weight
|
107
|
+
|
108
|
+
#computing the score matrix
|
109
|
+
ambiguous_inst = self._gen_ambiguous(y_true_sw, y_pred_sw)
|
110
|
+
overlap_score_mat_org = np.zeros((len(y_true_sw), len(y_pred_sw)))
|
111
|
+
for anomaly_id in range(len(y_true_sw)):
|
112
|
+
for prediction_id in range(len(y_pred_sw)):
|
113
|
+
overlap_score_mat_org[anomaly_id, prediction_id] = \
|
114
|
+
float(self._overlap_and_subsequent_score(y_true_sw[anomaly_id], ambiguous_inst[anomaly_id], y_pred_sw[prediction_id]))
|
115
|
+
|
116
|
+
#computing the maximum scores for each anomaly or prediction
|
117
|
+
max_anomaly_score = []
|
118
|
+
max_prediction_score = []
|
119
|
+
for an_anomaly in y_true_sw:
|
120
|
+
start, end = an_anomaly
|
121
|
+
max_anomaly_score.append(float(self._sum_of_func(start, end, start, end, self._uniform_func)))
|
122
|
+
for a_prediction in y_pred_sw:
|
123
|
+
max_prediction_score.append(a_prediction[1]-a_prediction[0] + 1)
|
124
|
+
|
125
|
+
|
126
|
+
return predictions_weight, predictions_total_weight, overlap_score_mat_org, max_anomaly_score, max_prediction_score
|
127
|
+
|
128
|
+
def _pruning(self, y_true_sw, y_pred_sw, overlap_score_mat_elm, max_anomaly_score, max_prediction_score):
|
129
|
+
|
130
|
+
|
131
|
+
while True:
|
132
|
+
tars = overlap_score_mat_elm.sum(axis=1)/max_anomaly_score
|
133
|
+
elem_anomaly_ids = set(np.where(tars<self.params['theta_r'])[0]) - set(np.where(tars==0.0)[0])
|
134
|
+
for id in elem_anomaly_ids:
|
135
|
+
overlap_score_mat_elm[id] = np.zeros(len(y_pred_sw))
|
136
|
+
taps = overlap_score_mat_elm.sum(axis=0)/max_prediction_score
|
137
|
+
elem_prediction_ids = set(np.where(taps<self.params['theta_p'])[0]) - set(np.where(taps==0.0)[0])
|
138
|
+
for id in elem_prediction_ids:
|
139
|
+
overlap_score_mat_elm[:, id] = np.zeros(len(y_true_sw))
|
140
|
+
|
141
|
+
if len(elem_anomaly_ids) == 0 and len(elem_prediction_ids) == 0:
|
142
|
+
break
|
143
|
+
return overlap_score_mat_elm
|
144
|
+
|
145
|
+
def _etar_p(self, y_true_sw, y_pred_sw, overlap_score_mat_elm, predictions_weight, predictions_total_weight, max_prediction_score):
|
146
|
+
"""
|
147
|
+
Calculate precision for the enhanced time series aware F-score.
|
148
|
+
|
149
|
+
Parameters:
|
150
|
+
y_true_sw (np.array):
|
151
|
+
The ground truth binary labels for the time series data, in segment-wise format.
|
152
|
+
y_pred_sw (np.array):
|
153
|
+
The predicted binary labels for the time series data, in segment-wise format.
|
154
|
+
overlap_score_mat_org (np.array):
|
155
|
+
The original overlap score matrix.
|
156
|
+
max_anomaly_score (list):
|
157
|
+
The maximum scores for each anomaly segment.
|
158
|
+
max_prediction_score (list):
|
159
|
+
The maximum scores for each prediction segment.
|
160
|
+
Returns:
|
161
|
+
float: The precision value.
|
162
|
+
"""
|
163
|
+
etap_d = 0
|
164
|
+
etap_p = 0
|
165
|
+
if len(y_true_sw) == 0.0 or len(y_pred_sw) == 0.0:
|
166
|
+
etap_d,etap_p = 0.0, 0.0
|
167
|
+
|
168
|
+
etap_d = overlap_score_mat_elm.sum(axis=0) / max_prediction_score
|
169
|
+
etap_p = etap_d
|
170
|
+
|
171
|
+
etap_d = np.where(etap_d >= self.params['theta_p'], 1.0, etap_d)
|
172
|
+
etap_d = np.where(etap_d < self.params['theta_p'], 0.0, etap_d)
|
173
|
+
corrected_id_list = np.where(etap_d >= self.params['theta_p'])[0]
|
174
|
+
|
175
|
+
detection_scores = etap_d
|
176
|
+
portion_scores = etap_p
|
177
|
+
|
178
|
+
|
179
|
+
scores = (detection_scores + detection_scores * portion_scores)/2
|
180
|
+
final_score = 0.0
|
181
|
+
for i in range(max(len(scores),len(etap_d),len(corrected_id_list))):
|
182
|
+
if i < len(scores):
|
183
|
+
final_score += float(predictions_weight[i]) * scores[i]
|
184
|
+
|
185
|
+
|
186
|
+
final_score /= float(predictions_total_weight)
|
187
|
+
return final_score
|
188
|
+
|
189
|
+
|
190
|
+
def _etar_d(self, y_true_sw, y_pred_sw, overlap_score_mat_elm, max_anomaly_score, max_prediction_score):
|
191
|
+
"""
|
192
|
+
Calculate recall for the enhanced time series aware F-score.
|
193
|
+
|
194
|
+
Parameters:
|
195
|
+
y_true_sw (np.array):
|
196
|
+
The ground truth binary labels for the time series data, in segment-wise format.
|
197
|
+
y_pred_sw (np.array):
|
198
|
+
The predicted binary labels for the time series data, in segment-wise format.
|
199
|
+
overlap_score_mat_org (np.array):
|
200
|
+
The original overlap score matrix.
|
201
|
+
max_anomaly_score (list):
|
202
|
+
The maximum scores for each anomaly segment.
|
203
|
+
max_prediction_score (list):
|
204
|
+
The maximum scores for each prediction segment.
|
205
|
+
Returns:
|
206
|
+
float: The recall value.
|
207
|
+
"""
|
208
|
+
if len(y_true_sw) == 0.0 or len(y_pred_sw) == 0.0:
|
209
|
+
return np.zeros(len(y_true_sw)), []
|
210
|
+
theta = self.params['theta_r']
|
211
|
+
scores = overlap_score_mat_elm.sum(axis=1) / max_anomaly_score
|
212
|
+
scores = np.where(scores >= theta, 1.0, scores)
|
213
|
+
scores = np.where(scores < theta, 0.0, scores)
|
214
|
+
detected_id_list = np.where(scores >= theta)[0]
|
215
|
+
|
216
|
+
return scores, detected_id_list
|
217
|
+
def _compute(self, y_true, y_pred):
|
218
|
+
"""
|
219
|
+
Calculate the enhanced time series aware F-score.
|
220
|
+
|
221
|
+
Parameters:
|
222
|
+
y_true (np.array):
|
223
|
+
The ground truth binary labels for the time series data.
|
224
|
+
y_pred (np.array):
|
225
|
+
The predicted binary labels for the time series data.
|
226
|
+
|
227
|
+
Returns:
|
228
|
+
float: The time series aware F-score, which is the harmonic mean of precision and recall, adjusted by the beta value.
|
229
|
+
"""
|
230
|
+
|
231
|
+
if np.sum(y_pred) == 0:
|
232
|
+
return 0
|
233
|
+
y_true_sw = np.array(full_series_to_segmentwise(y_true))
|
234
|
+
y_pred_sw = np.array(full_series_to_segmentwise(y_pred))
|
235
|
+
|
236
|
+
predictions_weight, predictions_total_weight, overlap_score_mat_org, max_anomaly_score, max_prediction_score = self._compute_overlap_scores_and_weights(y_true_sw,y_pred_sw)
|
237
|
+
overlap_score_mat_elm = self._pruning(y_true_sw, y_pred_sw, overlap_score_mat_org, max_anomaly_score, max_prediction_score)
|
238
|
+
detection_scores, detected_id_list = self._etar_d(y_true_sw, y_pred_sw, overlap_score_mat_elm, max_anomaly_score, max_prediction_score)
|
239
|
+
precision = self._etar_p(y_true_sw, y_pred_sw, overlap_score_mat_elm, predictions_weight, predictions_total_weight, max_prediction_score)
|
240
|
+
|
241
|
+
if len(y_true_sw) == 0 or len(y_pred_sw) == 0:
|
242
|
+
portion_scores = 0.0
|
243
|
+
else:
|
244
|
+
portion_scores = overlap_score_mat_elm.sum(axis=1) / max_anomaly_score
|
245
|
+
portion_scores = np.where(portion_scores > 1.0, 1.0, portion_scores)
|
246
|
+
recall = ((detection_scores + detection_scores * portion_scores)/2).mean()
|
247
|
+
|
248
|
+
if precision + recall == 0:
|
249
|
+
return 0.0
|
250
|
+
else:
|
251
|
+
return (2 * recall * precision) / (recall + precision)
|
252
|
+
|
@@ -0,0 +1,68 @@
|
|
1
|
+
from ....base.Metric import Metric
|
2
|
+
import numpy as np
|
3
|
+
from ....utils.functions_conversion import full_series_to_pointwise
|
4
|
+
|
5
|
+
class TemporalDistance(Metric):
|
6
|
+
"""
|
7
|
+
Calculate temporal distance for anomaly detection in time series.
|
8
|
+
|
9
|
+
This metric computes the sum of the distances from each labelled anomaly point to
|
10
|
+
the closest predicted anomaly point, and from each predicted anomaly point to the
|
11
|
+
closest labelled anomaly point.
|
12
|
+
|
13
|
+
Implementation of https://link.springer.com/article/10.1007/s10618-023-00988-8
|
14
|
+
|
15
|
+
For more information, see the original paper:
|
16
|
+
https://sciendo.com/article/10.2478/ausi-2019-0008
|
17
|
+
|
18
|
+
Parameters:
|
19
|
+
distance (int):
|
20
|
+
The distance type parameter for the temporal distance calculation.
|
21
|
+
- 0: Euclidean distance
|
22
|
+
- 1: Squared Euclidean distance
|
23
|
+
"""
|
24
|
+
name = "td"
|
25
|
+
binary_prediction = True
|
26
|
+
param_schema = {
|
27
|
+
"distance": {
|
28
|
+
"default": 0,
|
29
|
+
"type": int
|
30
|
+
}
|
31
|
+
}
|
32
|
+
|
33
|
+
def __init__(self, **kwargs):
|
34
|
+
super().__init__(name="td", **kwargs)
|
35
|
+
|
36
|
+
def _compute(self, y_true, y_pred):
|
37
|
+
"""
|
38
|
+
Calculate the temporal distance.
|
39
|
+
|
40
|
+
Parameters:
|
41
|
+
y_true (np.array):
|
42
|
+
The ground truth binary labels for the time series data.
|
43
|
+
y_pred (np.array):
|
44
|
+
The predicted binary labels for the time series data.
|
45
|
+
|
46
|
+
Returns:
|
47
|
+
float: The temporal distance.
|
48
|
+
"""
|
49
|
+
|
50
|
+
def _dist(a, b):
|
51
|
+
dist = 0
|
52
|
+
for pt in a:
|
53
|
+
if len(b) > 0:
|
54
|
+
dist += min(abs(b - pt))
|
55
|
+
else:
|
56
|
+
dist += len(y_true)
|
57
|
+
return dist
|
58
|
+
|
59
|
+
y_true_pw = np.array(full_series_to_pointwise(y_true))
|
60
|
+
y_pred_pw = np.array(full_series_to_pointwise(y_pred))
|
61
|
+
|
62
|
+
distance = self.params['distance']
|
63
|
+
if distance == 0:
|
64
|
+
return _dist(y_true_pw, y_pred_pw) + _dist(y_pred_pw, y_true_pw)
|
65
|
+
elif distance == 1:
|
66
|
+
return _dist(y_true_pw, y_pred_pw)**2 + _dist(y_pred_pw, y_true_pw)**2
|
67
|
+
else:
|
68
|
+
raise ValueError(f"Distance {distance} not supported")
|
@@ -0,0 +1,9 @@
|
|
1
|
+
from .AbsoluteDetectionDistance import AbsoluteDetectionDistance
|
2
|
+
from .TemporalDistance import TemporalDistance
|
3
|
+
from .EnhancedTimeseriesAwareFScore import EnhancedTimeseriesAwareFScore
|
4
|
+
|
5
|
+
__all__ = [
|
6
|
+
"AbsoluteDetectionDistance",
|
7
|
+
"TemporalDistance",
|
8
|
+
"EnhancedTimeseriesAwareFScore"
|
9
|
+
]
|
@@ -0,0 +1,104 @@
|
|
1
|
+
from ....base.Metric import Metric
|
2
|
+
import numpy as np
|
3
|
+
from ....utils.functions_conversion import full_series_to_segmentwise
|
4
|
+
|
5
|
+
class CompositeFScore(Metric):
|
6
|
+
"""
|
7
|
+
Composite F-score for anomaly detection in time series.
|
8
|
+
|
9
|
+
This metric combines aspects of the point-wise F-score and the segment-wise
|
10
|
+
F-score. It is defined as the harmonic mean of point-wise precision and
|
11
|
+
segment-wise recall. Using point-wise precision ensures that false positives
|
12
|
+
are properly penalized, a limitation often found in purely segment-wise
|
13
|
+
metrics.
|
14
|
+
|
15
|
+
Reference:
|
16
|
+
Implementation based on:
|
17
|
+
https://ieeexplore.ieee.org/document/9525836
|
18
|
+
|
19
|
+
For more details, see:
|
20
|
+
https://doi.org/10.1109/TNNLS.2021.3105827
|
21
|
+
|
22
|
+
Attributes:
|
23
|
+
name (str):
|
24
|
+
Fixed name identifier for this metric: `"cf"`.
|
25
|
+
binary_prediction (bool):
|
26
|
+
Indicates that this metric requires binary predictions.
|
27
|
+
param_schema (dict):
|
28
|
+
Defines supported parameters. Includes:
|
29
|
+
- beta (float): Weighting factor for recall in the F-score
|
30
|
+
calculation. Default = 1.0.
|
31
|
+
|
32
|
+
Raises:
|
33
|
+
ValueError:
|
34
|
+
If inputs are invalid or improperly shaped (checked by the base class).
|
35
|
+
TypeError:
|
36
|
+
If inputs are not array-like.
|
37
|
+
"""
|
38
|
+
|
39
|
+
name = "cf"
|
40
|
+
binary_prediction = True
|
41
|
+
param_schema = {
|
42
|
+
"beta": {
|
43
|
+
"default": 1.0,
|
44
|
+
"type": float
|
45
|
+
}
|
46
|
+
}
|
47
|
+
|
48
|
+
def __init__(self, **kwargs):
|
49
|
+
"""
|
50
|
+
Initialize the CompositeFScore metric.
|
51
|
+
|
52
|
+
Parameters:
|
53
|
+
**kwargs:
|
54
|
+
Optional keyword arguments passed to the base `Metric` class.
|
55
|
+
Supported parameter:
|
56
|
+
- beta (float): Weight factor for recall in the F-score.
|
57
|
+
"""
|
58
|
+
super().__init__(name="cf", **kwargs)
|
59
|
+
|
60
|
+
def _compute(self, y_true, y_pred):
|
61
|
+
"""
|
62
|
+
Compute the composite F-score.
|
63
|
+
|
64
|
+
The score is computed as:
|
65
|
+
F_beta = (1 + beta^2) * (precision * recall) / (beta^2 * precision + recall)
|
66
|
+
|
67
|
+
where:
|
68
|
+
- precision is computed point-wise.
|
69
|
+
- recall is computed segment-wise, meaning a segment is counted as
|
70
|
+
correctly detected if any point within it is predicted as anomalous.
|
71
|
+
|
72
|
+
Parameters:
|
73
|
+
y_true (np.ndarray):
|
74
|
+
Ground-truth binary labels for the time series (0 = normal, 1 = anomaly).
|
75
|
+
y_pred (np.ndarray):
|
76
|
+
Predicted binary labels for the time series.
|
77
|
+
|
78
|
+
Returns:
|
79
|
+
float:
|
80
|
+
The composite F-score. Returns 0 if either precision or recall is 0.
|
81
|
+
"""
|
82
|
+
tp = np.sum(y_pred * y_true)
|
83
|
+
fp = np.sum(y_pred * (1 - y_true))
|
84
|
+
|
85
|
+
tp_sw = 0
|
86
|
+
fn_sw = 0
|
87
|
+
for gt_anomaly in full_series_to_segmentwise(y_true):
|
88
|
+
found = False
|
89
|
+
for i_index in range(gt_anomaly[0], gt_anomaly[1] + 1):
|
90
|
+
if y_pred[i_index] == 1:
|
91
|
+
tp_sw += 1
|
92
|
+
found = True
|
93
|
+
break
|
94
|
+
if not found:
|
95
|
+
fn_sw += 1
|
96
|
+
|
97
|
+
precision = tp / (tp + fp) if (tp + fp) > 0 else 0
|
98
|
+
recall = tp_sw / (tp_sw + fn_sw) if (tp_sw + fn_sw) > 0 else 0
|
99
|
+
|
100
|
+
if precision == 0 or recall == 0:
|
101
|
+
return 0
|
102
|
+
|
103
|
+
beta = self.params['beta']
|
104
|
+
return ((1 + beta**2) * precision * recall) / (beta**2 * precision + recall)
|
@@ -0,0 +1,123 @@
|
|
1
|
+
from ....base.Metric import Metric
|
2
|
+
import numpy as np
|
3
|
+
from ....utils.functions_conversion import full_series_to_segmentwise
|
4
|
+
from ....utils.functions_auc import auc
|
5
|
+
|
6
|
+
class PointadjustedAucPr(Metric):
|
7
|
+
"""
|
8
|
+
Point-adjusted Area Under the Precision-Recall Curve (AUC-PR) for anomaly detection.
|
9
|
+
|
10
|
+
Unlike the standard point-wise AUC-PR, this variant uses a point-adjusted evaluation:
|
11
|
+
|
12
|
+
- Each anomalous segment in `y_true` is considered correctly detected if **at least
|
13
|
+
one point** within that segment is predicted as anomalous.
|
14
|
+
- Once a segment is detected, all its points are marked as detected in the adjusted
|
15
|
+
prediction.
|
16
|
+
|
17
|
+
This adjustment accounts for the fact that detecting any part of an anomalous
|
18
|
+
segment is often sufficient in practice.
|
19
|
+
|
20
|
+
Reference:
|
21
|
+
Implementation of:
|
22
|
+
https://link.springer.com/article/10.1007/s10618-023-00988-8
|
23
|
+
|
24
|
+
Attributes:
|
25
|
+
name (str):
|
26
|
+
Fixed name identifier for this metric: `"pa_auc_pr"`.
|
27
|
+
binary_prediction (bool):
|
28
|
+
Indicates whether this metric requires binary predictions.
|
29
|
+
Always `False`, as it expects continuous anomaly scores.
|
30
|
+
param_schema (dict):
|
31
|
+
Empty schema since this metric has no tunable parameters.
|
32
|
+
|
33
|
+
Raises:
|
34
|
+
ValueError:
|
35
|
+
If `y_true` and `y_anomaly_scores` have mismatched lengths.
|
36
|
+
TypeError:
|
37
|
+
If inputs are not array-like.
|
38
|
+
"""
|
39
|
+
|
40
|
+
name = "pa_auc_pr"
|
41
|
+
binary_prediction = False
|
42
|
+
param_schema = {}
|
43
|
+
|
44
|
+
def __init__(self, **kwargs):
|
45
|
+
"""
|
46
|
+
Initialize the PointadjustedAucPr metric.
|
47
|
+
|
48
|
+
Parameters:
|
49
|
+
**kwargs:
|
50
|
+
Optional keyword arguments passed to the base `Metric` class.
|
51
|
+
"""
|
52
|
+
super().__init__(name="pa_auc_pr", **kwargs)
|
53
|
+
|
54
|
+
def compute_point_adjusted(self, y_true, y_pred):
|
55
|
+
"""
|
56
|
+
Apply point-adjustment to predictions and compute precision/recall.
|
57
|
+
|
58
|
+
For each ground-truth anomalous segment, if any point is predicted as
|
59
|
+
anomalous, the entire segment is marked as detected.
|
60
|
+
|
61
|
+
Parameters:
|
62
|
+
y_true (np.ndarray):
|
63
|
+
Ground-truth binary labels (0 = normal, 1 = anomaly).
|
64
|
+
y_pred (np.ndarray):
|
65
|
+
Binary predictions (0 = normal, 1 = anomaly).
|
66
|
+
|
67
|
+
Returns:
|
68
|
+
tuple[float, float]:
|
69
|
+
- precision (float): Adjusted precision score.
|
70
|
+
- recall (float): Adjusted recall score.
|
71
|
+
"""
|
72
|
+
adjusted_prediction = y_pred.copy()
|
73
|
+
|
74
|
+
for start, end in full_series_to_segmentwise(y_true):
|
75
|
+
if np.any(adjusted_prediction[start:end + 1]):
|
76
|
+
adjusted_prediction[start:end + 1] = 1
|
77
|
+
else:
|
78
|
+
adjusted_prediction[start:end + 1] = 0
|
79
|
+
|
80
|
+
tp = np.sum(adjusted_prediction * y_true)
|
81
|
+
fp = np.sum(adjusted_prediction * (1 - y_true))
|
82
|
+
fn = np.sum((1 - adjusted_prediction) * y_true)
|
83
|
+
|
84
|
+
precision = tp / (tp + fp) if (tp + fp) > 0 else 0.0
|
85
|
+
recall = tp / (tp + fn) if (tp + fn) > 0 else 0.0
|
86
|
+
|
87
|
+
return precision, recall
|
88
|
+
|
89
|
+
def _compute(self, y_true, y_anomaly_scores):
|
90
|
+
"""
|
91
|
+
Compute the point-adjusted AUC-PR score.
|
92
|
+
|
93
|
+
Parameters:
|
94
|
+
y_true (np.ndarray):
|
95
|
+
Ground-truth binary labels for the time series.
|
96
|
+
y_anomaly_scores (np.ndarray):
|
97
|
+
Continuous anomaly scores assigned to each point.
|
98
|
+
|
99
|
+
Returns:
|
100
|
+
float:
|
101
|
+
The point-adjusted AUC-PR score.
|
102
|
+
"""
|
103
|
+
unique_thresholds = np.unique(y_anomaly_scores)
|
104
|
+
unique_thresholds = np.sort(unique_thresholds)[::-1] # descending
|
105
|
+
|
106
|
+
precisions, recalls = [], []
|
107
|
+
|
108
|
+
for threshold in unique_thresholds:
|
109
|
+
y_pred_binary = (y_anomaly_scores >= threshold).astype(int)
|
110
|
+
precision, recall = self.compute_point_adjusted(y_true, y_pred_binary)
|
111
|
+
precisions.append(precision)
|
112
|
+
recalls.append(recall)
|
113
|
+
|
114
|
+
# Add endpoints for PR curve
|
115
|
+
recalls = [0.0] + recalls + [1.0]
|
116
|
+
precisions = [1.0] + precisions + [0.0]
|
117
|
+
|
118
|
+
# Sort by recall (increasing order)
|
119
|
+
sorted_indices = np.argsort(recalls)
|
120
|
+
recalls_sorted = np.array(recalls)[sorted_indices]
|
121
|
+
precisions_sorted = np.array(precisions)[sorted_indices]
|
122
|
+
|
123
|
+
return auc(recalls_sorted, precisions_sorted)
|
@@ -0,0 +1,119 @@
|
|
1
|
+
from ....base.Metric import Metric
|
2
|
+
import numpy as np
|
3
|
+
from ....utils.functions_conversion import full_series_to_segmentwise
|
4
|
+
from ....utils.functions_auc import auc
|
5
|
+
|
6
|
+
class PointadjustedAucRoc(Metric):
|
7
|
+
"""
|
8
|
+
Point-adjusted Area Under the ROC Curve (AUC-ROC) for anomaly detection in time series.
|
9
|
+
|
10
|
+
Unlike standard point-wise AUC-ROC, this metric applies **point-adjusted evaluation**:
|
11
|
+
|
12
|
+
- Each anomalous segment in `y_true` is considered correctly detected if **at least one
|
13
|
+
point** within that segment is predicted as anomalous.
|
14
|
+
- Once a segment is detected, all its points are marked as detected in the adjusted
|
15
|
+
predictions.
|
16
|
+
- Adjusted predictions are then used to compute true positive rate (TPR) and false
|
17
|
+
positive rate (FPR) at multiple thresholds to construct the ROC curve.
|
18
|
+
|
19
|
+
Reference:
|
20
|
+
Implementation based on:
|
21
|
+
https://link.springer.com/article/10.1007/s10618-023-00988-8
|
22
|
+
|
23
|
+
Attributes:
|
24
|
+
name (str):
|
25
|
+
Fixed name identifier for this metric: `"pa_auc_roc"`.
|
26
|
+
binary_prediction (bool):
|
27
|
+
Indicates that this metric expects continuous anomaly scores.
|
28
|
+
param_schema (dict):
|
29
|
+
Empty schema since this metric has no tunable parameters.
|
30
|
+
|
31
|
+
Raises:
|
32
|
+
ValueError:
|
33
|
+
If `y_true` and `y_anomaly_scores` have mismatched lengths.
|
34
|
+
TypeError:
|
35
|
+
If inputs are not array-like.
|
36
|
+
"""
|
37
|
+
|
38
|
+
name = "pa_auc_roc"
|
39
|
+
binary_prediction = False
|
40
|
+
param_schema = {}
|
41
|
+
|
42
|
+
def __init__(self, **kwargs):
|
43
|
+
"""
|
44
|
+
Initialize the PointadjustedAucRoc metric.
|
45
|
+
|
46
|
+
Parameters:
|
47
|
+
**kwargs:
|
48
|
+
Optional keyword arguments passed to the base `Metric` class.
|
49
|
+
"""
|
50
|
+
super().__init__(name="pa_auc_roc", **kwargs)
|
51
|
+
|
52
|
+
def compute_point_adjusted(self, y_true, y_pred):
|
53
|
+
"""
|
54
|
+
Apply point-adjustment and compute TPR and FPR.
|
55
|
+
|
56
|
+
For each ground-truth anomalous segment, if any point is predicted as
|
57
|
+
anomalous, the entire segment is marked as detected.
|
58
|
+
|
59
|
+
Parameters:
|
60
|
+
y_true (np.ndarray):
|
61
|
+
Ground-truth binary labels (0 = normal, 1 = anomaly).
|
62
|
+
y_pred (np.ndarray):
|
63
|
+
Binary predictions (0 = normal, 1 = anomaly).
|
64
|
+
|
65
|
+
Returns:
|
66
|
+
tuple[float, float]:
|
67
|
+
- tpr (float): True positive rate.
|
68
|
+
- fpr (float): False positive rate.
|
69
|
+
"""
|
70
|
+
adjusted_prediction = y_pred.copy()
|
71
|
+
|
72
|
+
for start, end in full_series_to_segmentwise(y_true):
|
73
|
+
if np.any(adjusted_prediction[start:end + 1]):
|
74
|
+
adjusted_prediction[start:end + 1] = 1
|
75
|
+
else:
|
76
|
+
adjusted_prediction[start:end + 1] = 0
|
77
|
+
|
78
|
+
tp = np.sum(adjusted_prediction * y_true)
|
79
|
+
fp = np.sum(adjusted_prediction * (1 - y_true))
|
80
|
+
fn = np.sum((1 - adjusted_prediction) * y_true)
|
81
|
+
tpr = tp / (tp + fn) if (tp + fn) > 0 else 0.0
|
82
|
+
fpr = fp / (fp + (len(y_true) - np.sum(y_true) - fp)) if (fp + (len(y_true) - np.sum(y_true) - fp)) > 0 else 0.0
|
83
|
+
return tpr, fpr
|
84
|
+
|
85
|
+
def _compute(self, y_true, y_anomaly_scores):
|
86
|
+
"""
|
87
|
+
Compute the point-adjusted AUC-ROC score.
|
88
|
+
|
89
|
+
Parameters:
|
90
|
+
y_true (np.ndarray):
|
91
|
+
Ground-truth binary labels for the time series.
|
92
|
+
y_anomaly_scores (np.ndarray):
|
93
|
+
Continuous anomaly scores assigned to each point.
|
94
|
+
|
95
|
+
Returns:
|
96
|
+
float:
|
97
|
+
The point-adjusted AUC-ROC score.
|
98
|
+
"""
|
99
|
+
unique_thresholds = np.unique(y_anomaly_scores)
|
100
|
+
unique_thresholds = np.sort(unique_thresholds)[::-1] # descending
|
101
|
+
|
102
|
+
tprs, fprs = [], []
|
103
|
+
|
104
|
+
for threshold in unique_thresholds:
|
105
|
+
y_pred_binary = (y_anomaly_scores >= threshold).astype(int)
|
106
|
+
tpr, fpr = self.compute_point_adjusted(y_true, y_pred_binary)
|
107
|
+
tprs.append(tpr)
|
108
|
+
fprs.append(fpr)
|
109
|
+
|
110
|
+
# Add endpoints for ROC curve
|
111
|
+
tprs = [0.0] + tprs + [1.0]
|
112
|
+
fprs = [0.0] + fprs + [1.0]
|
113
|
+
|
114
|
+
# Sort by FPR to ensure monotonic increasing for AUC calculation
|
115
|
+
sorted_indices = np.argsort(fprs)
|
116
|
+
fprs_sorted = np.array(fprs)[sorted_indices]
|
117
|
+
tprs_sorted = np.array(tprs)[sorted_indices]
|
118
|
+
|
119
|
+
return auc(fprs_sorted, tprs_sorted)
|