pyharbinger 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pyharbinger/__init__.py +63 -0
- pyharbinger/anomaly.py +238 -0
- pyharbinger/changepoint.py +260 -0
- pyharbinger/clustering.py +84 -0
- pyharbinger/data/examples_anomalies.csv +3713 -0
- pyharbinger/data/examples_changepoints.csv +2304 -0
- pyharbinger/datasets.py +64 -0
- pyharbinger/evaluation.py +188 -0
- pyharbinger/harbinger.py +123 -0
- pyharbinger/utils.py +297 -0
- pyharbinger-0.1.0.dist-info/METADATA +123 -0
- pyharbinger-0.1.0.dist-info/RECORD +14 -0
- pyharbinger-0.1.0.dist-info/WHEEL +4 -0
- pyharbinger-0.1.0.dist-info/licenses/LICENSE +21 -0
pyharbinger/__init__.py
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
"""pyharbinger -- A unified time series event detection framework.
|
|
2
|
+
|
|
3
|
+
Python port of the R ``harbinger`` framework, the capstone of the DAL family.
|
|
4
|
+
It builds on ``pydaltoolbox`` (the Experiment Line foundation) and ``pytspredit``
|
|
5
|
+
(time series prediction) to unify many event-detection methods -- anomalies,
|
|
6
|
+
change points, and motifs/discords -- under one interface (``fit`` -> ``detect``
|
|
7
|
+
-> ``evaluate``) and a common evaluation protocol, wrapping numpy / pandas /
|
|
8
|
+
scikit-learn / scipy / ruptures with thin adapters.
|
|
9
|
+
"""
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from . import datasets, utils
|
|
13
|
+
from .anomaly import HanrFbiad, HanrFft, HanrHistogram, HanrMl
|
|
14
|
+
from .changepoint import HcpAmoc, HcpBinseg, HcpKswin, HcpPageHinkley, HcpPelt
|
|
15
|
+
from .clustering import HanctKmeans
|
|
16
|
+
from .evaluation import HarEval, HarEvalSoft
|
|
17
|
+
from .harbinger import Harbinger
|
|
18
|
+
from .utils import (
|
|
19
|
+
har_candidate_selection_firstgroup,
|
|
20
|
+
har_candidate_selection_highgroup,
|
|
21
|
+
har_candidate_selection_referencedistribution,
|
|
22
|
+
har_deviation_huber,
|
|
23
|
+
har_deviation_l1,
|
|
24
|
+
har_deviation_l2,
|
|
25
|
+
har_filter_boxplot,
|
|
26
|
+
har_filter_gaussian,
|
|
27
|
+
har_filter_grubbs,
|
|
28
|
+
har_filter_mad,
|
|
29
|
+
har_filter_none,
|
|
30
|
+
har_filter_ratio,
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
__version__ = "0.1.0"
|
|
34
|
+
|
|
35
|
+
__all__ = [
|
|
36
|
+
"HanctKmeans",
|
|
37
|
+
"HanrFbiad",
|
|
38
|
+
"HanrFft",
|
|
39
|
+
"HanrHistogram",
|
|
40
|
+
"HanrMl",
|
|
41
|
+
"HarEval",
|
|
42
|
+
"HarEvalSoft",
|
|
43
|
+
"Harbinger",
|
|
44
|
+
"HcpAmoc",
|
|
45
|
+
"HcpBinseg",
|
|
46
|
+
"HcpKswin",
|
|
47
|
+
"HcpPageHinkley",
|
|
48
|
+
"HcpPelt",
|
|
49
|
+
"datasets",
|
|
50
|
+
"har_candidate_selection_firstgroup",
|
|
51
|
+
"har_candidate_selection_highgroup",
|
|
52
|
+
"har_candidate_selection_referencedistribution",
|
|
53
|
+
"har_deviation_huber",
|
|
54
|
+
"har_deviation_l1",
|
|
55
|
+
"har_deviation_l2",
|
|
56
|
+
"har_filter_boxplot",
|
|
57
|
+
"har_filter_gaussian",
|
|
58
|
+
"har_filter_grubbs",
|
|
59
|
+
"har_filter_mad",
|
|
60
|
+
"har_filter_none",
|
|
61
|
+
"har_filter_ratio",
|
|
62
|
+
"utils",
|
|
63
|
+
]
|
pyharbinger/anomaly.py
ADDED
|
@@ -0,0 +1,238 @@
|
|
|
1
|
+
"""Anomaly detectors (Python port of the R ``hanr_*`` family).
|
|
2
|
+
|
|
3
|
+
This module provides the statistical / machine-learning anomaly detectors that
|
|
4
|
+
are thin wrappers over numpy / pandas / pytspredit:
|
|
5
|
+
|
|
6
|
+
* :class:`HanrMl` -- forecasting-based: wraps a ``pytspredit`` regressor, predicts
|
|
7
|
+
one step ahead over sliding windows, and flags large residuals. This is the
|
|
8
|
+
bridge that proves the pydaltoolbox -> pytspredit -> pyharbinger stack.
|
|
9
|
+
* :class:`HanrFft` -- high-pass FFT filtering: flags high-frequency departures.
|
|
10
|
+
* :class:`HanrHistogram` -- histogram density: flags low-density / out-of-range bins.
|
|
11
|
+
* :class:`HanrFbiad` -- Forward and Backward Inertial Anomaly Detector.
|
|
12
|
+
|
|
13
|
+
All of them reuse the shared :class:`~pyharbinger.harbinger.Harbinger` hooks
|
|
14
|
+
(``har_distance`` / ``har_outliers`` / ``har_outliers_check``).
|
|
15
|
+
"""
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import numpy as np
|
|
19
|
+
from pytspredit import ts_data, ts_projection
|
|
20
|
+
|
|
21
|
+
from .harbinger import Harbinger
|
|
22
|
+
|
|
23
|
+
__all__ = ["HanrFbiad", "HanrFft", "HanrHistogram", "HanrMl"]
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
# --- hanr_ml ----------------------------------------------------------------
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class HanrMl(Harbinger):
|
|
30
|
+
"""Anomaly detector based on ML regression (wraps a pytspredit regressor).
|
|
31
|
+
|
|
32
|
+
Parameters
|
|
33
|
+
----------
|
|
34
|
+
model:
|
|
35
|
+
A ``pytspredit`` regression model (e.g. ``TsElm``, ``TsKnn``, ``TsRf``).
|
|
36
|
+
sw_size:
|
|
37
|
+
Sliding-window size used to build the supervised one-step problem.
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
def __init__(self, model, sw_size: int = 15):
|
|
41
|
+
super().__init__()
|
|
42
|
+
self.model = model
|
|
43
|
+
self.sw_size = sw_size
|
|
44
|
+
|
|
45
|
+
def fit(self, serie, **kwargs) -> HanrMl:
|
|
46
|
+
arr = _as_array(serie)
|
|
47
|
+
ts = ts_data(arr, self.sw_size)
|
|
48
|
+
io = ts_projection(ts)
|
|
49
|
+
self.model = self.model.fit(x=io.input, y=io.output)
|
|
50
|
+
return self
|
|
51
|
+
|
|
52
|
+
def detect(self, serie, **kwargs):
|
|
53
|
+
self._store_refs(serie)
|
|
54
|
+
ts = ts_data(self.serie, self.sw_size)
|
|
55
|
+
io = ts_projection(ts)
|
|
56
|
+
|
|
57
|
+
adjust = np.asarray(self.model.predict(io.input)).ravel()
|
|
58
|
+
values = np.asarray(io.output).ravel()
|
|
59
|
+
res = values - adjust
|
|
60
|
+
|
|
61
|
+
res = self.har_distance(res)
|
|
62
|
+
outliers = self.har_outliers(res)
|
|
63
|
+
selection = self.har_outliers_check(outliers, res, values)
|
|
64
|
+
|
|
65
|
+
res = np.concatenate([np.zeros(self.sw_size - 1), res])
|
|
66
|
+
anomalies = np.concatenate([np.zeros(self.sw_size - 1, dtype=bool), selection.mask])
|
|
67
|
+
|
|
68
|
+
return self._restore_refs(anomalies=anomalies, res=res, threshold=selection.threshold)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
# --- hanr_fft ---------------------------------------------------------------
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _compute_cut_index(freqs: np.ndarray) -> int:
|
|
75
|
+
"""High-pass cutoff index (1-based, as in R ``compute_cut_index``)."""
|
|
76
|
+
cutindex = int(np.argmax(freqs)) + 1
|
|
77
|
+
if freqs.min() != freqs.max():
|
|
78
|
+
threshold = freqs.mean() + 2.698 * np.std(freqs, ddof=1)
|
|
79
|
+
f = freqs.copy()
|
|
80
|
+
f[f < threshold] = freqs.min() + freqs.max()
|
|
81
|
+
cutindex = int(np.argmin(f)) + 1
|
|
82
|
+
return cutindex
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
class HanrFft(Harbinger):
|
|
86
|
+
"""Anomaly detector using high-pass FFT filtering."""
|
|
87
|
+
|
|
88
|
+
def detect(self, serie, **kwargs):
|
|
89
|
+
if serie is None:
|
|
90
|
+
raise ValueError("No data was provided for computation")
|
|
91
|
+
self._store_refs(serie)
|
|
92
|
+
|
|
93
|
+
fft_signal = np.fft.fft(self.serie)
|
|
94
|
+
spectrum = np.abs(fft_signal) ** 2
|
|
95
|
+
half_spectrum = spectrum[: len(self.serie) // 2 + 1]
|
|
96
|
+
|
|
97
|
+
cutindex = _compute_cut_index(half_spectrum)
|
|
98
|
+
n = len(fft_signal)
|
|
99
|
+
|
|
100
|
+
# Zero out low frequencies (high-pass), both symmetric halves.
|
|
101
|
+
fft_signal[:cutindex] = 0
|
|
102
|
+
fft_signal[n - cutindex - 1 :] = 0
|
|
103
|
+
|
|
104
|
+
filtered_series = np.real(np.fft.ifft(fft_signal))
|
|
105
|
+
|
|
106
|
+
res = self.har_distance(filtered_series)
|
|
107
|
+
outliers = self.har_outliers(res)
|
|
108
|
+
selection = self.har_outliers_check(outliers, res, self.serie)
|
|
109
|
+
|
|
110
|
+
return self._restore_refs(anomalies=selection.mask, res=res, threshold=selection.threshold)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
# --- hanr_histogram ---------------------------------------------------------
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def _nclass_sturges(n: int) -> int:
|
|
117
|
+
"""R ``nclass.Sturges``: ceil(log2(n) + 1)."""
|
|
118
|
+
return int(np.ceil(np.log2(n) + 1))
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def r_pretty(lo: float, hi: float, n: int) -> np.ndarray:
|
|
122
|
+
"""Equally spaced 'nice' breakpoints covering [lo, hi], like R ``pretty()``.
|
|
123
|
+
|
|
124
|
+
A compact port of the common path of R's ``pretty`` / ``hist`` break
|
|
125
|
+
computation: pick a step among {1, 2, 2.5, 5} * 10^k that yields about ``n``
|
|
126
|
+
intervals, then extend the range outward to multiples of that step.
|
|
127
|
+
"""
|
|
128
|
+
if hi == lo:
|
|
129
|
+
hi = lo + 1
|
|
130
|
+
dx = hi - lo
|
|
131
|
+
raw_step = dx / max(n, 1)
|
|
132
|
+
mag = 10 ** np.floor(np.log10(raw_step))
|
|
133
|
+
norm = raw_step / mag
|
|
134
|
+
# R favors these nice multiples (with a mild bias toward larger steps).
|
|
135
|
+
if norm < 1.5:
|
|
136
|
+
nice = 1.0
|
|
137
|
+
elif norm < 3:
|
|
138
|
+
nice = 2.0
|
|
139
|
+
elif norm < 7:
|
|
140
|
+
nice = 5.0
|
|
141
|
+
else:
|
|
142
|
+
nice = 10.0
|
|
143
|
+
step = nice * mag
|
|
144
|
+
start = np.floor(lo / step) * step
|
|
145
|
+
end = np.ceil(hi / step) * step
|
|
146
|
+
# Guard floating point so the last break covers hi.
|
|
147
|
+
n_steps = int(np.round((end - start) / step))
|
|
148
|
+
return start + step * np.arange(n_steps + 1)
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
class HanrHistogram(Harbinger):
|
|
152
|
+
"""Anomaly detector using histogram bin density.
|
|
153
|
+
|
|
154
|
+
Parameters
|
|
155
|
+
----------
|
|
156
|
+
density_threshold:
|
|
157
|
+
Minimum bin density for an observation not to be flagged (default 0.05).
|
|
158
|
+
"""
|
|
159
|
+
|
|
160
|
+
def __init__(self, density_threshold: float = 0.05):
|
|
161
|
+
super().__init__()
|
|
162
|
+
self.density_threshold = density_threshold
|
|
163
|
+
|
|
164
|
+
def detect(self, serie, **kwargs):
|
|
165
|
+
if serie is None:
|
|
166
|
+
raise ValueError("No data was provided for computation")
|
|
167
|
+
self._store_refs(serie)
|
|
168
|
+
values = self.serie
|
|
169
|
+
n = len(values)
|
|
170
|
+
|
|
171
|
+
bin_edges = r_pretty(float(values.min()), float(values.max()), _nclass_sturges(n))
|
|
172
|
+
counts, _ = np.histogram(values, bins=bin_edges)
|
|
173
|
+
widths = np.diff(bin_edges)
|
|
174
|
+
density = counts / (n * widths)
|
|
175
|
+
n_bins = len(density)
|
|
176
|
+
|
|
177
|
+
anomalies = np.zeros(n, dtype=bool)
|
|
178
|
+
for i in range(n):
|
|
179
|
+
# findInterval(..., all.inside=TRUE) clamped to [1, n_bins] (1-based).
|
|
180
|
+
bin_index = int(np.searchsorted(bin_edges, values[i], side="right"))
|
|
181
|
+
bin_index = min(max(bin_index, 1), n_bins)
|
|
182
|
+
lower = bin_edges[bin_index - 1]
|
|
183
|
+
upper = bin_edges[bin_index]
|
|
184
|
+
if values[i] < lower or values[i] > upper or density[bin_index - 1] < self.density_threshold:
|
|
185
|
+
anomalies[i] = True
|
|
186
|
+
|
|
187
|
+
return self._restore_refs(anomalies=anomalies)
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
# --- hanr_fbiad -------------------------------------------------------------
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
class HanrFbiad(Harbinger):
|
|
194
|
+
"""Forward and Backward Inertial Anomaly Detector.
|
|
195
|
+
|
|
196
|
+
Parameters
|
|
197
|
+
----------
|
|
198
|
+
sw_size:
|
|
199
|
+
Window size for the forward/backward inertial context.
|
|
200
|
+
"""
|
|
201
|
+
|
|
202
|
+
def __init__(self, sw_size: int = 30):
|
|
203
|
+
super().__init__()
|
|
204
|
+
self.sw_size = sw_size
|
|
205
|
+
|
|
206
|
+
def _direction(self, serie: np.ndarray):
|
|
207
|
+
sx = np.asarray(ts_data(serie, self.sw_size))
|
|
208
|
+
ma = sx.mean(axis=1)
|
|
209
|
+
values = sx[:, -1]
|
|
210
|
+
res = self.har_distance(sx[:, -1] - ma)
|
|
211
|
+
outliers = self.har_outliers(res)
|
|
212
|
+
selection = self.har_outliers_check(outliers, res, values)
|
|
213
|
+
return selection.mask, res
|
|
214
|
+
|
|
215
|
+
def detect(self, serie, **kwargs):
|
|
216
|
+
if serie is None:
|
|
217
|
+
raise ValueError("No data was provided for computation")
|
|
218
|
+
self._store_refs(serie)
|
|
219
|
+
pad = self.sw_size - 1
|
|
220
|
+
|
|
221
|
+
i_f, res_f = self._direction(self.serie)
|
|
222
|
+
i_f = np.concatenate([np.zeros(pad, dtype=bool), i_f])
|
|
223
|
+
res_f = np.concatenate([np.zeros(pad), res_f])
|
|
224
|
+
|
|
225
|
+
i_b, res_b = self._direction(self.serie[::-1])
|
|
226
|
+
i_b = np.concatenate([i_b[::-1], np.zeros(pad, dtype=bool)])
|
|
227
|
+
res_b = np.concatenate([res_b[::-1], np.zeros(pad)])
|
|
228
|
+
|
|
229
|
+
res = (res_b + res_f) / 2
|
|
230
|
+
anomalies = i_f | i_b
|
|
231
|
+
|
|
232
|
+
return self._restore_refs(anomalies=anomalies, res=res)
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def _as_array(serie) -> np.ndarray:
|
|
236
|
+
from .harbinger import as_series_array
|
|
237
|
+
|
|
238
|
+
return as_series_array(serie)
|
|
@@ -0,0 +1,260 @@
|
|
|
1
|
+
"""Change-point detectors (Python port of the R ``hcp_*`` family).
|
|
2
|
+
|
|
3
|
+
Two groups are provided:
|
|
4
|
+
|
|
5
|
+
* **Offline, cost-based** detectors that segment the whole series by changes in
|
|
6
|
+
mean/variance. The R package wraps ``changepoint::cpt.meanvar`` (Gaussian cost,
|
|
7
|
+
MBIC penalty); here they wrap the ``ruptures`` library with its Gaussian
|
|
8
|
+
(``"normal"``) cost. Because ``ruptures`` does not implement the MBIC penalty,
|
|
9
|
+
parity with R is **approximate** -- compare breakpoint positions, not exact
|
|
10
|
+
equality.
|
|
11
|
+
|
|
12
|
+
* :class:`HcpAmoc` -- at most one change (``ruptures.Dynp``, ``n_bkps=1``).
|
|
13
|
+
* :class:`HcpBinseg` -- binary segmentation (``ruptures.Binseg``, ``n_bkps=Q``).
|
|
14
|
+
* :class:`HcpPelt` -- PELT (``ruptures.Pelt`` with a penalty).
|
|
15
|
+
|
|
16
|
+
* **Online, streaming** detectors that scan the series once:
|
|
17
|
+
|
|
18
|
+
* :class:`HcpKswin` -- Kolmogorov-Smirnov windowing (uses a random subsample, so
|
|
19
|
+
results are RNG-dependent; set ``random_state`` for reproducibility).
|
|
20
|
+
* :class:`HcpPageHinkley` -- the classical Page-Hinkley cumulative statistic
|
|
21
|
+
(fully deterministic).
|
|
22
|
+
"""
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
import numpy as np
|
|
26
|
+
import ruptures as rpt
|
|
27
|
+
from scipy import stats
|
|
28
|
+
|
|
29
|
+
from .harbinger import Harbinger, as_series_array
|
|
30
|
+
|
|
31
|
+
__all__ = ["HcpAmoc", "HcpBinseg", "HcpKswin", "HcpPageHinkley", "HcpPelt"]
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class _RupturesChangePoint(Harbinger):
|
|
35
|
+
"""Shared machinery for the ruptures-backed offline detectors."""
|
|
36
|
+
|
|
37
|
+
#: ruptures cost model (Gaussian mean/variance, like R's test.stat="Normal").
|
|
38
|
+
model = "normal"
|
|
39
|
+
min_size = 2
|
|
40
|
+
|
|
41
|
+
def _predict_bkps(self, signal: np.ndarray) -> list[int]: # pragma: no cover
|
|
42
|
+
raise NotImplementedError
|
|
43
|
+
|
|
44
|
+
def detect(self, serie, **kwargs):
|
|
45
|
+
if serie is None:
|
|
46
|
+
raise ValueError("No data was provided for computation")
|
|
47
|
+
self._store_refs(serie)
|
|
48
|
+
signal = self.serie.astype(float)
|
|
49
|
+
n = len(signal)
|
|
50
|
+
|
|
51
|
+
bkps = self._predict_bkps(signal)
|
|
52
|
+
cp = np.zeros(n, dtype=bool)
|
|
53
|
+
for b in bkps:
|
|
54
|
+
# ruptures breakpoints are segment end indices (exclusive); the last
|
|
55
|
+
# one equals n and is dropped. A breakpoint b marks a change after
|
|
56
|
+
# position b-1 (0-based), matching R's cpts[i] (1-based) -> b-1.
|
|
57
|
+
if 0 < b < n:
|
|
58
|
+
cp[b - 1] = True
|
|
59
|
+
|
|
60
|
+
return self._restore_refs(change_points=cp)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class HcpAmoc(_RupturesChangePoint):
|
|
64
|
+
"""At Most One Change (AMOC): the single most significant change in mean/variance."""
|
|
65
|
+
|
|
66
|
+
def _predict_bkps(self, signal):
|
|
67
|
+
algo = rpt.Dynp(model=self.model, min_size=self.min_size, jump=1).fit(signal)
|
|
68
|
+
return algo.predict(n_bkps=1)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
class HcpBinseg(_RupturesChangePoint):
|
|
72
|
+
"""Binary Segmentation (BinSeg).
|
|
73
|
+
|
|
74
|
+
Parameters
|
|
75
|
+
----------
|
|
76
|
+
Q:
|
|
77
|
+
Maximum number of change points to search for.
|
|
78
|
+
"""
|
|
79
|
+
|
|
80
|
+
def __init__(self, Q: int = 2):
|
|
81
|
+
super().__init__()
|
|
82
|
+
self.Q = Q
|
|
83
|
+
|
|
84
|
+
def _predict_bkps(self, signal):
|
|
85
|
+
algo = rpt.Binseg(model=self.model, min_size=self.min_size, jump=1).fit(signal)
|
|
86
|
+
return algo.predict(n_bkps=self.Q)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
class HcpPelt(_RupturesChangePoint):
|
|
90
|
+
"""Pruned Exact Linear Time (PELT).
|
|
91
|
+
|
|
92
|
+
Parameters
|
|
93
|
+
----------
|
|
94
|
+
penalty:
|
|
95
|
+
Penalty value passed to ``ruptures.Pelt.predict``. When ``None`` a default
|
|
96
|
+
of ``3 * log(n)`` is used -- a BIC-style penalty for the Gaussian
|
|
97
|
+
mean/variance cost (two parameters per segment), chosen to give a
|
|
98
|
+
practically useful segmentation. R uses the MBIC penalty, which is not
|
|
99
|
+
reproduced exactly; tune ``penalty`` to control granularity.
|
|
100
|
+
"""
|
|
101
|
+
|
|
102
|
+
def __init__(self, penalty: float | None = None):
|
|
103
|
+
super().__init__()
|
|
104
|
+
self.penalty = penalty
|
|
105
|
+
|
|
106
|
+
def _predict_bkps(self, signal):
|
|
107
|
+
n = len(signal)
|
|
108
|
+
pen = self.penalty if self.penalty is not None else 3.0 * float(np.log(n))
|
|
109
|
+
algo = rpt.Pelt(model=self.model, min_size=self.min_size, jump=1).fit(signal)
|
|
110
|
+
return algo.predict(pen=pen)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
class HcpKswin(Harbinger):
|
|
114
|
+
"""KSWIN: Kolmogorov-Smirnov windowing change detector (online, univariate).
|
|
115
|
+
|
|
116
|
+
A sliding window is split into an early part and the most recent
|
|
117
|
+
``stat_size`` observations; a KS test between a random subsample of the early
|
|
118
|
+
part and the recent part flags a change. Because of the random subsample the
|
|
119
|
+
result depends on ``random_state`` and is only approximately comparable to R.
|
|
120
|
+
|
|
121
|
+
Parameters
|
|
122
|
+
----------
|
|
123
|
+
window_size:
|
|
124
|
+
Size of the sliding window.
|
|
125
|
+
stat_size:
|
|
126
|
+
Size of the statistic subwindow used for the KS test.
|
|
127
|
+
alpha:
|
|
128
|
+
Significance level for the KS test.
|
|
129
|
+
random_state:
|
|
130
|
+
Seed for the random subsample.
|
|
131
|
+
"""
|
|
132
|
+
|
|
133
|
+
def __init__(
|
|
134
|
+
self,
|
|
135
|
+
window_size: int = 100,
|
|
136
|
+
stat_size: int = 30,
|
|
137
|
+
alpha: float = 0.005,
|
|
138
|
+
random_state: int | None = None,
|
|
139
|
+
):
|
|
140
|
+
super().__init__()
|
|
141
|
+
if alpha < 0 or alpha > 1:
|
|
142
|
+
raise ValueError("alpha must be between 0 and 1.")
|
|
143
|
+
if window_size <= 0:
|
|
144
|
+
raise ValueError("window_size must be greater than 0.")
|
|
145
|
+
if stat_size <= 0:
|
|
146
|
+
raise ValueError("stat_size must be greater than 0.")
|
|
147
|
+
if window_size <= 2 * stat_size:
|
|
148
|
+
raise ValueError("window_size must be greater than 2 * stat_size.")
|
|
149
|
+
self.window_size = window_size
|
|
150
|
+
self.stat_size = stat_size
|
|
151
|
+
self.alpha = alpha
|
|
152
|
+
self.random_state = random_state
|
|
153
|
+
|
|
154
|
+
def detect(self, serie, **kwargs):
|
|
155
|
+
if serie is None:
|
|
156
|
+
raise ValueError("No data was provided for computation")
|
|
157
|
+
serie = as_series_array(serie)
|
|
158
|
+
serie = serie[~np.isnan(serie)]
|
|
159
|
+
n = len(serie)
|
|
160
|
+
if n == 0:
|
|
161
|
+
raise ValueError("No non-missing observations were provided.")
|
|
162
|
+
|
|
163
|
+
rng = np.random.default_rng(self.random_state)
|
|
164
|
+
window = np.array([], dtype=float)
|
|
165
|
+
drift_active = False
|
|
166
|
+
result = np.zeros(n, dtype=bool)
|
|
167
|
+
|
|
168
|
+
for i in range(n):
|
|
169
|
+
window = np.append(window, serie[i])
|
|
170
|
+
if len(window) < self.window_size:
|
|
171
|
+
continue
|
|
172
|
+
if len(window) > self.window_size:
|
|
173
|
+
window = window[-self.window_size :]
|
|
174
|
+
|
|
175
|
+
early = window[: self.window_size - self.stat_size]
|
|
176
|
+
recent = window[self.window_size - self.stat_size :]
|
|
177
|
+
sample = rng.choice(early, size=self.stat_size, replace=True)
|
|
178
|
+
ks = stats.ks_2samp(sample, recent, method="exact")
|
|
179
|
+
|
|
180
|
+
drift_now = np.isfinite(ks.pvalue) and ks.pvalue < self.alpha and ks.statistic > 0.1
|
|
181
|
+
pred = bool(drift_now and not drift_active)
|
|
182
|
+
drift_active = bool(drift_now)
|
|
183
|
+
if pred:
|
|
184
|
+
window = recent
|
|
185
|
+
result[i] = pred
|
|
186
|
+
|
|
187
|
+
return self._build_cp_table(n, result)
|
|
188
|
+
|
|
189
|
+
@staticmethod
|
|
190
|
+
def _build_cp_table(n, result):
|
|
191
|
+
type_arr = np.array([""] * n, dtype=object)
|
|
192
|
+
type_arr[result] = "changepoint"
|
|
193
|
+
import pandas as pd
|
|
194
|
+
|
|
195
|
+
return pd.DataFrame({"idx": np.arange(n), "event": result, "type": type_arr})
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
class HcpPageHinkley(Harbinger):
|
|
199
|
+
"""Page-Hinkley online change-point detector (univariate, deterministic).
|
|
200
|
+
|
|
201
|
+
Parameters
|
|
202
|
+
----------
|
|
203
|
+
min_instances:
|
|
204
|
+
Minimum number of observations before a change can be reported.
|
|
205
|
+
delta:
|
|
206
|
+
Slack term subtracted from the deviation score.
|
|
207
|
+
threshold:
|
|
208
|
+
Detection threshold for the cumulative statistic.
|
|
209
|
+
alpha:
|
|
210
|
+
Forgetting factor applied to the cumulative score.
|
|
211
|
+
"""
|
|
212
|
+
|
|
213
|
+
def __init__(
|
|
214
|
+
self,
|
|
215
|
+
min_instances: int = 30,
|
|
216
|
+
delta: float = 0.005,
|
|
217
|
+
threshold: float = 50,
|
|
218
|
+
alpha: float = 1 - 1e-4,
|
|
219
|
+
):
|
|
220
|
+
super().__init__()
|
|
221
|
+
self.min_instances = min_instances
|
|
222
|
+
self.delta = delta
|
|
223
|
+
self.threshold = threshold
|
|
224
|
+
self.alpha = alpha
|
|
225
|
+
|
|
226
|
+
def detect(self, serie, **kwargs):
|
|
227
|
+
if serie is None:
|
|
228
|
+
raise ValueError("No data was provided for computation")
|
|
229
|
+
serie = as_series_array(serie)
|
|
230
|
+
serie = serie[~np.isnan(serie)]
|
|
231
|
+
n = len(serie)
|
|
232
|
+
if n == 0:
|
|
233
|
+
raise ValueError("No non-missing observations were provided.")
|
|
234
|
+
|
|
235
|
+
state = {"x_mean": 0.0, "sum": 0.0, "sample_count": 1}
|
|
236
|
+
|
|
237
|
+
def update(x):
|
|
238
|
+
state["x_mean"] += (x - state["x_mean"]) / state["sample_count"]
|
|
239
|
+
state["sum"] = max(0.0, self.alpha * state["sum"] + (x - state["x_mean"] - self.delta))
|
|
240
|
+
state["sample_count"] += 1
|
|
241
|
+
if state["sample_count"] < self.min_instances:
|
|
242
|
+
return False
|
|
243
|
+
if state["sum"] > self.threshold:
|
|
244
|
+
state["x_mean"] = 0.0
|
|
245
|
+
state["sum"] = 0.0
|
|
246
|
+
state["sample_count"] = 1
|
|
247
|
+
return True
|
|
248
|
+
return False
|
|
249
|
+
|
|
250
|
+
result = np.zeros(n, dtype=bool)
|
|
251
|
+
# R primes the statistic with the first observation before the main loop.
|
|
252
|
+
update(serie[0])
|
|
253
|
+
for i in range(n):
|
|
254
|
+
result[i] = update(serie[i])
|
|
255
|
+
|
|
256
|
+
type_arr = np.array([""] * n, dtype=object)
|
|
257
|
+
type_arr[result] = "changepoint"
|
|
258
|
+
import pandas as pd
|
|
259
|
+
|
|
260
|
+
return pd.DataFrame({"idx": np.arange(n), "event": result, "type": type_arr})
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
"""Clustering-based anomaly/discord detector (Python port of ``hanct_kmeans``).
|
|
2
|
+
|
|
3
|
+
:class:`HanctKmeans` clusters the sliding-window subsequences of a series with
|
|
4
|
+
k-means and flags the ones that are far from their nearest centroid. With
|
|
5
|
+
``seq == 1`` it labels isolated observations as anomalies; with ``seq > 1`` it
|
|
6
|
+
labels whole subsequences as discords.
|
|
7
|
+
|
|
8
|
+
Because scikit-learn's k-means uses a different initialization and RNG than R's
|
|
9
|
+
``stats::kmeans``, results are only approximately comparable to R; set
|
|
10
|
+
``random_state`` for reproducibility.
|
|
11
|
+
"""
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import numpy as np
|
|
15
|
+
from pytspredit import ts_data
|
|
16
|
+
from sklearn.cluster import KMeans
|
|
17
|
+
|
|
18
|
+
from .harbinger import Harbinger, as_series_array
|
|
19
|
+
|
|
20
|
+
__all__ = ["HanctKmeans"]
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class HanctKmeans(Harbinger):
|
|
24
|
+
"""k-means distance-based anomaly (``seq == 1``) / discord (``seq > 1``) detector.
|
|
25
|
+
|
|
26
|
+
Parameters
|
|
27
|
+
----------
|
|
28
|
+
seq:
|
|
29
|
+
Subsequence (sliding-window) size.
|
|
30
|
+
centers:
|
|
31
|
+
Number of centroids. When ``None`` it defaults to ``ceil(log10(n))``.
|
|
32
|
+
random_state:
|
|
33
|
+
Seed for the k-means initialization.
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
def __init__(self, seq: int = 1, centers: int | None = None, random_state: int | None = None):
|
|
37
|
+
super().__init__()
|
|
38
|
+
self.seq = seq
|
|
39
|
+
self.centers = centers
|
|
40
|
+
self.random_state = random_state
|
|
41
|
+
self.centroids: np.ndarray | None = None
|
|
42
|
+
|
|
43
|
+
def fit(self, serie, **kwargs) -> HanctKmeans:
|
|
44
|
+
arr = as_series_array(serie)
|
|
45
|
+
arr = arr[~np.isnan(arr)]
|
|
46
|
+
centers = self.centers
|
|
47
|
+
if centers is None:
|
|
48
|
+
centers = int(np.ceil(np.log10(len(arr))))
|
|
49
|
+
data = np.asarray(ts_data(arr, self.seq))
|
|
50
|
+
km = KMeans(n_clusters=centers, n_init=1, random_state=self.random_state).fit(data)
|
|
51
|
+
self.centers = centers
|
|
52
|
+
self.centroids = km.cluster_centers_
|
|
53
|
+
return self
|
|
54
|
+
|
|
55
|
+
def detect(self, serie, **kwargs):
|
|
56
|
+
if serie is None:
|
|
57
|
+
raise ValueError("No data was provided for computation")
|
|
58
|
+
self._store_refs(serie)
|
|
59
|
+
|
|
60
|
+
sx = np.asarray(ts_data(self.serie, self.seq))
|
|
61
|
+
values = sx[:, -1]
|
|
62
|
+
|
|
63
|
+
# Distance from each window to its nearest centroid.
|
|
64
|
+
d2 = ((sx[:, None, :] - self.centroids[None, :, :]) ** 2).sum(axis=2)
|
|
65
|
+
res = np.sqrt(d2.min(axis=1))
|
|
66
|
+
|
|
67
|
+
res = self.har_distance(res)
|
|
68
|
+
outliers = self.har_outliers(res)
|
|
69
|
+
selection = self.har_outliers_check(outliers, res, values)
|
|
70
|
+
threshold = selection.threshold
|
|
71
|
+
|
|
72
|
+
pad = self.seq - 1
|
|
73
|
+
res = np.concatenate([np.zeros(pad), res])
|
|
74
|
+
anomalies = np.concatenate([np.zeros(pad, dtype=bool), selection.mask])
|
|
75
|
+
|
|
76
|
+
detection = self._restore_refs(anomalies=anomalies, res=res, threshold=threshold)
|
|
77
|
+
|
|
78
|
+
if self.seq != 1:
|
|
79
|
+
mask = detection["type"] == "anomaly"
|
|
80
|
+
detection.loc[mask, "type"] = "discord"
|
|
81
|
+
detection["seq"] = np.where(mask, self.seq, np.nan)
|
|
82
|
+
detection["seqlen"] = np.where(mask, self.seq, np.nan)
|
|
83
|
+
|
|
84
|
+
return detection
|