deep-river 0.3.2__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {deep_river-0.3.2 → deep_river-0.4.0}/PKG-INFO +15 -55
- {deep_river-0.3.2 → deep_river-0.4.0}/README.md +6 -6
- deep_river-0.4.0/deep_river/__init__.py +7 -0
- {deep_river-0.3.2 → deep_river-0.4.0}/deep_river/anomaly/__init__.py +5 -4
- {deep_river-0.3.2 → deep_river-0.4.0}/deep_river/anomaly/ae.py +13 -5
- {deep_river-0.3.2 → deep_river-0.4.0}/deep_river/anomaly/probability_weighted_ae.py +24 -14
- {deep_river-0.3.2 → deep_river-0.4.0}/deep_river/anomaly/rolling_ae.py +22 -45
- {deep_river-0.3.2 → deep_river-0.4.0}/deep_river/anomaly/scaler.py +55 -48
- {deep_river-0.3.2 → deep_river-0.4.0}/deep_river/base.py +110 -44
- {deep_river-0.3.2 → deep_river-0.4.0}/deep_river/classification/__init__.py +4 -3
- {deep_river-0.3.2 → deep_river-0.4.0}/deep_river/classification/classifier.py +15 -5
- {deep_river-0.3.2 → deep_river-0.4.0}/deep_river/classification/rolling_classifier.py +17 -33
- {deep_river-0.3.2 → deep_river-0.4.0}/deep_river/classification/zoo.py +14 -5
- deep_river-0.4.0/deep_river/forecasting/__init__.py +23 -0
- deep_river-0.4.0/deep_river/forecasting/forecaster.py +189 -0
- deep_river-0.4.0/deep_river/forecasting/zoo.py +617 -0
- {deep_river-0.3.2 → deep_river-0.4.0}/deep_river/regression/__init__.py +4 -3
- {deep_river-0.3.2 → deep_river-0.4.0}/deep_river/regression/multioutput.py +30 -37
- {deep_river-0.3.2 → deep_river-0.4.0}/deep_river/regression/regressor.py +8 -4
- {deep_river-0.3.2 → deep_river-0.4.0}/deep_river/regression/rolling_regressor.py +16 -52
- {deep_river-0.3.2 → deep_river-0.4.0}/deep_river/regression/zoo.py +15 -5
- deep_river-0.4.0/deep_river/utils/estimator_checks.py +1589 -0
- deep_river-0.4.0/deep_river/utils/inspect.py +13 -0
- deep_river-0.4.0/deep_river/utils/ordered_set.py +42 -0
- {deep_river-0.3.2 → deep_river-0.4.0}/deep_river/utils/tensor_conversion.py +20 -6
- {deep_river-0.3.2 → deep_river-0.4.0}/deep_river.egg-info/PKG-INFO +15 -55
- {deep_river-0.3.2 → deep_river-0.4.0}/deep_river.egg-info/SOURCES.txt +5 -3
- deep_river-0.4.0/deep_river.egg-info/requires.txt +13 -0
- {deep_river-0.3.2 → deep_river-0.4.0}/pyproject.toml +26 -60
- deep_river-0.4.0/tests/test_rolling_batch.py +84 -0
- deep_river-0.3.2/deep_river/__init__.py +0 -4
- deep_river-0.3.2/deep_river/__version__.py +0 -3
- deep_river-0.3.2/deep_river/utils/estimator_checks.py +0 -320
- deep_river-0.3.2/deep_river/utils/inspect.py +0 -7
- deep_river-0.3.2/deep_river/utils/test_estimators.py +0 -50
- deep_river-0.3.2/deep_river/utils/test_tensor_conversion.py +0 -150
- deep_river-0.3.2/deep_river.egg-info/requires.txt +0 -46
- {deep_river-0.3.2 → deep_river-0.4.0}/LICENSE +0 -0
- {deep_river-0.3.2 → deep_river-0.4.0}/deep_river/utils/__init__.py +0 -0
- {deep_river-0.3.2 → deep_river-0.4.0}/deep_river/utils/hooks.py +0 -0
- {deep_river-0.3.2 → deep_river-0.4.0}/deep_river/utils/params.py +0 -0
- {deep_river-0.3.2 → deep_river-0.4.0}/deep_river.egg-info/dependency_links.txt +0 -0
- {deep_river-0.3.2 → deep_river-0.4.0}/deep_river.egg-info/top_level.txt +0 -0
- {deep_river-0.3.2 → deep_river-0.4.0}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: deep_river
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Summary: Online Deep Learning for river
|
|
5
5
|
Author-email: Cedric Kulbach <cedric.kulbach@googlemail.com>, Lucas Cazzonelli <Cazzonelli@fzi.de>, Hoang-Anh Ngo <hoang-anh.ngo@alumni.polytechnique.org>, Max Halford <maxhalford25@gmail.com>, Saulo Martiello Mastelini <saulomastelini@gmail.com>
|
|
6
6
|
License: BSD-3-Clause
|
|
@@ -8,71 +8,31 @@ Project-URL: Homepage, https://github.com/online-ml/deep-river
|
|
|
8
8
|
Project-URL: Repository, https://github.com/online-ml/deep-river
|
|
9
9
|
Project-URL: Documentation, https://github.com/online-ml/deep-river
|
|
10
10
|
Keywords: online-learning,deep-learning,data-streams
|
|
11
|
-
Classifier: Programming Language :: Python
|
|
12
11
|
Classifier: Programming Language :: Python :: 3
|
|
13
|
-
Classifier: Programming Language :: Python :: 3.10
|
|
14
12
|
Classifier: Programming Language :: Python :: 3.11
|
|
15
13
|
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
-
|
|
17
|
-
Classifier: Programming Language :: Python :: Implementation :: PyPy
|
|
18
|
-
Requires-Python: <3.13,>=3.10
|
|
14
|
+
Requires-Python: <3.13,>=3.11
|
|
19
15
|
Description-Content-Type: text/markdown
|
|
20
16
|
License-File: LICENSE
|
|
21
|
-
Requires-Dist:
|
|
22
|
-
Requires-Dist:
|
|
23
|
-
Requires-Dist:
|
|
24
|
-
Requires-Dist:
|
|
25
|
-
Requires-Dist:
|
|
17
|
+
Requires-Dist: numpy<3,>=2.3.4
|
|
18
|
+
Requires-Dist: pandas<3,>=2.2
|
|
19
|
+
Requires-Dist: river<0.27,>=0.26
|
|
20
|
+
Requires-Dist: scikit-learn<2,>=1.5
|
|
21
|
+
Requires-Dist: sortedcontainers>=2.4.0
|
|
22
|
+
Requires-Dist: torch==2.2.2; sys_platform == "darwin" and platform_machine == "x86_64"
|
|
23
|
+
Requires-Dist: torch<2.7,>=2.5; sys_platform != "darwin" or platform_machine != "x86_64"
|
|
26
24
|
Requires-Dist: tqdm~=4.66.5
|
|
27
25
|
Requires-Dist: torchviz~=0.0.2
|
|
28
|
-
Requires-Dist: mkdocstrings[python]>=0.19.0
|
|
29
|
-
Requires-Dist: pytkdocs[numpy-style]>=0.5.0
|
|
30
|
-
Requires-Dist: sortedcontainers>=2.4.0
|
|
31
|
-
Provides-Extra: dev
|
|
32
|
-
Requires-Dist: graphviz>=0.20.3; extra == "dev"
|
|
33
|
-
Requires-Dist: matplotlib>=3.9.2; extra == "dev"
|
|
34
|
-
Requires-Dist: mypy>=1.11.1; extra == "dev"
|
|
35
|
-
Requires-Dist: codecov>=2.1.13; extra == "dev"
|
|
36
|
-
Requires-Dist: pre-commit>=3.8.0; extra == "dev"
|
|
37
|
-
Requires-Dist: pytest>=8.3.2; extra == "dev"
|
|
38
|
-
Requires-Dist: pytest-cov>=5.0.0; extra == "dev"
|
|
39
|
-
Requires-Dist: black>=24.8.0; extra == "dev"
|
|
40
|
-
Requires-Dist: flake8>=7.1.1; extra == "dev"
|
|
41
|
-
Requires-Dist: isort>=5.13.2; extra == "dev"
|
|
42
|
-
Requires-Dist: jupyter>=1.0.0; extra == "dev"
|
|
43
|
-
Requires-Dist: pyupgrade==3.17.0; extra == "dev"
|
|
44
|
-
Requires-Dist: flask>=3.0.2; extra == "dev"
|
|
45
|
-
Requires-Dist: ipykernel>=6.9.0; extra == "dev"
|
|
46
|
-
Requires-Dist: mike>=0.5.3; extra == "dev"
|
|
47
|
-
Requires-Dist: mkdocs>=1.2.3; extra == "dev"
|
|
48
|
-
Requires-Dist: mkdocs-awesome-pages-plugin>=2.7.0; extra == "dev"
|
|
49
|
-
Requires-Dist: mkdocs-gen-files>=0.3.5; extra == "dev"
|
|
50
|
-
Requires-Dist: mkdocs-charts-plugin>=0.0.8; extra == "dev"
|
|
51
|
-
Requires-Dist: mkdocs-literate-nav>=0.4.1; extra == "dev"
|
|
52
|
-
Requires-Dist: mkdocs-material>=8.1.11; extra == "dev"
|
|
53
|
-
Requires-Dist: ipython_genutils>=0.1.0; extra == "dev"
|
|
54
|
-
Requires-Dist: mkdocs-jupyter>=0.20.0; extra == "dev"
|
|
55
|
-
Requires-Dist: nbconvert>=6.4.2; extra == "dev"
|
|
56
|
-
Requires-Dist: jupyter_contrib_nbextensions==0.7.0; extra == "dev"
|
|
57
|
-
Requires-Dist: numpydoc>=1.2; extra == "dev"
|
|
58
|
-
Requires-Dist: notebook==6.4.3; extra == "dev"
|
|
59
|
-
Requires-Dist: spacy>=3.2.2; extra == "dev"
|
|
60
|
-
Requires-Dist: jinja2>=3.0.3; extra == "dev"
|
|
61
|
-
Requires-Dist: dominate; extra == "dev"
|
|
62
|
-
Requires-Dist: jupyter-client; extra == "dev"
|
|
63
|
-
Requires-Dist: python-slugify; extra == "dev"
|
|
64
|
-
Requires-Dist: watermark==2.3.1; extra == "dev"
|
|
65
|
-
Requires-Dist: plotly>=5.0.0; extra == "dev"
|
|
66
26
|
Dynamic: license-file
|
|
67
27
|
|
|
68
28
|
<p align="center">
|
|
69
|
-
<img height="150px" src="https://raw.githubusercontent.com/online-ml/deep-river/
|
|
29
|
+
<img height="150px" src="https://raw.githubusercontent.com/online-ml/deep-river/main/docs/img/logo.png" alt="incremental dl logo">
|
|
70
30
|
</p>
|
|
71
31
|
<p align="center">
|
|
72
32
|
<img alt="PyPI" src="https://img.shields.io/pypi/v/deep-river">
|
|
73
33
|
<!-- Single Coverage Badge (Shields.io) -->
|
|
74
34
|
<a href="https://app.codecov.io/gh/online-ml/deep-river" >
|
|
75
|
-
<img alt="Coverage" src="https://img.shields.io/codecov/c/github/online-ml/deep-river?branch=
|
|
35
|
+
<img alt="Coverage" src="https://img.shields.io/codecov/c/github/online-ml/deep-river?branch=main&logo=codecov&label=coverage" />
|
|
76
36
|
</a>
|
|
77
37
|
<img alt="PyPI - Python Version" src="https://img.shields.io/pypi/pyversions/deep-river">
|
|
78
38
|
<img alt="PyPI - Downloads" src="https://img.shields.io/pypi/dm/deep-river">
|
|
@@ -88,7 +48,7 @@ Dynamic: license-file
|
|
|
88
48
|
|
|
89
49
|
## 📚 [Documentation](https://online-ml.github.io/deep-river/)
|
|
90
50
|
The [documentation](https://online-ml.github.io/deep-river/) contains an overview of all features of this repository as well as the repository's full features list.
|
|
91
|
-
In each of these, the git repo reference is listed in a section that shows [examples](https://github.com/online-ml/deep-river/blob/
|
|
51
|
+
In each of these, the git repo reference is listed in a section that shows [examples](https://github.com/online-ml/deep-river/blob/main/docs/examples) of the features and functionality.
|
|
92
52
|
As we are always looking for further use cases and examples, feel free to contribute to the documentation or the repository itself via a pull request
|
|
93
53
|
|
|
94
54
|
## 💈 Installation
|
|
@@ -103,7 +63,7 @@ pip install "river[deep]"
|
|
|
103
63
|
You can install the latest development version from GitHub as so:
|
|
104
64
|
|
|
105
65
|
```shell
|
|
106
|
-
pip install https://github.com/online-ml/deep-river/archive/refs/heads/
|
|
66
|
+
pip install https://github.com/online-ml/deep-river/archive/refs/heads/main.zip
|
|
107
67
|
```
|
|
108
68
|
|
|
109
69
|
### Development Environment
|
|
@@ -119,7 +79,7 @@ git clone https://github.com/online-ml/deep-river.git
|
|
|
119
79
|
cd deep-river
|
|
120
80
|
|
|
121
81
|
# Install all dependencies (including dev dependencies)
|
|
122
|
-
uv sync --
|
|
82
|
+
uv sync --group dev
|
|
123
83
|
|
|
124
84
|
# Run tests
|
|
125
85
|
make test
|
|
@@ -316,7 +276,7 @@ To acknowledge the use of the `DeepRiver` library in your research, please refer
|
|
|
316
276
|
## 🏫 Affiliations
|
|
317
277
|
|
|
318
278
|
<p align="center">
|
|
319
|
-
<img src="https://upload.wikimedia.org/wikipedia/de/
|
|
279
|
+
<img src="https://upload.wikimedia.org/wikipedia/de/4/44/Fzi_logo.svg" alt="FZI Logo" height="200"/>
|
|
320
280
|
</p>
|
|
321
281
|
|
|
322
282
|
<p align="center">
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
<p align="center">
|
|
2
|
-
<img height="150px" src="https://raw.githubusercontent.com/online-ml/deep-river/
|
|
2
|
+
<img height="150px" src="https://raw.githubusercontent.com/online-ml/deep-river/main/docs/img/logo.png" alt="incremental dl logo">
|
|
3
3
|
</p>
|
|
4
4
|
<p align="center">
|
|
5
5
|
<img alt="PyPI" src="https://img.shields.io/pypi/v/deep-river">
|
|
6
6
|
<!-- Single Coverage Badge (Shields.io) -->
|
|
7
7
|
<a href="https://app.codecov.io/gh/online-ml/deep-river" >
|
|
8
|
-
<img alt="Coverage" src="https://img.shields.io/codecov/c/github/online-ml/deep-river?branch=
|
|
8
|
+
<img alt="Coverage" src="https://img.shields.io/codecov/c/github/online-ml/deep-river?branch=main&logo=codecov&label=coverage" />
|
|
9
9
|
</a>
|
|
10
10
|
<img alt="PyPI - Python Version" src="https://img.shields.io/pypi/pyversions/deep-river">
|
|
11
11
|
<img alt="PyPI - Downloads" src="https://img.shields.io/pypi/dm/deep-river">
|
|
@@ -21,7 +21,7 @@
|
|
|
21
21
|
|
|
22
22
|
## 📚 [Documentation](https://online-ml.github.io/deep-river/)
|
|
23
23
|
The [documentation](https://online-ml.github.io/deep-river/) contains an overview of all features of this repository as well as the repository's full features list.
|
|
24
|
-
In each of these, the git repo reference is listed in a section that shows [examples](https://github.com/online-ml/deep-river/blob/
|
|
24
|
+
In each of these, the git repo reference is listed in a section that shows [examples](https://github.com/online-ml/deep-river/blob/main/docs/examples) of the features and functionality.
|
|
25
25
|
As we are always looking for further use cases and examples, feel free to contribute to the documentation or the repository itself via a pull request
|
|
26
26
|
|
|
27
27
|
## 💈 Installation
|
|
@@ -36,7 +36,7 @@ pip install "river[deep]"
|
|
|
36
36
|
You can install the latest development version from GitHub as so:
|
|
37
37
|
|
|
38
38
|
```shell
|
|
39
|
-
pip install https://github.com/online-ml/deep-river/archive/refs/heads/
|
|
39
|
+
pip install https://github.com/online-ml/deep-river/archive/refs/heads/main.zip
|
|
40
40
|
```
|
|
41
41
|
|
|
42
42
|
### Development Environment
|
|
@@ -52,7 +52,7 @@ git clone https://github.com/online-ml/deep-river.git
|
|
|
52
52
|
cd deep-river
|
|
53
53
|
|
|
54
54
|
# Install all dependencies (including dev dependencies)
|
|
55
|
-
uv sync --
|
|
55
|
+
uv sync --group dev
|
|
56
56
|
|
|
57
57
|
# Run tests
|
|
58
58
|
make test
|
|
@@ -249,7 +249,7 @@ To acknowledge the use of the `DeepRiver` library in your research, please refer
|
|
|
249
249
|
## 🏫 Affiliations
|
|
250
250
|
|
|
251
251
|
<p align="center">
|
|
252
|
-
<img src="https://upload.wikimedia.org/wikipedia/de/
|
|
252
|
+
<img src="https://upload.wikimedia.org/wikipedia/de/4/44/Fzi_logo.svg" alt="FZI Logo" height="200"/>
|
|
253
253
|
</p>
|
|
254
254
|
|
|
255
255
|
<p align="center">
|
|
@@ -1,12 +1,13 @@
|
|
|
1
|
+
"""
|
|
2
|
+
This module contains the anomaly detection algorithms for the
|
|
3
|
+
deep_river package.
|
|
4
|
+
"""
|
|
5
|
+
|
|
1
6
|
from .ae import Autoencoder
|
|
2
7
|
from .probability_weighted_ae import ProbabilityWeightedAutoencoder
|
|
3
8
|
from .rolling_ae import RollingAutoencoder
|
|
4
9
|
from .scaler import AnomalyMeanScaler, AnomalyMinMaxScaler, AnomalyStandardScaler
|
|
5
10
|
|
|
6
|
-
"""
|
|
7
|
-
This module contains the anomaly detection algorithms for the
|
|
8
|
-
deep_river package.
|
|
9
|
-
"""
|
|
10
11
|
__all__ = [
|
|
11
12
|
"AnomalyStandardScaler",
|
|
12
13
|
"AnomalyMeanScaler",
|
|
@@ -3,7 +3,7 @@ from typing import Any, Callable, Union
|
|
|
3
3
|
import numpy as np
|
|
4
4
|
import pandas as pd
|
|
5
5
|
import torch
|
|
6
|
-
from river.
|
|
6
|
+
from river.base import AnomalyDetector
|
|
7
7
|
from torch import nn
|
|
8
8
|
|
|
9
9
|
from deep_river.base import DeepEstimator
|
|
@@ -108,7 +108,17 @@ class Autoencoder(DeepEstimator, AnomalyDetector):
|
|
|
108
108
|
"""
|
|
109
109
|
return set()
|
|
110
110
|
|
|
111
|
-
def
|
|
111
|
+
def _update_observed_features(self, x: dict | pd.DataFrame) -> bool:
|
|
112
|
+
updated = super()._update_observed_features(x)
|
|
113
|
+
if (
|
|
114
|
+
self.is_feature_incremental
|
|
115
|
+
and self.output_layer is not None
|
|
116
|
+
and self._get_output_size() < self._get_input_size()
|
|
117
|
+
):
|
|
118
|
+
self._expand_layer(self.output_layer, self._get_input_size(), output=True)
|
|
119
|
+
return updated
|
|
120
|
+
|
|
121
|
+
def learn_one(self, x: dict, y: Any = None) -> None:
|
|
112
122
|
"""
|
|
113
123
|
Performs one step of training with a single example.
|
|
114
124
|
|
|
@@ -116,8 +126,6 @@ class Autoencoder(DeepEstimator, AnomalyDetector):
|
|
|
116
126
|
----------
|
|
117
127
|
x
|
|
118
128
|
Input example.
|
|
119
|
-
|
|
120
|
-
**kwargs
|
|
121
129
|
"""
|
|
122
130
|
self._update_observed_features(x)
|
|
123
131
|
self._learn(self._dict2tensor(x))
|
|
@@ -188,5 +196,5 @@ class Autoencoder(DeepEstimator, AnomalyDetector):
|
|
|
188
196
|
self.loss_func(x_pred, x_t, reduction="none"),
|
|
189
197
|
dim=list(range(1, x_t.dim())),
|
|
190
198
|
)
|
|
191
|
-
score = loss.cpu().detach().
|
|
199
|
+
score = np.asarray(loss.cpu().detach().tolist())
|
|
192
200
|
return score
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import math
|
|
2
2
|
from typing import Any, Callable, Union
|
|
3
3
|
|
|
4
|
+
import numpy as np
|
|
4
5
|
import pandas as pd
|
|
5
6
|
import torch
|
|
6
7
|
from river import stats, utils
|
|
@@ -21,9 +22,13 @@ class ProbabilityWeightedAutoencoder(ae.Autoencoder):
|
|
|
21
22
|
device: str = "cpu",
|
|
22
23
|
seed: int = 42,
|
|
23
24
|
skip_threshold: float = 0.9,
|
|
24
|
-
window_size=250,
|
|
25
|
+
window_size: int = 250,
|
|
25
26
|
**kwargs,
|
|
26
27
|
):
|
|
28
|
+
if not 0 < skip_threshold <= 1:
|
|
29
|
+
raise ValueError("skip_threshold must be greater than 0 and at most 1.")
|
|
30
|
+
if window_size < 1:
|
|
31
|
+
raise ValueError("window_size must be positive.")
|
|
27
32
|
super().__init__(
|
|
28
33
|
module=module,
|
|
29
34
|
loss_fn=loss_fn,
|
|
@@ -38,7 +43,7 @@ class ProbabilityWeightedAutoencoder(ae.Autoencoder):
|
|
|
38
43
|
self.rolling_mean = utils.Rolling(stats.Mean(), window_size=window_size)
|
|
39
44
|
self.rolling_var = utils.Rolling(stats.Var(), window_size=window_size)
|
|
40
45
|
|
|
41
|
-
def learn_one(self, x: dict, y: Any = None
|
|
46
|
+
def learn_one(self, x: dict, y: Any = None) -> None:
|
|
42
47
|
"""
|
|
43
48
|
Performs one step of training with a single example,
|
|
44
49
|
scaling the employed learning rate based on the outlier
|
|
@@ -46,7 +51,6 @@ class ProbabilityWeightedAutoencoder(ae.Autoencoder):
|
|
|
46
51
|
|
|
47
52
|
Parameters
|
|
48
53
|
----------
|
|
49
|
-
**kwargs
|
|
50
54
|
x
|
|
51
55
|
Input example.
|
|
52
56
|
|
|
@@ -65,23 +69,29 @@ class ProbabilityWeightedAutoencoder(ae.Autoencoder):
|
|
|
65
69
|
self._apply_loss(loss)
|
|
66
70
|
|
|
67
71
|
def _apply_loss(self, loss):
|
|
68
|
-
losses_numpy = loss.detach().
|
|
72
|
+
losses_numpy = np.asarray(loss.detach().cpu().tolist())
|
|
69
73
|
mean = self.rolling_mean.get()
|
|
70
74
|
var = self.rolling_var.get() if self.rolling_var.get() > 0 else 1
|
|
71
|
-
|
|
72
|
-
self.rolling_mean.update(
|
|
73
|
-
self.rolling_var.update(
|
|
74
|
-
else:
|
|
75
|
-
for loss_numpy in range(len(losses_numpy)):
|
|
76
|
-
self.rolling_mean.update(loss_numpy)
|
|
77
|
-
self.rolling_var.update(loss_numpy)
|
|
75
|
+
for loss_numpy in np.atleast_1d(losses_numpy):
|
|
76
|
+
self.rolling_mean.update(float(loss_numpy))
|
|
77
|
+
self.rolling_var.update(float(loss_numpy))
|
|
78
78
|
|
|
79
79
|
loss_scaled = (losses_numpy - mean) / math.sqrt(var)
|
|
80
80
|
prob = ndtr(loss_scaled)
|
|
81
|
-
|
|
82
|
-
|
|
81
|
+
weights = loss.new_tensor(
|
|
82
|
+
np.maximum(0, (self.skip_threshold - prob) / self.skip_threshold).tolist()
|
|
83
|
+
)
|
|
83
84
|
self.optimizer.zero_grad()
|
|
85
|
+
if not weights.numel():
|
|
86
|
+
return
|
|
87
|
+
loss = (weights * loss).mean()
|
|
84
88
|
loss.backward()
|
|
89
|
+
if not torch.any(weights > 0):
|
|
90
|
+
return
|
|
91
|
+
if self.gradient_clip_value is not None:
|
|
92
|
+
torch.nn.utils.clip_grad_norm_(
|
|
93
|
+
self.module.parameters(), self.gradient_clip_value
|
|
94
|
+
)
|
|
85
95
|
self.optimizer.step()
|
|
86
96
|
|
|
87
97
|
def learn_many(self, X: pd.DataFrame) -> None:
|
|
@@ -89,7 +99,7 @@ class ProbabilityWeightedAutoencoder(ae.Autoencoder):
|
|
|
89
99
|
X_t = self._df2tensor(X)
|
|
90
100
|
|
|
91
101
|
self.module.train()
|
|
92
|
-
x_pred = self.module(
|
|
102
|
+
x_pred = self.module(X_t)
|
|
93
103
|
loss = torch.mean(
|
|
94
104
|
self.loss_func(x_pred, X_t, reduction="none"),
|
|
95
105
|
dim=list(range(1, X_t.dim())),
|
|
@@ -1,9 +1,8 @@
|
|
|
1
1
|
from typing import Any, Callable, List, Union
|
|
2
2
|
|
|
3
|
-
import numpy as np
|
|
4
3
|
import pandas as pd
|
|
5
4
|
import torch
|
|
6
|
-
from river import
|
|
5
|
+
from river.base import AnomalyDetector
|
|
7
6
|
from torch import nn
|
|
8
7
|
|
|
9
8
|
from deep_river.base import RollingDeepEstimator
|
|
@@ -55,7 +54,7 @@ class _TestLSTMAutoencoder(nn.Module):
|
|
|
55
54
|
return output
|
|
56
55
|
|
|
57
56
|
|
|
58
|
-
class RollingAutoencoder(RollingDeepEstimator,
|
|
57
|
+
class RollingAutoencoder(RollingDeepEstimator, AnomalyDetector):
|
|
59
58
|
"""Rolling window autoencoder for streaming anomaly detection.
|
|
60
59
|
|
|
61
60
|
Maintains a fixed-size deque of the latest ``window_size`` observations and
|
|
@@ -141,7 +140,17 @@ class RollingAutoencoder(RollingDeepEstimator, anomaly.base.AnomalyDetector):
|
|
|
141
140
|
"check_predict_proba_one_binary",
|
|
142
141
|
}
|
|
143
142
|
|
|
144
|
-
def
|
|
143
|
+
def _update_observed_features(self, x: dict | pd.DataFrame) -> bool:
|
|
144
|
+
updated = super()._update_observed_features(x)
|
|
145
|
+
if (
|
|
146
|
+
self.is_feature_incremental
|
|
147
|
+
and self.output_layer is not None
|
|
148
|
+
and self._get_output_size() < self._get_input_size()
|
|
149
|
+
):
|
|
150
|
+
self._expand_layer(self.output_layer, self._get_input_size(), output=True)
|
|
151
|
+
return updated
|
|
152
|
+
|
|
153
|
+
def learn_one(self, x: dict, y: Any = None) -> None:
|
|
145
154
|
"""Update model using a single sample appended to the rolling window.
|
|
146
155
|
|
|
147
156
|
Parameters
|
|
@@ -150,31 +159,17 @@ class RollingAutoencoder(RollingDeepEstimator, anomaly.base.AnomalyDetector):
|
|
|
150
159
|
Dictionary containing feature name-value pairs for the sample.
|
|
151
160
|
y : Any, optional
|
|
152
161
|
Target value (not used in autoencoder training).
|
|
153
|
-
**kwargs
|
|
154
|
-
Additional keyword arguments.
|
|
155
162
|
"""
|
|
156
163
|
self._update_observed_features(x)
|
|
157
|
-
self._x_window.append(
|
|
164
|
+
self._x_window.append([x.get(feature, 0) for feature in self.observed_features])
|
|
158
165
|
|
|
159
166
|
x_t = deque2rolling_tensor(self._x_window, device=self.device)
|
|
160
167
|
self._learn(x=x_t)
|
|
161
168
|
|
|
162
169
|
def learn_many(self, X: pd.DataFrame, y=None) -> None:
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
----------
|
|
167
|
-
X : pd.DataFrame
|
|
168
|
-
DataFrame containing the input features for each sample.
|
|
169
|
-
y : None
|
|
170
|
-
Ignored, present for compatibility.
|
|
171
|
-
"""
|
|
172
|
-
self._update_observed_features(X)
|
|
173
|
-
|
|
174
|
-
self._x_window.append(X.values.tolist())
|
|
175
|
-
if len(self._x_window) == self.window_size:
|
|
176
|
-
X_t = deque2rolling_tensor(self._x_window, device=self.device)
|
|
177
|
-
self._learn(x=X_t)
|
|
170
|
+
columns = X.columns
|
|
171
|
+
for values in X.to_numpy(dtype=object).tolist():
|
|
172
|
+
self.learn_one(dict(zip(columns, values)))
|
|
178
173
|
|
|
179
174
|
def score_one(self, x: dict) -> float:
|
|
180
175
|
"""Return reconstruction error for current window + candidate sample.
|
|
@@ -193,7 +188,7 @@ class RollingAutoencoder(RollingDeepEstimator, anomaly.base.AnomalyDetector):
|
|
|
193
188
|
self._update_observed_features(x)
|
|
194
189
|
if len(self._x_window) == self.window_size:
|
|
195
190
|
x_win = self._x_window.copy()
|
|
196
|
-
x_win.append(
|
|
191
|
+
x_win.append([x.get(feature, 0) for feature in self.observed_features])
|
|
197
192
|
x_t = deque2rolling_tensor(x_win, device=self.device)
|
|
198
193
|
self.module.eval()
|
|
199
194
|
with torch.inference_mode():
|
|
@@ -202,7 +197,9 @@ class RollingAutoencoder(RollingDeepEstimator, anomaly.base.AnomalyDetector):
|
|
|
202
197
|
res = loss.item()
|
|
203
198
|
|
|
204
199
|
if self.append_predict:
|
|
205
|
-
self._x_window.append(
|
|
200
|
+
self._x_window.append(
|
|
201
|
+
[x.get(feature, 0) for feature in self.observed_features]
|
|
202
|
+
)
|
|
206
203
|
return res
|
|
207
204
|
|
|
208
205
|
def score_many(self, X: pd.DataFrame) -> List[Any]:
|
|
@@ -220,24 +217,4 @@ class RollingAutoencoder(RollingDeepEstimator, anomaly.base.AnomalyDetector):
|
|
|
220
217
|
List[float]
|
|
221
218
|
List of computed anomaly scores (reconstruction errors) for each sample in X.
|
|
222
219
|
"""
|
|
223
|
-
self.
|
|
224
|
-
x_win = self._x_window.copy()
|
|
225
|
-
x_win.append(X.values.tolist())
|
|
226
|
-
if self.append_predict:
|
|
227
|
-
self._x_window.append(X.values.tolist())
|
|
228
|
-
|
|
229
|
-
if len(self._x_window) == self.window_size:
|
|
230
|
-
X_t = deque2rolling_tensor(x_win, device=self.device)
|
|
231
|
-
self.module.eval()
|
|
232
|
-
with torch.inference_mode():
|
|
233
|
-
x_pred = self.module(X_t)
|
|
234
|
-
loss = torch.mean(
|
|
235
|
-
self.loss_func(x_pred, x_pred, reduction="none"),
|
|
236
|
-
dim=list(range(1, x_pred.dim())),
|
|
237
|
-
)
|
|
238
|
-
losses = loss.detach().numpy()
|
|
239
|
-
if len(losses) < len(X):
|
|
240
|
-
losses = np.pad(losses, (len(X) - len(losses), 0))
|
|
241
|
-
return losses.tolist()
|
|
242
|
-
else:
|
|
243
|
-
return np.zeros(len(X)).tolist()
|
|
220
|
+
return [self.score_one(row) for row in X.to_dict(orient="records")]
|
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import abc
|
|
2
2
|
|
|
3
3
|
import numpy as np
|
|
4
|
+
import pandas as pd
|
|
4
5
|
from river import base, utils
|
|
5
6
|
from river.anomaly import HalfSpaceTrees
|
|
6
|
-
from river.
|
|
7
|
+
from river.base import AnomalyDetector
|
|
7
8
|
from river.stats import Mean, Min, RollingMin
|
|
8
9
|
|
|
9
10
|
|
|
@@ -21,7 +22,7 @@ class AnomalyScaler(base.Wrapper, AnomalyDetector):
|
|
|
21
22
|
self.anomaly_detector = anomaly_detector
|
|
22
23
|
|
|
23
24
|
@classmethod
|
|
24
|
-
def _unit_test_params(
|
|
25
|
+
def _unit_test_params(cls):
|
|
25
26
|
"""
|
|
26
27
|
Returns a dictionary of parameters to be used for unit testing
|
|
27
28
|
the respective class.
|
|
@@ -32,7 +33,7 @@ class AnomalyScaler(base.Wrapper, AnomalyDetector):
|
|
|
32
33
|
Dictionary of parameters to be used for unit testing the
|
|
33
34
|
respective class.
|
|
34
35
|
"""
|
|
35
|
-
|
|
36
|
+
yield {"anomaly_detector": HalfSpaceTrees(seed=42, window_size=10)}
|
|
36
37
|
|
|
37
38
|
@classmethod
|
|
38
39
|
def _unit_test_skips(self) -> set:
|
|
@@ -60,7 +61,7 @@ class AnomalyScaler(base.Wrapper, AnomalyDetector):
|
|
|
60
61
|
return self.anomaly_detector
|
|
61
62
|
|
|
62
63
|
@abc.abstractmethod
|
|
63
|
-
def score_one(self, *args) -> float:
|
|
64
|
+
def score_one(self, *args, **kwargs) -> float:
|
|
64
65
|
"""Return a scaled anomaly score based on raw score provided by
|
|
65
66
|
the wrapped anomaly detector.
|
|
66
67
|
|
|
@@ -79,7 +80,7 @@ class AnomalyScaler(base.Wrapper, AnomalyDetector):
|
|
|
79
80
|
more anomalous examples.
|
|
80
81
|
"""
|
|
81
82
|
|
|
82
|
-
def learn_one(self, *args) -> None:
|
|
83
|
+
def learn_one(self, *args, **kwargs) -> None:
|
|
83
84
|
"""
|
|
84
85
|
Update the scaler and the underlying anomaly scaler.
|
|
85
86
|
|
|
@@ -95,26 +96,22 @@ class AnomalyScaler(base.Wrapper, AnomalyDetector):
|
|
|
95
96
|
The model itself.
|
|
96
97
|
"""
|
|
97
98
|
|
|
98
|
-
self.anomaly_detector.
|
|
99
|
+
self._update_score(self.anomaly_detector.score_one(*args, **kwargs))
|
|
100
|
+
self.anomaly_detector.learn_one(*args, **kwargs)
|
|
99
101
|
|
|
100
102
|
@abc.abstractmethod
|
|
101
|
-
def
|
|
102
|
-
|
|
103
|
-
the wrapped anomaly detector.
|
|
103
|
+
def _update_score(self, score: float) -> None:
|
|
104
|
+
pass
|
|
104
105
|
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
*args
|
|
111
|
-
Depends on whether the underlying anomaly detector is
|
|
112
|
-
supervised or not.
|
|
106
|
+
def score_many(self, X: pd.DataFrame, **kwargs) -> np.ndarray:
|
|
107
|
+
return np.asarray(
|
|
108
|
+
[self.score_one(x, **kwargs) for x in X.to_dict(orient="records")],
|
|
109
|
+
dtype=float,
|
|
110
|
+
)
|
|
113
111
|
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
"""
|
|
112
|
+
def learn_many(self, X: pd.DataFrame, **kwargs) -> None:
|
|
113
|
+
for x in X.to_dict(orient="records"):
|
|
114
|
+
self.learn_one(x, **kwargs)
|
|
118
115
|
|
|
119
116
|
|
|
120
117
|
class AnomalyStandardScaler(AnomalyScaler):
|
|
@@ -141,16 +138,20 @@ class AnomalyStandardScaler(AnomalyScaler):
|
|
|
141
138
|
rolling: bool = True,
|
|
142
139
|
window_size: int = 250,
|
|
143
140
|
):
|
|
141
|
+
from river.stats import Var
|
|
142
|
+
|
|
144
143
|
super().__init__(anomaly_detector)
|
|
145
144
|
self.rolling = rolling
|
|
146
145
|
self.window_size = window_size
|
|
147
146
|
self.mean = utils.Rolling(Mean(), self.window_size) if self.rolling else Mean()
|
|
148
|
-
self.
|
|
149
|
-
utils.Rolling(
|
|
147
|
+
self.var = (
|
|
148
|
+
utils.Rolling(Var(ddof=0), self.window_size)
|
|
149
|
+
if self.rolling
|
|
150
|
+
else Var(ddof=0)
|
|
150
151
|
)
|
|
151
152
|
self.with_std = with_std
|
|
152
153
|
|
|
153
|
-
def score_one(self, *args):
|
|
154
|
+
def score_one(self, *args, **kwargs):
|
|
154
155
|
"""
|
|
155
156
|
Return a scaled anomaly score based on raw score provided by the
|
|
156
157
|
wrapped anomaly detector. Larger values indicate more
|
|
@@ -167,17 +168,16 @@ class AnomalyStandardScaler(AnomalyScaler):
|
|
|
167
168
|
An scaled anomaly score. Larger values indicate more
|
|
168
169
|
anomalous examples.
|
|
169
170
|
"""
|
|
170
|
-
raw_score = self.anomaly_detector.score_one(*args)
|
|
171
|
-
mean = self.mean.
|
|
172
|
-
if self.with_std:
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
score = (raw_score - mean) / var**0.5
|
|
177
|
-
else:
|
|
178
|
-
score = raw_score - mean
|
|
171
|
+
raw_score = self.anomaly_detector.score_one(*args, **kwargs)
|
|
172
|
+
mean = self.mean.get()
|
|
173
|
+
if not self.with_std:
|
|
174
|
+
return raw_score - mean
|
|
175
|
+
var = self.var.get()
|
|
176
|
+
return (raw_score - mean) / var**0.5 if var > 0 else 0.0
|
|
179
177
|
|
|
180
|
-
|
|
178
|
+
def _update_score(self, score: float) -> None:
|
|
179
|
+
self.mean.update(score)
|
|
180
|
+
self.var.update(score)
|
|
181
181
|
|
|
182
182
|
|
|
183
183
|
class AnomalyMeanScaler(AnomalyScaler):
|
|
@@ -200,14 +200,14 @@ class AnomalyMeanScaler(AnomalyScaler):
|
|
|
200
200
|
self,
|
|
201
201
|
anomaly_detector: AnomalyDetector,
|
|
202
202
|
rolling: bool = True,
|
|
203
|
-
window_size=250,
|
|
203
|
+
window_size: int = 250,
|
|
204
204
|
):
|
|
205
205
|
super().__init__(anomaly_detector=anomaly_detector)
|
|
206
206
|
self.rolling = rolling
|
|
207
207
|
self.window_size = window_size
|
|
208
208
|
self.mean = utils.Rolling(Mean(), self.window_size) if self.rolling else Mean()
|
|
209
209
|
|
|
210
|
-
def score_one(self, *args):
|
|
210
|
+
def score_one(self, *args, **kwargs):
|
|
211
211
|
"""
|
|
212
212
|
Return a scaled anomaly score based on raw score provided by the
|
|
213
213
|
wrapped anomaly detector. Larger values indicate more
|
|
@@ -224,11 +224,12 @@ class AnomalyMeanScaler(AnomalyScaler):
|
|
|
224
224
|
An scaled anomaly score. Larger values indicate more
|
|
225
225
|
anomalous examples.
|
|
226
226
|
"""
|
|
227
|
-
raw_score = self.anomaly_detector.score_one(*args)
|
|
228
|
-
mean = self.mean.
|
|
229
|
-
|
|
227
|
+
raw_score = self.anomaly_detector.score_one(*args, **kwargs)
|
|
228
|
+
mean = self.mean.get()
|
|
229
|
+
return raw_score / mean if mean else 0.0
|
|
230
230
|
|
|
231
|
-
|
|
231
|
+
def _update_score(self, score: float) -> None:
|
|
232
|
+
self.mean.update(score)
|
|
232
233
|
|
|
233
234
|
|
|
234
235
|
class AnomalyMinMaxScaler(AnomalyScaler):
|
|
@@ -251,13 +252,15 @@ class AnomalyMinMaxScaler(AnomalyScaler):
|
|
|
251
252
|
rolling: bool = True,
|
|
252
253
|
window_size: int = 250,
|
|
253
254
|
):
|
|
255
|
+
from river.stats import Max, RollingMax
|
|
256
|
+
|
|
254
257
|
super().__init__(anomaly_detector)
|
|
255
258
|
self.rolling = rolling
|
|
256
259
|
self.window_size = window_size
|
|
257
260
|
self.min = RollingMin(self.window_size) if self.rolling else Min()
|
|
258
|
-
self.max =
|
|
261
|
+
self.max = RollingMax(self.window_size) if self.rolling else Max()
|
|
259
262
|
|
|
260
|
-
def score_one(self, *args):
|
|
263
|
+
def score_one(self, *args, **kwargs):
|
|
261
264
|
"""
|
|
262
265
|
Return a scaled anomaly score based on raw score provided by the
|
|
263
266
|
wrapped anomaly detector. Larger values indicate more
|
|
@@ -274,9 +277,13 @@ class AnomalyMinMaxScaler(AnomalyScaler):
|
|
|
274
277
|
An scaled anomaly score. Larger values indicate more
|
|
275
278
|
anomalous examples.
|
|
276
279
|
"""
|
|
277
|
-
raw_score = self.anomaly_detector.score_one(*args)
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
return
|
|
280
|
+
raw_score = self.anomaly_detector.score_one(*args, **kwargs)
|
|
281
|
+
minimum = self.min.get()
|
|
282
|
+
maximum = self.max.get()
|
|
283
|
+
if minimum is None or maximum is None:
|
|
284
|
+
return 0.0
|
|
285
|
+
return (raw_score - minimum) / (maximum - minimum) if maximum > minimum else 0.0
|
|
286
|
+
|
|
287
|
+
def _update_score(self, score: float) -> None:
|
|
288
|
+
self.min.update(score)
|
|
289
|
+
self.max.update(score)
|