scada-fault-bench 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- scada_fault_bench-0.1.0/LICENSE +21 -0
- scada_fault_bench-0.1.0/PKG-INFO +254 -0
- scada_fault_bench-0.1.0/README.md +219 -0
- scada_fault_bench-0.1.0/pyproject.toml +65 -0
- scada_fault_bench-0.1.0/setup.cfg +4 -0
- scada_fault_bench-0.1.0/src/scada_fault_bench/__init__.py +6 -0
- scada_fault_bench-0.1.0/src/scada_fault_bench/__main__.py +3 -0
- scada_fault_bench-0.1.0/src/scada_fault_bench/_version.py +1 -0
- scada_fault_bench-0.1.0/src/scada_fault_bench/base.py +156 -0
- scada_fault_bench-0.1.0/src/scada_fault_bench/baselines.py +274 -0
- scada_fault_bench-0.1.0/src/scada_fault_bench/cli.py +162 -0
- scada_fault_bench-0.1.0/src/scada_fault_bench/data.py +154 -0
- scada_fault_bench-0.1.0/src/scada_fault_bench/evaluate.py +167 -0
- scada_fault_bench-0.1.0/src/scada_fault_bench/faults.py +485 -0
- scada_fault_bench-0.1.0/src/scada_fault_bench/features.py +365 -0
- scada_fault_bench-0.1.0/src/scada_fault_bench/portfolio.py +365 -0
- scada_fault_bench-0.1.0/src/scada_fault_bench/prepare.py +51 -0
- scada_fault_bench-0.1.0/src/scada_fault_bench/real.py +156 -0
- scada_fault_bench-0.1.0/src/scada_fault_bench/report.py +155 -0
- scada_fault_bench-0.1.0/src/scada_fault_bench/runner.py +168 -0
- scada_fault_bench-0.1.0/src/scada_fault_bench/scenario.py +136 -0
- scada_fault_bench-0.1.0/src/scada_fault_bench/solar.py +37 -0
- scada_fault_bench-0.1.0/src/scada_fault_bench/stats.py +89 -0
- scada_fault_bench-0.1.0/src/scada_fault_bench/summary.py +163 -0
- scada_fault_bench-0.1.0/src/scada_fault_bench/synthetic.py +103 -0
- scada_fault_bench-0.1.0/src/scada_fault_bench.egg-info/PKG-INFO +254 -0
- scada_fault_bench-0.1.0/src/scada_fault_bench.egg-info/SOURCES.txt +35 -0
- scada_fault_bench-0.1.0/src/scada_fault_bench.egg-info/dependency_links.txt +1 -0
- scada_fault_bench-0.1.0/src/scada_fault_bench.egg-info/entry_points.txt +2 -0
- scada_fault_bench-0.1.0/src/scada_fault_bench.egg-info/requires.txt +14 -0
- scada_fault_bench-0.1.0/src/scada_fault_bench.egg-info/top_level.txt +1 -0
- scada_fault_bench-0.1.0/tests/test_data.py +73 -0
- scada_fault_bench-0.1.0/tests/test_docs.py +163 -0
- scada_fault_bench-0.1.0/tests/test_evaluate.py +344 -0
- scada_fault_bench-0.1.0/tests/test_faults.py +211 -0
- scada_fault_bench-0.1.0/tests/test_methods.py +193 -0
- scada_fault_bench-0.1.0/tests/test_pipeline.py +171 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Konrad Waleszczyński
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,254 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: scada-fault-bench
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: An open benchmark for detecting faults in wind and PV SCADA data: injected faults with known ground truth on open datasets in the scada-schema format, one interface for detectors, one evaluation with block-bootstrap confidence intervals.
|
|
5
|
+
Author: Konrad Waleszczyński
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/kwwaleszczynski-del/scada-fault-bench
|
|
8
|
+
Project-URL: Documentation, https://kwwaleszczynski-del.github.io/scada-fault-bench/
|
|
9
|
+
Project-URL: Issues, https://github.com/kwwaleszczynski-del/scada-fault-bench/issues
|
|
10
|
+
Keywords: scada,wind-energy,photovoltaics,fault-detection,anomaly-detection,benchmark,scada-schema
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Intended Audience :: Science/Research
|
|
13
|
+
Classifier: Operating System :: OS Independent
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
18
|
+
Classifier: Topic :: Scientific/Engineering
|
|
19
|
+
Requires-Python: >=3.11
|
|
20
|
+
Description-Content-Type: text/markdown
|
|
21
|
+
License-File: LICENSE
|
|
22
|
+
Requires-Dist: numpy>=1.26
|
|
23
|
+
Requires-Dist: pandas>=2.1
|
|
24
|
+
Requires-Dist: scikit-learn>=1.4
|
|
25
|
+
Requires-Dist: scipy>=1.11
|
|
26
|
+
Requires-Dist: scada-schema>=0.1.0
|
|
27
|
+
Provides-Extra: plots
|
|
28
|
+
Requires-Dist: matplotlib>=3.8; extra == "plots"
|
|
29
|
+
Provides-Extra: dev
|
|
30
|
+
Requires-Dist: pytest>=8; extra == "dev"
|
|
31
|
+
Requires-Dist: ruff>=0.6; extra == "dev"
|
|
32
|
+
Requires-Dist: build>=1.2; extra == "dev"
|
|
33
|
+
Requires-Dist: matplotlib>=3.8; extra == "dev"
|
|
34
|
+
Dynamic: license-file
|
|
35
|
+
|
|
36
|
+
# scada-fault-bench
|
|
37
|
+
|
|
38
|
+
An open benchmark for fault detection in wind and PV SCADA data. Faults with a known position in time, device and size are injected into blocks cut from open datasets (the data are not selected for health: known stops, night and real outages are left out of the count of false alarms, nothing else); any detector that reads a table of the [`scada-schema`](https://github.com/kwwaleszczynski-del/scada-schema) format is scored by one evaluation, with confidence intervals. A second part scores the same methods on real, labelled faults of CARE to Compare.
|
|
39
|
+
|
|
40
|
+
Not on PyPI yet. A DOI is assigned by Zenodo when the first release is archived. Live results: [kwwaleszczynski-del.github.io/scada-fault-bench](https://kwwaleszczynski-del.github.io/scada-fault-bench/).
|
|
41
|
+
|
|
42
|
+
**How to read the numbers.** An alarm that happens to fall on the affected rows of an event is not a detection of the fault. Every table therefore shows the *chance* level next to recall (the recall that the same method reaches on the untouched test block, matched to the same events) and the *excess recall* over it; the other main columns (row F1, AUROC and average precision on affected against healthy rows, false alarm hours) do not depend on where alarms fall by luck. Methods marked with an asterisk are scored only on the fault types whose changed channel they read (column *Fault types scored* in [The methods](#the-methods)), so their rows cannot be compared with rows of methods scored on all types.
|
|
43
|
+
|
|
44
|
+
**What this is not.** The injected faults are a signal-level model, not recorded failures. A method that does well here is not shown to be better in general, and a method that does badly may be badly matched to these models rather than bad. The tables describe this benchmark, these plants and these fault models, and nothing else. See [Limitations](#limitations).
|
|
45
|
+
|
|
46
|
+
## What was run
|
|
47
|
+
|
|
48
|
+
12 methods (6 simple baselines and 6 reference methods called from other repositories of this portfolio) on 7 datasets: 3 wind farms and 4 PV systems. 2,213 faults were injected: 956 into the wind farms and 1,257 into the PV systems, 12 fault types in all (8 for wind, 8 for PV), each at 5 sizes. 4 test blocks per dataset; the rest of the year (or two years), including the weeks after a test block, is training data. 1 injection seed per fold for the wind farms and 2 for the PV systems (a PV plant has one inverter or met station to carry all events of a block, so it gets fewer events per seed). The real-fault part has 12 labelled anomalies and 10 normal datasets of CARE farm A. Every number below is produced by `scada-fault-bench run` and `summarise` and written to `results/summary.json`; this README is rendered from it and a test fails when they differ.
|
|
49
|
+
|
|
50
|
+
### Wind farms
|
|
51
|
+
|
|
52
|
+
All datasets and the fault types each method is scored on, added. Values in brackets are 95 % block-bootstrap intervals. The excess recall has an interval above zero for: `zscore`, `ewma`, `cusum`, `iforest`, `rules`, `scada_qc`, `nbm_rc`, `nbm_gbm`, `power_curve_kpi`. The highest AUROC is that of `nbm_gbm`; the methods whose AUROC interval overlaps its interval are `nbm_gbm` (the intervals describe the sampling of weeks and events in this benchmark, so a method outside that list is not shown to be worse on other data, and one inside it is not shown to be equal).
|
|
53
|
+
|
|
54
|
+
| Method | Excess recall | Recall | Chance | Precision | Row F1 | AUROC | AP | Delay (h) | False alarms (h/yr) |
|
|
55
|
+
| :--- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
|
|
56
|
+
| Robust z-score (`zscore`) | 0.23 (0.20-0.26) | 0.91 (0.89-0.93) | 0.68 | 0.14 (0.12-0.16) | 0.37 (0.33-0.42) | 0.68 (0.66-0.69) | 0.31 (0.26-0.36) | 49 (43-56) | 180 (117-256) |
|
|
57
|
+
| EWMA chart (`ewma`) | 0.38 (0.33-0.44) | 0.70 (0.65-0.75) | 0.32 | 0.23 (0.19-0.29) | 0.39 (0.32-0.46) | 0.75 (0.73-0.78) | 0.32 (0.23-0.44) | 81 (73-90) | 265 (144-411) |
|
|
58
|
+
| CUSUM chart (`cusum`) | 0.42 (0.36-0.48) | 0.57 (0.51-0.63) | 0.16 | 0.27 (0.20-0.38) | 0.39 (0.32-0.47) | 0.77 (0.74-0.79) | 0.35 (0.27-0.43) | 97 (88-108) | 327 (175-527) |
|
|
59
|
+
| Isolation Forest (`iforest`) | 0.28 (0.23-0.32) | 0.82 (0.77-0.86) | 0.55 | 0.19 (0.16-0.23) | 0.28 (0.24-0.32) | 0.70 (0.68-0.71) | 0.21 (0.17-0.27) | 68 (60-76) | 185 (105-288) |
|
|
60
|
+
| Expert rules (`rules`) | 0.26 (0.23-0.28) | 0.35 (0.30-0.39) | 0.09 | 0.15 (0.11-0.25) | 0.09 (0.08-0.11) | 0.62 (0.59-0.65) | 0.13 (0.11-0.15) | 45 (32-56) | 97 (26-180) |
|
|
61
|
+
| scada-schema QC (`scada_qc`) | 0.22 (0.20-0.25) | 0.53 (0.47-0.60) | 0.31 | 0.12 (0.10-0.15) | 0.06 (0.05-0.08) | 0.60 (0.55-0.64) | 0.09 (0.07-0.12) | 93 (81-105) | 113 (56-181) |
|
|
62
|
+
| Thermal model (RC) (`nbm_rc`) * | 0.53 (0.47-0.59) | 0.87 (0.81-0.92) | 0.35 | 0.13 (0.11-0.17) | 0.30 (0.25-0.34) | 0.72 (0.69-0.75) | 0.27 (0.23-0.32) | 77 (62-92) | 171 (117-230) |
|
|
63
|
+
| Thermal model (GBM) (`nbm_gbm`) * | 0.51 (0.44-0.58) | 0.88 (0.83-0.93) | 0.37 | 0.12 (0.10-0.15) | 0.34 (0.29-0.40) | 0.85 (0.82-0.87) | 0.39 (0.32-0.47) | 75 (61-91) | 133 (81-193) |
|
|
64
|
+
| Power curve (method of bins) (`power_curve_kpi`) * | 0.24 (0.19-0.30) | 0.69 (0.61-0.77) | 0.45 | 0.14 (0.11-0.19) | 0.16 (0.13-0.19) | 0.68 (0.66-0.70) | 0.12 (0.10-0.16) | 80 (68-93) | 160 (83-242) |
|
|
65
|
+
|
|
66
|
+
### PV systems
|
|
67
|
+
|
|
68
|
+
The excess recall has an interval above zero for: `zscore`, `ewma`, `cusum`, `iforest`, `rules`, `scada_qc`, `pv_autoencoder`, `pv_outage_expected`, `pv_outage_iforest`. The highest AUROC is that of `iforest`; the methods whose AUROC interval overlaps its interval are `ewma`, `cusum`, `iforest`, `pv_outage_iforest` (the intervals describe the sampling of weeks and events in this benchmark, so a method outside that list is not shown to be worse on other data, and one inside it is not shown to be equal).
|
|
69
|
+
|
|
70
|
+
| Method | Excess recall | Recall | Chance | Precision | Row F1 | AUROC | AP | Delay (h) | False alarms (h/yr) |
|
|
71
|
+
| :--- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
|
|
72
|
+
| Robust z-score (`zscore`) | 0.30 (0.27-0.34) | 0.65 (0.61-0.69) | 0.35 | 0.36 (0.33-0.40) | 0.22 (0.20-0.25) | 0.59 (0.58-0.60) | 0.30 (0.28-0.32) | 40 (35-44) | 173 (84-316) |
|
|
73
|
+
| EWMA chart (`ewma`) | 0.32 (0.27-0.36) | 0.48 (0.43-0.52) | 0.16 | 0.53 (0.43-0.66) | 0.24 (0.21-0.27) | 0.66 (0.64-0.67) | 0.34 (0.30-0.40) | 49 (42-56) | 175 (83-319) |
|
|
74
|
+
| CUSUM chart (`cusum`) | 0.30 (0.26-0.33) | 0.43 (0.38-0.49) | 0.14 | 0.46 (0.38-0.56) | 0.27 (0.24-0.30) | 0.66 (0.64-0.67) | 0.32 (0.28-0.38) | 48 (41-56) | 317 (164-544) |
|
|
75
|
+
| Isolation Forest (`iforest`) | 0.37 (0.33-0.42) | 0.63 (0.57-0.67) | 0.25 | 0.45 (0.40-0.52) | 0.22 (0.20-0.25) | 0.66 (0.65-0.67) | 0.32 (0.28-0.38) | 38 (34-43) | 214 (87-409) |
|
|
76
|
+
| Expert rules (`rules`) | 0.32 (0.29-0.35) | 0.36 (0.33-0.40) | 0.05 | 0.58 (0.46-0.76) | 0.13 (0.11-0.15) | 0.54 (0.53-0.55) | 0.24 (0.22-0.25) | 10 (6-14) | 85 (14-195) |
|
|
77
|
+
| scada-schema QC (`scada_qc`) | 0.14 (0.12-0.17) | 0.48 (0.42-0.55) | 0.34 | 0.20 (0.19-0.22) | 0.12 (0.10-0.13) | 0.52 (0.51-0.53) | 0.19 (0.18-0.20) | 34 (29-40) | 443 (317-577) |
|
|
78
|
+
| Autoencoder (PV) (`pv_autoencoder`) | 0.31 (0.28-0.35) | 0.58 (0.54-0.63) | 0.27 | 0.38 (0.34-0.43) | 0.19 (0.17-0.21) | 0.60 (0.59-0.61) | 0.29 (0.27-0.31) | 37 (31-42) | 173 (118-249) |
|
|
79
|
+
| Shortfall vs PVWatts model (`pv_outage_expected`) * | 0.31 (0.27-0.36) | 0.59 (0.53-0.66) | 0.28 | 0.37 (0.32-0.41) | 0.20 (0.18-0.22) | 0.62 (0.61-0.63) | 0.26 (0.23-0.31) | 30 (26-35) | 212 (110-374) |
|
|
80
|
+
| Outage features + Isolation Forest (`pv_outage_iforest`) * | 0.24 (0.20-0.28) | 0.44 (0.40-0.49) | 0.20 | 0.45 (0.35-0.56) | 0.19 (0.17-0.22) | 0.66 (0.64-0.67) | 0.29 (0.26-0.32) | 36 (31-42) | 143 (83-228) |
|
|
81
|
+
|
|
82
|
+
### Excess recall by fault type
|
|
83
|
+
|
|
84
|
+
Share of injected events that were detected, minus the share that the same alarms hit by chance, all sizes and datasets of the profile added; a dash is a type the method is not scored on. The sensitivity curve (excess recall against the size of the fault) is on the [project page](https://kwwaleszczynski-del.github.io/scada-fault-bench/).
|
|
85
|
+
|
|
86
|
+
Wind:
|
|
87
|
+
|
|
88
|
+
| Method | anemometer drift | nacelle direction offset | derating | power loss | temperature drift | flatline | unit jump | clock shift |
|
|
89
|
+
| :--- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
|
|
90
|
+
| `zscore` | 0.13 | 0.06 | 0.11 | 0.01 | 0.04 | 0.81 | 0.65 | 0.07 |
|
|
91
|
+
| `ewma` | 0.58 | 0.47 | 0.42 | 0.28 | 0.29 | 0.47 | 0.47 | 0.08 |
|
|
92
|
+
| `cusum` | 0.71 | 0.66 | 0.37 | 0.47 | 0.42 | 0.27 | 0.36 | 0.07 |
|
|
93
|
+
| `iforest` | 0.29 | 0.24 | 0.19 | 0.04 | 0.07 | 0.86 | 0.30 | 0.23 |
|
|
94
|
+
| `rules` | 0.19 | 0.13 | 0.00 | 0.00 | 0.00 | 0.85 | 0.88 | 0.01 |
|
|
95
|
+
| `scada_qc` | 0.03 | 0.01 | 0.62 | 0.01 | 0.00 | 0.95 | 0.03 | 0.17 |
|
|
96
|
+
| `nbm_rc` | - | - | - | - | 0.15 | - | 0.90 | - |
|
|
97
|
+
| `nbm_gbm` | - | - | - | - | 0.16 | - | 0.87 | - |
|
|
98
|
+
| `power_curve_kpi` | 0.20 | - | 0.29 | 0.11 | - | 0.59 | - | 0.02 |
|
|
99
|
+
|
|
100
|
+
PV:
|
|
101
|
+
|
|
102
|
+
| Method | temperature drift | flatline | unit jump | clock shift | string loss | shading | inverter outage | pyranometer drift |
|
|
103
|
+
| :--- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
|
|
104
|
+
| `zscore` | 0.04 | 0.55 | 0.88 | 0.46 | -0.01 | 0.04 | 0.47 | 0.06 |
|
|
105
|
+
| `ewma` | 0.27 | 0.39 | 0.73 | 0.15 | 0.24 | -0.04 | 0.66 | 0.17 |
|
|
106
|
+
| `cusum` | 0.24 | 0.37 | 0.65 | 0.24 | 0.23 | -0.01 | 0.57 | 0.11 |
|
|
107
|
+
| `iforest` | 0.14 | 0.59 | 0.56 | 0.59 | 0.16 | 0.16 | 0.73 | 0.07 |
|
|
108
|
+
| `rules` | 0.00 | 0.52 | 0.77 | 0.38 | -0.02 | 0.00 | 0.93 | 0.00 |
|
|
109
|
+
| `scada_qc` | 0.01 | 0.77 | 0.35 | 0.08 | 0.00 | 0.00 | -0.03 | 0.00 |
|
|
110
|
+
| `pv_autoencoder` | 0.00 | 0.59 | 0.89 | 0.44 | -0.05 | -0.02 | 0.74 | -0.01 |
|
|
111
|
+
| `pv_outage_expected` | - | 0.37 | - | 0.39 | 0.16 | 0.21 | 0.76 | -0.01 |
|
|
112
|
+
| `pv_outage_iforest` | - | 0.35 | - | 0.24 | 0.04 | 0.02 | 0.78 | 0.02 |
|
|
113
|
+
|
|
114
|
+
### Real faults: CARE to Compare, farm A
|
|
115
|
+
|
|
116
|
+
The same methods, trained on the first year of each turbine and scored on the prediction section; the truth is every row of the labelled anomaly that has a power reading (the turbine is usually in a downtime state inside an event, which is part of the fault and not a reason to ignore those rows). The chance level is the recall that the alarms of the method reach when they are shifted in time at random over the prediction section. These are single-turbine tables, so the residuals that compare a turbine with its neighbours do not exist and the methods that need neighbours work with less. 12 anomalies and 10 normal datasets are too few to rank methods; the intervals show how little this part can say. Precision is not reported: only the normal datasets contribute healthy time with no fault in it, and that is not a precision.
|
|
117
|
+
|
|
118
|
+
| Method | Excess recall | Recall | Chance | AUROC | AP | Delay (h) | False alarms (h/yr) |
|
|
119
|
+
| :--- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
|
|
120
|
+
| Robust z-score (`zscore`) | 0.09 (-0.05-0.20) | 0.83 (0.58-1.00) | 0.75 | 0.53 (0.49-0.57) | 0.44 (0.24-0.66) | 99 (48-162) | 30 (14-49) |
|
|
121
|
+
| EWMA chart (`ewma`) | -0.02 (-0.21-0.11) | 0.25 (0.00-0.53) | 0.27 | 0.65 (0.56-0.75) | 0.56 (0.32-0.77) | n/a | 16 (0-58) |
|
|
122
|
+
| CUSUM chart (`cusum`) | -0.12 (-0.30-0.00) | 0.00 (0.00-0.00) | 0.12 | 0.66 (0.54-0.78) | 0.56 (0.31-0.80) | n/a | 27 (0-68) |
|
|
123
|
+
| Isolation Forest (`iforest`) | -0.04 (-0.24-0.10) | 0.50 (0.22-0.79) | 0.54 | 0.54 (0.49-0.59) | 0.45 (0.25-0.67) | 165 (37-363) | 9 (1-23) |
|
|
124
|
+
| Expert rules (`rules`) | 0.00 (0.00-0.00) | 0.00 (0.00-0.00) | 0.00 | 0.50 (0.50-0.50) | 0.41 (0.22-0.63) | n/a | 0 (0-0) |
|
|
125
|
+
| scada-schema QC (`scada_qc`) | 0.06 (-0.09-0.18) | 0.67 (0.38-0.92) | 0.60 | 0.48 (0.34-0.59) | 0.42 (0.25-0.65) | 129 (75-187) | 19 (2-42) |
|
|
126
|
+
| Thermal model (RC) (`nbm_rc`) | -0.03 (-0.17-0.08) | 0.17 (0.00-0.40) | 0.20 | 0.57 (0.52-0.62) | 0.51 (0.27-0.72) | n/a | 59 (0-147) |
|
|
127
|
+
| Thermal model (GBM) (`nbm_gbm`) | -0.01 (-0.18-0.12) | 0.25 (0.00-0.50) | 0.26 | 0.54 (0.47-0.60) | 0.49 (0.25-0.70) | n/a | 63 (0-156) |
|
|
128
|
+
| Power curve (method of bins) (`power_curve_kpi`) | 0.03 (-0.09-0.13) | 0.67 (0.38-0.92) | 0.64 | 0.54 (0.47-0.60) | 0.44 (0.23-0.66) | 117 (66-172) | 64 (3-166) |
|
|
129
|
+
|
|
130
|
+
## The faults
|
|
131
|
+
|
|
132
|
+
Every fault is a change of the logged signals of one device. The start, the device and the size of an event are drawn from a seeded generator (the same table, type and seed give the same events), events of one device keep 2 days apart. The truth of an event starts at the first row where the change exceeds a tolerance and ends at the last such row, leaving out the rows where an alarm is neither right nor wrong (night, a known stop, missing data); an event that would show in fewer than 3 hours of data is redrawn. Where the change would move other channels by simple arithmetic (power, current and energy, the energy meter counter) they are changed together.
|
|
133
|
+
|
|
134
|
+
| Type | Profile | What is injected | Five levels |
|
|
135
|
+
| :--- | :--- | :--- | :--- |
|
|
136
|
+
| `anemometer_drift` | wind | nacelle anemometer gain drifts linearly to +-m % (sign drawn per event); the second anemometer is untouched | 3, 6, 9, 12, 15 % (size) |
|
|
137
|
+
| `nacelle_dir_offset` | wind | nacelle position sensor offset by +-m degrees (step); the vane and the absolute direction are untouched | 3, 6, 10, 15, 25 deg (size) |
|
|
138
|
+
| `derating` | wind | active power capped at (1 - m %) of rated power; power, current and energy move together | 5, 10, 20, 30, 40 % of rated (size) |
|
|
139
|
+
| `power_loss` | wind | active power reads (1 - m %) of the true value at every power level (underperformance) | 2, 4, 6, 10, 15 % (size) |
|
|
140
|
+
| `thermal_drift` | wind+pv | temperature channel drifts linearly to +m K (gearbox oil of a turbine, heat sink of an inverter) | 2, 4, 6, 10, 15 K (size) |
|
|
141
|
+
| `flatline` | wind+pv | sensor stuck at its first value of the window (wind speed of a turbine, plane-of-array irradiance of a met station) | 3, 6, 12, 24, 48 h (duration) |
|
|
142
|
+
| `unit_jump` | wind+pv | temperature logged in degF instead of degC (gearbox oil of a turbine, module temperature of a met station) | 6, 12, 24, 48, 96 h (duration) |
|
|
143
|
+
| `clock_shift` | wind+pv | all channels of one device are delayed by m minutes against the others (logger clock error) | 10, 30, 60, 120, 360 min (size) |
|
|
144
|
+
| `string_loss` | pv | a share of the strings of an inverter is lost: AC and DC power and current scale by (1 - m %) | 2, 5, 10, 20, 35 % (size) |
|
|
145
|
+
| `shading` | pv | a fixed obstacle in the west shades the plant at low sun in the afternoon: power scales by (1 - m %) when the sun is below 30 deg and west of south-west | 5, 10, 20, 35, 50 % (size) |
|
|
146
|
+
| `inverter_outage` | pv | inverter produces nothing for the duration of the window (power and currents zero) | 2, 6, 12, 24, 72 h (duration) |
|
|
147
|
+
| `pyranometer_drift` | pv | plane-of-array irradiance gain drifts linearly to +-m % (soiling, ageing; sign drawn per event) | 2, 4, 6, 10, 15 % (size) |
|
|
148
|
+
|
|
149
|
+
## The methods
|
|
150
|
+
|
|
151
|
+
| Name | Profile | Family | Threshold | Fault types scored | Description |
|
|
152
|
+
| :--- | :--- | :--- | :--- | :--- | :--- |
|
|
153
|
+
| `zscore` | wind+pv | baseline | calibrated | all | largest absolute residual of the shared features (one-hour median), each divided by its 99.5 % quantile on the training weeks so that no single heavy-tailed feature decides the score |
|
|
154
|
+
| `ewma` | wind+pv | baseline | calibrated | all | exponentially weighted moving average of every residual clipped at 5 sigma (time constant about one day), largest standardised value |
|
|
155
|
+
| `cusum` | wind+pv | baseline | calibrated | all | two-sided CUSUM of every residual clipped at 5 sigma, on hourly values (allowance 1 sigma, memory 3 days), largest statistic |
|
|
156
|
+
| `iforest` | wind+pv | baseline | calibrated | all | Isolation Forest (100 trees, 4096 samples) on the vector of standardised residuals; missing residuals set to 0 |
|
|
157
|
+
| `rules` | wind+pv | baseline | fixed | all | fixed physical rules, no learning: the two anemometers differ by more than 1.5 m/s, a wind speed with zero variation over an hour while the turbine produces, gearbox oil outside -20 to 100 degC, vane + nacelle - absolute direction beyond 25 deg, power above 110 % of rated; PV: irradiance frozen in daylight, inverter below 2 % of rated in sun above 300 W/m2, module temperature outside -30 to 90 degC, power above 110 % of rated |
|
|
158
|
+
| `scada_qc` | wind+pv | baseline | fixed | all | rows flagged by the row-level checks of scada_schema.qc (flatline, spikes, out of range, units, power curve, wind direction, energy counter, energy against power) |
|
|
159
|
+
| `nbm_rc` | wind | portfolio | calibrated | temperature drift, unit jump | wind-thermal-nbm RCModel: first-order thermal model of the gearbox oil temperature from load, rotor speed, nacelle and ambient temperature; the standardised residual is the score Source: wind-thermal-nbm: wind_thermal_nbm.thermal.RCModel. |
|
|
160
|
+
| `nbm_gbm` | wind | portfolio | calibrated | temperature drift, unit jump | wind-thermal-nbm GBMModel: gradient boosting on the same inputs and their trailing means (1-12 h) for the gearbox oil temperature; the standardised residual is the score Source: wind-thermal-nbm: wind_thermal_nbm.ml.GBMModel. |
|
|
161
|
+
| `power_curve_kpi` | wind | portfolio | calibrated | anemometer drift, derating, power loss, flatline, clock shift | res-asset-kpi bin_power_curve per turbine on the training weeks; the standardised residual of power against the curve at the measured wind speed is the score Source: res-asset-kpi: res_asset_kpi.wind.bin_power_curve, curve_power (the method of wind-power-curve-kelmarsh). |
|
|
162
|
+
| `pv_autoencoder` | pv | portfolio | calibrated | all | pv-anomaly-autoencoder AutoencoderDetector per inverter: windows of 8 steps of power, performance ratio, irradiance, clear-sky index, temperatures and sun position; the reconstruction error (mse), smoothed over one hour, is the score; trained for at most 100 epochs on 80 % of the training days Source: pv-anomaly-autoencoder: pv_anomaly.autoencoder.AutoencoderDetector, pv_anomaly.features.build_features. |
|
|
163
|
+
| `pv_outage_expected` | pv | portfolio | calibrated | string loss, shading, inverter outage, pyranometer drift, flatline, clock shift | pv-outage-classifier detectors.score('expected'): shortfall of inverter power against a PVWatts-style model driven by the met station's irradiance and module temperature, calibrated per bin on the table that is scored (as in that repository); shortfall only Source: pv-outage-classifier: pv_outage.features.compute, pv_outage.detectors.score('expected'). |
|
|
164
|
+
| `pv_outage_iforest` | pv | portfolio | calibrated | string loss, shading, inverter outage, pyranometer drift, flatline, clock shift | pv-outage-classifier detectors.score('iforest'): Isolation Forest on that repository's step features (residuals, level, sun position, ...), fitted on the table that is scored, as in that repository Source: pv-outage-classifier: pv_outage.features.compute, pv_outage.detectors.score('iforest'). |
|
|
165
|
+
|
|
166
|
+
The other repositories of the portfolio were looked at and are not wrapped, for reasons that are properties of their inputs and not of their quality: `wind-wake-aware-verification` and `wind-bayes-twin` work on daily farm-level statistics with wake models, `pv-string-iv-faults` needs per-MPPT voltage and current channels that none of these datasets has, `pv-shade-fingerprint` produces a one-off diagnosis of a whole record and not a score per time stamp. Details in [docs/METHODS.md](docs/METHODS.md).
|
|
167
|
+
|
|
168
|
+
## The datasets
|
|
169
|
+
|
|
170
|
+
| Name | Dataset | Profile | Period (UTC) | Injected events |
|
|
171
|
+
| :--- | :--- | :--- | :--- | ---: |
|
|
172
|
+
| `kelmarsh_2019` | Kelmarsh 2019 | wind | 2019-01-01 to 2020-01-01 | 320 |
|
|
173
|
+
| `penmanshiel_2020` | Penmanshiel 2020 | wind | 2020-01-01 to 2021-01-01 | 320 |
|
|
174
|
+
| `hill_of_towie_2021` | Hill of Towie 2021 | wind | 2021-01-01 to 2022-01-01 | 316 |
|
|
175
|
+
| `pvdaq_34` | PVDAQ system 34 | pv | 2016-01-01 to 2018-01-01 | 318 |
|
|
176
|
+
| `pvdaq_1276` | PVDAQ system 1276 | pv | 2016-01-01 to 2018-01-01 | 308 |
|
|
177
|
+
| `pvdaq_1278` | PVDAQ system 1278 | pv | 2016-01-01 to 2018-01-01 | 317 |
|
|
178
|
+
| `pvdaq_35` | PVDAQ system 35 | pv | 2016-01-01 to 2018-01-01 | 314 |
|
|
179
|
+
|
|
180
|
+
The tables are built from the original open files with the loaders of `scada-schema` (`scada-fault-bench prepare`); nothing is redistributed. Kelmarsh and Penmanshiel are Cubico (CC BY 4.0), Hill of Towie is RES (CC BY 4.0), PVDAQ is NREL (CC BY 4.0). CARE to Compare is CC BY-SA 4.0, which also applies to datasets derived from it.
|
|
181
|
+
|
|
182
|
+
## The evaluation
|
|
183
|
+
|
|
184
|
+
For every dataset and fold the table is cut into a test block (91 days for wind farms, 150 for PV systems, which have one inverter or met station to carry all events), a margin of 7 days and the training part. A method is fitted on the training part without its calibration weeks (one week in four) and scored on the calibration weeks; the alarm threshold is the lowest one at which the merged alarms (below) cover at most 1 % of the healthy rows of the calibration weeks, so every calibrated method works at about the same operating point (the same hours of alarm on healthy data). Rule-based methods have a fixed threshold, and their false alarm rate is a result. Alarms closer than 6 hours are merged and alarms with less than 0.5 hours of flagged rows are dropped.
|
|
185
|
+
|
|
186
|
+
An event is detected when an alarm of its device covers at least one affected row of it (a row where the injected change is above its tolerance and an alarm can be right or wrong). The delay is the time from the first affected row to the first alarm on an affected row. An alarm is true when it covers an affected row, ignored when it lies only on rows where an alarm is neither right nor wrong (known stops, night, real outages of the open data), and false otherwise. The *chance* recall of a method is the share of the events that the alarms of the same method on the untouched test block would have hit; the *excess* is the difference. A wind alarm belongs to a turbine; a PV alarm belongs to the plant, because one met station and one or two inverters do not let a plant-level fault be attributed to a device. False alarms are counted in hours per year of healthy time. AUROC and average precision compare the ranks of the scores of affected and of healthy rows, where the rank of a score is its position among the scores of the healthy calibration rows of the same fold; they do not use the alarm threshold, but they do depend on how finely a score separates values near the healthy range, and a score with few distinct values (the rules) has a coarse rank.
|
|
187
|
+
|
|
188
|
+
Confidence intervals come from a bootstrap over time blocks: the unit of resampling is 4 consecutive weeks of a test block of one dataset and fold (longer than most events), with all devices, fault types and methods of those weeks drawn together, 1000 resamples; for the CARE part a block is one dataset. A mean delay over fewer than five detected events is not shown. They describe the sampling of weeks, devices and events inside this benchmark and nothing beyond it.
|
|
189
|
+
|
|
190
|
+
## Use it
|
|
191
|
+
|
|
192
|
+
```bash
|
|
193
|
+
pip install -e . # installs scada-schema from PyPI
|
|
194
|
+
scada-fault-bench list
|
|
195
|
+
scada-fault-bench prepare --datasets kelmarsh_2019 pvdaq_34 --fetch # builds data/<name>.parquet with scada-schema
|
|
196
|
+
scada-fault-bench prepare --datasets care_A --fetch # the 22 CARE datasets of farm A (real faults)
|
|
197
|
+
scada-fault-bench run --datasets kelmarsh_2019 --folds 0 1 --out results
|
|
198
|
+
scada-fault-bench summarise --out results
|
|
199
|
+
```
|
|
200
|
+
|
|
201
|
+
A method is a class with `fit(train, meta)` and `score(table, meta)`; the score is one non-negative number per row of the table (higher means more anomalous, `NaN` means not assessable):
|
|
202
|
+
|
|
203
|
+
```python
|
|
204
|
+
import numpy as np
|
|
205
|
+
from scada_fault_bench.base import Detector
|
|
206
|
+
|
|
207
|
+
class MyMethod(Detector):
|
|
208
|
+
name = "my_method"
|
|
209
|
+
title = "My method"
|
|
210
|
+
description = "what it does"
|
|
211
|
+
profiles = ("wind",)
|
|
212
|
+
|
|
213
|
+
def fit(self, train, meta):
|
|
214
|
+
self.mean = train["p_ac"].mean()
|
|
215
|
+
return self
|
|
216
|
+
|
|
217
|
+
def score(self, table, meta):
|
|
218
|
+
return np.abs(table["p_ac"].to_numpy() - self.mean)
|
|
219
|
+
```
|
|
220
|
+
|
|
221
|
+
```bash
|
|
222
|
+
scada-fault-bench run --datasets kelmarsh_2019 --plugin my_package.my_module:MyMethod --out results
|
|
223
|
+
```
|
|
224
|
+
|
|
225
|
+
`scada-fault-bench inject --dataset kelmarsh_2019 --type derating --out out/derating` writes one injected test block, its truth and the row labels, for use with any other tool. `power_curve_kpi` needs `res-asset-kpi`, which `scada-schema` installs from PyPI. The other reference methods are optional dependencies that are installed from GitHub (they are not on PyPI), and `pv_autoencoder` also needs `torch`; a method whose repository cannot be imported is skipped and listed as unavailable in `results/summary.json`. The full run in this repository used all of them:
|
|
226
|
+
|
|
227
|
+
```bash
|
|
228
|
+
pip install torch # for pv_autoencoder
|
|
229
|
+
pip install git+https://github.com/kwwaleszczynski-del/wind-thermal-nbm git+https://github.com/kwwaleszczynski-del/pv-anomaly-autoencoder git+https://github.com/kwwaleszczynski-del/pv-outage-classifier
|
|
230
|
+
```
|
|
231
|
+
|
|
232
|
+
## Limitations
|
|
233
|
+
|
|
234
|
+
* The faults are injected into recorded signals. Physical faults change more channels, and in ways that depend on the machine; the models here favour detectors that look at the channel that was changed. Nothing here says how a method finds a real fault except the CARE part, which is small.
|
|
235
|
+
* Faults of one type are drawn from one simple model with five sizes. The sensitivity curves describe that model.
|
|
236
|
+
* The open data are not guaranteed to be healthy. Real stops that the status log shows, night, and real inverter outages of PVDAQ are removed from the count of false alarms; real faults that are not visible in them are not, so alarms on them are counted as false. This is conservative.
|
|
237
|
+
* PV plants are one met station with one or two inverters: there is no string level and no device-level attribution. The PV results are plant-level.
|
|
238
|
+
* Calibrated methods share a false alarm budget on a calibration sample from the same year; a method whose scores drift with the seasons will exceed it, and the false alarm column shows by how much.
|
|
239
|
+
* An alarm on an affected row counts as a detection whatever its cause, so a method with many alarms also "detects" events by coincidence. The chance column and the excess recall show how much that is; the sensitivity curves show the excess and are not always monotone in the size of the fault. A method whose excess interval contains zero has not been shown to detect these faults.
|
|
240
|
+
* The simple baselines read a residual matrix that was built with the fault models in view: roughly one residual for each kind of fault (power against the curve, wind speed against neighbours and the second anemometer, direction against neighbours, gearbox temperature, frozen signals; for PV the ratio against irradiance, sun position, irradiance variation, temperatures). That makes them well suited to these faults by construction. `scada_qc` is run on the channels that the fault models touch, because it would otherwise flag quantised channels the faults do not concern. Nothing was tuned on the results, but the design was not blind to the faults.
|
|
241
|
+
* `nbm_rc`, `nbm_gbm` and `power_curve_kpi` are models taken from their repositories, not the alarm methods of those repositories (which average residuals over days and chart them with a zero-false-alarm limit), and they are scored only on the fault types in their channels; `pv_outage_*` likewise. Their overall rows say nothing about the other fault types.
|
|
242
|
+
* The calibrated methods share an operating point on healthy calibration data, but their scores have different autocorrelation, so the hours of alarm on the untouched test block still differ from method to method; the false alarm column shows it. A method whose score is too persistent to meet the budget gets the highest threshold of the search and may raise almost no alarm.
|
|
243
|
+
* The rule that hides real PV outages of the open data from the false alarm count (a whole calendar day on which at least four fifths of the sunny rows have zero power, or no power reading) is not the one-hour zero-power rule of the `rules` method; short real outages are counted as false alarms for every method.
|
|
244
|
+
* The reference methods are called through adapters that rename and rescale columns. A weak result can come from the adapter or from a method that was written for other data; the repositories that own the methods are the authority on what they do.
|
|
245
|
+
* 1000 bootstrap resamples of weeks are not independent of the fault draws: another seed gives other events. Differences within the intervals should not be read as differences.
|
|
246
|
+
* The file formats and the fault models are at version 0.1.0; results of different versions are not comparable.
|
|
247
|
+
|
|
248
|
+
## Reproduce
|
|
249
|
+
|
|
250
|
+
The run in this repository is `scada-fault-bench prepare` (all datasets, `--fetch` the first time), then `scada-fault-bench run --out results` (all datasets, all folds, 1 injection seed for wind and 2 for PV, which are the defaults) and `scada-fault-bench summarise --out results`, with one process per dataset to save time. `results/` holds the shards and `summary.json` of that run; `python scripts/render_readme.py` rebuilds this file and `python scripts/build_pages.py` the page from them. The tests in `tests/test_docs.py` compare both with the summary, and `tests/test_data.py` (marked `data`) repeats a small part of the run on the real tables.
|
|
251
|
+
|
|
252
|
+
## Citation and licence
|
|
253
|
+
|
|
254
|
+
Code: MIT. Cite the repository (`CITATION.cff`) and the datasets above. `scada-fault-bench` is built on `scada-schema` 0.1.1, whose loaders read the open datasets.
|
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
# scada-fault-bench
|
|
2
|
+
|
|
3
|
+
An open benchmark for fault detection in wind and PV SCADA data. Faults with a known position in time, device and size are injected into blocks cut from open datasets (the data are not selected for health: known stops, night and real outages are left out of the count of false alarms, nothing else); any detector that reads a table of the [`scada-schema`](https://github.com/kwwaleszczynski-del/scada-schema) format is scored by one evaluation, with confidence intervals. A second part scores the same methods on real, labelled faults of CARE to Compare.
|
|
4
|
+
|
|
5
|
+
Not on PyPI yet. A DOI is assigned by Zenodo when the first release is archived. Live results: [kwwaleszczynski-del.github.io/scada-fault-bench](https://kwwaleszczynski-del.github.io/scada-fault-bench/).
|
|
6
|
+
|
|
7
|
+
**How to read the numbers.** An alarm that happens to fall on the affected rows of an event is not a detection of the fault. Every table therefore shows the *chance* level next to recall (the recall that the same method reaches on the untouched test block, matched to the same events) and the *excess recall* over it; the other main columns (row F1, AUROC and average precision on affected against healthy rows, false alarm hours) do not depend on where alarms fall by luck. Methods marked with an asterisk are scored only on the fault types whose changed channel they read (column *Fault types scored* in [The methods](#the-methods)), so their rows cannot be compared with rows of methods scored on all types.
|
|
8
|
+
|
|
9
|
+
**What this is not.** The injected faults are a signal-level model, not recorded failures. A method that does well here is not shown to be better in general, and a method that does badly may be badly matched to these models rather than bad. The tables describe this benchmark, these plants and these fault models, and nothing else. See [Limitations](#limitations).
|
|
10
|
+
|
|
11
|
+
## What was run
|
|
12
|
+
|
|
13
|
+
12 methods (6 simple baselines and 6 reference methods called from other repositories of this portfolio) on 7 datasets: 3 wind farms and 4 PV systems. 2,213 faults were injected: 956 into the wind farms and 1,257 into the PV systems, 12 fault types in all (8 for wind, 8 for PV), each at 5 sizes. 4 test blocks per dataset; the rest of the year (or two years), including the weeks after a test block, is training data. 1 injection seed per fold for the wind farms and 2 for the PV systems (a PV plant has one inverter or met station to carry all events of a block, so it gets fewer events per seed). The real-fault part has 12 labelled anomalies and 10 normal datasets of CARE farm A. Every number below is produced by `scada-fault-bench run` and `summarise` and written to `results/summary.json`; this README is rendered from it and a test fails when they differ.
|
|
14
|
+
|
|
15
|
+
### Wind farms
|
|
16
|
+
|
|
17
|
+
All datasets and the fault types each method is scored on, added. Values in brackets are 95 % block-bootstrap intervals. The excess recall has an interval above zero for: `zscore`, `ewma`, `cusum`, `iforest`, `rules`, `scada_qc`, `nbm_rc`, `nbm_gbm`, `power_curve_kpi`. The highest AUROC is that of `nbm_gbm`; the methods whose AUROC interval overlaps its interval are `nbm_gbm` (the intervals describe the sampling of weeks and events in this benchmark, so a method outside that list is not shown to be worse on other data, and one inside it is not shown to be equal).
|
|
18
|
+
|
|
19
|
+
| Method | Excess recall | Recall | Chance | Precision | Row F1 | AUROC | AP | Delay (h) | False alarms (h/yr) |
|
|
20
|
+
| :--- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
|
|
21
|
+
| Robust z-score (`zscore`) | 0.23 (0.20-0.26) | 0.91 (0.89-0.93) | 0.68 | 0.14 (0.12-0.16) | 0.37 (0.33-0.42) | 0.68 (0.66-0.69) | 0.31 (0.26-0.36) | 49 (43-56) | 180 (117-256) |
|
|
22
|
+
| EWMA chart (`ewma`) | 0.38 (0.33-0.44) | 0.70 (0.65-0.75) | 0.32 | 0.23 (0.19-0.29) | 0.39 (0.32-0.46) | 0.75 (0.73-0.78) | 0.32 (0.23-0.44) | 81 (73-90) | 265 (144-411) |
|
|
23
|
+
| CUSUM chart (`cusum`) | 0.42 (0.36-0.48) | 0.57 (0.51-0.63) | 0.16 | 0.27 (0.20-0.38) | 0.39 (0.32-0.47) | 0.77 (0.74-0.79) | 0.35 (0.27-0.43) | 97 (88-108) | 327 (175-527) |
|
|
24
|
+
| Isolation Forest (`iforest`) | 0.28 (0.23-0.32) | 0.82 (0.77-0.86) | 0.55 | 0.19 (0.16-0.23) | 0.28 (0.24-0.32) | 0.70 (0.68-0.71) | 0.21 (0.17-0.27) | 68 (60-76) | 185 (105-288) |
|
|
25
|
+
| Expert rules (`rules`) | 0.26 (0.23-0.28) | 0.35 (0.30-0.39) | 0.09 | 0.15 (0.11-0.25) | 0.09 (0.08-0.11) | 0.62 (0.59-0.65) | 0.13 (0.11-0.15) | 45 (32-56) | 97 (26-180) |
|
|
26
|
+
| scada-schema QC (`scada_qc`) | 0.22 (0.20-0.25) | 0.53 (0.47-0.60) | 0.31 | 0.12 (0.10-0.15) | 0.06 (0.05-0.08) | 0.60 (0.55-0.64) | 0.09 (0.07-0.12) | 93 (81-105) | 113 (56-181) |
|
|
27
|
+
| Thermal model (RC) (`nbm_rc`) * | 0.53 (0.47-0.59) | 0.87 (0.81-0.92) | 0.35 | 0.13 (0.11-0.17) | 0.30 (0.25-0.34) | 0.72 (0.69-0.75) | 0.27 (0.23-0.32) | 77 (62-92) | 171 (117-230) |
|
|
28
|
+
| Thermal model (GBM) (`nbm_gbm`) * | 0.51 (0.44-0.58) | 0.88 (0.83-0.93) | 0.37 | 0.12 (0.10-0.15) | 0.34 (0.29-0.40) | 0.85 (0.82-0.87) | 0.39 (0.32-0.47) | 75 (61-91) | 133 (81-193) |
|
|
29
|
+
| Power curve (method of bins) (`power_curve_kpi`) * | 0.24 (0.19-0.30) | 0.69 (0.61-0.77) | 0.45 | 0.14 (0.11-0.19) | 0.16 (0.13-0.19) | 0.68 (0.66-0.70) | 0.12 (0.10-0.16) | 80 (68-93) | 160 (83-242) |
|
|
30
|
+
|
|
31
|
+
### PV systems
|
|
32
|
+
|
|
33
|
+
The excess recall has an interval above zero for: `zscore`, `ewma`, `cusum`, `iforest`, `rules`, `scada_qc`, `pv_autoencoder`, `pv_outage_expected`, `pv_outage_iforest`. The highest AUROC is that of `iforest`; the methods whose AUROC interval overlaps its interval are `ewma`, `cusum`, `iforest`, `pv_outage_iforest` (the intervals describe the sampling of weeks and events in this benchmark, so a method outside that list is not shown to be worse on other data, and one inside it is not shown to be equal).
|
|
34
|
+
|
|
35
|
+
| Method | Excess recall | Recall | Chance | Precision | Row F1 | AUROC | AP | Delay (h) | False alarms (h/yr) |
|
|
36
|
+
| :--- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
|
|
37
|
+
| Robust z-score (`zscore`) | 0.30 (0.27-0.34) | 0.65 (0.61-0.69) | 0.35 | 0.36 (0.33-0.40) | 0.22 (0.20-0.25) | 0.59 (0.58-0.60) | 0.30 (0.28-0.32) | 40 (35-44) | 173 (84-316) |
|
|
38
|
+
| EWMA chart (`ewma`) | 0.32 (0.27-0.36) | 0.48 (0.43-0.52) | 0.16 | 0.53 (0.43-0.66) | 0.24 (0.21-0.27) | 0.66 (0.64-0.67) | 0.34 (0.30-0.40) | 49 (42-56) | 175 (83-319) |
|
|
39
|
+
| CUSUM chart (`cusum`) | 0.30 (0.26-0.33) | 0.43 (0.38-0.49) | 0.14 | 0.46 (0.38-0.56) | 0.27 (0.24-0.30) | 0.66 (0.64-0.67) | 0.32 (0.28-0.38) | 48 (41-56) | 317 (164-544) |
|
|
40
|
+
| Isolation Forest (`iforest`) | 0.37 (0.33-0.42) | 0.63 (0.57-0.67) | 0.25 | 0.45 (0.40-0.52) | 0.22 (0.20-0.25) | 0.66 (0.65-0.67) | 0.32 (0.28-0.38) | 38 (34-43) | 214 (87-409) |
|
|
41
|
+
| Expert rules (`rules`) | 0.32 (0.29-0.35) | 0.36 (0.33-0.40) | 0.05 | 0.58 (0.46-0.76) | 0.13 (0.11-0.15) | 0.54 (0.53-0.55) | 0.24 (0.22-0.25) | 10 (6-14) | 85 (14-195) |
|
|
42
|
+
| scada-schema QC (`scada_qc`) | 0.14 (0.12-0.17) | 0.48 (0.42-0.55) | 0.34 | 0.20 (0.19-0.22) | 0.12 (0.10-0.13) | 0.52 (0.51-0.53) | 0.19 (0.18-0.20) | 34 (29-40) | 443 (317-577) |
|
|
43
|
+
| Autoencoder (PV) (`pv_autoencoder`) | 0.31 (0.28-0.35) | 0.58 (0.54-0.63) | 0.27 | 0.38 (0.34-0.43) | 0.19 (0.17-0.21) | 0.60 (0.59-0.61) | 0.29 (0.27-0.31) | 37 (31-42) | 173 (118-249) |
|
|
44
|
+
| Shortfall vs PVWatts model (`pv_outage_expected`) * | 0.31 (0.27-0.36) | 0.59 (0.53-0.66) | 0.28 | 0.37 (0.32-0.41) | 0.20 (0.18-0.22) | 0.62 (0.61-0.63) | 0.26 (0.23-0.31) | 30 (26-35) | 212 (110-374) |
|
|
45
|
+
| Outage features + Isolation Forest (`pv_outage_iforest`) * | 0.24 (0.20-0.28) | 0.44 (0.40-0.49) | 0.20 | 0.45 (0.35-0.56) | 0.19 (0.17-0.22) | 0.66 (0.64-0.67) | 0.29 (0.26-0.32) | 36 (31-42) | 143 (83-228) |
|
|
46
|
+
|
|
47
|
+
### Excess recall by fault type
|
|
48
|
+
|
|
49
|
+
Share of injected events that were detected, minus the share that the same alarms hit by chance, all sizes and datasets of the profile added; a dash is a type the method is not scored on. The sensitivity curve (excess recall against the size of the fault) is on the [project page](https://kwwaleszczynski-del.github.io/scada-fault-bench/).
|
|
50
|
+
|
|
51
|
+
Wind:
|
|
52
|
+
|
|
53
|
+
| Method | anemometer drift | nacelle direction offset | derating | power loss | temperature drift | flatline | unit jump | clock shift |
|
|
54
|
+
| :--- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
|
|
55
|
+
| `zscore` | 0.13 | 0.06 | 0.11 | 0.01 | 0.04 | 0.81 | 0.65 | 0.07 |
|
|
56
|
+
| `ewma` | 0.58 | 0.47 | 0.42 | 0.28 | 0.29 | 0.47 | 0.47 | 0.08 |
|
|
57
|
+
| `cusum` | 0.71 | 0.66 | 0.37 | 0.47 | 0.42 | 0.27 | 0.36 | 0.07 |
|
|
58
|
+
| `iforest` | 0.29 | 0.24 | 0.19 | 0.04 | 0.07 | 0.86 | 0.30 | 0.23 |
|
|
59
|
+
| `rules` | 0.19 | 0.13 | 0.00 | 0.00 | 0.00 | 0.85 | 0.88 | 0.01 |
|
|
60
|
+
| `scada_qc` | 0.03 | 0.01 | 0.62 | 0.01 | 0.00 | 0.95 | 0.03 | 0.17 |
|
|
61
|
+
| `nbm_rc` | - | - | - | - | 0.15 | - | 0.90 | - |
|
|
62
|
+
| `nbm_gbm` | - | - | - | - | 0.16 | - | 0.87 | - |
|
|
63
|
+
| `power_curve_kpi` | 0.20 | - | 0.29 | 0.11 | - | 0.59 | - | 0.02 |
|
|
64
|
+
|
|
65
|
+
PV:
|
|
66
|
+
|
|
67
|
+
| Method | temperature drift | flatline | unit jump | clock shift | string loss | shading | inverter outage | pyranometer drift |
|
|
68
|
+
| :--- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
|
|
69
|
+
| `zscore` | 0.04 | 0.55 | 0.88 | 0.46 | -0.01 | 0.04 | 0.47 | 0.06 |
|
|
70
|
+
| `ewma` | 0.27 | 0.39 | 0.73 | 0.15 | 0.24 | -0.04 | 0.66 | 0.17 |
|
|
71
|
+
| `cusum` | 0.24 | 0.37 | 0.65 | 0.24 | 0.23 | -0.01 | 0.57 | 0.11 |
|
|
72
|
+
| `iforest` | 0.14 | 0.59 | 0.56 | 0.59 | 0.16 | 0.16 | 0.73 | 0.07 |
|
|
73
|
+
| `rules` | 0.00 | 0.52 | 0.77 | 0.38 | -0.02 | 0.00 | 0.93 | 0.00 |
|
|
74
|
+
| `scada_qc` | 0.01 | 0.77 | 0.35 | 0.08 | 0.00 | 0.00 | -0.03 | 0.00 |
|
|
75
|
+
| `pv_autoencoder` | 0.00 | 0.59 | 0.89 | 0.44 | -0.05 | -0.02 | 0.74 | -0.01 |
|
|
76
|
+
| `pv_outage_expected` | - | 0.37 | - | 0.39 | 0.16 | 0.21 | 0.76 | -0.01 |
|
|
77
|
+
| `pv_outage_iforest` | - | 0.35 | - | 0.24 | 0.04 | 0.02 | 0.78 | 0.02 |
|
|
78
|
+
|
|
79
|
+
### Real faults: CARE to Compare, farm A
|
|
80
|
+
|
|
81
|
+
The same methods, trained on the first year of each turbine and scored on the prediction section; the truth is every row of the labelled anomaly that has a power reading (the turbine is usually in a downtime state inside an event, which is part of the fault and not a reason to ignore those rows). The chance level is the recall that the alarms of the method reach when they are shifted in time at random over the prediction section. These are single-turbine tables, so the residuals that compare a turbine with its neighbours do not exist and the methods that need neighbours work with less. 12 anomalies and 10 normal datasets are too few to rank methods; the intervals show how little this part can say. Precision is not reported: only the normal datasets contribute healthy time with no fault in it, and that is not a precision.
|
|
82
|
+
|
|
83
|
+
| Method | Excess recall | Recall | Chance | AUROC | AP | Delay (h) | False alarms (h/yr) |
|
|
84
|
+
| :--- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
|
|
85
|
+
| Robust z-score (`zscore`) | 0.09 (-0.05-0.20) | 0.83 (0.58-1.00) | 0.75 | 0.53 (0.49-0.57) | 0.44 (0.24-0.66) | 99 (48-162) | 30 (14-49) |
|
|
86
|
+
| EWMA chart (`ewma`) | -0.02 (-0.21-0.11) | 0.25 (0.00-0.53) | 0.27 | 0.65 (0.56-0.75) | 0.56 (0.32-0.77) | n/a | 16 (0-58) |
|
|
87
|
+
| CUSUM chart (`cusum`) | -0.12 (-0.30-0.00) | 0.00 (0.00-0.00) | 0.12 | 0.66 (0.54-0.78) | 0.56 (0.31-0.80) | n/a | 27 (0-68) |
|
|
88
|
+
| Isolation Forest (`iforest`) | -0.04 (-0.24-0.10) | 0.50 (0.22-0.79) | 0.54 | 0.54 (0.49-0.59) | 0.45 (0.25-0.67) | 165 (37-363) | 9 (1-23) |
|
|
89
|
+
| Expert rules (`rules`) | 0.00 (0.00-0.00) | 0.00 (0.00-0.00) | 0.00 | 0.50 (0.50-0.50) | 0.41 (0.22-0.63) | n/a | 0 (0-0) |
|
|
90
|
+
| scada-schema QC (`scada_qc`) | 0.06 (-0.09-0.18) | 0.67 (0.38-0.92) | 0.60 | 0.48 (0.34-0.59) | 0.42 (0.25-0.65) | 129 (75-187) | 19 (2-42) |
|
|
91
|
+
| Thermal model (RC) (`nbm_rc`) | -0.03 (-0.17-0.08) | 0.17 (0.00-0.40) | 0.20 | 0.57 (0.52-0.62) | 0.51 (0.27-0.72) | n/a | 59 (0-147) |
|
|
92
|
+
| Thermal model (GBM) (`nbm_gbm`) | -0.01 (-0.18-0.12) | 0.25 (0.00-0.50) | 0.26 | 0.54 (0.47-0.60) | 0.49 (0.25-0.70) | n/a | 63 (0-156) |
|
|
93
|
+
| Power curve (method of bins) (`power_curve_kpi`) | 0.03 (-0.09-0.13) | 0.67 (0.38-0.92) | 0.64 | 0.54 (0.47-0.60) | 0.44 (0.23-0.66) | 117 (66-172) | 64 (3-166) |
|
|
94
|
+
|
|
95
|
+
## The faults
|
|
96
|
+
|
|
97
|
+
Every fault is a change of the logged signals of one device. The start, the device and the size of an event are drawn from a seeded generator (the same table, type and seed give the same events), events of one device keep 2 days apart. The truth of an event starts at the first row where the change exceeds a tolerance and ends at the last such row, leaving out the rows where an alarm is neither right nor wrong (night, a known stop, missing data); an event that would show in fewer than 3 hours of data is redrawn. Where the change would move other channels by simple arithmetic (power, current and energy, the energy meter counter) they are changed together.
|
|
98
|
+
|
|
99
|
+
| Type | Profile | What is injected | Five levels |
|
|
100
|
+
| :--- | :--- | :--- | :--- |
|
|
101
|
+
| `anemometer_drift` | wind | nacelle anemometer gain drifts linearly to +-m % (sign drawn per event); the second anemometer is untouched | 3, 6, 9, 12, 15 % (size) |
|
|
102
|
+
| `nacelle_dir_offset` | wind | nacelle position sensor offset by +-m degrees (step); the vane and the absolute direction are untouched | 3, 6, 10, 15, 25 deg (size) |
|
|
103
|
+
| `derating` | wind | active power capped at (1 - m %) of rated power; power, current and energy move together | 5, 10, 20, 30, 40 % of rated (size) |
|
|
104
|
+
| `power_loss` | wind | active power reads (1 - m %) of the true value at every power level (underperformance) | 2, 4, 6, 10, 15 % (size) |
|
|
105
|
+
| `thermal_drift` | wind+pv | temperature channel drifts linearly to +m K (gearbox oil of a turbine, heat sink of an inverter) | 2, 4, 6, 10, 15 K (size) |
|
|
106
|
+
| `flatline` | wind+pv | sensor stuck at its first value of the window (wind speed of a turbine, plane-of-array irradiance of a met station) | 3, 6, 12, 24, 48 h (duration) |
|
|
107
|
+
| `unit_jump` | wind+pv | temperature logged in degF instead of degC (gearbox oil of a turbine, module temperature of a met station) | 6, 12, 24, 48, 96 h (duration) |
|
|
108
|
+
| `clock_shift` | wind+pv | all channels of one device are delayed by m minutes against the others (logger clock error) | 10, 30, 60, 120, 360 min (size) |
|
|
109
|
+
| `string_loss` | pv | a share of the strings of an inverter is lost: AC and DC power and current scale by (1 - m %) | 2, 5, 10, 20, 35 % (size) |
|
|
110
|
+
| `shading` | pv | a fixed obstacle in the west shades the plant at low sun in the afternoon: power scales by (1 - m %) when the sun is below 30 deg and west of south-west | 5, 10, 20, 35, 50 % (size) |
|
|
111
|
+
| `inverter_outage` | pv | inverter produces nothing for the duration of the window (power and currents zero) | 2, 6, 12, 24, 72 h (duration) |
|
|
112
|
+
| `pyranometer_drift` | pv | plane-of-array irradiance gain drifts linearly to +-m % (soiling, ageing; sign drawn per event) | 2, 4, 6, 10, 15 % (size) |
|
|
113
|
+
|
|
114
|
+
## The methods
|
|
115
|
+
|
|
116
|
+
| Name | Profile | Family | Threshold | Fault types scored | Description |
|
|
117
|
+
| :--- | :--- | :--- | :--- | :--- | :--- |
|
|
118
|
+
| `zscore` | wind+pv | baseline | calibrated | all | largest absolute residual of the shared features (one-hour median), each divided by its 99.5 % quantile on the training weeks so that no single heavy-tailed feature decides the score |
|
|
119
|
+
| `ewma` | wind+pv | baseline | calibrated | all | exponentially weighted moving average of every residual clipped at 5 sigma (time constant about one day), largest standardised value |
|
|
120
|
+
| `cusum` | wind+pv | baseline | calibrated | all | two-sided CUSUM of every residual clipped at 5 sigma, on hourly values (allowance 1 sigma, memory 3 days), largest statistic |
|
|
121
|
+
| `iforest` | wind+pv | baseline | calibrated | all | Isolation Forest (100 trees, 4096 samples) on the vector of standardised residuals; missing residuals set to 0 |
|
|
122
|
+
| `rules` | wind+pv | baseline | fixed | all | fixed physical rules, no learning: the two anemometers differ by more than 1.5 m/s, a wind speed with zero variation over an hour while the turbine produces, gearbox oil outside -20 to 100 degC, vane + nacelle - absolute direction beyond 25 deg, power above 110 % of rated; PV: irradiance frozen in daylight, inverter below 2 % of rated in sun above 300 W/m2, module temperature outside -30 to 90 degC, power above 110 % of rated |
|
|
123
|
+
| `scada_qc` | wind+pv | baseline | fixed | all | rows flagged by the row-level checks of scada_schema.qc (flatline, spikes, out of range, units, power curve, wind direction, energy counter, energy against power) |
|
|
124
|
+
| `nbm_rc` | wind | portfolio | calibrated | temperature drift, unit jump | wind-thermal-nbm RCModel: first-order thermal model of the gearbox oil temperature from load, rotor speed, nacelle and ambient temperature; the standardised residual is the score Source: wind-thermal-nbm: wind_thermal_nbm.thermal.RCModel. |
|
|
125
|
+
| `nbm_gbm` | wind | portfolio | calibrated | temperature drift, unit jump | wind-thermal-nbm GBMModel: gradient boosting on the same inputs and their trailing means (1-12 h) for the gearbox oil temperature; the standardised residual is the score Source: wind-thermal-nbm: wind_thermal_nbm.ml.GBMModel. |
|
|
126
|
+
| `power_curve_kpi` | wind | portfolio | calibrated | anemometer drift, derating, power loss, flatline, clock shift | res-asset-kpi bin_power_curve per turbine on the training weeks; the standardised residual of power against the curve at the measured wind speed is the score Source: res-asset-kpi: res_asset_kpi.wind.bin_power_curve, curve_power (the method of wind-power-curve-kelmarsh). |
|
|
127
|
+
| `pv_autoencoder` | pv | portfolio | calibrated | all | pv-anomaly-autoencoder AutoencoderDetector per inverter: windows of 8 steps of power, performance ratio, irradiance, clear-sky index, temperatures and sun position; the reconstruction error (mse), smoothed over one hour, is the score; trained for at most 100 epochs on 80 % of the training days Source: pv-anomaly-autoencoder: pv_anomaly.autoencoder.AutoencoderDetector, pv_anomaly.features.build_features. |
|
|
128
|
+
| `pv_outage_expected` | pv | portfolio | calibrated | string loss, shading, inverter outage, pyranometer drift, flatline, clock shift | pv-outage-classifier detectors.score('expected'): shortfall of inverter power against a PVWatts-style model driven by the met station's irradiance and module temperature, calibrated per bin on the table that is scored (as in that repository); shortfall only Source: pv-outage-classifier: pv_outage.features.compute, pv_outage.detectors.score('expected'). |
|
|
129
|
+
| `pv_outage_iforest` | pv | portfolio | calibrated | string loss, shading, inverter outage, pyranometer drift, flatline, clock shift | pv-outage-classifier detectors.score('iforest'): Isolation Forest on that repository's step features (residuals, level, sun position, ...), fitted on the table that is scored, as in that repository Source: pv-outage-classifier: pv_outage.features.compute, pv_outage.detectors.score('iforest'). |
|
|
130
|
+
|
|
131
|
+
The other repositories of the portfolio were looked at and are not wrapped, for reasons that are properties of their inputs and not of their quality: `wind-wake-aware-verification` and `wind-bayes-twin` work on daily farm-level statistics with wake models, `pv-string-iv-faults` needs per-MPPT voltage and current channels that none of these datasets has, `pv-shade-fingerprint` produces a one-off diagnosis of a whole record and not a score per time stamp. Details in [docs/METHODS.md](docs/METHODS.md).
|
|
132
|
+
|
|
133
|
+
## The datasets
|
|
134
|
+
|
|
135
|
+
| Name | Dataset | Profile | Period (UTC) | Injected events |
|
|
136
|
+
| :--- | :--- | :--- | :--- | ---: |
|
|
137
|
+
| `kelmarsh_2019` | Kelmarsh 2019 | wind | 2019-01-01 to 2020-01-01 | 320 |
|
|
138
|
+
| `penmanshiel_2020` | Penmanshiel 2020 | wind | 2020-01-01 to 2021-01-01 | 320 |
|
|
139
|
+
| `hill_of_towie_2021` | Hill of Towie 2021 | wind | 2021-01-01 to 2022-01-01 | 316 |
|
|
140
|
+
| `pvdaq_34` | PVDAQ system 34 | pv | 2016-01-01 to 2018-01-01 | 318 |
|
|
141
|
+
| `pvdaq_1276` | PVDAQ system 1276 | pv | 2016-01-01 to 2018-01-01 | 308 |
|
|
142
|
+
| `pvdaq_1278` | PVDAQ system 1278 | pv | 2016-01-01 to 2018-01-01 | 317 |
|
|
143
|
+
| `pvdaq_35` | PVDAQ system 35 | pv | 2016-01-01 to 2018-01-01 | 314 |
|
|
144
|
+
|
|
145
|
+
The tables are built from the original open files with the loaders of `scada-schema` (`scada-fault-bench prepare`); nothing is redistributed. Kelmarsh and Penmanshiel are Cubico (CC BY 4.0), Hill of Towie is RES (CC BY 4.0), PVDAQ is NREL (CC BY 4.0). CARE to Compare is CC BY-SA 4.0, which also applies to datasets derived from it.
|
|
146
|
+
|
|
147
|
+
## The evaluation
|
|
148
|
+
|
|
149
|
+
For every dataset and fold the table is cut into a test block (91 days for wind farms, 150 for PV systems, which have one inverter or met station to carry all events), a margin of 7 days and the training part. A method is fitted on the training part without its calibration weeks (one week in four) and scored on the calibration weeks; the alarm threshold is the lowest one at which the merged alarms (below) cover at most 1 % of the healthy rows of the calibration weeks, so every calibrated method works at about the same operating point (the same hours of alarm on healthy data). Rule-based methods have a fixed threshold, and their false alarm rate is a result. Alarms closer than 6 hours are merged and alarms with less than 0.5 hours of flagged rows are dropped.
|
|
150
|
+
|
|
151
|
+
An event is detected when an alarm of its device covers at least one affected row of it (a row where the injected change is above its tolerance and an alarm can be right or wrong). The delay is the time from the first affected row to the first alarm on an affected row. An alarm is true when it covers an affected row, ignored when it lies only on rows where an alarm is neither right nor wrong (known stops, night, real outages of the open data), and false otherwise. The *chance* recall of a method is the share of the events that the alarms of the same method on the untouched test block would have hit; the *excess* is the difference. A wind alarm belongs to a turbine; a PV alarm belongs to the plant, because one met station and one or two inverters do not let a plant-level fault be attributed to a device. False alarms are counted in hours per year of healthy time. AUROC and average precision compare the ranks of the scores of affected and of healthy rows, where the rank of a score is its position among the scores of the healthy calibration rows of the same fold; they do not use the alarm threshold, but they do depend on how finely a score separates values near the healthy range, and a score with few distinct values (the rules) has a coarse rank.
|
|
152
|
+
|
|
153
|
+
Confidence intervals come from a bootstrap over time blocks: the unit of resampling is 4 consecutive weeks of a test block of one dataset and fold (longer than most events), with all devices, fault types and methods of those weeks drawn together, 1000 resamples; for the CARE part a block is one dataset. A mean delay over fewer than five detected events is not shown. They describe the sampling of weeks, devices and events inside this benchmark and nothing beyond it.
|
|
154
|
+
|
|
155
|
+
## Use it
|
|
156
|
+
|
|
157
|
+
```bash
|
|
158
|
+
pip install -e . # installs scada-schema from PyPI
|
|
159
|
+
scada-fault-bench list
|
|
160
|
+
scada-fault-bench prepare --datasets kelmarsh_2019 pvdaq_34 --fetch # builds data/<name>.parquet with scada-schema
|
|
161
|
+
scada-fault-bench prepare --datasets care_A --fetch # the 22 CARE datasets of farm A (real faults)
|
|
162
|
+
scada-fault-bench run --datasets kelmarsh_2019 --folds 0 1 --out results
|
|
163
|
+
scada-fault-bench summarise --out results
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
A method is a class with `fit(train, meta)` and `score(table, meta)`; the score is one non-negative number per row of the table (higher means more anomalous, `NaN` means not assessable):
|
|
167
|
+
|
|
168
|
+
```python
|
|
169
|
+
import numpy as np
|
|
170
|
+
from scada_fault_bench.base import Detector
|
|
171
|
+
|
|
172
|
+
class MyMethod(Detector):
|
|
173
|
+
name = "my_method"
|
|
174
|
+
title = "My method"
|
|
175
|
+
description = "what it does"
|
|
176
|
+
profiles = ("wind",)
|
|
177
|
+
|
|
178
|
+
def fit(self, train, meta):
|
|
179
|
+
self.mean = train["p_ac"].mean()
|
|
180
|
+
return self
|
|
181
|
+
|
|
182
|
+
def score(self, table, meta):
|
|
183
|
+
return np.abs(table["p_ac"].to_numpy() - self.mean)
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
```bash
|
|
187
|
+
scada-fault-bench run --datasets kelmarsh_2019 --plugin my_package.my_module:MyMethod --out results
|
|
188
|
+
```
|
|
189
|
+
|
|
190
|
+
`scada-fault-bench inject --dataset kelmarsh_2019 --type derating --out out/derating` writes one injected test block, its truth and the row labels, for use with any other tool. `power_curve_kpi` needs `res-asset-kpi`, which `scada-schema` installs from PyPI. The other reference methods are optional dependencies that are installed from GitHub (they are not on PyPI), and `pv_autoencoder` also needs `torch`; a method whose repository cannot be imported is skipped and listed as unavailable in `results/summary.json`. The full run in this repository used all of them:
|
|
191
|
+
|
|
192
|
+
```bash
|
|
193
|
+
pip install torch # for pv_autoencoder
|
|
194
|
+
pip install git+https://github.com/kwwaleszczynski-del/wind-thermal-nbm git+https://github.com/kwwaleszczynski-del/pv-anomaly-autoencoder git+https://github.com/kwwaleszczynski-del/pv-outage-classifier
|
|
195
|
+
```
|
|
196
|
+
|
|
197
|
+
## Limitations
|
|
198
|
+
|
|
199
|
+
* The faults are injected into recorded signals. Physical faults change more channels, and in ways that depend on the machine; the models here favour detectors that look at the channel that was changed. Nothing here says how a method finds a real fault except the CARE part, which is small.
|
|
200
|
+
* Faults of one type are drawn from one simple model with five sizes. The sensitivity curves describe that model.
|
|
201
|
+
* The open data are not guaranteed to be healthy. Real stops that the status log shows, night, and real inverter outages of PVDAQ are removed from the count of false alarms; real faults that are not visible in them are not, so alarms on them are counted as false. This is conservative.
|
|
202
|
+
* PV plants are one met station with one or two inverters: there is no string level and no device-level attribution. The PV results are plant-level.
|
|
203
|
+
* Calibrated methods share a false alarm budget on a calibration sample from the same year; a method whose scores drift with the seasons will exceed it, and the false alarm column shows by how much.
|
|
204
|
+
* An alarm on an affected row counts as a detection whatever its cause, so a method with many alarms also "detects" events by coincidence. The chance column and the excess recall show how much that is; the sensitivity curves show the excess and are not always monotone in the size of the fault. A method whose excess interval contains zero has not been shown to detect these faults.
|
|
205
|
+
* The simple baselines read a residual matrix that was built with the fault models in view: roughly one residual for each kind of fault (power against the curve, wind speed against neighbours and the second anemometer, direction against neighbours, gearbox temperature, frozen signals; for PV the ratio against irradiance, sun position, irradiance variation, temperatures). That makes them well suited to these faults by construction. `scada_qc` is run on the channels that the fault models touch, because it would otherwise flag quantised channels the faults do not concern. Nothing was tuned on the results, but the design was not blind to the faults.
|
|
206
|
+
* `nbm_rc`, `nbm_gbm` and `power_curve_kpi` are models taken from their repositories, not the alarm methods of those repositories (which average residuals over days and chart them with a zero-false-alarm limit), and they are scored only on the fault types in their channels; `pv_outage_*` likewise. Their overall rows say nothing about the other fault types.
|
|
207
|
+
* The calibrated methods share an operating point on healthy calibration data, but their scores have different autocorrelation, so the hours of alarm on the untouched test block still differ from method to method; the false alarm column shows it. A method whose score is too persistent to meet the budget gets the highest threshold of the search and may raise almost no alarm.
|
|
208
|
+
* The rule that hides real PV outages of the open data from the false alarm count (a whole calendar day on which at least four fifths of the sunny rows have zero power, or no power reading) is not the one-hour zero-power rule of the `rules` method; short real outages are counted as false alarms for every method.
|
|
209
|
+
* The reference methods are called through adapters that rename and rescale columns. A weak result can come from the adapter or from a method that was written for other data; the repositories that own the methods are the authority on what they do.
|
|
210
|
+
* 1000 bootstrap resamples of weeks are not independent of the fault draws: another seed gives other events. Differences within the intervals should not be read as differences.
|
|
211
|
+
* The file formats and the fault models are at version 0.1.0; results of different versions are not comparable.
|
|
212
|
+
|
|
213
|
+
## Reproduce
|
|
214
|
+
|
|
215
|
+
The run in this repository is `scada-fault-bench prepare` (all datasets, `--fetch` the first time), then `scada-fault-bench run --out results` (all datasets, all folds, 1 injection seed for wind and 2 for PV, which are the defaults) and `scada-fault-bench summarise --out results`, with one process per dataset to save time. `results/` holds the shards and `summary.json` of that run; `python scripts/render_readme.py` rebuilds this file and `python scripts/build_pages.py` the page from them. The tests in `tests/test_docs.py` compare both with the summary, and `tests/test_data.py` (marked `data`) repeats a small part of the run on the real tables.
|
|
216
|
+
|
|
217
|
+
## Citation and licence
|
|
218
|
+
|
|
219
|
+
Code: MIT. Cite the repository (`CITATION.cff`) and the datasets above. `scada-fault-bench` is built on `scada-schema` 0.1.1, whose loaders read the open datasets.
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "scada-fault-bench"
|
|
7
|
+
dynamic = ["version"]
|
|
8
|
+
description = "An open benchmark for detecting faults in wind and PV SCADA data: injected faults with known ground truth on open datasets in the scada-schema format, one interface for detectors, one evaluation with block-bootstrap confidence intervals."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.11"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
authors = [{ name = "Konrad Waleszczyński" }]
|
|
14
|
+
keywords = ["scada", "wind-energy", "photovoltaics", "fault-detection", "anomaly-detection", "benchmark", "scada-schema"]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Development Status :: 3 - Alpha",
|
|
17
|
+
"Intended Audience :: Science/Research",
|
|
18
|
+
"Operating System :: OS Independent",
|
|
19
|
+
"Programming Language :: Python :: 3",
|
|
20
|
+
"Programming Language :: Python :: 3.11",
|
|
21
|
+
"Programming Language :: Python :: 3.12",
|
|
22
|
+
"Programming Language :: Python :: 3.13",
|
|
23
|
+
"Topic :: Scientific/Engineering",
|
|
24
|
+
]
|
|
25
|
+
dependencies = [
|
|
26
|
+
"numpy>=1.26",
|
|
27
|
+
"pandas>=2.1",
|
|
28
|
+
"scikit-learn>=1.4",
|
|
29
|
+
"scipy>=1.11",
|
|
30
|
+
"scada-schema>=0.1.0",
|
|
31
|
+
]
|
|
32
|
+
|
|
33
|
+
[project.optional-dependencies]
|
|
34
|
+
# methods from other repositories of the portfolio; none is on PyPI, so they are installed from GitHub (see README, "Reference methods")
|
|
35
|
+
plots = ["matplotlib>=3.8"]
|
|
36
|
+
dev = ["pytest>=8", "ruff>=0.6", "build>=1.2", "matplotlib>=3.8"]
|
|
37
|
+
|
|
38
|
+
[project.scripts]
|
|
39
|
+
scada-fault-bench = "scada_fault_bench.cli:main"
|
|
40
|
+
|
|
41
|
+
[project.urls]
|
|
42
|
+
Homepage = "https://github.com/kwwaleszczynski-del/scada-fault-bench"
|
|
43
|
+
Documentation = "https://kwwaleszczynski-del.github.io/scada-fault-bench/"
|
|
44
|
+
Issues = "https://github.com/kwwaleszczynski-del/scada-fault-bench/issues"
|
|
45
|
+
|
|
46
|
+
[tool.setuptools.dynamic]
|
|
47
|
+
version = { attr = "scada_fault_bench._version.__version__" }
|
|
48
|
+
|
|
49
|
+
[tool.setuptools.packages.find]
|
|
50
|
+
where = ["src"]
|
|
51
|
+
|
|
52
|
+
[tool.pytest.ini_options]
|
|
53
|
+
testpaths = ["tests"]
|
|
54
|
+
markers = [
|
|
55
|
+
"data: needs the open datasets prepared on disk (SCADA_FAULT_BENCH_DATA); skipped otherwise",
|
|
56
|
+
"portfolio: needs a reference method from another portfolio repository; skipped when it is not importable",
|
|
57
|
+
]
|
|
58
|
+
|
|
59
|
+
[tool.ruff]
|
|
60
|
+
line-length = 160
|
|
61
|
+
target-version = "py311"
|
|
62
|
+
|
|
63
|
+
[tool.ruff.lint]
|
|
64
|
+
select = ["E", "F", "W", "I", "UP", "B"]
|
|
65
|
+
ignore = ["E501", "B905"]
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
"""An open benchmark for fault detection in wind and PV SCADA data (see the README)."""
|
|
2
|
+
|
|
3
|
+
from scada_fault_bench import baselines, portfolio # noqa: E402,F401 - importing the package registers the methods in base.METHODS
|
|
4
|
+
from scada_fault_bench._version import __version__
|
|
5
|
+
|
|
6
|
+
__all__ = ["__version__"]
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.1.0"
|