scada-schema 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- scada_schema-0.1.0/CHANGELOG.md +19 -0
- scada_schema-0.1.0/CITATION.cff +33 -0
- scada_schema-0.1.0/LICENSE +21 -0
- scada_schema-0.1.0/MANIFEST.in +6 -0
- scada_schema-0.1.0/PKG-INFO +275 -0
- scada_schema-0.1.0/README.md +225 -0
- scada_schema-0.1.0/SPEC.md +161 -0
- scada_schema-0.1.0/pyproject.toml +80 -0
- scada_schema-0.1.0/release.json +7 -0
- scada_schema-0.1.0/schema/columns.json +2513 -0
- scada_schema-0.1.0/schema/dataset-meta.schema.json +300 -0
- scada_schema-0.1.0/schema/mapping.schema.json +428 -0
- scada_schema-0.1.0/setup.cfg +4 -0
- scada_schema-0.1.0/src/scada_schema/__init__.py +31 -0
- scada_schema-0.1.0/src/scada_schema/__main__.py +3 -0
- scada_schema-0.1.0/src/scada_schema/_steps.py +50 -0
- scada_schema-0.1.0/src/scada_schema/_time.py +12 -0
- scada_schema-0.1.0/src/scada_schema/_version.py +1 -0
- scada_schema-0.1.0/src/scada_schema/cli.py +295 -0
- scada_schema-0.1.0/src/scada_schema/convert/__init__.py +8 -0
- scada_schema-0.1.0/src/scada_schema/convert/openoa.py +80 -0
- scada_schema-0.1.0/src/scada_schema/convert/pvlib.py +98 -0
- scada_schema-0.1.0/src/scada_schema/convert/windup.py +175 -0
- scada_schema-0.1.0/src/scada_schema/data/codes_senvion_mm.csv +269 -0
- scada_schema-0.1.0/src/scada_schema/data/codes_siemens_swt_hot.csv +147 -0
- scada_schema-0.1.0/src/scada_schema/data/iec_labels.csv +17 -0
- scada_schema-0.1.0/src/scada_schema/io.py +101 -0
- scada_schema-0.1.0/src/scada_schema/jsonschema.py +57 -0
- scada_schema-0.1.0/src/scada_schema/loaders/__init__.py +24 -0
- scada_schema-0.1.0/src/scada_schema/loaders/_common.py +112 -0
- scada_schema-0.1.0/src/scada_schema/loaders/_paths.py +21 -0
- scada_schema-0.1.0/src/scada_schema/loaders/_remotezip.py +67 -0
- scada_schema-0.1.0/src/scada_schema/loaders/care.py +167 -0
- scada_schema-0.1.0/src/scada_schema/loaders/generic.py +315 -0
- scada_schema-0.1.0/src/scada_schema/loaders/greenbyte.py +202 -0
- scada_schema-0.1.0/src/scada_schema/loaders/hill_of_towie.py +200 -0
- scada_schema-0.1.0/src/scada_schema/loaders/kelmarsh.py +53 -0
- scada_schema-0.1.0/src/scada_schema/loaders/la_haute_borne.py +81 -0
- scada_schema-0.1.0/src/scada_schema/loaders/penmanshiel.py +48 -0
- scada_schema-0.1.0/src/scada_schema/loaders/pvdaq.py +208 -0
- scada_schema-0.1.0/src/scada_schema/model.py +174 -0
- scada_schema-0.1.0/src/scada_schema/parity.py +181 -0
- scada_schema-0.1.0/src/scada_schema/py.typed +0 -0
- scada_schema-0.1.0/src/scada_schema/qc/__init__.py +14 -0
- scada_schema-0.1.0/src/scada_schema/qc/checks.py +580 -0
- scada_schema-0.1.0/src/scada_schema/qc/core.py +308 -0
- scada_schema-0.1.0/src/scada_schema/qc/report.py +167 -0
- scada_schema-0.1.0/src/scada_schema/resample.py +122 -0
- scada_schema-0.1.0/src/scada_schema/signals.py +169 -0
- scada_schema-0.1.0/src/scada_schema/spec.py +150 -0
- scada_schema-0.1.0/src/scada_schema/states.py +221 -0
- scada_schema-0.1.0/src/scada_schema/study.py +274 -0
- scada_schema-0.1.0/src/scada_schema/validate.py +207 -0
- scada_schema-0.1.0/src/scada_schema.egg-info/PKG-INFO +275 -0
- scada_schema-0.1.0/src/scada_schema.egg-info/SOURCES.txt +68 -0
- scada_schema-0.1.0/src/scada_schema.egg-info/dependency_links.txt +1 -0
- scada_schema-0.1.0/src/scada_schema.egg-info/entry_points.txt +2 -0
- scada_schema-0.1.0/src/scada_schema.egg-info/requires.txt +25 -0
- scada_schema-0.1.0/src/scada_schema.egg-info/top_level.txt +1 -0
- scada_schema-0.1.0/tests/conftest.py +147 -0
- scada_schema-0.1.0/tests/test_cli.py +112 -0
- scada_schema-0.1.0/tests/test_core.py +98 -0
- scada_schema-0.1.0/tests/test_docs.py +146 -0
- scada_schema-0.1.0/tests/test_generic.py +191 -0
- scada_schema-0.1.0/tests/test_loaders.py +299 -0
- scada_schema-0.1.0/tests/test_misc.py +45 -0
- scada_schema-0.1.0/tests/test_parity.py +172 -0
- scada_schema-0.1.0/tests/test_qc.py +212 -0
- scada_schema-0.1.0/tests/test_resample.py +89 -0
- scada_schema-0.1.0/tests/test_states.py +122 -0
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project are documented here. The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and the project uses [semantic versioning](https://semver.org/).
|
|
4
|
+
|
|
5
|
+
## [0.1.0] - 2026-11-06
|
|
6
|
+
|
|
7
|
+
First release.
|
|
8
|
+
|
|
9
|
+
### Added
|
|
10
|
+
|
|
11
|
+
- The schema (version 1.0, wind and PV profiles), `SPEC.md` generated from the code, JSON Schema files and a column list (`schema/`).
|
|
12
|
+
- Loaders for Kelmarsh, Penmanshiel, Hill of Towie, La Haute Borne, CARE to Compare and NREL PVDAQ (with `fetch` and checksum verification).
|
|
13
|
+
- A loader driven by a YAML mapping (`scada-schema import`) and resampling to a coarser interval (`scada-schema resample`).
|
|
14
|
+
- 15 quality checks with a self-contained HTML report.
|
|
15
|
+
- IEC 61400-26 state mapping (code tables, label tables, event projection) and time-based availability through res-asset-kpi.
|
|
16
|
+
- Converters to wind-up, OpenOA and pvlib with parity checks.
|
|
17
|
+
- Command line interface, the study that produces every number of the README, tests on Linux and Windows, CITATION.cff and Zenodo metadata, and a release workflow for PyPI Trusted Publishing (disabled until `PYPI_PUBLISH` is set).
|
|
18
|
+
|
|
19
|
+
[0.1.0]: https://github.com/kwwaleszczynski-del/scada-schema/releases/tag/v0.1.0
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
cff-version: 1.2.0
|
|
2
|
+
message: "If you use this software, please cite it using the metadata below."
|
|
3
|
+
type: software
|
|
4
|
+
title: "scada-schema: an open data standard for wind and PV SCADA"
|
|
5
|
+
abstract: >-
|
|
6
|
+
One table schema for wind turbine and PV SCADA (names, units, sign and angle conventions, UTC interval start, device types),
|
|
7
|
+
loaders that read open datasets (Kelmarsh, Penmanshiel, Hill of Towie, La Haute Borne, CARE to Compare, NREL PVDAQ) from their
|
|
8
|
+
original files, a loader driven by a YAML mapping for other exports, resampling, 15 data quality checks with an HTML report,
|
|
9
|
+
mapping of status codes to IEC 61400-26 categories, and converters to wind-up, OpenOA and pvlib with parity checks.
|
|
10
|
+
authors:
|
|
11
|
+
- family-names: "Waleszczyński"
|
|
12
|
+
given-names: "Konrad"
|
|
13
|
+
version: 0.1.0
|
|
14
|
+
date-released: "2026-11-06"
|
|
15
|
+
license: MIT
|
|
16
|
+
repository-code: "https://github.com/kwwaleszczynski-del/scada-schema"
|
|
17
|
+
url: "https://kwwaleszczynski-del.github.io/scada-schema/"
|
|
18
|
+
keywords:
|
|
19
|
+
- SCADA
|
|
20
|
+
- wind energy
|
|
21
|
+
- photovoltaics
|
|
22
|
+
- data schema
|
|
23
|
+
- data quality
|
|
24
|
+
- IEC 61400-26
|
|
25
|
+
- open data
|
|
26
|
+
references:
|
|
27
|
+
- type: software
|
|
28
|
+
title: "res-asset-kpi: key performance indicators for operating PV, wind and battery assets"
|
|
29
|
+
authors:
|
|
30
|
+
- family-names: "Waleszczyński"
|
|
31
|
+
given-names: "Konrad"
|
|
32
|
+
doi: 10.5281/zenodo.23742958
|
|
33
|
+
repository-code: "https://github.com/kwwaleszczynski-del/res-asset-kpi"
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Konrad Waleszczyński
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,275 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: scada-schema
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: An open data standard for wind and PV SCADA: one schema, loaders for open datasets, quality control with an HTML report, IEC 61400-26 state mapping and converters to wind-up, OpenOA and pvlib.
|
|
5
|
+
Author: Konrad Waleszczyński
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/kwwaleszczynski-del/scada-schema
|
|
8
|
+
Project-URL: Documentation, https://kwwaleszczynski-del.github.io/scada-schema/
|
|
9
|
+
Project-URL: Source, https://github.com/kwwaleszczynski-del/scada-schema
|
|
10
|
+
Project-URL: Changelog, https://github.com/kwwaleszczynski-del/scada-schema/blob/main/CHANGELOG.md
|
|
11
|
+
Project-URL: Specification, https://github.com/kwwaleszczynski-del/scada-schema/blob/main/SPEC.md
|
|
12
|
+
Project-URL: Issues, https://github.com/kwwaleszczynski-del/scada-schema/issues
|
|
13
|
+
Keywords: scada,wind-energy,photovoltaics,data-schema,data-quality,iec-61400-26,wind-up,openoa,pvlib,parquet,open-data
|
|
14
|
+
Classifier: Development Status :: 3 - Alpha
|
|
15
|
+
Classifier: Intended Audience :: Science/Research
|
|
16
|
+
Classifier: Intended Audience :: Developers
|
|
17
|
+
Classifier: Operating System :: OS Independent
|
|
18
|
+
Classifier: Programming Language :: Python :: 3
|
|
19
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
23
|
+
Classifier: Topic :: Scientific/Engineering
|
|
24
|
+
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
25
|
+
Classifier: Typing :: Typed
|
|
26
|
+
Requires-Python: >=3.11
|
|
27
|
+
Description-Content-Type: text/markdown
|
|
28
|
+
License-File: LICENSE
|
|
29
|
+
Requires-Dist: numpy>=1.26
|
|
30
|
+
Requires-Dist: pandas>=2.1
|
|
31
|
+
Requires-Dist: pyarrow>=14
|
|
32
|
+
Requires-Dist: pydantic>=2.5
|
|
33
|
+
Requires-Dist: pyyaml>=6.0
|
|
34
|
+
Requires-Dist: requests>=2.31
|
|
35
|
+
Requires-Dist: res-asset-kpi>=0.1.0
|
|
36
|
+
Provides-Extra: windup
|
|
37
|
+
Requires-Dist: res-wind-up>=0.5; extra == "windup"
|
|
38
|
+
Provides-Extra: openoa
|
|
39
|
+
Requires-Dist: openoa>=3.1; extra == "openoa"
|
|
40
|
+
Provides-Extra: pvlib
|
|
41
|
+
Requires-Dist: pvlib>=0.11; extra == "pvlib"
|
|
42
|
+
Provides-Extra: converters
|
|
43
|
+
Requires-Dist: scada-schema[openoa,pvlib,windup]; extra == "converters"
|
|
44
|
+
Provides-Extra: dev
|
|
45
|
+
Requires-Dist: pytest>=8; extra == "dev"
|
|
46
|
+
Requires-Dist: ruff>=0.6; extra == "dev"
|
|
47
|
+
Requires-Dist: build>=1.2; extra == "dev"
|
|
48
|
+
Requires-Dist: twine>=5; extra == "dev"
|
|
49
|
+
Dynamic: license-file
|
|
50
|
+
|
|
51
|
+
# scada-schema: an open data standard for wind and PV SCADA
|
|
52
|
+
|
|
53
|
+
[](https://github.com/kwwaleszczynski-del/scada-schema/actions/workflows/ci.yml) [](https://github.com/kwwaleszczynski-del/scada-schema/blob/main/LICENSE)
|
|
54
|
+
|
|
55
|
+
**Project page with the results: [kwwaleszczynski-del.github.io/scada-schema](https://kwwaleszczynski-del.github.io/scada-schema/)**
|
|
56
|
+
- [specification](https://github.com/kwwaleszczynski-del/scada-schema/blob/main/SPEC.md) - [example quality report](https://kwwaleszczynski-del.github.io/scada-schema/qc/pvdaq_34_raw_units.html).
|
|
57
|
+
|
|
58
|
+
SCADA exports from two wind farms never look alike: columns named after a vendor's tag list, power in hectowatts or watts, time in local
|
|
59
|
+
time or at the end of the interval, a status code that only the turbine manual explains. Every analysis tool then starts with its own
|
|
60
|
+
cleaning script. This package fixes **one table layout** (names, units, sign and angle conventions, time zone and interval, device types) for
|
|
61
|
+
wind turbines and PV plants, reads **open datasets from 6 sources** into it from their original files, checks the data, maps status codes to
|
|
62
|
+
**IEC 61400-26 categories** and converts the table to the inputs of **wind-up, OpenOA and pvlib**, with a test that each tool gives the same result
|
|
63
|
+
on the library's input as on a hand-built one.
|
|
64
|
+
|
|
65
|
+
It is a proposal, not a standard of any body, and working material: the formulas and category names follow public descriptions, and nothing
|
|
66
|
+
here is a certified implementation of an IEC standard.
|
|
67
|
+
|
|
68
|
+
## What is in it
|
|
69
|
+
|
|
70
|
+
1. **Schema** ([SPEC.md](https://github.com/kwwaleszczynski-del/scada-schema/blob/main/SPEC.md), generated from the code): 40 canonical signals (31 in the wind profile, 23 in the PV profile),
|
|
71
|
+
one row per device and interval, `timestamp` in UTC at the start of the interval, canonical units, power positive into the grid, compass
|
|
72
|
+
angles clockwise from true north. Minimum, maximum and standard deviation columns where a source has them. The metadata (profile, interval, the
|
|
73
|
+
time zone and label of the *source*, devices with ratings and positions, licence, every conversion the loader made) are stored inside the
|
|
74
|
+
Parquet file. pydantic describes the metadata; the table is validated by a vectorised check that reports every violation with a count (pandera
|
|
75
|
+
stops at the first failure and adds a dependency, and a per-row quality report is needed anyway).
|
|
76
|
+
2. **Loaders** for Kelmarsh, Penmanshiel, Hill of Towie, La Haute Borne, CARE to Compare and NREL PVDAQ. They read the original archives (Zenodo, OEDI) and
|
|
77
|
+
`fetch` downloads them with checksum verification; nothing large is in this repository. Hill of Towie's 1.4 GB yearly archives are read with range
|
|
78
|
+
requests, so only the needed tables are transferred.
|
|
79
|
+
3. **Quality control** with an HTML report: 15 checks - duplicate and unordered time stamps, time jumps, gaps, daylight saving artefacts, missing values,
|
|
80
|
+
physical limits, flat lines, spikes, power-curve outliers, wind direction (vane, nacelle and north offsets between turbines), energy counters and interval energy,
|
|
81
|
+
icing, and units (order of magnitude and a switch of unit inside a record). The report is one file without scripts.
|
|
82
|
+
4. **IEC 61400-26 states**: code tables for the Senvion MM series (268 codes) and the Siemens SWT 2.3 (146 codes) as logged in the open
|
|
83
|
+
datasets, label tables for Greenbyte and CARE, projection of an event log onto the intervals, and time-based availability through
|
|
84
|
+
[res-asset-kpi](https://github.com/kwwaleszczynski-del/res-asset-kpi) (which also supplies the unit, time-zone and category code used here).
|
|
85
|
+
5. **Converters** to wind-up, OpenOA and pvlib and the parity checks below.
|
|
86
|
+
6. **Your own data without code**: `scada-schema import mapping.yaml` turns a CSV, Excel or Parquet export into a canonical table from a YAML mapping (columns, units,
|
|
87
|
+
time zone and label, devices); see below.
|
|
88
|
+
7. **JSON Schema** of the metadata and of the mapping file, and a list of every canonical column, in [`schema/`](https://github.com/kwwaleszczynski-del/scada-schema/blob/main/schema) for tools that are not written in Python.
|
|
89
|
+
8. **Resampling** to a coarser interval with rules by signal kind (mean, vector mean of angles, sum, last meter reading, pooled standard deviation, coverage threshold): `scada-schema resample`.
|
|
90
|
+
9. **Command line**: `scada-schema fetch | load | import | resample | validate | qc | convert | jsonschema | spec | study`.
|
|
91
|
+
|
|
92
|
+
## Results
|
|
93
|
+
|
|
94
|
+
All numbers on this page come from `python -m scada_schema study`, which writes `results/study.json`; `tests/test_docs.py` renders this README from the
|
|
95
|
+
template and that file and fails when they differ.
|
|
96
|
+
|
|
97
|
+
12 tables from 6 sources (3,870,948 rows, 59 devices) were loaded from the original files; the schema accepts 12 of 12.
|
|
98
|
+
|
|
99
|
+
| Dataset | Source (licence) | Profile | Interval | Rows | Devices | Signals | Schema | QC findings (error / warning / info) |
|
|
100
|
+
|---|---|---|---|---:|---:|---:|---|---|
|
|
101
|
+
| [kelmarsh_2019](https://kwwaleszczynski-del.github.io/scada-schema/qc/kelmarsh_2019.html) | Kelmarsh wind farm data (Cubico Sustainable Investments) (CC-BY-4.0) | wind | 10min | 315,360 | 6 | 23 (+39 statistics) | valid | 0 / 19 / 49 |
|
|
102
|
+
| [penmanshiel_2020](https://kwwaleszczynski-del.github.io/scada-schema/qc/penmanshiel_2020.html) | Penmanshiel wind farm data (Cubico Sustainable Investments) (CC-BY-4.0) | wind | 10min | 737,856 | 14 | 23 (+60 statistics) | valid | 0 / 359 / 107 |
|
|
103
|
+
| [hill_of_towie_2021](https://kwwaleszczynski-del.github.io/scada-schema/qc/hill_of_towie_2021.html) | Hill of Towie wind farm open dataset (RES) (CC-BY-4.0) | wind | 10min | 1,090,921 | 21 | 20 (+42 statistics) | valid | 2 / 115 / 155 |
|
|
104
|
+
| [la_haute_borne](https://kwwaleszczynski-del.github.io/scada-schema/qc/la_haute_borne.html) | La Haute Borne wind farm data (ENGIE, copy shipped with OpenOA) (Etalab open licence (as stated by ENGIE, not re-verified)) | wind | 10min | 420,432 | 4 | 7 (+0 statistics) | valid | 0 / 11 / 18 |
|
|
105
|
+
| [care_A](https://kwwaleszczynski-del.github.io/scada-schema/qc/care_A.html) | CARE to Compare (CC-BY-SA-4.0) | wind | 10min | 54,358 | 1 | 16 (+15 statistics) | valid | 0 / 3 / 6 |
|
|
106
|
+
| [care_B](https://kwwaleszczynski-del.github.io/scada-schema/qc/care_B.html) | CARE to Compare (CC-BY-SA-4.0) | wind | 10min | 56,564 | 1 | 15 (+39 statistics) | valid | 1 / 6 / 5 |
|
|
107
|
+
| [care_C](https://kwwaleszczynski-del.github.io/scada-schema/qc/care_C.html) | CARE to Compare (CC-BY-SA-4.0) | wind | 10min | 55,753 | 1 | 8 (+18 statistics) | valid | 0 / 3 / 1 |
|
|
108
|
+
| [pvdaq_34](https://kwwaleszczynski-del.github.io/scada-schema/qc/pvdaq_34.html) | NREL PVDAQ public datasets (OEDI) (CC-BY-4.0) | pv | 15min | 210,300 | 2 | 15 (+0 statistics) | valid | 2 / 7 / 7 |
|
|
109
|
+
| [pvdaq_1276](https://kwwaleszczynski-del.github.io/scada-schema/qc/pvdaq_1276.html) | NREL PVDAQ public datasets (OEDI) (CC-BY-4.0) | pv | 15min | 209,162 | 2 | 14 (+0 statistics) | valid | 2 / 11 / 5 |
|
|
110
|
+
| [pvdaq_1278](https://kwwaleszczynski-del.github.io/scada-schema/qc/pvdaq_1278.html) | NREL PVDAQ public datasets (OEDI) (CC-BY-4.0) | pv | 15min | 315,456 | 3 | 14 (+0 statistics) | valid | 3 / 12 / 10 |
|
|
111
|
+
| [pvdaq_35](https://kwwaleszczynski-del.github.io/scada-schema/qc/pvdaq_35.html) | NREL PVDAQ public datasets (OEDI) (CC-BY-4.0) | pv | 15min | 194,486 | 2 | 15 (+0 statistics) | valid | 2 / 13 / 7 |
|
|
112
|
+
| [pvdaq_34_raw_units](https://kwwaleszczynski-del.github.io/scada-schema/qc/pvdaq_34_raw_units.html) | NREL PVDAQ public datasets (OEDI) (CC-BY-4.0) | pv | 15min | 210,300 | 2 | 15 (+0 statistics) | valid | 4 / 7 / 7 |
|
|
113
|
+
|
|
114
|
+
Number of devices with at least one finding, by check (the quality reports list every finding with its longest episodes):
|
|
115
|
+
|
|
116
|
+
| Check | kelmarsh_2019 | penmanshiel_2020 | hill_of_towie_2021 | la_haute_borne | care_A | care_B | care_C | pvdaq_34 | pvdaq_1276 | pvdaq_1278 | pvdaq_35 | pvdaq_34_raw_units |
|
|
117
|
+
|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|
|
|
118
|
+
| `duplicates` | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 |
|
|
119
|
+
| `ordering` | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 |
|
|
120
|
+
| `time_jumps` | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 |
|
|
121
|
+
| `gaps` | 0 | 0 | 21 | 4 | 1 | 1 | 0 | 2 | 2 | 3 | 2 | 2 |
|
|
122
|
+
| `dst` | 0 | 0 | 0 | 4 | skipped | skipped | skipped | skipped | skipped | skipped | skipped | skipped |
|
|
123
|
+
| `missing_values` | 6 | 14 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 |
|
|
124
|
+
| `out_of_range` | 5 | 14 | 0 | 3 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 |
|
|
125
|
+
| `flatline` | 1 | 13 | 21 | 1 | 1 | 1 | 1 | 2 | 2 | 3 | 2 | 2 |
|
|
126
|
+
| `spikes` | 6 | 14 | 21 | 1 | 1 | 1 | 1 | 2 | 2 | 3 | 2 | 2 |
|
|
127
|
+
| `power_curve` | 6 | 14 | 21 | 4 | 1 | 1 | 1 | skipped | skipped | skipped | skipped | skipped |
|
|
128
|
+
| `wind_direction` | 6 | 14 | 21 | 4 | 0 | 0 | skipped | 0 | 0 | 0 | 0 | 0 |
|
|
129
|
+
| `icing` | 1 | 8 | 21 | 4 | 0 | 1 | 0 | skipped | skipped | skipped | skipped | skipped |
|
|
130
|
+
| `energy_counter` | 2 | 0 | skipped | skipped | skipped | skipped | skipped | skipped | skipped | skipped | skipped | skipped |
|
|
131
|
+
| `energy_power` | 0 | 0 | skipped | skipped | skipped | skipped | skipped | skipped | skipped | skipped | skipped | skipped |
|
|
132
|
+
| `units` | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 |
|
|
133
|
+
|
|
134
|
+
What the checks found on real data:
|
|
135
|
+
|
|
136
|
+
- **A unit switch is found where the dataset documents it.** PVDAQ's power channels change from hectowatts to watts in August 2018. Read as hectowatts
|
|
137
|
+
throughout, system 34 shows a step of a factor of 98 in power over irradiance on 2018-08-05; the loader finds the same switch from the data
|
|
138
|
+
(`p_ac, p_dc: hectowatts to kW (x0.1) until 2018-08-04 20:00 UTC, watts to kW (x0.001) from then on`), and the check on the corrected table reports 0 switches.
|
|
139
|
+
- **North offsets.** On Hill of Towie the nacelle position of 12 of 21 turbines is more than 10 degrees off the fleet mean direction;
|
|
140
|
+
the largest, T16, is 84.7 degrees off. On Kelmarsh the largest offset is 3.8 degrees.
|
|
141
|
+
- **A real daylight saving artefact.** La Haute Borne stamps its data with the UTC offset, so the time is exact, but at both autumn changes (2014-10-26 and 2015-10-25) one hour is missing in UTC (00:00 to 01:00, the first occurrence of
|
|
142
|
+
the repeated local hour): the source holds only one of the two repeated hours. The `dst` check, run against Europe/Paris, finds 2 of 4 changes on every turbine.
|
|
143
|
+
- **Time elsewhere.** The three UK farms are checked against Europe/London (UTC time stamps, no artefact found); for the others the check does not apply (fixed offset or anonymised time).
|
|
144
|
+
The table above shows what each dataset triggered.
|
|
145
|
+
|
|
146
|
+
### IEC 61400-26 mapping
|
|
147
|
+
|
|
148
|
+
A stop event labels an interval when the interval's midpoint lies in the event. On the Greenbyte farms the operator also labels each stop with an IEC category,
|
|
149
|
+
which tests the code tables:
|
|
150
|
+
|
|
151
|
+
| Dataset | Code table | Events | Events with a code in the table | Stop events | Compared with the operator's category | Agreement | Time-based availability, IEC operational / OEM style |
|
|
152
|
+
|---|---|---:|---:|---:|---:|---:|---|
|
|
153
|
+
| kelmarsh_2019 | `senvion_mm` | 59,313 | 100.0 % | 673 | 673 | 100.0 % | 99.5 % / 99.8 % |
|
|
154
|
+
| penmanshiel_2020 | `senvion_mm` | 158,509 | 100.0 % | 3,267 | 3,264 | 100.0 % | 97.8 % / 98.1 % |
|
|
155
|
+
| hill_of_towie_2021 | `siemens_swt_hot` | 798,406 | 94.9 % | 4,034 | - | no operator category | not computed |
|
|
156
|
+
|
|
157
|
+
On Kelmarsh the Senvion table agrees with the operator's category on 100.0 % of 673 compared stop events, on Penmanshiel on 100.0 % of 3,264. **This is a
|
|
158
|
+
consistency check, not an independent test:** the Senvion table was written from the same open logs. The Hill of Towie alarm log has no operator category; the dataset describes only
|
|
159
|
+
12 alarm codes (83.6 % of its 798,406 alarm events), the rest of the Siemens table was inferred from SCADA in
|
|
160
|
+
[wind-om-reliability](https://github.com/kwwaleszczynski-del/wind-om-reliability) (column `source` of the table), and 94.9 % of the events have a code in the table. No availability is
|
|
161
|
+
computed for it. The availability figures of the Greenbyte farms are what the operator's categories imply under two illustrative treatments
|
|
162
|
+
(`iec_operational`: every operative state counts as available; `oem_warranty`: grid, environment and requested stops excluded, 60 h of maintenance excused per turbine and year); they are not a reading of any contract.
|
|
163
|
+
|
|
164
|
+
### Converters: the same result through the library and by hand
|
|
165
|
+
|
|
166
|
+
| Target | Input | What runs on both inputs | Largest difference |
|
|
167
|
+
|---|---|---|---|
|
|
168
|
+
| wind-up 0.5 | Kelmarsh 2019, turbines T01 to T03, 157,680 rows | the SCADA frame itself | 1.2e-04 (float32 rounding) |
|
|
169
|
+
| | | `filter_scada_df`: same rows removed (30.2 % of power values) | 1.2e-04; missing pattern identical |
|
|
170
|
+
| | | `calc_pc_and_rated_ws_one_ttype` power curve | 6.5e-06 kW |
|
|
171
|
+
| OpenOA 3 | Kelmarsh 2019, turbines T01 to T03, 157,680 rows | `PlantData.scada` | 1.2e-04 |
|
|
172
|
+
| | | `power_curve.IEC` | 1.6e-02 kW |
|
|
173
|
+
| pvlib 0.16 | PVDAQ system 34, 2017, 35,040 intervals | PVWatts `ModelChain` (modelled plant up to 141 kW) | 8.3e-08 relative in daylight (6.7e-03 W) |
|
|
174
|
+
|
|
175
|
+
OpenOA's default power-curve bins start at 0 (centres 0.25, 0.75, ...) while the bins of IEC 61400-12-1 and res-asset-kpi are centred on multiples of 0.5; with the bins aligned
|
|
176
|
+
the two implementations agree to 0.0e+00 kW, with OpenOA's default bins the curves differ by up to 4.8 % of rated power. The converters do not hide that: it is
|
|
177
|
+
a property of the tool.
|
|
178
|
+
|
|
179
|
+
## Install and use
|
|
180
|
+
|
|
181
|
+
Not on PyPI yet; install from GitHub:
|
|
182
|
+
|
|
183
|
+
```bash
|
|
184
|
+
pip install "scada-schema @ git+https://github.com/kwwaleszczynski-del/scada-schema"
|
|
185
|
+
pip install "scada-schema[converters] @ git+https://github.com/kwwaleszczynski-del/scada-schema" # with wind-up, OpenOA and pvlib
|
|
186
|
+
```
|
|
187
|
+
|
|
188
|
+
From a clone of the repository (development):
|
|
189
|
+
|
|
190
|
+
```bash
|
|
191
|
+
pip install -e ".[converters]" # core: pip install -e .
|
|
192
|
+
```
|
|
193
|
+
|
|
194
|
+
```python
|
|
195
|
+
from scada_schema.loaders import kelmarsh
|
|
196
|
+
from scada_schema import qc, validate
|
|
197
|
+
from scada_schema.states import attach_states
|
|
198
|
+
|
|
199
|
+
loaded = kelmarsh.load("data/kelmarsh", years=[2019]) # fetch first: scada-schema fetch kelmarsh --years 2019
|
|
200
|
+
attach_states(loaded) # status_code and iec_category from the event log
|
|
201
|
+
assert validate(loaded.table, loaded.meta).ok
|
|
202
|
+
report = qc.run(loaded.table, loaded.meta)
|
|
203
|
+
report.to_html("kelmarsh_2019_qc.html")
|
|
204
|
+
|
|
205
|
+
from scada_schema.convert import windup, openoa
|
|
206
|
+
scada, notes = windup.to_windup_scada(loaded.table, loaded.meta)
|
|
207
|
+
plant = openoa.to_openoa(loaded.table, loaded.meta)
|
|
208
|
+
```
|
|
209
|
+
|
|
210
|
+
```bash
|
|
211
|
+
scada-schema load pvdaq --system 34 --years 2016 2017 2018 --out out/pvdaq_34.parquet
|
|
212
|
+
scada-schema qc out/pvdaq_34.parquet --html out/pvdaq_34.html
|
|
213
|
+
scada-schema convert out/pvdaq_34.parquet --to pvlib --out out/pvlib
|
|
214
|
+
```
|
|
215
|
+
|
|
216
|
+
Raw files are looked for in the folder given with `--data-dir`, then in `$SCADA_SCHEMA_<NAME>_DIR`, `$SCADA_SCHEMA_DATA/<name>` and `./data/<name>`.
|
|
217
|
+
|
|
218
|
+
## Your own data
|
|
219
|
+
|
|
220
|
+
A loader for a new export is a mapping file, not code (examples: [wind, stacked CSV](https://github.com/kwwaleszczynski-del/scada-schema/blob/main/docs/examples/wind_semicolon.yaml), [PV, wide CSV](https://github.com/kwwaleszczynski-del/scada-schema/blob/main/docs/examples/pv_wide.yaml)):
|
|
221
|
+
|
|
222
|
+
```yaml
|
|
223
|
+
name: my_wind_farm
|
|
224
|
+
profile: wind
|
|
225
|
+
interval: 10min
|
|
226
|
+
files: ["export/*.csv"]
|
|
227
|
+
read: {sep: ";", decimal: ",", encoding: utf-8}
|
|
228
|
+
timestamp: {column: Time, timezone: Europe/Warsaw, label: end, format: "%d.%m.%Y %H:%M"}
|
|
229
|
+
device_column: Turbine
|
|
230
|
+
columns:
|
|
231
|
+
"Active power [W]": {signal: p_ac, unit: W}
|
|
232
|
+
"Wind [km/h]": {signal: wind_speed, unit: km/h}
|
|
233
|
+
"Ambient [F]": {signal: t_amb, unit: degF}
|
|
234
|
+
"Status": {signal: status_code}
|
|
235
|
+
devices:
|
|
236
|
+
T01: {type: turbine, rated_kw: 2050}
|
|
237
|
+
```
|
|
238
|
+
|
|
239
|
+
```bash
|
|
240
|
+
scada-schema import mapping.yaml --out my_farm.parquet
|
|
241
|
+
scada-schema qc my_farm.parquet --html my_farm_qc.html
|
|
242
|
+
```
|
|
243
|
+
|
|
244
|
+
Time stamps are converted to UTC with the daylight saving rules of the zone (the repeated autumn hour is resolved by order), moved to the start of the interval and checked against the
|
|
245
|
+
grid; units are converted through the registry of res-asset-kpi; every conversion lands in the metadata. A wrong time zone, a missing column or a unit that does not fit the signal stops
|
|
246
|
+
the import with a message that names it. [`schema/mapping.schema.json`](https://github.com/kwwaleszczynski-del/scada-schema/blob/main/schema/mapping.schema.json) lets a YAML editor complete and check the mapping.
|
|
247
|
+
|
|
248
|
+
## What is not verified
|
|
249
|
+
|
|
250
|
+
- The wind-up check uses wind-up 0.5.1 (beta; its v1 input is not fixed) and the filter stage with a minimal configuration; the analysis periods in `windup_config` are placeholders
|
|
251
|
+
and the pitch and generator speed ranges are estimated from the data. A full wind-up assessment was not run.
|
|
252
|
+
- The code tables are written from the public code texts of the open datasets; they are not OEM documents. IEC categories are the top levels of the information model, not the whole tree.
|
|
253
|
+
- The quality thresholds are defaults chosen to be conservative; they were tuned on these datasets and are not validated on others. Findings describe data, not the condition of a sensor.
|
|
254
|
+
- CARE's sensor names are anonymised; the sensor tables of farms A, B and C cover the signals listed in `loaders/care.py` and each entry is checked against the benchmark's own
|
|
255
|
+
description when loading. CARE time stamps are anonymised, so checks that need real time do not apply to it.
|
|
256
|
+
- Hill of Towie's SCADA tables used here have no absolute wind direction; only the nacelle position is mapped.
|
|
257
|
+
- The schema has been applied to open datasets only. Operator exports from other vendors need their own loader (a loader is a column mapping, a unit table and a time convention).
|
|
258
|
+
- `pse-opendata` is not used: the schema holds SCADA, and no PSE data are needed for it.
|
|
259
|
+
|
|
260
|
+
## Reproduce
|
|
261
|
+
|
|
262
|
+
```bash
|
|
263
|
+
scada-schema fetch kelmarsh --years 2019 && scada-schema fetch penmanshiel --years 2020 # and hill_of_towie, care, pvdaq
|
|
264
|
+
python -m scada_schema study --out results
|
|
265
|
+
python scripts/build_pages.py && python scripts/render_readme.py
|
|
266
|
+
pytest -q # tests with real data: set SCADA_SCHEMA_<NAME>_DIR
|
|
267
|
+
```
|
|
268
|
+
|
|
269
|
+
Datasets and licences: La Haute Borne (ENGIE, Etalab open licence as stated by ENGIE, not re-verified; read from the copy shipped with OpenOA, BSD-3-Clause repository), Kelmarsh and Penmanshiel (Cubico Sustainable Investments, CC BY 4.0, doi 10.5281/zenodo.5841833 and 10.5281/zenodo.5946807), Hill of Towie (RES, CC BY 4.0,
|
|
270
|
+
doi 10.5281/zenodo.14870021), CARE to Compare (CC BY-SA 4.0, doi 10.5281/zenodo.15846963; the share-alike condition applies to the data and to datasets derived from it, not to this
|
|
271
|
+
package's code), NREL PVDAQ (CC BY 4.0, doi 10.25984/1846021). The code is MIT licensed.
|
|
272
|
+
|
|
273
|
+
## Citation
|
|
274
|
+
|
|
275
|
+
Cite the software through [CITATION.cff](https://github.com/kwwaleszczynski-del/scada-schema/blob/main/CITATION.cff) (GitHub's "Cite this repository" reads it). A DOI is assigned by Zenodo when the first release is archived; until then cite the repository and the version.
|
|
@@ -0,0 +1,225 @@
|
|
|
1
|
+
# scada-schema: an open data standard for wind and PV SCADA
|
|
2
|
+
|
|
3
|
+
[](https://github.com/kwwaleszczynski-del/scada-schema/actions/workflows/ci.yml) [](https://github.com/kwwaleszczynski-del/scada-schema/blob/main/LICENSE)
|
|
4
|
+
|
|
5
|
+
**Project page with the results: [kwwaleszczynski-del.github.io/scada-schema](https://kwwaleszczynski-del.github.io/scada-schema/)**
|
|
6
|
+
- [specification](https://github.com/kwwaleszczynski-del/scada-schema/blob/main/SPEC.md) - [example quality report](https://kwwaleszczynski-del.github.io/scada-schema/qc/pvdaq_34_raw_units.html).
|
|
7
|
+
|
|
8
|
+
SCADA exports from two wind farms never look alike: columns named after a vendor's tag list, power in hectowatts or watts, time in local
|
|
9
|
+
time or at the end of the interval, a status code that only the turbine manual explains. Every analysis tool then starts with its own
|
|
10
|
+
cleaning script. This package fixes **one table layout** (names, units, sign and angle conventions, time zone and interval, device types) for
|
|
11
|
+
wind turbines and PV plants, reads **open datasets from 6 sources** into it from their original files, checks the data, maps status codes to
|
|
12
|
+
**IEC 61400-26 categories** and converts the table to the inputs of **wind-up, OpenOA and pvlib**, with a test that each tool gives the same result
|
|
13
|
+
on the library's input as on a hand-built one.
|
|
14
|
+
|
|
15
|
+
It is a proposal, not a standard of any body, and working material: the formulas and category names follow public descriptions, and nothing
|
|
16
|
+
here is a certified implementation of an IEC standard.
|
|
17
|
+
|
|
18
|
+
## What is in it
|
|
19
|
+
|
|
20
|
+
1. **Schema** ([SPEC.md](https://github.com/kwwaleszczynski-del/scada-schema/blob/main/SPEC.md), generated from the code): 40 canonical signals (31 in the wind profile, 23 in the PV profile),
|
|
21
|
+
one row per device and interval, `timestamp` in UTC at the start of the interval, canonical units, power positive into the grid, compass
|
|
22
|
+
angles clockwise from true north. Minimum, maximum and standard deviation columns where a source has them. The metadata (profile, interval, the
|
|
23
|
+
time zone and label of the *source*, devices with ratings and positions, licence, every conversion the loader made) are stored inside the
|
|
24
|
+
Parquet file. pydantic describes the metadata; the table is validated by a vectorised check that reports every violation with a count (pandera
|
|
25
|
+
stops at the first failure and adds a dependency, and a per-row quality report is needed anyway).
|
|
26
|
+
2. **Loaders** for Kelmarsh, Penmanshiel, Hill of Towie, La Haute Borne, CARE to Compare and NREL PVDAQ. They read the original archives (Zenodo, OEDI) and
|
|
27
|
+
`fetch` downloads them with checksum verification; nothing large is in this repository. Hill of Towie's 1.4 GB yearly archives are read with range
|
|
28
|
+
requests, so only the needed tables are transferred.
|
|
29
|
+
3. **Quality control** with an HTML report: 15 checks - duplicate and unordered time stamps, time jumps, gaps, daylight saving artefacts, missing values,
|
|
30
|
+
physical limits, flat lines, spikes, power-curve outliers, wind direction (vane, nacelle and north offsets between turbines), energy counters and interval energy,
|
|
31
|
+
icing, and units (order of magnitude and a switch of unit inside a record). The report is one file without scripts.
|
|
32
|
+
4. **IEC 61400-26 states**: code tables for the Senvion MM series (268 codes) and the Siemens SWT 2.3 (146 codes) as logged in the open
|
|
33
|
+
datasets, label tables for Greenbyte and CARE, projection of an event log onto the intervals, and time-based availability through
|
|
34
|
+
[res-asset-kpi](https://github.com/kwwaleszczynski-del/res-asset-kpi) (which also supplies the unit, time-zone and category code used here).
|
|
35
|
+
5. **Converters** to wind-up, OpenOA and pvlib and the parity checks below.
|
|
36
|
+
6. **Your own data without code**: `scada-schema import mapping.yaml` turns a CSV, Excel or Parquet export into a canonical table from a YAML mapping (columns, units,
|
|
37
|
+
time zone and label, devices); see below.
|
|
38
|
+
7. **JSON Schema** of the metadata and of the mapping file, and a list of every canonical column, in [`schema/`](https://github.com/kwwaleszczynski-del/scada-schema/blob/main/schema) for tools that are not written in Python.
|
|
39
|
+
8. **Resampling** to a coarser interval with rules by signal kind (mean, vector mean of angles, sum, last meter reading, pooled standard deviation, coverage threshold): `scada-schema resample`.
|
|
40
|
+
9. **Command line**: `scada-schema fetch | load | import | resample | validate | qc | convert | jsonschema | spec | study`.
|
|
41
|
+
|
|
42
|
+
## Results
|
|
43
|
+
|
|
44
|
+
All numbers on this page come from `python -m scada_schema study`, which writes `results/study.json`; `tests/test_docs.py` renders this README from the
|
|
45
|
+
template and that file and fails when they differ.
|
|
46
|
+
|
|
47
|
+
12 tables from 6 sources (3,870,948 rows, 59 devices) were loaded from the original files; the schema accepts 12 of 12.
|
|
48
|
+
|
|
49
|
+
| Dataset | Source (licence) | Profile | Interval | Rows | Devices | Signals | Schema | QC findings (error / warning / info) |
|
|
50
|
+
|---|---|---|---|---:|---:|---:|---|---|
|
|
51
|
+
| [kelmarsh_2019](https://kwwaleszczynski-del.github.io/scada-schema/qc/kelmarsh_2019.html) | Kelmarsh wind farm data (Cubico Sustainable Investments) (CC-BY-4.0) | wind | 10min | 315,360 | 6 | 23 (+39 statistics) | valid | 0 / 19 / 49 |
|
|
52
|
+
| [penmanshiel_2020](https://kwwaleszczynski-del.github.io/scada-schema/qc/penmanshiel_2020.html) | Penmanshiel wind farm data (Cubico Sustainable Investments) (CC-BY-4.0) | wind | 10min | 737,856 | 14 | 23 (+60 statistics) | valid | 0 / 359 / 107 |
|
|
53
|
+
| [hill_of_towie_2021](https://kwwaleszczynski-del.github.io/scada-schema/qc/hill_of_towie_2021.html) | Hill of Towie wind farm open dataset (RES) (CC-BY-4.0) | wind | 10min | 1,090,921 | 21 | 20 (+42 statistics) | valid | 2 / 115 / 155 |
|
|
54
|
+
| [la_haute_borne](https://kwwaleszczynski-del.github.io/scada-schema/qc/la_haute_borne.html) | La Haute Borne wind farm data (ENGIE, copy shipped with OpenOA) (Etalab open licence (as stated by ENGIE, not re-verified)) | wind | 10min | 420,432 | 4 | 7 (+0 statistics) | valid | 0 / 11 / 18 |
|
|
55
|
+
| [care_A](https://kwwaleszczynski-del.github.io/scada-schema/qc/care_A.html) | CARE to Compare (CC-BY-SA-4.0) | wind | 10min | 54,358 | 1 | 16 (+15 statistics) | valid | 0 / 3 / 6 |
|
|
56
|
+
| [care_B](https://kwwaleszczynski-del.github.io/scada-schema/qc/care_B.html) | CARE to Compare (CC-BY-SA-4.0) | wind | 10min | 56,564 | 1 | 15 (+39 statistics) | valid | 1 / 6 / 5 |
|
|
57
|
+
| [care_C](https://kwwaleszczynski-del.github.io/scada-schema/qc/care_C.html) | CARE to Compare (CC-BY-SA-4.0) | wind | 10min | 55,753 | 1 | 8 (+18 statistics) | valid | 0 / 3 / 1 |
|
|
58
|
+
| [pvdaq_34](https://kwwaleszczynski-del.github.io/scada-schema/qc/pvdaq_34.html) | NREL PVDAQ public datasets (OEDI) (CC-BY-4.0) | pv | 15min | 210,300 | 2 | 15 (+0 statistics) | valid | 2 / 7 / 7 |
|
|
59
|
+
| [pvdaq_1276](https://kwwaleszczynski-del.github.io/scada-schema/qc/pvdaq_1276.html) | NREL PVDAQ public datasets (OEDI) (CC-BY-4.0) | pv | 15min | 209,162 | 2 | 14 (+0 statistics) | valid | 2 / 11 / 5 |
|
|
60
|
+
| [pvdaq_1278](https://kwwaleszczynski-del.github.io/scada-schema/qc/pvdaq_1278.html) | NREL PVDAQ public datasets (OEDI) (CC-BY-4.0) | pv | 15min | 315,456 | 3 | 14 (+0 statistics) | valid | 3 / 12 / 10 |
|
|
61
|
+
| [pvdaq_35](https://kwwaleszczynski-del.github.io/scada-schema/qc/pvdaq_35.html) | NREL PVDAQ public datasets (OEDI) (CC-BY-4.0) | pv | 15min | 194,486 | 2 | 15 (+0 statistics) | valid | 2 / 13 / 7 |
|
|
62
|
+
| [pvdaq_34_raw_units](https://kwwaleszczynski-del.github.io/scada-schema/qc/pvdaq_34_raw_units.html) | NREL PVDAQ public datasets (OEDI) (CC-BY-4.0) | pv | 15min | 210,300 | 2 | 15 (+0 statistics) | valid | 4 / 7 / 7 |
|
|
63
|
+
|
|
64
|
+
Number of devices with at least one finding, by check (the quality reports list every finding with its longest episodes):
|
|
65
|
+
|
|
66
|
+
| Check | kelmarsh_2019 | penmanshiel_2020 | hill_of_towie_2021 | la_haute_borne | care_A | care_B | care_C | pvdaq_34 | pvdaq_1276 | pvdaq_1278 | pvdaq_35 | pvdaq_34_raw_units |
|
|
67
|
+
|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|
|
|
68
|
+
| `duplicates` | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 |
|
|
69
|
+
| `ordering` | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 |
|
|
70
|
+
| `time_jumps` | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 |
|
|
71
|
+
| `gaps` | 0 | 0 | 21 | 4 | 1 | 1 | 0 | 2 | 2 | 3 | 2 | 2 |
|
|
72
|
+
| `dst` | 0 | 0 | 0 | 4 | skipped | skipped | skipped | skipped | skipped | skipped | skipped | skipped |
|
|
73
|
+
| `missing_values` | 6 | 14 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 |
|
|
74
|
+
| `out_of_range` | 5 | 14 | 0 | 3 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 |
|
|
75
|
+
| `flatline` | 1 | 13 | 21 | 1 | 1 | 1 | 1 | 2 | 2 | 3 | 2 | 2 |
|
|
76
|
+
| `spikes` | 6 | 14 | 21 | 1 | 1 | 1 | 1 | 2 | 2 | 3 | 2 | 2 |
|
|
77
|
+
| `power_curve` | 6 | 14 | 21 | 4 | 1 | 1 | 1 | skipped | skipped | skipped | skipped | skipped |
|
|
78
|
+
| `wind_direction` | 6 | 14 | 21 | 4 | 0 | 0 | skipped | 0 | 0 | 0 | 0 | 0 |
|
|
79
|
+
| `icing` | 1 | 8 | 21 | 4 | 0 | 1 | 0 | skipped | skipped | skipped | skipped | skipped |
|
|
80
|
+
| `energy_counter` | 2 | 0 | skipped | skipped | skipped | skipped | skipped | skipped | skipped | skipped | skipped | skipped |
|
|
81
|
+
| `energy_power` | 0 | 0 | skipped | skipped | skipped | skipped | skipped | skipped | skipped | skipped | skipped | skipped |
|
|
82
|
+
| `units` | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 |
|
|
83
|
+
|
|
84
|
+
What the checks found on real data:
|
|
85
|
+
|
|
86
|
+
- **A unit switch is found where the dataset documents it.** PVDAQ's power channels change from hectowatts to watts in August 2018. Read as hectowatts
|
|
87
|
+
throughout, system 34 shows a step of a factor of 98 in power over irradiance on 2018-08-05; the loader finds the same switch from the data
|
|
88
|
+
(`p_ac, p_dc: hectowatts to kW (x0.1) until 2018-08-04 20:00 UTC, watts to kW (x0.001) from then on`), and the check on the corrected table reports 0 switches.
|
|
89
|
+
- **North offsets.** On Hill of Towie the nacelle position of 12 of 21 turbines is more than 10 degrees off the fleet mean direction;
|
|
90
|
+
the largest, T16, is 84.7 degrees off. On Kelmarsh the largest offset is 3.8 degrees.
|
|
91
|
+
- **A real daylight saving artefact.** La Haute Borne stamps its data with the UTC offset, so the time is exact, but at both autumn changes (2014-10-26 and 2015-10-25) one hour is missing in UTC (00:00 to 01:00, the first occurrence of
|
|
92
|
+
the repeated local hour): the source holds only one of the two repeated hours. The `dst` check, run against Europe/Paris, finds 2 of 4 changes on every turbine.
|
|
93
|
+
- **Time elsewhere.** The three UK farms are checked against Europe/London (UTC time stamps, no artefact found); for the others the check does not apply (fixed offset or anonymised time).
|
|
94
|
+
The table above shows what each dataset triggered.
|
|
95
|
+
|
|
96
|
+
### IEC 61400-26 mapping
|
|
97
|
+
|
|
98
|
+
A stop event labels an interval when the interval's midpoint lies in the event. On the Greenbyte farms the operator also labels each stop with an IEC category,
|
|
99
|
+
which tests the code tables:
|
|
100
|
+
|
|
101
|
+
| Dataset | Code table | Events | Events with a code in the table | Stop events | Compared with the operator's category | Agreement | Time-based availability, IEC operational / OEM style |
|
|
102
|
+
|---|---|---:|---:|---:|---:|---:|---|
|
|
103
|
+
| kelmarsh_2019 | `senvion_mm` | 59,313 | 100.0 % | 673 | 673 | 100.0 % | 99.5 % / 99.8 % |
|
|
104
|
+
| penmanshiel_2020 | `senvion_mm` | 158,509 | 100.0 % | 3,267 | 3,264 | 100.0 % | 97.8 % / 98.1 % |
|
|
105
|
+
| hill_of_towie_2021 | `siemens_swt_hot` | 798,406 | 94.9 % | 4,034 | - | no operator category | not computed |
|
|
106
|
+
|
|
107
|
+
On Kelmarsh the Senvion table agrees with the operator's category on 100.0 % of 673 compared stop events, on Penmanshiel on 100.0 % of 3,264. **This is a
|
|
108
|
+
consistency check, not an independent test:** the Senvion table was written from the same open logs. The Hill of Towie alarm log has no operator category; the dataset describes only
|
|
109
|
+
12 alarm codes (83.6 % of its 798,406 alarm events), the rest of the Siemens table was inferred from SCADA in
|
|
110
|
+
[wind-om-reliability](https://github.com/kwwaleszczynski-del/wind-om-reliability) (column `source` of the table), and 94.9 % of the events have a code in the table. No availability is
|
|
111
|
+
computed for it. The availability figures of the Greenbyte farms are what the operator's categories imply under two illustrative treatments
|
|
112
|
+
(`iec_operational`: every operative state counts as available; `oem_warranty`: grid, environment and requested stops excluded, 60 h of maintenance excused per turbine and year); they are not a reading of any contract.
|
|
113
|
+
|
|
114
|
+
### Converters: the same result through the library and by hand
|
|
115
|
+
|
|
116
|
+
| Target | Input | What runs on both inputs | Largest difference |
|
|
117
|
+
|---|---|---|---|
|
|
118
|
+
| wind-up 0.5 | Kelmarsh 2019, turbines T01 to T03, 157,680 rows | the SCADA frame itself | 1.2e-04 (float32 rounding) |
|
|
119
|
+
| | | `filter_scada_df`: same rows removed (30.2 % of power values) | 1.2e-04; missing pattern identical |
|
|
120
|
+
| | | `calc_pc_and_rated_ws_one_ttype` power curve | 6.5e-06 kW |
|
|
121
|
+
| OpenOA 3 | Kelmarsh 2019, turbines T01 to T03, 157,680 rows | `PlantData.scada` | 1.2e-04 |
|
|
122
|
+
| | | `power_curve.IEC` | 1.6e-02 kW |
|
|
123
|
+
| pvlib 0.16 | PVDAQ system 34, 2017, 35,040 intervals | PVWatts `ModelChain` (modelled plant up to 141 kW) | 8.3e-08 relative in daylight (6.7e-03 W) |
|
|
124
|
+
|
|
125
|
+
OpenOA's default power-curve bins start at 0 (centres 0.25, 0.75, ...) while the bins of IEC 61400-12-1 and res-asset-kpi are centred on multiples of 0.5; with the bins aligned
|
|
126
|
+
the two implementations agree to 0.0e+00 kW, with OpenOA's default bins the curves differ by up to 4.8 % of rated power. The converters do not hide that: it is
|
|
127
|
+
a property of the tool.
|
|
128
|
+
|
|
129
|
+
## Install and use
|
|
130
|
+
|
|
131
|
+
Not on PyPI yet; install from GitHub:
|
|
132
|
+
|
|
133
|
+
```bash
|
|
134
|
+
pip install "scada-schema @ git+https://github.com/kwwaleszczynski-del/scada-schema"
|
|
135
|
+
pip install "scada-schema[converters] @ git+https://github.com/kwwaleszczynski-del/scada-schema" # with wind-up, OpenOA and pvlib
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
From a clone of the repository (development):
|
|
139
|
+
|
|
140
|
+
```bash
|
|
141
|
+
pip install -e ".[converters]" # core: pip install -e .
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
```python
|
|
145
|
+
from scada_schema.loaders import kelmarsh
|
|
146
|
+
from scada_schema import qc, validate
|
|
147
|
+
from scada_schema.states import attach_states
|
|
148
|
+
|
|
149
|
+
loaded = kelmarsh.load("data/kelmarsh", years=[2019]) # fetch first: scada-schema fetch kelmarsh --years 2019
|
|
150
|
+
attach_states(loaded) # status_code and iec_category from the event log
|
|
151
|
+
assert validate(loaded.table, loaded.meta).ok
|
|
152
|
+
report = qc.run(loaded.table, loaded.meta)
|
|
153
|
+
report.to_html("kelmarsh_2019_qc.html")
|
|
154
|
+
|
|
155
|
+
from scada_schema.convert import windup, openoa
|
|
156
|
+
scada, notes = windup.to_windup_scada(loaded.table, loaded.meta)
|
|
157
|
+
plant = openoa.to_openoa(loaded.table, loaded.meta)
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
```bash
|
|
161
|
+
scada-schema load pvdaq --system 34 --years 2016 2017 2018 --out out/pvdaq_34.parquet
|
|
162
|
+
scada-schema qc out/pvdaq_34.parquet --html out/pvdaq_34.html
|
|
163
|
+
scada-schema convert out/pvdaq_34.parquet --to pvlib --out out/pvlib
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
Raw files are looked for in the folder given with `--data-dir`, then in `$SCADA_SCHEMA_<NAME>_DIR`, `$SCADA_SCHEMA_DATA/<name>` and `./data/<name>`.
|
|
167
|
+
|
|
168
|
+
## Your own data
|
|
169
|
+
|
|
170
|
+
A loader for a new export is a mapping file, not code (examples: [wind, stacked CSV](https://github.com/kwwaleszczynski-del/scada-schema/blob/main/docs/examples/wind_semicolon.yaml), [PV, wide CSV](https://github.com/kwwaleszczynski-del/scada-schema/blob/main/docs/examples/pv_wide.yaml)):
|
|
171
|
+
|
|
172
|
+
```yaml
|
|
173
|
+
name: my_wind_farm
|
|
174
|
+
profile: wind
|
|
175
|
+
interval: 10min
|
|
176
|
+
files: ["export/*.csv"]
|
|
177
|
+
read: {sep: ";", decimal: ",", encoding: utf-8}
|
|
178
|
+
timestamp: {column: Time, timezone: Europe/Warsaw, label: end, format: "%d.%m.%Y %H:%M"}
|
|
179
|
+
device_column: Turbine
|
|
180
|
+
columns:
|
|
181
|
+
"Active power [W]": {signal: p_ac, unit: W}
|
|
182
|
+
"Wind [km/h]": {signal: wind_speed, unit: km/h}
|
|
183
|
+
"Ambient [F]": {signal: t_amb, unit: degF}
|
|
184
|
+
"Status": {signal: status_code}
|
|
185
|
+
devices:
|
|
186
|
+
T01: {type: turbine, rated_kw: 2050}
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
```bash
|
|
190
|
+
scada-schema import mapping.yaml --out my_farm.parquet
|
|
191
|
+
scada-schema qc my_farm.parquet --html my_farm_qc.html
|
|
192
|
+
```
|
|
193
|
+
|
|
194
|
+
Time stamps are converted to UTC with the daylight saving rules of the zone (the repeated autumn hour is resolved by order), moved to the start of the interval and checked against the
|
|
195
|
+
grid; units are converted through the registry of res-asset-kpi; every conversion lands in the metadata. A wrong time zone, a missing column or a unit that does not fit the signal stops
|
|
196
|
+
the import with a message that names it. [`schema/mapping.schema.json`](https://github.com/kwwaleszczynski-del/scada-schema/blob/main/schema/mapping.schema.json) lets a YAML editor complete and check the mapping.
|
|
197
|
+
|
|
198
|
+
## What is not verified
|
|
199
|
+
|
|
200
|
+
- The wind-up check uses wind-up 0.5.1 (beta; its v1 input is not fixed) and the filter stage with a minimal configuration; the analysis periods in `windup_config` are placeholders
|
|
201
|
+
and the pitch and generator speed ranges are estimated from the data. A full wind-up assessment was not run.
|
|
202
|
+
- The code tables are written from the public code texts of the open datasets; they are not OEM documents. IEC categories are the top levels of the information model, not the whole tree.
|
|
203
|
+
- The quality thresholds are defaults chosen to be conservative; they were tuned on these datasets and are not validated on others. Findings describe data, not the condition of a sensor.
|
|
204
|
+
- CARE's sensor names are anonymised; the sensor tables of farms A, B and C cover the signals listed in `loaders/care.py` and each entry is checked against the benchmark's own
|
|
205
|
+
description when loading. CARE time stamps are anonymised, so checks that need real time do not apply to it.
|
|
206
|
+
- Hill of Towie's SCADA tables used here have no absolute wind direction; only the nacelle position is mapped.
|
|
207
|
+
- The schema has been applied to open datasets only. Operator exports from other vendors need their own loader (a loader is a column mapping, a unit table and a time convention).
|
|
208
|
+
- `pse-opendata` is not used: the schema holds SCADA, and no PSE data are needed for it.
|
|
209
|
+
|
|
210
|
+
## Reproduce
|
|
211
|
+
|
|
212
|
+
```bash
|
|
213
|
+
scada-schema fetch kelmarsh --years 2019 && scada-schema fetch penmanshiel --years 2020 # and hill_of_towie, care, pvdaq
|
|
214
|
+
python -m scada_schema study --out results
|
|
215
|
+
python scripts/build_pages.py && python scripts/render_readme.py
|
|
216
|
+
pytest -q # tests with real data: set SCADA_SCHEMA_<NAME>_DIR
|
|
217
|
+
```
|
|
218
|
+
|
|
219
|
+
Datasets and licences: La Haute Borne (ENGIE, Etalab open licence as stated by ENGIE, not re-verified; read from the copy shipped with OpenOA, BSD-3-Clause repository), Kelmarsh and Penmanshiel (Cubico Sustainable Investments, CC BY 4.0, doi 10.5281/zenodo.5841833 and 10.5281/zenodo.5946807), Hill of Towie (RES, CC BY 4.0,
|
|
220
|
+
doi 10.5281/zenodo.14870021), CARE to Compare (CC BY-SA 4.0, doi 10.5281/zenodo.15846963; the share-alike condition applies to the data and to datasets derived from it, not to this
|
|
221
|
+
package's code), NREL PVDAQ (CC BY 4.0, doi 10.25984/1846021). The code is MIT licensed.
|
|
222
|
+
|
|
223
|
+
## Citation
|
|
224
|
+
|
|
225
|
+
Cite the software through [CITATION.cff](https://github.com/kwwaleszczynski-del/scada-schema/blob/main/CITATION.cff) (GitHub's "Cite this repository" reads it). A DOI is assigned by Zenodo when the first release is archived; until then cite the repository and the version.
|