scada-schema 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. scada_schema-0.1.0/CHANGELOG.md +19 -0
  2. scada_schema-0.1.0/CITATION.cff +33 -0
  3. scada_schema-0.1.0/LICENSE +21 -0
  4. scada_schema-0.1.0/MANIFEST.in +6 -0
  5. scada_schema-0.1.0/PKG-INFO +275 -0
  6. scada_schema-0.1.0/README.md +225 -0
  7. scada_schema-0.1.0/SPEC.md +161 -0
  8. scada_schema-0.1.0/pyproject.toml +80 -0
  9. scada_schema-0.1.0/release.json +7 -0
  10. scada_schema-0.1.0/schema/columns.json +2513 -0
  11. scada_schema-0.1.0/schema/dataset-meta.schema.json +300 -0
  12. scada_schema-0.1.0/schema/mapping.schema.json +428 -0
  13. scada_schema-0.1.0/setup.cfg +4 -0
  14. scada_schema-0.1.0/src/scada_schema/__init__.py +31 -0
  15. scada_schema-0.1.0/src/scada_schema/__main__.py +3 -0
  16. scada_schema-0.1.0/src/scada_schema/_steps.py +50 -0
  17. scada_schema-0.1.0/src/scada_schema/_time.py +12 -0
  18. scada_schema-0.1.0/src/scada_schema/_version.py +1 -0
  19. scada_schema-0.1.0/src/scada_schema/cli.py +295 -0
  20. scada_schema-0.1.0/src/scada_schema/convert/__init__.py +8 -0
  21. scada_schema-0.1.0/src/scada_schema/convert/openoa.py +80 -0
  22. scada_schema-0.1.0/src/scada_schema/convert/pvlib.py +98 -0
  23. scada_schema-0.1.0/src/scada_schema/convert/windup.py +175 -0
  24. scada_schema-0.1.0/src/scada_schema/data/codes_senvion_mm.csv +269 -0
  25. scada_schema-0.1.0/src/scada_schema/data/codes_siemens_swt_hot.csv +147 -0
  26. scada_schema-0.1.0/src/scada_schema/data/iec_labels.csv +17 -0
  27. scada_schema-0.1.0/src/scada_schema/io.py +101 -0
  28. scada_schema-0.1.0/src/scada_schema/jsonschema.py +57 -0
  29. scada_schema-0.1.0/src/scada_schema/loaders/__init__.py +24 -0
  30. scada_schema-0.1.0/src/scada_schema/loaders/_common.py +112 -0
  31. scada_schema-0.1.0/src/scada_schema/loaders/_paths.py +21 -0
  32. scada_schema-0.1.0/src/scada_schema/loaders/_remotezip.py +67 -0
  33. scada_schema-0.1.0/src/scada_schema/loaders/care.py +167 -0
  34. scada_schema-0.1.0/src/scada_schema/loaders/generic.py +315 -0
  35. scada_schema-0.1.0/src/scada_schema/loaders/greenbyte.py +202 -0
  36. scada_schema-0.1.0/src/scada_schema/loaders/hill_of_towie.py +200 -0
  37. scada_schema-0.1.0/src/scada_schema/loaders/kelmarsh.py +53 -0
  38. scada_schema-0.1.0/src/scada_schema/loaders/la_haute_borne.py +81 -0
  39. scada_schema-0.1.0/src/scada_schema/loaders/penmanshiel.py +48 -0
  40. scada_schema-0.1.0/src/scada_schema/loaders/pvdaq.py +208 -0
  41. scada_schema-0.1.0/src/scada_schema/model.py +174 -0
  42. scada_schema-0.1.0/src/scada_schema/parity.py +181 -0
  43. scada_schema-0.1.0/src/scada_schema/py.typed +0 -0
  44. scada_schema-0.1.0/src/scada_schema/qc/__init__.py +14 -0
  45. scada_schema-0.1.0/src/scada_schema/qc/checks.py +580 -0
  46. scada_schema-0.1.0/src/scada_schema/qc/core.py +308 -0
  47. scada_schema-0.1.0/src/scada_schema/qc/report.py +167 -0
  48. scada_schema-0.1.0/src/scada_schema/resample.py +122 -0
  49. scada_schema-0.1.0/src/scada_schema/signals.py +169 -0
  50. scada_schema-0.1.0/src/scada_schema/spec.py +150 -0
  51. scada_schema-0.1.0/src/scada_schema/states.py +221 -0
  52. scada_schema-0.1.0/src/scada_schema/study.py +274 -0
  53. scada_schema-0.1.0/src/scada_schema/validate.py +207 -0
  54. scada_schema-0.1.0/src/scada_schema.egg-info/PKG-INFO +275 -0
  55. scada_schema-0.1.0/src/scada_schema.egg-info/SOURCES.txt +68 -0
  56. scada_schema-0.1.0/src/scada_schema.egg-info/dependency_links.txt +1 -0
  57. scada_schema-0.1.0/src/scada_schema.egg-info/entry_points.txt +2 -0
  58. scada_schema-0.1.0/src/scada_schema.egg-info/requires.txt +25 -0
  59. scada_schema-0.1.0/src/scada_schema.egg-info/top_level.txt +1 -0
  60. scada_schema-0.1.0/tests/conftest.py +147 -0
  61. scada_schema-0.1.0/tests/test_cli.py +112 -0
  62. scada_schema-0.1.0/tests/test_core.py +98 -0
  63. scada_schema-0.1.0/tests/test_docs.py +146 -0
  64. scada_schema-0.1.0/tests/test_generic.py +191 -0
  65. scada_schema-0.1.0/tests/test_loaders.py +299 -0
  66. scada_schema-0.1.0/tests/test_misc.py +45 -0
  67. scada_schema-0.1.0/tests/test_parity.py +172 -0
  68. scada_schema-0.1.0/tests/test_qc.py +212 -0
  69. scada_schema-0.1.0/tests/test_resample.py +89 -0
  70. scada_schema-0.1.0/tests/test_states.py +122 -0
@@ -0,0 +1,19 @@
1
+ # Changelog
2
+
3
+ All notable changes to this project are documented here. The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and the project uses [semantic versioning](https://semver.org/).
4
+
5
+ ## [0.1.0] - 2026-11-06
6
+
7
+ First release.
8
+
9
+ ### Added
10
+
11
+ - The schema (version 1.0, wind and PV profiles), `SPEC.md` generated from the code, JSON Schema files and a column list (`schema/`).
12
+ - Loaders for Kelmarsh, Penmanshiel, Hill of Towie, La Haute Borne, CARE to Compare and NREL PVDAQ (with `fetch` and checksum verification).
13
+ - A loader driven by a YAML mapping (`scada-schema import`) and resampling to a coarser interval (`scada-schema resample`).
14
+ - 15 quality checks with a self-contained HTML report.
15
+ - IEC 61400-26 state mapping (code tables, label tables, event projection) and time-based availability through res-asset-kpi.
16
+ - Converters to wind-up, OpenOA and pvlib with parity checks.
17
+ - Command line interface, the study that produces every number of the README, tests on Linux and Windows, CITATION.cff and Zenodo metadata, and a release workflow for PyPI Trusted Publishing (disabled until `PYPI_PUBLISH` is set).
18
+
19
+ [0.1.0]: https://github.com/kwwaleszczynski-del/scada-schema/releases/tag/v0.1.0
@@ -0,0 +1,33 @@
1
+ cff-version: 1.2.0
2
+ message: "If you use this software, please cite it using the metadata below."
3
+ type: software
4
+ title: "scada-schema: an open data standard for wind and PV SCADA"
5
+ abstract: >-
6
+ One table schema for wind turbine and PV SCADA (names, units, sign and angle conventions, UTC interval start, device types),
7
+ loaders that read open datasets (Kelmarsh, Penmanshiel, Hill of Towie, La Haute Borne, CARE to Compare, NREL PVDAQ) from their
8
+ original files, a loader driven by a YAML mapping for other exports, resampling, 15 data quality checks with an HTML report,
9
+ mapping of status codes to IEC 61400-26 categories, and converters to wind-up, OpenOA and pvlib with parity checks.
10
+ authors:
11
+ - family-names: "Waleszczyński"
12
+ given-names: "Konrad"
13
+ version: 0.1.0
14
+ date-released: "2026-11-06"
15
+ license: MIT
16
+ repository-code: "https://github.com/kwwaleszczynski-del/scada-schema"
17
+ url: "https://kwwaleszczynski-del.github.io/scada-schema/"
18
+ keywords:
19
+ - SCADA
20
+ - wind energy
21
+ - photovoltaics
22
+ - data schema
23
+ - data quality
24
+ - IEC 61400-26
25
+ - open data
26
+ references:
27
+ - type: software
28
+ title: "res-asset-kpi: key performance indicators for operating PV, wind and battery assets"
29
+ authors:
30
+ - family-names: "Waleszczyński"
31
+ given-names: "Konrad"
32
+ doi: 10.5281/zenodo.23742958
33
+ repository-code: "https://github.com/kwwaleszczynski-del/res-asset-kpi"
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Konrad Waleszczyński
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,6 @@
1
+ include CHANGELOG.md
2
+ include CITATION.cff
3
+ include SPEC.md
4
+ include release.json
5
+ graft schema
6
+ recursive-include tests *.py
@@ -0,0 +1,275 @@
1
+ Metadata-Version: 2.4
2
+ Name: scada-schema
3
+ Version: 0.1.0
4
+ Summary: An open data standard for wind and PV SCADA: one schema, loaders for open datasets, quality control with an HTML report, IEC 61400-26 state mapping and converters to wind-up, OpenOA and pvlib.
5
+ Author: Konrad Waleszczyński
6
+ License-Expression: MIT
7
+ Project-URL: Homepage, https://github.com/kwwaleszczynski-del/scada-schema
8
+ Project-URL: Documentation, https://kwwaleszczynski-del.github.io/scada-schema/
9
+ Project-URL: Source, https://github.com/kwwaleszczynski-del/scada-schema
10
+ Project-URL: Changelog, https://github.com/kwwaleszczynski-del/scada-schema/blob/main/CHANGELOG.md
11
+ Project-URL: Specification, https://github.com/kwwaleszczynski-del/scada-schema/blob/main/SPEC.md
12
+ Project-URL: Issues, https://github.com/kwwaleszczynski-del/scada-schema/issues
13
+ Keywords: scada,wind-energy,photovoltaics,data-schema,data-quality,iec-61400-26,wind-up,openoa,pvlib,parquet,open-data
14
+ Classifier: Development Status :: 3 - Alpha
15
+ Classifier: Intended Audience :: Science/Research
16
+ Classifier: Intended Audience :: Developers
17
+ Classifier: Operating System :: OS Independent
18
+ Classifier: Programming Language :: Python :: 3
19
+ Classifier: Programming Language :: Python :: 3 :: Only
20
+ Classifier: Programming Language :: Python :: 3.11
21
+ Classifier: Programming Language :: Python :: 3.12
22
+ Classifier: Programming Language :: Python :: 3.13
23
+ Classifier: Topic :: Scientific/Engineering
24
+ Classifier: Topic :: Scientific/Engineering :: Information Analysis
25
+ Classifier: Typing :: Typed
26
+ Requires-Python: >=3.11
27
+ Description-Content-Type: text/markdown
28
+ License-File: LICENSE
29
+ Requires-Dist: numpy>=1.26
30
+ Requires-Dist: pandas>=2.1
31
+ Requires-Dist: pyarrow>=14
32
+ Requires-Dist: pydantic>=2.5
33
+ Requires-Dist: pyyaml>=6.0
34
+ Requires-Dist: requests>=2.31
35
+ Requires-Dist: res-asset-kpi>=0.1.0
36
+ Provides-Extra: windup
37
+ Requires-Dist: res-wind-up>=0.5; extra == "windup"
38
+ Provides-Extra: openoa
39
+ Requires-Dist: openoa>=3.1; extra == "openoa"
40
+ Provides-Extra: pvlib
41
+ Requires-Dist: pvlib>=0.11; extra == "pvlib"
42
+ Provides-Extra: converters
43
+ Requires-Dist: scada-schema[openoa,pvlib,windup]; extra == "converters"
44
+ Provides-Extra: dev
45
+ Requires-Dist: pytest>=8; extra == "dev"
46
+ Requires-Dist: ruff>=0.6; extra == "dev"
47
+ Requires-Dist: build>=1.2; extra == "dev"
48
+ Requires-Dist: twine>=5; extra == "dev"
49
+ Dynamic: license-file
50
+
51
+ # scada-schema: an open data standard for wind and PV SCADA
52
+
53
+ [![tests](https://github.com/kwwaleszczynski-del/scada-schema/actions/workflows/ci.yml/badge.svg)](https://github.com/kwwaleszczynski-del/scada-schema/actions/workflows/ci.yml) [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/kwwaleszczynski-del/scada-schema/blob/main/LICENSE)
54
+
55
+ **Project page with the results: [kwwaleszczynski-del.github.io/scada-schema](https://kwwaleszczynski-del.github.io/scada-schema/)**
56
+ - [specification](https://github.com/kwwaleszczynski-del/scada-schema/blob/main/SPEC.md) - [example quality report](https://kwwaleszczynski-del.github.io/scada-schema/qc/pvdaq_34_raw_units.html).
57
+
58
+ SCADA exports from two wind farms never look alike: columns named after a vendor's tag list, power in hectowatts or watts, time in local
59
+ time or at the end of the interval, a status code that only the turbine manual explains. Every analysis tool then starts with its own
60
+ cleaning script. This package fixes **one table layout** (names, units, sign and angle conventions, time zone and interval, device types) for
61
+ wind turbines and PV plants, reads **open datasets from 6 sources** into it from their original files, checks the data, maps status codes to
62
+ **IEC 61400-26 categories** and converts the table to the inputs of **wind-up, OpenOA and pvlib**, with a test that each tool gives the same result
63
+ on the library's input as on a hand-built one.
64
+
65
+ It is a proposal, not a standard of any body, and working material: the formulas and category names follow public descriptions, and nothing
66
+ here is a certified implementation of an IEC standard.
67
+
68
+ ## What is in it
69
+
70
+ 1. **Schema** ([SPEC.md](https://github.com/kwwaleszczynski-del/scada-schema/blob/main/SPEC.md), generated from the code): 40 canonical signals (31 in the wind profile, 23 in the PV profile),
71
+ one row per device and interval, `timestamp` in UTC at the start of the interval, canonical units, power positive into the grid, compass
72
+ angles clockwise from true north. Minimum, maximum and standard deviation columns where a source has them. The metadata (profile, interval, the
73
+ time zone and label of the *source*, devices with ratings and positions, licence, every conversion the loader made) are stored inside the
74
+ Parquet file. pydantic describes the metadata; the table is validated by a vectorised check that reports every violation with a count (pandera
75
+ stops at the first failure and adds a dependency, and a per-row quality report is needed anyway).
76
+ 2. **Loaders** for Kelmarsh, Penmanshiel, Hill of Towie, La Haute Borne, CARE to Compare and NREL PVDAQ. They read the original archives (Zenodo, OEDI) and
77
+ `fetch` downloads them with checksum verification; nothing large is in this repository. Hill of Towie's 1.4 GB yearly archives are read with range
78
+ requests, so only the needed tables are transferred.
79
+ 3. **Quality control** with an HTML report: 15 checks - duplicate and unordered time stamps, time jumps, gaps, daylight saving artefacts, missing values,
80
+ physical limits, flat lines, spikes, power-curve outliers, wind direction (vane, nacelle and north offsets between turbines), energy counters and interval energy,
81
+ icing, and units (order of magnitude and a switch of unit inside a record). The report is one file without scripts.
82
+ 4. **IEC 61400-26 states**: code tables for the Senvion MM series (268 codes) and the Siemens SWT 2.3 (146 codes) as logged in the open
83
+ datasets, label tables for Greenbyte and CARE, projection of an event log onto the intervals, and time-based availability through
84
+ [res-asset-kpi](https://github.com/kwwaleszczynski-del/res-asset-kpi) (which also supplies the unit, time-zone and category code used here).
85
+ 5. **Converters** to wind-up, OpenOA and pvlib and the parity checks below.
86
+ 6. **Your own data without code**: `scada-schema import mapping.yaml` turns a CSV, Excel or Parquet export into a canonical table from a YAML mapping (columns, units,
87
+ time zone and label, devices); see below.
88
+ 7. **JSON Schema** of the metadata and of the mapping file, and a list of every canonical column, in [`schema/`](https://github.com/kwwaleszczynski-del/scada-schema/blob/main/schema) for tools that are not written in Python.
89
+ 8. **Resampling** to a coarser interval with rules by signal kind (mean, vector mean of angles, sum, last meter reading, pooled standard deviation, coverage threshold): `scada-schema resample`.
90
+ 9. **Command line**: `scada-schema fetch | load | import | resample | validate | qc | convert | jsonschema | spec | study`.
91
+
92
+ ## Results
93
+
94
+ All numbers on this page come from `python -m scada_schema study`, which writes `results/study.json`; `tests/test_docs.py` renders this README from the
95
+ template and that file and fails when they differ.
96
+
97
+ 12 tables from 6 sources (3,870,948 rows, 59 devices) were loaded from the original files; the schema accepts 12 of 12.
98
+
99
+ | Dataset | Source (licence) | Profile | Interval | Rows | Devices | Signals | Schema | QC findings (error / warning / info) |
100
+ |---|---|---|---|---:|---:|---:|---|---|
101
+ | [kelmarsh_2019](https://kwwaleszczynski-del.github.io/scada-schema/qc/kelmarsh_2019.html) | Kelmarsh wind farm data (Cubico Sustainable Investments) (CC-BY-4.0) | wind | 10min | 315,360 | 6 | 23 (+39 statistics) | valid | 0 / 19 / 49 |
102
+ | [penmanshiel_2020](https://kwwaleszczynski-del.github.io/scada-schema/qc/penmanshiel_2020.html) | Penmanshiel wind farm data (Cubico Sustainable Investments) (CC-BY-4.0) | wind | 10min | 737,856 | 14 | 23 (+60 statistics) | valid | 0 / 359 / 107 |
103
+ | [hill_of_towie_2021](https://kwwaleszczynski-del.github.io/scada-schema/qc/hill_of_towie_2021.html) | Hill of Towie wind farm open dataset (RES) (CC-BY-4.0) | wind | 10min | 1,090,921 | 21 | 20 (+42 statistics) | valid | 2 / 115 / 155 |
104
+ | [la_haute_borne](https://kwwaleszczynski-del.github.io/scada-schema/qc/la_haute_borne.html) | La Haute Borne wind farm data (ENGIE, copy shipped with OpenOA) (Etalab open licence (as stated by ENGIE, not re-verified)) | wind | 10min | 420,432 | 4 | 7 (+0 statistics) | valid | 0 / 11 / 18 |
105
+ | [care_A](https://kwwaleszczynski-del.github.io/scada-schema/qc/care_A.html) | CARE to Compare (CC-BY-SA-4.0) | wind | 10min | 54,358 | 1 | 16 (+15 statistics) | valid | 0 / 3 / 6 |
106
+ | [care_B](https://kwwaleszczynski-del.github.io/scada-schema/qc/care_B.html) | CARE to Compare (CC-BY-SA-4.0) | wind | 10min | 56,564 | 1 | 15 (+39 statistics) | valid | 1 / 6 / 5 |
107
+ | [care_C](https://kwwaleszczynski-del.github.io/scada-schema/qc/care_C.html) | CARE to Compare (CC-BY-SA-4.0) | wind | 10min | 55,753 | 1 | 8 (+18 statistics) | valid | 0 / 3 / 1 |
108
+ | [pvdaq_34](https://kwwaleszczynski-del.github.io/scada-schema/qc/pvdaq_34.html) | NREL PVDAQ public datasets (OEDI) (CC-BY-4.0) | pv | 15min | 210,300 | 2 | 15 (+0 statistics) | valid | 2 / 7 / 7 |
109
+ | [pvdaq_1276](https://kwwaleszczynski-del.github.io/scada-schema/qc/pvdaq_1276.html) | NREL PVDAQ public datasets (OEDI) (CC-BY-4.0) | pv | 15min | 209,162 | 2 | 14 (+0 statistics) | valid | 2 / 11 / 5 |
110
+ | [pvdaq_1278](https://kwwaleszczynski-del.github.io/scada-schema/qc/pvdaq_1278.html) | NREL PVDAQ public datasets (OEDI) (CC-BY-4.0) | pv | 15min | 315,456 | 3 | 14 (+0 statistics) | valid | 3 / 12 / 10 |
111
+ | [pvdaq_35](https://kwwaleszczynski-del.github.io/scada-schema/qc/pvdaq_35.html) | NREL PVDAQ public datasets (OEDI) (CC-BY-4.0) | pv | 15min | 194,486 | 2 | 15 (+0 statistics) | valid | 2 / 13 / 7 |
112
+ | [pvdaq_34_raw_units](https://kwwaleszczynski-del.github.io/scada-schema/qc/pvdaq_34_raw_units.html) | NREL PVDAQ public datasets (OEDI) (CC-BY-4.0) | pv | 15min | 210,300 | 2 | 15 (+0 statistics) | valid | 4 / 7 / 7 |
113
+
114
+ Number of devices with at least one finding, by check (the quality reports list every finding with its longest episodes):
115
+
116
+ | Check | kelmarsh_2019 | penmanshiel_2020 | hill_of_towie_2021 | la_haute_borne | care_A | care_B | care_C | pvdaq_34 | pvdaq_1276 | pvdaq_1278 | pvdaq_35 | pvdaq_34_raw_units |
117
+ |---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|
118
+ | `duplicates` | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 |
119
+ | `ordering` | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 |
120
+ | `time_jumps` | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 |
121
+ | `gaps` | 0 | 0 | 21 | 4 | 1 | 1 | 0 | 2 | 2 | 3 | 2 | 2 |
122
+ | `dst` | 0 | 0 | 0 | 4 | skipped | skipped | skipped | skipped | skipped | skipped | skipped | skipped |
123
+ | `missing_values` | 6 | 14 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 |
124
+ | `out_of_range` | 5 | 14 | 0 | 3 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 |
125
+ | `flatline` | 1 | 13 | 21 | 1 | 1 | 1 | 1 | 2 | 2 | 3 | 2 | 2 |
126
+ | `spikes` | 6 | 14 | 21 | 1 | 1 | 1 | 1 | 2 | 2 | 3 | 2 | 2 |
127
+ | `power_curve` | 6 | 14 | 21 | 4 | 1 | 1 | 1 | skipped | skipped | skipped | skipped | skipped |
128
+ | `wind_direction` | 6 | 14 | 21 | 4 | 0 | 0 | skipped | 0 | 0 | 0 | 0 | 0 |
129
+ | `icing` | 1 | 8 | 21 | 4 | 0 | 1 | 0 | skipped | skipped | skipped | skipped | skipped |
130
+ | `energy_counter` | 2 | 0 | skipped | skipped | skipped | skipped | skipped | skipped | skipped | skipped | skipped | skipped |
131
+ | `energy_power` | 0 | 0 | skipped | skipped | skipped | skipped | skipped | skipped | skipped | skipped | skipped | skipped |
132
+ | `units` | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 |
133
+
134
+ What the checks found on real data:
135
+
136
+ - **A unit switch is found where the dataset documents it.** PVDAQ's power channels change from hectowatts to watts in August 2018. Read as hectowatts
137
+ throughout, system 34 shows a step of a factor of 98 in power over irradiance on 2018-08-05; the loader finds the same switch from the data
138
+ (`p_ac, p_dc: hectowatts to kW (x0.1) until 2018-08-04 20:00 UTC, watts to kW (x0.001) from then on`), and the check on the corrected table reports 0 switches.
139
+ - **North offsets.** On Hill of Towie the nacelle position of 12 of 21 turbines is more than 10 degrees off the fleet mean direction;
140
+ the largest, T16, is 84.7 degrees off. On Kelmarsh the largest offset is 3.8 degrees.
141
+ - **A real daylight saving artefact.** La Haute Borne stamps its data with the UTC offset, so the time is exact, but at both autumn changes (2014-10-26 and 2015-10-25) one hour is missing in UTC (00:00 to 01:00, the first occurrence of
142
+ the repeated local hour): the source holds only one of the two repeated hours. The `dst` check, run against Europe/Paris, finds 2 of 4 changes on every turbine.
143
+ - **Time elsewhere.** The three UK farms are checked against Europe/London (UTC time stamps, no artefact found); for the others the check does not apply (fixed offset or anonymised time).
144
+ The table above shows what each dataset triggered.
145
+
146
+ ### IEC 61400-26 mapping
147
+
148
+ A stop event labels an interval when the interval's midpoint lies in the event. On the Greenbyte farms the operator also labels each stop with an IEC category,
149
+ which tests the code tables:
150
+
151
+ | Dataset | Code table | Events | Events with a code in the table | Stop events | Compared with the operator's category | Agreement | Time-based availability, IEC operational / OEM style |
152
+ |---|---|---:|---:|---:|---:|---:|---|
153
+ | kelmarsh_2019 | `senvion_mm` | 59,313 | 100.0 % | 673 | 673 | 100.0 % | 99.5 % / 99.8 % |
154
+ | penmanshiel_2020 | `senvion_mm` | 158,509 | 100.0 % | 3,267 | 3,264 | 100.0 % | 97.8 % / 98.1 % |
155
+ | hill_of_towie_2021 | `siemens_swt_hot` | 798,406 | 94.9 % | 4,034 | - | no operator category | not computed |
156
+
157
+ On Kelmarsh the Senvion table agrees with the operator's category on 100.0 % of 673 compared stop events, on Penmanshiel on 100.0 % of 3,264. **This is a
158
+ consistency check, not an independent test:** the Senvion table was written from the same open logs. The Hill of Towie alarm log has no operator category; the dataset describes only
159
+ 12 alarm codes (83.6 % of its 798,406 alarm events), the rest of the Siemens table was inferred from SCADA in
160
+ [wind-om-reliability](https://github.com/kwwaleszczynski-del/wind-om-reliability) (column `source` of the table), and 94.9 % of the events have a code in the table. No availability is
161
+ computed for it. The availability figures of the Greenbyte farms are what the operator's categories imply under two illustrative treatments
162
+ (`iec_operational`: every operative state counts as available; `oem_warranty`: grid, environment and requested stops excluded, 60 h of maintenance excused per turbine and year); they are not a reading of any contract.
163
+
164
+ ### Converters: the same result through the library and by hand
165
+
166
+ | Target | Input | What runs on both inputs | Largest difference |
167
+ |---|---|---|---|
168
+ | wind-up 0.5 | Kelmarsh 2019, turbines T01 to T03, 157,680 rows | the SCADA frame itself | 1.2e-04 (float32 rounding) |
169
+ | | | `filter_scada_df`: same rows removed (30.2 % of power values) | 1.2e-04; missing pattern identical |
170
+ | | | `calc_pc_and_rated_ws_one_ttype` power curve | 6.5e-06 kW |
171
+ | OpenOA 3 | Kelmarsh 2019, turbines T01 to T03, 157,680 rows | `PlantData.scada` | 1.2e-04 |
172
+ | | | `power_curve.IEC` | 1.6e-02 kW |
173
+ | pvlib 0.16 | PVDAQ system 34, 2017, 35,040 intervals | PVWatts `ModelChain` (modelled plant up to 141 kW) | 8.3e-08 relative in daylight (6.7e-03 W) |
174
+
175
+ OpenOA's default power-curve bins start at 0 (centres 0.25, 0.75, ...) while the bins of IEC 61400-12-1 and res-asset-kpi are centred on multiples of 0.5; with the bins aligned
176
+ the two implementations agree to 0.0e+00 kW, with OpenOA's default bins the curves differ by up to 4.8 % of rated power. The converters do not hide that: it is
177
+ a property of the tool.
178
+
179
+ ## Install and use
180
+
181
+ Not on PyPI yet; install from GitHub:
182
+
183
+ ```bash
184
+ pip install "scada-schema @ git+https://github.com/kwwaleszczynski-del/scada-schema"
185
+ pip install "scada-schema[converters] @ git+https://github.com/kwwaleszczynski-del/scada-schema" # with wind-up, OpenOA and pvlib
186
+ ```
187
+
188
+ From a clone of the repository (development):
189
+
190
+ ```bash
191
+ pip install -e ".[converters]" # core: pip install -e .
192
+ ```
193
+
194
+ ```python
195
+ from scada_schema.loaders import kelmarsh
196
+ from scada_schema import qc, validate
197
+ from scada_schema.states import attach_states
198
+
199
+ loaded = kelmarsh.load("data/kelmarsh", years=[2019]) # fetch first: scada-schema fetch kelmarsh --years 2019
200
+ attach_states(loaded) # status_code and iec_category from the event log
201
+ assert validate(loaded.table, loaded.meta).ok
202
+ report = qc.run(loaded.table, loaded.meta)
203
+ report.to_html("kelmarsh_2019_qc.html")
204
+
205
+ from scada_schema.convert import windup, openoa
206
+ scada, notes = windup.to_windup_scada(loaded.table, loaded.meta)
207
+ plant = openoa.to_openoa(loaded.table, loaded.meta)
208
+ ```
209
+
210
+ ```bash
211
+ scada-schema load pvdaq --system 34 --years 2016 2017 2018 --out out/pvdaq_34.parquet
212
+ scada-schema qc out/pvdaq_34.parquet --html out/pvdaq_34.html
213
+ scada-schema convert out/pvdaq_34.parquet --to pvlib --out out/pvlib
214
+ ```
215
+
216
+ Raw files are looked for in the folder given with `--data-dir`, then in `$SCADA_SCHEMA_<NAME>_DIR`, `$SCADA_SCHEMA_DATA/<name>` and `./data/<name>`.
217
+
218
+ ## Your own data
219
+
220
+ A loader for a new export is a mapping file, not code (examples: [wind, stacked CSV](https://github.com/kwwaleszczynski-del/scada-schema/blob/main/docs/examples/wind_semicolon.yaml), [PV, wide CSV](https://github.com/kwwaleszczynski-del/scada-schema/blob/main/docs/examples/pv_wide.yaml)):
221
+
222
+ ```yaml
223
+ name: my_wind_farm
224
+ profile: wind
225
+ interval: 10min
226
+ files: ["export/*.csv"]
227
+ read: {sep: ";", decimal: ",", encoding: utf-8}
228
+ timestamp: {column: Time, timezone: Europe/Warsaw, label: end, format: "%d.%m.%Y %H:%M"}
229
+ device_column: Turbine
230
+ columns:
231
+ "Active power [W]": {signal: p_ac, unit: W}
232
+ "Wind [km/h]": {signal: wind_speed, unit: km/h}
233
+ "Ambient [F]": {signal: t_amb, unit: degF}
234
+ "Status": {signal: status_code}
235
+ devices:
236
+ T01: {type: turbine, rated_kw: 2050}
237
+ ```
238
+
239
+ ```bash
240
+ scada-schema import mapping.yaml --out my_farm.parquet
241
+ scada-schema qc my_farm.parquet --html my_farm_qc.html
242
+ ```
243
+
244
+ Time stamps are converted to UTC with the daylight saving rules of the zone (the repeated autumn hour is resolved by order), moved to the start of the interval and checked against the
245
+ grid; units are converted through the registry of res-asset-kpi; every conversion lands in the metadata. A wrong time zone, a missing column or a unit that does not fit the signal stops
246
+ the import with a message that names it. [`schema/mapping.schema.json`](https://github.com/kwwaleszczynski-del/scada-schema/blob/main/schema/mapping.schema.json) lets a YAML editor complete and check the mapping.
247
+
248
+ ## What is not verified
249
+
250
+ - The wind-up check uses wind-up 0.5.1 (beta; its v1 input is not fixed) and the filter stage with a minimal configuration; the analysis periods in `windup_config` are placeholders
251
+ and the pitch and generator speed ranges are estimated from the data. A full wind-up assessment was not run.
252
+ - The code tables are written from the public code texts of the open datasets; they are not OEM documents. IEC categories are the top levels of the information model, not the whole tree.
253
+ - The quality thresholds are defaults chosen to be conservative; they were tuned on these datasets and are not validated on others. Findings describe data, not the condition of a sensor.
254
+ - CARE's sensor names are anonymised; the sensor tables of farms A, B and C cover the signals listed in `loaders/care.py` and each entry is checked against the benchmark's own
255
+ description when loading. CARE time stamps are anonymised, so checks that need real time do not apply to it.
256
+ - Hill of Towie's SCADA tables used here have no absolute wind direction; only the nacelle position is mapped.
257
+ - The schema has been applied to open datasets only. Operator exports from other vendors need their own loader (a loader is a column mapping, a unit table and a time convention).
258
+ - `pse-opendata` is not used: the schema holds SCADA, and no PSE data are needed for it.
259
+
260
+ ## Reproduce
261
+
262
+ ```bash
263
+ scada-schema fetch kelmarsh --years 2019 && scada-schema fetch penmanshiel --years 2020 # and hill_of_towie, care, pvdaq
264
+ python -m scada_schema study --out results
265
+ python scripts/build_pages.py && python scripts/render_readme.py
266
+ pytest -q # tests with real data: set SCADA_SCHEMA_<NAME>_DIR
267
+ ```
268
+
269
+ Datasets and licences: La Haute Borne (ENGIE, Etalab open licence as stated by ENGIE, not re-verified; read from the copy shipped with OpenOA, BSD-3-Clause repository), Kelmarsh and Penmanshiel (Cubico Sustainable Investments, CC BY 4.0, doi 10.5281/zenodo.5841833 and 10.5281/zenodo.5946807), Hill of Towie (RES, CC BY 4.0,
270
+ doi 10.5281/zenodo.14870021), CARE to Compare (CC BY-SA 4.0, doi 10.5281/zenodo.15846963; the share-alike condition applies to the data and to datasets derived from it, not to this
271
+ package's code), NREL PVDAQ (CC BY 4.0, doi 10.25984/1846021). The code is MIT licensed.
272
+
273
+ ## Citation
274
+
275
+ Cite the software through [CITATION.cff](https://github.com/kwwaleszczynski-del/scada-schema/blob/main/CITATION.cff) (GitHub's "Cite this repository" reads it). A DOI is assigned by Zenodo when the first release is archived; until then cite the repository and the version.
@@ -0,0 +1,225 @@
1
+ # scada-schema: an open data standard for wind and PV SCADA
2
+
3
+ [![tests](https://github.com/kwwaleszczynski-del/scada-schema/actions/workflows/ci.yml/badge.svg)](https://github.com/kwwaleszczynski-del/scada-schema/actions/workflows/ci.yml) [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/kwwaleszczynski-del/scada-schema/blob/main/LICENSE)
4
+
5
+ **Project page with the results: [kwwaleszczynski-del.github.io/scada-schema](https://kwwaleszczynski-del.github.io/scada-schema/)**
6
+ - [specification](https://github.com/kwwaleszczynski-del/scada-schema/blob/main/SPEC.md) - [example quality report](https://kwwaleszczynski-del.github.io/scada-schema/qc/pvdaq_34_raw_units.html).
7
+
8
+ SCADA exports from two wind farms never look alike: columns named after a vendor's tag list, power in hectowatts or watts, time in local
9
+ time or at the end of the interval, a status code that only the turbine manual explains. Every analysis tool then starts with its own
10
+ cleaning script. This package fixes **one table layout** (names, units, sign and angle conventions, time zone and interval, device types) for
11
+ wind turbines and PV plants, reads **open datasets from 6 sources** into it from their original files, checks the data, maps status codes to
12
+ **IEC 61400-26 categories** and converts the table to the inputs of **wind-up, OpenOA and pvlib**, with a test that each tool gives the same result
13
+ on the library's input as on a hand-built one.
14
+
15
+ It is a proposal, not a standard of any body, and working material: the formulas and category names follow public descriptions, and nothing
16
+ here is a certified implementation of an IEC standard.
17
+
18
+ ## What is in it
19
+
20
+ 1. **Schema** ([SPEC.md](https://github.com/kwwaleszczynski-del/scada-schema/blob/main/SPEC.md), generated from the code): 40 canonical signals (31 in the wind profile, 23 in the PV profile),
21
+ one row per device and interval, `timestamp` in UTC at the start of the interval, canonical units, power positive into the grid, compass
22
+ angles clockwise from true north. Minimum, maximum and standard deviation columns where a source has them. The metadata (profile, interval, the
23
+ time zone and label of the *source*, devices with ratings and positions, licence, every conversion the loader made) are stored inside the
24
+ Parquet file. pydantic describes the metadata; the table is validated by a vectorised check that reports every violation with a count (pandera
25
+ stops at the first failure and adds a dependency, and a per-row quality report is needed anyway).
26
+ 2. **Loaders** for Kelmarsh, Penmanshiel, Hill of Towie, La Haute Borne, CARE to Compare and NREL PVDAQ. They read the original archives (Zenodo, OEDI) and
27
+ `fetch` downloads them with checksum verification; nothing large is in this repository. Hill of Towie's 1.4 GB yearly archives are read with range
28
+ requests, so only the needed tables are transferred.
29
+ 3. **Quality control** with an HTML report: 15 checks - duplicate and unordered time stamps, time jumps, gaps, daylight saving artefacts, missing values,
30
+ physical limits, flat lines, spikes, power-curve outliers, wind direction (vane, nacelle and north offsets between turbines), energy counters and interval energy,
31
+ icing, and units (order of magnitude and a switch of unit inside a record). The report is one file without scripts.
32
+ 4. **IEC 61400-26 states**: code tables for the Senvion MM series (268 codes) and the Siemens SWT 2.3 (146 codes) as logged in the open
33
+ datasets, label tables for Greenbyte and CARE, projection of an event log onto the intervals, and time-based availability through
34
+ [res-asset-kpi](https://github.com/kwwaleszczynski-del/res-asset-kpi) (which also supplies the unit, time-zone and category code used here).
35
+ 5. **Converters** to wind-up, OpenOA and pvlib and the parity checks below.
36
+ 6. **Your own data without code**: `scada-schema import mapping.yaml` turns a CSV, Excel or Parquet export into a canonical table from a YAML mapping (columns, units,
37
+ time zone and label, devices); see below.
38
+ 7. **JSON Schema** of the metadata and of the mapping file, and a list of every canonical column, in [`schema/`](https://github.com/kwwaleszczynski-del/scada-schema/blob/main/schema) for tools that are not written in Python.
39
+ 8. **Resampling** to a coarser interval with rules by signal kind (mean, vector mean of angles, sum, last meter reading, pooled standard deviation, coverage threshold): `scada-schema resample`.
40
+ 9. **Command line**: `scada-schema fetch | load | import | resample | validate | qc | convert | jsonschema | spec | study`.
41
+
42
+ ## Results
43
+
44
+ All numbers on this page come from `python -m scada_schema study`, which writes `results/study.json`; `tests/test_docs.py` renders this README from the
45
+ template and that file and fails when they differ.
46
+
47
+ 12 tables from 6 sources (3,870,948 rows, 59 devices) were loaded from the original files; the schema accepts 12 of 12.
48
+
49
+ | Dataset | Source (licence) | Profile | Interval | Rows | Devices | Signals | Schema | QC findings (error / warning / info) |
50
+ |---|---|---|---|---:|---:|---:|---|---|
51
+ | [kelmarsh_2019](https://kwwaleszczynski-del.github.io/scada-schema/qc/kelmarsh_2019.html) | Kelmarsh wind farm data (Cubico Sustainable Investments) (CC-BY-4.0) | wind | 10min | 315,360 | 6 | 23 (+39 statistics) | valid | 0 / 19 / 49 |
52
+ | [penmanshiel_2020](https://kwwaleszczynski-del.github.io/scada-schema/qc/penmanshiel_2020.html) | Penmanshiel wind farm data (Cubico Sustainable Investments) (CC-BY-4.0) | wind | 10min | 737,856 | 14 | 23 (+60 statistics) | valid | 0 / 359 / 107 |
53
+ | [hill_of_towie_2021](https://kwwaleszczynski-del.github.io/scada-schema/qc/hill_of_towie_2021.html) | Hill of Towie wind farm open dataset (RES) (CC-BY-4.0) | wind | 10min | 1,090,921 | 21 | 20 (+42 statistics) | valid | 2 / 115 / 155 |
54
+ | [la_haute_borne](https://kwwaleszczynski-del.github.io/scada-schema/qc/la_haute_borne.html) | La Haute Borne wind farm data (ENGIE, copy shipped with OpenOA) (Etalab open licence (as stated by ENGIE, not re-verified)) | wind | 10min | 420,432 | 4 | 7 (+0 statistics) | valid | 0 / 11 / 18 |
55
+ | [care_A](https://kwwaleszczynski-del.github.io/scada-schema/qc/care_A.html) | CARE to Compare (CC-BY-SA-4.0) | wind | 10min | 54,358 | 1 | 16 (+15 statistics) | valid | 0 / 3 / 6 |
56
+ | [care_B](https://kwwaleszczynski-del.github.io/scada-schema/qc/care_B.html) | CARE to Compare (CC-BY-SA-4.0) | wind | 10min | 56,564 | 1 | 15 (+39 statistics) | valid | 1 / 6 / 5 |
57
+ | [care_C](https://kwwaleszczynski-del.github.io/scada-schema/qc/care_C.html) | CARE to Compare (CC-BY-SA-4.0) | wind | 10min | 55,753 | 1 | 8 (+18 statistics) | valid | 0 / 3 / 1 |
58
+ | [pvdaq_34](https://kwwaleszczynski-del.github.io/scada-schema/qc/pvdaq_34.html) | NREL PVDAQ public datasets (OEDI) (CC-BY-4.0) | pv | 15min | 210,300 | 2 | 15 (+0 statistics) | valid | 2 / 7 / 7 |
59
+ | [pvdaq_1276](https://kwwaleszczynski-del.github.io/scada-schema/qc/pvdaq_1276.html) | NREL PVDAQ public datasets (OEDI) (CC-BY-4.0) | pv | 15min | 209,162 | 2 | 14 (+0 statistics) | valid | 2 / 11 / 5 |
60
+ | [pvdaq_1278](https://kwwaleszczynski-del.github.io/scada-schema/qc/pvdaq_1278.html) | NREL PVDAQ public datasets (OEDI) (CC-BY-4.0) | pv | 15min | 315,456 | 3 | 14 (+0 statistics) | valid | 3 / 12 / 10 |
61
+ | [pvdaq_35](https://kwwaleszczynski-del.github.io/scada-schema/qc/pvdaq_35.html) | NREL PVDAQ public datasets (OEDI) (CC-BY-4.0) | pv | 15min | 194,486 | 2 | 15 (+0 statistics) | valid | 2 / 13 / 7 |
62
+ | [pvdaq_34_raw_units](https://kwwaleszczynski-del.github.io/scada-schema/qc/pvdaq_34_raw_units.html) | NREL PVDAQ public datasets (OEDI) (CC-BY-4.0) | pv | 15min | 210,300 | 2 | 15 (+0 statistics) | valid | 4 / 7 / 7 |
63
+
64
+ Number of devices with at least one finding, by check (the quality reports list every finding with its longest episodes):
65
+
66
+ | Check | kelmarsh_2019 | penmanshiel_2020 | hill_of_towie_2021 | la_haute_borne | care_A | care_B | care_C | pvdaq_34 | pvdaq_1276 | pvdaq_1278 | pvdaq_35 | pvdaq_34_raw_units |
67
+ |---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|
68
+ | `duplicates` | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 |
69
+ | `ordering` | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 |
70
+ | `time_jumps` | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 |
71
+ | `gaps` | 0 | 0 | 21 | 4 | 1 | 1 | 0 | 2 | 2 | 3 | 2 | 2 |
72
+ | `dst` | 0 | 0 | 0 | 4 | skipped | skipped | skipped | skipped | skipped | skipped | skipped | skipped |
73
+ | `missing_values` | 6 | 14 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 |
74
+ | `out_of_range` | 5 | 14 | 0 | 3 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 |
75
+ | `flatline` | 1 | 13 | 21 | 1 | 1 | 1 | 1 | 2 | 2 | 3 | 2 | 2 |
76
+ | `spikes` | 6 | 14 | 21 | 1 | 1 | 1 | 1 | 2 | 2 | 3 | 2 | 2 |
77
+ | `power_curve` | 6 | 14 | 21 | 4 | 1 | 1 | 1 | skipped | skipped | skipped | skipped | skipped |
78
+ | `wind_direction` | 6 | 14 | 21 | 4 | 0 | 0 | skipped | 0 | 0 | 0 | 0 | 0 |
79
+ | `icing` | 1 | 8 | 21 | 4 | 0 | 1 | 0 | skipped | skipped | skipped | skipped | skipped |
80
+ | `energy_counter` | 2 | 0 | skipped | skipped | skipped | skipped | skipped | skipped | skipped | skipped | skipped | skipped |
81
+ | `energy_power` | 0 | 0 | skipped | skipped | skipped | skipped | skipped | skipped | skipped | skipped | skipped | skipped |
82
+ | `units` | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 |
83
+
84
+ What the checks found on real data:
85
+
86
+ - **A unit switch is found where the dataset documents it.** PVDAQ's power channels change from hectowatts to watts in August 2018. Read as hectowatts
87
+ throughout, system 34 shows a step of a factor of 98 in power over irradiance on 2018-08-05; the loader finds the same switch from the data
88
+ (`p_ac, p_dc: hectowatts to kW (x0.1) until 2018-08-04 20:00 UTC, watts to kW (x0.001) from then on`), and the check on the corrected table reports 0 switches.
89
+ - **North offsets.** On Hill of Towie the nacelle position of 12 of 21 turbines is more than 10 degrees off the fleet mean direction;
90
+ the largest, T16, is 84.7 degrees off. On Kelmarsh the largest offset is 3.8 degrees.
91
+ - **A real daylight saving artefact.** La Haute Borne stamps its data with the UTC offset, so the time is exact, but at both autumn changes (2014-10-26 and 2015-10-25) one hour is missing in UTC (00:00 to 01:00, the first occurrence of
92
+ the repeated local hour): the source holds only one of the two repeated hours. The `dst` check, run against Europe/Paris, finds 2 of 4 changes on every turbine.
93
+ - **Time elsewhere.** The three UK farms are checked against Europe/London (UTC time stamps, no artefact found); for the others the check does not apply (fixed offset or anonymised time).
94
+ The table above shows what each dataset triggered.
95
+
96
+ ### IEC 61400-26 mapping
97
+
98
+ A stop event labels an interval when the interval's midpoint lies in the event. On the Greenbyte farms the operator also labels each stop with an IEC category,
99
+ which tests the code tables:
100
+
101
+ | Dataset | Code table | Events | Events with a code in the table | Stop events | Compared with the operator's category | Agreement | Time-based availability, IEC operational / OEM style |
102
+ |---|---|---:|---:|---:|---:|---:|---|
103
+ | kelmarsh_2019 | `senvion_mm` | 59,313 | 100.0 % | 673 | 673 | 100.0 % | 99.5 % / 99.8 % |
104
+ | penmanshiel_2020 | `senvion_mm` | 158,509 | 100.0 % | 3,267 | 3,264 | 100.0 % | 97.8 % / 98.1 % |
105
+ | hill_of_towie_2021 | `siemens_swt_hot` | 798,406 | 94.9 % | 4,034 | - | no operator category | not computed |
106
+
107
+ On Kelmarsh the Senvion table agrees with the operator's category on 100.0 % of 673 compared stop events, on Penmanshiel on 100.0 % of 3,264. **This is a
108
+ consistency check, not an independent test:** the Senvion table was written from the same open logs. The Hill of Towie alarm log has no operator category; the dataset describes only
109
+ 12 alarm codes (83.6 % of its 798,406 alarm events), the rest of the Siemens table was inferred from SCADA in
110
+ [wind-om-reliability](https://github.com/kwwaleszczynski-del/wind-om-reliability) (column `source` of the table), and 94.9 % of the events have a code in the table. No availability is
111
+ computed for it. The availability figures of the Greenbyte farms are what the operator's categories imply under two illustrative treatments
112
+ (`iec_operational`: every operative state counts as available; `oem_warranty`: grid, environment and requested stops excluded, 60 h of maintenance excused per turbine and year); they are not a reading of any contract.
113
+
114
+ ### Converters: the same result through the library and by hand
115
+
116
+ | Target | Input | What runs on both inputs | Largest difference |
117
+ |---|---|---|---|
118
+ | wind-up 0.5 | Kelmarsh 2019, turbines T01 to T03, 157,680 rows | the SCADA frame itself | 1.2e-04 (float32 rounding) |
119
+ | | | `filter_scada_df`: same rows removed (30.2 % of power values) | 1.2e-04; missing pattern identical |
120
+ | | | `calc_pc_and_rated_ws_one_ttype` power curve | 6.5e-06 kW |
121
+ | OpenOA 3 | Kelmarsh 2019, turbines T01 to T03, 157,680 rows | `PlantData.scada` | 1.2e-04 |
122
+ | | | `power_curve.IEC` | 1.6e-02 kW |
123
+ | pvlib 0.16 | PVDAQ system 34, 2017, 35,040 intervals | PVWatts `ModelChain` (modelled plant up to 141 kW) | 8.3e-08 relative in daylight (6.7e-03 W) |
124
+
125
+ OpenOA's default power-curve bins start at 0 (centres 0.25, 0.75, ...) while the bins of IEC 61400-12-1 and res-asset-kpi are centred on multiples of 0.5; with the bins aligned
126
+ the two implementations agree to 0.0e+00 kW, with OpenOA's default bins the curves differ by up to 4.8 % of rated power. The converters do not hide that: it is
127
+ a property of the tool.
128
+
129
+ ## Install and use
130
+
131
+ Not on PyPI yet; install from GitHub:
132
+
133
+ ```bash
134
+ pip install "scada-schema @ git+https://github.com/kwwaleszczynski-del/scada-schema"
135
+ pip install "scada-schema[converters] @ git+https://github.com/kwwaleszczynski-del/scada-schema" # with wind-up, OpenOA and pvlib
136
+ ```
137
+
138
+ From a clone of the repository (development):
139
+
140
+ ```bash
141
+ pip install -e ".[converters]" # core: pip install -e .
142
+ ```
143
+
144
+ ```python
145
+ from scada_schema.loaders import kelmarsh
146
+ from scada_schema import qc, validate
147
+ from scada_schema.states import attach_states
148
+
149
+ loaded = kelmarsh.load("data/kelmarsh", years=[2019]) # fetch first: scada-schema fetch kelmarsh --years 2019
150
+ attach_states(loaded) # status_code and iec_category from the event log
151
+ assert validate(loaded.table, loaded.meta).ok
152
+ report = qc.run(loaded.table, loaded.meta)
153
+ report.to_html("kelmarsh_2019_qc.html")
154
+
155
+ from scada_schema.convert import windup, openoa
156
+ scada, notes = windup.to_windup_scada(loaded.table, loaded.meta)
157
+ plant = openoa.to_openoa(loaded.table, loaded.meta)
158
+ ```
159
+
160
+ ```bash
161
+ scada-schema load pvdaq --system 34 --years 2016 2017 2018 --out out/pvdaq_34.parquet
162
+ scada-schema qc out/pvdaq_34.parquet --html out/pvdaq_34.html
163
+ scada-schema convert out/pvdaq_34.parquet --to pvlib --out out/pvlib
164
+ ```
165
+
166
+ Raw files are looked for in the folder given with `--data-dir`, then in `$SCADA_SCHEMA_<NAME>_DIR`, `$SCADA_SCHEMA_DATA/<name>` and `./data/<name>`.
167
+
168
+ ## Your own data
169
+
170
+ A loader for a new export is a mapping file, not code (examples: [wind, stacked CSV](https://github.com/kwwaleszczynski-del/scada-schema/blob/main/docs/examples/wind_semicolon.yaml), [PV, wide CSV](https://github.com/kwwaleszczynski-del/scada-schema/blob/main/docs/examples/pv_wide.yaml)):
171
+
172
+ ```yaml
173
+ name: my_wind_farm
174
+ profile: wind
175
+ interval: 10min
176
+ files: ["export/*.csv"]
177
+ read: {sep: ";", decimal: ",", encoding: utf-8}
178
+ timestamp: {column: Time, timezone: Europe/Warsaw, label: end, format: "%d.%m.%Y %H:%M"}
179
+ device_column: Turbine
180
+ columns:
181
+ "Active power [W]": {signal: p_ac, unit: W}
182
+ "Wind [km/h]": {signal: wind_speed, unit: km/h}
183
+ "Ambient [F]": {signal: t_amb, unit: degF}
184
+ "Status": {signal: status_code}
185
+ devices:
186
+ T01: {type: turbine, rated_kw: 2050}
187
+ ```
188
+
189
+ ```bash
190
+ scada-schema import mapping.yaml --out my_farm.parquet
191
+ scada-schema qc my_farm.parquet --html my_farm_qc.html
192
+ ```
193
+
194
+ Time stamps are converted to UTC with the daylight saving rules of the zone (the repeated autumn hour is resolved by order), moved to the start of the interval and checked against the
195
+ grid; units are converted through the registry of res-asset-kpi; every conversion lands in the metadata. A wrong time zone, a missing column or a unit that does not fit the signal stops
196
+ the import with a message that names it. [`schema/mapping.schema.json`](https://github.com/kwwaleszczynski-del/scada-schema/blob/main/schema/mapping.schema.json) lets a YAML editor complete and check the mapping.
197
+
198
+ ## What is not verified
199
+
200
+ - The wind-up check uses wind-up 0.5.1 (beta; its v1 input is not fixed) and the filter stage with a minimal configuration; the analysis periods in `windup_config` are placeholders
201
+ and the pitch and generator speed ranges are estimated from the data. A full wind-up assessment was not run.
202
+ - The code tables are written from the public code texts of the open datasets; they are not OEM documents. IEC categories are the top levels of the information model, not the whole tree.
203
+ - The quality thresholds are defaults chosen to be conservative; they were tuned on these datasets and are not validated on others. Findings describe data, not the condition of a sensor.
204
+ - CARE's sensor names are anonymised; the sensor tables of farms A, B and C cover the signals listed in `loaders/care.py` and each entry is checked against the benchmark's own
205
+ description when loading. CARE time stamps are anonymised, so checks that need real time do not apply to it.
206
+ - Hill of Towie's SCADA tables used here have no absolute wind direction; only the nacelle position is mapped.
207
+ - The schema has been applied to open datasets only. Operator exports from other vendors need their own loader (a loader is a column mapping, a unit table and a time convention).
208
+ - `pse-opendata` is not used: the schema holds SCADA, and no PSE data are needed for it.
209
+
210
+ ## Reproduce
211
+
212
+ ```bash
213
+ scada-schema fetch kelmarsh --years 2019 && scada-schema fetch penmanshiel --years 2020 # and hill_of_towie, care, pvdaq
214
+ python -m scada_schema study --out results
215
+ python scripts/build_pages.py && python scripts/render_readme.py
216
+ pytest -q # tests with real data: set SCADA_SCHEMA_<NAME>_DIR
217
+ ```
218
+
219
+ Datasets and licences: La Haute Borne (ENGIE, Etalab open licence as stated by ENGIE, not re-verified; read from the copy shipped with OpenOA, BSD-3-Clause repository), Kelmarsh and Penmanshiel (Cubico Sustainable Investments, CC BY 4.0, doi 10.5281/zenodo.5841833 and 10.5281/zenodo.5946807), Hill of Towie (RES, CC BY 4.0,
220
+ doi 10.5281/zenodo.14870021), CARE to Compare (CC BY-SA 4.0, doi 10.5281/zenodo.15846963; the share-alike condition applies to the data and to datasets derived from it, not to this
221
+ package's code), NREL PVDAQ (CC BY 4.0, doi 10.25984/1846021). The code is MIT licensed.
222
+
223
+ ## Citation
224
+
225
+ Cite the software through [CITATION.cff](https://github.com/kwwaleszczynski-del/scada-schema/blob/main/CITATION.cff) (GitHub's "Cite this repository" reads it). A DOI is assigned by Zenodo when the first release is archived; until then cite the repository and the version.