scada-schema 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. scada_schema/__init__.py +31 -0
  2. scada_schema/__main__.py +3 -0
  3. scada_schema/_steps.py +50 -0
  4. scada_schema/_time.py +12 -0
  5. scada_schema/_version.py +1 -0
  6. scada_schema/cli.py +295 -0
  7. scada_schema/convert/__init__.py +8 -0
  8. scada_schema/convert/openoa.py +80 -0
  9. scada_schema/convert/pvlib.py +98 -0
  10. scada_schema/convert/windup.py +175 -0
  11. scada_schema/data/codes_senvion_mm.csv +269 -0
  12. scada_schema/data/codes_siemens_swt_hot.csv +147 -0
  13. scada_schema/data/iec_labels.csv +17 -0
  14. scada_schema/io.py +101 -0
  15. scada_schema/jsonschema.py +57 -0
  16. scada_schema/loaders/__init__.py +24 -0
  17. scada_schema/loaders/_common.py +112 -0
  18. scada_schema/loaders/_paths.py +21 -0
  19. scada_schema/loaders/_remotezip.py +67 -0
  20. scada_schema/loaders/care.py +167 -0
  21. scada_schema/loaders/generic.py +315 -0
  22. scada_schema/loaders/greenbyte.py +202 -0
  23. scada_schema/loaders/hill_of_towie.py +200 -0
  24. scada_schema/loaders/kelmarsh.py +53 -0
  25. scada_schema/loaders/la_haute_borne.py +81 -0
  26. scada_schema/loaders/penmanshiel.py +48 -0
  27. scada_schema/loaders/pvdaq.py +208 -0
  28. scada_schema/model.py +174 -0
  29. scada_schema/parity.py +181 -0
  30. scada_schema/py.typed +0 -0
  31. scada_schema/qc/__init__.py +14 -0
  32. scada_schema/qc/checks.py +580 -0
  33. scada_schema/qc/core.py +308 -0
  34. scada_schema/qc/report.py +167 -0
  35. scada_schema/resample.py +122 -0
  36. scada_schema/signals.py +169 -0
  37. scada_schema/spec.py +150 -0
  38. scada_schema/states.py +221 -0
  39. scada_schema/study.py +274 -0
  40. scada_schema/validate.py +207 -0
  41. scada_schema-0.1.0.dist-info/METADATA +275 -0
  42. scada_schema-0.1.0.dist-info/RECORD +46 -0
  43. scada_schema-0.1.0.dist-info/WHEEL +5 -0
  44. scada_schema-0.1.0.dist-info/entry_points.txt +2 -0
  45. scada_schema-0.1.0.dist-info/licenses/LICENSE +21 -0
  46. scada_schema-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,31 @@
1
+ """An open data standard for wind and PV SCADA.
2
+
3
+ * [`signals`][scada_schema.signals] - the registry of canonical signals (names, units, conventions, limits);
4
+ * [`model`][scada_schema.model] - dataset metadata and profiles (wind, PV);
5
+ * [`validate`][scada_schema.validate] - is a table in the schema?
6
+ * [`io`][scada_schema.io] - self-describing Parquet;
7
+ * [`loaders`][scada_schema.loaders] - Kelmarsh, Penmanshiel, Hill of Towie, La Haute Borne, CARE and PVDAQ to the schema;
8
+ * [`qc`][scada_schema.qc] - data quality checks and an HTML report;
9
+ * [`states`][scada_schema.states] - status codes to IEC 61400-26 categories;
10
+ * [`convert`][scada_schema.convert] - wind-up, OpenOA and pvlib.
11
+ """
12
+
13
+ from scada_schema._version import __version__
14
+ from scada_schema.io import finalise, read_meta, read_parquet, write_parquet
15
+ from scada_schema.model import DatasetMeta, Device, Source
16
+ from scada_schema.signals import SIGNALS
17
+ from scada_schema.validate import SchemaError, validate
18
+
19
+ __all__ = [
20
+ "SIGNALS",
21
+ "DatasetMeta",
22
+ "Device",
23
+ "SchemaError",
24
+ "Source",
25
+ "__version__",
26
+ "finalise",
27
+ "read_meta",
28
+ "read_parquet",
29
+ "validate",
30
+ "write_parquet",
31
+ ]
@@ -0,0 +1,3 @@
1
+ from scada_schema.cli import main
2
+
3
+ raise SystemExit(main())
scada_schema/_steps.py ADDED
@@ -0,0 +1,50 @@
1
+ """Finding a step in the level of a series: the common core of the unit-switch check of the quality control and of the
2
+ loader that corrects PVDAQ's hectowatt-to-watt change."""
3
+
4
+ from __future__ import annotations
5
+
6
+ import numpy as np
7
+ import pandas as pd
8
+
9
+ __all__ = ["daily_log_ratio", "level_step"]
10
+
11
+
12
+ def daily_log_ratio(power: pd.Series, reference: pd.Series, min_reference: float = 200.0, min_points: int = 8) -> pd.Series | None:
13
+ """log10 of the daily median of ``power / reference`` where the reference (irradiance) exceeds ``min_reference``.
14
+
15
+ Both series are indexed by time stamp; days with fewer than ``min_points`` such intervals are dropped. ``None`` when fewer
16
+ than 14 days remain.
17
+ """
18
+ ref = reference.reindex(power.index)
19
+ r = (power.where(power > 0) / ref.where(ref > min_reference)).dropna()
20
+ if r.empty:
21
+ return None
22
+ g = r.groupby(r.index.floor("D"))
23
+ d, n = g.median(), g.size()
24
+ d = d[(n >= min_points) & (d > 0)]
25
+ return np.log10(d) if len(d) >= 14 else None
26
+
27
+
28
+ def level_step(levels: pd.Series, window: int = 7, min_decades: float = 0.8) -> tuple[pd.Timestamp, float] | None:
29
+ """The largest step of at least ``min_decades`` (log10) in a daily series, as ``(first day at the new level, step)``.
30
+
31
+ The step is the difference between the median of the next ``window`` days and of the previous ``window`` days; the median
32
+ windows smear a step over +-``window`` days, so the change is put at the first day past the midpoint level.
33
+ """
34
+ vals = levels.to_numpy(dtype=float)
35
+ n = len(vals)
36
+ if n < 2 * window + 1:
37
+ return None
38
+ jump = np.full(n, np.nan)
39
+ for i in range(window, n - window):
40
+ jump[i] = np.median(vals[i:i + window]) - np.median(vals[i - window:i])
41
+ if not np.isfinite(jump).any():
42
+ return None
43
+ i = int(np.nanargmax(np.abs(jump)))
44
+ step = float(jump[i])
45
+ if abs(step) < min_decades:
46
+ return None
47
+ mid = (np.median(vals[max(i - window, 0):i]) + np.median(vals[i:i + window])) / 2
48
+ span = np.arange(max(i - window, 0), min(i + window, n))
49
+ cross = span[(vals[span] > mid) if step > 0 else (vals[span] < mid)]
50
+ return levels.index[int(cross[0]) if len(cross) else i], step
scada_schema/_time.py ADDED
@@ -0,0 +1,12 @@
1
+ """Time helpers: pandas 3 no longer defaults to nanosecond resolution, the code below does arithmetic on nanoseconds."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import numpy as np
6
+ import pandas as pd
7
+
8
+
9
+ def to_ns(s: pd.Series | pd.DatetimeIndex) -> np.ndarray:
10
+ """Time stamps as int64 nanoseconds since the epoch, whatever the resolution of the datetime column."""
11
+ s = pd.Series(s) if isinstance(s, pd.DatetimeIndex) else s
12
+ return s.dt.as_unit("ns").astype("int64").to_numpy()
@@ -0,0 +1 @@
1
+ __version__ = "0.1.0"
scada_schema/cli.py ADDED
@@ -0,0 +1,295 @@
1
+ """Command line interface of scada-schema.
2
+
3
+ Usage:
4
+ scada-schema signals --profile wind
5
+ scada-schema fetch kelmarsh --years 2019 --data-dir data/kelmarsh
6
+ scada-schema load kelmarsh --years 2019 --data-dir data/kelmarsh --states --out out/kelmarsh_2019.parquet
7
+ scada-schema validate out/kelmarsh_2019.parquet
8
+ scada-schema qc out/kelmarsh_2019.parquet --html out/qc.html --dst-zone Europe/London
9
+ scada-schema convert out/kelmarsh_2019.parquet --to windup --out out/windup
10
+ scada-schema resample plant.parquet --to 1h --out plant_1h.parquet
11
+ scada-schema import mapping.yaml --out my_farm.parquet
12
+ scada-schema jsonschema --out schema
13
+ scada-schema spec > SPEC.md
14
+ scada-schema study --out results
15
+ """
16
+
17
+ from __future__ import annotations
18
+
19
+ import argparse
20
+ import json
21
+ import logging
22
+ import sys
23
+ from pathlib import Path
24
+
25
+ import pandas as pd
26
+
27
+ from scada_schema import __version__
28
+ from scada_schema.io import read_parquet, write_parquet
29
+ from scada_schema.loaders import DATASETS
30
+ from scada_schema.signals import PROFILE_SIGNALS, SIGNALS
31
+ from scada_schema.validate import validate
32
+
33
+ log = logging.getLogger("scada_schema")
34
+
35
+
36
+ def _signals(args) -> int:
37
+ rows = [s for n, s in SIGNALS.items() if args.profile in (None, *s.profiles)]
38
+ df = pd.DataFrame([{"signal": s.name, "unit": s.unit, "kind": s.kind, "profiles": "+".join(s.profiles), "devices": ",".join(s.devices),
39
+ "lo": s.lo, "hi": s.hi, "stats": s.stats, "description": s.description} for s in rows])
40
+ if args.format == "csv":
41
+ print(df.to_csv(index=False))
42
+ elif args.format == "json":
43
+ print(df.to_json(orient="records", indent=1))
44
+ else:
45
+ print(df.to_string(index=False, max_colwidth=70))
46
+ log.info("%d signals (wind %d, pv %d)", len(df), len(PROFILE_SIGNALS["wind"]), len(PROFILE_SIGNALS["pv"]))
47
+ return 0
48
+
49
+
50
+ def _loader_kwargs(args) -> dict:
51
+ mod = DATASETS[args.dataset]
52
+ kw: dict = {}
53
+ if args.dataset == "pvdaq":
54
+ kw["system"] = args.system
55
+ if args.no_fix_units:
56
+ kw["fix_units"] = False
57
+ elif args.dataset == "care":
58
+ kw.update(farm=args.farm, event_id=args.event)
59
+ if args.years and args.dataset not in ("care", "la_haute_borne"):
60
+ kw["years"] = args.years
61
+ del mod
62
+ return kw
63
+
64
+
65
+ def _fetch(args) -> int:
66
+ mod = DATASETS[args.dataset]
67
+ if args.dataset == "pvdaq":
68
+ n = mod.fetch(args.data_dir, system=args.system, years=args.years or [2016])
69
+ log.info("fetched %d daily files", n)
70
+ elif args.dataset == "care":
71
+ p = mod.fetch(args.data_dir, datasets={args.farm: [args.event]})
72
+ log.info("wrote %s", p)
73
+ elif args.dataset == "la_haute_borne":
74
+ for p in mod.fetch(args.data_dir):
75
+ log.info("ready %s", p)
76
+ else:
77
+ for p in mod.fetch(args.data_dir, years=args.years):
78
+ log.info("ready %s", p)
79
+ return 0
80
+
81
+
82
+ def _load(args) -> int:
83
+ mod = DATASETS[args.dataset]
84
+ loaded = mod.load(args.data_dir, **_loader_kwargs(args))
85
+ if args.states:
86
+ from scada_schema.states import attach_states
87
+
88
+ if len(loaded.events):
89
+ attach_states(loaded)
90
+ else:
91
+ log.warning("no event log: --states ignored")
92
+ rep = validate(loaded.table, loaded.meta)
93
+ for i in rep.issues:
94
+ log.log(logging.ERROR if i.level == "error" else logging.WARNING, "%s: %s", i.code, i.message)
95
+ out = write_parquet(loaded.table, loaded.meta, args.out)
96
+ if len(loaded.events):
97
+ loaded.events.to_parquet(Path(args.out).with_suffix(".events.parquet"))
98
+ log.info("wrote %s: %d rows, %d devices, %d events", out, len(loaded.table), loaded.table["device_id"].nunique(), len(loaded.events))
99
+ return 0
100
+
101
+
102
+ def _validate(args) -> int:
103
+ df, meta = read_parquet(args.file)
104
+ rep = validate(df, meta)
105
+ print(rep.to_frame().to_string(index=False) if rep.issues else "no issues")
106
+ print(f"{len(df):,} rows, {df['device_id'].nunique()} devices, profile {meta.profile}, interval {meta.interval}: "
107
+ + ("valid" if rep.ok else f"{len(rep.errors)} error(s)"))
108
+ return 0 if rep.ok else 1
109
+
110
+
111
+ def _qc(args) -> int:
112
+ from scada_schema import qc
113
+
114
+ df, meta = read_parquet(args.file)
115
+ cfg = qc.QCConfig(dst_zone=args.dst_zone) if args.dst_zone else None
116
+ report = qc.run(df, meta, cfg, checks=args.checks)
117
+ s = report.summary()
118
+ print(s.to_string(index=False) if len(s) else "no findings")
119
+ print("severity counts:", report.counts(), "| skipped:", report.checks_skipped or "none")
120
+ if args.html:
121
+ report.to_html(args.html)
122
+ log.info("wrote %s", args.html)
123
+ if args.json:
124
+ report.to_json(args.json)
125
+ log.info("wrote %s", args.json)
126
+ return 1 if args.fail_on_error and report.counts()["error"] else 0
127
+
128
+
129
+ def _convert(args) -> int:
130
+ df, meta = read_parquet(args.file)
131
+ out = Path(args.out)
132
+ out.mkdir(parents=True, exist_ok=True)
133
+ if args.to == "windup":
134
+ from scada_schema.convert import windup
135
+
136
+ scada, notes = windup.to_windup_scada(df, meta)
137
+ scada.to_parquet(out / "scada.parquet")
138
+ spec = {"notes": notes}
139
+ try:
140
+ spec["turbine_type"] = windup.windup_turbine_types(df, meta, args.turbine_type)
141
+ spec["turbines"] = windup.windup_turbines(meta, spec["turbine_type"])
142
+ except ValueError as e:
143
+ spec["turbine_type_error"] = str(e)
144
+ log.warning("%s", e)
145
+ (out / "windup.json").write_text(json.dumps(spec, indent=1), encoding="utf-8")
146
+ elif args.to == "openoa":
147
+ from scada_schema.convert import openoa
148
+
149
+ scada, asset = openoa.to_openoa_frames(df, meta)
150
+ scada.to_parquet(out / "scada.parquet")
151
+ asset.to_parquet(out / "asset.parquet")
152
+ else:
153
+ from scada_schema.convert import pvlib
154
+
155
+ inp = pvlib.to_pvlib(df, meta, inverter=args.inverter, at=args.at)
156
+ inp.weather.to_parquet(out / "weather.parquet")
157
+ inp.p_ac_kw.to_frame().to_parquet(out / "p_ac_kw.parquet")
158
+ (out / "pvlib.json").write_text(json.dumps({"location": inp.location, "system": inp.system, "notes": inp.notes}, indent=1), encoding="utf-8")
159
+ log.info("wrote %s", out)
160
+ return 0
161
+
162
+
163
+ def _resample(args) -> int:
164
+ from scada_schema.resample import resample
165
+
166
+ df, meta = read_parquet(args.file)
167
+ out, m = resample(df, meta, args.to, args.min_coverage)
168
+ write_parquet(out, m, args.out)
169
+ log.info("wrote %s: %d rows at %s (from %d at %s)", args.out, len(out), args.to, len(df), meta.interval)
170
+ return 0
171
+
172
+
173
+ def _import(args) -> int:
174
+ from scada_schema.loaders.generic import import_table, load_spec
175
+
176
+ loaded = import_table(load_spec(args.mapping), validate_result=not args.no_validate)
177
+ out = write_parquet(loaded.table, loaded.meta, args.out, check=not args.no_validate)
178
+ log.info("wrote %s: %d rows, %d devices", out, len(loaded.table), loaded.table["device_id"].nunique())
179
+ return 0
180
+
181
+
182
+ def _jsonschema(args) -> int:
183
+ from scada_schema import jsonschema
184
+
185
+ for p in jsonschema.write(args.out):
186
+ log.info("wrote %s", p)
187
+ return 0
188
+
189
+
190
+ def _spec(args) -> int:
191
+ from scada_schema.spec import render_spec
192
+
193
+ sys.stdout.write(render_spec())
194
+ return 0
195
+
196
+
197
+ def _study(args) -> int:
198
+ from scada_schema import study
199
+
200
+ study.run(Path(args.out), only=args.only, skip_converters=args.skip_converters)
201
+ return 0
202
+
203
+
204
+ def build_parser() -> argparse.ArgumentParser:
205
+ p = argparse.ArgumentParser(prog="scada-schema", description="Open data standard for wind and PV SCADA.")
206
+ p.add_argument("--version", action="version", version=f"scada-schema {__version__}")
207
+ p.add_argument("-v", "--verbose", action="store_true", help="debug logging")
208
+ sub = p.add_subparsers(dest="command", required=True)
209
+
210
+ s = sub.add_parser("signals", help="list the canonical signals")
211
+ s.add_argument("--profile", choices=["wind", "pv"])
212
+ s.add_argument("--format", choices=["table", "csv", "json"], default="table")
213
+ s.set_defaults(func=_signals)
214
+
215
+ def dataset_args(sp):
216
+ sp.add_argument("dataset", choices=sorted(DATASETS))
217
+ sp.add_argument("--data-dir", help="folder with the raw files (default: $SCADA_SCHEMA_<NAME>_DIR, $SCADA_SCHEMA_DATA/<name>, ./data/<name>)")
218
+ sp.add_argument("--years", type=int, nargs="+")
219
+ sp.add_argument("--system", type=int, default=34, choices=[34, 1276, 1278, 35], help="PVDAQ system")
220
+ sp.add_argument("--farm", default="A", choices=["A", "B", "C"], help="CARE wind farm")
221
+ sp.add_argument("--event", type=int, default=0, help="CARE event (dataset) id")
222
+
223
+ s = sub.add_parser("fetch", help="download an open dataset from its original source (checksums verified)")
224
+ dataset_args(s)
225
+ s.set_defaults(func=_fetch)
226
+
227
+ s = sub.add_parser("load", help="read an open dataset into a canonical Parquet file")
228
+ dataset_args(s)
229
+ s.add_argument("--out", required=True)
230
+ s.add_argument("--states", action="store_true", help="add status_code and iec_category from the event log")
231
+ s.add_argument("--no-fix-units", action="store_true", help="PVDAQ: keep the hectowatt reading of power (for demonstrating the unit check)")
232
+ s.set_defaults(func=_load)
233
+
234
+ s = sub.add_parser("validate", help="check a canonical Parquet file against the schema")
235
+ s.add_argument("file")
236
+ s.set_defaults(func=_validate)
237
+
238
+ s = sub.add_parser("qc", help="run the quality checks")
239
+ s.add_argument("file")
240
+ s.add_argument("--html")
241
+ s.add_argument("--json")
242
+ s.add_argument("--checks", nargs="+")
243
+ s.add_argument("--dst-zone", help="IANA zone whose daylight saving changes are looked for, e.g. Europe/Warsaw")
244
+ s.add_argument("--fail-on-error", action="store_true", help="exit with 1 when any finding has severity error")
245
+ s.set_defaults(func=_qc)
246
+
247
+ s = sub.add_parser("convert", help="convert a canonical file to the input of wind-up, OpenOA or pvlib")
248
+ s.add_argument("file")
249
+ s.add_argument("--to", required=True, choices=["windup", "openoa", "pvlib"])
250
+ s.add_argument("--out", required=True, help="output folder")
251
+ s.add_argument("--turbine-type", default="turbine", help="wind-up turbine type name")
252
+ s.add_argument("--inverter", help="pvlib: inverter id")
253
+ s.add_argument("--at", default="center", choices=["start", "center", "end"], help="pvlib: instant of the interval that indexes the weather")
254
+ s.set_defaults(func=_convert)
255
+
256
+ s = sub.add_parser("resample", help="move a canonical file to a coarser interval (rules by signal kind)")
257
+ s.add_argument("file")
258
+ s.add_argument("--to", required=True, choices=["10min", "15min", "30min", "1h"])
259
+ s.add_argument("--min-coverage", type=float, default=0.8, help="share of source intervals needed to fill a target interval")
260
+ s.add_argument("--out", required=True)
261
+ s.set_defaults(func=_resample)
262
+
263
+ s = sub.add_parser("import", help="turn your own export into a canonical Parquet file with a YAML mapping")
264
+ s.add_argument("mapping")
265
+ s.add_argument("--out", required=True)
266
+ s.add_argument("--no-validate", action="store_true")
267
+ s.set_defaults(func=_import)
268
+
269
+ s = sub.add_parser("jsonschema", help="write JSON Schema files and the column list of the standard")
270
+ s.add_argument("--out", default="schema")
271
+ s.set_defaults(func=_jsonschema)
272
+
273
+ s = sub.add_parser("spec", help="print the specification (SPEC.md) generated from the code")
274
+ s.set_defaults(func=_spec)
275
+
276
+ s = sub.add_parser("study", help="run the schema, QC, mapping and parity study on the open datasets and write results/")
277
+ s.add_argument("--out", default="results")
278
+ s.add_argument("--only", nargs="+", help="names of the study datasets to run")
279
+ s.add_argument("--skip-converters", action="store_true")
280
+ s.set_defaults(func=_study)
281
+ return p
282
+
283
+
284
+ def main(argv: list[str] | None = None) -> int:
285
+ args = build_parser().parse_args(argv)
286
+ logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO, format="%(levelname)s %(message)s", stream=sys.stderr)
287
+ try:
288
+ return int(args.func(args))
289
+ except (FileNotFoundError, ValueError, KeyError) as e:
290
+ log.error("%s", e)
291
+ return 2
292
+
293
+
294
+ if __name__ == "__main__":
295
+ raise SystemExit(main())
@@ -0,0 +1,8 @@
1
+ """Converters from the canonical table to the inputs of wind-up, OpenOA and pvlib.
2
+
3
+ Each module imports its target lazily, so the package works without them; install ``scada-schema[converters]``.
4
+ """
5
+
6
+ from scada_schema.convert import openoa, pvlib, windup
7
+
8
+ __all__ = ["openoa", "pvlib", "windup"]
@@ -0,0 +1,80 @@
1
+ """Canonical wind table to an [OpenOA](https://github.com/NREL/OpenOA) ``PlantData`` (v3).
2
+
3
+ OpenOA's SCADA schema names the columns ``time``, ``asset_id``, ``WTUR_W`` (kW), ``WMET_HorWdSpd`` (m/s),
4
+ ``WMET_HorWdDir`` (deg), ``WMET_HorWdDirRel`` (deg), ``WTUR_TurSt`` (status, text), ``WROT_BlPthAngVal`` (deg) and
5
+ ``WMET_EnvTmp`` (degC); the asset table has one row per turbine (``asset_id``, ``latitude``, ``longitude``,
6
+ ``rated_power`` in kW, ``hub_height``, ``rotor_diameter``, ``elevation``, ``type``).
7
+
8
+ ============================ ========================== ==============================================
9
+ OpenOA column canonical signal note
10
+ ============================ ========================== ==============================================
11
+ ``WTUR_W`` ``p_ac`` kW
12
+ ``WMET_HorWdSpd`` ``wind_speed`` m/s
13
+ ``WMET_HorWdDir`` ``wind_dir`` absolute, from; absent if the source has none
14
+ ``WMET_HorWdDirRel`` ``wind_dir_rel``
15
+ ``WTUR_TurSt`` ``status_code`` text
16
+ ``WROT_BlPthAngVal`` ``pitch`` deg
17
+ ``WMET_EnvTmp`` ``t_amb`` degC
18
+ ============================ ========================== ==============================================
19
+
20
+ The ``time`` column is the canonical time stamp as naive UTC (OpenOA's schema does not state which instant of the interval
21
+ it marks; the canonical table marks the start). Elevation is not part of the schema and is left empty unless given.
22
+
23
+ Usage:
24
+ plant = to_openoa(df, meta) # needs openoa
25
+ plant.scada, plant.asset
26
+ """
27
+
28
+ from __future__ import annotations
29
+
30
+ import numpy as np
31
+ import pandas as pd
32
+
33
+ from scada_schema.model import DatasetMeta
34
+
35
+ __all__ = ["COLUMN_MAP", "to_openoa", "to_openoa_frames"]
36
+
37
+ COLUMN_MAP = {
38
+ "p_ac": "WTUR_W", "wind_speed": "WMET_HorWdSpd", "wind_dir": "WMET_HorWdDir", "wind_dir_rel": "WMET_HorWdDirRel",
39
+ "status_code": "WTUR_TurSt", "pitch": "WROT_BlPthAngVal", "t_amb": "WMET_EnvTmp",
40
+ }
41
+
42
+
43
+ def to_openoa_frames(df: pd.DataFrame, meta: DatasetMeta, turbines: list[str] | None = None,
44
+ elevation_m: float | None = None) -> tuple[pd.DataFrame, pd.DataFrame]:
45
+ """The SCADA and asset frames of OpenOA, without needing OpenOA installed."""
46
+ if meta.profile != "wind":
47
+ raise ValueError("OpenOA's PlantData needs a wind table")
48
+ for need in ("p_ac", "wind_speed"):
49
+ if need not in df.columns or not df[need].notna().any():
50
+ raise ValueError(f"OpenOA's SCADA schema needs {need}")
51
+ devs = [d for d in meta.devices if d.type == "turbine" and (turbines is None or d.id in turbines)]
52
+ ids = [d.id for d in devs]
53
+ missing = [d.id for d in devs if d.lat is None or d.lon is None or d.rated_kw is None]
54
+ if missing:
55
+ raise ValueError(f"OpenOA's asset table needs latitude, longitude and rated power; missing for {missing[:5]}")
56
+ sub = df[df["device_id"].isin(ids)]
57
+ scada = pd.DataFrame({"time": sub["timestamp"].dt.tz_convert("UTC").dt.tz_localize(None).to_numpy(), "asset_id": sub["device_id"].to_numpy()})
58
+ for src, dst in COLUMN_MAP.items():
59
+ if src in sub.columns and sub[src].notna().any():
60
+ scada[dst] = sub[src].to_numpy(dtype="float64", na_value=np.nan) if src != "status_code" else sub[src].astype(object).to_numpy()
61
+ if "WTUR_TurSt" in scada:
62
+ scada["WTUR_TurSt"] = scada["WTUR_TurSt"].astype(object).where(scada["WTUR_TurSt"].notna(), None)
63
+ asset = pd.DataFrame({
64
+ "asset_id": ids, "latitude": [d.lat for d in devs], "longitude": [d.lon for d in devs],
65
+ "rated_power": [d.rated_kw for d in devs], "hub_height": [d.hub_height_m if d.hub_height_m else np.nan for d in devs],
66
+ "rotor_diameter": [d.rotor_diameter_m if d.rotor_diameter_m else np.nan for d in devs],
67
+ "elevation": [elevation_m if elevation_m is not None else np.nan] * len(devs), "type": "turbine",
68
+ })
69
+ return scada, asset
70
+
71
+
72
+ def to_openoa(df: pd.DataFrame, meta: DatasetMeta, turbines: list[str] | None = None, elevation_m: float | None = None,
73
+ analysis_type: list[str] | None = None):
74
+ """An OpenOA ``PlantData`` from a canonical wind table (needs the ``openoa`` extra)."""
75
+ from openoa import PlantData
76
+
77
+ scada, asset = to_openoa_frames(df, meta, turbines, elevation_m)
78
+ md = {"latitude": float(asset["latitude"].mean()), "longitude": float(asset["longitude"].mean()),
79
+ "capacity": float(asset["rated_power"].sum() / 1000.0), "scada": {"frequency": meta.interval}, "asset": {}}
80
+ return PlantData(analysis_type=analysis_type, metadata=md, scada=scada, asset=asset)
@@ -0,0 +1,98 @@
1
+ """Canonical PV table to [pvlib](https://pvlib-python.readthedocs.io) inputs: weather, location, array geometry, measured power.
2
+
3
+ pvlib works with instants and its own column names. The converter
4
+
5
+ * takes the weather from the met device (``met`` or ``plant``) of the table: ``ghi``, ``dni``, ``dhi``, ``poa_global`` (from ``poa``),
6
+ ``temp_air`` (from ``t_amb``), ``wind_speed``, ``temp_module`` (from ``t_module``, the measured back-of-module value);
7
+ * indexes it by the **centre of the interval** (``at="center"``, the right instant for a mean over the interval; ``"start"`` and
8
+ ``"end"`` are also available) in UTC;
9
+ * gives the location and the array geometry of an inverter (``surface_tilt``, ``surface_azimuth`` clockwise from north, 180 =
10
+ south, ``pdc0`` in W from ``capacity_kwp``);
11
+ * returns the measured AC power of the inverter in kW on the same index, to compare with a model.
12
+
13
+ Units already agree (W/m2, degC, m/s); no conversion happens here.
14
+
15
+ Usage:
16
+ inputs = to_pvlib(df, meta, inverter="INV")
17
+ loc = pvlib.location.Location(**inputs.location)
18
+ weather, p_meas = inputs.weather, inputs.p_ac_kw
19
+ """
20
+
21
+ from __future__ import annotations
22
+
23
+ from dataclasses import dataclass
24
+
25
+ import numpy as np
26
+ import pandas as pd
27
+
28
+ from scada_schema.model import DatasetMeta
29
+
30
+ __all__ = ["COLUMN_MAP", "PvlibInputs", "to_pvlib"]
31
+
32
+ COLUMN_MAP = {"ghi": "ghi", "dni": "dni", "dhi": "dhi", "poa": "poa_global", "t_amb": "temp_air", "wind_speed": "wind_speed",
33
+ "t_module": "temp_module"}
34
+ _SHIFT = {"start": 0.0, "center": 0.5, "end": 1.0}
35
+
36
+
37
+ @dataclass
38
+ class PvlibInputs:
39
+ weather: pd.DataFrame
40
+ location: dict
41
+ system: dict
42
+ p_ac_kw: pd.Series
43
+ notes: list[str]
44
+
45
+
46
+ def to_pvlib(df: pd.DataFrame, meta: DatasetMeta, inverter: str | None = None, met: str | None = None, at: str = "center") -> PvlibInputs:
47
+ """pvlib inputs for one inverter of a canonical PV table.
48
+
49
+ Parameters
50
+ ----------
51
+ df, meta
52
+ Canonical PV table and its metadata.
53
+ inverter : str, optional
54
+ Inverter id (the only one by default).
55
+ met : str, optional
56
+ Met device id (the only ``met`` or ``plant`` device by default).
57
+ at : {"center", "start", "end"}
58
+ Which instant of the interval indexes the result.
59
+ """
60
+ if meta.profile != "pv":
61
+ raise ValueError("to_pvlib needs a PV table")
62
+ if at not in _SHIFT:
63
+ raise ValueError(f"at must be one of {list(_SHIFT)}")
64
+ invs = [d for d in meta.devices if d.type == "inverter"]
65
+ mets = [d for d in meta.devices if d.type in ("met", "plant")]
66
+ inv = _pick(invs, inverter, "inverter")
67
+ met_dev = _pick(mets, met, "met")
68
+ shift = pd.Timedelta(meta.step * _SHIFT[at])
69
+ w = df[df["device_id"] == met_dev.id].set_index("timestamp")
70
+ weather = pd.DataFrame(index=w.index + shift)
71
+ for src, dst in COLUMN_MAP.items():
72
+ if src in w.columns and w[src].notna().any():
73
+ weather[dst] = w[src].to_numpy(dtype="float64", na_value=np.nan)
74
+ weather.index.name = "time"
75
+ p = df[df["device_id"] == inv.id].set_index("timestamp")
76
+ if "p_ac" not in p.columns:
77
+ raise ValueError(f"inverter {inv.id} has no p_ac")
78
+ p_ac = pd.Series(p["p_ac"].to_numpy(dtype="float64", na_value=np.nan), index=p.index + shift, name="p_ac_kw")
79
+ p_ac.index.name = "time"
80
+ loc_src = inv if inv.lat is not None else met_dev
81
+ if loc_src.lat is None or loc_src.lon is None:
82
+ raise ValueError("pvlib needs a latitude and longitude on the inverter or the met device")
83
+ location = {"latitude": loc_src.lat, "longitude": loc_src.lon, "tz": "UTC"}
84
+ system = {k: v for k, v in {"surface_tilt": inv.tilt_deg, "surface_azimuth": inv.azimuth_deg,
85
+ "pdc0": None if inv.capacity_kwp is None else inv.capacity_kwp * 1000.0}.items() if v is not None}
86
+ notes = [f"index: {at} of the interval (UTC)"]
87
+ return PvlibInputs(weather, location, system, p_ac, notes)
88
+
89
+
90
+ def _pick(devs, wanted, kind):
91
+ if wanted is not None:
92
+ for d in devs:
93
+ if d.id == wanted:
94
+ return d
95
+ raise KeyError(f"no {kind} device {wanted!r}")
96
+ if len(devs) != 1:
97
+ raise ValueError(f"the table has {len(devs)} {kind} devices; name one")
98
+ return devs[0]