protocolith 0.6.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. protocolith/__init__.py +38 -0
  2. protocolith/baseline_groups.py +294 -0
  3. protocolith/baselines.py +288 -0
  4. protocolith/card_svg_render.py +281 -0
  5. protocolith/card_svg_styles.py +178 -0
  6. protocolith/cli.py +796 -0
  7. protocolith/core/__init__.py +63 -0
  8. protocolith/core/_lock_posix.py +29 -0
  9. protocolith/core/_lock_windows.py +29 -0
  10. protocolith/core/atlas.py +406 -0
  11. protocolith/core/atlas_assets/atlas.css +86 -0
  12. protocolith/core/atlas_assets/atlas.html +84 -0
  13. protocolith/core/atlas_assets/atlas.js +370 -0
  14. protocolith/core/atlas_i18n/en.json +61 -0
  15. protocolith/core/canonical.py +124 -0
  16. protocolith/core/capture.py +68 -0
  17. protocolith/core/clock.py +25 -0
  18. protocolith/core/commands.py +356 -0
  19. protocolith/core/errors.py +116 -0
  20. protocolith/core/issue.py +98 -0
  21. protocolith/core/ledger.py +75 -0
  22. protocolith/core/mcp.py +371 -0
  23. protocolith/core/ops.py +183 -0
  24. protocolith/core/rules.py +352 -0
  25. protocolith/core/schema.py +137 -0
  26. protocolith/core/store.py +101 -0
  27. protocolith/core/views.py +136 -0
  28. protocolith/doctor.py +135 -0
  29. protocolith/evidence_card.py +354 -0
  30. protocolith/family_ledger.py +489 -0
  31. protocolith/generic_ohlcv_batch_pre_registration_meta.py +123 -0
  32. protocolith/inspect.py +48 -0
  33. protocolith/nasa_power.py +266 -0
  34. protocolith/nasa_power_fetch.py +64 -0
  35. protocolith/nasa_power_parser.py +133 -0
  36. protocolith/nasa_power_prereg_runner.py +383 -0
  37. protocolith/noaa_coops_fetch.py +188 -0
  38. protocolith/noaa_coops_parser.py +120 -0
  39. protocolith/noaa_coops_prereg_runner.py +341 -0
  40. protocolith/public_ai_protocol.py +176 -0
  41. protocolith/py.typed +0 -0
  42. protocolith/registry.py +139 -0
  43. protocolith/registry_view.py +302 -0
  44. protocolith/rerun_report.py +111 -0
  45. protocolith/run_root.py +243 -0
  46. protocolith/scaffold.py +565 -0
  47. protocolith/source_probe.py +230 -0
  48. protocolith/svg_a11y.py +100 -0
  49. protocolith/tree_overlay.py +343 -0
  50. protocolith/verify_packs.py +251 -0
  51. protocolith/workflows.py +233 -0
  52. protocolith-0.6.0.dist-info/METADATA +283 -0
  53. protocolith-0.6.0.dist-info/RECORD +57 -0
  54. protocolith-0.6.0.dist-info/WHEEL +4 -0
  55. protocolith-0.6.0.dist-info/entry_points.txt +3 -0
  56. protocolith-0.6.0.dist-info/licenses/LICENSE +201 -0
  57. protocolith-0.6.0.dist-info/licenses/NOTICE +13 -0
@@ -0,0 +1,383 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+
3
+ """NASA POWER D-103 frozen pre-registration evaluator.
4
+
5
+ This runner evaluates operator-supplied local NASA POWER Daily JSON files under
6
+ the frozen 1.0.143 charter. It performs no network I/O and does not change gates.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import hashlib
12
+ from dataclasses import dataclass, field
13
+ from datetime import date
14
+ from pathlib import Path
15
+ from collections.abc import Mapping
16
+
17
+ import numpy as np
18
+ from numpy.typing import NDArray
19
+
20
+ from protocolith.baseline_groups import (
21
+ Doc11AcceptanceConfig,
22
+ WindowObservation,
23
+ evaluate_group_vs_baseline_pool,
24
+ )
25
+ from protocolith.generic_ohlcv_batch_pre_registration_meta import (
26
+ build_generic_ohlcv_batch_reproducibility,
27
+ )
28
+ from protocolith.nasa_power import (
29
+ NASA_POWER_POINTS,
30
+ normalize_point_id,
31
+ parse_nasa_power_json_file,
32
+ solar_7d_mean_from_values,
33
+ )
34
+
35
+ NASA_POWER_PREREG_REPORT_SCHEMA = "nasa_power_prereg_report_v1"
36
+ NASA_POWER_CANDIDATE_NAME = "nasa_power_solar_resource_shortfall_zscore"
37
+
38
+
39
+ @dataclass(frozen=True)
40
+ class NasaPowerPreregConfig:
41
+ horizon_days: int = 7
42
+ threshold_quantile: float = 0.10
43
+ trailing_window_days: int = 7
44
+ train_days: int = 5 * 365
45
+ test_days: int = 365
46
+ step_days: int = 365
47
+ minimum_test_event_count: int = 15
48
+ solar_sigma_floor: float = 1e-9
49
+ acceptance: Doc11AcceptanceConfig = field(
50
+ default_factory=lambda: Doc11AcceptanceConfig(
51
+ top_rate=0.10,
52
+ min_windows=8,
53
+ min_positive_rate=0.65,
54
+ min_event_rate=0.03,
55
+ max_event_rate=0.40,
56
+ )
57
+ )
58
+
59
+
60
+ def _sha256_of_file(path: Path) -> str:
61
+ digest = hashlib.sha256()
62
+ with path.open("rb") as fh:
63
+ for chunk in iter(lambda: fh.read(64 * 1024), b""):
64
+ digest.update(chunk)
65
+ return digest.hexdigest()
66
+
67
+
68
+ def _sha256_of_text(text: str) -> str:
69
+ return hashlib.sha256(text.encode("utf-8")).hexdigest()
70
+
71
+
72
+ def _zscore_low(values: NDArray[np.float64], train_idx: NDArray[np.int64], test_idx: NDArray[np.int64], floor: float) -> NDArray[np.float64]:
73
+ train = np.asarray(values[train_idx], dtype=np.float64)
74
+ mu = float(np.mean(train)) if train.size else 0.0
75
+ sigma = max(float(np.std(train)) if train.size else 0.0, floor)
76
+ return (mu - np.asarray(values[test_idx], dtype=np.float64)) / sigma
77
+
78
+
79
+ def _zscore_high(values: NDArray[np.float64], train_idx: NDArray[np.int64], test_idx: NDArray[np.int64], floor: float) -> NDArray[np.float64]:
80
+ train = np.asarray(values[train_idx], dtype=np.float64)
81
+ mu = float(np.mean(train)) if train.size else 0.0
82
+ sigma = max(float(np.std(train)) if train.size else 0.0, floor)
83
+ return (np.asarray(values[test_idx], dtype=np.float64) - mu) / sigma
84
+
85
+
86
+ def _rolling_low_zscore(values: NDArray[np.float64], *, window: int, floor: float) -> NDArray[np.float64]:
87
+ arr = np.asarray(values, dtype=np.float64)
88
+ out = np.zeros_like(arr)
89
+ for idx in range(len(arr)):
90
+ start = max(0, idx - window)
91
+ hist = arr[start:idx]
92
+ if hist.size < 2:
93
+ continue
94
+ mu = float(np.mean(hist))
95
+ sigma = max(float(np.std(hist)), floor)
96
+ out[idx] = (mu - float(arr[idx])) / sigma
97
+ return out
98
+
99
+
100
+ def _seasonal_dayofyear_low_zscore(
101
+ values: NDArray[np.float64],
102
+ days: list[date],
103
+ *,
104
+ train_idx: NDArray[np.int64],
105
+ test_idx: NDArray[np.int64],
106
+ floor: float,
107
+ ) -> NDArray[np.float64]:
108
+ arr = np.asarray(values, dtype=np.float64)
109
+ by_day: dict[int, list[float]] = {idx: [] for idx in range(1, 367)}
110
+ for idx in train_idx.tolist():
111
+ by_day[days[idx].timetuple().tm_yday].append(float(arr[idx]))
112
+ global_train = arr[train_idx]
113
+ fallback_mu = float(np.mean(global_train)) if global_train.size else 0.0
114
+ fallback_sigma = max(float(np.std(global_train)) if global_train.size else 0.0, floor)
115
+ out = np.zeros(len(test_idx), dtype=np.float64)
116
+ for pos, idx in enumerate(test_idx.tolist()):
117
+ doy = days[idx].timetuple().tm_yday
118
+ bucket_values: list[float] = []
119
+ for neighbor in (doy - 1, doy, doy + 1):
120
+ if neighbor == 0:
121
+ key = 366
122
+ elif neighbor == 367:
123
+ key = 1
124
+ else:
125
+ key = neighbor
126
+ bucket_values.extend(by_day.get(key, []))
127
+ bucket = np.asarray(bucket_values, dtype=np.float64)
128
+ if bucket.size >= 2:
129
+ mu = float(np.mean(bucket))
130
+ sigma = max(float(np.std(bucket)), floor)
131
+ else:
132
+ mu = fallback_mu
133
+ sigma = fallback_sigma
134
+ out[pos] = (mu - float(arr[idx])) / sigma
135
+ return out
136
+
137
+
138
+ def _build_forward_events(
139
+ solar_7d_mean: NDArray[np.float64],
140
+ *,
141
+ train_idx: NDArray[np.int64],
142
+ test_idx: NDArray[np.int64],
143
+ horizon_days: int,
144
+ threshold_quantile: float,
145
+ ) -> NDArray[np.bool_]:
146
+ threshold = float(np.quantile(solar_7d_mean[train_idx], threshold_quantile))
147
+ events = np.zeros(len(test_idx), dtype=bool)
148
+ n = len(solar_7d_mean)
149
+ for pos, idx in enumerate(test_idx.tolist()):
150
+ end = min(n, idx + horizon_days + 1)
151
+ if idx + 1 >= end:
152
+ continue
153
+ events[pos] = bool(np.min(solar_7d_mean[idx + 1 : end]) < threshold)
154
+ return events
155
+
156
+
157
+ def _walk_forward_splits(n: int, cfg: NasaPowerPreregConfig) -> list[tuple[NDArray[np.int64], NDArray[np.int64]]]:
158
+ splits: list[tuple[NDArray[np.int64], NDArray[np.int64]]] = []
159
+ start = 0
160
+ while start + cfg.train_days + cfg.test_days <= n:
161
+ train = np.arange(start, start + cfg.train_days, dtype=np.int64)
162
+ test = np.arange(
163
+ start + cfg.train_days,
164
+ start + cfg.train_days + cfg.test_days,
165
+ dtype=np.int64,
166
+ )
167
+ splits.append((train, test))
168
+ start += cfg.step_days
169
+ return splits
170
+
171
+
172
+ def _baseline_scores_for_split(
173
+ *,
174
+ solar_7d_mean: NDArray[np.float64],
175
+ t2m: NDArray[np.float64],
176
+ ws10m: NDArray[np.float64],
177
+ prectotcorr: NDArray[np.float64],
178
+ days: list[date],
179
+ train_idx: NDArray[np.int64],
180
+ test_idx: NDArray[np.int64],
181
+ floor: float,
182
+ ) -> Mapping[str, NDArray[np.float64]]:
183
+ rolling_30d = _rolling_low_zscore(solar_7d_mean, window=30, floor=floor)[test_idx]
184
+ rolling_90d = _rolling_low_zscore(solar_7d_mean, window=90, floor=floor)[test_idx]
185
+ shifted = np.roll(solar_7d_mean, 1)
186
+ shifted[0] = solar_7d_mean[0]
187
+ persistence = _zscore_low(shifted, train_idx, test_idx, floor)
188
+ seasonal = _seasonal_dayofyear_low_zscore(
189
+ solar_7d_mean, days, train_idx=train_idx, test_idx=test_idx, floor=floor
190
+ )
191
+ temperature = _zscore_high(t2m, train_idx, test_idx, floor)
192
+ wind = _zscore_high(ws10m, train_idx, test_idx, floor)
193
+ precipitation = _zscore_high(prectotcorr, train_idx, test_idx, floor)
194
+ shuffled = np.asarray(solar_7d_mean[test_idx], dtype=np.float64).copy()
195
+ rng = np.random.default_rng(20260428)
196
+ rng.shuffle(shuffled)
197
+ shuffled = _zscore_low(shuffled, np.arange(len(shuffled), dtype=np.int64), np.arange(len(shuffled), dtype=np.int64), floor)
198
+ reversed_scores = _zscore_low(
199
+ solar_7d_mean[::-1],
200
+ np.arange(min(len(train_idx), len(solar_7d_mean)), dtype=np.int64),
201
+ np.arange(len(test_idx), dtype=np.int64),
202
+ floor,
203
+ )
204
+ no_solar_context = (
205
+ temperature + wind + precipitation
206
+ ) / 3.0
207
+ return {
208
+ "rolling_solar_shortfall_zscore_30d": rolling_30d,
209
+ "rolling_solar_shortfall_zscore_90d": rolling_90d,
210
+ "persistence_solar_shortfall": persistence,
211
+ "seasonal_dayofyear_solar_shortfall": seasonal,
212
+ "temperature_anomaly_zscore": temperature,
213
+ "wind_speed_anomaly_zscore": wind,
214
+ "precipitation_anomaly_zscore": precipitation,
215
+ "shuffled_candidate": shuffled,
216
+ "time_reversed_candidate": reversed_scores,
217
+ "candidate_no_solar_context": no_solar_context,
218
+ }
219
+
220
+
221
+ def _load_point_payload(
222
+ json_path: Path,
223
+ *,
224
+ point_id: str,
225
+ ) -> tuple[list[date], dict[str, NDArray[np.float64]], NDArray[np.float64], dict[str, object]]:
226
+ pid = normalize_point_id(point_id)
227
+ days, values, parse_summary = parse_nasa_power_json_file(json_path)
228
+ solar_7d = solar_7d_mean_from_values(values)
229
+ point = NASA_POWER_POINTS[pid]
230
+ source = {
231
+ "point_id": pid,
232
+ "point_name": point.name,
233
+ "point_location_redacted": True,
234
+ "json": json_path.name,
235
+ "json_sha256": _sha256_of_file(json_path),
236
+ "rows": parse_summary["rows"],
237
+ "skipped": parse_summary["skipped"],
238
+ "first_date": parse_summary["first_date"],
239
+ "last_date": parse_summary["last_date"],
240
+ "finite_parameter_coverage": 1.0,
241
+ }
242
+ return days, values, solar_7d, source
243
+
244
+
245
+ def evaluate_nasa_power_prereg(
246
+ *,
247
+ point_ids: list[str],
248
+ json_paths: list[Path],
249
+ config: NasaPowerPreregConfig | None = None,
250
+ pre_registration_doc_path: Path | None = None,
251
+ ) -> dict[str, object]:
252
+ if len(point_ids) != len(json_paths):
253
+ raise ValueError("point_ids and json_paths counts must match")
254
+ cfg = config or NasaPowerPreregConfig()
255
+
256
+ observations: list[WindowObservation] = []
257
+ point_reports: list[dict[str, object]] = []
258
+ source_audit_passed = True
259
+ window_id = 0
260
+ for point_id, json_path in zip(point_ids, json_paths, strict=True):
261
+ days, values, solar_7d, source = _load_point_payload(Path(json_path), point_id=point_id)
262
+ point = NASA_POWER_POINTS[normalize_point_id(point_id)]
263
+ point_windows: list[dict[str, object]] = []
264
+ coverage_ok = bool(int(source["rows"]) >= int(0.95 * len(days)))
265
+ source_audit_passed = source_audit_passed and coverage_ok
266
+ for train_idx, test_idx in _walk_forward_splits(len(solar_7d), cfg):
267
+ events = _build_forward_events(
268
+ solar_7d,
269
+ train_idx=train_idx,
270
+ test_idx=test_idx,
271
+ horizon_days=cfg.horizon_days,
272
+ threshold_quantile=cfg.threshold_quantile,
273
+ )
274
+ event_count = int(np.sum(events))
275
+ event_rate = float(np.mean(events)) if len(events) else None
276
+ eligible = event_count >= cfg.minimum_test_event_count
277
+ point_windows.append(
278
+ {
279
+ "window_id": window_id,
280
+ "rows": int(len(test_idx)),
281
+ "events": event_count,
282
+ "event_rate": event_rate,
283
+ "eligible": bool(eligible),
284
+ }
285
+ )
286
+ if eligible:
287
+ candidate = _zscore_low(solar_7d, train_idx, test_idx, cfg.solar_sigma_floor)
288
+ baselines = _baseline_scores_for_split(
289
+ solar_7d_mean=solar_7d,
290
+ t2m=values["T2M"],
291
+ ws10m=values["WS10M"],
292
+ prectotcorr=values["PRECTOTCORR"],
293
+ days=days,
294
+ train_idx=train_idx,
295
+ test_idx=test_idx,
296
+ floor=cfg.solar_sigma_floor,
297
+ )
298
+ observations.append(
299
+ WindowObservation(
300
+ candidate_scores=candidate,
301
+ baseline_scores=baselines,
302
+ events=events,
303
+ window_id=window_id,
304
+ )
305
+ )
306
+ window_id += 1
307
+ point_reports.append(
308
+ {
309
+ "point_id": point.point_id,
310
+ "point_name": point.name,
311
+ "point_location_redacted": True,
312
+ "source": source,
313
+ "coverage_ok": coverage_ok,
314
+ "windows": point_windows,
315
+ }
316
+ )
317
+
318
+ acceptance = evaluate_group_vs_baseline_pool(observations, config=cfg.acceptance)
319
+ eligible_point_count = sum(1 for point in point_reports if point["coverage_ok"])
320
+ eligible_windows = len(observations)
321
+ pre_reg_hash: str | None = None
322
+ if pre_registration_doc_path is not None and pre_registration_doc_path.is_file():
323
+ pre_reg_hash = _sha256_of_text(pre_registration_doc_path.read_text(encoding="utf-8"))
324
+ source_audit_passed = source_audit_passed and eligible_point_count >= 2
325
+ overall = (
326
+ source_audit_passed
327
+ and eligible_windows >= 8
328
+ and bool(acceptance.get("overall_go_no_go", False))
329
+ )
330
+ result_status = "PASSED_UNDER_PROTOCOL" if overall else "NEGATIVE_RESULT"
331
+ if not source_audit_passed:
332
+ result_status = "BLOCKED_SOURCE"
333
+ return {
334
+ "schema": NASA_POWER_PREREG_REPORT_SCHEMA,
335
+ "candidate": NASA_POWER_CANDIDATE_NAME,
336
+ "pre_registration": {
337
+ "doc": "NASA_POWER_D103_PREREG_CHARTER.md",
338
+ "doc_sha256": pre_reg_hash,
339
+ "config": {
340
+ "horizon_days": cfg.horizon_days,
341
+ "threshold_quantile": cfg.threshold_quantile,
342
+ "trailing_window_days": cfg.trailing_window_days,
343
+ "train_days": cfg.train_days,
344
+ "test_days": cfg.test_days,
345
+ "step_days": cfg.step_days,
346
+ "minimum_test_event_count": cfg.minimum_test_event_count,
347
+ "top_rate": cfg.acceptance.top_rate,
348
+ "min_positive_rate": cfg.acceptance.min_positive_rate,
349
+ },
350
+ },
351
+ "reproducibility": build_generic_ohlcv_batch_reproducibility(),
352
+ "source_audit_passed": bool(source_audit_passed),
353
+ "eligible_point_count": int(eligible_point_count),
354
+ "eligible_windows": int(eligible_windows),
355
+ "points": point_reports,
356
+ "acceptance": acceptance,
357
+ "overall_go_no_go": bool(overall),
358
+ "result_status": result_status,
359
+ "allowed_narrow_claim": (
360
+ "passed the pre-registered NASA POWER D-103 gate under protocol 1.0.143"
361
+ if overall
362
+ else None
363
+ ),
364
+ "forbidden_claims": [
365
+ "broad or universal forecasting performance",
366
+ "deployment readiness",
367
+ "energy-grid operational readiness",
368
+ "non-public background performance claim",
369
+ ],
370
+ "honest_boundary": (
371
+ "This report evaluates local NASA POWER JSON payloads under the frozen "
372
+ "D-103 protocol. A positive claim requires overall_go_no_go=true; "
373
+ "otherwise the result is negative or source-blocked evidence."
374
+ ),
375
+ }
376
+
377
+
378
+ __all__ = [
379
+ "NASA_POWER_CANDIDATE_NAME",
380
+ "NASA_POWER_PREREG_REPORT_SCHEMA",
381
+ "NasaPowerPreregConfig",
382
+ "evaluate_nasa_power_prereg",
383
+ ]
@@ -0,0 +1,188 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ """Download NOAA CO-OPS D-131 payloads in API-compliant chunks.
3
+
4
+ NOAA's Data API limits many products (including hourly_height) to 365 days per
5
+ request. D-131 spans 2018-01-01..2024-12-31, so operators must fetch by year (or
6
+ smaller windows) and merge payloads locally outside the repository.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import json
12
+ import urllib.error
13
+ import urllib.parse
14
+ import urllib.request
15
+ from dataclasses import dataclass
16
+ from datetime import date
17
+ from pathlib import Path
18
+ from collections.abc import Callable
19
+
20
+
21
+ NOAA_COOPS_D131_STATIONS: tuple[str, ...] = ("8518750", "9414290", "8638610")
22
+ NOAA_COOPS_D131_BEGIN = date(2018, 1, 1)
23
+ NOAA_COOPS_D131_END = date(2024, 12, 31)
24
+ NOAA_COOPS_API_BASE = "https://api.tidesandcurrents.noaa.gov/api/prod/datagetter"
25
+ NOAA_COOPS_APPLICATION = "NOS.COOPS.TAC.WL"
26
+
27
+
28
+ @dataclass(frozen=True)
29
+ class NoaaCoopsFetchConfig:
30
+ station: str
31
+ product: str
32
+ begin: date
33
+ end: date
34
+ datum: str = "MLLW"
35
+ time_zone: str = "gmt"
36
+ units: str = "metric"
37
+ interval: str | None = None
38
+ application: str = NOAA_COOPS_APPLICATION
39
+
40
+
41
+ def year_chunks(begin: date, end: date) -> list[tuple[date, date]]:
42
+ if begin > end:
43
+ raise ValueError("begin must be <= end")
44
+
45
+ chunks: list[tuple[date, date]] = []
46
+ for year in range(begin.year, end.year + 1):
47
+ chunk_start = date(year, 1, 1) if year > begin.year else begin
48
+ chunk_end = date(year, 12, 31) if year < end.year else end
49
+ chunks.append((chunk_start, chunk_end))
50
+ return chunks
51
+
52
+
53
+ def build_datagetter_url(config: NoaaCoopsFetchConfig, chunk_begin: date, chunk_end: date) -> str:
54
+ params: dict[str, str] = {
55
+ "product": config.product,
56
+ "application": config.application,
57
+ "begin_date": chunk_begin.strftime("%Y%m%d"),
58
+ "end_date": chunk_end.strftime("%Y%m%d"),
59
+ "datum": config.datum,
60
+ "station": config.station,
61
+ "time_zone": config.time_zone,
62
+ "units": config.units,
63
+ "format": "json",
64
+ }
65
+ if config.product == "predictions":
66
+ params["interval"] = config.interval or "h"
67
+ query = urllib.parse.urlencode(params)
68
+ return f"{NOAA_COOPS_API_BASE}?{query}"
69
+
70
+
71
+ def merge_hourly_height_payloads(payloads: list[dict[str, object]]) -> dict[str, object]:
72
+ if not payloads:
73
+ raise ValueError("hourly_height payloads must not be empty")
74
+
75
+ merged_data: list[object] = []
76
+ metadata: object | None = None
77
+ for payload in payloads:
78
+ if "error" in payload:
79
+ raise ValueError(str(payload["error"]))
80
+ if metadata is None and "metadata" in payload:
81
+ metadata = payload["metadata"]
82
+ data = payload.get("data")
83
+ if not isinstance(data, list):
84
+ raise ValueError("hourly_height payload missing data list")
85
+ merged_data.extend(data)
86
+
87
+ out: dict[str, object] = {"data": merged_data}
88
+ if metadata is not None:
89
+ out["metadata"] = metadata
90
+ return out
91
+
92
+
93
+ def merge_predictions_payloads(payloads: list[dict[str, object]]) -> dict[str, object]:
94
+ if not payloads:
95
+ raise ValueError("predictions payloads must not be empty")
96
+
97
+ merged_predictions: list[object] = []
98
+ for payload in payloads:
99
+ if "error" in payload:
100
+ raise ValueError(str(payload["error"]))
101
+ predictions = payload.get("predictions")
102
+ if not isinstance(predictions, list):
103
+ raise ValueError("predictions payload missing predictions list")
104
+ merged_predictions.extend(predictions)
105
+
106
+ return {"predictions": merged_predictions}
107
+
108
+
109
+ def merge_product_payloads(product: str, payloads: list[dict[str, object]]) -> dict[str, object]:
110
+ if product == "hourly_height":
111
+ return merge_hourly_height_payloads(payloads)
112
+ if product == "predictions":
113
+ return merge_predictions_payloads(payloads)
114
+ raise ValueError(f"unsupported NOAA product {product!r}")
115
+
116
+
117
+ def default_fetch_json(url: str) -> dict[str, object]:
118
+ request = urllib.request.Request(url, headers={"User-Agent": "Protocolith-noaa-fetch/0.1"})
119
+ try:
120
+ with urllib.request.urlopen(request, timeout=120) as response:
121
+ body = response.read()
122
+ except urllib.error.HTTPError as exc:
123
+ detail = exc.read().decode("utf-8", errors="replace")
124
+ raise RuntimeError(f"NOAA HTTP {exc.code} for {url}: {detail}") from exc
125
+
126
+ payload = json.loads(body.decode("utf-8"))
127
+ if not isinstance(payload, dict):
128
+ raise RuntimeError(f"NOAA response is not a JSON object: {url}")
129
+ return payload
130
+
131
+
132
+ def fetch_product_payload(
133
+ config: NoaaCoopsFetchConfig,
134
+ *,
135
+ fetch_json: Callable[[str], dict[str, object]] = default_fetch_json,
136
+ ) -> dict[str, object]:
137
+ chunk_payloads: list[dict[str, object]] = []
138
+ for chunk_begin, chunk_end in year_chunks(config.begin, config.end):
139
+ url = build_datagetter_url(config, chunk_begin, chunk_end)
140
+ chunk_payloads.append(fetch_json(url))
141
+ return merge_product_payloads(config.product, chunk_payloads)
142
+
143
+
144
+ def write_json(path: Path, payload: dict[str, object]) -> Path:
145
+ path.parent.mkdir(parents=True, exist_ok=True)
146
+ path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
147
+ return path
148
+
149
+
150
+ def fetch_d131_station_payloads(
151
+ station: str,
152
+ out_dir: Path,
153
+ *,
154
+ begin: date = NOAA_COOPS_D131_BEGIN,
155
+ end: date = NOAA_COOPS_D131_END,
156
+ fetch_json: Callable[[str], dict[str, object]] = default_fetch_json,
157
+ ) -> list[Path]:
158
+ written: list[Path] = []
159
+ for product, suffix in (("hourly_height", "observed"), ("predictions", "predictions")):
160
+ payload = fetch_product_payload(
161
+ NoaaCoopsFetchConfig(
162
+ station=station,
163
+ product=product,
164
+ begin=begin,
165
+ end=end,
166
+ ),
167
+ fetch_json=fetch_json,
168
+ )
169
+ written.append(write_json(out_dir / f"{station}_{suffix}.json", payload))
170
+ return written
171
+
172
+
173
+ __all__ = [
174
+ "NOAA_COOPS_API_BASE",
175
+ "NOAA_COOPS_D131_BEGIN",
176
+ "NOAA_COOPS_D131_END",
177
+ "NOAA_COOPS_D131_STATIONS",
178
+ "NoaaCoopsFetchConfig",
179
+ "build_datagetter_url",
180
+ "default_fetch_json",
181
+ "fetch_d131_station_payloads",
182
+ "fetch_product_payload",
183
+ "merge_hourly_height_payloads",
184
+ "merge_predictions_payloads",
185
+ "merge_product_payloads",
186
+ "write_json",
187
+ "year_chunks",
188
+ ]
@@ -0,0 +1,120 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+
3
+ """Parse and join NOAA CO-OPS D-131 local JSON payloads."""
4
+
5
+ from __future__ import annotations
6
+
7
+ import hashlib
8
+ import json
9
+ from datetime import datetime
10
+ from pathlib import Path
11
+
12
+ import numpy as np
13
+ from numpy.typing import NDArray
14
+
15
+ from protocolith.noaa_coops_fetch import NOAA_COOPS_D131_STATIONS
16
+
17
+
18
+ def sha256_of_file(path: Path) -> str:
19
+ digest = hashlib.sha256()
20
+ with path.open("rb") as fh:
21
+ for chunk in iter(lambda: fh.read(64 * 1024), b""):
22
+ digest.update(chunk)
23
+ return digest.hexdigest()
24
+
25
+
26
+ def parse_noaa_coops_series(path: Path) -> tuple[list[datetime], NDArray[np.float64], dict[str, object]]:
27
+ """Parse one NOAA CO-OPS hourly_height or predictions JSON file."""
28
+
29
+ payload = json.loads(path.read_text(encoding="utf-8"))
30
+ if "error" in payload:
31
+ raise ValueError(str(payload["error"]))
32
+
33
+ rows = payload.get("data")
34
+ if rows is None:
35
+ rows = payload.get("predictions")
36
+ if not isinstance(rows, list):
37
+ raise ValueError(f"{path.name}: expected data or predictions list")
38
+
39
+ timestamps: list[datetime] = []
40
+ values: list[float] = []
41
+ skipped = 0
42
+ for row in rows:
43
+ if not isinstance(row, dict):
44
+ skipped += 1
45
+ continue
46
+ raw_value = row.get("v")
47
+ if raw_value in (None, "", " "):
48
+ skipped += 1
49
+ continue
50
+ timestamps.append(datetime.strptime(str(row["t"]), "%Y-%m-%d %H:%M"))
51
+ values.append(float(raw_value))
52
+
53
+ if not timestamps:
54
+ raise ValueError(f"{path.name}: no finite observations")
55
+
56
+ return (
57
+ timestamps,
58
+ np.asarray(values, dtype=np.float64),
59
+ {
60
+ "json": path.name,
61
+ "json_sha256": sha256_of_file(path),
62
+ "rows": len(rows),
63
+ "finite_rows": len(values),
64
+ "skipped": skipped,
65
+ "first_timestamp": timestamps[0].isoformat(sep=" "),
66
+ "last_timestamp": timestamps[-1].isoformat(sep=" "),
67
+ },
68
+ )
69
+
70
+
71
+ def join_observed_predictions(
72
+ observed_path: Path,
73
+ predictions_path: Path,
74
+ ) -> tuple[list[datetime], NDArray[np.float64], NDArray[np.float64], NDArray[np.float64], dict[str, object]]:
75
+ """Join observed and prediction series on timestamp."""
76
+
77
+ obs_ts, obs_values, obs_summary = parse_noaa_coops_series(observed_path)
78
+ pred_ts, pred_values, pred_summary = parse_noaa_coops_series(predictions_path)
79
+
80
+ obs_map = {ts: value for ts, value in zip(obs_ts, obs_values, strict=True)}
81
+ pred_map = {ts: value for ts, value in zip(pred_ts, pred_values, strict=True)}
82
+ shared = sorted(set(obs_map) & set(pred_map))
83
+ if not shared:
84
+ raise ValueError(
85
+ f"no shared timestamps between {observed_path.name} and {predictions_path.name}"
86
+ )
87
+
88
+ observed = np.asarray([obs_map[ts] for ts in shared], dtype=np.float64)
89
+ predicted = np.asarray([pred_map[ts] for ts in shared], dtype=np.float64)
90
+ residual = observed - predicted
91
+ return (
92
+ shared,
93
+ observed,
94
+ predicted,
95
+ residual,
96
+ {
97
+ "observed": obs_summary,
98
+ "predictions": pred_summary,
99
+ "joined_rows": len(shared),
100
+ "observed_sha256": obs_summary["json_sha256"],
101
+ "predictions_sha256": pred_summary["json_sha256"],
102
+ },
103
+ )
104
+
105
+
106
+ def default_station_paths(raw_dir: Path, station: str) -> tuple[Path, Path]:
107
+ if station not in NOAA_COOPS_D131_STATIONS:
108
+ raise ValueError(f"unknown D-131 station: {station!r}")
109
+ return (
110
+ raw_dir / f"{station}_observed.json",
111
+ raw_dir / f"{station}_predictions.json",
112
+ )
113
+
114
+
115
+ __all__ = [
116
+ "default_station_paths",
117
+ "join_observed_predictions",
118
+ "parse_noaa_coops_series",
119
+ "sha256_of_file",
120
+ ]