protocolith 0.6.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- protocolith/__init__.py +38 -0
- protocolith/baseline_groups.py +294 -0
- protocolith/baselines.py +288 -0
- protocolith/card_svg_render.py +281 -0
- protocolith/card_svg_styles.py +178 -0
- protocolith/cli.py +796 -0
- protocolith/core/__init__.py +63 -0
- protocolith/core/_lock_posix.py +29 -0
- protocolith/core/_lock_windows.py +29 -0
- protocolith/core/atlas.py +406 -0
- protocolith/core/atlas_assets/atlas.css +86 -0
- protocolith/core/atlas_assets/atlas.html +84 -0
- protocolith/core/atlas_assets/atlas.js +370 -0
- protocolith/core/atlas_i18n/en.json +61 -0
- protocolith/core/canonical.py +124 -0
- protocolith/core/capture.py +68 -0
- protocolith/core/clock.py +25 -0
- protocolith/core/commands.py +356 -0
- protocolith/core/errors.py +116 -0
- protocolith/core/issue.py +98 -0
- protocolith/core/ledger.py +75 -0
- protocolith/core/mcp.py +371 -0
- protocolith/core/ops.py +183 -0
- protocolith/core/rules.py +352 -0
- protocolith/core/schema.py +137 -0
- protocolith/core/store.py +101 -0
- protocolith/core/views.py +136 -0
- protocolith/doctor.py +135 -0
- protocolith/evidence_card.py +354 -0
- protocolith/family_ledger.py +489 -0
- protocolith/generic_ohlcv_batch_pre_registration_meta.py +123 -0
- protocolith/inspect.py +48 -0
- protocolith/nasa_power.py +266 -0
- protocolith/nasa_power_fetch.py +64 -0
- protocolith/nasa_power_parser.py +133 -0
- protocolith/nasa_power_prereg_runner.py +383 -0
- protocolith/noaa_coops_fetch.py +188 -0
- protocolith/noaa_coops_parser.py +120 -0
- protocolith/noaa_coops_prereg_runner.py +341 -0
- protocolith/public_ai_protocol.py +176 -0
- protocolith/py.typed +0 -0
- protocolith/registry.py +139 -0
- protocolith/registry_view.py +302 -0
- protocolith/rerun_report.py +111 -0
- protocolith/run_root.py +243 -0
- protocolith/scaffold.py +565 -0
- protocolith/source_probe.py +230 -0
- protocolith/svg_a11y.py +100 -0
- protocolith/tree_overlay.py +343 -0
- protocolith/verify_packs.py +251 -0
- protocolith/workflows.py +233 -0
- protocolith-0.6.0.dist-info/METADATA +283 -0
- protocolith-0.6.0.dist-info/RECORD +57 -0
- protocolith-0.6.0.dist-info/WHEEL +4 -0
- protocolith-0.6.0.dist-info/entry_points.txt +3 -0
- protocolith-0.6.0.dist-info/licenses/LICENSE +201 -0
- protocolith-0.6.0.dist-info/licenses/NOTICE +13 -0
|
@@ -0,0 +1,383 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
|
|
3
|
+
"""NASA POWER D-103 frozen pre-registration evaluator.
|
|
4
|
+
|
|
5
|
+
This runner evaluates operator-supplied local NASA POWER Daily JSON files under
|
|
6
|
+
the frozen 1.0.143 charter. It performs no network I/O and does not change gates.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import hashlib
|
|
12
|
+
from dataclasses import dataclass, field
|
|
13
|
+
from datetime import date
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from collections.abc import Mapping
|
|
16
|
+
|
|
17
|
+
import numpy as np
|
|
18
|
+
from numpy.typing import NDArray
|
|
19
|
+
|
|
20
|
+
from protocolith.baseline_groups import (
|
|
21
|
+
Doc11AcceptanceConfig,
|
|
22
|
+
WindowObservation,
|
|
23
|
+
evaluate_group_vs_baseline_pool,
|
|
24
|
+
)
|
|
25
|
+
from protocolith.generic_ohlcv_batch_pre_registration_meta import (
|
|
26
|
+
build_generic_ohlcv_batch_reproducibility,
|
|
27
|
+
)
|
|
28
|
+
from protocolith.nasa_power import (
|
|
29
|
+
NASA_POWER_POINTS,
|
|
30
|
+
normalize_point_id,
|
|
31
|
+
parse_nasa_power_json_file,
|
|
32
|
+
solar_7d_mean_from_values,
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
NASA_POWER_PREREG_REPORT_SCHEMA = "nasa_power_prereg_report_v1"
|
|
36
|
+
NASA_POWER_CANDIDATE_NAME = "nasa_power_solar_resource_shortfall_zscore"
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@dataclass(frozen=True)
|
|
40
|
+
class NasaPowerPreregConfig:
|
|
41
|
+
horizon_days: int = 7
|
|
42
|
+
threshold_quantile: float = 0.10
|
|
43
|
+
trailing_window_days: int = 7
|
|
44
|
+
train_days: int = 5 * 365
|
|
45
|
+
test_days: int = 365
|
|
46
|
+
step_days: int = 365
|
|
47
|
+
minimum_test_event_count: int = 15
|
|
48
|
+
solar_sigma_floor: float = 1e-9
|
|
49
|
+
acceptance: Doc11AcceptanceConfig = field(
|
|
50
|
+
default_factory=lambda: Doc11AcceptanceConfig(
|
|
51
|
+
top_rate=0.10,
|
|
52
|
+
min_windows=8,
|
|
53
|
+
min_positive_rate=0.65,
|
|
54
|
+
min_event_rate=0.03,
|
|
55
|
+
max_event_rate=0.40,
|
|
56
|
+
)
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _sha256_of_file(path: Path) -> str:
|
|
61
|
+
digest = hashlib.sha256()
|
|
62
|
+
with path.open("rb") as fh:
|
|
63
|
+
for chunk in iter(lambda: fh.read(64 * 1024), b""):
|
|
64
|
+
digest.update(chunk)
|
|
65
|
+
return digest.hexdigest()
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _sha256_of_text(text: str) -> str:
|
|
69
|
+
return hashlib.sha256(text.encode("utf-8")).hexdigest()
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _zscore_low(values: NDArray[np.float64], train_idx: NDArray[np.int64], test_idx: NDArray[np.int64], floor: float) -> NDArray[np.float64]:
|
|
73
|
+
train = np.asarray(values[train_idx], dtype=np.float64)
|
|
74
|
+
mu = float(np.mean(train)) if train.size else 0.0
|
|
75
|
+
sigma = max(float(np.std(train)) if train.size else 0.0, floor)
|
|
76
|
+
return (mu - np.asarray(values[test_idx], dtype=np.float64)) / sigma
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _zscore_high(values: NDArray[np.float64], train_idx: NDArray[np.int64], test_idx: NDArray[np.int64], floor: float) -> NDArray[np.float64]:
|
|
80
|
+
train = np.asarray(values[train_idx], dtype=np.float64)
|
|
81
|
+
mu = float(np.mean(train)) if train.size else 0.0
|
|
82
|
+
sigma = max(float(np.std(train)) if train.size else 0.0, floor)
|
|
83
|
+
return (np.asarray(values[test_idx], dtype=np.float64) - mu) / sigma
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _rolling_low_zscore(values: NDArray[np.float64], *, window: int, floor: float) -> NDArray[np.float64]:
|
|
87
|
+
arr = np.asarray(values, dtype=np.float64)
|
|
88
|
+
out = np.zeros_like(arr)
|
|
89
|
+
for idx in range(len(arr)):
|
|
90
|
+
start = max(0, idx - window)
|
|
91
|
+
hist = arr[start:idx]
|
|
92
|
+
if hist.size < 2:
|
|
93
|
+
continue
|
|
94
|
+
mu = float(np.mean(hist))
|
|
95
|
+
sigma = max(float(np.std(hist)), floor)
|
|
96
|
+
out[idx] = (mu - float(arr[idx])) / sigma
|
|
97
|
+
return out
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _seasonal_dayofyear_low_zscore(
|
|
101
|
+
values: NDArray[np.float64],
|
|
102
|
+
days: list[date],
|
|
103
|
+
*,
|
|
104
|
+
train_idx: NDArray[np.int64],
|
|
105
|
+
test_idx: NDArray[np.int64],
|
|
106
|
+
floor: float,
|
|
107
|
+
) -> NDArray[np.float64]:
|
|
108
|
+
arr = np.asarray(values, dtype=np.float64)
|
|
109
|
+
by_day: dict[int, list[float]] = {idx: [] for idx in range(1, 367)}
|
|
110
|
+
for idx in train_idx.tolist():
|
|
111
|
+
by_day[days[idx].timetuple().tm_yday].append(float(arr[idx]))
|
|
112
|
+
global_train = arr[train_idx]
|
|
113
|
+
fallback_mu = float(np.mean(global_train)) if global_train.size else 0.0
|
|
114
|
+
fallback_sigma = max(float(np.std(global_train)) if global_train.size else 0.0, floor)
|
|
115
|
+
out = np.zeros(len(test_idx), dtype=np.float64)
|
|
116
|
+
for pos, idx in enumerate(test_idx.tolist()):
|
|
117
|
+
doy = days[idx].timetuple().tm_yday
|
|
118
|
+
bucket_values: list[float] = []
|
|
119
|
+
for neighbor in (doy - 1, doy, doy + 1):
|
|
120
|
+
if neighbor == 0:
|
|
121
|
+
key = 366
|
|
122
|
+
elif neighbor == 367:
|
|
123
|
+
key = 1
|
|
124
|
+
else:
|
|
125
|
+
key = neighbor
|
|
126
|
+
bucket_values.extend(by_day.get(key, []))
|
|
127
|
+
bucket = np.asarray(bucket_values, dtype=np.float64)
|
|
128
|
+
if bucket.size >= 2:
|
|
129
|
+
mu = float(np.mean(bucket))
|
|
130
|
+
sigma = max(float(np.std(bucket)), floor)
|
|
131
|
+
else:
|
|
132
|
+
mu = fallback_mu
|
|
133
|
+
sigma = fallback_sigma
|
|
134
|
+
out[pos] = (mu - float(arr[idx])) / sigma
|
|
135
|
+
return out
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def _build_forward_events(
|
|
139
|
+
solar_7d_mean: NDArray[np.float64],
|
|
140
|
+
*,
|
|
141
|
+
train_idx: NDArray[np.int64],
|
|
142
|
+
test_idx: NDArray[np.int64],
|
|
143
|
+
horizon_days: int,
|
|
144
|
+
threshold_quantile: float,
|
|
145
|
+
) -> NDArray[np.bool_]:
|
|
146
|
+
threshold = float(np.quantile(solar_7d_mean[train_idx], threshold_quantile))
|
|
147
|
+
events = np.zeros(len(test_idx), dtype=bool)
|
|
148
|
+
n = len(solar_7d_mean)
|
|
149
|
+
for pos, idx in enumerate(test_idx.tolist()):
|
|
150
|
+
end = min(n, idx + horizon_days + 1)
|
|
151
|
+
if idx + 1 >= end:
|
|
152
|
+
continue
|
|
153
|
+
events[pos] = bool(np.min(solar_7d_mean[idx + 1 : end]) < threshold)
|
|
154
|
+
return events
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def _walk_forward_splits(n: int, cfg: NasaPowerPreregConfig) -> list[tuple[NDArray[np.int64], NDArray[np.int64]]]:
|
|
158
|
+
splits: list[tuple[NDArray[np.int64], NDArray[np.int64]]] = []
|
|
159
|
+
start = 0
|
|
160
|
+
while start + cfg.train_days + cfg.test_days <= n:
|
|
161
|
+
train = np.arange(start, start + cfg.train_days, dtype=np.int64)
|
|
162
|
+
test = np.arange(
|
|
163
|
+
start + cfg.train_days,
|
|
164
|
+
start + cfg.train_days + cfg.test_days,
|
|
165
|
+
dtype=np.int64,
|
|
166
|
+
)
|
|
167
|
+
splits.append((train, test))
|
|
168
|
+
start += cfg.step_days
|
|
169
|
+
return splits
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def _baseline_scores_for_split(
|
|
173
|
+
*,
|
|
174
|
+
solar_7d_mean: NDArray[np.float64],
|
|
175
|
+
t2m: NDArray[np.float64],
|
|
176
|
+
ws10m: NDArray[np.float64],
|
|
177
|
+
prectotcorr: NDArray[np.float64],
|
|
178
|
+
days: list[date],
|
|
179
|
+
train_idx: NDArray[np.int64],
|
|
180
|
+
test_idx: NDArray[np.int64],
|
|
181
|
+
floor: float,
|
|
182
|
+
) -> Mapping[str, NDArray[np.float64]]:
|
|
183
|
+
rolling_30d = _rolling_low_zscore(solar_7d_mean, window=30, floor=floor)[test_idx]
|
|
184
|
+
rolling_90d = _rolling_low_zscore(solar_7d_mean, window=90, floor=floor)[test_idx]
|
|
185
|
+
shifted = np.roll(solar_7d_mean, 1)
|
|
186
|
+
shifted[0] = solar_7d_mean[0]
|
|
187
|
+
persistence = _zscore_low(shifted, train_idx, test_idx, floor)
|
|
188
|
+
seasonal = _seasonal_dayofyear_low_zscore(
|
|
189
|
+
solar_7d_mean, days, train_idx=train_idx, test_idx=test_idx, floor=floor
|
|
190
|
+
)
|
|
191
|
+
temperature = _zscore_high(t2m, train_idx, test_idx, floor)
|
|
192
|
+
wind = _zscore_high(ws10m, train_idx, test_idx, floor)
|
|
193
|
+
precipitation = _zscore_high(prectotcorr, train_idx, test_idx, floor)
|
|
194
|
+
shuffled = np.asarray(solar_7d_mean[test_idx], dtype=np.float64).copy()
|
|
195
|
+
rng = np.random.default_rng(20260428)
|
|
196
|
+
rng.shuffle(shuffled)
|
|
197
|
+
shuffled = _zscore_low(shuffled, np.arange(len(shuffled), dtype=np.int64), np.arange(len(shuffled), dtype=np.int64), floor)
|
|
198
|
+
reversed_scores = _zscore_low(
|
|
199
|
+
solar_7d_mean[::-1],
|
|
200
|
+
np.arange(min(len(train_idx), len(solar_7d_mean)), dtype=np.int64),
|
|
201
|
+
np.arange(len(test_idx), dtype=np.int64),
|
|
202
|
+
floor,
|
|
203
|
+
)
|
|
204
|
+
no_solar_context = (
|
|
205
|
+
temperature + wind + precipitation
|
|
206
|
+
) / 3.0
|
|
207
|
+
return {
|
|
208
|
+
"rolling_solar_shortfall_zscore_30d": rolling_30d,
|
|
209
|
+
"rolling_solar_shortfall_zscore_90d": rolling_90d,
|
|
210
|
+
"persistence_solar_shortfall": persistence,
|
|
211
|
+
"seasonal_dayofyear_solar_shortfall": seasonal,
|
|
212
|
+
"temperature_anomaly_zscore": temperature,
|
|
213
|
+
"wind_speed_anomaly_zscore": wind,
|
|
214
|
+
"precipitation_anomaly_zscore": precipitation,
|
|
215
|
+
"shuffled_candidate": shuffled,
|
|
216
|
+
"time_reversed_candidate": reversed_scores,
|
|
217
|
+
"candidate_no_solar_context": no_solar_context,
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def _load_point_payload(
|
|
222
|
+
json_path: Path,
|
|
223
|
+
*,
|
|
224
|
+
point_id: str,
|
|
225
|
+
) -> tuple[list[date], dict[str, NDArray[np.float64]], NDArray[np.float64], dict[str, object]]:
|
|
226
|
+
pid = normalize_point_id(point_id)
|
|
227
|
+
days, values, parse_summary = parse_nasa_power_json_file(json_path)
|
|
228
|
+
solar_7d = solar_7d_mean_from_values(values)
|
|
229
|
+
point = NASA_POWER_POINTS[pid]
|
|
230
|
+
source = {
|
|
231
|
+
"point_id": pid,
|
|
232
|
+
"point_name": point.name,
|
|
233
|
+
"point_location_redacted": True,
|
|
234
|
+
"json": json_path.name,
|
|
235
|
+
"json_sha256": _sha256_of_file(json_path),
|
|
236
|
+
"rows": parse_summary["rows"],
|
|
237
|
+
"skipped": parse_summary["skipped"],
|
|
238
|
+
"first_date": parse_summary["first_date"],
|
|
239
|
+
"last_date": parse_summary["last_date"],
|
|
240
|
+
"finite_parameter_coverage": 1.0,
|
|
241
|
+
}
|
|
242
|
+
return days, values, solar_7d, source
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
def evaluate_nasa_power_prereg(
|
|
246
|
+
*,
|
|
247
|
+
point_ids: list[str],
|
|
248
|
+
json_paths: list[Path],
|
|
249
|
+
config: NasaPowerPreregConfig | None = None,
|
|
250
|
+
pre_registration_doc_path: Path | None = None,
|
|
251
|
+
) -> dict[str, object]:
|
|
252
|
+
if len(point_ids) != len(json_paths):
|
|
253
|
+
raise ValueError("point_ids and json_paths counts must match")
|
|
254
|
+
cfg = config or NasaPowerPreregConfig()
|
|
255
|
+
|
|
256
|
+
observations: list[WindowObservation] = []
|
|
257
|
+
point_reports: list[dict[str, object]] = []
|
|
258
|
+
source_audit_passed = True
|
|
259
|
+
window_id = 0
|
|
260
|
+
for point_id, json_path in zip(point_ids, json_paths, strict=True):
|
|
261
|
+
days, values, solar_7d, source = _load_point_payload(Path(json_path), point_id=point_id)
|
|
262
|
+
point = NASA_POWER_POINTS[normalize_point_id(point_id)]
|
|
263
|
+
point_windows: list[dict[str, object]] = []
|
|
264
|
+
coverage_ok = bool(int(source["rows"]) >= int(0.95 * len(days)))
|
|
265
|
+
source_audit_passed = source_audit_passed and coverage_ok
|
|
266
|
+
for train_idx, test_idx in _walk_forward_splits(len(solar_7d), cfg):
|
|
267
|
+
events = _build_forward_events(
|
|
268
|
+
solar_7d,
|
|
269
|
+
train_idx=train_idx,
|
|
270
|
+
test_idx=test_idx,
|
|
271
|
+
horizon_days=cfg.horizon_days,
|
|
272
|
+
threshold_quantile=cfg.threshold_quantile,
|
|
273
|
+
)
|
|
274
|
+
event_count = int(np.sum(events))
|
|
275
|
+
event_rate = float(np.mean(events)) if len(events) else None
|
|
276
|
+
eligible = event_count >= cfg.minimum_test_event_count
|
|
277
|
+
point_windows.append(
|
|
278
|
+
{
|
|
279
|
+
"window_id": window_id,
|
|
280
|
+
"rows": int(len(test_idx)),
|
|
281
|
+
"events": event_count,
|
|
282
|
+
"event_rate": event_rate,
|
|
283
|
+
"eligible": bool(eligible),
|
|
284
|
+
}
|
|
285
|
+
)
|
|
286
|
+
if eligible:
|
|
287
|
+
candidate = _zscore_low(solar_7d, train_idx, test_idx, cfg.solar_sigma_floor)
|
|
288
|
+
baselines = _baseline_scores_for_split(
|
|
289
|
+
solar_7d_mean=solar_7d,
|
|
290
|
+
t2m=values["T2M"],
|
|
291
|
+
ws10m=values["WS10M"],
|
|
292
|
+
prectotcorr=values["PRECTOTCORR"],
|
|
293
|
+
days=days,
|
|
294
|
+
train_idx=train_idx,
|
|
295
|
+
test_idx=test_idx,
|
|
296
|
+
floor=cfg.solar_sigma_floor,
|
|
297
|
+
)
|
|
298
|
+
observations.append(
|
|
299
|
+
WindowObservation(
|
|
300
|
+
candidate_scores=candidate,
|
|
301
|
+
baseline_scores=baselines,
|
|
302
|
+
events=events,
|
|
303
|
+
window_id=window_id,
|
|
304
|
+
)
|
|
305
|
+
)
|
|
306
|
+
window_id += 1
|
|
307
|
+
point_reports.append(
|
|
308
|
+
{
|
|
309
|
+
"point_id": point.point_id,
|
|
310
|
+
"point_name": point.name,
|
|
311
|
+
"point_location_redacted": True,
|
|
312
|
+
"source": source,
|
|
313
|
+
"coverage_ok": coverage_ok,
|
|
314
|
+
"windows": point_windows,
|
|
315
|
+
}
|
|
316
|
+
)
|
|
317
|
+
|
|
318
|
+
acceptance = evaluate_group_vs_baseline_pool(observations, config=cfg.acceptance)
|
|
319
|
+
eligible_point_count = sum(1 for point in point_reports if point["coverage_ok"])
|
|
320
|
+
eligible_windows = len(observations)
|
|
321
|
+
pre_reg_hash: str | None = None
|
|
322
|
+
if pre_registration_doc_path is not None and pre_registration_doc_path.is_file():
|
|
323
|
+
pre_reg_hash = _sha256_of_text(pre_registration_doc_path.read_text(encoding="utf-8"))
|
|
324
|
+
source_audit_passed = source_audit_passed and eligible_point_count >= 2
|
|
325
|
+
overall = (
|
|
326
|
+
source_audit_passed
|
|
327
|
+
and eligible_windows >= 8
|
|
328
|
+
and bool(acceptance.get("overall_go_no_go", False))
|
|
329
|
+
)
|
|
330
|
+
result_status = "PASSED_UNDER_PROTOCOL" if overall else "NEGATIVE_RESULT"
|
|
331
|
+
if not source_audit_passed:
|
|
332
|
+
result_status = "BLOCKED_SOURCE"
|
|
333
|
+
return {
|
|
334
|
+
"schema": NASA_POWER_PREREG_REPORT_SCHEMA,
|
|
335
|
+
"candidate": NASA_POWER_CANDIDATE_NAME,
|
|
336
|
+
"pre_registration": {
|
|
337
|
+
"doc": "NASA_POWER_D103_PREREG_CHARTER.md",
|
|
338
|
+
"doc_sha256": pre_reg_hash,
|
|
339
|
+
"config": {
|
|
340
|
+
"horizon_days": cfg.horizon_days,
|
|
341
|
+
"threshold_quantile": cfg.threshold_quantile,
|
|
342
|
+
"trailing_window_days": cfg.trailing_window_days,
|
|
343
|
+
"train_days": cfg.train_days,
|
|
344
|
+
"test_days": cfg.test_days,
|
|
345
|
+
"step_days": cfg.step_days,
|
|
346
|
+
"minimum_test_event_count": cfg.minimum_test_event_count,
|
|
347
|
+
"top_rate": cfg.acceptance.top_rate,
|
|
348
|
+
"min_positive_rate": cfg.acceptance.min_positive_rate,
|
|
349
|
+
},
|
|
350
|
+
},
|
|
351
|
+
"reproducibility": build_generic_ohlcv_batch_reproducibility(),
|
|
352
|
+
"source_audit_passed": bool(source_audit_passed),
|
|
353
|
+
"eligible_point_count": int(eligible_point_count),
|
|
354
|
+
"eligible_windows": int(eligible_windows),
|
|
355
|
+
"points": point_reports,
|
|
356
|
+
"acceptance": acceptance,
|
|
357
|
+
"overall_go_no_go": bool(overall),
|
|
358
|
+
"result_status": result_status,
|
|
359
|
+
"allowed_narrow_claim": (
|
|
360
|
+
"passed the pre-registered NASA POWER D-103 gate under protocol 1.0.143"
|
|
361
|
+
if overall
|
|
362
|
+
else None
|
|
363
|
+
),
|
|
364
|
+
"forbidden_claims": [
|
|
365
|
+
"broad or universal forecasting performance",
|
|
366
|
+
"deployment readiness",
|
|
367
|
+
"energy-grid operational readiness",
|
|
368
|
+
"non-public background performance claim",
|
|
369
|
+
],
|
|
370
|
+
"honest_boundary": (
|
|
371
|
+
"This report evaluates local NASA POWER JSON payloads under the frozen "
|
|
372
|
+
"D-103 protocol. A positive claim requires overall_go_no_go=true; "
|
|
373
|
+
"otherwise the result is negative or source-blocked evidence."
|
|
374
|
+
),
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
|
|
378
|
+
__all__ = [
|
|
379
|
+
"NASA_POWER_CANDIDATE_NAME",
|
|
380
|
+
"NASA_POWER_PREREG_REPORT_SCHEMA",
|
|
381
|
+
"NasaPowerPreregConfig",
|
|
382
|
+
"evaluate_nasa_power_prereg",
|
|
383
|
+
]
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
"""Download NOAA CO-OPS D-131 payloads in API-compliant chunks.
|
|
3
|
+
|
|
4
|
+
NOAA's Data API limits many products (including hourly_height) to 365 days per
|
|
5
|
+
request. D-131 spans 2018-01-01..2024-12-31, so operators must fetch by year (or
|
|
6
|
+
smaller windows) and merge payloads locally outside the repository.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
import urllib.error
|
|
13
|
+
import urllib.parse
|
|
14
|
+
import urllib.request
|
|
15
|
+
from dataclasses import dataclass
|
|
16
|
+
from datetime import date
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
from collections.abc import Callable
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
NOAA_COOPS_D131_STATIONS: tuple[str, ...] = ("8518750", "9414290", "8638610")
|
|
22
|
+
NOAA_COOPS_D131_BEGIN = date(2018, 1, 1)
|
|
23
|
+
NOAA_COOPS_D131_END = date(2024, 12, 31)
|
|
24
|
+
NOAA_COOPS_API_BASE = "https://api.tidesandcurrents.noaa.gov/api/prod/datagetter"
|
|
25
|
+
NOAA_COOPS_APPLICATION = "NOS.COOPS.TAC.WL"
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@dataclass(frozen=True)
|
|
29
|
+
class NoaaCoopsFetchConfig:
|
|
30
|
+
station: str
|
|
31
|
+
product: str
|
|
32
|
+
begin: date
|
|
33
|
+
end: date
|
|
34
|
+
datum: str = "MLLW"
|
|
35
|
+
time_zone: str = "gmt"
|
|
36
|
+
units: str = "metric"
|
|
37
|
+
interval: str | None = None
|
|
38
|
+
application: str = NOAA_COOPS_APPLICATION
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def year_chunks(begin: date, end: date) -> list[tuple[date, date]]:
|
|
42
|
+
if begin > end:
|
|
43
|
+
raise ValueError("begin must be <= end")
|
|
44
|
+
|
|
45
|
+
chunks: list[tuple[date, date]] = []
|
|
46
|
+
for year in range(begin.year, end.year + 1):
|
|
47
|
+
chunk_start = date(year, 1, 1) if year > begin.year else begin
|
|
48
|
+
chunk_end = date(year, 12, 31) if year < end.year else end
|
|
49
|
+
chunks.append((chunk_start, chunk_end))
|
|
50
|
+
return chunks
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def build_datagetter_url(config: NoaaCoopsFetchConfig, chunk_begin: date, chunk_end: date) -> str:
|
|
54
|
+
params: dict[str, str] = {
|
|
55
|
+
"product": config.product,
|
|
56
|
+
"application": config.application,
|
|
57
|
+
"begin_date": chunk_begin.strftime("%Y%m%d"),
|
|
58
|
+
"end_date": chunk_end.strftime("%Y%m%d"),
|
|
59
|
+
"datum": config.datum,
|
|
60
|
+
"station": config.station,
|
|
61
|
+
"time_zone": config.time_zone,
|
|
62
|
+
"units": config.units,
|
|
63
|
+
"format": "json",
|
|
64
|
+
}
|
|
65
|
+
if config.product == "predictions":
|
|
66
|
+
params["interval"] = config.interval or "h"
|
|
67
|
+
query = urllib.parse.urlencode(params)
|
|
68
|
+
return f"{NOAA_COOPS_API_BASE}?{query}"
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def merge_hourly_height_payloads(payloads: list[dict[str, object]]) -> dict[str, object]:
|
|
72
|
+
if not payloads:
|
|
73
|
+
raise ValueError("hourly_height payloads must not be empty")
|
|
74
|
+
|
|
75
|
+
merged_data: list[object] = []
|
|
76
|
+
metadata: object | None = None
|
|
77
|
+
for payload in payloads:
|
|
78
|
+
if "error" in payload:
|
|
79
|
+
raise ValueError(str(payload["error"]))
|
|
80
|
+
if metadata is None and "metadata" in payload:
|
|
81
|
+
metadata = payload["metadata"]
|
|
82
|
+
data = payload.get("data")
|
|
83
|
+
if not isinstance(data, list):
|
|
84
|
+
raise ValueError("hourly_height payload missing data list")
|
|
85
|
+
merged_data.extend(data)
|
|
86
|
+
|
|
87
|
+
out: dict[str, object] = {"data": merged_data}
|
|
88
|
+
if metadata is not None:
|
|
89
|
+
out["metadata"] = metadata
|
|
90
|
+
return out
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def merge_predictions_payloads(payloads: list[dict[str, object]]) -> dict[str, object]:
|
|
94
|
+
if not payloads:
|
|
95
|
+
raise ValueError("predictions payloads must not be empty")
|
|
96
|
+
|
|
97
|
+
merged_predictions: list[object] = []
|
|
98
|
+
for payload in payloads:
|
|
99
|
+
if "error" in payload:
|
|
100
|
+
raise ValueError(str(payload["error"]))
|
|
101
|
+
predictions = payload.get("predictions")
|
|
102
|
+
if not isinstance(predictions, list):
|
|
103
|
+
raise ValueError("predictions payload missing predictions list")
|
|
104
|
+
merged_predictions.extend(predictions)
|
|
105
|
+
|
|
106
|
+
return {"predictions": merged_predictions}
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def merge_product_payloads(product: str, payloads: list[dict[str, object]]) -> dict[str, object]:
|
|
110
|
+
if product == "hourly_height":
|
|
111
|
+
return merge_hourly_height_payloads(payloads)
|
|
112
|
+
if product == "predictions":
|
|
113
|
+
return merge_predictions_payloads(payloads)
|
|
114
|
+
raise ValueError(f"unsupported NOAA product {product!r}")
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def default_fetch_json(url: str) -> dict[str, object]:
|
|
118
|
+
request = urllib.request.Request(url, headers={"User-Agent": "Protocolith-noaa-fetch/0.1"})
|
|
119
|
+
try:
|
|
120
|
+
with urllib.request.urlopen(request, timeout=120) as response:
|
|
121
|
+
body = response.read()
|
|
122
|
+
except urllib.error.HTTPError as exc:
|
|
123
|
+
detail = exc.read().decode("utf-8", errors="replace")
|
|
124
|
+
raise RuntimeError(f"NOAA HTTP {exc.code} for {url}: {detail}") from exc
|
|
125
|
+
|
|
126
|
+
payload = json.loads(body.decode("utf-8"))
|
|
127
|
+
if not isinstance(payload, dict):
|
|
128
|
+
raise RuntimeError(f"NOAA response is not a JSON object: {url}")
|
|
129
|
+
return payload
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def fetch_product_payload(
|
|
133
|
+
config: NoaaCoopsFetchConfig,
|
|
134
|
+
*,
|
|
135
|
+
fetch_json: Callable[[str], dict[str, object]] = default_fetch_json,
|
|
136
|
+
) -> dict[str, object]:
|
|
137
|
+
chunk_payloads: list[dict[str, object]] = []
|
|
138
|
+
for chunk_begin, chunk_end in year_chunks(config.begin, config.end):
|
|
139
|
+
url = build_datagetter_url(config, chunk_begin, chunk_end)
|
|
140
|
+
chunk_payloads.append(fetch_json(url))
|
|
141
|
+
return merge_product_payloads(config.product, chunk_payloads)
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def write_json(path: Path, payload: dict[str, object]) -> Path:
|
|
145
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
146
|
+
path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
|
147
|
+
return path
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def fetch_d131_station_payloads(
|
|
151
|
+
station: str,
|
|
152
|
+
out_dir: Path,
|
|
153
|
+
*,
|
|
154
|
+
begin: date = NOAA_COOPS_D131_BEGIN,
|
|
155
|
+
end: date = NOAA_COOPS_D131_END,
|
|
156
|
+
fetch_json: Callable[[str], dict[str, object]] = default_fetch_json,
|
|
157
|
+
) -> list[Path]:
|
|
158
|
+
written: list[Path] = []
|
|
159
|
+
for product, suffix in (("hourly_height", "observed"), ("predictions", "predictions")):
|
|
160
|
+
payload = fetch_product_payload(
|
|
161
|
+
NoaaCoopsFetchConfig(
|
|
162
|
+
station=station,
|
|
163
|
+
product=product,
|
|
164
|
+
begin=begin,
|
|
165
|
+
end=end,
|
|
166
|
+
),
|
|
167
|
+
fetch_json=fetch_json,
|
|
168
|
+
)
|
|
169
|
+
written.append(write_json(out_dir / f"{station}_{suffix}.json", payload))
|
|
170
|
+
return written
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
__all__ = [
|
|
174
|
+
"NOAA_COOPS_API_BASE",
|
|
175
|
+
"NOAA_COOPS_D131_BEGIN",
|
|
176
|
+
"NOAA_COOPS_D131_END",
|
|
177
|
+
"NOAA_COOPS_D131_STATIONS",
|
|
178
|
+
"NoaaCoopsFetchConfig",
|
|
179
|
+
"build_datagetter_url",
|
|
180
|
+
"default_fetch_json",
|
|
181
|
+
"fetch_d131_station_payloads",
|
|
182
|
+
"fetch_product_payload",
|
|
183
|
+
"merge_hourly_height_payloads",
|
|
184
|
+
"merge_predictions_payloads",
|
|
185
|
+
"merge_product_payloads",
|
|
186
|
+
"write_json",
|
|
187
|
+
"year_chunks",
|
|
188
|
+
]
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
|
|
3
|
+
"""Parse and join NOAA CO-OPS D-131 local JSON payloads."""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
import hashlib
|
|
8
|
+
import json
|
|
9
|
+
from datetime import datetime
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
|
|
12
|
+
import numpy as np
|
|
13
|
+
from numpy.typing import NDArray
|
|
14
|
+
|
|
15
|
+
from protocolith.noaa_coops_fetch import NOAA_COOPS_D131_STATIONS
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def sha256_of_file(path: Path) -> str:
|
|
19
|
+
digest = hashlib.sha256()
|
|
20
|
+
with path.open("rb") as fh:
|
|
21
|
+
for chunk in iter(lambda: fh.read(64 * 1024), b""):
|
|
22
|
+
digest.update(chunk)
|
|
23
|
+
return digest.hexdigest()
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def parse_noaa_coops_series(path: Path) -> tuple[list[datetime], NDArray[np.float64], dict[str, object]]:
|
|
27
|
+
"""Parse one NOAA CO-OPS hourly_height or predictions JSON file."""
|
|
28
|
+
|
|
29
|
+
payload = json.loads(path.read_text(encoding="utf-8"))
|
|
30
|
+
if "error" in payload:
|
|
31
|
+
raise ValueError(str(payload["error"]))
|
|
32
|
+
|
|
33
|
+
rows = payload.get("data")
|
|
34
|
+
if rows is None:
|
|
35
|
+
rows = payload.get("predictions")
|
|
36
|
+
if not isinstance(rows, list):
|
|
37
|
+
raise ValueError(f"{path.name}: expected data or predictions list")
|
|
38
|
+
|
|
39
|
+
timestamps: list[datetime] = []
|
|
40
|
+
values: list[float] = []
|
|
41
|
+
skipped = 0
|
|
42
|
+
for row in rows:
|
|
43
|
+
if not isinstance(row, dict):
|
|
44
|
+
skipped += 1
|
|
45
|
+
continue
|
|
46
|
+
raw_value = row.get("v")
|
|
47
|
+
if raw_value in (None, "", " "):
|
|
48
|
+
skipped += 1
|
|
49
|
+
continue
|
|
50
|
+
timestamps.append(datetime.strptime(str(row["t"]), "%Y-%m-%d %H:%M"))
|
|
51
|
+
values.append(float(raw_value))
|
|
52
|
+
|
|
53
|
+
if not timestamps:
|
|
54
|
+
raise ValueError(f"{path.name}: no finite observations")
|
|
55
|
+
|
|
56
|
+
return (
|
|
57
|
+
timestamps,
|
|
58
|
+
np.asarray(values, dtype=np.float64),
|
|
59
|
+
{
|
|
60
|
+
"json": path.name,
|
|
61
|
+
"json_sha256": sha256_of_file(path),
|
|
62
|
+
"rows": len(rows),
|
|
63
|
+
"finite_rows": len(values),
|
|
64
|
+
"skipped": skipped,
|
|
65
|
+
"first_timestamp": timestamps[0].isoformat(sep=" "),
|
|
66
|
+
"last_timestamp": timestamps[-1].isoformat(sep=" "),
|
|
67
|
+
},
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def join_observed_predictions(
|
|
72
|
+
observed_path: Path,
|
|
73
|
+
predictions_path: Path,
|
|
74
|
+
) -> tuple[list[datetime], NDArray[np.float64], NDArray[np.float64], NDArray[np.float64], dict[str, object]]:
|
|
75
|
+
"""Join observed and prediction series on timestamp."""
|
|
76
|
+
|
|
77
|
+
obs_ts, obs_values, obs_summary = parse_noaa_coops_series(observed_path)
|
|
78
|
+
pred_ts, pred_values, pred_summary = parse_noaa_coops_series(predictions_path)
|
|
79
|
+
|
|
80
|
+
obs_map = {ts: value for ts, value in zip(obs_ts, obs_values, strict=True)}
|
|
81
|
+
pred_map = {ts: value for ts, value in zip(pred_ts, pred_values, strict=True)}
|
|
82
|
+
shared = sorted(set(obs_map) & set(pred_map))
|
|
83
|
+
if not shared:
|
|
84
|
+
raise ValueError(
|
|
85
|
+
f"no shared timestamps between {observed_path.name} and {predictions_path.name}"
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
observed = np.asarray([obs_map[ts] for ts in shared], dtype=np.float64)
|
|
89
|
+
predicted = np.asarray([pred_map[ts] for ts in shared], dtype=np.float64)
|
|
90
|
+
residual = observed - predicted
|
|
91
|
+
return (
|
|
92
|
+
shared,
|
|
93
|
+
observed,
|
|
94
|
+
predicted,
|
|
95
|
+
residual,
|
|
96
|
+
{
|
|
97
|
+
"observed": obs_summary,
|
|
98
|
+
"predictions": pred_summary,
|
|
99
|
+
"joined_rows": len(shared),
|
|
100
|
+
"observed_sha256": obs_summary["json_sha256"],
|
|
101
|
+
"predictions_sha256": pred_summary["json_sha256"],
|
|
102
|
+
},
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def default_station_paths(raw_dir: Path, station: str) -> tuple[Path, Path]:
|
|
107
|
+
if station not in NOAA_COOPS_D131_STATIONS:
|
|
108
|
+
raise ValueError(f"unknown D-131 station: {station!r}")
|
|
109
|
+
return (
|
|
110
|
+
raw_dir / f"{station}_observed.json",
|
|
111
|
+
raw_dir / f"{station}_predictions.json",
|
|
112
|
+
)
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
__all__ = [
|
|
116
|
+
"default_station_paths",
|
|
117
|
+
"join_observed_predictions",
|
|
118
|
+
"parse_noaa_coops_series",
|
|
119
|
+
"sha256_of_file",
|
|
120
|
+
]
|