protocolith 0.6.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. protocolith/__init__.py +38 -0
  2. protocolith/baseline_groups.py +294 -0
  3. protocolith/baselines.py +288 -0
  4. protocolith/card_svg_render.py +281 -0
  5. protocolith/card_svg_styles.py +178 -0
  6. protocolith/cli.py +796 -0
  7. protocolith/core/__init__.py +63 -0
  8. protocolith/core/_lock_posix.py +29 -0
  9. protocolith/core/_lock_windows.py +29 -0
  10. protocolith/core/atlas.py +406 -0
  11. protocolith/core/atlas_assets/atlas.css +86 -0
  12. protocolith/core/atlas_assets/atlas.html +84 -0
  13. protocolith/core/atlas_assets/atlas.js +370 -0
  14. protocolith/core/atlas_i18n/en.json +61 -0
  15. protocolith/core/canonical.py +124 -0
  16. protocolith/core/capture.py +68 -0
  17. protocolith/core/clock.py +25 -0
  18. protocolith/core/commands.py +356 -0
  19. protocolith/core/errors.py +116 -0
  20. protocolith/core/issue.py +98 -0
  21. protocolith/core/ledger.py +75 -0
  22. protocolith/core/mcp.py +371 -0
  23. protocolith/core/ops.py +183 -0
  24. protocolith/core/rules.py +352 -0
  25. protocolith/core/schema.py +137 -0
  26. protocolith/core/store.py +101 -0
  27. protocolith/core/views.py +136 -0
  28. protocolith/doctor.py +135 -0
  29. protocolith/evidence_card.py +354 -0
  30. protocolith/family_ledger.py +489 -0
  31. protocolith/generic_ohlcv_batch_pre_registration_meta.py +123 -0
  32. protocolith/inspect.py +48 -0
  33. protocolith/nasa_power.py +266 -0
  34. protocolith/nasa_power_fetch.py +64 -0
  35. protocolith/nasa_power_parser.py +133 -0
  36. protocolith/nasa_power_prereg_runner.py +383 -0
  37. protocolith/noaa_coops_fetch.py +188 -0
  38. protocolith/noaa_coops_parser.py +120 -0
  39. protocolith/noaa_coops_prereg_runner.py +341 -0
  40. protocolith/public_ai_protocol.py +176 -0
  41. protocolith/py.typed +0 -0
  42. protocolith/registry.py +139 -0
  43. protocolith/registry_view.py +302 -0
  44. protocolith/rerun_report.py +111 -0
  45. protocolith/run_root.py +243 -0
  46. protocolith/scaffold.py +565 -0
  47. protocolith/source_probe.py +230 -0
  48. protocolith/svg_a11y.py +100 -0
  49. protocolith/tree_overlay.py +343 -0
  50. protocolith/verify_packs.py +251 -0
  51. protocolith/workflows.py +233 -0
  52. protocolith-0.6.0.dist-info/METADATA +283 -0
  53. protocolith-0.6.0.dist-info/RECORD +57 -0
  54. protocolith-0.6.0.dist-info/WHEEL +4 -0
  55. protocolith-0.6.0.dist-info/entry_points.txt +3 -0
  56. protocolith-0.6.0.dist-info/licenses/LICENSE +201 -0
  57. protocolith-0.6.0.dist-info/licenses/NOTICE +13 -0
@@ -0,0 +1,341 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+
3
+ """NOAA CO-OPS D-131 frozen pre-registration evaluator.
4
+
5
+ Evaluates operator-supplied local NOAA CO-OPS JSON files under protocol 1.0.139.
6
+ Performs no network I/O and does not change gates.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import hashlib
12
+ from dataclasses import dataclass, field
13
+ from datetime import datetime
14
+ from pathlib import Path
15
+
16
+ import numpy as np
17
+ from numpy.typing import NDArray
18
+
19
+ from protocolith.baseline_groups import (
20
+ Doc11AcceptanceConfig,
21
+ WindowObservation,
22
+ evaluate_group_vs_baseline_pool,
23
+ )
24
+ from protocolith.noaa_coops_fetch import NOAA_COOPS_D131_STATIONS
25
+ from protocolith.noaa_coops_parser import join_observed_predictions, sha256_of_file
26
+
27
+ NOAA_COOPS_PREREG_REPORT_SCHEMA = "noaa_coops_prereg_report_v1"
28
+ NOAA_COOPS_NEGATIVE_SUMMARY_SCHEMA = "protocolith_negative_result_summary_v1"
29
+ NOAA_COOPS_CANDIDATE_NAME = "noaa_coops_d131_abs_prediction_residual"
30
+ NOAA_COOPS_PROTOCOL_VERSION = "1.0.139"
31
+ HOURS_PER_YEAR = 365 * 24
32
+ # Shortest committed D-131 station row count (9414290) from the baseline summary.
33
+ NOAA_COOPS_MIN_JOINED_ROWS = 56424
34
+
35
+
36
+ @dataclass(frozen=True)
37
+ class NoaaCoopsPreregConfig:
38
+ train_hours: int = 3 * HOURS_PER_YEAR
39
+ test_hours: int = HOURS_PER_YEAR
40
+ step_hours: int = HOURS_PER_YEAR
41
+ event_quantile: float = 0.90
42
+ minimum_test_event_count: int = 15
43
+ min_joined_rows: int = int(0.95 * NOAA_COOPS_MIN_JOINED_ROWS)
44
+ acceptance: Doc11AcceptanceConfig = field(
45
+ default_factory=lambda: Doc11AcceptanceConfig(
46
+ top_rate=0.10,
47
+ min_windows=3,
48
+ min_positive_rate=0.65,
49
+ min_event_rate=0.03,
50
+ max_event_rate=0.40,
51
+ rng_seed=20260430,
52
+ )
53
+ )
54
+
55
+
56
+ def _sha256_of_text(text: str) -> str:
57
+ return hashlib.sha256(text.encode("utf-8")).hexdigest()
58
+
59
+
60
+ def _hour_of_year_index(timestamp: datetime) -> int:
61
+ return (timestamp.timetuple().tm_yday - 1) * 24 + timestamp.hour
62
+
63
+
64
+ def _seasonal_residual_scores(
65
+ residual: NDArray[np.float64],
66
+ timestamps: list[datetime],
67
+ *,
68
+ train_idx: NDArray[np.int64],
69
+ test_idx: NDArray[np.int64],
70
+ ) -> NDArray[np.float64]:
71
+ by_hour: dict[int, list[float]] = {}
72
+ for idx in train_idx.tolist():
73
+ by_hour.setdefault(_hour_of_year_index(timestamps[idx]), []).append(float(residual[idx]))
74
+ global_mean = float(np.mean(residual[train_idx]))
75
+ scores = np.zeros(len(test_idx), dtype=np.float64)
76
+ for pos, idx in enumerate(test_idx.tolist()):
77
+ bucket = by_hour.get(_hour_of_year_index(timestamps[idx]), [])
78
+ if len(bucket) >= 2:
79
+ expected = float(np.mean(bucket))
80
+ else:
81
+ expected = global_mean
82
+ scores[pos] = abs(float(residual[idx]) - expected)
83
+ return scores
84
+
85
+
86
+ def _persistence_residual_scores(
87
+ residual: NDArray[np.float64],
88
+ test_idx: NDArray[np.int64],
89
+ ) -> NDArray[np.float64]:
90
+ shifted = np.roll(residual, 1)
91
+ shifted[0] = residual[0]
92
+ return np.abs(shifted[test_idx])
93
+
94
+
95
+ def _walk_forward_splits(n: int, cfg: NoaaCoopsPreregConfig) -> list[tuple[NDArray[np.int64], NDArray[np.int64]]]:
96
+ splits: list[tuple[NDArray[np.int64], NDArray[np.int64]]] = []
97
+ start = 0
98
+ while start + cfg.train_hours + cfg.test_hours <= n:
99
+ train = np.arange(start, start + cfg.train_hours, dtype=np.int64)
100
+ test = np.arange(start + cfg.train_hours, start + cfg.train_hours + cfg.test_hours, dtype=np.int64)
101
+ splits.append((train, test))
102
+ start += cfg.step_hours
103
+ return splits
104
+
105
+
106
+ def derive_failed_gates(acceptance: dict[str, object], *, cfg: NoaaCoopsPreregConfig) -> list[str]:
107
+ failed: list[str] = []
108
+ baseline_acceptance = acceptance.get("baseline_acceptance", {})
109
+ if isinstance(baseline_acceptance, dict):
110
+ seasonal = baseline_acceptance.get("seasonal_hour_of_year_residual", {})
111
+ if isinstance(seasonal, dict) and not seasonal.get("meets_paired_gate", False):
112
+ failed.append("seasonal hour-of-year residual control was not beaten")
113
+
114
+ for record in acceptance.get("windows", []):
115
+ if not isinstance(record, dict) or record.get("event_rate_in_range") is not False:
116
+ continue
117
+ rate = record.get("event_rate")
118
+ if not isinstance(rate, (int, float)):
119
+ continue
120
+ if rate < cfg.acceptance.min_event_rate and "event rate below minimum in one window" not in failed:
121
+ failed.append("event rate below minimum in one window")
122
+ if rate > cfg.acceptance.max_event_rate and "event rate above maximum in one window" not in failed:
123
+ failed.append("event rate above maximum in one window")
124
+ return failed
125
+
126
+
127
+ def build_negative_result_summary(
128
+ report: dict[str, object],
129
+ *,
130
+ report_path: Path,
131
+ ) -> dict[str, object]:
132
+ inputs = report.get("inputs", [])
133
+ result = report.get("result", {})
134
+ acceptance = report.get("acceptance", {})
135
+ if not isinstance(inputs, list) or not isinstance(result, dict):
136
+ raise ValueError("report missing inputs or result")
137
+ if not isinstance(acceptance, dict):
138
+ acceptance = {}
139
+
140
+ cfg = report.get("config")
141
+ protocol_version = NOAA_COOPS_PROTOCOL_VERSION
142
+ if isinstance(cfg, dict):
143
+ protocol_version = str(cfg.get("protocol_version", protocol_version))
144
+
145
+ return {
146
+ "schema": NOAA_COOPS_NEGATIVE_SUMMARY_SCHEMA,
147
+ "domain": "NOAA_COOPS_D131",
148
+ "protocol_version": protocol_version,
149
+ "source": report.get("source", {}),
150
+ "inputs": inputs,
151
+ "report_sha256": sha256_of_file(report_path),
152
+ "external_payload_manifest_sha256": report.get("external_payload_manifest_sha256"),
153
+ "result": result,
154
+ "failed_gates": report.get("failed_gates", []),
155
+ "claim_boundary": {
156
+ "allowed": "executed under a frozen protocol and recorded as NEGATIVE_RESULT_UNDER_PROTOCOL",
157
+ "forbidden": [
158
+ "accepted by protocol",
159
+ "coastal-warning performance proven",
160
+ "universal forecasting edge",
161
+ "deployment readiness",
162
+ "all statistical methods beaten",
163
+ ],
164
+ },
165
+ }
166
+
167
+
168
+ def evaluate_noaa_coops_prereg(
169
+ *,
170
+ station_ids: list[str],
171
+ observed_paths: list[Path],
172
+ predictions_paths: list[Path],
173
+ config: NoaaCoopsPreregConfig | None = None,
174
+ pre_registration_doc_path: Path | None = None,
175
+ external_payload_manifest_sha256: str | None = None,
176
+ ) -> dict[str, object]:
177
+ if not (len(station_ids) == len(observed_paths) == len(predictions_paths)):
178
+ raise ValueError("station_ids, observed_paths and predictions_paths counts must match")
179
+
180
+ cfg = config or NoaaCoopsPreregConfig()
181
+ observations: list[WindowObservation] = []
182
+ station_reports: list[dict[str, object]] = []
183
+ input_summaries: list[dict[str, object]] = []
184
+ source_audit_passed = True
185
+ window_id = 0
186
+
187
+ for station_id, observed_path, predictions_path in zip(
188
+ station_ids, observed_paths, predictions_paths, strict=True
189
+ ):
190
+ timestamps, _observed, _predicted, residual, join_summary = join_observed_predictions(
191
+ Path(observed_path),
192
+ Path(predictions_path),
193
+ )
194
+ coverage_ok = int(join_summary["joined_rows"]) >= cfg.min_joined_rows
195
+ source_audit_passed = source_audit_passed and coverage_ok
196
+ station_windows: list[dict[str, object]] = []
197
+
198
+ for train_idx, test_idx in _walk_forward_splits(len(residual), cfg):
199
+ threshold = float(np.quantile(np.abs(residual[train_idx]), cfg.event_quantile))
200
+ events = np.abs(residual[test_idx]) >= threshold
201
+ event_count = int(np.sum(events))
202
+ event_rate = float(np.mean(events)) if len(events) else None
203
+ eligible = event_count >= cfg.minimum_test_event_count
204
+ station_windows.append(
205
+ {
206
+ "window_id": window_id,
207
+ "rows": int(len(test_idx)),
208
+ "events": event_count,
209
+ "event_rate": event_rate,
210
+ "eligible": bool(eligible),
211
+ }
212
+ )
213
+ if eligible:
214
+ observations.append(
215
+ WindowObservation(
216
+ candidate_scores=np.abs(residual[test_idx]),
217
+ baseline_scores={
218
+ "persistence_residual": _persistence_residual_scores(residual, test_idx),
219
+ "seasonal_hour_of_year_residual": _seasonal_residual_scores(
220
+ residual,
221
+ timestamps,
222
+ train_idx=train_idx,
223
+ test_idx=test_idx,
224
+ ),
225
+ },
226
+ events=events,
227
+ window_id=window_id,
228
+ )
229
+ )
230
+ window_id += 1
231
+
232
+ input_summaries.append(
233
+ {
234
+ "station": station_id,
235
+ "observed_sha256": join_summary["observed_sha256"],
236
+ "predictions_sha256": join_summary["predictions_sha256"],
237
+ "joined_rows": join_summary["joined_rows"],
238
+ }
239
+ )
240
+ station_reports.append(
241
+ {
242
+ "station_id": station_id,
243
+ "coverage_ok": coverage_ok,
244
+ "join": join_summary,
245
+ "windows": station_windows,
246
+ }
247
+ )
248
+
249
+ acceptance = evaluate_group_vs_baseline_pool(observations, config=cfg.acceptance)
250
+ eligible_station_count = sum(1 for station in station_reports if station["coverage_ok"])
251
+ eligible_windows = len(observations)
252
+ failed_gates = derive_failed_gates(acceptance, cfg=cfg)
253
+
254
+ pre_reg_hash: str | None = None
255
+ if pre_registration_doc_path is not None and pre_registration_doc_path.is_file():
256
+ pre_reg_hash = _sha256_of_text(pre_registration_doc_path.read_text(encoding="utf-8"))
257
+
258
+ source_audit_passed = source_audit_passed and eligible_station_count >= 3
259
+ gate_passed = bool(acceptance.get("overall_go_no_go", False))
260
+ overall = source_audit_passed and eligible_windows >= cfg.acceptance.min_windows and gate_passed
261
+ if not source_audit_passed:
262
+ result_status = "BLOCKED_SOURCE"
263
+ elif overall:
264
+ result_status = "PASSED_UNDER_PROTOCOL"
265
+ else:
266
+ result_status = "NEGATIVE_RESULT_UNDER_PROTOCOL"
267
+
268
+ return {
269
+ "schema": NOAA_COOPS_PREREG_REPORT_SCHEMA,
270
+ "candidate": NOAA_COOPS_CANDIDATE_NAME,
271
+ "config": {
272
+ "protocol_version": NOAA_COOPS_PROTOCOL_VERSION,
273
+ "train_hours": cfg.train_hours,
274
+ "test_hours": cfg.test_hours,
275
+ "step_hours": cfg.step_hours,
276
+ "event_quantile": cfg.event_quantile,
277
+ "minimum_test_event_count": cfg.minimum_test_event_count,
278
+ "acceptance": acceptance.get("config", {}),
279
+ },
280
+ "pre_registration": {
281
+ "doc": "NOAA_COOPS_D131_PREREG_CHARTER.md",
282
+ "doc_sha256": pre_reg_hash,
283
+ },
284
+ "source": {
285
+ "official_base_url": "https://api.tidesandcurrents.noaa.gov/api/prod/datagetter",
286
+ "observed_product": "hourly_height",
287
+ "prediction_product": "predictions",
288
+ "datum": "MLLW",
289
+ "time_zone": "gmt",
290
+ "units": "metric",
291
+ "format": "json",
292
+ "period": "2018-01-01..2024-12-31",
293
+ "stations": list(NOAA_COOPS_D131_STATIONS),
294
+ "raw_payload_committed": False,
295
+ },
296
+ "source_audit_passed": bool(source_audit_passed),
297
+ "eligible_station_count": int(eligible_station_count),
298
+ "eligible_windows": int(eligible_windows),
299
+ "stations": station_reports,
300
+ "acceptance": acceptance,
301
+ "failed_gates": failed_gates,
302
+ "inputs": input_summaries,
303
+ "external_payload_manifest_sha256": external_payload_manifest_sha256,
304
+ "overall_go_no_go": bool(overall),
305
+ "result": {
306
+ "source_audit_passed": bool(source_audit_passed),
307
+ "eligible_station_count": int(eligible_station_count),
308
+ "eligible_windows": int(eligible_windows),
309
+ "overall_go_no_go": bool(overall),
310
+ "result_status": result_status,
311
+ },
312
+ "result_status": result_status,
313
+ "allowed_narrow_claim": (
314
+ "passed the pre-registered NOAA CO-OPS D-131 gate under protocol 1.0.139"
315
+ if overall
316
+ else None
317
+ ),
318
+ "forbidden_claims": [
319
+ "coastal-warning deployment readiness",
320
+ "universal forecasting edge",
321
+ "broad model superiority",
322
+ "all statistical methods beaten",
323
+ ],
324
+ "honest_boundary": (
325
+ "This report evaluates local NOAA CO-OPS JSON payloads under the frozen "
326
+ "D-131 protocol. A positive claim requires overall_go_no_go=true; "
327
+ "otherwise the result is negative or source-blocked evidence."
328
+ ),
329
+ }
330
+
331
+
332
+ __all__ = [
333
+ "NOAA_COOPS_CANDIDATE_NAME",
334
+ "NOAA_COOPS_NEGATIVE_SUMMARY_SCHEMA",
335
+ "NOAA_COOPS_PREREG_REPORT_SCHEMA",
336
+ "NOAA_COOPS_PROTOCOL_VERSION",
337
+ "NoaaCoopsPreregConfig",
338
+ "build_negative_result_summary",
339
+ "derive_failed_gates",
340
+ "evaluate_noaa_coops_prereg",
341
+ ]
@@ -0,0 +1,176 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ """Stdlib validator for public AI claim source-audit metadata.
3
+
4
+ Mirrors ``specs/public_ai_claim_metadata.schema.json`` and
5
+ ``docs/PUBLIC_AI_CLAIM_PROTOCOL.md``. It checks metadata shape and honesty
6
+ constraints only; it never assigns a result status.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import re
12
+ from collections.abc import Mapping
13
+ from typing import Any
14
+ from urllib.parse import urlparse
15
+
16
+ SCHEMA_VERSION = "protocolith-public-ai-claim-v1"
17
+
18
+ SOURCE_KINDS = (
19
+ "official_docs",
20
+ "model_card",
21
+ "api_reference",
22
+ "system_card",
23
+ "repository",
24
+ "blog_post",
25
+ "screenshot_only",
26
+ "other",
27
+ )
28
+ CAPTURE_METHODS = ("http_fetch", "manual_copy", "archive_snapshot", "screenshot")
29
+ CLAIM_CLASSES = ("source_audit", "source_audit_screenshot_limited")
30
+ API_METADATA_SOURCES = ("documented", "observed", "not_stated")
31
+ HALT_CONDITIONS = (
32
+ "none",
33
+ "source_blocked",
34
+ "rights_unclear",
35
+ "hash_missing",
36
+ "model_identity_unverifiable",
37
+ "screenshot_only_coverage",
38
+ "protocol_changed_after_outcome",
39
+ )
40
+ AUDIT_OUTCOMES = ("recorded", "BLOCKED_SOURCE", "INSUFFICIENT_COVERAGE")
41
+ REQUIRED_FORBIDDEN_IMPLICATIONS = (
42
+ "model_safety",
43
+ "model_quality",
44
+ "runtime_behavior",
45
+ "benchmark_superiority",
46
+ )
47
+ REQUIRED_FIELDS = (
48
+ "schema_version",
49
+ "protocol_id",
50
+ "source_url",
51
+ "source_name",
52
+ "source_kind",
53
+ "access_date",
54
+ "capture_method",
55
+ "content_sha256",
56
+ "model",
57
+ "prompt_sha256",
58
+ "transcript_sha256",
59
+ "raw_prompt_committed",
60
+ "raw_transcript_committed",
61
+ "screenshot_only",
62
+ "claim_class",
63
+ "claim_boundary",
64
+ "forbidden_implications",
65
+ "stop_rule_triggered",
66
+ "audit_outcome",
67
+ )
68
+ MODEL_FIELDS = ("provider", "model_name", "model_version", "api_metadata_source")
69
+
70
+ _SHA256_RE = re.compile(r"^[0-9a-f]{64}$")
71
+ _DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$")
72
+ _PROTOCOL_ID_RE = re.compile(r"^[A-Z0-9][A-Z0-9_]{2,63}$")
73
+ # Value used when a prompt or transcript legitimately does not exist.
74
+ NOT_APPLICABLE = "NOT_APPLICABLE"
75
+ _OVERCLAIM_RE = re.compile(
76
+ r"\b(is safe|are safe|safer than|outperforms?|superior|state[- ]of[- ]the[- ]art|"
77
+ r"best model|guarantees?|certified|production[- ]ready)\b",
78
+ re.IGNORECASE,
79
+ )
80
+
81
+
82
+ def _hash_or_na(value: object, name: str, errors: list[str]) -> None:
83
+ if value == NOT_APPLICABLE:
84
+ return
85
+ if not isinstance(value, str) or not _SHA256_RE.match(value):
86
+ errors.append(f"{name} must be a lowercase sha256 hex digest or {NOT_APPLICABLE}")
87
+
88
+
89
+ def validate_public_ai_claim_metadata(data: Mapping[str, Any]) -> list[str]:
90
+ """Return a list of problems; an empty list means the metadata conforms."""
91
+
92
+ errors: list[str] = []
93
+ if not isinstance(data, Mapping):
94
+ return ["metadata must be an object"]
95
+
96
+ for field in REQUIRED_FIELDS:
97
+ if field not in data:
98
+ errors.append(f"missing required field: {field}")
99
+ if errors:
100
+ return errors
101
+
102
+ if data["schema_version"] != SCHEMA_VERSION:
103
+ errors.append(f"schema_version must be {SCHEMA_VERSION}")
104
+ if not isinstance(data["protocol_id"], str) or not _PROTOCOL_ID_RE.match(data["protocol_id"]):
105
+ errors.append("protocol_id must match [A-Z0-9][A-Z0-9_]{2,63}")
106
+
107
+ parsed = urlparse(data["source_url"]) if isinstance(data["source_url"], str) else None
108
+ if parsed is None or parsed.scheme not in ("http", "https") or not parsed.netloc:
109
+ errors.append("source_url must be an http or https URL")
110
+ for name in ("source_name", "claim_boundary"):
111
+ if not isinstance(data[name], str) or not data[name].strip():
112
+ errors.append(f"{name} must be a non-empty string")
113
+ if data["source_kind"] not in SOURCE_KINDS:
114
+ errors.append(f"source_kind must be one of {list(SOURCE_KINDS)}")
115
+ if not isinstance(data["access_date"], str) or not _DATE_RE.match(data["access_date"]):
116
+ errors.append("access_date must be YYYY-MM-DD")
117
+ if data["capture_method"] not in CAPTURE_METHODS:
118
+ errors.append(f"capture_method must be one of {list(CAPTURE_METHODS)}")
119
+
120
+ _hash_or_na(data["prompt_sha256"], "prompt_sha256", errors)
121
+ _hash_or_na(data["transcript_sha256"], "transcript_sha256", errors)
122
+ if not isinstance(data["content_sha256"], str) or not _SHA256_RE.match(data["content_sha256"]):
123
+ errors.append("content_sha256 must be a lowercase sha256 hex digest")
124
+
125
+ for name in ("raw_prompt_committed", "raw_transcript_committed"):
126
+ if data[name] is not False:
127
+ errors.append(f"{name} must be false (store hashes, never raw text)")
128
+
129
+ model = data["model"]
130
+ if not isinstance(model, Mapping):
131
+ errors.append("model must be an object")
132
+ else:
133
+ for field in MODEL_FIELDS:
134
+ if not isinstance(model.get(field), str) or not model[field].strip():
135
+ errors.append(f"model.{field} must be a non-empty string (use NOT_STATED if unknown)")
136
+ if model.get("api_metadata_source") not in API_METADATA_SOURCES:
137
+ errors.append(f"model.api_metadata_source must be one of {list(API_METADATA_SOURCES)}")
138
+ endpoint = model.get("api_endpoint")
139
+ if endpoint is not None and not isinstance(endpoint, str):
140
+ errors.append("model.api_endpoint must be a string when present")
141
+
142
+ if not isinstance(data["screenshot_only"], bool):
143
+ errors.append("screenshot_only must be a boolean")
144
+ if data["claim_class"] not in CLAIM_CLASSES:
145
+ errors.append(f"claim_class must be one of {list(CLAIM_CLASSES)}")
146
+ shot_only = data["screenshot_only"] is True or data["capture_method"] == "screenshot" or data["source_kind"] == "screenshot_only"
147
+ if shot_only:
148
+ if data["claim_class"] != "source_audit_screenshot_limited":
149
+ errors.append("screenshot-only sources must use claim_class source_audit_screenshot_limited")
150
+ if data["screenshot_only"] is not True:
151
+ errors.append("screenshot capture requires screenshot_only to be true")
152
+ if not isinstance(data.get("screenshot_sha256"), str) or not _SHA256_RE.match(data.get("screenshot_sha256", "")):
153
+ errors.append("screenshot-only sources require screenshot_sha256")
154
+ if data["audit_outcome"] == "recorded" and data["stop_rule_triggered"] == "none":
155
+ errors.append("screenshot-only sources must trigger stop rule screenshot_only_coverage or end as INSUFFICIENT_COVERAGE")
156
+ elif data["claim_class"] == "source_audit_screenshot_limited":
157
+ errors.append("claim_class source_audit_screenshot_limited requires screenshot_only")
158
+
159
+ forbidden = data["forbidden_implications"]
160
+ if not isinstance(forbidden, list):
161
+ errors.append("forbidden_implications must be a list")
162
+ else:
163
+ missing = [item for item in REQUIRED_FORBIDDEN_IMPLICATIONS if item not in forbidden]
164
+ if missing:
165
+ errors.append(f"forbidden_implications must include: {missing}")
166
+
167
+ if data["stop_rule_triggered"] not in HALT_CONDITIONS:
168
+ errors.append(f"stop_rule_triggered must be one of {list(HALT_CONDITIONS)}")
169
+ if data["audit_outcome"] not in AUDIT_OUTCOMES:
170
+ errors.append(f"audit_outcome must be one of {list(AUDIT_OUTCOMES)}")
171
+ if data["stop_rule_triggered"] not in ("none",) and data["audit_outcome"] == "recorded" and not shot_only:
172
+ errors.append("a triggered stop rule cannot end with audit_outcome recorded")
173
+
174
+ if isinstance(data["claim_boundary"], str) and _OVERCLAIM_RE.search(data["claim_boundary"]):
175
+ errors.append("claim_boundary contains an overclaim phrase (safety, quality, superiority or certification)")
176
+ return errors
protocolith/py.typed ADDED
File without changes
@@ -0,0 +1,139 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ """Validation helpers for the Protocolith public evidence registry."""
3
+
4
+ from __future__ import annotations
5
+
6
+ import json
7
+ from collections import Counter
8
+ from pathlib import Path
9
+ from typing import Any
10
+
11
+ from protocolith.evidence_card import validate_evidence_card, warn_evidence_card
12
+
13
+
14
+ ENTRY_CARD_FIELDS = {
15
+ "domain",
16
+ "evidence_id",
17
+ "official_source_name",
18
+ "protocol_id",
19
+ "record_type",
20
+ "reproduction_level",
21
+ "result_status",
22
+ "sanitized_report_path",
23
+ "registry_sequence",
24
+ "operator",
25
+ "last_verified_date",
26
+ "verification_count",
27
+ "verification_level",
28
+ }
29
+ # Note: created_at is intentionally not cross-checked yet — thousands of CB7K
30
+ # registry rows still carry the original campaign date while card JSON was
31
+ # later updated. Rebuild evidence_index from cards before enforcing created_at.
32
+
33
+
34
+
35
+ def load_registry(path: Path) -> dict[str, Any]:
36
+ data = json.loads(path.read_text(encoding="utf-8"))
37
+ if not isinstance(data, dict):
38
+ raise ValueError("registry must be a JSON object")
39
+ return data
40
+
41
+
42
+ def validate_registry(
43
+ registry: dict[str, Any],
44
+ repo_root: Path,
45
+ warnings: list[str] | None = None,
46
+ ) -> list[str]:
47
+ """Return registry violations; append non-fatal hardening findings to ``warnings``."""
48
+ violations: list[str] = []
49
+ warn: list[str] = warnings if warnings is not None else []
50
+ cards = registry.get("cards")
51
+ if not isinstance(cards, list):
52
+ return ["registry.cards must be a list"]
53
+
54
+ if registry.get("card_count") != len(cards):
55
+ violations.append("card_count must match number of registry cards")
56
+
57
+ seen_sequences: set[int] = set()
58
+ seen_ids: set[str] = set()
59
+ entries_for_stats: list[dict[str, Any]] = []
60
+
61
+ for index, entry in enumerate(cards, start=1):
62
+ if not isinstance(entry, dict):
63
+ violations.append(f"registry card #{index} must be an object")
64
+ continue
65
+
66
+ evidence_id = str(entry.get("evidence_id", f"#{index}"))
67
+ if evidence_id in seen_ids:
68
+ violations.append(f"duplicate evidence_id in registry: {evidence_id}")
69
+ seen_ids.add(evidence_id)
70
+
71
+ sequence = entry.get("registry_sequence")
72
+ if not isinstance(sequence, int) or sequence < 1:
73
+ violations.append(f"{evidence_id}: registry_sequence must be positive integer")
74
+ elif sequence in seen_sequences:
75
+ violations.append(f"{evidence_id}: duplicate registry_sequence {sequence}")
76
+ else:
77
+ seen_sequences.add(sequence)
78
+
79
+ path_value = entry.get("path")
80
+ if not isinstance(path_value, str) or not path_value:
81
+ violations.append(f"{evidence_id}: missing registry path")
82
+ continue
83
+
84
+ card_path = repo_root / path_value
85
+ if not card_path.is_file():
86
+ violations.append(f"{evidence_id}: card path does not exist: {path_value}")
87
+ continue
88
+
89
+ card = json.loads(card_path.read_text(encoding="utf-8"))
90
+ card_violations = validate_evidence_card(card)
91
+ violations.extend(f"{evidence_id}: {item}" for item in card_violations)
92
+ warn.extend(f"{evidence_id}: {item}" for item in warn_evidence_card(card))
93
+ if card.get("evidence_id") != evidence_id:
94
+ warn.append(f"{evidence_id}: card evidence_id differs from registry entry")
95
+ if card_path.stem != str(card.get("evidence_id", "")):
96
+ warn.append(f"{evidence_id}: card file name does not match evidence_id")
97
+
98
+ for field in sorted(ENTRY_CARD_FIELDS):
99
+ if entry.get(field) != card.get(field):
100
+ violations.append(f"{evidence_id}: registry field mismatch: {field}")
101
+
102
+ svg_url = entry.get("svg_url")
103
+ if isinstance(svg_url, str) and svg_url:
104
+ svg_path = _github_blob_url_to_repo_path(svg_url)
105
+ if svg_path is not None and not (repo_root / svg_path).is_file():
106
+ violations.append(f"{evidence_id}: svg_url path does not exist: {svg_path}")
107
+
108
+ entries_for_stats.append(entry)
109
+
110
+ if seen_sequences and seen_sequences != set(range(1, len(cards) + 1)):
111
+ violations.append("registry_sequence values must be contiguous from 1")
112
+
113
+ expected_statistics = _compute_statistics(entries_for_stats)
114
+ if registry.get("statistics") != expected_statistics:
115
+ violations.append("statistics must match registry card metadata")
116
+
117
+ return violations
118
+
119
+
120
+ def _compute_statistics(entries: list[dict[str, Any]]) -> dict[str, dict[str, int]]:
121
+ return {
122
+ "by_domain": _count(entries, "domain"),
123
+ "by_record_type": _count(entries, "record_type"),
124
+ "by_result_status": _count(entries, "result_status"),
125
+ "by_source": _count(entries, "official_source_name"),
126
+ }
127
+
128
+
129
+ def _count(entries: list[dict[str, Any]], field: str) -> dict[str, int]:
130
+ counter = Counter(str(entry.get(field, "")) for entry in entries)
131
+ counter.pop("", None)
132
+ return dict(sorted(counter.items()))
133
+
134
+
135
+ def _github_blob_url_to_repo_path(url: str) -> str | None:
136
+ marker = "/blob/main/"
137
+ if marker not in url:
138
+ return None
139
+ return url.split(marker, 1)[1]