tigrcorn-certification 0.3.16.dev5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tigrcorn_certification/__init__.py +55 -0
- tigrcorn_certification/aioquic_preflight.py +449 -0
- tigrcorn_certification/certification_env.py +419 -0
- tigrcorn_certification/conformance.py +42 -0
- tigrcorn_certification/explicit_surfaces.py +130 -0
- tigrcorn_certification/interop_runner.py +2017 -0
- tigrcorn_certification/perf_runner.py +725 -0
- tigrcorn_certification/py.typed +1 -0
- tigrcorn_certification/release_gates.py +1354 -0
- tigrcorn_certification-0.3.16.dev5.dist-info/METADATA +242 -0
- tigrcorn_certification-0.3.16.dev5.dist-info/RECORD +14 -0
- tigrcorn_certification-0.3.16.dev5.dist-info/WHEEL +5 -0
- tigrcorn_certification-0.3.16.dev5.dist-info/licenses/LICENSE +163 -0
- tigrcorn_certification-0.3.16.dev5.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,1354 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from dataclasses import dataclass, field
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Any, Iterable, Mapping
|
|
7
|
+
|
|
8
|
+
from .interop_runner import InteropScenario, load_external_matrix
|
|
9
|
+
|
|
10
|
+
DEFAULT_BOUNDARY_PATH = Path('docs/review/conformance/certification_boundary.json')
|
|
11
|
+
DEFAULT_CORPUS_PATH = Path('docs/review/conformance/corpus.json')
|
|
12
|
+
DEFAULT_INDEPENDENT_MATRIX_PATH = Path('docs/review/conformance/external_matrix.release.json')
|
|
13
|
+
DEFAULT_SAME_STACK_MATRIX_PATH = Path('docs/review/conformance/external_matrix.same_stack_replay.json')
|
|
14
|
+
DEFAULT_STRICT_TARGET_BOUNDARY_PATH = Path('docs/review/conformance/certification_boundary.strict_target.json')
|
|
15
|
+
DEFAULT_PROMOTION_TARGET_PATH = Path('docs/review/conformance/promotion_gate.target.json')
|
|
16
|
+
DEFAULT_TLS_WRAPPER_PATH = Path('src/tigrcorn/security/tls.py')
|
|
17
|
+
DEFAULT_CLAIMS_REGISTRY_PATH = Path('docs/review/conformance/claims_registry.json')
|
|
18
|
+
DEFAULT_RISK_REGISTER_PATH = Path('docs/conformance/risk/RISK_REGISTER.json')
|
|
19
|
+
DEFAULT_RISK_TRACEABILITY_PATH = Path('docs/conformance/risk/RISK_TRACEABILITY.json')
|
|
20
|
+
DEFAULT_LEGACY_UNITTEST_INVENTORY_PATH = Path('LEGACY_UNITTEST_INVENTORY.json')
|
|
21
|
+
DEFAULT_SSOT_REGISTRY_PATH = Path('.ssot/registry.json')
|
|
22
|
+
VALID_EVIDENCE_TIERS = ('local_conformance', 'same_stack_replay', 'independent_certification')
|
|
23
|
+
EVIDENCE_TIER_ORDER = {name: index for index, name in enumerate(VALID_EVIDENCE_TIERS, start=1)}
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@dataclass(slots=True)
|
|
27
|
+
class ReleaseGateReport:
|
|
28
|
+
passed: bool
|
|
29
|
+
failures: list[str] = field(default_factory=list)
|
|
30
|
+
checked_files: list[str] = field(default_factory=list)
|
|
31
|
+
rfc_status: dict[str, dict[str, Any]] = field(default_factory=dict)
|
|
32
|
+
artifact_status: dict[str, dict[str, Any]] = field(default_factory=dict)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@dataclass(slots=True)
|
|
36
|
+
class IndependentBundleReport:
|
|
37
|
+
passed: bool
|
|
38
|
+
failures: list[str] = field(default_factory=list)
|
|
39
|
+
checked_files: list[str] = field(default_factory=list)
|
|
40
|
+
scenario_status: dict[str, dict[str, Any]] = field(default_factory=dict)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
INDEPENDENT_BUNDLE_REQUIRED_ROOT_FILES = ('manifest.json', 'summary.json', 'index.json')
|
|
44
|
+
INDEPENDENT_BUNDLE_REQUIRED_SCENARIO_FILES = (
|
|
45
|
+
'summary.json',
|
|
46
|
+
'index.json',
|
|
47
|
+
'result.json',
|
|
48
|
+
'scenario.json',
|
|
49
|
+
'command.json',
|
|
50
|
+
'env.json',
|
|
51
|
+
'versions.json',
|
|
52
|
+
'wire_capture.json',
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class ReleaseGateError(RuntimeError):
|
|
57
|
+
pass
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def load_certification_boundary(path: str | Path) -> dict[str, Any]:
|
|
61
|
+
return json.loads(Path(path).read_text(encoding='utf-8'))
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def load_conformance_corpus(path: str | Path) -> dict[str, Any]:
|
|
65
|
+
return json.loads(Path(path).read_text(encoding='utf-8'))
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def evaluate_release_gates(
|
|
69
|
+
source_root: str | Path,
|
|
70
|
+
*,
|
|
71
|
+
boundary_path: str | Path | None = None,
|
|
72
|
+
corpus_path: str | Path | None = None,
|
|
73
|
+
independent_matrix_path: str | Path | None = None,
|
|
74
|
+
same_stack_matrix_path: str | Path | None = None,
|
|
75
|
+
) -> ReleaseGateReport:
|
|
76
|
+
source_root = Path(source_root)
|
|
77
|
+
boundary_file = source_root / (Path(boundary_path) if boundary_path is not None else DEFAULT_BOUNDARY_PATH)
|
|
78
|
+
corpus_file = source_root / (Path(corpus_path) if corpus_path is not None else DEFAULT_CORPUS_PATH)
|
|
79
|
+
independent_file = source_root / (Path(independent_matrix_path) if independent_matrix_path is not None else DEFAULT_INDEPENDENT_MATRIX_PATH)
|
|
80
|
+
same_stack_file = source_root / (Path(same_stack_matrix_path) if same_stack_matrix_path is not None else DEFAULT_SAME_STACK_MATRIX_PATH)
|
|
81
|
+
|
|
82
|
+
failures: list[str] = []
|
|
83
|
+
checked_files: list[str] = [str(boundary_file), str(corpus_file), str(independent_file), str(same_stack_file)]
|
|
84
|
+
rfc_status: dict[str, dict[str, Any]] = {}
|
|
85
|
+
artifact_status: dict[str, dict[str, Any]] = {}
|
|
86
|
+
|
|
87
|
+
if not boundary_file.exists():
|
|
88
|
+
failures.append(f'missing certification boundary file: {boundary_file}')
|
|
89
|
+
return ReleaseGateReport(False, failures, checked_files, rfc_status, artifact_status)
|
|
90
|
+
|
|
91
|
+
boundary = load_certification_boundary(boundary_file)
|
|
92
|
+
canonical_doc = str(boundary.get('canonical_doc', 'docs/review/conformance/CERTIFICATION_BOUNDARY.md'))
|
|
93
|
+
gates = dict(boundary.get('gates', {}))
|
|
94
|
+
docs_to_check = [source_root / Path(item) for item in boundary.get('docs_that_must_reference_boundary', [])]
|
|
95
|
+
checked_files.extend(str(path) for path in docs_to_check)
|
|
96
|
+
|
|
97
|
+
if gates.get('require_docs_reference_canonical_boundary', False):
|
|
98
|
+
failures.extend(_validate_boundary_references(canonical_doc=canonical_doc, docs_to_check=docs_to_check))
|
|
99
|
+
|
|
100
|
+
corpus_payload: dict[str, Any] | None
|
|
101
|
+
if gates.get('require_conformance_corpus', False):
|
|
102
|
+
if not corpus_file.exists():
|
|
103
|
+
failures.append(f'missing conformance corpus: {corpus_file}')
|
|
104
|
+
corpus_payload = None
|
|
105
|
+
else:
|
|
106
|
+
corpus_payload = load_conformance_corpus(corpus_file)
|
|
107
|
+
else:
|
|
108
|
+
corpus_payload = load_conformance_corpus(corpus_file) if corpus_file.exists() else None
|
|
109
|
+
|
|
110
|
+
independent_matrix = None
|
|
111
|
+
if gates.get('require_independent_matrix', False):
|
|
112
|
+
if not independent_file.exists():
|
|
113
|
+
failures.append(f'missing independent certification matrix: {independent_file}')
|
|
114
|
+
else:
|
|
115
|
+
independent_matrix = load_external_matrix(independent_file)
|
|
116
|
+
failures.extend(_fail_closed_for_matrix_metadata(independent_matrix, matrix_name='independent certification matrix'))
|
|
117
|
+
if not independent_matrix.scenarios:
|
|
118
|
+
failures.append('independent certification matrix does not include any declared scenarios')
|
|
119
|
+
elif independent_file.exists():
|
|
120
|
+
independent_matrix = load_external_matrix(independent_file)
|
|
121
|
+
|
|
122
|
+
same_stack_matrix = None
|
|
123
|
+
if same_stack_file.exists():
|
|
124
|
+
same_stack_matrix = load_external_matrix(same_stack_file)
|
|
125
|
+
failures.extend(_fail_closed_for_matrix_metadata(same_stack_matrix, matrix_name='same-stack replay matrix'))
|
|
126
|
+
if any(scenario.evidence_tier != 'same_stack_replay' for scenario in same_stack_matrix.scenarios):
|
|
127
|
+
failures.append('same-stack replay matrix contains a scenario outside the same_stack_replay tier')
|
|
128
|
+
elif gates.get('require_docs_reference_canonical_boundary', False):
|
|
129
|
+
failures.append(f'missing same-stack replay matrix: {same_stack_file}')
|
|
130
|
+
|
|
131
|
+
if independent_matrix is not None:
|
|
132
|
+
failures.extend(_evaluate_independent_matrix(independent_matrix.scenarios, gates=gates))
|
|
133
|
+
|
|
134
|
+
if gates.get('require_package_owned_tls13_subsystem', False):
|
|
135
|
+
tls_wrapper_path = source_root / DEFAULT_TLS_WRAPPER_PATH
|
|
136
|
+
checked_files.append(str(tls_wrapper_path))
|
|
137
|
+
if not tls_wrapper_path.exists():
|
|
138
|
+
failures.append(f'missing TLS wrapper module: {tls_wrapper_path}')
|
|
139
|
+
else:
|
|
140
|
+
tls_wrapper_text = tls_wrapper_path.read_text(encoding='utf-8')
|
|
141
|
+
if 'ssl.create_default_context' in tls_wrapper_text:
|
|
142
|
+
failures.append(
|
|
143
|
+
'package-owned TLS 1.3 release gate failed because src/tigrcorn/security/tls.py still delegates TCP/TLS to ssl.create_default_context'
|
|
144
|
+
)
|
|
145
|
+
|
|
146
|
+
if gates.get('require_rfc_evidence_map', False) and corpus_payload is not None and independent_matrix is not None and same_stack_matrix is not None:
|
|
147
|
+
failures.extend(
|
|
148
|
+
_evaluate_rfc_evidence(
|
|
149
|
+
source_root=source_root,
|
|
150
|
+
boundary=boundary,
|
|
151
|
+
corpus_payload=corpus_payload,
|
|
152
|
+
independent_matrix_scenarios=independent_matrix.scenarios,
|
|
153
|
+
same_stack_matrix_scenarios=same_stack_matrix.scenarios,
|
|
154
|
+
checked_files=checked_files,
|
|
155
|
+
rfc_status=rfc_status,
|
|
156
|
+
artifact_status=artifact_status,
|
|
157
|
+
)
|
|
158
|
+
)
|
|
159
|
+
|
|
160
|
+
if gates.get('require_governance_graph', False):
|
|
161
|
+
failures.extend(_evaluate_governance_graph(source_root=source_root, checked_files=checked_files))
|
|
162
|
+
|
|
163
|
+
return ReleaseGateReport(not failures, failures, checked_files, rfc_status, artifact_status)
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def assert_release_ready(
|
|
167
|
+
source_root: str | Path,
|
|
168
|
+
*,
|
|
169
|
+
boundary_path: str | Path | None = None,
|
|
170
|
+
corpus_path: str | Path | None = None,
|
|
171
|
+
independent_matrix_path: str | Path | None = None,
|
|
172
|
+
same_stack_matrix_path: str | Path | None = None,
|
|
173
|
+
) -> None:
|
|
174
|
+
report = evaluate_release_gates(
|
|
175
|
+
source_root,
|
|
176
|
+
boundary_path=boundary_path,
|
|
177
|
+
corpus_path=corpus_path,
|
|
178
|
+
independent_matrix_path=independent_matrix_path,
|
|
179
|
+
same_stack_matrix_path=same_stack_matrix_path,
|
|
180
|
+
)
|
|
181
|
+
if not report.passed:
|
|
182
|
+
details = '\n'.join(f'- {item}' for item in report.failures)
|
|
183
|
+
raise ReleaseGateError(f'release gates failed:\n{details}')
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def _validate_boundary_references(*, canonical_doc: str, docs_to_check: list[Path]) -> list[str]:
|
|
187
|
+
failures: list[str] = []
|
|
188
|
+
for doc in docs_to_check:
|
|
189
|
+
if not doc.exists():
|
|
190
|
+
failures.append(f'missing documentation file: {doc}')
|
|
191
|
+
continue
|
|
192
|
+
text = doc.read_text(encoding='utf-8')
|
|
193
|
+
if canonical_doc not in text:
|
|
194
|
+
failures.append(f'{doc} does not reference the canonical certification boundary {canonical_doc}')
|
|
195
|
+
return failures
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def _evaluate_independent_matrix(scenarios: list[InteropScenario], *, gates: dict[str, Any]) -> list[str]:
|
|
199
|
+
failures: list[str] = []
|
|
200
|
+
independent_scenarios = [scenario for scenario in scenarios if scenario.evidence_tier == 'independent_certification']
|
|
201
|
+
if not independent_scenarios:
|
|
202
|
+
failures.append('independent certification matrix does not contain any independent_certification scenarios')
|
|
203
|
+
return failures
|
|
204
|
+
|
|
205
|
+
for scenario in independent_scenarios:
|
|
206
|
+
failures.extend(_fail_closed_for_scenario_metadata(scenario))
|
|
207
|
+
peer_kind = scenario.peer_process.provenance_kind
|
|
208
|
+
if peer_kind == 'same_stack_fixture':
|
|
209
|
+
failures.append(f'independent scenario {scenario.id} incorrectly uses a same_stack_fixture peer')
|
|
210
|
+
if peer_kind not in {'third_party_library', 'third_party_binary'}:
|
|
211
|
+
failures.append(f'independent scenario {scenario.id} is not backed by a true third-party peer: {peer_kind!r}')
|
|
212
|
+
|
|
213
|
+
if gates.get('require_third_party_http3_request_response', False) and not _has_third_party_http3_request_response(independent_scenarios):
|
|
214
|
+
failures.append('independent certification matrix does not declare a true third-party HTTP/3 request/response scenario')
|
|
215
|
+
|
|
216
|
+
if gates.get('require_third_party_http3_websocket', False) and not _has_third_party_http3_websocket(independent_scenarios):
|
|
217
|
+
failures.append('independent certification matrix does not declare a true third-party RFC 9220 WebSocket-over-HTTP/3 scenario')
|
|
218
|
+
|
|
219
|
+
return failures
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def _fail_closed_for_matrix_metadata(matrix: Any, *, matrix_name: str) -> list[str]:
|
|
223
|
+
failures: list[str] = []
|
|
224
|
+
metadata = dict(getattr(matrix, 'metadata', {}) or {})
|
|
225
|
+
pending_ids = metadata.get('pending_third_party_http3_scenarios', [])
|
|
226
|
+
if isinstance(pending_ids, list) and pending_ids:
|
|
227
|
+
failures.append(
|
|
228
|
+
f'{matrix_name} declares blocked pending_third_party_http3_scenarios and therefore is not release-gate eligible: {sorted(str(item) for item in pending_ids)}'
|
|
229
|
+
)
|
|
230
|
+
blocked_ids = metadata.get('blocked_scenarios', [])
|
|
231
|
+
if isinstance(blocked_ids, list) and blocked_ids:
|
|
232
|
+
failures.append(
|
|
233
|
+
f'{matrix_name} declares blocked_scenarios and therefore is not release-gate eligible: {sorted(str(item) for item in blocked_ids)}'
|
|
234
|
+
)
|
|
235
|
+
return failures
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def _fail_closed_for_scenario_metadata(scenario: InteropScenario) -> list[str]:
|
|
239
|
+
failures: list[str] = []
|
|
240
|
+
metadata = dict(scenario.metadata or {})
|
|
241
|
+
certification_status = str(metadata.get('certification_status', '')).strip().lower()
|
|
242
|
+
blocked_statuses = {
|
|
243
|
+
'blocked',
|
|
244
|
+
'failed',
|
|
245
|
+
'incomplete',
|
|
246
|
+
'not_ready',
|
|
247
|
+
'not_release_ready',
|
|
248
|
+
'pending',
|
|
249
|
+
'provisional',
|
|
250
|
+
}
|
|
251
|
+
if certification_status in blocked_statuses:
|
|
252
|
+
failures.append(
|
|
253
|
+
f'independent scenario {scenario.id} is blocked by certification_status={metadata.get("certification_status")!r}'
|
|
254
|
+
)
|
|
255
|
+
for key in ('blocked', 'pending'):
|
|
256
|
+
if metadata.get(key) is True:
|
|
257
|
+
failures.append(f'independent scenario {scenario.id} is blocked by metadata flag {key}=true')
|
|
258
|
+
for key in ('blocked_reason', 'pending_reason', 'blocker'):
|
|
259
|
+
value = metadata.get(key)
|
|
260
|
+
if isinstance(value, str) and value.strip():
|
|
261
|
+
failures.append(f'independent scenario {scenario.id} is blocked by metadata {key}={value!r}')
|
|
262
|
+
return failures
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
def _evaluate_rfc_evidence(
|
|
266
|
+
*,
|
|
267
|
+
source_root: Path,
|
|
268
|
+
boundary: dict[str, Any],
|
|
269
|
+
corpus_payload: dict[str, Any],
|
|
270
|
+
independent_matrix_scenarios: list[InteropScenario],
|
|
271
|
+
same_stack_matrix_scenarios: list[InteropScenario],
|
|
272
|
+
checked_files: list[str],
|
|
273
|
+
rfc_status: dict[str, dict[str, Any]],
|
|
274
|
+
artifact_status: dict[str, dict[str, Any]],
|
|
275
|
+
) -> list[str]:
|
|
276
|
+
failures: list[str] = []
|
|
277
|
+
required_rfcs = [str(item) for item in boundary.get('required_rfcs', [])]
|
|
278
|
+
rfc_evidence_map = dict(boundary.get('required_rfc_evidence', {}))
|
|
279
|
+
corpus_vectors = _index_corpus_vectors(corpus_payload)
|
|
280
|
+
independent_index = {scenario.id: scenario for scenario in independent_matrix_scenarios}
|
|
281
|
+
same_stack_index = {scenario.id: scenario for scenario in same_stack_matrix_scenarios}
|
|
282
|
+
artifact_bundles = {tier: source_root / Path(path) for tier, path in dict(boundary.get('artifact_bundles', {})).items()}
|
|
283
|
+
|
|
284
|
+
for bundle_root in artifact_bundles.values():
|
|
285
|
+
checked_files.extend(str(path) for path in [bundle_root, bundle_root / 'index.json', bundle_root / 'manifest.json'])
|
|
286
|
+
|
|
287
|
+
preserved_artifacts = {
|
|
288
|
+
tier: _load_preserved_artifacts(bundle_root, artifact_status=artifact_status)
|
|
289
|
+
for tier, bundle_root in artifact_bundles.items()
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
for required_rfc in required_rfcs:
|
|
293
|
+
policy = rfc_evidence_map.get(required_rfc)
|
|
294
|
+
if not isinstance(policy, Mapping):
|
|
295
|
+
failures.append(f'boundary required RFC is missing evidence policy: {required_rfc}')
|
|
296
|
+
continue
|
|
297
|
+
highest_tier = str(policy.get('highest_required_evidence_tier', '')).strip()
|
|
298
|
+
declared_evidence = dict(policy.get('declared_evidence', {}))
|
|
299
|
+
rfc_failures, status = _evaluate_single_rfc_policy(
|
|
300
|
+
required_rfc=required_rfc,
|
|
301
|
+
highest_tier=highest_tier,
|
|
302
|
+
declared_evidence=declared_evidence,
|
|
303
|
+
corpus_vectors=corpus_vectors,
|
|
304
|
+
independent_index=independent_index,
|
|
305
|
+
same_stack_index=same_stack_index,
|
|
306
|
+
preserved_artifacts=preserved_artifacts,
|
|
307
|
+
)
|
|
308
|
+
failures.extend(rfc_failures)
|
|
309
|
+
rfc_status[required_rfc] = status
|
|
310
|
+
|
|
311
|
+
extra_policies = sorted(set(rfc_evidence_map) - set(required_rfcs))
|
|
312
|
+
for item in extra_policies:
|
|
313
|
+
failures.append(f'boundary contains evidence policy for non-required RFC: {item}')
|
|
314
|
+
|
|
315
|
+
return failures
|
|
316
|
+
|
|
317
|
+
|
|
318
|
+
def _evaluate_single_rfc_policy(
|
|
319
|
+
*,
|
|
320
|
+
required_rfc: str,
|
|
321
|
+
highest_tier: str,
|
|
322
|
+
declared_evidence: dict[str, Any],
|
|
323
|
+
corpus_vectors: dict[str, dict[str, Any]],
|
|
324
|
+
independent_index: dict[str, InteropScenario],
|
|
325
|
+
same_stack_index: dict[str, InteropScenario],
|
|
326
|
+
preserved_artifacts: dict[str, dict[str, dict[str, Any]]],
|
|
327
|
+
) -> tuple[list[str], dict[str, Any]]:
|
|
328
|
+
failures: list[str] = []
|
|
329
|
+
status: dict[str, Any] = {
|
|
330
|
+
'highest_required_evidence_tier': highest_tier,
|
|
331
|
+
'declared_evidence': declared_evidence,
|
|
332
|
+
'resolved_evidence': {},
|
|
333
|
+
'highest_observed_evidence_tier': None,
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
if highest_tier not in EVIDENCE_TIER_ORDER:
|
|
337
|
+
failures.append(f'{required_rfc} has invalid highest_required_evidence_tier {highest_tier!r}')
|
|
338
|
+
return failures, status
|
|
339
|
+
|
|
340
|
+
if highest_tier not in declared_evidence:
|
|
341
|
+
failures.append(f'{required_rfc} requires {highest_tier} evidence but does not declare any {highest_tier} sources')
|
|
342
|
+
|
|
343
|
+
observed_rank = 0
|
|
344
|
+
for tier_name, entries in declared_evidence.items():
|
|
345
|
+
if tier_name not in EVIDENCE_TIER_ORDER:
|
|
346
|
+
failures.append(f'{required_rfc} declares invalid evidence tier {tier_name!r}')
|
|
347
|
+
continue
|
|
348
|
+
if not isinstance(entries, list) or not all(isinstance(item, str) for item in entries):
|
|
349
|
+
failures.append(f'{required_rfc} declares malformed evidence list for {tier_name}')
|
|
350
|
+
continue
|
|
351
|
+
resolved: list[dict[str, Any]] = []
|
|
352
|
+
tier_failures, tier_satisfied = _resolve_declared_evidence(
|
|
353
|
+
required_rfc=required_rfc,
|
|
354
|
+
tier_name=tier_name,
|
|
355
|
+
entries=entries,
|
|
356
|
+
corpus_vectors=corpus_vectors,
|
|
357
|
+
independent_index=independent_index,
|
|
358
|
+
same_stack_index=same_stack_index,
|
|
359
|
+
preserved_artifacts=preserved_artifacts,
|
|
360
|
+
resolved=resolved,
|
|
361
|
+
)
|
|
362
|
+
failures.extend(tier_failures)
|
|
363
|
+
status['resolved_evidence'][tier_name] = resolved
|
|
364
|
+
if tier_satisfied:
|
|
365
|
+
observed_rank = max(observed_rank, EVIDENCE_TIER_ORDER[tier_name])
|
|
366
|
+
|
|
367
|
+
if observed_rank == 0:
|
|
368
|
+
failures.append(f'{required_rfc} does not resolve any declared evidence')
|
|
369
|
+
elif observed_rank < EVIDENCE_TIER_ORDER[highest_tier]:
|
|
370
|
+
observed_tier = VALID_EVIDENCE_TIERS[observed_rank - 1]
|
|
371
|
+
failures.append(
|
|
372
|
+
f'{required_rfc} requires {highest_tier} evidence, but the resolved evidence only reaches {observed_tier}'
|
|
373
|
+
)
|
|
374
|
+
status['highest_observed_evidence_tier'] = observed_tier
|
|
375
|
+
else:
|
|
376
|
+
status['highest_observed_evidence_tier'] = VALID_EVIDENCE_TIERS[observed_rank - 1]
|
|
377
|
+
|
|
378
|
+
return failures, status
|
|
379
|
+
|
|
380
|
+
|
|
381
|
+
def _resolve_declared_evidence(
|
|
382
|
+
*,
|
|
383
|
+
required_rfc: str,
|
|
384
|
+
tier_name: str,
|
|
385
|
+
entries: list[str],
|
|
386
|
+
corpus_vectors: dict[str, dict[str, Any]],
|
|
387
|
+
independent_index: dict[str, InteropScenario],
|
|
388
|
+
same_stack_index: dict[str, InteropScenario],
|
|
389
|
+
preserved_artifacts: dict[str, dict[str, dict[str, Any]]],
|
|
390
|
+
resolved: list[dict[str, Any]],
|
|
391
|
+
) -> tuple[list[str], bool]:
|
|
392
|
+
failures: list[str] = []
|
|
393
|
+
tier_satisfied = True
|
|
394
|
+
for entry in entries:
|
|
395
|
+
if tier_name == 'local_conformance':
|
|
396
|
+
vector = corpus_vectors.get(entry)
|
|
397
|
+
if vector is None:
|
|
398
|
+
failures.append(f'{required_rfc} references unknown local_conformance vector {entry}')
|
|
399
|
+
tier_satisfied = False
|
|
400
|
+
continue
|
|
401
|
+
resolved.append({'vector': entry, 'rfc': _normalize_rfc_from_corpus(vector.get('rfc'))})
|
|
402
|
+
continue
|
|
403
|
+
|
|
404
|
+
scenario_index = independent_index if tier_name == 'independent_certification' else same_stack_index
|
|
405
|
+
scenario = scenario_index.get(entry)
|
|
406
|
+
if scenario is None:
|
|
407
|
+
failures.append(f'{required_rfc} references unknown {tier_name} scenario {entry}')
|
|
408
|
+
tier_satisfied = False
|
|
409
|
+
continue
|
|
410
|
+
rfcs = set(_scenario_rfcs(scenario))
|
|
411
|
+
if required_rfc not in rfcs:
|
|
412
|
+
failures.append(f'{required_rfc} references scenario {entry} but that scenario metadata does not declare {required_rfc}')
|
|
413
|
+
tier_satisfied = False
|
|
414
|
+
scenario_payload = {
|
|
415
|
+
'scenario_id': entry,
|
|
416
|
+
'enabled': bool(scenario.enabled and scenario.peer_process.enabled and scenario.sut.enabled),
|
|
417
|
+
'peer_kind': scenario.peer_process.provenance_kind,
|
|
418
|
+
}
|
|
419
|
+
if tier_name == 'independent_certification':
|
|
420
|
+
bundle_status = preserved_artifacts.get('independent_certification', {}).get(entry)
|
|
421
|
+
if bundle_status is None:
|
|
422
|
+
failures.append(
|
|
423
|
+
f'{required_rfc} independent_certification scenario {entry} is missing preserved artifacts under the canonical independent release bundle'
|
|
424
|
+
)
|
|
425
|
+
scenario_payload['artifact_status'] = 'missing'
|
|
426
|
+
tier_satisfied = False
|
|
427
|
+
elif not bundle_status.get('passed', False):
|
|
428
|
+
failures.append(
|
|
429
|
+
f'{required_rfc} independent_certification scenario {entry} has preserved artifacts but they are not marked passing'
|
|
430
|
+
)
|
|
431
|
+
scenario_payload['artifact_status'] = 'failed'
|
|
432
|
+
tier_satisfied = False
|
|
433
|
+
else:
|
|
434
|
+
scenario_payload['artifact_status'] = 'passed'
|
|
435
|
+
if not scenario_payload['enabled']:
|
|
436
|
+
failures.append(f'{required_rfc} independent_certification scenario {entry} is declared but disabled')
|
|
437
|
+
tier_satisfied = False
|
|
438
|
+
elif tier_name == 'same_stack_replay':
|
|
439
|
+
bundle_status = preserved_artifacts.get('same_stack_replay', {}).get(entry)
|
|
440
|
+
scenario_payload['artifact_status'] = 'passed' if bundle_status and bundle_status.get('passed', False) else 'missing'
|
|
441
|
+
if bundle_status is None:
|
|
442
|
+
failures.append(f'{required_rfc} same_stack_replay scenario {entry} is missing preserved artifacts under the canonical same-stack bundle')
|
|
443
|
+
tier_satisfied = False
|
|
444
|
+
resolved.append(scenario_payload)
|
|
445
|
+
return failures, tier_satisfied
|
|
446
|
+
|
|
447
|
+
|
|
448
|
+
def _load_preserved_artifacts(bundle_root: Path, *, artifact_status: dict[str, dict[str, Any]]) -> dict[str, dict[str, Any]]:
|
|
449
|
+
if not bundle_root.exists():
|
|
450
|
+
return {}
|
|
451
|
+
index_path = bundle_root / 'index.json'
|
|
452
|
+
if not index_path.exists():
|
|
453
|
+
return {}
|
|
454
|
+
payload = json.loads(index_path.read_text(encoding='utf-8'))
|
|
455
|
+
scenarios: dict[str, dict[str, Any]] = {}
|
|
456
|
+
for entry in payload.get('scenarios', []):
|
|
457
|
+
scenario_id = str(entry.get('id'))
|
|
458
|
+
if not scenario_id or scenario_id == 'None':
|
|
459
|
+
continue
|
|
460
|
+
result_path = bundle_root / scenario_id / 'result.json'
|
|
461
|
+
passed = bool(entry.get('passed', False))
|
|
462
|
+
if result_path.exists():
|
|
463
|
+
try:
|
|
464
|
+
result_payload = json.loads(result_path.read_text(encoding='utf-8'))
|
|
465
|
+
passed = bool(result_payload.get('passed', passed))
|
|
466
|
+
except Exception:
|
|
467
|
+
pass
|
|
468
|
+
status = {
|
|
469
|
+
'artifact_dir': str(bundle_root / scenario_id),
|
|
470
|
+
'passed': passed,
|
|
471
|
+
'result_path': str(result_path),
|
|
472
|
+
'exists': result_path.exists(),
|
|
473
|
+
}
|
|
474
|
+
scenarios[scenario_id] = status
|
|
475
|
+
artifact_status[str(bundle_root / scenario_id)] = status
|
|
476
|
+
return scenarios
|
|
477
|
+
|
|
478
|
+
|
|
479
|
+
def validate_independent_certification_bundle(
|
|
480
|
+
bundle_root: str | Path,
|
|
481
|
+
*,
|
|
482
|
+
required_scenarios: Iterable[str] | None = None,
|
|
483
|
+
required_root_files: Iterable[str] = INDEPENDENT_BUNDLE_REQUIRED_ROOT_FILES,
|
|
484
|
+
required_scenario_files: Iterable[str] = INDEPENDENT_BUNDLE_REQUIRED_SCENARIO_FILES,
|
|
485
|
+
) -> IndependentBundleReport:
|
|
486
|
+
bundle_root = Path(bundle_root)
|
|
487
|
+
failures: list[str] = []
|
|
488
|
+
checked_files: list[str] = []
|
|
489
|
+
scenario_status: dict[str, dict[str, Any]] = {}
|
|
490
|
+
|
|
491
|
+
if not bundle_root.exists():
|
|
492
|
+
failures.append(f'missing independent-certification bundle root: {bundle_root}')
|
|
493
|
+
return IndependentBundleReport(False, failures, checked_files, scenario_status)
|
|
494
|
+
|
|
495
|
+
for filename in required_root_files:
|
|
496
|
+
checked_files.append(str(bundle_root / filename))
|
|
497
|
+
if not (bundle_root / filename).exists():
|
|
498
|
+
failures.append(f'missing bundle file: {bundle_root / filename}')
|
|
499
|
+
|
|
500
|
+
if failures:
|
|
501
|
+
return IndependentBundleReport(False, failures, checked_files, scenario_status)
|
|
502
|
+
|
|
503
|
+
manifest = json.loads((bundle_root / 'manifest.json').read_text(encoding='utf-8'))
|
|
504
|
+
summary = json.loads((bundle_root / 'summary.json').read_text(encoding='utf-8'))
|
|
505
|
+
index = json.loads((bundle_root / 'index.json').read_text(encoding='utf-8'))
|
|
506
|
+
|
|
507
|
+
if str(index.get('matrix_name', '')) != str(summary.get('matrix_name', '')):
|
|
508
|
+
failures.append('bundle summary and index disagree on matrix_name')
|
|
509
|
+
if str(index.get('commit_hash', '')) != str(summary.get('commit_hash', '')):
|
|
510
|
+
failures.append('bundle summary and index disagree on commit_hash')
|
|
511
|
+
if str(index.get('commit_hash', '')) != str(manifest.get('commit_hash', '')):
|
|
512
|
+
failures.append('bundle manifest and index disagree on commit_hash')
|
|
513
|
+
|
|
514
|
+
index_ids = {str(entry.get('id')) for entry in index.get('scenarios', []) if entry.get('id') is not None}
|
|
515
|
+
summary_ids = {str(item) for item in summary.get('scenario_ids', []) if item is not None}
|
|
516
|
+
if summary_ids and index_ids != summary_ids:
|
|
517
|
+
failures.append('bundle summary scenario_ids do not match bundle index scenarios')
|
|
518
|
+
|
|
519
|
+
if required_scenarios is not None:
|
|
520
|
+
for scenario_id in required_scenarios:
|
|
521
|
+
if scenario_id not in index_ids:
|
|
522
|
+
failures.append(f'required proof scenario missing from bundle index: {scenario_id}')
|
|
523
|
+
|
|
524
|
+
for entry in index.get('scenarios', []):
|
|
525
|
+
scenario_id = str(entry.get('id', '')).strip()
|
|
526
|
+
if not scenario_id:
|
|
527
|
+
failures.append('bundle index contains a scenario entry without an id')
|
|
528
|
+
continue
|
|
529
|
+
scenario_dir = bundle_root / scenario_id
|
|
530
|
+
checked_files.append(str(scenario_dir))
|
|
531
|
+
if not scenario_dir.exists():
|
|
532
|
+
failures.append(f'missing scenario directory: {scenario_dir}')
|
|
533
|
+
continue
|
|
534
|
+
status: dict[str, Any] = {
|
|
535
|
+
'artifact_dir': str(scenario_dir),
|
|
536
|
+
'required_files_present': True,
|
|
537
|
+
'passed': bool(entry.get('passed', False)),
|
|
538
|
+
}
|
|
539
|
+
scenario_status[scenario_id] = status
|
|
540
|
+
|
|
541
|
+
for filename in required_scenario_files:
|
|
542
|
+
file_path = scenario_dir / filename
|
|
543
|
+
checked_files.append(str(file_path))
|
|
544
|
+
if not file_path.exists():
|
|
545
|
+
failures.append(f'{scenario_id} missing required artifact file: {file_path}')
|
|
546
|
+
status['required_files_present'] = False
|
|
547
|
+
|
|
548
|
+
if not status['required_files_present']:
|
|
549
|
+
continue
|
|
550
|
+
|
|
551
|
+
result_payload = json.loads((scenario_dir / 'result.json').read_text(encoding='utf-8'))
|
|
552
|
+
summary_payload = json.loads((scenario_dir / 'summary.json').read_text(encoding='utf-8'))
|
|
553
|
+
scenario_index_payload = json.loads((scenario_dir / 'index.json').read_text(encoding='utf-8'))
|
|
554
|
+
command_payload = json.loads((scenario_dir / 'command.json').read_text(encoding='utf-8'))
|
|
555
|
+
env_payload = json.loads((scenario_dir / 'env.json').read_text(encoding='utf-8'))
|
|
556
|
+
versions_payload = json.loads((scenario_dir / 'versions.json').read_text(encoding='utf-8'))
|
|
557
|
+
wire_payload = json.loads((scenario_dir / 'wire_capture.json').read_text(encoding='utf-8'))
|
|
558
|
+
|
|
559
|
+
status['passed'] = bool(result_payload.get('passed', False))
|
|
560
|
+
if bool(entry.get('passed', False)) != bool(result_payload.get('passed', False)):
|
|
561
|
+
failures.append(f'{scenario_id} bundle index passed flag disagrees with result.json')
|
|
562
|
+
if bool(summary_payload.get('passed', False)) != bool(result_payload.get('passed', False)):
|
|
563
|
+
failures.append(f'{scenario_id} summary.json passed flag disagrees with result.json')
|
|
564
|
+
if bool(scenario_index_payload.get('passed', False)) != bool(result_payload.get('passed', False)):
|
|
565
|
+
failures.append(f'{scenario_id} index.json passed flag disagrees with result.json')
|
|
566
|
+
|
|
567
|
+
artifact_files = scenario_index_payload.get('artifact_files')
|
|
568
|
+
if not isinstance(artifact_files, Mapping) or not artifact_files:
|
|
569
|
+
failures.append(f'{scenario_id} index.json is missing a populated artifact_files inventory')
|
|
570
|
+
else:
|
|
571
|
+
for filename in required_scenario_files:
|
|
572
|
+
metadata = artifact_files.get(filename)
|
|
573
|
+
if not isinstance(metadata, Mapping) or not bool(metadata.get('exists', False)):
|
|
574
|
+
failures.append(f'{scenario_id} index.json does not record {filename} as an existing artifact')
|
|
575
|
+
|
|
576
|
+
if 'sut' not in command_payload or 'peer' not in command_payload:
|
|
577
|
+
failures.append(f'{scenario_id} command.json must contain sut and peer command records')
|
|
578
|
+
if 'sut' not in env_payload or 'peer' not in env_payload:
|
|
579
|
+
failures.append(f'{scenario_id} env.json must contain sut and peer environment records')
|
|
580
|
+
if 'sut' not in versions_payload or 'peer' not in versions_payload:
|
|
581
|
+
failures.append(f'{scenario_id} versions.json must contain sut and peer version records')
|
|
582
|
+
if 'packet_trace' not in wire_payload or 'logs' not in wire_payload:
|
|
583
|
+
failures.append(f'{scenario_id} wire_capture.json must contain packet_trace and logs sections')
|
|
584
|
+
|
|
585
|
+
passed = not failures
|
|
586
|
+
return IndependentBundleReport(passed, failures, checked_files, scenario_status)
|
|
587
|
+
|
|
588
|
+
|
|
589
|
+
def assert_independent_certification_bundle_ready(
|
|
590
|
+
bundle_root: str | Path,
|
|
591
|
+
*,
|
|
592
|
+
required_scenarios: Iterable[str] | None = None,
|
|
593
|
+
required_root_files: Iterable[str] = INDEPENDENT_BUNDLE_REQUIRED_ROOT_FILES,
|
|
594
|
+
required_scenario_files: Iterable[str] = INDEPENDENT_BUNDLE_REQUIRED_SCENARIO_FILES,
|
|
595
|
+
) -> None:
|
|
596
|
+
report = validate_independent_certification_bundle(
|
|
597
|
+
bundle_root,
|
|
598
|
+
required_scenarios=required_scenarios,
|
|
599
|
+
required_root_files=required_root_files,
|
|
600
|
+
required_scenario_files=required_scenario_files,
|
|
601
|
+
)
|
|
602
|
+
if report.passed:
|
|
603
|
+
return
|
|
604
|
+
raise ReleaseGateError('independent-certification bundle validation failed: ' + '; '.join(report.failures))
|
|
605
|
+
|
|
606
|
+
|
|
607
|
+
def _has_third_party_http3_request_response(scenarios: list[InteropScenario]) -> bool:
|
|
608
|
+
for scenario in scenarios:
|
|
609
|
+
if scenario.protocol != 'http3':
|
|
610
|
+
continue
|
|
611
|
+
if scenario.peer_process.provenance_kind not in {'third_party_library', 'third_party_binary'}:
|
|
612
|
+
continue
|
|
613
|
+
rfcs = set(_scenario_rfcs(scenario))
|
|
614
|
+
feature = scenario.feature.lower()
|
|
615
|
+
if 'RFC 9220' in rfcs or 'websocket' in feature:
|
|
616
|
+
continue
|
|
617
|
+
if 'RFC 9114' not in rfcs and not any(token in feature for token in ('request', 'response', 'post', 'get', 'echo')):
|
|
618
|
+
continue
|
|
619
|
+
return True
|
|
620
|
+
return False
|
|
621
|
+
|
|
622
|
+
|
|
623
|
+
def _has_third_party_http3_websocket(scenarios: list[InteropScenario]) -> bool:
|
|
624
|
+
for scenario in scenarios:
|
|
625
|
+
if scenario.protocol != 'http3':
|
|
626
|
+
continue
|
|
627
|
+
if scenario.peer_process.provenance_kind not in {'third_party_library', 'third_party_binary'}:
|
|
628
|
+
continue
|
|
629
|
+
rfcs = set(_scenario_rfcs(scenario))
|
|
630
|
+
if 'RFC 9220' in rfcs or 'websocket' in scenario.feature.lower():
|
|
631
|
+
return True
|
|
632
|
+
return False
|
|
633
|
+
|
|
634
|
+
|
|
635
|
+
def _scenario_rfcs(scenario: InteropScenario) -> list[str]:
|
|
636
|
+
metadata = scenario.metadata
|
|
637
|
+
rfcs = metadata.get('rfc', []) if isinstance(metadata, dict) else []
|
|
638
|
+
return [str(item) for item in rfcs]
|
|
639
|
+
|
|
640
|
+
|
|
641
|
+
def _index_corpus_vectors(corpus_payload: dict[str, Any]) -> dict[str, dict[str, Any]]:
|
|
642
|
+
vectors = corpus_payload.get('vectors', [])
|
|
643
|
+
index: dict[str, dict[str, Any]] = {}
|
|
644
|
+
for entry in vectors:
|
|
645
|
+
if not isinstance(entry, dict) or 'name' not in entry:
|
|
646
|
+
continue
|
|
647
|
+
index[str(entry['name'])] = dict(entry)
|
|
648
|
+
return index
|
|
649
|
+
|
|
650
|
+
|
|
651
|
+
def _normalize_rfc_from_corpus(value: Any) -> str | None:
|
|
652
|
+
if value is None:
|
|
653
|
+
return None
|
|
654
|
+
text = str(value)
|
|
655
|
+
if text.startswith('9110-connect'):
|
|
656
|
+
return 'RFC 9110 §9.3.6'
|
|
657
|
+
if text.startswith('9110-trailers'):
|
|
658
|
+
return 'RFC 9110 §6.5'
|
|
659
|
+
if text.startswith('9110-content-coding'):
|
|
660
|
+
return 'RFC 9110 §8'
|
|
661
|
+
if text.isdigit():
|
|
662
|
+
return f'RFC {text}'
|
|
663
|
+
return text
|
|
664
|
+
|
|
665
|
+
|
|
666
|
+
@dataclass(slots=True)
|
|
667
|
+
class PromotionSectionReport:
|
|
668
|
+
name: str
|
|
669
|
+
passed: bool
|
|
670
|
+
failures: list[str] = field(default_factory=list)
|
|
671
|
+
checked_files: list[str] = field(default_factory=list)
|
|
672
|
+
details: dict[str, Any] = field(default_factory=dict)
|
|
673
|
+
|
|
674
|
+
|
|
675
|
+
@dataclass(slots=True)
|
|
676
|
+
class PromotionTargetReport:
|
|
677
|
+
passed: bool
|
|
678
|
+
failures: list[str] = field(default_factory=list)
|
|
679
|
+
checked_files: list[str] = field(default_factory=list)
|
|
680
|
+
authoritative_boundary: PromotionSectionReport | None = None
|
|
681
|
+
strict_target_boundary: PromotionSectionReport | None = None
|
|
682
|
+
flag_surface: PromotionSectionReport | None = None
|
|
683
|
+
operator_surface: PromotionSectionReport | None = None
|
|
684
|
+
performance: PromotionSectionReport | None = None
|
|
685
|
+
documentation: PromotionSectionReport | None = None
|
|
686
|
+
|
|
687
|
+
|
|
688
|
+
class PromotionTargetError(RuntimeError):
|
|
689
|
+
pass
|
|
690
|
+
|
|
691
|
+
|
|
692
|
+
def load_promotion_target(path: str | Path) -> dict[str, Any]:
|
|
693
|
+
return json.loads(Path(path).read_text(encoding='utf-8'))
|
|
694
|
+
|
|
695
|
+
|
|
696
|
+
def evaluate_promotion_target(
|
|
697
|
+
source_root: str | Path,
|
|
698
|
+
*,
|
|
699
|
+
target_path: str | Path | None = None,
|
|
700
|
+
) -> PromotionTargetReport:
|
|
701
|
+
source_root = Path(source_root)
|
|
702
|
+
target_file = source_root / (Path(target_path) if target_path is not None else DEFAULT_PROMOTION_TARGET_PATH)
|
|
703
|
+
checked_files: list[str] = [str(target_file)]
|
|
704
|
+
if not target_file.exists():
|
|
705
|
+
failure = f'missing promotion target file: {target_file}'
|
|
706
|
+
return PromotionTargetReport(False, [failure], checked_files)
|
|
707
|
+
|
|
708
|
+
target = load_promotion_target(target_file)
|
|
709
|
+
|
|
710
|
+
authoritative_config = dict(target.get('authoritative_boundary', {}))
|
|
711
|
+
authoritative_report = evaluate_release_gates(
|
|
712
|
+
source_root,
|
|
713
|
+
boundary_path=authoritative_config.get('boundary_path'),
|
|
714
|
+
corpus_path=authoritative_config.get('corpus_path'),
|
|
715
|
+
independent_matrix_path=authoritative_config.get('independent_matrix_path'),
|
|
716
|
+
same_stack_matrix_path=authoritative_config.get('same_stack_matrix_path'),
|
|
717
|
+
)
|
|
718
|
+
authoritative_section = PromotionSectionReport(
|
|
719
|
+
name='authoritative_boundary',
|
|
720
|
+
passed=authoritative_report.passed,
|
|
721
|
+
failures=list(authoritative_report.failures),
|
|
722
|
+
checked_files=list(authoritative_report.checked_files),
|
|
723
|
+
details={
|
|
724
|
+
'boundary_path': authoritative_config.get('boundary_path', str(DEFAULT_BOUNDARY_PATH)),
|
|
725
|
+
'required_rfcs': sorted(authoritative_report.rfc_status),
|
|
726
|
+
},
|
|
727
|
+
)
|
|
728
|
+
|
|
729
|
+
strict_config = dict(target.get('strict_target_boundary', {}))
|
|
730
|
+
strict_report = evaluate_release_gates(
|
|
731
|
+
source_root,
|
|
732
|
+
boundary_path=strict_config.get('boundary_path', str(DEFAULT_STRICT_TARGET_BOUNDARY_PATH)),
|
|
733
|
+
corpus_path=strict_config.get('corpus_path'),
|
|
734
|
+
independent_matrix_path=strict_config.get('independent_matrix_path'),
|
|
735
|
+
same_stack_matrix_path=strict_config.get('same_stack_matrix_path'),
|
|
736
|
+
)
|
|
737
|
+
strict_section = PromotionSectionReport(
|
|
738
|
+
name='strict_target_boundary',
|
|
739
|
+
passed=strict_report.passed,
|
|
740
|
+
failures=list(strict_report.failures),
|
|
741
|
+
checked_files=list(strict_report.checked_files),
|
|
742
|
+
details={
|
|
743
|
+
'boundary_path': strict_config.get('boundary_path', str(DEFAULT_STRICT_TARGET_BOUNDARY_PATH)),
|
|
744
|
+
'required_rfcs': sorted(strict_report.rfc_status),
|
|
745
|
+
},
|
|
746
|
+
)
|
|
747
|
+
|
|
748
|
+
flag_section = _evaluate_flag_contract_target(source_root, dict(target.get('flag_surface', {})))
|
|
749
|
+
operator_section = _evaluate_operator_surface_target(source_root, dict(target.get('operator_surface', {})))
|
|
750
|
+
performance_section = _evaluate_performance_target(source_root, dict(target.get('performance', {})))
|
|
751
|
+
documentation_section = _evaluate_documentation_claim_consistency(source_root, dict(target.get('documentation', {})))
|
|
752
|
+
|
|
753
|
+
sections = [
|
|
754
|
+
authoritative_section,
|
|
755
|
+
strict_section,
|
|
756
|
+
flag_section,
|
|
757
|
+
operator_section,
|
|
758
|
+
performance_section,
|
|
759
|
+
documentation_section,
|
|
760
|
+
]
|
|
761
|
+
|
|
762
|
+
failures: list[str] = []
|
|
763
|
+
for section in sections:
|
|
764
|
+
checked_files.extend(section.checked_files)
|
|
765
|
+
failures.extend(f'[{section.name}] {failure}' for failure in section.failures)
|
|
766
|
+
|
|
767
|
+
checked_files = list(dict.fromkeys(checked_files))
|
|
768
|
+
return PromotionTargetReport(
|
|
769
|
+
passed=all(section.passed for section in sections),
|
|
770
|
+
failures=failures,
|
|
771
|
+
checked_files=checked_files,
|
|
772
|
+
authoritative_boundary=authoritative_section,
|
|
773
|
+
strict_target_boundary=strict_section,
|
|
774
|
+
flag_surface=flag_section,
|
|
775
|
+
operator_surface=operator_section,
|
|
776
|
+
performance=performance_section,
|
|
777
|
+
documentation=documentation_section,
|
|
778
|
+
)
|
|
779
|
+
|
|
780
|
+
|
|
781
|
+
def assert_promotion_target_ready(
|
|
782
|
+
source_root: str | Path,
|
|
783
|
+
*,
|
|
784
|
+
target_path: str | Path | None = None,
|
|
785
|
+
) -> None:
|
|
786
|
+
report = evaluate_promotion_target(source_root, target_path=target_path)
|
|
787
|
+
if not report.passed:
|
|
788
|
+
details = '\n'.join(f'- {item}' for item in report.failures)
|
|
789
|
+
raise PromotionTargetError(f'promotion target failed:\n{details}')
|
|
790
|
+
|
|
791
|
+
|
|
792
|
+
def _evaluate_flag_contract_target(source_root: Path, config: Mapping[str, Any]) -> PromotionSectionReport:
|
|
793
|
+
contracts_file = source_root / Path(str(config.get('contracts_path', 'docs/review/conformance/flag_contracts.json')))
|
|
794
|
+
covering_file = source_root / Path(str(config.get('covering_array_path', 'docs/review/conformance/flag_covering_array.json')))
|
|
795
|
+
checked_files = [str(contracts_file), str(covering_file), str(source_root / 'src/tigrcorn/cli.py')]
|
|
796
|
+
failures: list[str] = []
|
|
797
|
+
details: dict[str, Any] = {}
|
|
798
|
+
|
|
799
|
+
if not contracts_file.exists():
|
|
800
|
+
failures.append(f'missing flag contracts file: {contracts_file}')
|
|
801
|
+
return PromotionSectionReport('flag_surface', False, failures, checked_files, details)
|
|
802
|
+
if not covering_file.exists():
|
|
803
|
+
failures.append(f'missing flag covering-array file: {covering_file}')
|
|
804
|
+
return PromotionSectionReport('flag_surface', False, failures, checked_files, details)
|
|
805
|
+
|
|
806
|
+
contracts_payload = json.loads(contracts_file.read_text(encoding='utf-8'))
|
|
807
|
+
covering_payload = json.loads(covering_file.read_text(encoding='utf-8'))
|
|
808
|
+
public_flags = _load_public_parser_flags()
|
|
809
|
+
required_fields = [str(item) for item in config.get('required_contract_fields', [])]
|
|
810
|
+
|
|
811
|
+
contracts = list(contracts_payload.get('contracts', []))
|
|
812
|
+
if contracts_payload.get('contract_mode') != 'one_row_per_concrete_public_flag':
|
|
813
|
+
failures.append('flag contracts must declare contract_mode=one_row_per_concrete_public_flag')
|
|
814
|
+
|
|
815
|
+
seen: dict[str, int] = {}
|
|
816
|
+
non_ready: list[str] = []
|
|
817
|
+
runtime_gaps: list[str] = []
|
|
818
|
+
for row in contracts:
|
|
819
|
+
for field_name in required_fields:
|
|
820
|
+
if field_name not in row:
|
|
821
|
+
failures.append(f'flag contract is missing required field {field_name!r}: {row!r}')
|
|
822
|
+
flag_strings = row.get('flag_strings', [])
|
|
823
|
+
if not isinstance(flag_strings, list) or len(flag_strings) != 1 or not isinstance(flag_strings[0], str):
|
|
824
|
+
failures.append(f'flag contract must contain exactly one concrete flag string: {row!r}')
|
|
825
|
+
continue
|
|
826
|
+
flag = flag_strings[0]
|
|
827
|
+
seen[flag] = seen.get(flag, 0) + 1
|
|
828
|
+
status = dict(row.get('status', {})) if isinstance(row.get('status'), Mapping) else {}
|
|
829
|
+
if not bool(status.get('contract_defined', False)):
|
|
830
|
+
failures.append(f'{flag} contract is not marked contract_defined=true')
|
|
831
|
+
if not bool(status.get('promotion_ready', False)):
|
|
832
|
+
non_ready.append(flag)
|
|
833
|
+
runtime_state = str(status.get('current_runtime_state', 'unknown'))
|
|
834
|
+
if runtime_state in {'parse_only', 'partially_wired', 'runtime_gap'}:
|
|
835
|
+
runtime_gaps.append(flag)
|
|
836
|
+
|
|
837
|
+
public_flag_set = set(public_flags)
|
|
838
|
+
documented_flag_set = set(seen)
|
|
839
|
+
missing_contracts = sorted(public_flag_set - documented_flag_set)
|
|
840
|
+
extra_contracts = sorted(documented_flag_set - public_flag_set)
|
|
841
|
+
duplicate_contracts = sorted(flag for flag, count in seen.items() if count > 1)
|
|
842
|
+
if missing_contracts:
|
|
843
|
+
failures.append(f'flag contracts are missing concrete public flags: {missing_contracts}')
|
|
844
|
+
if extra_contracts:
|
|
845
|
+
failures.append(f'flag contracts declare non-public flags: {extra_contracts}')
|
|
846
|
+
if duplicate_contracts:
|
|
847
|
+
failures.append(f'flag contracts declare duplicate rows: {duplicate_contracts}')
|
|
848
|
+
expected_public_count = int(contracts_payload.get('public_flag_string_count', len(public_flag_set)))
|
|
849
|
+
if expected_public_count != len(public_flag_set):
|
|
850
|
+
failures.append(
|
|
851
|
+
f'flag contracts public_flag_string_count={expected_public_count} does not match parser public flag count={len(public_flag_set)}'
|
|
852
|
+
)
|
|
853
|
+
if len(contracts) != len(public_flag_set):
|
|
854
|
+
failures.append(
|
|
855
|
+
f'flag contracts contain {len(contracts)} rows but the parser exposes {len(public_flag_set)} concrete public flags'
|
|
856
|
+
)
|
|
857
|
+
|
|
858
|
+
cases = list(covering_payload.get('cases', []))
|
|
859
|
+
covered_flags: set[str] = set()
|
|
860
|
+
for case in cases:
|
|
861
|
+
for dimension in case.get('dimensions', []):
|
|
862
|
+
if not isinstance(dimension, Mapping):
|
|
863
|
+
continue
|
|
864
|
+
flag = dimension.get('flag')
|
|
865
|
+
if isinstance(flag, str):
|
|
866
|
+
covered_flags.add(flag)
|
|
867
|
+
missing_coverage = sorted(public_flag_set - covered_flags)
|
|
868
|
+
if missing_coverage:
|
|
869
|
+
failures.append(f'flag covering array does not exercise every public flag: {missing_coverage}')
|
|
870
|
+
|
|
871
|
+
declared_hazard_clusters = {
|
|
872
|
+
str(cluster.get('cluster_id'))
|
|
873
|
+
for cluster in covering_payload.get('hazard_clusters', [])
|
|
874
|
+
if isinstance(cluster, Mapping) and cluster.get('cluster_id')
|
|
875
|
+
}
|
|
876
|
+
for cluster_id in [str(item) for item in config.get('required_hazard_clusters', [])]:
|
|
877
|
+
if cluster_id not in declared_hazard_clusters:
|
|
878
|
+
failures.append(f'flag covering array is missing required hazard cluster {cluster_id!r}')
|
|
879
|
+
|
|
880
|
+
if non_ready:
|
|
881
|
+
failures.append(
|
|
882
|
+
'flag surface still has non-promotion-ready contracts: ' + ', '.join(sorted(non_ready))
|
|
883
|
+
)
|
|
884
|
+
|
|
885
|
+
details.update(
|
|
886
|
+
{
|
|
887
|
+
'public_flag_count': len(public_flag_set),
|
|
888
|
+
'contract_row_count': len(contracts),
|
|
889
|
+
'promotion_ready_count': len(contracts) - len(non_ready),
|
|
890
|
+
'runtime_gap_flags': sorted(runtime_gaps),
|
|
891
|
+
'missing_contracts': missing_contracts,
|
|
892
|
+
'missing_coverage': missing_coverage,
|
|
893
|
+
'hazard_cluster_count': len(declared_hazard_clusters),
|
|
894
|
+
}
|
|
895
|
+
)
|
|
896
|
+
return PromotionSectionReport('flag_surface', not failures, failures, checked_files, details)
|
|
897
|
+
|
|
898
|
+
|
|
899
|
+
|
|
900
|
+
def _evaluate_operator_surface_target(source_root: Path, config: Mapping[str, Any]) -> PromotionSectionReport:
|
|
901
|
+
index_file = source_root / Path(str(config.get('bundle_index', 'docs/review/conformance/releases/0.3.7/release-0.3.7/tigrcorn-operator-surface-certification-bundle/index.json')))
|
|
902
|
+
checked_files = [str(index_file)]
|
|
903
|
+
failures: list[str] = []
|
|
904
|
+
details: dict[str, Any] = {}
|
|
905
|
+
if not index_file.exists():
|
|
906
|
+
failures.append(f'missing operator-surface bundle index: {index_file}')
|
|
907
|
+
return PromotionSectionReport('operator_surface', False, failures, checked_files, details)
|
|
908
|
+
payload = json.loads(index_file.read_text(encoding='utf-8'))
|
|
909
|
+
implemented = dict(payload.get('implemented', {}))
|
|
910
|
+
required_keys = [str(item) for item in config.get('required_implemented_keys', [])]
|
|
911
|
+
if not bool(payload.get('release_gate_eligible', False)):
|
|
912
|
+
failures.append('operator-surface certification bundle is not release_gate_eligible')
|
|
913
|
+
missing_keys = [key for key in required_keys if key not in implemented]
|
|
914
|
+
false_keys = [key for key in required_keys if implemented.get(key) is not True]
|
|
915
|
+
if missing_keys:
|
|
916
|
+
failures.append(f'operator-surface bundle is missing required implementation keys: {missing_keys}')
|
|
917
|
+
if false_keys:
|
|
918
|
+
failures.append(f'operator-surface bundle contains non-green required implementation keys: {false_keys}')
|
|
919
|
+
details.update(
|
|
920
|
+
{
|
|
921
|
+
'implemented_count': int(payload.get('implemented_count', len([item for item in implemented.values() if item]))),
|
|
922
|
+
'required_implemented_keys': required_keys,
|
|
923
|
+
'implemented_keys': sorted(implemented),
|
|
924
|
+
}
|
|
925
|
+
)
|
|
926
|
+
return PromotionSectionReport('operator_surface', not failures, failures, checked_files, details)
|
|
927
|
+
|
|
928
|
+
|
|
929
|
+
|
|
930
|
+
def _evaluate_performance_target(source_root: Path, config: Mapping[str, Any]) -> PromotionSectionReport:
|
|
931
|
+
from .perf_runner import load_performance_matrix, validate_performance_artifacts
|
|
932
|
+
|
|
933
|
+
matrix_path = Path(str(config.get('matrix_path', 'docs/review/performance/performance_matrix.json')))
|
|
934
|
+
slos_path = Path(str(config.get('slos_path', 'docs/review/performance/performance_slos.json')))
|
|
935
|
+
current_artifact_root = Path(str(config.get('current_artifact_root', 'docs/review/performance/artifacts/phase6_current_release')))
|
|
936
|
+
baseline_artifact_root = Path(str(config.get('baseline_artifact_root', 'docs/review/performance/artifacts/phase6_reference_baseline')))
|
|
937
|
+
checked_files = [str(source_root / matrix_path), str(source_root / slos_path), str(source_root / current_artifact_root)]
|
|
938
|
+
failures: list[str] = []
|
|
939
|
+
details: dict[str, Any] = {}
|
|
940
|
+
|
|
941
|
+
if not (source_root / matrix_path).exists():
|
|
942
|
+
failures.append(f'missing performance matrix file: {source_root / matrix_path}')
|
|
943
|
+
return PromotionSectionReport('performance', False, failures, checked_files, details)
|
|
944
|
+
if not (source_root / slos_path).exists():
|
|
945
|
+
failures.append(f'missing performance SLO target file: {source_root / slos_path}')
|
|
946
|
+
return PromotionSectionReport('performance', False, failures, checked_files, details)
|
|
947
|
+
|
|
948
|
+
matrix = load_performance_matrix(source_root / matrix_path)
|
|
949
|
+
slos_payload = json.loads((source_root / slos_path).read_text(encoding='utf-8'))
|
|
950
|
+
artifact_failures = validate_performance_artifacts(
|
|
951
|
+
source_root,
|
|
952
|
+
matrix_path=matrix_path,
|
|
953
|
+
artifact_root=current_artifact_root,
|
|
954
|
+
baseline_root=baseline_artifact_root,
|
|
955
|
+
require_relative_regression=bool(config.get('require_relative_regression', False)),
|
|
956
|
+
)
|
|
957
|
+
failures.extend(artifact_failures)
|
|
958
|
+
|
|
959
|
+
required_metric_keys = {str(item) for item in slos_payload.get('required_metric_keys', [])}
|
|
960
|
+
required_threshold_keys = {str(item) for item in slos_payload.get('required_threshold_keys', [])}
|
|
961
|
+
required_relative_budget_keys = {str(item) for item in slos_payload.get('required_relative_regression_budget_keys', [])}
|
|
962
|
+
required_artifact_files = {str(item) for item in slos_payload.get('required_artifact_files', [])}
|
|
963
|
+
required_matrix_lanes = {str(item) for item in slos_payload.get('required_matrix_lanes', [])}
|
|
964
|
+
promotion_requirements = dict(slos_payload.get('promotion_requirements', {}))
|
|
965
|
+
|
|
966
|
+
require_full_declared_strict_contract = bool(config.get('require_full_declared_strict_contract', False))
|
|
967
|
+
require_artifact_files = require_full_declared_strict_contract or bool(config.get('require_required_artifact_files', False))
|
|
968
|
+
require_matrix_lanes = require_full_declared_strict_contract or bool(config.get('require_required_matrix_lanes', False))
|
|
969
|
+
require_certification_platforms = (
|
|
970
|
+
require_full_declared_strict_contract
|
|
971
|
+
or bool(config.get('require_certification_platform_declarations', False))
|
|
972
|
+
or bool(promotion_requirements.get('require_certification_platforms', False))
|
|
973
|
+
)
|
|
974
|
+
require_documented_slos_per_profile = (
|
|
975
|
+
require_full_declared_strict_contract
|
|
976
|
+
or bool(config.get('require_documented_slos_per_profile', False))
|
|
977
|
+
or bool(promotion_requirements.get('require_documented_slos_per_profile', False))
|
|
978
|
+
)
|
|
979
|
+
require_correctness_for_rfc_targets = (
|
|
980
|
+
require_full_declared_strict_contract
|
|
981
|
+
or bool(config.get('require_correctness_for_rfc_profiles', False))
|
|
982
|
+
or bool(promotion_requirements.get('require_correctness_under_load_for_rfc_targets', False))
|
|
983
|
+
)
|
|
984
|
+
require_live_listener_metadata = (
|
|
985
|
+
require_full_declared_strict_contract
|
|
986
|
+
or bool(config.get('require_live_listener_metadata_for_end_to_end_profiles', False))
|
|
987
|
+
or bool(promotion_requirements.get('require_end_to_end_live_listener_profiles', False))
|
|
988
|
+
)
|
|
989
|
+
|
|
990
|
+
observed_metric_keys = _load_performance_metric_keys(source_root / current_artifact_root, [profile.profile_id for profile in matrix.profiles])
|
|
991
|
+
declared_threshold_keys = {key for profile in matrix.profiles for key in profile.thresholds}
|
|
992
|
+
declared_relative_keys = {key for profile in matrix.profiles for key in profile.relative_regression_budget}
|
|
993
|
+
|
|
994
|
+
missing_metric_keys = sorted(required_metric_keys - observed_metric_keys)
|
|
995
|
+
missing_threshold_keys = sorted(required_threshold_keys - declared_threshold_keys)
|
|
996
|
+
missing_relative_keys = sorted(required_relative_budget_keys - declared_relative_keys)
|
|
997
|
+
if missing_metric_keys:
|
|
998
|
+
failures.append(f'performance artifacts are missing required SLO metric keys: {missing_metric_keys}')
|
|
999
|
+
if missing_threshold_keys:
|
|
1000
|
+
failures.append(f'performance matrix is missing required absolute threshold keys: {missing_threshold_keys}')
|
|
1001
|
+
if missing_relative_keys:
|
|
1002
|
+
failures.append(f'performance matrix is missing required relative regression budget keys: {missing_relative_keys}')
|
|
1003
|
+
|
|
1004
|
+
artifact_root_path = source_root / current_artifact_root
|
|
1005
|
+
root_summary_path = artifact_root_path / 'summary.json'
|
|
1006
|
+
root_index_path = artifact_root_path / 'index.json'
|
|
1007
|
+
root_summary = _load_json_payload(root_summary_path) if root_summary_path.exists() else {}
|
|
1008
|
+
root_index = _load_json_payload(root_index_path) if root_index_path.exists() else {}
|
|
1009
|
+
|
|
1010
|
+
if require_artifact_files:
|
|
1011
|
+
required_root_files, required_profile_files = _split_required_performance_artifact_files(required_artifact_files)
|
|
1012
|
+
missing_root_files = sorted(filename for filename in required_root_files if not (artifact_root_path / filename).exists())
|
|
1013
|
+
if missing_root_files:
|
|
1014
|
+
failures.append(f'performance artifact root is missing required files: {missing_root_files}')
|
|
1015
|
+
for profile in matrix.profiles:
|
|
1016
|
+
profile_dir = artifact_root_path / profile.profile_id
|
|
1017
|
+
missing_profile_files = sorted(filename for filename in required_profile_files if not (profile_dir / filename).exists())
|
|
1018
|
+
if missing_profile_files:
|
|
1019
|
+
failures.append(f'{profile.profile_id} performance artifact directory is missing required files: {missing_profile_files}')
|
|
1020
|
+
|
|
1021
|
+
if require_matrix_lanes:
|
|
1022
|
+
declared_lanes = {profile.lane for profile in matrix.profiles}
|
|
1023
|
+
missing_lanes = sorted(required_matrix_lanes - declared_lanes)
|
|
1024
|
+
if missing_lanes:
|
|
1025
|
+
failures.append(f'performance matrix is missing required lanes: {missing_lanes}')
|
|
1026
|
+
lane_counts = root_summary.get('lane_counts', {}) if isinstance(root_summary, Mapping) else {}
|
|
1027
|
+
lane_count_keys = {str(key) for key in lane_counts} if isinstance(lane_counts, Mapping) else set()
|
|
1028
|
+
missing_lane_counts = sorted(required_matrix_lanes - lane_count_keys)
|
|
1029
|
+
if missing_lane_counts:
|
|
1030
|
+
failures.append(f'performance artifact summary is missing required lane counts: {missing_lane_counts}')
|
|
1031
|
+
for lane in sorted(required_matrix_lanes & lane_count_keys):
|
|
1032
|
+
try:
|
|
1033
|
+
if int(lane_counts[lane]) <= 0:
|
|
1034
|
+
failures.append(f'performance artifact summary declares non-positive count for required lane {lane!r}')
|
|
1035
|
+
except Exception:
|
|
1036
|
+
failures.append(f'performance artifact summary carries a non-integer lane count for required lane {lane!r}')
|
|
1037
|
+
|
|
1038
|
+
matrix_platforms = [str(item) for item in matrix.metadata.get('certification_platforms', [])]
|
|
1039
|
+
if require_certification_platforms and not matrix_platforms:
|
|
1040
|
+
failures.append('performance matrix metadata is missing certification_platforms declarations')
|
|
1041
|
+
root_certification_platform = ''
|
|
1042
|
+
if isinstance(root_summary, Mapping):
|
|
1043
|
+
if root_summary.get('certification_platform') is not None:
|
|
1044
|
+
root_certification_platform = str(root_summary.get('certification_platform', ''))
|
|
1045
|
+
elif root_summary.get('certification_platforms'):
|
|
1046
|
+
platforms = root_summary.get('certification_platforms')
|
|
1047
|
+
if isinstance(platforms, list) and platforms:
|
|
1048
|
+
root_certification_platform = str(platforms[0])
|
|
1049
|
+
if require_certification_platforms and not root_certification_platform:
|
|
1050
|
+
failures.append('performance artifact summary is missing certification platform declarations')
|
|
1051
|
+
|
|
1052
|
+
if require_matrix_lanes and isinstance(root_index, Mapping):
|
|
1053
|
+
summary_profiles = root_index.get('profiles', []) or root_index.get('scenarios', []) or []
|
|
1054
|
+
details['artifact_profile_entry_count'] = len(summary_profiles) if isinstance(summary_profiles, list) else 0
|
|
1055
|
+
|
|
1056
|
+
profile_failures: dict[str, list[str]] = {}
|
|
1057
|
+
for profile in matrix.profiles:
|
|
1058
|
+
profile_dir = artifact_root_path / profile.profile_id
|
|
1059
|
+
result_payload = _load_json_payload(profile_dir / 'result.json') if (profile_dir / 'result.json').exists() else {}
|
|
1060
|
+
summary_payload = _load_json_payload(profile_dir / 'summary.json') if (profile_dir / 'summary.json').exists() else {}
|
|
1061
|
+
command_payload = _load_json_payload(profile_dir / 'command.json') if (profile_dir / 'command.json').exists() else {}
|
|
1062
|
+
env_payload = _load_json_payload(profile_dir / 'env.json') if (profile_dir / 'env.json').exists() else {}
|
|
1063
|
+
correctness_payload = _load_json_payload(profile_dir / 'correctness.json') if (profile_dir / 'correctness.json').exists() else {}
|
|
1064
|
+
|
|
1065
|
+
current_profile_failures: list[str] = []
|
|
1066
|
+
|
|
1067
|
+
if require_documented_slos_per_profile:
|
|
1068
|
+
if not str(profile.description).strip():
|
|
1069
|
+
current_profile_failures.append('missing non-empty profile description for documented SLO coverage')
|
|
1070
|
+
missing_profile_threshold_keys = sorted(required_threshold_keys - set(profile.thresholds))
|
|
1071
|
+
if missing_profile_threshold_keys:
|
|
1072
|
+
current_profile_failures.append(f'missing required threshold keys: {missing_profile_threshold_keys}')
|
|
1073
|
+
missing_profile_relative_keys = sorted(required_relative_budget_keys - set(profile.relative_regression_budget))
|
|
1074
|
+
if missing_profile_relative_keys:
|
|
1075
|
+
current_profile_failures.append(f'missing required relative regression budget keys: {missing_profile_relative_keys}')
|
|
1076
|
+
|
|
1077
|
+
if require_certification_platforms:
|
|
1078
|
+
if not profile.certification_platforms:
|
|
1079
|
+
current_profile_failures.append('missing profile certification_platforms declarations in matrix')
|
|
1080
|
+
if not result_payload.get('certification_platforms'):
|
|
1081
|
+
current_profile_failures.append('missing result.json certification_platforms declarations')
|
|
1082
|
+
if not summary_payload.get('certification_platforms'):
|
|
1083
|
+
current_profile_failures.append('missing summary.json certification_platforms declarations')
|
|
1084
|
+
if not command_payload.get('certification_platforms'):
|
|
1085
|
+
current_profile_failures.append('missing command.json certification_platforms declarations')
|
|
1086
|
+
if not env_payload.get('certification_platform'):
|
|
1087
|
+
current_profile_failures.append('missing env.json certification_platform declaration')
|
|
1088
|
+
if not env_payload.get('matrix_declared_platforms'):
|
|
1089
|
+
current_profile_failures.append('missing env.json matrix_declared_platforms declaration')
|
|
1090
|
+
|
|
1091
|
+
if require_correctness_for_rfc_targets and profile.rfc_targets:
|
|
1092
|
+
if not profile.correctness_required:
|
|
1093
|
+
current_profile_failures.append('RFC-scoped profile is not marked correctness_required=true in the matrix')
|
|
1094
|
+
checks = correctness_payload.get('checks', {}) if isinstance(correctness_payload, Mapping) else {}
|
|
1095
|
+
if not bool(correctness_payload.get('required', False)):
|
|
1096
|
+
current_profile_failures.append('correctness.json is not marked required=true for an RFC-scoped profile')
|
|
1097
|
+
if not bool(correctness_payload.get('passed', False)):
|
|
1098
|
+
current_profile_failures.append('correctness.json does not record passed=true for an RFC-scoped profile')
|
|
1099
|
+
if not isinstance(checks, Mapping) or not checks:
|
|
1100
|
+
current_profile_failures.append('correctness.json is missing correctness checks for an RFC-scoped profile')
|
|
1101
|
+
|
|
1102
|
+
if require_live_listener_metadata and profile.lane == 'end_to_end_release':
|
|
1103
|
+
if not profile.live_listener_required:
|
|
1104
|
+
current_profile_failures.append('end_to_end_release profile is not marked live_listener_required=true in the matrix')
|
|
1105
|
+
for filename, payload in [
|
|
1106
|
+
('result.json', result_payload),
|
|
1107
|
+
('summary.json', summary_payload),
|
|
1108
|
+
('command.json', command_payload),
|
|
1109
|
+
('correctness.json', correctness_payload),
|
|
1110
|
+
]:
|
|
1111
|
+
if payload and payload.get('live_listener_required') is not True:
|
|
1112
|
+
current_profile_failures.append(f'{filename} does not preserve live_listener_required=true for an end_to_end_release profile')
|
|
1113
|
+
if payload and str(payload.get('lane', '')) != 'end_to_end_release':
|
|
1114
|
+
current_profile_failures.append(f'{filename} does not preserve lane="end_to_end_release" for an end_to_end_release profile')
|
|
1115
|
+
|
|
1116
|
+
if current_profile_failures:
|
|
1117
|
+
profile_failures[profile.profile_id] = list(current_profile_failures)
|
|
1118
|
+
failures.extend(f'{profile.profile_id} {message}' for message in current_profile_failures)
|
|
1119
|
+
|
|
1120
|
+
details.update(
|
|
1121
|
+
{
|
|
1122
|
+
'profile_count': len(matrix.profiles),
|
|
1123
|
+
'required_metric_keys': sorted(required_metric_keys),
|
|
1124
|
+
'observed_metric_keys': sorted(observed_metric_keys),
|
|
1125
|
+
'missing_metric_keys': missing_metric_keys,
|
|
1126
|
+
'missing_threshold_keys': missing_threshold_keys,
|
|
1127
|
+
'missing_relative_budget_keys': missing_relative_keys,
|
|
1128
|
+
'required_artifact_files': sorted(required_artifact_files),
|
|
1129
|
+
'required_matrix_lanes': sorted(required_matrix_lanes),
|
|
1130
|
+
'certification_platforms': matrix_platforms,
|
|
1131
|
+
'profile_failures': profile_failures,
|
|
1132
|
+
'require_full_declared_strict_contract': require_full_declared_strict_contract,
|
|
1133
|
+
'require_certification_platforms': require_certification_platforms,
|
|
1134
|
+
'require_documented_slos_per_profile': require_documented_slos_per_profile,
|
|
1135
|
+
'require_correctness_for_rfc_targets': require_correctness_for_rfc_targets,
|
|
1136
|
+
'require_live_listener_metadata': require_live_listener_metadata,
|
|
1137
|
+
}
|
|
1138
|
+
)
|
|
1139
|
+
return PromotionSectionReport('performance', not failures, failures, checked_files, details)
|
|
1140
|
+
|
|
1141
|
+
|
|
1142
|
+
|
|
1143
|
+
def _evaluate_documentation_claim_consistency(source_root: Path, config: Mapping[str, Any]) -> PromotionSectionReport:
|
|
1144
|
+
checks = list(config.get('required_phrase_checks', []))
|
|
1145
|
+
failures: list[str] = []
|
|
1146
|
+
checked_files: list[str] = []
|
|
1147
|
+
details: dict[str, Any] = {'documents_checked': len(checks)}
|
|
1148
|
+
|
|
1149
|
+
for check in checks:
|
|
1150
|
+
if not isinstance(check, Mapping):
|
|
1151
|
+
failures.append(f'malformed documentation phrase check: {check!r}')
|
|
1152
|
+
continue
|
|
1153
|
+
doc_file = source_root / Path(str(check.get('path', '')))
|
|
1154
|
+
checked_files.append(str(doc_file))
|
|
1155
|
+
if not doc_file.exists():
|
|
1156
|
+
failures.append(f'missing documentation file for claim-consistency check: {doc_file}')
|
|
1157
|
+
continue
|
|
1158
|
+
text = doc_file.read_text(encoding='utf-8')
|
|
1159
|
+
for needle in [str(item) for item in check.get('must_contain', [])]:
|
|
1160
|
+
if needle not in text:
|
|
1161
|
+
failures.append(f'{doc_file} is missing required phrase: {needle!r}')
|
|
1162
|
+
for needle in [str(item) for item in check.get('must_not_contain', [])]:
|
|
1163
|
+
if needle in text:
|
|
1164
|
+
failures.append(f'{doc_file} contains forbidden phrase: {needle!r}')
|
|
1165
|
+
|
|
1166
|
+
return PromotionSectionReport('documentation', not failures, failures, checked_files, details)
|
|
1167
|
+
|
|
1168
|
+
|
|
1169
|
+
def _evaluate_governance_graph(*, source_root: Path, checked_files: list[str]) -> list[str]:
|
|
1170
|
+
failures: list[str] = []
|
|
1171
|
+
ssot_registry_path = source_root / DEFAULT_SSOT_REGISTRY_PATH
|
|
1172
|
+
claims_path = source_root / DEFAULT_CLAIMS_REGISTRY_PATH
|
|
1173
|
+
risk_register_path = source_root / DEFAULT_RISK_REGISTER_PATH
|
|
1174
|
+
risk_traceability_path = source_root / DEFAULT_RISK_TRACEABILITY_PATH
|
|
1175
|
+
legacy_inventory_path = source_root / DEFAULT_LEGACY_UNITTEST_INVENTORY_PATH
|
|
1176
|
+
checked_files.extend(str(path) for path in (ssot_registry_path, claims_path, risk_register_path, risk_traceability_path, legacy_inventory_path))
|
|
1177
|
+
|
|
1178
|
+
for path in (ssot_registry_path, claims_path, risk_register_path, risk_traceability_path, legacy_inventory_path):
|
|
1179
|
+
if not path.exists():
|
|
1180
|
+
failures.append(f'missing governance graph input: {path}')
|
|
1181
|
+
if failures:
|
|
1182
|
+
return failures
|
|
1183
|
+
|
|
1184
|
+
ssot_payload = _load_json_payload(ssot_registry_path)
|
|
1185
|
+
repo = ssot_payload.get('repo', {})
|
|
1186
|
+
if str(repo.get('name', '')).strip() != 'tigrcorn':
|
|
1187
|
+
failures.append('ssot registry repo.name is not "tigrcorn"')
|
|
1188
|
+
active_boundary_id = str(ssot_payload.get('program', {}).get('active_boundary_id', '')).strip()
|
|
1189
|
+
active_release_id = str(ssot_payload.get('program', {}).get('active_release_id', '')).strip()
|
|
1190
|
+
if not active_boundary_id:
|
|
1191
|
+
failures.append('ssot registry is missing program.active_boundary_id')
|
|
1192
|
+
if not active_release_id:
|
|
1193
|
+
failures.append('ssot registry is missing program.active_release_id')
|
|
1194
|
+
boundaries = {
|
|
1195
|
+
str(row.get('id', '')): row
|
|
1196
|
+
for row in ssot_payload.get('boundaries', [])
|
|
1197
|
+
if isinstance(row, Mapping)
|
|
1198
|
+
}
|
|
1199
|
+
releases = {
|
|
1200
|
+
str(row.get('id', '')): row
|
|
1201
|
+
for row in ssot_payload.get('releases', [])
|
|
1202
|
+
if isinstance(row, Mapping)
|
|
1203
|
+
}
|
|
1204
|
+
if active_boundary_id not in boundaries:
|
|
1205
|
+
failures.append('ssot registry active boundary id does not resolve to a boundary row')
|
|
1206
|
+
if active_release_id not in releases:
|
|
1207
|
+
failures.append('ssot registry active release id does not resolve to a release row')
|
|
1208
|
+
if active_boundary_id in boundaries and boundaries[active_boundary_id].get('canonical_registry_source') != '.ssot/registry.json':
|
|
1209
|
+
failures.append('ssot registry active boundary does not self-identify .ssot/registry.json as canonical_registry_source')
|
|
1210
|
+
|
|
1211
|
+
claims_payload = _load_json_payload(claims_path)
|
|
1212
|
+
claim_ids = {str(row.get('id', '')) for row in claims_payload.get('current_and_candidate_claims', []) if isinstance(row, Mapping)}
|
|
1213
|
+
|
|
1214
|
+
register_payload = _load_json_payload(risk_register_path)
|
|
1215
|
+
traceability_payload = _load_json_payload(risk_traceability_path)
|
|
1216
|
+
inventory_payload = _load_json_payload(legacy_inventory_path)
|
|
1217
|
+
|
|
1218
|
+
register_rows = register_payload.get('register', [])
|
|
1219
|
+
traceability_rows = traceability_payload.get('risks', [])
|
|
1220
|
+
if not isinstance(register_rows, list) or not isinstance(traceability_rows, list):
|
|
1221
|
+
failures.append('risk register or risk traceability payload is malformed')
|
|
1222
|
+
return failures
|
|
1223
|
+
|
|
1224
|
+
register_ids = {str(row.get('risk_id', '')) for row in register_rows if isinstance(row, Mapping)}
|
|
1225
|
+
traceability_ids = {str(row.get('risk_id', '')) for row in traceability_rows if isinstance(row, Mapping)}
|
|
1226
|
+
if register_ids != traceability_ids:
|
|
1227
|
+
failures.append('risk register and risk traceability files disagree on declared risk ids')
|
|
1228
|
+
|
|
1229
|
+
open_blocking_statuses = {'open', 'active', 'unmitigated', 'planned'}
|
|
1230
|
+
for row in register_rows:
|
|
1231
|
+
if not isinstance(row, Mapping):
|
|
1232
|
+
continue
|
|
1233
|
+
if bool(row.get('release_gate_blocking', False)) and str(row.get('status', '')).strip().lower() in open_blocking_statuses:
|
|
1234
|
+
failures.append(f'blocking risk {row.get("risk_id")} remains open with status={row.get("status")!r}')
|
|
1235
|
+
|
|
1236
|
+
for row in traceability_rows:
|
|
1237
|
+
if not isinstance(row, Mapping):
|
|
1238
|
+
continue
|
|
1239
|
+
risk_id = str(row.get('risk_id', ''))
|
|
1240
|
+
for claim_ref in row.get('claim_refs', []):
|
|
1241
|
+
if str(claim_ref) not in claim_ids:
|
|
1242
|
+
failures.append(f'risk traceability row {risk_id} references unknown claim {claim_ref!r}')
|
|
1243
|
+
for test_ref in row.get('test_refs', []):
|
|
1244
|
+
test_path = source_root / Path(str(test_ref).split('::', 1)[0])
|
|
1245
|
+
if not test_path.exists():
|
|
1246
|
+
failures.append(f'risk traceability row {risk_id} references missing test {test_ref!r}')
|
|
1247
|
+
for evidence_ref in row.get('evidence_refs', []):
|
|
1248
|
+
evidence_path = source_root / Path(str(evidence_ref))
|
|
1249
|
+
if not evidence_path.exists():
|
|
1250
|
+
failures.append(f'risk traceability row {risk_id} references missing evidence {evidence_ref!r}')
|
|
1251
|
+
|
|
1252
|
+
for group_name in ('interop_retention_bundles', 'performance_retention_bundles'):
|
|
1253
|
+
for row in traceability_payload.get(group_name, []):
|
|
1254
|
+
if not isinstance(row, Mapping):
|
|
1255
|
+
failures.append(f'{group_name} contains a malformed row')
|
|
1256
|
+
continue
|
|
1257
|
+
retained_path = source_root / Path(str(row.get('path', '')))
|
|
1258
|
+
if not retained_path.exists():
|
|
1259
|
+
failures.append(f'{group_name} references missing retained input {row.get("path")!r}')
|
|
1260
|
+
|
|
1261
|
+
approved_legacy = list(inventory_payload.get('approved_legacy_files', []))
|
|
1262
|
+
detected_legacy = list(inventory_payload.get('detected_legacy_files', []))
|
|
1263
|
+
unexpected_legacy = list(inventory_payload.get('unexpected_legacy_files', []))
|
|
1264
|
+
if inventory_payload.get('forward_runner') != 'pytest':
|
|
1265
|
+
failures.append('legacy unittest inventory does not declare pytest as the forward runner')
|
|
1266
|
+
if unexpected_legacy:
|
|
1267
|
+
failures.append(f'legacy unittest inventory contains unexpected files: {sorted(str(item) for item in unexpected_legacy)}')
|
|
1268
|
+
if set(approved_legacy) != set(detected_legacy):
|
|
1269
|
+
failures.append('legacy unittest inventory detected files do not match the approved grandfathered inventory')
|
|
1270
|
+
|
|
1271
|
+
return failures
|
|
1272
|
+
|
|
1273
|
+
|
|
1274
|
+
|
|
1275
|
+
|
|
1276
|
+
|
|
1277
|
+
|
|
1278
|
+
def _split_required_performance_artifact_files(required_files: Iterable[str]) -> tuple[set[str], set[str]]:
|
|
1279
|
+
required = {str(item) for item in required_files}
|
|
1280
|
+
root_files = {'summary.json', 'index.json'} & required
|
|
1281
|
+
profile_files = required - {'index.json'}
|
|
1282
|
+
return root_files, profile_files
|
|
1283
|
+
|
|
1284
|
+
|
|
1285
|
+
|
|
1286
|
+
def _load_json_payload(path: Path) -> dict[str, Any]:
|
|
1287
|
+
return json.loads(path.read_text(encoding='utf-8'))
|
|
1288
|
+
|
|
1289
|
+
def _load_public_parser_flags() -> dict[str, dict[str, Any]]:
|
|
1290
|
+
import argparse
|
|
1291
|
+
|
|
1292
|
+
from tigrcorn_runtime.cli import build_parser
|
|
1293
|
+
|
|
1294
|
+
parser = build_parser()
|
|
1295
|
+
public_flags: dict[str, dict[str, Any]] = {}
|
|
1296
|
+
for group in parser._action_groups:
|
|
1297
|
+
title = getattr(group, 'title', None)
|
|
1298
|
+
for action in getattr(group, '_group_actions', []):
|
|
1299
|
+
if isinstance(action, argparse._HelpAction):
|
|
1300
|
+
continue
|
|
1301
|
+
if action.help == argparse.SUPPRESS:
|
|
1302
|
+
continue
|
|
1303
|
+
for flag in action.option_strings:
|
|
1304
|
+
if not flag.startswith('--'):
|
|
1305
|
+
continue
|
|
1306
|
+
public_flags[flag] = {
|
|
1307
|
+
'dest': action.dest,
|
|
1308
|
+
'group': title,
|
|
1309
|
+
'choices': list(action.choices) if action.choices is not None else [],
|
|
1310
|
+
'nargs': action.nargs,
|
|
1311
|
+
'default': action.default,
|
|
1312
|
+
}
|
|
1313
|
+
return public_flags
|
|
1314
|
+
|
|
1315
|
+
|
|
1316
|
+
|
|
1317
|
+
def _load_performance_metric_keys(artifact_root: Path, profile_ids: list[str]) -> set[str]:
|
|
1318
|
+
metric_keys: set[str] = set()
|
|
1319
|
+
for profile_id in profile_ids:
|
|
1320
|
+
result_file = artifact_root / profile_id / 'result.json'
|
|
1321
|
+
if not result_file.exists():
|
|
1322
|
+
continue
|
|
1323
|
+
payload = json.loads(result_file.read_text(encoding='utf-8'))
|
|
1324
|
+
metrics = payload.get('metrics', {})
|
|
1325
|
+
if isinstance(metrics, Mapping):
|
|
1326
|
+
metric_keys.update(str(key) for key in metrics)
|
|
1327
|
+
return metric_keys
|
|
1328
|
+
|
|
1329
|
+
|
|
1330
|
+
__all__ = [
|
|
1331
|
+
'DEFAULT_BOUNDARY_PATH',
|
|
1332
|
+
'DEFAULT_CLAIMS_REGISTRY_PATH',
|
|
1333
|
+
'DEFAULT_CORPUS_PATH',
|
|
1334
|
+
'DEFAULT_INDEPENDENT_MATRIX_PATH',
|
|
1335
|
+
'DEFAULT_LEGACY_UNITTEST_INVENTORY_PATH',
|
|
1336
|
+
'DEFAULT_SAME_STACK_MATRIX_PATH',
|
|
1337
|
+
'DEFAULT_STRICT_TARGET_BOUNDARY_PATH',
|
|
1338
|
+
'DEFAULT_PROMOTION_TARGET_PATH',
|
|
1339
|
+
'DEFAULT_RISK_REGISTER_PATH',
|
|
1340
|
+
'DEFAULT_RISK_TRACEABILITY_PATH',
|
|
1341
|
+
'DEFAULT_SSOT_REGISTRY_PATH',
|
|
1342
|
+
'PromotionSectionReport',
|
|
1343
|
+
'PromotionTargetError',
|
|
1344
|
+
'PromotionTargetReport',
|
|
1345
|
+
'ReleaseGateError',
|
|
1346
|
+
'ReleaseGateReport',
|
|
1347
|
+
'assert_promotion_target_ready',
|
|
1348
|
+
'assert_release_ready',
|
|
1349
|
+
'evaluate_promotion_target',
|
|
1350
|
+
'evaluate_release_gates',
|
|
1351
|
+
'load_certification_boundary',
|
|
1352
|
+
'load_conformance_corpus',
|
|
1353
|
+
'load_promotion_target',
|
|
1354
|
+
]
|