sable-sphynx 0.7.0.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sable_sphynx-0.7.0.dev0.dist-info/METADATA +152 -0
- sable_sphynx-0.7.0.dev0.dist-info/RECORD +174 -0
- sable_sphynx-0.7.0.dev0.dist-info/WHEEL +5 -0
- sable_sphynx-0.7.0.dev0.dist-info/top_level.txt +1 -0
- sphynx/__init__.py +469 -0
- sphynx/__init__.pyi +376 -0
- sphynx/_analytical_client.py +77 -0
- sphynx/_analytical_values.py +252 -0
- sphynx/_audit_client.py +173 -0
- sphynx/_comparison_adapters.py +113 -0
- sphynx/_componentized_full_rag.py +34 -0
- sphynx/_composed_pipeline.py +109 -0
- sphynx/_full_rag_grading.py +218 -0
- sphynx/_full_rag_trace.py +221 -0
- sphynx/_local_eval.py +268 -0
- sphynx/_pipeline_core.py +243 -0
- sphynx/_pr_client.py +237 -0
- sphynx/_rag_diagnostics_calibration.py +285 -0
- sphynx/_rag_diagnostics_catalog.py +39 -0
- sphynx/_rag_diagnostics_common.py +26 -0
- sphynx/_rag_diagnostics_contracts.py +186 -0
- sphynx/_rag_diagnostics_engine.py +243 -0
- sphynx/_rag_diagnostics_importers.py +242 -0
- sphynx/_rag_diagnostics_ranking.py +81 -0
- sphynx/_rag_diagnostics_semantic.py +325 -0
- sphynx/_rag_ranking.py +149 -0
- sphynx/_speech_benchmark_scoring.py +140 -0
- sphynx/_speech_benchmark_sources.py +275 -0
- sphynx/_speech_complete_scoring.py +218 -0
- sphynx/_speech_complete_source.py +118 -0
- sphynx/_speech_kido.py +294 -0
- sphynx/_speech_scoring.py +77 -0
- sphynx/_track_capture.py +213 -0
- sphynx/analytical_views.py +515 -0
- sphynx/assessment_contracts.py +378 -0
- sphynx/assessment_selection.py +57 -0
- sphynx/audit_access.py +245 -0
- sphynx/audit_contracts.py +204 -0
- sphynx/audit_requests.py +180 -0
- sphynx/authoring.py +188 -0
- sphynx/benchmarks.py +568 -0
- sphynx/candidates.py +62 -0
- sphynx/catalog.py +272 -0
- sphynx/chunking.py +372 -0
- sphynx/clickhouse_reader.py +169 -0
- sphynx/client.py +486 -0
- sphynx/comparison_core.py +149 -0
- sphynx/comparison_views.py +223 -0
- sphynx/components.py +1661 -0
- sphynx/contracts.py +514 -0
- sphynx/customer_rerank.py +299 -0
- sphynx/customer_retrieval.py +146 -0
- sphynx/data/benchmark-releases.v1.json +164 -0
- sphynx/data/public-benchmark-release.v1.json +354 -0
- sphynx/data/rag-diagnostics-synthetic.v1.json +6982 -0
- sphynx/data/rag-paper-references.v1.json +176 -0
- sphynx/data/rag-paper-schemas.v1.json +1069 -0
- sphynx/data/rag-rerank-v1.json +507 -0
- sphynx/dataset_catalog_data.py +1214 -0
- sphynx/dataset_readiness.py +138 -0
- sphynx/dataset_v3.py +999 -0
- sphynx/demo.py +130 -0
- sphynx/demo_candidates/__init__.py +1 -0
- sphynx/demo_candidates/dense.py +11 -0
- sphynx/demo_candidates/lexical.py +21 -0
- sphynx/diagnostics.py +207 -0
- sphynx/documentation.py +737 -0
- sphynx/documentation_candidate.py +939 -0
- sphynx/evals.py +538 -0
- sphynx/evaluation_protocols.py +124 -0
- sphynx/evaluator_bindings.py +370 -0
- sphynx/evidence.py +352 -0
- sphynx/execution_capabilities.py +109 -0
- sphynx/fast_search.py +285 -0
- sphynx/full_rag.py +341 -0
- sphynx/hosted/__init__.py +18 -0
- sphynx/hosted/benchmark_intake.py +218 -0
- sphynx/hosted/contracts.py +361 -0
- sphynx/hosted/model_admission.py +130 -0
- sphynx/hosted/model_artifacts.py +298 -0
- sphynx/hosted/model_contracts.py +63 -0
- sphynx/hosted/model_gcs.py +181 -0
- sphynx/hosted/model_history.py +84 -0
- sphynx/hosted/model_session.py +233 -0
- sphynx/hosted/public_suite.py +74 -0
- sphynx/hosted/registry.py +97 -0
- sphynx/hosted/revisions.py +126 -0
- sphynx/hosted/sandbox.py +105 -0
- sphynx/hosted/submission.py +287 -0
- sphynx/hosted_workflow.py +451 -0
- sphynx/http_transport.py +86 -0
- sphynx/integrated_rag.py +352 -0
- sphynx/integrated_workflow.py +552 -0
- sphynx/integration_contracts.py +474 -0
- sphynx/invocation.py +791 -0
- sphynx/invocation_candidates.py +392 -0
- sphynx/invocation_catalog_data.py +40 -0
- sphynx/learned_catalog_data.py +30 -0
- sphynx/learned_rerank.py +267 -0
- sphynx/legacy_full_rag.py +67 -0
- sphynx/lifecycle.py +885 -0
- sphynx/lifecycle_workflow.py +490 -0
- sphynx/metric_views.py +474 -0
- sphynx/metrics.py +282 -0
- sphynx/model_adapters.py +438 -0
- sphynx/model_catalog_data.py +30 -0
- sphynx/official_benchmarks.py +385 -0
- sphynx/packs.py +258 -0
- sphynx/pipeline.py +313 -0
- sphynx/pipeline_workflow.py +551 -0
- sphynx/pr_evaluations.py +560 -0
- sphynx/progress.py +89 -0
- sphynx/public_bundle.py +173 -0
- sphynx/public_eval.py +62 -0
- sphynx/public_rerank.py +355 -0
- sphynx/py.typed +0 -0
- sphynx/pytest_plugin.py +62 -0
- sphynx/question_robustness.py +437 -0
- sphynx/rag.py +329 -0
- sphynx/rag_diagnostics.py +309 -0
- sphynx/rag_papers/__init__.py +29 -0
- sphynx/rag_papers/__main__.py +45 -0
- sphynx/rag_papers/candidates.py +19 -0
- sphynx/rag_papers/contracts.py +282 -0
- sphynx/rag_papers/datasets.py +291 -0
- sphynx/rag_papers/experiments.py +116 -0
- sphynx/rag_papers/judges.py +308 -0
- sphynx/rag_papers/registry.py +33 -0
- sphynx/rag_papers/runner.py +176 -0
- sphynx/rag_papers/sable.py +135 -0
- sphynx/rag_papers/schemas.py +12 -0
- sphynx/rag_papers/scoring.py +249 -0
- sphynx/rag_papers/semantic.py +288 -0
- sphynx/rag_papers/trec.py +80 -0
- sphynx/recorded_scores.py +352 -0
- sphynx/records.py +124 -0
- sphynx/reflex_traces.py +193 -0
- sphynx/report_rendering.py +71 -0
- sphynx/rerank_family_contracts.py +150 -0
- sphynx/rerank_workflow.py +424 -0
- sphynx/research_candidate.py +190 -0
- sphynx/results.py +852 -0
- sphynx/retrieval.py +750 -0
- sphynx/retrieval_candidates.py +295 -0
- sphynx/retrieval_catalog_data.py +19 -0
- sphynx/retrieval_recordings.py +125 -0
- sphynx/reviewer_audit.py +266 -0
- sphynx/routing_policies.py +375 -0
- sphynx/runner.py +544 -0
- sphynx/sable_rerank.py +489 -0
- sphynx/score_reliability.py +393 -0
- sphynx/speech_benchmarks.py +204 -0
- sphynx/speech_campaigns.py +378 -0
- sphynx/speech_complete.py +236 -0
- sphynx/speech_contracts.py +225 -0
- sphynx/speech_evidence.py +429 -0
- sphynx/speech_listening.py +115 -0
- sphynx/speech_multilingual.py +71 -0
- sphynx/speech_tts.py +479 -0
- sphynx/sql/001_sync_v1.sql +30 -0
- sphynx/stage_contracts.py +199 -0
- sphynx/statistical_catalog_data.py +53 -0
- sphynx/statistical_rerank.py +232 -0
- sphynx/sync.py +797 -0
- sphynx/workbench.py +378 -0
- sphynx/workbench_catalog_data.py +13 -0
- sphynx/workbench_tracks.py +78 -0
- sphynx/workflow_async.py +197 -0
- sphynx/workflow_contracts.py +594 -0
- sphynx/workflow_decisions.py +211 -0
- sphynx/workflow_defaults.py +16 -0
- sphynx/workflow_extensions.py +212 -0
- sphynx/workflow_registry.py +513 -0
- sphynx/workflows.py +792 -0
|
@@ -0,0 +1,939 @@
|
|
|
1
|
+
"""Review-only documentation candidates, extracted without importing the SDK.
|
|
2
|
+
|
|
3
|
+
The frozen v1 exporter remains independent. This format binds the actual wheel,
|
|
4
|
+
the complete reviewed source snapshot, curated guides and a static HTML view.
|
|
5
|
+
AST extraction reads declarations; it never executes inspected source or examples.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import argparse
|
|
10
|
+
import ast
|
|
11
|
+
import copy
|
|
12
|
+
from email import policy
|
|
13
|
+
from email.parser import Parser
|
|
14
|
+
import hashlib
|
|
15
|
+
import html
|
|
16
|
+
import json
|
|
17
|
+
from pathlib import Path, PurePosixPath
|
|
18
|
+
import re
|
|
19
|
+
import runpy
|
|
20
|
+
import sys
|
|
21
|
+
import tomllib
|
|
22
|
+
from urllib.parse import urlsplit
|
|
23
|
+
import zipfile
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
EXPORTER_VERSION = "candidate.2"
|
|
27
|
+
CONTENT_SCHEMA = "sphynx.docs.candidate-content.v1"
|
|
28
|
+
CATALOG_SCHEMA = "sphynx.docs.candidate-catalog.v2"
|
|
29
|
+
BUNDLE_SCHEMA = "sphynx.docs.candidate-bundle.v1"
|
|
30
|
+
MANIFEST_SCHEMA = "sphynx.docs.candidate-manifest.v1"
|
|
31
|
+
MAX_JSON_BYTES = 12_000_000
|
|
32
|
+
MAX_SOURCE_BYTES = 100_000_000
|
|
33
|
+
_HASH = re.compile(r"[0-9a-f]{64}")
|
|
34
|
+
_IDENTIFIER = re.compile(r"[a-zA-Z0-9][a-zA-Z0-9._-]{0,255}")
|
|
35
|
+
_PRIVATE = re.compile(
|
|
36
|
+
r"gs://|[A-Za-z]:[\\/](?:Users|ProgramData)[\\/]|"
|
|
37
|
+
r"(?:^|[\\/])\.agent[\\/]|<\s*(?:script|iframe|object|svg)\b|javascript:|"
|
|
38
|
+
r"-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----|sk-[A-Za-z0-9]{20,}", re.I)
|
|
39
|
+
_SECRET_KEY = re.compile(
|
|
40
|
+
r"^(?:token|password|secret|credentials?|raw_questions?|raw_labels?|answer_capture|"
|
|
41
|
+
r"weights?|fitted_state|gcs_uri|internal_path|environment)$", re.I)
|
|
42
|
+
_FILES = ("catalog.json", "content.json", "source-identity.json", "index.html")
|
|
43
|
+
_MEMBERS = {*_FILES, "bundle.json", "manifest.json"}
|
|
44
|
+
_CORE_SOURCE = {
|
|
45
|
+
"pyproject.toml", "tools/release/source_allowlist.json", "src/sphynx/__init__.py",
|
|
46
|
+
"src/sphynx/documentation.py", "src/sphynx/documentation_candidate.py",
|
|
47
|
+
"src/sphynx/workflows.py", "src/sphynx/workflow_contracts.py",
|
|
48
|
+
"src/sphynx/workflow_registry.py", "src/sphynx/assessment_contracts.py",
|
|
49
|
+
"src/sphynx/workflow_defaults.py", "src/sphynx/rerank_workflow.py",
|
|
50
|
+
"tools/export_documentation.py",
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class CandidateDocumentationError(ValueError):
|
|
55
|
+
"""Malformed, stale, incompatible or unapproved candidate documentation."""
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _require(condition, message):
|
|
59
|
+
if not condition:
|
|
60
|
+
raise CandidateDocumentationError(message)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _sha(raw):
|
|
64
|
+
return hashlib.sha256(raw).hexdigest()
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _canonical(value):
|
|
68
|
+
try:
|
|
69
|
+
raw = json.dumps(value, ensure_ascii=False, sort_keys=True,
|
|
70
|
+
separators=(",", ":"), allow_nan=False).encode("utf-8")
|
|
71
|
+
except (ValueError, TypeError, UnicodeError, RecursionError) as exc:
|
|
72
|
+
raise CandidateDocumentationError("content is outside safe JSON") from exc
|
|
73
|
+
_require(len(raw) <= MAX_JSON_BYTES, "JSON exceeds candidate size limit")
|
|
74
|
+
return raw
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _safe_json(value, *, confidentiality=True):
|
|
78
|
+
"""Bound structure and reject non-finite values, executable/private content."""
|
|
79
|
+
nodes = 0
|
|
80
|
+
|
|
81
|
+
def visit(item, depth):
|
|
82
|
+
nonlocal nodes
|
|
83
|
+
nodes += 1
|
|
84
|
+
_require(depth <= 40 and nodes <= 300_000, "JSON exceeds structural limits")
|
|
85
|
+
if item is None or type(item) in (bool, int):
|
|
86
|
+
return
|
|
87
|
+
if type(item) is float:
|
|
88
|
+
_require(item == item and abs(item) != float("inf"), "non-finite JSON number")
|
|
89
|
+
elif type(item) is str:
|
|
90
|
+
_require(len(item) <= 200_000, "JSON string exceeds size limit")
|
|
91
|
+
_require(not confidentiality or not _PRIVATE.search(item),
|
|
92
|
+
"unapproved private locator or executable content")
|
|
93
|
+
elif type(item) is list:
|
|
94
|
+
for child in item:
|
|
95
|
+
visit(child, depth + 1)
|
|
96
|
+
elif type(item) is dict:
|
|
97
|
+
for key, child in item.items():
|
|
98
|
+
_require(type(key) is str, "JSON keys must be strings")
|
|
99
|
+
_require(not confidentiality or not _SECRET_KEY.fullmatch(key),
|
|
100
|
+
"unapproved protected-data field")
|
|
101
|
+
visit(key, depth + 1)
|
|
102
|
+
visit(child, depth + 1)
|
|
103
|
+
else:
|
|
104
|
+
raise CandidateDocumentationError("value is outside safe JSON")
|
|
105
|
+
visit(value, 0)
|
|
106
|
+
_canonical(value)
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _pairs(items):
|
|
110
|
+
result = {}
|
|
111
|
+
for key, value in items:
|
|
112
|
+
_require(key not in result, "duplicate JSON key")
|
|
113
|
+
result[key] = value
|
|
114
|
+
return result
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _read(path, limit=MAX_JSON_BYTES):
|
|
118
|
+
path = Path(path)
|
|
119
|
+
_require(path.is_file() and not path.is_symlink(), "input is missing or symlinked")
|
|
120
|
+
_require(path.stat().st_size <= limit, "input exceeds size limit")
|
|
121
|
+
raw = path.read_bytes()
|
|
122
|
+
_require(len(raw) <= limit, "input exceeds size limit")
|
|
123
|
+
return raw
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def _json(path):
|
|
127
|
+
try:
|
|
128
|
+
value = json.loads(_read(path).decode("utf-8"), object_pairs_hook=_pairs,
|
|
129
|
+
parse_constant=lambda _: (_ for _ in ()).throw(
|
|
130
|
+
CandidateDocumentationError("non-finite JSON number")))
|
|
131
|
+
except (ValueError, UnicodeError, RecursionError) as exc:
|
|
132
|
+
raise CandidateDocumentationError("invalid candidate JSON") from exc
|
|
133
|
+
_safe_json(value)
|
|
134
|
+
return value
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def _keys(value, keys, name):
|
|
138
|
+
_require(type(value) is dict and set(value) == set(keys.split()),
|
|
139
|
+
f"{name} has missing or unapproved keys")
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def _text(value, name):
|
|
143
|
+
_require(type(value) is str and bool(value.strip()) and len(value) <= 20_000,
|
|
144
|
+
f"{name} must be nonempty bounded text")
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def _relative(value):
|
|
148
|
+
_require(type(value) is str and value and "\\" not in value and ":" not in value,
|
|
149
|
+
"unsafe relative path")
|
|
150
|
+
path = PurePosixPath(value)
|
|
151
|
+
_require(not path.is_absolute() and all(part not in ("", ".", "..") for part in value.split("/")),
|
|
152
|
+
"unsafe relative path")
|
|
153
|
+
for part in path.parts:
|
|
154
|
+
_require(part.rstrip(" .") == part and not any(ord(c) < 32 or c in '<>"|?*' for c in part),
|
|
155
|
+
"nonportable relative path")
|
|
156
|
+
_require(part.split(".")[0].upper() not in
|
|
157
|
+
{"CON", "PRN", "AUX", "NUL", "CONIN$", "CONOUT$",
|
|
158
|
+
*(f"COM{i}" for i in "123456789"), *(f"LPT{i}" for i in "123456789")},
|
|
159
|
+
"reserved relative path")
|
|
160
|
+
return path
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def _source_path(root, relative):
|
|
164
|
+
relative = _relative(relative)
|
|
165
|
+
path = root.joinpath(*relative.parts)
|
|
166
|
+
_require(path.is_file() and path.resolve().is_relative_to(root),
|
|
167
|
+
"source input is missing or outside the source root")
|
|
168
|
+
_require(not any(parent.is_symlink() for parent in (path, *path.parents) if parent != root.parent),
|
|
169
|
+
"symlinked source input")
|
|
170
|
+
return path
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def _unique(items, name):
|
|
174
|
+
_require(type(items) is list and bool(items), f"{name} must be a nonempty array")
|
|
175
|
+
identifiers = []
|
|
176
|
+
for item in items:
|
|
177
|
+
_require(type(item) is dict and type(item.get("id")) is str
|
|
178
|
+
and _IDENTIFIER.fullmatch(item["id"]), f"{name} has invalid IDs")
|
|
179
|
+
identifiers.append(item["id"])
|
|
180
|
+
_require(len(identifiers) == len(set(identifiers)), f"{name} has duplicate IDs")
|
|
181
|
+
return set(identifiers)
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def validate_content(content):
|
|
185
|
+
"""Validate the exact curated candidate grammar; never execute its examples."""
|
|
186
|
+
_safe_json(content)
|
|
187
|
+
_keys(content, "schema_version id sdk_version pages", "candidate content")
|
|
188
|
+
_require(content["schema_version"] == CONTENT_SCHEMA, "unsupported candidate content schema")
|
|
189
|
+
_require(type(content["id"]) is str and _IDENTIFIER.fullmatch(content["id"]), "invalid candidate ID")
|
|
190
|
+
_text(content["sdk_version"], "SDK version")
|
|
191
|
+
_unique(content["pages"], "pages")
|
|
192
|
+
pages = {page["id"]: page for page in content["pages"]}
|
|
193
|
+
for page in content["pages"]:
|
|
194
|
+
_keys(page, "id title category summary sections", "page")
|
|
195
|
+
for key in ("title", "category", "summary"):
|
|
196
|
+
_text(page[key], "page " + key)
|
|
197
|
+
_unique(page["sections"], "sections")
|
|
198
|
+
for section in page["sections"]:
|
|
199
|
+
_keys(section, "id title paragraphs blocks tables links", "section")
|
|
200
|
+
_text(section["title"], "section title")
|
|
201
|
+
for key in ("paragraphs", "blocks", "tables", "links"):
|
|
202
|
+
_require(type(section[key]) is list, "section members must be arrays")
|
|
203
|
+
for paragraph in section["paragraphs"]:
|
|
204
|
+
_text(paragraph, "paragraph")
|
|
205
|
+
for block in section["blocks"]:
|
|
206
|
+
_keys(block, "language code execution", "block")
|
|
207
|
+
_require(block["language"] in {"python", "powershell", "bash", "text", "json"},
|
|
208
|
+
"unsupported example language")
|
|
209
|
+
_require(block["execution"] in {"fixture", "read_only", "metadata_only", "paid_remote"},
|
|
210
|
+
"unsupported execution boundary")
|
|
211
|
+
_text(block["code"], "example code")
|
|
212
|
+
_require(not block["code"].startswith((">>>", "PS ", "$ ")), "examples must omit prompts")
|
|
213
|
+
try:
|
|
214
|
+
if block["language"] == "python":
|
|
215
|
+
ast.parse(block["code"], feature_version=(3, 11))
|
|
216
|
+
elif block["language"] == "json":
|
|
217
|
+
_safe_json(json.loads(block["code"], object_pairs_hook=_pairs))
|
|
218
|
+
except (SyntaxError, ValueError, RecursionError) as exc:
|
|
219
|
+
raise CandidateDocumentationError("invalid example syntax") from exc
|
|
220
|
+
for table in section["tables"]:
|
|
221
|
+
_keys(table, "columns rows", "table")
|
|
222
|
+
_require(type(table["columns"]) is list and bool(table["columns"])
|
|
223
|
+
and type(table["rows"]) is list, "table must have columns and rows")
|
|
224
|
+
for cell in table["columns"]:
|
|
225
|
+
_text(cell, "table column")
|
|
226
|
+
for row in table["rows"]:
|
|
227
|
+
_require(type(row) is list and len(row) == len(table["columns"]), "table row width differs")
|
|
228
|
+
for cell in row:
|
|
229
|
+
_require(type(cell) is str, "table cells must be strings")
|
|
230
|
+
for link in section["links"]:
|
|
231
|
+
_keys(link, "label url", "link")
|
|
232
|
+
_text(link["label"], "link label")
|
|
233
|
+
url = link["url"]
|
|
234
|
+
_require(type(url) is str, "link URL must be text")
|
|
235
|
+
if url.startswith("#"):
|
|
236
|
+
pieces = url[1:].split("/")
|
|
237
|
+
target = pages.get(pieces[0])
|
|
238
|
+
_require(1 <= len(pieces) <= 2 and target is not None, "invalid internal page link")
|
|
239
|
+
_require(len(pieces) == 1 or pieces[1] in {item["id"] for item in target["sections"]},
|
|
240
|
+
"invalid internal section link")
|
|
241
|
+
else:
|
|
242
|
+
try:
|
|
243
|
+
parsed = urlsplit(url)
|
|
244
|
+
except ValueError as exc:
|
|
245
|
+
raise CandidateDocumentationError("invalid external link") from exc
|
|
246
|
+
_require(parsed.scheme == "https" and bool(parsed.hostname)
|
|
247
|
+
and parsed.username is None and parsed.password is None,
|
|
248
|
+
"external links must be credential-free HTTPS")
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def _parameters(args, *, bound=False):
|
|
252
|
+
args = copy.deepcopy(args)
|
|
253
|
+
if bound and args.args and args.args[0].arg in {"self", "cls"}:
|
|
254
|
+
args.args.pop(0)
|
|
255
|
+
positions = [*args.posonlyargs, *args.args]
|
|
256
|
+
required = len(positions) - len(args.defaults)
|
|
257
|
+
rows = []
|
|
258
|
+
for index, arg in enumerate(positions):
|
|
259
|
+
default = None if index < required else args.defaults[index - required]
|
|
260
|
+
rows.append({"name": arg.arg, "annotation": ast.unparse(arg.annotation) if arg.annotation else None,
|
|
261
|
+
"kind": "positional_only" if index < len(args.posonlyargs) else "positional_or_keyword",
|
|
262
|
+
"required": index < required, "default": ast.unparse(default) if default is not None else None})
|
|
263
|
+
if args.vararg:
|
|
264
|
+
arg = args.vararg
|
|
265
|
+
rows.append({"name": arg.arg, "annotation": ast.unparse(arg.annotation) if arg.annotation else None,
|
|
266
|
+
"kind": "var_positional", "required": False, "default": None})
|
|
267
|
+
for arg, default in zip(args.kwonlyargs, args.kw_defaults):
|
|
268
|
+
rows.append({"name": arg.arg, "annotation": ast.unparse(arg.annotation) if arg.annotation else None,
|
|
269
|
+
"kind": "keyword_only", "required": default is None,
|
|
270
|
+
"default": ast.unparse(default) if default is not None else None})
|
|
271
|
+
if args.kwarg:
|
|
272
|
+
arg = args.kwarg
|
|
273
|
+
rows.append({"name": arg.arg, "annotation": ast.unparse(arg.annotation) if arg.annotation else None,
|
|
274
|
+
"kind": "var_keyword", "required": False, "default": None})
|
|
275
|
+
return ast.unparse(args), rows
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
class _Source:
|
|
279
|
+
def __init__(self, root):
|
|
280
|
+
self.root = Path(root).resolve(strict=True)
|
|
281
|
+
self.trees = {}
|
|
282
|
+
|
|
283
|
+
def tree(self, module):
|
|
284
|
+
if module not in self.trees:
|
|
285
|
+
relative = "src/sphynx/" + (module.replace(".", "/") + ".py" if module else "__init__.py")
|
|
286
|
+
path = _source_path(self.root, relative)
|
|
287
|
+
try:
|
|
288
|
+
tree = ast.parse(_read(path).decode("utf-8-sig"), filename=relative, feature_version=(3, 11))
|
|
289
|
+
except (SyntaxError, UnicodeError) as exc:
|
|
290
|
+
raise CandidateDocumentationError("invalid source syntax: " + relative) from exc
|
|
291
|
+
self.trees[module] = (tree, relative)
|
|
292
|
+
return self.trees[module][0]
|
|
293
|
+
|
|
294
|
+
def imported(self, module, node, name):
|
|
295
|
+
if node.level:
|
|
296
|
+
parent = module.split(".")[:-node.level] if module else []
|
|
297
|
+
destination = ".".join([*parent, *([node.module] if node.module else [])])
|
|
298
|
+
else:
|
|
299
|
+
_require(node.module is not None and node.module.startswith("sphynx."),
|
|
300
|
+
"public API resolves outside the SDK")
|
|
301
|
+
destination = node.module.removeprefix("sphynx.")
|
|
302
|
+
return destination, name
|
|
303
|
+
|
|
304
|
+
def resolve(self, module, name, trail=()):
|
|
305
|
+
key = (module, name)
|
|
306
|
+
_require(key not in trail and len(trail) < 30, "cyclic or excessive public alias chain")
|
|
307
|
+
tree = self.tree(module)
|
|
308
|
+
# Class-body aliases are resolved before their enclosing class import.
|
|
309
|
+
if "." in name:
|
|
310
|
+
head, tail = name.split(".", 1)
|
|
311
|
+
owner_module, owner_name, owner = self.resolve(module, head, (*trail, key))
|
|
312
|
+
_require(isinstance(owner, ast.ClassDef), "public method owner is not a class")
|
|
313
|
+
for node in reversed(owner.body):
|
|
314
|
+
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name == tail:
|
|
315
|
+
return owner_module, owner_name + "." + tail, node
|
|
316
|
+
if isinstance(node, ast.Assign) and any(isinstance(t, ast.Name) and t.id == tail for t in node.targets):
|
|
317
|
+
_require(isinstance(node.value, ast.Name), "unsupported class-body alias")
|
|
318
|
+
return self.resolve(owner_module, owner_name + "." + node.value.id, (*trail, key))
|
|
319
|
+
raise CandidateDocumentationError("missing declared public method: " + name)
|
|
320
|
+
for node in reversed(tree.body):
|
|
321
|
+
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)) and node.name == name:
|
|
322
|
+
return module, name, node
|
|
323
|
+
if isinstance(node, (ast.Assign, ast.AnnAssign)):
|
|
324
|
+
targets = node.targets if isinstance(node, ast.Assign) else [node.target]
|
|
325
|
+
if any(isinstance(t, ast.Name) and t.id == name for t in targets):
|
|
326
|
+
if isinstance(node.value, ast.Name):
|
|
327
|
+
return self.resolve(module, node.value.id, (*trail, key))
|
|
328
|
+
return module, name, node
|
|
329
|
+
if isinstance(node, ast.ImportFrom):
|
|
330
|
+
for alias in node.names:
|
|
331
|
+
if (alias.asname or alias.name) == name:
|
|
332
|
+
destination, original = self.imported(module, node, alias.name)
|
|
333
|
+
return self.resolve(destination, original, (*trail, key))
|
|
334
|
+
raise CandidateDocumentationError("missing declared public API: " + (module + "." + name))
|
|
335
|
+
|
|
336
|
+
def literal(self, module, node, trail=()):
|
|
337
|
+
if isinstance(node, (ast.List, ast.Tuple)):
|
|
338
|
+
return [self.literal(module, item, trail) for item in node.elts]
|
|
339
|
+
if isinstance(node, ast.Dict):
|
|
340
|
+
return {self.literal(module, key, trail): self.literal(module, item, trail)
|
|
341
|
+
for key, item in zip(node.keys, node.values)}
|
|
342
|
+
try:
|
|
343
|
+
return ast.literal_eval(node)
|
|
344
|
+
except (ValueError, TypeError):
|
|
345
|
+
pass
|
|
346
|
+
if isinstance(node, ast.Name):
|
|
347
|
+
resolved_module, name, declaration = self.resolve(module, node.id)
|
|
348
|
+
_require((resolved_module, name) not in trail, "cyclic metadata constant")
|
|
349
|
+
_require(isinstance(declaration, (ast.Assign, ast.AnnAssign)), "metadata constant is not literal")
|
|
350
|
+
return self.literal(resolved_module, declaration.value, (*trail, (resolved_module, name)))
|
|
351
|
+
raise CandidateDocumentationError("metadata expression is not a declared literal")
|
|
352
|
+
|
|
353
|
+
def exports(self):
|
|
354
|
+
bindings, values, public = {}, {}, []
|
|
355
|
+
for node in self.tree("").body:
|
|
356
|
+
if isinstance(node, ast.ImportFrom):
|
|
357
|
+
for alias in node.names:
|
|
358
|
+
bindings[alias.asname or alias.name] = self.imported("", node, alias.name)
|
|
359
|
+
elif isinstance(node, ast.Assign):
|
|
360
|
+
for target in node.targets:
|
|
361
|
+
if not isinstance(target, ast.Name):
|
|
362
|
+
continue
|
|
363
|
+
if target.id in {"_WORKFLOW_EXPORTS", "_RESEARCH_EXPORTS"}:
|
|
364
|
+
values[target.id] = ast.literal_eval(node.value)
|
|
365
|
+
for name, value in values[target.id].items():
|
|
366
|
+
bindings[name] = (value, name) if isinstance(value, str) else tuple(value)
|
|
367
|
+
elif target.id == "__all__":
|
|
368
|
+
public = ast.literal_eval(node.value)
|
|
369
|
+
elif isinstance(node.value, ast.Name) and node.value.id in bindings:
|
|
370
|
+
bindings[target.id] = bindings[node.value.id]
|
|
371
|
+
elif isinstance(node, ast.AugAssign) and isinstance(node.target, ast.Name) and node.target.id == "__all__":
|
|
372
|
+
if isinstance(node.value, ast.Call) and isinstance(node.value.func, ast.Name) and node.value.func.id == "list":
|
|
373
|
+
public.extend(values[node.value.args[0].id])
|
|
374
|
+
else:
|
|
375
|
+
public.extend(ast.literal_eval(node.value))
|
|
376
|
+
_require(len(public) == len(set(public)), "root public exports contain duplicates")
|
|
377
|
+
_require(set(public).issubset(bindings), "root public export has no static binding")
|
|
378
|
+
return {name: bindings[name] for name in sorted(public)}
|
|
379
|
+
|
|
380
|
+
def function(self, module, canonical, node, public_name):
|
|
381
|
+
bound = "." in canonical
|
|
382
|
+
arguments, parameters = _parameters(node.args, bound=bound)
|
|
383
|
+
returns = ast.unparse(node.returns) if node.returns else None
|
|
384
|
+
return {"name": public_name, "target": "sphynx." + module + "." + canonical,
|
|
385
|
+
"alias_of": None if public_name == canonical.split(".")[-1] else canonical,
|
|
386
|
+
"signature": public_name + "(" + arguments + ")" + (" -> " + returns if returns else ""),
|
|
387
|
+
"parameters": parameters, "returns": returns,
|
|
388
|
+
"summary": (ast.get_docstring(node) or "").split("\n")[0],
|
|
389
|
+
"decorators": [ast.unparse(item) for item in node.decorator_list],
|
|
390
|
+
"source_line": node.lineno}
|
|
391
|
+
|
|
392
|
+
def entry(self, public_name, module, original):
|
|
393
|
+
defining_module, canonical, node = self.resolve(module, original)
|
|
394
|
+
path = self.trees[defining_module][1]
|
|
395
|
+
result = {"id": "sphynx." + public_name, "target": "sphynx." + defining_module + "." + canonical,
|
|
396
|
+
"kind": "constant", "alias_of": None if public_name == canonical else canonical,
|
|
397
|
+
"signature": None, "parameters": [], "returns": None, "summary": "",
|
|
398
|
+
"source_path": path, "source_line": node.lineno, "fields": [], "methods": [], "value": None}
|
|
399
|
+
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
|
400
|
+
function = self.function(defining_module, canonical, node, public_name)
|
|
401
|
+
result.update({key: function[key] for key in ("signature", "parameters", "returns", "summary")})
|
|
402
|
+
result["kind"] = "function"
|
|
403
|
+
elif isinstance(node, ast.ClassDef):
|
|
404
|
+
result.update(kind="class", summary=(ast.get_docstring(node) or "").split("\n")[0])
|
|
405
|
+
dataclass = next((item for item in node.decorator_list
|
|
406
|
+
if (isinstance(item, ast.Name) and item.id == "dataclass")
|
|
407
|
+
or (isinstance(item, ast.Call) and isinstance(item.func, ast.Name)
|
|
408
|
+
and item.func.id == "dataclass")), None)
|
|
409
|
+
options = {kw.arg: ast.literal_eval(kw.value) for kw in dataclass.keywords} if isinstance(dataclass, ast.Call) else {}
|
|
410
|
+
for item in node.body:
|
|
411
|
+
if not isinstance(item, ast.AnnAssign) or not isinstance(item.target, ast.Name):
|
|
412
|
+
continue
|
|
413
|
+
name, default, factory = item.target.id, item.value, None
|
|
414
|
+
init, keyword_only = True, options.get("kw_only", False)
|
|
415
|
+
if isinstance(default, ast.Call) and isinstance(default.func, ast.Name) and default.func.id == "field":
|
|
416
|
+
keywords = {kw.arg: kw.value for kw in default.keywords}
|
|
417
|
+
init = ast.literal_eval(keywords.get("init", ast.Constant(True)))
|
|
418
|
+
keyword_only = ast.literal_eval(keywords.get("kw_only", ast.Constant(keyword_only)))
|
|
419
|
+
factory = ast.unparse(keywords["default_factory"]) if "default_factory" in keywords else None
|
|
420
|
+
default = keywords.get("default")
|
|
421
|
+
result["fields"].append({"name": name, "annotation": ast.unparse(item.annotation),
|
|
422
|
+
"default": ast.unparse(default) if default is not None else None,
|
|
423
|
+
"default_factory": factory, "init": init,
|
|
424
|
+
"keyword_only": keyword_only, "private": name.startswith("_")})
|
|
425
|
+
methods = {}
|
|
426
|
+
for item in node.body:
|
|
427
|
+
if isinstance(item, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
|
428
|
+
if not item.name.startswith("_") or item.name == "__init__":
|
|
429
|
+
methods[item.name] = item.name
|
|
430
|
+
elif isinstance(item, ast.Assign) and isinstance(item.value, ast.Name):
|
|
431
|
+
for target in item.targets:
|
|
432
|
+
if isinstance(target, ast.Name) and not target.id.startswith("_"):
|
|
433
|
+
methods[target.id] = item.value.id
|
|
434
|
+
for name in sorted(methods):
|
|
435
|
+
m, c, function = self.resolve(defining_module, canonical + "." + name)
|
|
436
|
+
result["methods"].append(self.function(m, c, function, name))
|
|
437
|
+
constructor = next((item for item in result["methods"] if item["name"] == "__init__"), None)
|
|
438
|
+
if constructor:
|
|
439
|
+
result["signature"] = public_name + constructor["signature"][len("__init__"):]
|
|
440
|
+
result["parameters"] = constructor["parameters"]
|
|
441
|
+
elif dataclass is not None and options.get("init", True):
|
|
442
|
+
fields = [item for item in result["fields"] if item["init"]]
|
|
443
|
+
result["parameters"] = [
|
|
444
|
+
{"name": item["name"], "annotation": item["annotation"],
|
|
445
|
+
"kind": "keyword_only" if item["keyword_only"] else "positional_or_keyword",
|
|
446
|
+
"required": item["default"] is None and item["default_factory"] is None,
|
|
447
|
+
"default": item["default"] if item["default_factory"] is None else "<factory:" + item["default_factory"] + ">"}
|
|
448
|
+
for item in fields]
|
|
449
|
+
pieces, keyword_marker = [], False
|
|
450
|
+
for item in result["parameters"]:
|
|
451
|
+
if item["kind"] == "keyword_only" and not keyword_marker:
|
|
452
|
+
pieces.append("*")
|
|
453
|
+
keyword_marker = True
|
|
454
|
+
pieces.append(item["name"] + ": " + item["annotation"]
|
|
455
|
+
+ ("" if item["required"] else " = " + item["default"]))
|
|
456
|
+
result["signature"] = public_name + "(" + ", ".join(pieces) + ")"
|
|
457
|
+
else:
|
|
458
|
+
result["value"] = self.literal(defining_module, node.value)
|
|
459
|
+
return result
|
|
460
|
+
|
|
461
|
+
|
|
462
|
+
def extract_catalog(source_root):
|
|
463
|
+
"""Read root exports, real alias targets and extension declarations by AST."""
|
|
464
|
+
source = _Source(source_root)
|
|
465
|
+
project = tomllib.loads(_read(_source_path(source.root, "pyproject.toml")).decode("utf-8-sig"))["project"]
|
|
466
|
+
runtime_version = source.literal("", next(node.value for node in source.tree("").body
|
|
467
|
+
if isinstance(node, ast.Assign) and any(isinstance(t, ast.Name) and t.id == "__version__" for t in node.targets)))
|
|
468
|
+
_require(project["version"] == runtime_version, "package/runtime version mismatch")
|
|
469
|
+
exports = [source.entry(name, *binding) for name, binding in source.exports().items()]
|
|
470
|
+
# The default is one convenience binding. Configured registry factories are
|
|
471
|
+
# derived from the same public AST entries; their constructors remain the
|
|
472
|
+
# authority for required operator configuration and capability admission.
|
|
473
|
+
factories = [{key: entry[key] for key in (
|
|
474
|
+
"id", "target", "signature", "parameters", "summary", "source_path", "source_line")}
|
|
475
|
+
for entry in exports if entry["kind"] == "function" and entry["returns"] == "WorkflowRegistry"]
|
|
476
|
+
bootstrap = source.tree("workflow_defaults")
|
|
477
|
+
_require(any(isinstance(node, ast.ImportFrom) and node.module == "rerank_workflow"
|
|
478
|
+
and any(alias.name == "build_rerank_registry" for alias in node.names)
|
|
479
|
+
for node in ast.walk(bootstrap)), "built-in bootstrap declaration changed")
|
|
480
|
+
factory = next(node for node in bootstrap.body if isinstance(node, ast.FunctionDef)
|
|
481
|
+
and node.name == "default_workflow_registry")
|
|
482
|
+
returns = [node for node in ast.walk(factory) if isinstance(node, ast.Return)]
|
|
483
|
+
_require(len(returns) == 1 and isinstance(returns[0].value, ast.Call)
|
|
484
|
+
and isinstance(returns[0].value.func, ast.Name)
|
|
485
|
+
and returns[0].value.func.id == "build_rerank_registry"
|
|
486
|
+
and not returns[0].value.args and not returns[0].value.keywords,
|
|
487
|
+
"built-in bootstrap selection changed; review its metadata extractor")
|
|
488
|
+
backend_tree = source.tree("rerank_workflow")
|
|
489
|
+
descriptor = next(node.value for node in backend_tree.body if isinstance(node, ast.Assign)
|
|
490
|
+
and any(isinstance(t, ast.Name) and t.id == "_DESCRIPTOR" for t in node.targets))
|
|
491
|
+
_require(isinstance(descriptor, ast.Call) and isinstance(descriptor.func, ast.Name)
|
|
492
|
+
and descriptor.func.id == "BackendDescriptor" and not descriptor.args,
|
|
493
|
+
"built-in descriptor must be an explicit metadata declaration")
|
|
494
|
+
backend = {item.arg: source.literal("rerank_workflow", item.value) for item in descriptor.keywords}
|
|
495
|
+
# Reuse only the frozen exporter's pure AST schema reader, without its lock.
|
|
496
|
+
frozen = runpy.run_path(str(Path(__file__).with_name("documentation.py")))
|
|
497
|
+
parameters = frozen["extract_parameter_schemas"](source.root)
|
|
498
|
+
result = {"schema_version": CATALOG_SCHEMA, "sdk_version": runtime_version,
|
|
499
|
+
"distribution": project["name"], "exports": exports,
|
|
500
|
+
"builtin_backends": [backend], "registry_factories": factories,
|
|
501
|
+
"native_parameter_schemas": parameters,
|
|
502
|
+
"boundaries": {"metadata_only": True, "live_hosted_verified": False,
|
|
503
|
+
"catalog_entry_is_execution_admission": False,
|
|
504
|
+
"candidate_source_verified_by_local_builtin": False}}
|
|
505
|
+
_safe_json(result)
|
|
506
|
+
return result
|
|
507
|
+
|
|
508
|
+
|
|
509
|
+
def _validate_catalog(catalog):
|
|
510
|
+
_safe_json(catalog)
|
|
511
|
+
_keys(catalog, "schema_version sdk_version distribution exports builtin_backends registry_factories native_parameter_schemas boundaries",
|
|
512
|
+
"candidate catalog")
|
|
513
|
+
_require(catalog["schema_version"] == CATALOG_SCHEMA, "unsupported candidate catalog schema")
|
|
514
|
+
_unique(catalog["exports"], "public exports")
|
|
515
|
+
_text(catalog["sdk_version"], "catalog SDK version")
|
|
516
|
+
_text(catalog["distribution"], "catalog distribution")
|
|
517
|
+
for entry in catalog["exports"]:
|
|
518
|
+
_keys(entry, "id target kind alias_of signature parameters returns summary source_path source_line fields methods value",
|
|
519
|
+
"public API")
|
|
520
|
+
_require(entry["kind"] in {"class", "function", "constant"}, "invalid API kind")
|
|
521
|
+
_text(entry["target"], "API target")
|
|
522
|
+
_require(type(entry["summary"]) is str, "API summary must be text")
|
|
523
|
+
_require(all(entry[key] is None or type(entry[key]) is str
|
|
524
|
+
for key in ("alias_of", "signature", "returns")), "invalid optional API text")
|
|
525
|
+
_relative(entry["source_path"])
|
|
526
|
+
_require(type(entry["source_line"]) is int and entry["source_line"] > 0, "invalid source line")
|
|
527
|
+
_require(type(entry["parameters"]) is list and type(entry["fields"]) is list
|
|
528
|
+
and type(entry["methods"]) is list, "API collections must be arrays")
|
|
529
|
+
for method in entry["methods"]:
|
|
530
|
+
_keys(method, "name target alias_of signature parameters returns summary decorators source_line", "API method")
|
|
531
|
+
for key in ("name", "target", "signature"):
|
|
532
|
+
_text(method[key], "method " + key)
|
|
533
|
+
_require(type(method["summary"]) is str
|
|
534
|
+
and type(method["decorators"]) is list
|
|
535
|
+
and all(type(item) is str for item in method["decorators"])
|
|
536
|
+
and type(method["source_line"]) is int and method["source_line"] > 0,
|
|
537
|
+
"invalid method metadata")
|
|
538
|
+
_require(all(method[key] is None or type(method[key]) is str for key in ("alias_of", "returns")),
|
|
539
|
+
"invalid optional method text")
|
|
540
|
+
for parameters in [entry["parameters"], *(method["parameters"] for method in entry["methods"])]:
|
|
541
|
+
_require(type(parameters) is list, "API parameters must be arrays")
|
|
542
|
+
for parameter in parameters:
|
|
543
|
+
_keys(parameter, "name annotation kind required default", "API parameter")
|
|
544
|
+
_text(parameter["name"], "parameter name")
|
|
545
|
+
_require(all(parameter[key] is None or type(parameter[key]) is str
|
|
546
|
+
for key in ("annotation", "default")), "invalid parameter type/default")
|
|
547
|
+
_require(type(parameter["required"]) is bool, "invalid required parameter flag")
|
|
548
|
+
_require(parameter["kind"] in {"positional_only", "positional_or_keyword", "keyword_only",
|
|
549
|
+
"var_positional", "var_keyword"}, "invalid parameter kind")
|
|
550
|
+
for field in entry["fields"]:
|
|
551
|
+
_keys(field, "name annotation default default_factory init keyword_only private", "dataclass field")
|
|
552
|
+
_text(field["name"], "field name")
|
|
553
|
+
_text(field["annotation"], "field annotation")
|
|
554
|
+
_require(all(field[key] is None or type(field[key]) is str for key in ("default", "default_factory")),
|
|
555
|
+
"invalid field default")
|
|
556
|
+
_require(all(type(field[name]) is bool for name in ("init", "keyword_only", "private")),
|
|
557
|
+
"invalid dataclass field flags")
|
|
558
|
+
_require(type(catalog["builtin_backends"]) is list and bool(catalog["builtin_backends"]), "missing built-in metadata")
|
|
559
|
+
for backend in catalog["builtin_backends"]:
|
|
560
|
+
_keys(backend, "backend_id version implementation_id capabilities supported_scopes supported_modes suite_ids metadata",
|
|
561
|
+
"backend declaration")
|
|
562
|
+
for key in ("backend_id", "version", "implementation_id"):
|
|
563
|
+
_text(backend[key], "backend " + key)
|
|
564
|
+
for key in ("capabilities", "supported_scopes", "supported_modes", "suite_ids"):
|
|
565
|
+
_require(type(backend[key]) is list and bool(backend[key])
|
|
566
|
+
and all(type(item) is str and item for item in backend[key]), "invalid backend declarations")
|
|
567
|
+
_require(type(backend["metadata"]) is dict, "backend metadata must be an object")
|
|
568
|
+
_require(type(catalog["native_parameter_schemas"]) is dict, "native schemas must be an object")
|
|
569
|
+
expected_factories = [{key: entry[key] for key in (
|
|
570
|
+
"id", "target", "signature", "parameters", "summary", "source_path", "source_line")}
|
|
571
|
+
for entry in catalog["exports"] if entry["kind"] == "function" and entry["returns"] == "WorkflowRegistry"]
|
|
572
|
+
_require(catalog["registry_factories"] == expected_factories,
|
|
573
|
+
"configured registry factory metadata differs from public declarations")
|
|
574
|
+
_keys(catalog["boundaries"], "metadata_only live_hosted_verified catalog_entry_is_execution_admission candidate_source_verified_by_local_builtin",
|
|
575
|
+
"catalog boundaries")
|
|
576
|
+
_require(all(type(value) is bool for value in catalog["boundaries"].values()), "capability flags must be boolean")
|
|
577
|
+
_require(catalog["boundaries"] == {
|
|
578
|
+
"metadata_only": True, "live_hosted_verified": False,
|
|
579
|
+
"catalog_entry_is_execution_admission": False, "candidate_source_verified_by_local_builtin": False},
|
|
580
|
+
"unapproved current capability claim")
|
|
581
|
+
|
|
582
|
+
|
|
583
|
+
def check_candidate(source_root, content_path):
|
|
584
|
+
"""Validate current metadata and curated source without a wheel or output."""
|
|
585
|
+
catalog = extract_catalog(source_root)
|
|
586
|
+
_validate_catalog(catalog)
|
|
587
|
+
content = _json(content_path)
|
|
588
|
+
validate_content(content)
|
|
589
|
+
_require(content["sdk_version"] == catalog["sdk_version"], "curated/source version mismatch")
|
|
590
|
+
return {"status": "passed", "candidate_id": content["id"], "sdk_version": catalog["sdk_version"],
|
|
591
|
+
"export_count": len(catalog["exports"]),
|
|
592
|
+
"backend_ids": [item["backend_id"] for item in catalog["builtin_backends"]],
|
|
593
|
+
"catalog_sha256": _sha(_canonical(catalog)), "content_sha256": _sha(_canonical(content))}
|
|
594
|
+
|
|
595
|
+
|
|
596
|
+
def _identity(value, *, source_root=None, required=()):
|
|
597
|
+
_safe_json(value)
|
|
598
|
+
_keys(value, "schema_version sdk_version distribution source_revision files", "reviewed source identity")
|
|
599
|
+
_require(value["schema_version"] == "sphynx.release-source.v1", "unsupported reviewed source identity")
|
|
600
|
+
_text(value["sdk_version"], "source SDK version")
|
|
601
|
+
_text(value["distribution"], "source distribution")
|
|
602
|
+
revision = value["source_revision"]
|
|
603
|
+
_require(type(revision) is str and re.fullmatch(
|
|
604
|
+
r"git:[0-9a-f]{40,64}\+snapshot:sha256:[0-9a-f]{64}", revision),
|
|
605
|
+
"source identity must bind a base Git revision plus a reviewed snapshot")
|
|
606
|
+
_require(type(value["files"]) is list and 1 <= len(value["files"]) <= 5000, "invalid reviewed file inventory")
|
|
607
|
+
rows, folded = {}, set()
|
|
608
|
+
for row in value["files"]:
|
|
609
|
+
_keys(row, "path sha256", "reviewed source row")
|
|
610
|
+
relative = _relative(row["path"]).as_posix()
|
|
611
|
+
_require(relative.casefold() not in folded and type(row["sha256"]) is str and _HASH.fullmatch(row["sha256"]),
|
|
612
|
+
"duplicate or invalid reviewed source row")
|
|
613
|
+
rows[relative] = row["sha256"]
|
|
614
|
+
folded.add(relative.casefold())
|
|
615
|
+
canonical = "".join(path + " " + rows[path] + "\n" for path in sorted(rows)).encode("utf-8")
|
|
616
|
+
_require(revision.rsplit(":", 1)[1] == _sha(canonical), "reviewed snapshot digest mismatch")
|
|
617
|
+
_require(set(required).issubset(rows), "reviewed source omits a required documentation/runtime input")
|
|
618
|
+
if source_root is not None:
|
|
619
|
+
root = Path(source_root).resolve(strict=True)
|
|
620
|
+
total = 0
|
|
621
|
+
for relative, digest in rows.items():
|
|
622
|
+
raw = _read(_source_path(root, relative), MAX_SOURCE_BYTES)
|
|
623
|
+
total += len(raw)
|
|
624
|
+
_require(total <= MAX_SOURCE_BYTES, "reviewed source exceeds size limit")
|
|
625
|
+
_require(_sha(raw) == digest, "stale reviewed source: " + relative)
|
|
626
|
+
allowlist = _json(_source_path(root, "tools/release/source_allowlist.json"))
|
|
627
|
+
_require(allowlist.get("schema_version") == "sphynx.release-allowlist.v1"
|
|
628
|
+
and allowlist.get("sdk_version") == value["sdk_version"], "source allowlist version mismatch")
|
|
629
|
+
approved = [item.get("path") for item in allowlist.get("files", []) if isinstance(item, dict)]
|
|
630
|
+
_require(len(approved) == len(set(approved)) and set(approved) == set(rows),
|
|
631
|
+
"source identity must cover the complete reviewed packaging allowlist")
|
|
632
|
+
return rows
|
|
633
|
+
|
|
634
|
+
|
|
635
|
+
def _wheel(wheel, identity, rows):
|
|
636
|
+
path = Path(wheel)
|
|
637
|
+
raw = _read(path, MAX_SOURCE_BYTES)
|
|
638
|
+
expected = {relative.removeprefix("src/"): digest for relative, digest in rows.items()
|
|
639
|
+
if relative.startswith("src/sphynx/")}
|
|
640
|
+
distribution, version = identity["distribution"], identity["sdk_version"]
|
|
641
|
+
normalized = distribution.replace("-", "_")
|
|
642
|
+
_require(path.name == f"{normalized}-{version}-py3-none-any.whl", "wheel filename/version mismatch")
|
|
643
|
+
generated = {f"{normalized}-{version}.dist-info/{name}" for name in ("METADATA", "WHEEL", "top_level.txt", "RECORD")}
|
|
644
|
+
try:
|
|
645
|
+
with zipfile.ZipFile(path) as archive:
|
|
646
|
+
infos, folded, runtime = {}, set(), {}
|
|
647
|
+
total = 0
|
|
648
|
+
for info in archive.infolist():
|
|
649
|
+
name = _relative(info.filename).as_posix()
|
|
650
|
+
_require(not info.is_dir() and name.casefold() not in folded and not info.flag_bits & 1,
|
|
651
|
+
"duplicate, encrypted or directory wheel member")
|
|
652
|
+
_require((info.external_attr >> 16) & 0o170000 != 0o120000, "symlinked wheel member")
|
|
653
|
+
total += info.file_size
|
|
654
|
+
_require(total <= MAX_SOURCE_BYTES, "wheel exceeds expanded size limit")
|
|
655
|
+
infos[name] = info
|
|
656
|
+
folded.add(name.casefold())
|
|
657
|
+
if name.startswith("sphynx/"):
|
|
658
|
+
runtime[name] = _sha(archive.read(info))
|
|
659
|
+
_require(runtime == expected, "wheel runtime bytes differ from reviewed source")
|
|
660
|
+
_require(set(infos) == set(expected) | generated, "wheel has missing or unexpected generated members")
|
|
661
|
+
metadata_text = archive.read(infos[f"{normalized}-{version}.dist-info/METADATA"]).decode("utf-8")
|
|
662
|
+
metadata = Parser(policy=policy.default).parsestr(metadata_text, headersonly=True)
|
|
663
|
+
_require(not metadata.defects
|
|
664
|
+
and not any(getattr(header, "defects", ()) for header in metadata.values()),
|
|
665
|
+
"malformed wheel package metadata headers")
|
|
666
|
+
_require([str(value) for value in metadata.get_all("Name", [])] == [distribution]
|
|
667
|
+
and [str(value) for value in metadata.get_all("Version", [])] == [version],
|
|
668
|
+
"wheel package metadata differs from reviewed identity")
|
|
669
|
+
except (zipfile.BadZipFile, KeyError, UnicodeError, RuntimeError) as exc:
|
|
670
|
+
raise CandidateDocumentationError("invalid candidate wheel") from exc
|
|
671
|
+
return {"filename": path.name, "sha256": _sha(raw), "size_bytes": len(raw)}
|
|
672
|
+
|
|
673
|
+
|
|
674
|
+
def _table(columns, rows):
|
|
675
|
+
e = html.escape
|
|
676
|
+
return ("<div class='table-wrap'><table><thead><tr>"
|
|
677
|
+
+ "".join("<th>" + e(str(cell)) + "</th>" for cell in columns)
|
|
678
|
+
+ "</tr></thead><tbody>"
|
|
679
|
+
+ "".join("<tr>" + "".join("<td>" + e(str(cell)) + "</td>" for cell in row) + "</tr>" for row in rows)
|
|
680
|
+
+ "</tbody></table></div>")
|
|
681
|
+
|
|
682
|
+
|
|
683
|
+
def _parameter_table(rows):
|
|
684
|
+
return _table(["Parameter", "Type", "Kind", "Default"], [
|
|
685
|
+
[item["name"], item["annotation"] or "Unannotated", item["kind"],
|
|
686
|
+
"Required" if item["required"] else item["default"] or "Variadic"] for item in rows])
|
|
687
|
+
|
|
688
|
+
|
|
689
|
+
def _render(content, catalog, release):
|
|
690
|
+
"""Produce an offline, escaped summary-to-detail guide with no scripts."""
|
|
691
|
+
e = html.escape
|
|
692
|
+
navigation, pages = [], []
|
|
693
|
+
for page in content["pages"]:
|
|
694
|
+
navigation.append("<a href='#" + e(page["id"], quote=True) + "'>" + e(page["title"]) + "</a>")
|
|
695
|
+
sections = []
|
|
696
|
+
for section in page["sections"]:
|
|
697
|
+
anchor = page["id"] + "/" + section["id"]
|
|
698
|
+
parts = ["<section id='" + e(anchor, quote=True) + "'><h3>" + e(section["title"]) + "</h3>"]
|
|
699
|
+
parts.extend("<p>" + e(paragraph) + "</p>" for paragraph in section["paragraphs"])
|
|
700
|
+
for block in section["blocks"]:
|
|
701
|
+
parts.append("<div class='example-label'>" + e(block["language"]) + " · "
|
|
702
|
+
+ e(block["execution"]) + " example — not executed by the exporter</div><pre><code>"
|
|
703
|
+
+ e(block["code"]) + "</code></pre>")
|
|
704
|
+
parts.extend(_table(table["columns"], table["rows"]) for table in section["tables"])
|
|
705
|
+
parts.extend("<p><a href='" + e(link["url"], quote=True) + "'>" + e(link["label"]) + "</a></p>"
|
|
706
|
+
for link in section["links"])
|
|
707
|
+
parts.append("</section>")
|
|
708
|
+
sections.append("".join(parts))
|
|
709
|
+
pages.append("<article id='" + e(page["id"], quote=True) + "'><div class='eyebrow'>"
|
|
710
|
+
+ e(page["category"]) + "</div><h2>" + e(page["title"]) + "</h2><p class='summary'>"
|
|
711
|
+
+ e(page["summary"]) + "</p>" + "".join(sections) + "</article>")
|
|
712
|
+
entries = []
|
|
713
|
+
for entry in catalog["exports"]:
|
|
714
|
+
parts = ["<details id='api-" + e(entry["id"], quote=True) + "'><summary><code>"
|
|
715
|
+
+ e(entry["id"]) + "</code> <span>" + e(entry["kind"]) + "</span></summary>"]
|
|
716
|
+
parts.append("<p>" + e(entry["summary"]) + "</p><p class='provenance'>Source: "
|
|
717
|
+
+ e(entry["source_path"]) + ":" + str(entry["source_line"]) + "<br>Target: "
|
|
718
|
+
+ e(entry["target"]) + "</p>")
|
|
719
|
+
if entry["alias_of"]:
|
|
720
|
+
parts.append("<p>Equivalent native target: <code>" + e(entry["target"]) + "</code>.</p>")
|
|
721
|
+
if entry["signature"]:
|
|
722
|
+
parts.append("<pre><code>" + e(entry["signature"]) + "</code></pre>")
|
|
723
|
+
if entry["parameters"]:
|
|
724
|
+
parts.append(_parameter_table(entry["parameters"]))
|
|
725
|
+
if entry["fields"]:
|
|
726
|
+
parts.append("<h4>Declared fields</h4>" + _table(
|
|
727
|
+
["Field", "Type", "Default or factory", "Constructor", "Visibility"],
|
|
728
|
+
[[field["name"], field["annotation"], field["default"] or
|
|
729
|
+
("Factory: " + field["default_factory"] if field["default_factory"] else "Required"),
|
|
730
|
+
"Included" if field["init"] else "Excluded", "Internal" if field["private"] else "Public"]
|
|
731
|
+
for field in entry["fields"]]))
|
|
732
|
+
for method in entry["methods"]:
|
|
733
|
+
parts.append("<details class='method'><summary><code>" + e(method["signature"])
|
|
734
|
+
+ "</code></summary><p>" + e(method["summary"]) + "</p>"
|
|
735
|
+
+ ("<p>Decorators: " + e(", ".join(method["decorators"])) + "</p>" if method["decorators"] else "")
|
|
736
|
+
+ "<p class='provenance'>Target: " + e(method["target"]) + "</p>"
|
|
737
|
+
+ _parameter_table(method["parameters"]) + "</details>")
|
|
738
|
+
if entry["kind"] == "constant":
|
|
739
|
+
parts.append("<pre><code>" + e(json.dumps(entry["value"], ensure_ascii=False, sort_keys=True)) + "</code></pre>")
|
|
740
|
+
parts.append("</details>")
|
|
741
|
+
entries.append("".join(parts))
|
|
742
|
+
backends = _table(["Backend", "Scopes", "Modes", "Native suites"], [
|
|
743
|
+
[item["backend_id"], ", ".join(item["supported_scopes"]), ", ".join(item["supported_modes"]),
|
|
744
|
+
", ".join(item["suite_ids"])] for item in catalog["builtin_backends"]])
|
|
745
|
+
factories = _table(["Registry factory", "Required configuration", "Source"], [
|
|
746
|
+
[item["signature"], item["summary"], item["source_path"]]
|
|
747
|
+
for item in catalog["registry_factories"]])
|
|
748
|
+
schema_details = "".join("<details><summary>" + e(name) + "</summary><pre><code>"
|
|
749
|
+
+ e(json.dumps(schema, ensure_ascii=False, indent=2, sort_keys=True)) + "</code></pre></details>"
|
|
750
|
+
for name, schema in sorted(catalog["native_parameter_schemas"].items()))
|
|
751
|
+
style = """
|
|
752
|
+
body{margin:0;background:#f6f8fb;color:#17243b;font:16px/1.6 system-ui,sans-serif}
|
|
753
|
+
header{padding:28px 5%;background:#102746;color:white}header p{max-width:80ch}
|
|
754
|
+
.layout{display:grid;grid-template-columns:240px minmax(0,1050px);gap:32px;max-width:1400px;margin:auto;padding:28px}
|
|
755
|
+
nav{position:sticky;top:20px;align-self:start}nav a{display:block;padding:7px 10px;margin:3px 0}
|
|
756
|
+
a{color:#1761ad}header a{color:#bcdcff}main{min-width:0}article,.panel{background:white;padding:28px;margin:0 0 24px;border:1px solid #dce4ef;border-radius:12px}
|
|
757
|
+
h1,h2,h3,h4{line-height:1.25}h2{margin-top:0}.summary{font-size:18px;color:#465b77}
|
|
758
|
+
.eyebrow,.example-label,.provenance{font-size:13px;color:#526781}.eyebrow{text-transform:uppercase;letter-spacing:.08em}
|
|
759
|
+
pre{white-space:pre-wrap;overflow-wrap:anywhere;padding:18px;border-radius:8px;background:#edf3fa;font-size:13px}
|
|
760
|
+
code{font-family:ui-monospace,Consolas,monospace}details{padding:12px;border:1px solid #dce4ef;border-radius:8px;margin:10px 0}
|
|
761
|
+
summary{cursor:pointer;overflow-wrap:anywhere}summary span{font-size:12px;color:#64748b}.method{margin-left:10px}
|
|
762
|
+
table{border-collapse:collapse;width:100%;font-size:13px}th,td{text-align:left;vertical-align:top;padding:10px;border-bottom:1px solid #dce4ef}
|
|
763
|
+
th{background:#edf3fa}.table-wrap{overflow:auto}.flow{display:flex;gap:6px;flex-wrap:wrap;padding:12px 0}
|
|
764
|
+
.flow span{border:1px solid #a7bedb;border-radius:6px;padding:7px 10px}.badge{display:inline-block;background:#dce9fb;color:#183b68;padding:4px 9px;border-radius:20px;font-size:12px}
|
|
765
|
+
section,article,details{scroll-margin-top:20px}@media(max-width:850px){.layout{grid-template-columns:1fr;padding:16px}nav{position:static;display:flex;flex-wrap:wrap}article,.panel{padding:18px}}
|
|
766
|
+
"""
|
|
767
|
+
head = ("<!doctype html><html lang='en'><head><meta charset='utf-8'>"
|
|
768
|
+
"<meta name='viewport' content='width=device-width,initial-scale=1'>"
|
|
769
|
+
"<meta http-equiv='Content-Security-Policy' content=\"default-src 'none'; style-src 'unsafe-inline'; img-src 'none'; base-uri 'none'; form-action 'none'\">"
|
|
770
|
+
"<title>Sphynx " + e(content["sdk_version"]) + " review documentation</title><style>" + style + "</style></head><body>")
|
|
771
|
+
header = ("<header><span class='badge'>Unpublished review candidate</span><h1>Sphynx engineering evaluations</h1>"
|
|
772
|
+
"<p>Define a suite. Plan a candidate run. Retain evidence. Assess it and compare compatible reports.</p>"
|
|
773
|
+
"<p>" + e(content["id"]) + " · SDK " + e(content["sdk_version"]) + "</p></header>")
|
|
774
|
+
overview = ("<section class='panel'><h2>One workflow, explicit domain behavior</h2><div class='flow' aria-label='Workflow overview'>"
|
|
775
|
+
+ " → ".join("<span>" + label + "</span>" for label in
|
|
776
|
+
("Suite", "Plan", "Resolve", "Admit", "Run", "Assess", "Report", "Compare"))
|
|
777
|
+
+ "</div><p>The backend owns domain payloads, evidence verification and comparison policies. "
|
|
778
|
+
"Completion is not a quality verdict. Local source metadata does not verify a PR revision.</p></section>")
|
|
779
|
+
api = ("<article id='generated-api'><h2>Generated API reference</h2><p>Extracted by AST from the reviewed source. "
|
|
780
|
+
"Signatures retain source annotations/default expressions. Dataclass factories are identified explicitly. "
|
|
781
|
+
"No provider, candidate, evaluator or example ran during export.</p>" + "".join(entries) + "</article>")
|
|
782
|
+
capabilities = ("<article id='generated-capabilities'><h2>Workflow libraries</h2><h3>Default bindings</h3>" + backends
|
|
783
|
+
+ "<h3>Configured registry factories</h3><p>These public factories are derived from source declarations. "
|
|
784
|
+
"Supply their operator configuration, then inspect the registered backend descriptor. "
|
|
785
|
+
"Discovery does not grant asset access or verify a live deployment.</p>" + factories
|
|
786
|
+
+ "<p>Native library catalog entries do not imply a common backend. The default local adapter "
|
|
787
|
+
"does not verify candidate source bytes or provide OS isolation.</p><h3>Native parameter schemas</h3>"
|
|
788
|
+
"<p>These existing native contracts remain domain-specific. A schema declaration is not execution admission.</p>"
|
|
789
|
+
+ schema_details + "</article>")
|
|
790
|
+
receipt = ("<article id='candidate-receipt'><h2>Candidate provenance</h2>"
|
|
791
|
+
+ _table(["Field", "Value"], [
|
|
792
|
+
["Package", release["distribution"] + " " + release["sdk_version"]],
|
|
793
|
+
["Wheel SHA256", release["wheel"]["sha256"]],
|
|
794
|
+
["Source revision", release["source_revision"]],
|
|
795
|
+
["Base commit contains all reviewed source", "No; the snapshot digest identifies changed source"],
|
|
796
|
+
["Delivery", "Review only; not published or deployed"]])
|
|
797
|
+
+ "<p><a href='bundle.json'>Approved bundle identity</a> · <a href='manifest.json'>File checksums</a> · "
|
|
798
|
+
"<a href='source-identity.json'>Reviewed source inventory</a> · <a href='catalog.json'>Generated catalog</a></p></article>")
|
|
799
|
+
nav = "".join(navigation) + "<a href='#generated-api'>API reference</a><a href='#generated-capabilities'>Capabilities</a><a href='#candidate-receipt'>Provenance</a>"
|
|
800
|
+
raw = (head + header + "<div class='layout'><nav aria-label='Guide navigation'>" + nav
|
|
801
|
+
+ "</nav><main>" + overview + "".join(pages) + api + capabilities + receipt + "</main></div></body></html>").encode("utf-8")
|
|
802
|
+
_require(len(raw) <= MAX_JSON_BYTES, "rendered candidate exceeds size limit")
|
|
803
|
+
return raw
|
|
804
|
+
|
|
805
|
+
|
|
806
|
+
def export_candidate(source_root, content_path, wheel, source_identity, output):
|
|
807
|
+
"""Bind guides/catalog/HTML to actual wheel and reviewed source bytes."""
|
|
808
|
+
root = Path(source_root).resolve(strict=True)
|
|
809
|
+
content_path = Path(content_path).resolve(strict=True)
|
|
810
|
+
_require(content_path.is_relative_to(root), "curated source must be in the reviewed source root")
|
|
811
|
+
catalog, content = extract_catalog(root), _json(content_path)
|
|
812
|
+
_validate_catalog(catalog)
|
|
813
|
+
validate_content(content)
|
|
814
|
+
identity = _json(source_identity)
|
|
815
|
+
required = {*_CORE_SOURCE, content_path.relative_to(root).as_posix(),
|
|
816
|
+
*(entry["source_path"] for entry in catalog["exports"])}
|
|
817
|
+
rows = _identity(identity, source_root=root, required=required)
|
|
818
|
+
_require(content["sdk_version"] == catalog["sdk_version"] == identity["sdk_version"],
|
|
819
|
+
"candidate content/source/package version mismatch")
|
|
820
|
+
_require(identity["distribution"] == catalog["distribution"], "candidate distribution mismatch")
|
|
821
|
+
wheel_identity = _wheel(wheel, identity, rows)
|
|
822
|
+
release = {"id": content["id"], "sdk_version": catalog["sdk_version"],
|
|
823
|
+
"distribution": catalog["distribution"], "source_revision": identity["source_revision"],
|
|
824
|
+
"base_head_contains_all_reviewed_source": False, "wheel": wheel_identity,
|
|
825
|
+
"content_source_path": content_path.relative_to(root).as_posix()}
|
|
826
|
+
files = {"catalog.json": _canonical(catalog), "content.json": _canonical(content),
|
|
827
|
+
"source-identity.json": _canonical(identity), "index.html": _render(content, catalog, release)}
|
|
828
|
+
bundle = {"schema_version": BUNDLE_SCHEMA, "exporter_version": EXPORTER_VERSION,
|
|
829
|
+
"release": release, "files": {
|
|
830
|
+
name: {"sha256": _sha(raw), "size_bytes": len(raw)} for name, raw in sorted(files.items())}}
|
|
831
|
+
files["bundle.json"] = _canonical(bundle)
|
|
832
|
+
manifest = {"schema_version": MANIFEST_SCHEMA, "bundle_sha256": _sha(files["bundle.json"]),
|
|
833
|
+
"files": {name: {"sha256": _sha(raw), "size_bytes": len(raw)} for name, raw in sorted(files.items())}}
|
|
834
|
+
files["manifest.json"] = _canonical(manifest)
|
|
835
|
+
destination = Path(output)
|
|
836
|
+
_require(not destination.is_symlink(), "candidate output must not be symlinked")
|
|
837
|
+
if destination.exists():
|
|
838
|
+
_require(destination.is_dir() and {item.name for item in destination.iterdir()} == _MEMBERS,
|
|
839
|
+
"existing candidate output differs; choose a new directory")
|
|
840
|
+
_require(all(_read(destination / name) == raw for name, raw in files.items()),
|
|
841
|
+
"existing candidate bytes differ; choose a new directory")
|
|
842
|
+
else:
|
|
843
|
+
destination.mkdir(parents=True)
|
|
844
|
+
for name, raw in files.items():
|
|
845
|
+
with (destination / name).open("xb") as stream:
|
|
846
|
+
stream.write(raw)
|
|
847
|
+
return verify_candidate(destination, manifest["bundle_sha256"], source_root=root, wheel=wheel)
|
|
848
|
+
|
|
849
|
+
|
|
850
|
+
def verify_candidate(bundle_dir, expected_sha256, *, source_root=None, wheel=None):
|
|
851
|
+
"""Verify an approved bundle digest and optional current source/wheel bytes."""
|
|
852
|
+
_require(type(expected_sha256) is str and _HASH.fullmatch(expected_sha256),
|
|
853
|
+
"verification requires an approved bundle.json SHA256")
|
|
854
|
+
_require(not Path(bundle_dir).is_symlink(), "candidate bundle directory must not be symlinked")
|
|
855
|
+
directory = Path(bundle_dir).resolve(strict=True)
|
|
856
|
+
_require({item.name for item in directory.iterdir()} == _MEMBERS, "candidate bundle has missing or extra files")
|
|
857
|
+
bundle_raw = _read(directory / "bundle.json")
|
|
858
|
+
_require(_sha(bundle_raw) == expected_sha256, "approved candidate bundle digest mismatch")
|
|
859
|
+
bundle, manifest = _json(directory / "bundle.json"), _json(directory / "manifest.json")
|
|
860
|
+
_keys(bundle, "schema_version exporter_version release files", "candidate bundle")
|
|
861
|
+
_require(bundle["schema_version"] == BUNDLE_SCHEMA and bundle["exporter_version"] == EXPORTER_VERSION,
|
|
862
|
+
"unsupported candidate bundle/exporter version")
|
|
863
|
+
_keys(manifest, "schema_version bundle_sha256 files", "candidate manifest")
|
|
864
|
+
_require(manifest["schema_version"] == MANIFEST_SCHEMA and manifest["bundle_sha256"] == expected_sha256,
|
|
865
|
+
"candidate manifest identity mismatch")
|
|
866
|
+
_require(type(bundle["files"]) is dict and set(bundle["files"]) == set(_FILES), "unapproved candidate file set")
|
|
867
|
+
expected_files = dict(bundle["files"])
|
|
868
|
+
expected_files["bundle.json"] = {"sha256": expected_sha256, "size_bytes": len(bundle_raw)}
|
|
869
|
+
_require(manifest["files"] == expected_files, "manifest differs from approved bundle checksums")
|
|
870
|
+
for name, row in expected_files.items():
|
|
871
|
+
_keys(row, "sha256 size_bytes", "file receipt")
|
|
872
|
+
_require(type(row["sha256"]) is str and _HASH.fullmatch(row["sha256"])
|
|
873
|
+
and type(row["size_bytes"]) is int and 0 <= row["size_bytes"] <= MAX_JSON_BYTES, "invalid file receipt")
|
|
874
|
+
raw = _read(directory / name)
|
|
875
|
+
_require(len(raw) == row["size_bytes"] and _sha(raw) == row["sha256"], "candidate file digest mismatch: " + name)
|
|
876
|
+
content, catalog, identity = (_json(directory / name) for name in ("content.json", "catalog.json", "source-identity.json"))
|
|
877
|
+
validate_content(content)
|
|
878
|
+
_validate_catalog(catalog)
|
|
879
|
+
release = bundle["release"]
|
|
880
|
+
_keys(release, "id sdk_version distribution source_revision base_head_contains_all_reviewed_source wheel content_source_path",
|
|
881
|
+
"candidate release")
|
|
882
|
+
content_source = _relative(release["content_source_path"]).as_posix()
|
|
883
|
+
rows = _identity(identity, source_root=source_root,
|
|
884
|
+
required={*_CORE_SOURCE, content_source, *(entry["source_path"] for entry in catalog["exports"])})
|
|
885
|
+
_keys(release["wheel"], "filename sha256 size_bytes", "wheel receipt")
|
|
886
|
+
_relative(release["wheel"]["filename"])
|
|
887
|
+
_require(type(release["wheel"]["sha256"]) is str and _HASH.fullmatch(release["wheel"]["sha256"])
|
|
888
|
+
and type(release["wheel"]["size_bytes"]) is int
|
|
889
|
+
and 0 < release["wheel"]["size_bytes"] <= MAX_SOURCE_BYTES, "invalid wheel receipt")
|
|
890
|
+
_require(release["id"] == content["id"] and release["sdk_version"] == content["sdk_version"]
|
|
891
|
+
== catalog["sdk_version"] == identity["sdk_version"], "candidate SDK/content identity mismatch")
|
|
892
|
+
_require(release["source_revision"] == identity["source_revision"]
|
|
893
|
+
and release["distribution"] == catalog["distribution"] == identity["distribution"]
|
|
894
|
+
and release["base_head_contains_all_reviewed_source"] is False, "candidate source identity mismatch")
|
|
895
|
+
if source_root is not None:
|
|
896
|
+
_require(extract_catalog(source_root) == catalog, "stale candidate API/catalog metadata")
|
|
897
|
+
_require(_json(_source_path(Path(source_root).resolve(strict=True), content_source)) == content,
|
|
898
|
+
"candidate guides differ from their reviewed source")
|
|
899
|
+
if wheel is not None:
|
|
900
|
+
_require(_wheel(wheel, identity, rows) == release["wheel"], "candidate wheel differs from approved receipt")
|
|
901
|
+
_require(_render(content, catalog, release) == _read(directory / "index.html"), "candidate HTML differs from approved rendering")
|
|
902
|
+
return {"status": "passed", "candidate_id": release["id"], "sdk_version": release["sdk_version"],
|
|
903
|
+
"bundle_sha256": expected_sha256, "source_revision": release["source_revision"],
|
|
904
|
+
"wheel_sha256": release["wheel"]["sha256"], "source_file_count": len(rows),
|
|
905
|
+
"export_count": len(catalog["exports"]), "source_checked": source_root is not None,
|
|
906
|
+
"wheel_checked": wheel is not None}
|
|
907
|
+
|
|
908
|
+
|
|
909
|
+
def main(argv=None):
|
|
910
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
911
|
+
subparsers = parser.add_subparsers(dest="command", required=True)
|
|
912
|
+
root_default = Path(__file__).resolve().parents[2]
|
|
913
|
+
check = subparsers.add_parser("candidate-check", help="Validate source declarations and curated guides only")
|
|
914
|
+
check.add_argument("--source-root", type=Path, default=root_default)
|
|
915
|
+
check.add_argument("--content", type=Path, required=True)
|
|
916
|
+
export = subparsers.add_parser("candidate-export", help="Bind a review bundle to actual reviewed source and wheel")
|
|
917
|
+
for name in ("source-root", "content", "wheel", "source-identity", "output"):
|
|
918
|
+
export.add_argument("--" + name, type=Path, required=True)
|
|
919
|
+
verify = subparsers.add_parser("candidate-verify", help="Verify the approved bundle digest and optional source/wheel")
|
|
920
|
+
verify.add_argument("--bundle-dir", type=Path, required=True)
|
|
921
|
+
verify.add_argument("--expected-sha256", required=True)
|
|
922
|
+
verify.add_argument("--source-root", type=Path)
|
|
923
|
+
verify.add_argument("--wheel", type=Path)
|
|
924
|
+
args = parser.parse_args(argv)
|
|
925
|
+
try:
|
|
926
|
+
if args.command == "candidate-check":
|
|
927
|
+
result = check_candidate(args.source_root, args.content)
|
|
928
|
+
elif args.command == "candidate-export":
|
|
929
|
+
result = export_candidate(args.source_root, args.content, args.wheel, args.source_identity, args.output)
|
|
930
|
+
else:
|
|
931
|
+
result = verify_candidate(args.bundle_dir, args.expected_sha256, source_root=args.source_root, wheel=args.wheel)
|
|
932
|
+
except (CandidateDocumentationError, OSError, KeyError, TypeError) as exc:
|
|
933
|
+
parser.exit(2, "candidate documentation: " + str(exc) + "\n")
|
|
934
|
+
print(json.dumps(result, ensure_ascii=True, sort_keys=True))
|
|
935
|
+
return 0
|
|
936
|
+
|
|
937
|
+
|
|
938
|
+
if __name__ == "__main__":
|
|
939
|
+
raise SystemExit(main())
|