sable-sphynx 0.7.0.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sable_sphynx-0.7.0.dev0.dist-info/METADATA +152 -0
- sable_sphynx-0.7.0.dev0.dist-info/RECORD +174 -0
- sable_sphynx-0.7.0.dev0.dist-info/WHEEL +5 -0
- sable_sphynx-0.7.0.dev0.dist-info/top_level.txt +1 -0
- sphynx/__init__.py +469 -0
- sphynx/__init__.pyi +376 -0
- sphynx/_analytical_client.py +77 -0
- sphynx/_analytical_values.py +252 -0
- sphynx/_audit_client.py +173 -0
- sphynx/_comparison_adapters.py +113 -0
- sphynx/_componentized_full_rag.py +34 -0
- sphynx/_composed_pipeline.py +109 -0
- sphynx/_full_rag_grading.py +218 -0
- sphynx/_full_rag_trace.py +221 -0
- sphynx/_local_eval.py +268 -0
- sphynx/_pipeline_core.py +243 -0
- sphynx/_pr_client.py +237 -0
- sphynx/_rag_diagnostics_calibration.py +285 -0
- sphynx/_rag_diagnostics_catalog.py +39 -0
- sphynx/_rag_diagnostics_common.py +26 -0
- sphynx/_rag_diagnostics_contracts.py +186 -0
- sphynx/_rag_diagnostics_engine.py +243 -0
- sphynx/_rag_diagnostics_importers.py +242 -0
- sphynx/_rag_diagnostics_ranking.py +81 -0
- sphynx/_rag_diagnostics_semantic.py +325 -0
- sphynx/_rag_ranking.py +149 -0
- sphynx/_speech_benchmark_scoring.py +140 -0
- sphynx/_speech_benchmark_sources.py +275 -0
- sphynx/_speech_complete_scoring.py +218 -0
- sphynx/_speech_complete_source.py +118 -0
- sphynx/_speech_kido.py +294 -0
- sphynx/_speech_scoring.py +77 -0
- sphynx/_track_capture.py +213 -0
- sphynx/analytical_views.py +515 -0
- sphynx/assessment_contracts.py +378 -0
- sphynx/assessment_selection.py +57 -0
- sphynx/audit_access.py +245 -0
- sphynx/audit_contracts.py +204 -0
- sphynx/audit_requests.py +180 -0
- sphynx/authoring.py +188 -0
- sphynx/benchmarks.py +568 -0
- sphynx/candidates.py +62 -0
- sphynx/catalog.py +272 -0
- sphynx/chunking.py +372 -0
- sphynx/clickhouse_reader.py +169 -0
- sphynx/client.py +486 -0
- sphynx/comparison_core.py +149 -0
- sphynx/comparison_views.py +223 -0
- sphynx/components.py +1661 -0
- sphynx/contracts.py +514 -0
- sphynx/customer_rerank.py +299 -0
- sphynx/customer_retrieval.py +146 -0
- sphynx/data/benchmark-releases.v1.json +164 -0
- sphynx/data/public-benchmark-release.v1.json +354 -0
- sphynx/data/rag-diagnostics-synthetic.v1.json +6982 -0
- sphynx/data/rag-paper-references.v1.json +176 -0
- sphynx/data/rag-paper-schemas.v1.json +1069 -0
- sphynx/data/rag-rerank-v1.json +507 -0
- sphynx/dataset_catalog_data.py +1214 -0
- sphynx/dataset_readiness.py +138 -0
- sphynx/dataset_v3.py +999 -0
- sphynx/demo.py +130 -0
- sphynx/demo_candidates/__init__.py +1 -0
- sphynx/demo_candidates/dense.py +11 -0
- sphynx/demo_candidates/lexical.py +21 -0
- sphynx/diagnostics.py +207 -0
- sphynx/documentation.py +737 -0
- sphynx/documentation_candidate.py +939 -0
- sphynx/evals.py +538 -0
- sphynx/evaluation_protocols.py +124 -0
- sphynx/evaluator_bindings.py +370 -0
- sphynx/evidence.py +352 -0
- sphynx/execution_capabilities.py +109 -0
- sphynx/fast_search.py +285 -0
- sphynx/full_rag.py +341 -0
- sphynx/hosted/__init__.py +18 -0
- sphynx/hosted/benchmark_intake.py +218 -0
- sphynx/hosted/contracts.py +361 -0
- sphynx/hosted/model_admission.py +130 -0
- sphynx/hosted/model_artifacts.py +298 -0
- sphynx/hosted/model_contracts.py +63 -0
- sphynx/hosted/model_gcs.py +181 -0
- sphynx/hosted/model_history.py +84 -0
- sphynx/hosted/model_session.py +233 -0
- sphynx/hosted/public_suite.py +74 -0
- sphynx/hosted/registry.py +97 -0
- sphynx/hosted/revisions.py +126 -0
- sphynx/hosted/sandbox.py +105 -0
- sphynx/hosted/submission.py +287 -0
- sphynx/hosted_workflow.py +451 -0
- sphynx/http_transport.py +86 -0
- sphynx/integrated_rag.py +352 -0
- sphynx/integrated_workflow.py +552 -0
- sphynx/integration_contracts.py +474 -0
- sphynx/invocation.py +791 -0
- sphynx/invocation_candidates.py +392 -0
- sphynx/invocation_catalog_data.py +40 -0
- sphynx/learned_catalog_data.py +30 -0
- sphynx/learned_rerank.py +267 -0
- sphynx/legacy_full_rag.py +67 -0
- sphynx/lifecycle.py +885 -0
- sphynx/lifecycle_workflow.py +490 -0
- sphynx/metric_views.py +474 -0
- sphynx/metrics.py +282 -0
- sphynx/model_adapters.py +438 -0
- sphynx/model_catalog_data.py +30 -0
- sphynx/official_benchmarks.py +385 -0
- sphynx/packs.py +258 -0
- sphynx/pipeline.py +313 -0
- sphynx/pipeline_workflow.py +551 -0
- sphynx/pr_evaluations.py +560 -0
- sphynx/progress.py +89 -0
- sphynx/public_bundle.py +173 -0
- sphynx/public_eval.py +62 -0
- sphynx/public_rerank.py +355 -0
- sphynx/py.typed +0 -0
- sphynx/pytest_plugin.py +62 -0
- sphynx/question_robustness.py +437 -0
- sphynx/rag.py +329 -0
- sphynx/rag_diagnostics.py +309 -0
- sphynx/rag_papers/__init__.py +29 -0
- sphynx/rag_papers/__main__.py +45 -0
- sphynx/rag_papers/candidates.py +19 -0
- sphynx/rag_papers/contracts.py +282 -0
- sphynx/rag_papers/datasets.py +291 -0
- sphynx/rag_papers/experiments.py +116 -0
- sphynx/rag_papers/judges.py +308 -0
- sphynx/rag_papers/registry.py +33 -0
- sphynx/rag_papers/runner.py +176 -0
- sphynx/rag_papers/sable.py +135 -0
- sphynx/rag_papers/schemas.py +12 -0
- sphynx/rag_papers/scoring.py +249 -0
- sphynx/rag_papers/semantic.py +288 -0
- sphynx/rag_papers/trec.py +80 -0
- sphynx/recorded_scores.py +352 -0
- sphynx/records.py +124 -0
- sphynx/reflex_traces.py +193 -0
- sphynx/report_rendering.py +71 -0
- sphynx/rerank_family_contracts.py +150 -0
- sphynx/rerank_workflow.py +424 -0
- sphynx/research_candidate.py +190 -0
- sphynx/results.py +852 -0
- sphynx/retrieval.py +750 -0
- sphynx/retrieval_candidates.py +295 -0
- sphynx/retrieval_catalog_data.py +19 -0
- sphynx/retrieval_recordings.py +125 -0
- sphynx/reviewer_audit.py +266 -0
- sphynx/routing_policies.py +375 -0
- sphynx/runner.py +544 -0
- sphynx/sable_rerank.py +489 -0
- sphynx/score_reliability.py +393 -0
- sphynx/speech_benchmarks.py +204 -0
- sphynx/speech_campaigns.py +378 -0
- sphynx/speech_complete.py +236 -0
- sphynx/speech_contracts.py +225 -0
- sphynx/speech_evidence.py +429 -0
- sphynx/speech_listening.py +115 -0
- sphynx/speech_multilingual.py +71 -0
- sphynx/speech_tts.py +479 -0
- sphynx/sql/001_sync_v1.sql +30 -0
- sphynx/stage_contracts.py +199 -0
- sphynx/statistical_catalog_data.py +53 -0
- sphynx/statistical_rerank.py +232 -0
- sphynx/sync.py +797 -0
- sphynx/workbench.py +378 -0
- sphynx/workbench_catalog_data.py +13 -0
- sphynx/workbench_tracks.py +78 -0
- sphynx/workflow_async.py +197 -0
- sphynx/workflow_contracts.py +594 -0
- sphynx/workflow_decisions.py +211 -0
- sphynx/workflow_defaults.py +16 -0
- sphynx/workflow_extensions.py +212 -0
- sphynx/workflow_registry.py +513 -0
- sphynx/workflows.py +792 -0
|
@@ -0,0 +1,451 @@
|
|
|
1
|
+
"""Explicit adapter for native exact-revision hosted PR submissions.
|
|
2
|
+
|
|
3
|
+
No service is contacted during construction, registration or resolution. Local
|
|
4
|
+
admission validates a request, not remote permission or executable source bytes.
|
|
5
|
+
The configured owner API must authorize and verify its records. This adapter
|
|
6
|
+
checks their linkage and preserves native trust; it does not supply isolation,
|
|
7
|
+
read protected cases, or promote historical service notes into live authority.
|
|
8
|
+
"""
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
import re
|
|
13
|
+
from dataclasses import dataclass, field
|
|
14
|
+
from typing import Any, Mapping
|
|
15
|
+
|
|
16
|
+
from .client import Client, Experiment, HostedClientError, HostedEvaluation, HostedRun, _retry_key
|
|
17
|
+
from .evals import describe_eval, resolve_eval
|
|
18
|
+
from .hosted.registry import BASELINE_ID, PUBLIC_SUITE_ID, REGISTERED_MAIN_BASELINE_ID, SUITE_ID
|
|
19
|
+
from .hosted.revisions import require_commit_sha
|
|
20
|
+
from .hosted.submission import pr_identity
|
|
21
|
+
from .workflow_async import GradingSnapshot, SubmissionError, SubmissionIntent, SubmissionSnapshot
|
|
22
|
+
from .workflow_contracts import (
|
|
23
|
+
AdmissionBinding, BackendDescriptor, Criterion, EvaluatorDefinition, MetricSpec,
|
|
24
|
+
PlanBinding, ReferenceBinding, ResolvedBinding, SuiteDefinition,
|
|
25
|
+
WorkflowContractError, _object, _text, _thaw, contract_digest,
|
|
26
|
+
)
|
|
27
|
+
from .workflow_registry import WorkflowRegistry
|
|
28
|
+
|
|
29
|
+
HOSTED_BACKEND_ID = "sphynx.hosted.pr"
|
|
30
|
+
_SUITES = (SUITE_ID, PUBLIC_SUITE_ID)
|
|
31
|
+
_METHOD = re.compile(r"[A-Za-z0-9][A-Za-z0-9._:-]{0,127}\Z")
|
|
32
|
+
_RESULTS = {SUITE_ID: "sphynx.hosted.result.v1", PUBLIC_SUITE_ID: "sphynx.hosted.public-rerank.result.v1"}
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@dataclass(frozen=True)
|
|
36
|
+
class PullRequestCandidate:
|
|
37
|
+
"""Strict PR request; legacy Client.submit still permits an omitted SHA guard."""
|
|
38
|
+
|
|
39
|
+
candidate_pr: str
|
|
40
|
+
expected_head_sha: str
|
|
41
|
+
candidate_method: str | None = None
|
|
42
|
+
baseline: str = BASELINE_ID
|
|
43
|
+
expected_baseline_sha: str | None = None
|
|
44
|
+
|
|
45
|
+
def __post_init__(self) -> None:
|
|
46
|
+
pr_identity(self.candidate_pr)
|
|
47
|
+
object.__setattr__(self, "expected_head_sha", require_commit_sha(self.expected_head_sha))
|
|
48
|
+
if self.candidate_method is not None and (type(self.candidate_method) is not str
|
|
49
|
+
or not _METHOD.fullmatch(self.candidate_method)):
|
|
50
|
+
raise WorkflowContractError("candidate_method must be a bounded ASCII method ID")
|
|
51
|
+
if self.baseline not in (BASELINE_ID, REGISTERED_MAIN_BASELINE_ID):
|
|
52
|
+
raise WorkflowContractError("baseline is not supported by the native hosted adapter")
|
|
53
|
+
if self.expected_baseline_sha is not None:
|
|
54
|
+
if self.baseline != REGISTERED_MAIN_BASELINE_ID:
|
|
55
|
+
raise WorkflowContractError("expected_baseline_sha requires the registered main baseline")
|
|
56
|
+
object.__setattr__(self, "expected_baseline_sha", require_commit_sha(self.expected_baseline_sha))
|
|
57
|
+
|
|
58
|
+
def to_dict(self) -> dict[str, Any]:
|
|
59
|
+
result = {"candidate_pr": self.candidate_pr, "expected_head_sha": self.expected_head_sha,
|
|
60
|
+
"baseline": self.baseline}
|
|
61
|
+
if self.candidate_method is not None:
|
|
62
|
+
result["candidate_method"] = self.candidate_method
|
|
63
|
+
if self.expected_baseline_sha is not None:
|
|
64
|
+
result["expected_baseline_sha"] = self.expected_baseline_sha
|
|
65
|
+
return result
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
@dataclass(frozen=True)
|
|
69
|
+
class _Submission:
|
|
70
|
+
handle: Experiment = field(repr=False, compare=False)
|
|
71
|
+
request_identity: Mapping[str, Any]
|
|
72
|
+
idempotency_key: str
|
|
73
|
+
owner: Any = field(repr=False, compare=False)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
@dataclass(frozen=True)
|
|
77
|
+
class _Run:
|
|
78
|
+
handle: HostedRun = field(repr=False, compare=False)
|
|
79
|
+
submission: _Submission = field(repr=False, compare=False)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
@dataclass(frozen=True)
|
|
83
|
+
class _Grading:
|
|
84
|
+
handle: HostedEvaluation = field(repr=False, compare=False)
|
|
85
|
+
run: _Run = field(repr=False, compare=False)
|
|
86
|
+
idempotency_key: str
|
|
87
|
+
request_identity: Mapping[str, Any]
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
@dataclass(frozen=True)
|
|
91
|
+
class _CompletedGrading:
|
|
92
|
+
pending: _Grading = field(repr=False, compare=False)
|
|
93
|
+
payload: Mapping[str, Any] = field(repr=False)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
@dataclass(frozen=True)
|
|
97
|
+
class _Projection:
|
|
98
|
+
payload: Mapping[str, Any] = field(repr=False)
|
|
99
|
+
|
|
100
|
+
def to_dict(self) -> dict[str, Any]:
|
|
101
|
+
return _thaw(self.payload)
|
|
102
|
+
|
|
103
|
+
def to_json(self) -> str:
|
|
104
|
+
return json.dumps(self.to_dict(), ensure_ascii=False, allow_nan=False, sort_keys=True)
|
|
105
|
+
|
|
106
|
+
def to_markdown(self) -> str:
|
|
107
|
+
summary = self.payload.get("summary")
|
|
108
|
+
if not isinstance(summary, str):
|
|
109
|
+
raise WorkflowContractError("native hosted report has no readable summary")
|
|
110
|
+
return summary
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
class HostedComparison:
|
|
114
|
+
"""Native saved comparison handle with explicit result projection.
|
|
115
|
+
|
|
116
|
+
Pending retrieval remains pending; obtaining this handle never runs or grades
|
|
117
|
+
candidates. The native comparison ID remains available after a read error.
|
|
118
|
+
"""
|
|
119
|
+
|
|
120
|
+
def __init__(self, backend, native_ref, evaluation_a, evaluation_b):
|
|
121
|
+
self._backend, self.native_ref = backend, native_ref
|
|
122
|
+
self.evaluation_a, self.evaluation_b = evaluation_a, evaluation_b
|
|
123
|
+
|
|
124
|
+
@property
|
|
125
|
+
def status(self):
|
|
126
|
+
return self.native_ref.status
|
|
127
|
+
|
|
128
|
+
def refresh(self):
|
|
129
|
+
self._backend._origin()
|
|
130
|
+
return HostedComparison(self._backend, self._backend.client.get_comparison(self.native_ref.id),
|
|
131
|
+
self.evaluation_a, self.evaluation_b)
|
|
132
|
+
|
|
133
|
+
def to_dict(self):
|
|
134
|
+
self._backend._origin()
|
|
135
|
+
payload = self.native_ref.result().to_dict()
|
|
136
|
+
if payload.get("evaluation_a") != self.evaluation_a or payload.get("evaluation_b") != self.evaluation_b:
|
|
137
|
+
raise WorkflowContractError("hosted comparison refers to different grading passes")
|
|
138
|
+
return payload
|
|
139
|
+
|
|
140
|
+
def to_json(self):
|
|
141
|
+
return json.dumps(self.to_dict(), ensure_ascii=False, allow_nan=False, sort_keys=True)
|
|
142
|
+
|
|
143
|
+
def to_markdown(self):
|
|
144
|
+
return _Projection(_object(self.to_dict(), "comparison")).to_markdown()
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
class HostedWorkflowBackend:
|
|
148
|
+
"""Operator-enabled native hosted adapter; no service checkout is bundled."""
|
|
149
|
+
|
|
150
|
+
def __init__(self, client: Client, *, suite_ids: tuple[str, ...] = (SUITE_ID,)):
|
|
151
|
+
if not isinstance(client, Client):
|
|
152
|
+
raise WorkflowContractError("hosted backend requires an explicitly configured Client")
|
|
153
|
+
if not isinstance(suite_ids, tuple) or not suite_ids or len(set(suite_ids)) != len(suite_ids):
|
|
154
|
+
raise WorkflowContractError("suite_ids must be a nonempty unique tuple")
|
|
155
|
+
if any(identifier not in _SUITES for identifier in suite_ids):
|
|
156
|
+
raise WorkflowContractError("no native hosted adapter for the selected suite")
|
|
157
|
+
self.client, self._owner = client, object()
|
|
158
|
+
self._descriptor = BackendDescriptor(
|
|
159
|
+
HOSTED_BACKEND_ID, "v1", "sphynx.hosted.pr-adapter.v1",
|
|
160
|
+
capabilities=("sphynx.async-submission.v1", "sphynx.async-grading.v1",
|
|
161
|
+
"sphynx.saved-assessment.v1", "sphynx.native-report.v1",
|
|
162
|
+
"sphynx.paired-comparison.v1"),
|
|
163
|
+
supported_scopes=("component",), supported_modes=("hosted",), suite_ids=suite_ids,
|
|
164
|
+
metadata={"api_origin": client.api_url, "remote_admission": "server_owned",
|
|
165
|
+
"case_access": "not_exposed", "case_assessments": "not_exposed",
|
|
166
|
+
"live_service_verified": False},
|
|
167
|
+
)
|
|
168
|
+
|
|
169
|
+
@property
|
|
170
|
+
def descriptor(self):
|
|
171
|
+
return self._descriptor
|
|
172
|
+
|
|
173
|
+
def describe(self, suite_id):
|
|
174
|
+
if suite_id not in self.descriptor.suite_ids:
|
|
175
|
+
raise WorkflowContractError("suite is not enabled in this hosted backend")
|
|
176
|
+
native = describe_eval(suite_id)
|
|
177
|
+
ids = tuple(f"{suite_id}/{metric}" for metric in native["metric_ids"])
|
|
178
|
+
return SuiteDefinition(suite_id, native["version"], self.descriptor.backend_id,
|
|
179
|
+
native["title"], criterion_ids=ids, metric_ids=ids, evaluator_ids=(suite_id + "/grader",),
|
|
180
|
+
required_capabilities=("sphynx.async-submission.v1", "sphynx.async-grading.v1"),
|
|
181
|
+
metadata={"native_definition": native, "native_metric_mapping": dict(zip(ids, native["metric_ids"])),
|
|
182
|
+
"native_report_layout": "metrics.candidate and metrics.baseline",
|
|
183
|
+
"remote_admission": "pending_server_submission", "case_access": "not_exposed"})
|
|
184
|
+
|
|
185
|
+
def cases(self, suite):
|
|
186
|
+
raise WorkflowContractError("hosted case access is not exposed by this API")
|
|
187
|
+
|
|
188
|
+
def _plan(self, plan):
|
|
189
|
+
if not isinstance(plan, PlanBinding) or not isinstance(plan.candidate, PullRequestCandidate):
|
|
190
|
+
raise WorkflowContractError("hosted PR plans require PullRequestCandidate")
|
|
191
|
+
if plan.suite.definition_digest != self.describe(plan.suite.suite_id).definition_digest:
|
|
192
|
+
raise WorkflowContractError("hosted suite definition changed")
|
|
193
|
+
if plan.scope != "component" or plan.mode != "hosted" or plan.runtime:
|
|
194
|
+
raise WorkflowContractError("hosted adapter supports component/hosted with native parameter runtime only")
|
|
195
|
+
self._origin()
|
|
196
|
+
|
|
197
|
+
def _origin(self):
|
|
198
|
+
if self.client.api_url != self.descriptor.metadata["api_origin"]:
|
|
199
|
+
raise WorkflowContractError("configured hosted origin changed")
|
|
200
|
+
|
|
201
|
+
def resolve(self, plan):
|
|
202
|
+
self._plan(plan)
|
|
203
|
+
invocation = resolve_eval(plan.suite.suite_id, plan.parameters)
|
|
204
|
+
request = {**plan.candidate.to_dict(), "suite": plan.suite.suite_id,
|
|
205
|
+
"parameters": _thaw(invocation.parameters)}
|
|
206
|
+
return ResolvedBinding(identity={"backend": self.descriptor.to_dict(),
|
|
207
|
+
"suite_id": plan.suite.suite_id, "suite_digest": plan.suite.definition_digest,
|
|
208
|
+
"scope": plan.scope, "mode": plan.mode, "request": request,
|
|
209
|
+
"native_invocation": invocation.to_dict()}, native_ref=invocation)
|
|
210
|
+
|
|
211
|
+
def _resolution(self, plan, resolved):
|
|
212
|
+
if not isinstance(resolved, ResolvedBinding) or self.resolve(plan).resolution_digest != resolved.resolution_digest:
|
|
213
|
+
raise WorkflowContractError("hosted request identity changed; resolve again")
|
|
214
|
+
|
|
215
|
+
def admit(self, plan, resolved):
|
|
216
|
+
self._resolution(plan, resolved)
|
|
217
|
+
return AdmissionBinding(resolved.resolution_digest,
|
|
218
|
+
{"request_digest": contract_digest(resolved.identity),
|
|
219
|
+
"remote_admission": "pending_server_submission", "api_origin": self.client.api_url},
|
|
220
|
+
"unverified", False, False)
|
|
221
|
+
|
|
222
|
+
def recheck(self, plan, resolved, admission):
|
|
223
|
+
expected = self.admit(plan, resolved)
|
|
224
|
+
if not isinstance(admission, AdmissionBinding) or admission.to_dict() != expected.to_dict():
|
|
225
|
+
raise WorkflowContractError("hosted admission or request identity changed")
|
|
226
|
+
|
|
227
|
+
def execute(self, plan, resolved, admission, output_dir):
|
|
228
|
+
raise WorkflowContractError("hosted execution is asynchronous; use Plan.submit()")
|
|
229
|
+
|
|
230
|
+
def assess(self, native_run):
|
|
231
|
+
raise WorkflowContractError("hosted grading is asynchronous; use Run.start_grading()")
|
|
232
|
+
|
|
233
|
+
def submit(self, plan, resolved, admission, idempotency_key):
|
|
234
|
+
self.recheck(plan, resolved, admission)
|
|
235
|
+
key = _retry_key(idempotency_key)
|
|
236
|
+
intent = SubmissionIntent(key, resolved.identity)
|
|
237
|
+
try:
|
|
238
|
+
handle = self.client.submit(**_thaw(resolved.identity["request"]), idempotency_key=key)
|
|
239
|
+
except HostedClientError as exc:
|
|
240
|
+
raise SubmissionError("hosted submission did not return a handle; retain this intent for recovery",
|
|
241
|
+
intent=intent, code=exc.code) from None
|
|
242
|
+
ref = _Submission(handle, _object(resolved.identity, "request_identity"), key, self._owner)
|
|
243
|
+
self._submission(ref)
|
|
244
|
+
return ref
|
|
245
|
+
|
|
246
|
+
def _submission(self, ref):
|
|
247
|
+
self._origin()
|
|
248
|
+
if not isinstance(ref, _Submission) or ref.owner is not self._owner:
|
|
249
|
+
raise WorkflowContractError("submission belongs to another backend binding")
|
|
250
|
+
data, request = ref.handle.to_dict(), ref.request_identity["request"]
|
|
251
|
+
if data.get("id") != ref.handle.id:
|
|
252
|
+
raise WorkflowContractError("native submission identity changed")
|
|
253
|
+
if data.get("head_sha") is not None and data["head_sha"] != request["expected_head_sha"]:
|
|
254
|
+
raise WorkflowContractError("hosted submission head differs from expected revision")
|
|
255
|
+
if "request" in data and contract_digest(data["request"]) != contract_digest(request):
|
|
256
|
+
raise WorkflowContractError("server submission request differs from the selected request")
|
|
257
|
+
_text(data.get("status"), "native submission status")
|
|
258
|
+
return data
|
|
259
|
+
|
|
260
|
+
def refresh_submission(self, native_ref):
|
|
261
|
+
self._submission(native_ref)
|
|
262
|
+
native_ref.handle.refresh()
|
|
263
|
+
self._submission(native_ref)
|
|
264
|
+
return native_ref
|
|
265
|
+
|
|
266
|
+
def _manifest(self, manifest, identity):
|
|
267
|
+
request = identity["request"]
|
|
268
|
+
if not isinstance(manifest, Mapping):
|
|
269
|
+
raise WorkflowContractError("hosted record has no pinned manifest")
|
|
270
|
+
if (manifest.get("head_sha") != request["expected_head_sha"]
|
|
271
|
+
or manifest.get("candidate_pr") != request["candidate_pr"]
|
|
272
|
+
or not isinstance(manifest.get("suite"), Mapping)
|
|
273
|
+
or manifest["suite"].get("id") != request["suite"]
|
|
274
|
+
or not isinstance(manifest.get("baseline"), Mapping)
|
|
275
|
+
or manifest["baseline"].get("id") != request["baseline"]
|
|
276
|
+
or manifest.get("candidate_method") != request.get("candidate_method")):
|
|
277
|
+
raise WorkflowContractError("hosted manifest differs from pinned head, method, suite or baseline")
|
|
278
|
+
family = manifest.get("candidate_family")
|
|
279
|
+
if family is not None and (not isinstance(family, Mapping)
|
|
280
|
+
or family.get("method") != request.get("candidate_method")):
|
|
281
|
+
raise WorkflowContractError("hosted family selection differs from the requested method")
|
|
282
|
+
if request.get("expected_baseline_sha") is not None:
|
|
283
|
+
baseline = manifest["baseline"].get("registered_component", {})
|
|
284
|
+
source = baseline.get("source_revision", {}) if isinstance(baseline, Mapping) else {}
|
|
285
|
+
if source.get("head_sha") != request["expected_baseline_sha"]:
|
|
286
|
+
raise WorkflowContractError("hosted baseline differs from its expected revision")
|
|
287
|
+
|
|
288
|
+
def _run(self, ref):
|
|
289
|
+
if not isinstance(ref, _Run):
|
|
290
|
+
raise WorkflowContractError("expected a completed native hosted run")
|
|
291
|
+
self._submission(ref.submission)
|
|
292
|
+
current = self.client.get_run(ref.handle.id)
|
|
293
|
+
record = current.to_dict()
|
|
294
|
+
if record.get("status") not in ("completed", "captured"):
|
|
295
|
+
raise WorkflowContractError("native hosted capture is not complete")
|
|
296
|
+
if record.get("experiment_id") != ref.submission.handle.id:
|
|
297
|
+
raise WorkflowContractError("native run belongs to another submission")
|
|
298
|
+
self._manifest(record.get("manifest"), ref.submission.request_identity)
|
|
299
|
+
return record
|
|
300
|
+
|
|
301
|
+
def submission_snapshot(self, native_ref):
|
|
302
|
+
data = self._submission(native_ref)
|
|
303
|
+
status = data["status"]
|
|
304
|
+
run = grading = None
|
|
305
|
+
if data.get("run_id") is not None:
|
|
306
|
+
handle = self.client.get_run(data["run_id"])
|
|
307
|
+
if handle.status in ("completed", "captured"):
|
|
308
|
+
run = _Run(handle, native_ref)
|
|
309
|
+
self._run(run)
|
|
310
|
+
if status == "completed":
|
|
311
|
+
if run is None or data.get("evaluation_id") is None:
|
|
312
|
+
raise WorkflowContractError("completed submission has no complete native run/grading linkage")
|
|
313
|
+
pending = _Grading(self.client.get_evaluation(data["evaluation_id"]), run,
|
|
314
|
+
native_ref.idempotency_key, native_ref.request_identity)
|
|
315
|
+
grading = self.grading_snapshot(pending).completed_ref
|
|
316
|
+
if grading is None:
|
|
317
|
+
raise WorkflowContractError("completed submission has a pending grading pass")
|
|
318
|
+
error = data.get("error") if isinstance(data.get("error"), dict) else {}
|
|
319
|
+
return SubmissionSnapshot(native_ref.handle.id, status, native_ref.idempotency_key,
|
|
320
|
+
native_ref.request_identity, native_ref, run_id=data.get("run_id"),
|
|
321
|
+
grading_id=data.get("evaluation_id"), native_run=run, native_grading=grading,
|
|
322
|
+
complete=status == "completed", failed=status == "failed",
|
|
323
|
+
error_code=error.get("code") if isinstance(error.get("code"), str) else None,
|
|
324
|
+
reason="hosted submission failed; inspect its owned native status" if status == "failed" else None,
|
|
325
|
+
trust_tier=grading.payload.get("trust_tier", "unverified") if grading else "unverified")
|
|
326
|
+
|
|
327
|
+
def start_grading(self, native_run, idempotency_key):
|
|
328
|
+
self._run(native_run)
|
|
329
|
+
key = _retry_key(idempotency_key)
|
|
330
|
+
identity = _object({"run_id": native_run.handle.id,
|
|
331
|
+
"submission_request_digest": contract_digest(native_run.submission.request_identity),
|
|
332
|
+
"operation": "saved_grading"}, "grading request")
|
|
333
|
+
try:
|
|
334
|
+
handle = self.client.grade(native_run.handle.id, idempotency_key=key)
|
|
335
|
+
except HostedClientError as exc:
|
|
336
|
+
raise SubmissionError("hosted grading did not return a handle; retain this intent for recovery",
|
|
337
|
+
intent=SubmissionIntent(key, identity), code=exc.code) from None
|
|
338
|
+
return _Grading(handle, native_run, key, identity)
|
|
339
|
+
|
|
340
|
+
def _grading(self, ref):
|
|
341
|
+
if not isinstance(ref, _Grading):
|
|
342
|
+
raise WorkflowContractError("expected a native pending grading handle")
|
|
343
|
+
self._run(ref.run)
|
|
344
|
+
data = ref.handle.to_dict()
|
|
345
|
+
if data.get("run_id") != ref.run.handle.id:
|
|
346
|
+
raise WorkflowContractError("hosted grading refers to another run")
|
|
347
|
+
if data.get("experiment_id") is not None and data["experiment_id"] != ref.run.submission.handle.id:
|
|
348
|
+
raise WorkflowContractError("hosted grading belongs to another submission")
|
|
349
|
+
_text(data.get("status"), "native grading status")
|
|
350
|
+
return data
|
|
351
|
+
|
|
352
|
+
def refresh_grading(self, native_ref):
|
|
353
|
+
self._grading(native_ref)
|
|
354
|
+
native_ref.handle.refresh()
|
|
355
|
+
self._grading(native_ref)
|
|
356
|
+
return native_ref
|
|
357
|
+
|
|
358
|
+
def grading_snapshot(self, native_ref):
|
|
359
|
+
data = self._grading(native_ref)
|
|
360
|
+
status, completed = data["status"], None
|
|
361
|
+
trust = "unverified"
|
|
362
|
+
if status == "completed":
|
|
363
|
+
payload = self.client._request("GET", f"/v1/evaluations/{native_ref.handle.id}/result")
|
|
364
|
+
request = native_ref.run.submission.request_identity["request"]
|
|
365
|
+
if (payload.get("schema_version") != _RESULTS[request["suite"]]
|
|
366
|
+
or payload.get("experiment_id") != native_ref.run.submission.handle.id):
|
|
367
|
+
raise WorkflowContractError("hosted report has an unsupported schema or submission identity")
|
|
368
|
+
for key, expected in (("run_id", native_ref.run.handle.id), ("evaluation_id", native_ref.handle.id)):
|
|
369
|
+
if key in payload and payload[key] != expected:
|
|
370
|
+
raise WorkflowContractError("hosted report has a different native record identity")
|
|
371
|
+
self._manifest(payload.get("manifest"), native_ref.run.submission.request_identity)
|
|
372
|
+
capture_manifest = self._run(native_ref.run)["manifest"]
|
|
373
|
+
fields = ("head_sha", "tree_sha", "candidate_pr", "suite", "baseline", "candidate_method",
|
|
374
|
+
"candidate_family", "configuration_digest", "source_bundle_digest")
|
|
375
|
+
if contract_digest({key: capture_manifest.get(key) for key in fields}) != contract_digest(
|
|
376
|
+
{key: payload["manifest"].get(key) for key in fields}):
|
|
377
|
+
raise WorkflowContractError("hosted report differs from the captured source/cohort identity")
|
|
378
|
+
_text(payload.get("summary"), "native report summary")
|
|
379
|
+
trust = payload.get("trust_tier", "unverified")
|
|
380
|
+
_text(trust, "native trust_tier")
|
|
381
|
+
completed = _CompletedGrading(native_ref, _object(payload, "native report"))
|
|
382
|
+
error = data.get("error") if isinstance(data.get("error"), dict) else {}
|
|
383
|
+
return GradingSnapshot(ReferenceBinding(native_ref.handle.id, status, trust,
|
|
384
|
+
run_id=native_ref.run.handle.id), native_ref.idempotency_key,
|
|
385
|
+
native_ref.request_identity, native_ref, completed_ref=completed,
|
|
386
|
+
complete=status == "completed", failed=status == "failed",
|
|
387
|
+
error_code=error.get("code") if isinstance(error.get("code"), str) else None,
|
|
388
|
+
reason="hosted grading failed; inspect its owned native status" if status == "failed" else None)
|
|
389
|
+
|
|
390
|
+
def accepts_reference(self, native_ref, kind):
|
|
391
|
+
if kind == "run":
|
|
392
|
+
return isinstance(native_ref, _Run) and native_ref.submission.owner is self._owner
|
|
393
|
+
if kind == "grading":
|
|
394
|
+
return isinstance(native_ref, _CompletedGrading) and native_ref.pending.run.submission.owner is self._owner
|
|
395
|
+
return False
|
|
396
|
+
|
|
397
|
+
def reference(self, native_ref, kind):
|
|
398
|
+
if not self.accepts_reference(native_ref, kind):
|
|
399
|
+
raise WorkflowContractError("native hosted reference belongs to another backend or is pending")
|
|
400
|
+
if kind == "run":
|
|
401
|
+
record = self._run(native_ref)
|
|
402
|
+
return ReferenceBinding(native_ref.handle.id, record["status"], record.get("trust_tier", "unverified"),
|
|
403
|
+
run_id=native_ref.handle.id,
|
|
404
|
+
metadata={"submission_id": native_ref.submission.handle.id})
|
|
405
|
+
pending = native_ref.pending
|
|
406
|
+
pending.handle.refresh()
|
|
407
|
+
fresh = self.grading_snapshot(pending)
|
|
408
|
+
if not fresh.complete or contract_digest(fresh.completed_ref.payload) != contract_digest(native_ref.payload):
|
|
409
|
+
raise WorkflowContractError("saved hosted report changed or is no longer available")
|
|
410
|
+
return fresh.reference
|
|
411
|
+
|
|
412
|
+
def report(self, native_grading):
|
|
413
|
+
self.reference(native_grading, "grading")
|
|
414
|
+
return _Projection(native_grading.payload)
|
|
415
|
+
|
|
416
|
+
def assessments(self, native_grading, case_id=None):
|
|
417
|
+
self.reference(native_grading, "grading")
|
|
418
|
+
raise WorkflowContractError("typed hosted case assessments are not exposed; inspect the native report")
|
|
419
|
+
|
|
420
|
+
def compare(self, baseline_grading, candidate_grading):
|
|
421
|
+
a, b = self.reference(baseline_grading, "grading"), self.reference(candidate_grading, "grading")
|
|
422
|
+
handle = self.client.compare_saved(a.record_id, b.record_id)
|
|
423
|
+
return HostedComparison(self, handle, a.record_id, b.record_id)
|
|
424
|
+
|
|
425
|
+
|
|
426
|
+
def build_hosted_registry(client: Client, *, suite_ids: tuple[str, ...] = (SUITE_ID,)) -> WorkflowRegistry:
|
|
427
|
+
"""Enable selected native suites in a separate, explicit hosted registry.
|
|
428
|
+
|
|
429
|
+
The same suite ID can remain bound to the local backend in another registry.
|
|
430
|
+
This performs no HTTP request and does not grant hosted execution permission.
|
|
431
|
+
"""
|
|
432
|
+
backend = HostedWorkflowBackend(client, suite_ids=suite_ids)
|
|
433
|
+
registry = WorkflowRegistry()
|
|
434
|
+
registry.register_backend(backend)
|
|
435
|
+
for suite_id in suite_ids:
|
|
436
|
+
suite = backend.describe(suite_id)
|
|
437
|
+
native = suite.metadata["native_definition"]
|
|
438
|
+
for identifier, metric in suite.metadata["native_metric_mapping"].items():
|
|
439
|
+
registry.register_criterion(Criterion(identifier, native["version"], metric,
|
|
440
|
+
configuration={"native_metric_mapping": True}))
|
|
441
|
+
registry.register_metric(MetricSpec(identifier, native["implementation"]["metrics"],
|
|
442
|
+
"ms" if metric == "runner_latency_ms" else "ratio",
|
|
443
|
+
"lower" if metric in ("runner_latency_ms", "candidate_failure_rate") else "higher",
|
|
444
|
+
metadata={"native_metric_id": metric, "native_report_layout": "candidate/baseline lanes",
|
|
445
|
+
"aggregation": "server_owned", "comparison": "server_owned"}))
|
|
446
|
+
registry.register_evaluator(EvaluatorDefinition(suite_id + "/grader", native["version"],
|
|
447
|
+
native["implementation"]["grader"], criterion_ids=suite.criterion_ids,
|
|
448
|
+
metric_ids=suite.metric_ids, required_capabilities=("sphynx.async-grading.v1",),
|
|
449
|
+
configuration={"execution": "server_owned", "case_assessments": "not_exposed"}))
|
|
450
|
+
registry.register_suite(suite)
|
|
451
|
+
return registry
|
sphynx/http_transport.py
ADDED
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
"""HTTP opening policy for SDK requests that may carry credentials."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import ipaddress
|
|
6
|
+
import urllib.request
|
|
7
|
+
from urllib.parse import urlsplit
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class NoRedirectHandler(urllib.request.HTTPRedirectHandler):
|
|
11
|
+
"""Reject redirects so urllib never forwards credentials to another URL."""
|
|
12
|
+
|
|
13
|
+
def redirect_request(self, request, fp, code, msg, headers, newurl):
|
|
14
|
+
return None
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def validate_private_connect_proxy(proxy_url: str) -> str:
|
|
18
|
+
"""Accept only an explicit RFC1918 IPv4 CONNECT relay on port 3128."""
|
|
19
|
+
try:
|
|
20
|
+
if not isinstance(proxy_url, str):
|
|
21
|
+
raise ValueError()
|
|
22
|
+
parsed = urlsplit(proxy_url)
|
|
23
|
+
address = ipaddress.IPv4Address(parsed.hostname)
|
|
24
|
+
private_ranges = (ipaddress.IPv4Network("10.0.0.0/8"),
|
|
25
|
+
ipaddress.IPv4Network("172.16.0.0/12"),
|
|
26
|
+
ipaddress.IPv4Network("192.168.0.0/16"))
|
|
27
|
+
canonical = f"http://{address}:3128"
|
|
28
|
+
if (proxy_url != canonical or parsed.scheme != "http" or parsed.port != 3128
|
|
29
|
+
or parsed.username or parsed.password or parsed.path or parsed.query or parsed.fragment
|
|
30
|
+
or not any(address in network for network in private_ranges)):
|
|
31
|
+
raise ValueError()
|
|
32
|
+
return canonical
|
|
33
|
+
except (ValueError, TypeError, AttributeError):
|
|
34
|
+
raise ValueError("CONNECT proxy must be an RFC1918 IPv4 HTTP origin on port 3128") from None
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _https_origin(value: str, *, origin_only: bool = False):
|
|
38
|
+
try:
|
|
39
|
+
parsed = urlsplit(value)
|
|
40
|
+
if (parsed.scheme != "https" or not parsed.hostname or parsed.username or parsed.password
|
|
41
|
+
or parsed.fragment or origin_only and (parsed.path not in ("", "/") or parsed.query)):
|
|
42
|
+
raise ValueError()
|
|
43
|
+
return parsed.hostname.casefold(), parsed.port or 443
|
|
44
|
+
except (ValueError, TypeError, AttributeError):
|
|
45
|
+
raise ValueError("CONNECT proxy target must be the configured HTTPS origin") from None
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class _OriginConnectProxyHandler(urllib.request.ProxyHandler):
|
|
49
|
+
"""Explicit per-opener relay, confined to one TLS origin, ignoring NO_PROXY."""
|
|
50
|
+
|
|
51
|
+
def __init__(self, proxy_url: str, origin: str):
|
|
52
|
+
self.proxy_url = validate_private_connect_proxy(proxy_url)
|
|
53
|
+
self.origin = _https_origin(origin, origin_only=True)
|
|
54
|
+
self.authority = urlsplit(self.proxy_url).netloc
|
|
55
|
+
super().__init__({"https": self.proxy_url})
|
|
56
|
+
|
|
57
|
+
def proxy_open(self, request, proxy, type):
|
|
58
|
+
if (type != "https" or request.type != "https"
|
|
59
|
+
or _https_origin(request.full_url) != self.origin):
|
|
60
|
+
raise ValueError("CONNECT proxy request is outside its configured HTTPS origin")
|
|
61
|
+
if request._tunnel_host is None:
|
|
62
|
+
request.set_proxy(self.authority, "http")
|
|
63
|
+
elif (request.host != self.authority
|
|
64
|
+
or request._tunnel_host != urlsplit(request.full_url).netloc):
|
|
65
|
+
raise ValueError("CONNECT proxy request has an unexpected tunnel destination")
|
|
66
|
+
# HTTPSHandler establishes CONNECT, then validates TLS for _tunnel_host.
|
|
67
|
+
# Never send the origin Authorization header as a proxy authorization.
|
|
68
|
+
return None
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def open_no_redirect(request: urllib.request.Request, *, timeout: float,
|
|
72
|
+
proxy_url: str | None = None, proxy_origin: str | None = None):
|
|
73
|
+
"""Open without redirects; an optional private relay is bound to one origin.
|
|
74
|
+
|
|
75
|
+
No process-wide opener/proxy setting is changed. The normal HTTPS handler
|
|
76
|
+
retains certificate and hostname verification for the origin behind CONNECT.
|
|
77
|
+
"""
|
|
78
|
+
handlers = [NoRedirectHandler()]
|
|
79
|
+
if proxy_url is not None:
|
|
80
|
+
handler = _OriginConnectProxyHandler(proxy_url, proxy_origin)
|
|
81
|
+
if request.type != "https" or _https_origin(request.full_url) != handler.origin:
|
|
82
|
+
raise ValueError("CONNECT proxy request is outside its configured HTTPS origin")
|
|
83
|
+
handlers.append(handler)
|
|
84
|
+
elif proxy_origin is not None:
|
|
85
|
+
raise ValueError("CONNECT proxy origin requires an explicit proxy")
|
|
86
|
+
return urllib.request.build_opener(*handlers).open(request, timeout=timeout)
|