sable-sphynx 0.7.0.dev0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (174) hide show
  1. sable_sphynx-0.7.0.dev0.dist-info/METADATA +152 -0
  2. sable_sphynx-0.7.0.dev0.dist-info/RECORD +174 -0
  3. sable_sphynx-0.7.0.dev0.dist-info/WHEEL +5 -0
  4. sable_sphynx-0.7.0.dev0.dist-info/top_level.txt +1 -0
  5. sphynx/__init__.py +469 -0
  6. sphynx/__init__.pyi +376 -0
  7. sphynx/_analytical_client.py +77 -0
  8. sphynx/_analytical_values.py +252 -0
  9. sphynx/_audit_client.py +173 -0
  10. sphynx/_comparison_adapters.py +113 -0
  11. sphynx/_componentized_full_rag.py +34 -0
  12. sphynx/_composed_pipeline.py +109 -0
  13. sphynx/_full_rag_grading.py +218 -0
  14. sphynx/_full_rag_trace.py +221 -0
  15. sphynx/_local_eval.py +268 -0
  16. sphynx/_pipeline_core.py +243 -0
  17. sphynx/_pr_client.py +237 -0
  18. sphynx/_rag_diagnostics_calibration.py +285 -0
  19. sphynx/_rag_diagnostics_catalog.py +39 -0
  20. sphynx/_rag_diagnostics_common.py +26 -0
  21. sphynx/_rag_diagnostics_contracts.py +186 -0
  22. sphynx/_rag_diagnostics_engine.py +243 -0
  23. sphynx/_rag_diagnostics_importers.py +242 -0
  24. sphynx/_rag_diagnostics_ranking.py +81 -0
  25. sphynx/_rag_diagnostics_semantic.py +325 -0
  26. sphynx/_rag_ranking.py +149 -0
  27. sphynx/_speech_benchmark_scoring.py +140 -0
  28. sphynx/_speech_benchmark_sources.py +275 -0
  29. sphynx/_speech_complete_scoring.py +218 -0
  30. sphynx/_speech_complete_source.py +118 -0
  31. sphynx/_speech_kido.py +294 -0
  32. sphynx/_speech_scoring.py +77 -0
  33. sphynx/_track_capture.py +213 -0
  34. sphynx/analytical_views.py +515 -0
  35. sphynx/assessment_contracts.py +378 -0
  36. sphynx/assessment_selection.py +57 -0
  37. sphynx/audit_access.py +245 -0
  38. sphynx/audit_contracts.py +204 -0
  39. sphynx/audit_requests.py +180 -0
  40. sphynx/authoring.py +188 -0
  41. sphynx/benchmarks.py +568 -0
  42. sphynx/candidates.py +62 -0
  43. sphynx/catalog.py +272 -0
  44. sphynx/chunking.py +372 -0
  45. sphynx/clickhouse_reader.py +169 -0
  46. sphynx/client.py +486 -0
  47. sphynx/comparison_core.py +149 -0
  48. sphynx/comparison_views.py +223 -0
  49. sphynx/components.py +1661 -0
  50. sphynx/contracts.py +514 -0
  51. sphynx/customer_rerank.py +299 -0
  52. sphynx/customer_retrieval.py +146 -0
  53. sphynx/data/benchmark-releases.v1.json +164 -0
  54. sphynx/data/public-benchmark-release.v1.json +354 -0
  55. sphynx/data/rag-diagnostics-synthetic.v1.json +6982 -0
  56. sphynx/data/rag-paper-references.v1.json +176 -0
  57. sphynx/data/rag-paper-schemas.v1.json +1069 -0
  58. sphynx/data/rag-rerank-v1.json +507 -0
  59. sphynx/dataset_catalog_data.py +1214 -0
  60. sphynx/dataset_readiness.py +138 -0
  61. sphynx/dataset_v3.py +999 -0
  62. sphynx/demo.py +130 -0
  63. sphynx/demo_candidates/__init__.py +1 -0
  64. sphynx/demo_candidates/dense.py +11 -0
  65. sphynx/demo_candidates/lexical.py +21 -0
  66. sphynx/diagnostics.py +207 -0
  67. sphynx/documentation.py +737 -0
  68. sphynx/documentation_candidate.py +939 -0
  69. sphynx/evals.py +538 -0
  70. sphynx/evaluation_protocols.py +124 -0
  71. sphynx/evaluator_bindings.py +370 -0
  72. sphynx/evidence.py +352 -0
  73. sphynx/execution_capabilities.py +109 -0
  74. sphynx/fast_search.py +285 -0
  75. sphynx/full_rag.py +341 -0
  76. sphynx/hosted/__init__.py +18 -0
  77. sphynx/hosted/benchmark_intake.py +218 -0
  78. sphynx/hosted/contracts.py +361 -0
  79. sphynx/hosted/model_admission.py +130 -0
  80. sphynx/hosted/model_artifacts.py +298 -0
  81. sphynx/hosted/model_contracts.py +63 -0
  82. sphynx/hosted/model_gcs.py +181 -0
  83. sphynx/hosted/model_history.py +84 -0
  84. sphynx/hosted/model_session.py +233 -0
  85. sphynx/hosted/public_suite.py +74 -0
  86. sphynx/hosted/registry.py +97 -0
  87. sphynx/hosted/revisions.py +126 -0
  88. sphynx/hosted/sandbox.py +105 -0
  89. sphynx/hosted/submission.py +287 -0
  90. sphynx/hosted_workflow.py +451 -0
  91. sphynx/http_transport.py +86 -0
  92. sphynx/integrated_rag.py +352 -0
  93. sphynx/integrated_workflow.py +552 -0
  94. sphynx/integration_contracts.py +474 -0
  95. sphynx/invocation.py +791 -0
  96. sphynx/invocation_candidates.py +392 -0
  97. sphynx/invocation_catalog_data.py +40 -0
  98. sphynx/learned_catalog_data.py +30 -0
  99. sphynx/learned_rerank.py +267 -0
  100. sphynx/legacy_full_rag.py +67 -0
  101. sphynx/lifecycle.py +885 -0
  102. sphynx/lifecycle_workflow.py +490 -0
  103. sphynx/metric_views.py +474 -0
  104. sphynx/metrics.py +282 -0
  105. sphynx/model_adapters.py +438 -0
  106. sphynx/model_catalog_data.py +30 -0
  107. sphynx/official_benchmarks.py +385 -0
  108. sphynx/packs.py +258 -0
  109. sphynx/pipeline.py +313 -0
  110. sphynx/pipeline_workflow.py +551 -0
  111. sphynx/pr_evaluations.py +560 -0
  112. sphynx/progress.py +89 -0
  113. sphynx/public_bundle.py +173 -0
  114. sphynx/public_eval.py +62 -0
  115. sphynx/public_rerank.py +355 -0
  116. sphynx/py.typed +0 -0
  117. sphynx/pytest_plugin.py +62 -0
  118. sphynx/question_robustness.py +437 -0
  119. sphynx/rag.py +329 -0
  120. sphynx/rag_diagnostics.py +309 -0
  121. sphynx/rag_papers/__init__.py +29 -0
  122. sphynx/rag_papers/__main__.py +45 -0
  123. sphynx/rag_papers/candidates.py +19 -0
  124. sphynx/rag_papers/contracts.py +282 -0
  125. sphynx/rag_papers/datasets.py +291 -0
  126. sphynx/rag_papers/experiments.py +116 -0
  127. sphynx/rag_papers/judges.py +308 -0
  128. sphynx/rag_papers/registry.py +33 -0
  129. sphynx/rag_papers/runner.py +176 -0
  130. sphynx/rag_papers/sable.py +135 -0
  131. sphynx/rag_papers/schemas.py +12 -0
  132. sphynx/rag_papers/scoring.py +249 -0
  133. sphynx/rag_papers/semantic.py +288 -0
  134. sphynx/rag_papers/trec.py +80 -0
  135. sphynx/recorded_scores.py +352 -0
  136. sphynx/records.py +124 -0
  137. sphynx/reflex_traces.py +193 -0
  138. sphynx/report_rendering.py +71 -0
  139. sphynx/rerank_family_contracts.py +150 -0
  140. sphynx/rerank_workflow.py +424 -0
  141. sphynx/research_candidate.py +190 -0
  142. sphynx/results.py +852 -0
  143. sphynx/retrieval.py +750 -0
  144. sphynx/retrieval_candidates.py +295 -0
  145. sphynx/retrieval_catalog_data.py +19 -0
  146. sphynx/retrieval_recordings.py +125 -0
  147. sphynx/reviewer_audit.py +266 -0
  148. sphynx/routing_policies.py +375 -0
  149. sphynx/runner.py +544 -0
  150. sphynx/sable_rerank.py +489 -0
  151. sphynx/score_reliability.py +393 -0
  152. sphynx/speech_benchmarks.py +204 -0
  153. sphynx/speech_campaigns.py +378 -0
  154. sphynx/speech_complete.py +236 -0
  155. sphynx/speech_contracts.py +225 -0
  156. sphynx/speech_evidence.py +429 -0
  157. sphynx/speech_listening.py +115 -0
  158. sphynx/speech_multilingual.py +71 -0
  159. sphynx/speech_tts.py +479 -0
  160. sphynx/sql/001_sync_v1.sql +30 -0
  161. sphynx/stage_contracts.py +199 -0
  162. sphynx/statistical_catalog_data.py +53 -0
  163. sphynx/statistical_rerank.py +232 -0
  164. sphynx/sync.py +797 -0
  165. sphynx/workbench.py +378 -0
  166. sphynx/workbench_catalog_data.py +13 -0
  167. sphynx/workbench_tracks.py +78 -0
  168. sphynx/workflow_async.py +197 -0
  169. sphynx/workflow_contracts.py +594 -0
  170. sphynx/workflow_decisions.py +211 -0
  171. sphynx/workflow_defaults.py +16 -0
  172. sphynx/workflow_extensions.py +212 -0
  173. sphynx/workflow_registry.py +513 -0
  174. sphynx/workflows.py +792 -0
@@ -0,0 +1,451 @@
1
+ """Explicit adapter for native exact-revision hosted PR submissions.
2
+
3
+ No service is contacted during construction, registration or resolution. Local
4
+ admission validates a request, not remote permission or executable source bytes.
5
+ The configured owner API must authorize and verify its records. This adapter
6
+ checks their linkage and preserves native trust; it does not supply isolation,
7
+ read protected cases, or promote historical service notes into live authority.
8
+ """
9
+ from __future__ import annotations
10
+
11
+ import json
12
+ import re
13
+ from dataclasses import dataclass, field
14
+ from typing import Any, Mapping
15
+
16
+ from .client import Client, Experiment, HostedClientError, HostedEvaluation, HostedRun, _retry_key
17
+ from .evals import describe_eval, resolve_eval
18
+ from .hosted.registry import BASELINE_ID, PUBLIC_SUITE_ID, REGISTERED_MAIN_BASELINE_ID, SUITE_ID
19
+ from .hosted.revisions import require_commit_sha
20
+ from .hosted.submission import pr_identity
21
+ from .workflow_async import GradingSnapshot, SubmissionError, SubmissionIntent, SubmissionSnapshot
22
+ from .workflow_contracts import (
23
+ AdmissionBinding, BackendDescriptor, Criterion, EvaluatorDefinition, MetricSpec,
24
+ PlanBinding, ReferenceBinding, ResolvedBinding, SuiteDefinition,
25
+ WorkflowContractError, _object, _text, _thaw, contract_digest,
26
+ )
27
+ from .workflow_registry import WorkflowRegistry
28
+
29
+ HOSTED_BACKEND_ID = "sphynx.hosted.pr"
30
+ _SUITES = (SUITE_ID, PUBLIC_SUITE_ID)
31
+ _METHOD = re.compile(r"[A-Za-z0-9][A-Za-z0-9._:-]{0,127}\Z")
32
+ _RESULTS = {SUITE_ID: "sphynx.hosted.result.v1", PUBLIC_SUITE_ID: "sphynx.hosted.public-rerank.result.v1"}
33
+
34
+
35
+ @dataclass(frozen=True)
36
+ class PullRequestCandidate:
37
+ """Strict PR request; legacy Client.submit still permits an omitted SHA guard."""
38
+
39
+ candidate_pr: str
40
+ expected_head_sha: str
41
+ candidate_method: str | None = None
42
+ baseline: str = BASELINE_ID
43
+ expected_baseline_sha: str | None = None
44
+
45
+ def __post_init__(self) -> None:
46
+ pr_identity(self.candidate_pr)
47
+ object.__setattr__(self, "expected_head_sha", require_commit_sha(self.expected_head_sha))
48
+ if self.candidate_method is not None and (type(self.candidate_method) is not str
49
+ or not _METHOD.fullmatch(self.candidate_method)):
50
+ raise WorkflowContractError("candidate_method must be a bounded ASCII method ID")
51
+ if self.baseline not in (BASELINE_ID, REGISTERED_MAIN_BASELINE_ID):
52
+ raise WorkflowContractError("baseline is not supported by the native hosted adapter")
53
+ if self.expected_baseline_sha is not None:
54
+ if self.baseline != REGISTERED_MAIN_BASELINE_ID:
55
+ raise WorkflowContractError("expected_baseline_sha requires the registered main baseline")
56
+ object.__setattr__(self, "expected_baseline_sha", require_commit_sha(self.expected_baseline_sha))
57
+
58
+ def to_dict(self) -> dict[str, Any]:
59
+ result = {"candidate_pr": self.candidate_pr, "expected_head_sha": self.expected_head_sha,
60
+ "baseline": self.baseline}
61
+ if self.candidate_method is not None:
62
+ result["candidate_method"] = self.candidate_method
63
+ if self.expected_baseline_sha is not None:
64
+ result["expected_baseline_sha"] = self.expected_baseline_sha
65
+ return result
66
+
67
+
68
+ @dataclass(frozen=True)
69
+ class _Submission:
70
+ handle: Experiment = field(repr=False, compare=False)
71
+ request_identity: Mapping[str, Any]
72
+ idempotency_key: str
73
+ owner: Any = field(repr=False, compare=False)
74
+
75
+
76
+ @dataclass(frozen=True)
77
+ class _Run:
78
+ handle: HostedRun = field(repr=False, compare=False)
79
+ submission: _Submission = field(repr=False, compare=False)
80
+
81
+
82
+ @dataclass(frozen=True)
83
+ class _Grading:
84
+ handle: HostedEvaluation = field(repr=False, compare=False)
85
+ run: _Run = field(repr=False, compare=False)
86
+ idempotency_key: str
87
+ request_identity: Mapping[str, Any]
88
+
89
+
90
+ @dataclass(frozen=True)
91
+ class _CompletedGrading:
92
+ pending: _Grading = field(repr=False, compare=False)
93
+ payload: Mapping[str, Any] = field(repr=False)
94
+
95
+
96
+ @dataclass(frozen=True)
97
+ class _Projection:
98
+ payload: Mapping[str, Any] = field(repr=False)
99
+
100
+ def to_dict(self) -> dict[str, Any]:
101
+ return _thaw(self.payload)
102
+
103
+ def to_json(self) -> str:
104
+ return json.dumps(self.to_dict(), ensure_ascii=False, allow_nan=False, sort_keys=True)
105
+
106
+ def to_markdown(self) -> str:
107
+ summary = self.payload.get("summary")
108
+ if not isinstance(summary, str):
109
+ raise WorkflowContractError("native hosted report has no readable summary")
110
+ return summary
111
+
112
+
113
+ class HostedComparison:
114
+ """Native saved comparison handle with explicit result projection.
115
+
116
+ Pending retrieval remains pending; obtaining this handle never runs or grades
117
+ candidates. The native comparison ID remains available after a read error.
118
+ """
119
+
120
+ def __init__(self, backend, native_ref, evaluation_a, evaluation_b):
121
+ self._backend, self.native_ref = backend, native_ref
122
+ self.evaluation_a, self.evaluation_b = evaluation_a, evaluation_b
123
+
124
+ @property
125
+ def status(self):
126
+ return self.native_ref.status
127
+
128
+ def refresh(self):
129
+ self._backend._origin()
130
+ return HostedComparison(self._backend, self._backend.client.get_comparison(self.native_ref.id),
131
+ self.evaluation_a, self.evaluation_b)
132
+
133
+ def to_dict(self):
134
+ self._backend._origin()
135
+ payload = self.native_ref.result().to_dict()
136
+ if payload.get("evaluation_a") != self.evaluation_a or payload.get("evaluation_b") != self.evaluation_b:
137
+ raise WorkflowContractError("hosted comparison refers to different grading passes")
138
+ return payload
139
+
140
+ def to_json(self):
141
+ return json.dumps(self.to_dict(), ensure_ascii=False, allow_nan=False, sort_keys=True)
142
+
143
+ def to_markdown(self):
144
+ return _Projection(_object(self.to_dict(), "comparison")).to_markdown()
145
+
146
+
147
+ class HostedWorkflowBackend:
148
+ """Operator-enabled native hosted adapter; no service checkout is bundled."""
149
+
150
+ def __init__(self, client: Client, *, suite_ids: tuple[str, ...] = (SUITE_ID,)):
151
+ if not isinstance(client, Client):
152
+ raise WorkflowContractError("hosted backend requires an explicitly configured Client")
153
+ if not isinstance(suite_ids, tuple) or not suite_ids or len(set(suite_ids)) != len(suite_ids):
154
+ raise WorkflowContractError("suite_ids must be a nonempty unique tuple")
155
+ if any(identifier not in _SUITES for identifier in suite_ids):
156
+ raise WorkflowContractError("no native hosted adapter for the selected suite")
157
+ self.client, self._owner = client, object()
158
+ self._descriptor = BackendDescriptor(
159
+ HOSTED_BACKEND_ID, "v1", "sphynx.hosted.pr-adapter.v1",
160
+ capabilities=("sphynx.async-submission.v1", "sphynx.async-grading.v1",
161
+ "sphynx.saved-assessment.v1", "sphynx.native-report.v1",
162
+ "sphynx.paired-comparison.v1"),
163
+ supported_scopes=("component",), supported_modes=("hosted",), suite_ids=suite_ids,
164
+ metadata={"api_origin": client.api_url, "remote_admission": "server_owned",
165
+ "case_access": "not_exposed", "case_assessments": "not_exposed",
166
+ "live_service_verified": False},
167
+ )
168
+
169
+ @property
170
+ def descriptor(self):
171
+ return self._descriptor
172
+
173
+ def describe(self, suite_id):
174
+ if suite_id not in self.descriptor.suite_ids:
175
+ raise WorkflowContractError("suite is not enabled in this hosted backend")
176
+ native = describe_eval(suite_id)
177
+ ids = tuple(f"{suite_id}/{metric}" for metric in native["metric_ids"])
178
+ return SuiteDefinition(suite_id, native["version"], self.descriptor.backend_id,
179
+ native["title"], criterion_ids=ids, metric_ids=ids, evaluator_ids=(suite_id + "/grader",),
180
+ required_capabilities=("sphynx.async-submission.v1", "sphynx.async-grading.v1"),
181
+ metadata={"native_definition": native, "native_metric_mapping": dict(zip(ids, native["metric_ids"])),
182
+ "native_report_layout": "metrics.candidate and metrics.baseline",
183
+ "remote_admission": "pending_server_submission", "case_access": "not_exposed"})
184
+
185
+ def cases(self, suite):
186
+ raise WorkflowContractError("hosted case access is not exposed by this API")
187
+
188
+ def _plan(self, plan):
189
+ if not isinstance(plan, PlanBinding) or not isinstance(plan.candidate, PullRequestCandidate):
190
+ raise WorkflowContractError("hosted PR plans require PullRequestCandidate")
191
+ if plan.suite.definition_digest != self.describe(plan.suite.suite_id).definition_digest:
192
+ raise WorkflowContractError("hosted suite definition changed")
193
+ if plan.scope != "component" or plan.mode != "hosted" or plan.runtime:
194
+ raise WorkflowContractError("hosted adapter supports component/hosted with native parameter runtime only")
195
+ self._origin()
196
+
197
+ def _origin(self):
198
+ if self.client.api_url != self.descriptor.metadata["api_origin"]:
199
+ raise WorkflowContractError("configured hosted origin changed")
200
+
201
+ def resolve(self, plan):
202
+ self._plan(plan)
203
+ invocation = resolve_eval(plan.suite.suite_id, plan.parameters)
204
+ request = {**plan.candidate.to_dict(), "suite": plan.suite.suite_id,
205
+ "parameters": _thaw(invocation.parameters)}
206
+ return ResolvedBinding(identity={"backend": self.descriptor.to_dict(),
207
+ "suite_id": plan.suite.suite_id, "suite_digest": plan.suite.definition_digest,
208
+ "scope": plan.scope, "mode": plan.mode, "request": request,
209
+ "native_invocation": invocation.to_dict()}, native_ref=invocation)
210
+
211
+ def _resolution(self, plan, resolved):
212
+ if not isinstance(resolved, ResolvedBinding) or self.resolve(plan).resolution_digest != resolved.resolution_digest:
213
+ raise WorkflowContractError("hosted request identity changed; resolve again")
214
+
215
+ def admit(self, plan, resolved):
216
+ self._resolution(plan, resolved)
217
+ return AdmissionBinding(resolved.resolution_digest,
218
+ {"request_digest": contract_digest(resolved.identity),
219
+ "remote_admission": "pending_server_submission", "api_origin": self.client.api_url},
220
+ "unverified", False, False)
221
+
222
+ def recheck(self, plan, resolved, admission):
223
+ expected = self.admit(plan, resolved)
224
+ if not isinstance(admission, AdmissionBinding) or admission.to_dict() != expected.to_dict():
225
+ raise WorkflowContractError("hosted admission or request identity changed")
226
+
227
+ def execute(self, plan, resolved, admission, output_dir):
228
+ raise WorkflowContractError("hosted execution is asynchronous; use Plan.submit()")
229
+
230
+ def assess(self, native_run):
231
+ raise WorkflowContractError("hosted grading is asynchronous; use Run.start_grading()")
232
+
233
+ def submit(self, plan, resolved, admission, idempotency_key):
234
+ self.recheck(plan, resolved, admission)
235
+ key = _retry_key(idempotency_key)
236
+ intent = SubmissionIntent(key, resolved.identity)
237
+ try:
238
+ handle = self.client.submit(**_thaw(resolved.identity["request"]), idempotency_key=key)
239
+ except HostedClientError as exc:
240
+ raise SubmissionError("hosted submission did not return a handle; retain this intent for recovery",
241
+ intent=intent, code=exc.code) from None
242
+ ref = _Submission(handle, _object(resolved.identity, "request_identity"), key, self._owner)
243
+ self._submission(ref)
244
+ return ref
245
+
246
+ def _submission(self, ref):
247
+ self._origin()
248
+ if not isinstance(ref, _Submission) or ref.owner is not self._owner:
249
+ raise WorkflowContractError("submission belongs to another backend binding")
250
+ data, request = ref.handle.to_dict(), ref.request_identity["request"]
251
+ if data.get("id") != ref.handle.id:
252
+ raise WorkflowContractError("native submission identity changed")
253
+ if data.get("head_sha") is not None and data["head_sha"] != request["expected_head_sha"]:
254
+ raise WorkflowContractError("hosted submission head differs from expected revision")
255
+ if "request" in data and contract_digest(data["request"]) != contract_digest(request):
256
+ raise WorkflowContractError("server submission request differs from the selected request")
257
+ _text(data.get("status"), "native submission status")
258
+ return data
259
+
260
+ def refresh_submission(self, native_ref):
261
+ self._submission(native_ref)
262
+ native_ref.handle.refresh()
263
+ self._submission(native_ref)
264
+ return native_ref
265
+
266
+ def _manifest(self, manifest, identity):
267
+ request = identity["request"]
268
+ if not isinstance(manifest, Mapping):
269
+ raise WorkflowContractError("hosted record has no pinned manifest")
270
+ if (manifest.get("head_sha") != request["expected_head_sha"]
271
+ or manifest.get("candidate_pr") != request["candidate_pr"]
272
+ or not isinstance(manifest.get("suite"), Mapping)
273
+ or manifest["suite"].get("id") != request["suite"]
274
+ or not isinstance(manifest.get("baseline"), Mapping)
275
+ or manifest["baseline"].get("id") != request["baseline"]
276
+ or manifest.get("candidate_method") != request.get("candidate_method")):
277
+ raise WorkflowContractError("hosted manifest differs from pinned head, method, suite or baseline")
278
+ family = manifest.get("candidate_family")
279
+ if family is not None and (not isinstance(family, Mapping)
280
+ or family.get("method") != request.get("candidate_method")):
281
+ raise WorkflowContractError("hosted family selection differs from the requested method")
282
+ if request.get("expected_baseline_sha") is not None:
283
+ baseline = manifest["baseline"].get("registered_component", {})
284
+ source = baseline.get("source_revision", {}) if isinstance(baseline, Mapping) else {}
285
+ if source.get("head_sha") != request["expected_baseline_sha"]:
286
+ raise WorkflowContractError("hosted baseline differs from its expected revision")
287
+
288
+ def _run(self, ref):
289
+ if not isinstance(ref, _Run):
290
+ raise WorkflowContractError("expected a completed native hosted run")
291
+ self._submission(ref.submission)
292
+ current = self.client.get_run(ref.handle.id)
293
+ record = current.to_dict()
294
+ if record.get("status") not in ("completed", "captured"):
295
+ raise WorkflowContractError("native hosted capture is not complete")
296
+ if record.get("experiment_id") != ref.submission.handle.id:
297
+ raise WorkflowContractError("native run belongs to another submission")
298
+ self._manifest(record.get("manifest"), ref.submission.request_identity)
299
+ return record
300
+
301
+ def submission_snapshot(self, native_ref):
302
+ data = self._submission(native_ref)
303
+ status = data["status"]
304
+ run = grading = None
305
+ if data.get("run_id") is not None:
306
+ handle = self.client.get_run(data["run_id"])
307
+ if handle.status in ("completed", "captured"):
308
+ run = _Run(handle, native_ref)
309
+ self._run(run)
310
+ if status == "completed":
311
+ if run is None or data.get("evaluation_id") is None:
312
+ raise WorkflowContractError("completed submission has no complete native run/grading linkage")
313
+ pending = _Grading(self.client.get_evaluation(data["evaluation_id"]), run,
314
+ native_ref.idempotency_key, native_ref.request_identity)
315
+ grading = self.grading_snapshot(pending).completed_ref
316
+ if grading is None:
317
+ raise WorkflowContractError("completed submission has a pending grading pass")
318
+ error = data.get("error") if isinstance(data.get("error"), dict) else {}
319
+ return SubmissionSnapshot(native_ref.handle.id, status, native_ref.idempotency_key,
320
+ native_ref.request_identity, native_ref, run_id=data.get("run_id"),
321
+ grading_id=data.get("evaluation_id"), native_run=run, native_grading=grading,
322
+ complete=status == "completed", failed=status == "failed",
323
+ error_code=error.get("code") if isinstance(error.get("code"), str) else None,
324
+ reason="hosted submission failed; inspect its owned native status" if status == "failed" else None,
325
+ trust_tier=grading.payload.get("trust_tier", "unverified") if grading else "unverified")
326
+
327
+ def start_grading(self, native_run, idempotency_key):
328
+ self._run(native_run)
329
+ key = _retry_key(idempotency_key)
330
+ identity = _object({"run_id": native_run.handle.id,
331
+ "submission_request_digest": contract_digest(native_run.submission.request_identity),
332
+ "operation": "saved_grading"}, "grading request")
333
+ try:
334
+ handle = self.client.grade(native_run.handle.id, idempotency_key=key)
335
+ except HostedClientError as exc:
336
+ raise SubmissionError("hosted grading did not return a handle; retain this intent for recovery",
337
+ intent=SubmissionIntent(key, identity), code=exc.code) from None
338
+ return _Grading(handle, native_run, key, identity)
339
+
340
+ def _grading(self, ref):
341
+ if not isinstance(ref, _Grading):
342
+ raise WorkflowContractError("expected a native pending grading handle")
343
+ self._run(ref.run)
344
+ data = ref.handle.to_dict()
345
+ if data.get("run_id") != ref.run.handle.id:
346
+ raise WorkflowContractError("hosted grading refers to another run")
347
+ if data.get("experiment_id") is not None and data["experiment_id"] != ref.run.submission.handle.id:
348
+ raise WorkflowContractError("hosted grading belongs to another submission")
349
+ _text(data.get("status"), "native grading status")
350
+ return data
351
+
352
+ def refresh_grading(self, native_ref):
353
+ self._grading(native_ref)
354
+ native_ref.handle.refresh()
355
+ self._grading(native_ref)
356
+ return native_ref
357
+
358
+ def grading_snapshot(self, native_ref):
359
+ data = self._grading(native_ref)
360
+ status, completed = data["status"], None
361
+ trust = "unverified"
362
+ if status == "completed":
363
+ payload = self.client._request("GET", f"/v1/evaluations/{native_ref.handle.id}/result")
364
+ request = native_ref.run.submission.request_identity["request"]
365
+ if (payload.get("schema_version") != _RESULTS[request["suite"]]
366
+ or payload.get("experiment_id") != native_ref.run.submission.handle.id):
367
+ raise WorkflowContractError("hosted report has an unsupported schema or submission identity")
368
+ for key, expected in (("run_id", native_ref.run.handle.id), ("evaluation_id", native_ref.handle.id)):
369
+ if key in payload and payload[key] != expected:
370
+ raise WorkflowContractError("hosted report has a different native record identity")
371
+ self._manifest(payload.get("manifest"), native_ref.run.submission.request_identity)
372
+ capture_manifest = self._run(native_ref.run)["manifest"]
373
+ fields = ("head_sha", "tree_sha", "candidate_pr", "suite", "baseline", "candidate_method",
374
+ "candidate_family", "configuration_digest", "source_bundle_digest")
375
+ if contract_digest({key: capture_manifest.get(key) for key in fields}) != contract_digest(
376
+ {key: payload["manifest"].get(key) for key in fields}):
377
+ raise WorkflowContractError("hosted report differs from the captured source/cohort identity")
378
+ _text(payload.get("summary"), "native report summary")
379
+ trust = payload.get("trust_tier", "unverified")
380
+ _text(trust, "native trust_tier")
381
+ completed = _CompletedGrading(native_ref, _object(payload, "native report"))
382
+ error = data.get("error") if isinstance(data.get("error"), dict) else {}
383
+ return GradingSnapshot(ReferenceBinding(native_ref.handle.id, status, trust,
384
+ run_id=native_ref.run.handle.id), native_ref.idempotency_key,
385
+ native_ref.request_identity, native_ref, completed_ref=completed,
386
+ complete=status == "completed", failed=status == "failed",
387
+ error_code=error.get("code") if isinstance(error.get("code"), str) else None,
388
+ reason="hosted grading failed; inspect its owned native status" if status == "failed" else None)
389
+
390
+ def accepts_reference(self, native_ref, kind):
391
+ if kind == "run":
392
+ return isinstance(native_ref, _Run) and native_ref.submission.owner is self._owner
393
+ if kind == "grading":
394
+ return isinstance(native_ref, _CompletedGrading) and native_ref.pending.run.submission.owner is self._owner
395
+ return False
396
+
397
+ def reference(self, native_ref, kind):
398
+ if not self.accepts_reference(native_ref, kind):
399
+ raise WorkflowContractError("native hosted reference belongs to another backend or is pending")
400
+ if kind == "run":
401
+ record = self._run(native_ref)
402
+ return ReferenceBinding(native_ref.handle.id, record["status"], record.get("trust_tier", "unverified"),
403
+ run_id=native_ref.handle.id,
404
+ metadata={"submission_id": native_ref.submission.handle.id})
405
+ pending = native_ref.pending
406
+ pending.handle.refresh()
407
+ fresh = self.grading_snapshot(pending)
408
+ if not fresh.complete or contract_digest(fresh.completed_ref.payload) != contract_digest(native_ref.payload):
409
+ raise WorkflowContractError("saved hosted report changed or is no longer available")
410
+ return fresh.reference
411
+
412
+ def report(self, native_grading):
413
+ self.reference(native_grading, "grading")
414
+ return _Projection(native_grading.payload)
415
+
416
+ def assessments(self, native_grading, case_id=None):
417
+ self.reference(native_grading, "grading")
418
+ raise WorkflowContractError("typed hosted case assessments are not exposed; inspect the native report")
419
+
420
+ def compare(self, baseline_grading, candidate_grading):
421
+ a, b = self.reference(baseline_grading, "grading"), self.reference(candidate_grading, "grading")
422
+ handle = self.client.compare_saved(a.record_id, b.record_id)
423
+ return HostedComparison(self, handle, a.record_id, b.record_id)
424
+
425
+
426
+ def build_hosted_registry(client: Client, *, suite_ids: tuple[str, ...] = (SUITE_ID,)) -> WorkflowRegistry:
427
+ """Enable selected native suites in a separate, explicit hosted registry.
428
+
429
+ The same suite ID can remain bound to the local backend in another registry.
430
+ This performs no HTTP request and does not grant hosted execution permission.
431
+ """
432
+ backend = HostedWorkflowBackend(client, suite_ids=suite_ids)
433
+ registry = WorkflowRegistry()
434
+ registry.register_backend(backend)
435
+ for suite_id in suite_ids:
436
+ suite = backend.describe(suite_id)
437
+ native = suite.metadata["native_definition"]
438
+ for identifier, metric in suite.metadata["native_metric_mapping"].items():
439
+ registry.register_criterion(Criterion(identifier, native["version"], metric,
440
+ configuration={"native_metric_mapping": True}))
441
+ registry.register_metric(MetricSpec(identifier, native["implementation"]["metrics"],
442
+ "ms" if metric == "runner_latency_ms" else "ratio",
443
+ "lower" if metric in ("runner_latency_ms", "candidate_failure_rate") else "higher",
444
+ metadata={"native_metric_id": metric, "native_report_layout": "candidate/baseline lanes",
445
+ "aggregation": "server_owned", "comparison": "server_owned"}))
446
+ registry.register_evaluator(EvaluatorDefinition(suite_id + "/grader", native["version"],
447
+ native["implementation"]["grader"], criterion_ids=suite.criterion_ids,
448
+ metric_ids=suite.metric_ids, required_capabilities=("sphynx.async-grading.v1",),
449
+ configuration={"execution": "server_owned", "case_assessments": "not_exposed"}))
450
+ registry.register_suite(suite)
451
+ return registry
@@ -0,0 +1,86 @@
1
+ """HTTP opening policy for SDK requests that may carry credentials."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import ipaddress
6
+ import urllib.request
7
+ from urllib.parse import urlsplit
8
+
9
+
10
+ class NoRedirectHandler(urllib.request.HTTPRedirectHandler):
11
+ """Reject redirects so urllib never forwards credentials to another URL."""
12
+
13
+ def redirect_request(self, request, fp, code, msg, headers, newurl):
14
+ return None
15
+
16
+
17
+ def validate_private_connect_proxy(proxy_url: str) -> str:
18
+ """Accept only an explicit RFC1918 IPv4 CONNECT relay on port 3128."""
19
+ try:
20
+ if not isinstance(proxy_url, str):
21
+ raise ValueError()
22
+ parsed = urlsplit(proxy_url)
23
+ address = ipaddress.IPv4Address(parsed.hostname)
24
+ private_ranges = (ipaddress.IPv4Network("10.0.0.0/8"),
25
+ ipaddress.IPv4Network("172.16.0.0/12"),
26
+ ipaddress.IPv4Network("192.168.0.0/16"))
27
+ canonical = f"http://{address}:3128"
28
+ if (proxy_url != canonical or parsed.scheme != "http" or parsed.port != 3128
29
+ or parsed.username or parsed.password or parsed.path or parsed.query or parsed.fragment
30
+ or not any(address in network for network in private_ranges)):
31
+ raise ValueError()
32
+ return canonical
33
+ except (ValueError, TypeError, AttributeError):
34
+ raise ValueError("CONNECT proxy must be an RFC1918 IPv4 HTTP origin on port 3128") from None
35
+
36
+
37
+ def _https_origin(value: str, *, origin_only: bool = False):
38
+ try:
39
+ parsed = urlsplit(value)
40
+ if (parsed.scheme != "https" or not parsed.hostname or parsed.username or parsed.password
41
+ or parsed.fragment or origin_only and (parsed.path not in ("", "/") or parsed.query)):
42
+ raise ValueError()
43
+ return parsed.hostname.casefold(), parsed.port or 443
44
+ except (ValueError, TypeError, AttributeError):
45
+ raise ValueError("CONNECT proxy target must be the configured HTTPS origin") from None
46
+
47
+
48
+ class _OriginConnectProxyHandler(urllib.request.ProxyHandler):
49
+ """Explicit per-opener relay, confined to one TLS origin, ignoring NO_PROXY."""
50
+
51
+ def __init__(self, proxy_url: str, origin: str):
52
+ self.proxy_url = validate_private_connect_proxy(proxy_url)
53
+ self.origin = _https_origin(origin, origin_only=True)
54
+ self.authority = urlsplit(self.proxy_url).netloc
55
+ super().__init__({"https": self.proxy_url})
56
+
57
+ def proxy_open(self, request, proxy, type):
58
+ if (type != "https" or request.type != "https"
59
+ or _https_origin(request.full_url) != self.origin):
60
+ raise ValueError("CONNECT proxy request is outside its configured HTTPS origin")
61
+ if request._tunnel_host is None:
62
+ request.set_proxy(self.authority, "http")
63
+ elif (request.host != self.authority
64
+ or request._tunnel_host != urlsplit(request.full_url).netloc):
65
+ raise ValueError("CONNECT proxy request has an unexpected tunnel destination")
66
+ # HTTPSHandler establishes CONNECT, then validates TLS for _tunnel_host.
67
+ # Never send the origin Authorization header as a proxy authorization.
68
+ return None
69
+
70
+
71
+ def open_no_redirect(request: urllib.request.Request, *, timeout: float,
72
+ proxy_url: str | None = None, proxy_origin: str | None = None):
73
+ """Open without redirects; an optional private relay is bound to one origin.
74
+
75
+ No process-wide opener/proxy setting is changed. The normal HTTPS handler
76
+ retains certificate and hostname verification for the origin behind CONNECT.
77
+ """
78
+ handlers = [NoRedirectHandler()]
79
+ if proxy_url is not None:
80
+ handler = _OriginConnectProxyHandler(proxy_url, proxy_origin)
81
+ if request.type != "https" or _https_origin(request.full_url) != handler.origin:
82
+ raise ValueError("CONNECT proxy request is outside its configured HTTPS origin")
83
+ handlers.append(handler)
84
+ elif proxy_origin is not None:
85
+ raise ValueError("CONNECT proxy origin requires an explicit proxy")
86
+ return urllib.request.build_opener(*handlers).open(request, timeout=timeout)