sable-sphynx 0.7.0.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sable_sphynx-0.7.0.dev0.dist-info/METADATA +152 -0
- sable_sphynx-0.7.0.dev0.dist-info/RECORD +174 -0
- sable_sphynx-0.7.0.dev0.dist-info/WHEEL +5 -0
- sable_sphynx-0.7.0.dev0.dist-info/top_level.txt +1 -0
- sphynx/__init__.py +469 -0
- sphynx/__init__.pyi +376 -0
- sphynx/_analytical_client.py +77 -0
- sphynx/_analytical_values.py +252 -0
- sphynx/_audit_client.py +173 -0
- sphynx/_comparison_adapters.py +113 -0
- sphynx/_componentized_full_rag.py +34 -0
- sphynx/_composed_pipeline.py +109 -0
- sphynx/_full_rag_grading.py +218 -0
- sphynx/_full_rag_trace.py +221 -0
- sphynx/_local_eval.py +268 -0
- sphynx/_pipeline_core.py +243 -0
- sphynx/_pr_client.py +237 -0
- sphynx/_rag_diagnostics_calibration.py +285 -0
- sphynx/_rag_diagnostics_catalog.py +39 -0
- sphynx/_rag_diagnostics_common.py +26 -0
- sphynx/_rag_diagnostics_contracts.py +186 -0
- sphynx/_rag_diagnostics_engine.py +243 -0
- sphynx/_rag_diagnostics_importers.py +242 -0
- sphynx/_rag_diagnostics_ranking.py +81 -0
- sphynx/_rag_diagnostics_semantic.py +325 -0
- sphynx/_rag_ranking.py +149 -0
- sphynx/_speech_benchmark_scoring.py +140 -0
- sphynx/_speech_benchmark_sources.py +275 -0
- sphynx/_speech_complete_scoring.py +218 -0
- sphynx/_speech_complete_source.py +118 -0
- sphynx/_speech_kido.py +294 -0
- sphynx/_speech_scoring.py +77 -0
- sphynx/_track_capture.py +213 -0
- sphynx/analytical_views.py +515 -0
- sphynx/assessment_contracts.py +378 -0
- sphynx/assessment_selection.py +57 -0
- sphynx/audit_access.py +245 -0
- sphynx/audit_contracts.py +204 -0
- sphynx/audit_requests.py +180 -0
- sphynx/authoring.py +188 -0
- sphynx/benchmarks.py +568 -0
- sphynx/candidates.py +62 -0
- sphynx/catalog.py +272 -0
- sphynx/chunking.py +372 -0
- sphynx/clickhouse_reader.py +169 -0
- sphynx/client.py +486 -0
- sphynx/comparison_core.py +149 -0
- sphynx/comparison_views.py +223 -0
- sphynx/components.py +1661 -0
- sphynx/contracts.py +514 -0
- sphynx/customer_rerank.py +299 -0
- sphynx/customer_retrieval.py +146 -0
- sphynx/data/benchmark-releases.v1.json +164 -0
- sphynx/data/public-benchmark-release.v1.json +354 -0
- sphynx/data/rag-diagnostics-synthetic.v1.json +6982 -0
- sphynx/data/rag-paper-references.v1.json +176 -0
- sphynx/data/rag-paper-schemas.v1.json +1069 -0
- sphynx/data/rag-rerank-v1.json +507 -0
- sphynx/dataset_catalog_data.py +1214 -0
- sphynx/dataset_readiness.py +138 -0
- sphynx/dataset_v3.py +999 -0
- sphynx/demo.py +130 -0
- sphynx/demo_candidates/__init__.py +1 -0
- sphynx/demo_candidates/dense.py +11 -0
- sphynx/demo_candidates/lexical.py +21 -0
- sphynx/diagnostics.py +207 -0
- sphynx/documentation.py +737 -0
- sphynx/documentation_candidate.py +939 -0
- sphynx/evals.py +538 -0
- sphynx/evaluation_protocols.py +124 -0
- sphynx/evaluator_bindings.py +370 -0
- sphynx/evidence.py +352 -0
- sphynx/execution_capabilities.py +109 -0
- sphynx/fast_search.py +285 -0
- sphynx/full_rag.py +341 -0
- sphynx/hosted/__init__.py +18 -0
- sphynx/hosted/benchmark_intake.py +218 -0
- sphynx/hosted/contracts.py +361 -0
- sphynx/hosted/model_admission.py +130 -0
- sphynx/hosted/model_artifacts.py +298 -0
- sphynx/hosted/model_contracts.py +63 -0
- sphynx/hosted/model_gcs.py +181 -0
- sphynx/hosted/model_history.py +84 -0
- sphynx/hosted/model_session.py +233 -0
- sphynx/hosted/public_suite.py +74 -0
- sphynx/hosted/registry.py +97 -0
- sphynx/hosted/revisions.py +126 -0
- sphynx/hosted/sandbox.py +105 -0
- sphynx/hosted/submission.py +287 -0
- sphynx/hosted_workflow.py +451 -0
- sphynx/http_transport.py +86 -0
- sphynx/integrated_rag.py +352 -0
- sphynx/integrated_workflow.py +552 -0
- sphynx/integration_contracts.py +474 -0
- sphynx/invocation.py +791 -0
- sphynx/invocation_candidates.py +392 -0
- sphynx/invocation_catalog_data.py +40 -0
- sphynx/learned_catalog_data.py +30 -0
- sphynx/learned_rerank.py +267 -0
- sphynx/legacy_full_rag.py +67 -0
- sphynx/lifecycle.py +885 -0
- sphynx/lifecycle_workflow.py +490 -0
- sphynx/metric_views.py +474 -0
- sphynx/metrics.py +282 -0
- sphynx/model_adapters.py +438 -0
- sphynx/model_catalog_data.py +30 -0
- sphynx/official_benchmarks.py +385 -0
- sphynx/packs.py +258 -0
- sphynx/pipeline.py +313 -0
- sphynx/pipeline_workflow.py +551 -0
- sphynx/pr_evaluations.py +560 -0
- sphynx/progress.py +89 -0
- sphynx/public_bundle.py +173 -0
- sphynx/public_eval.py +62 -0
- sphynx/public_rerank.py +355 -0
- sphynx/py.typed +0 -0
- sphynx/pytest_plugin.py +62 -0
- sphynx/question_robustness.py +437 -0
- sphynx/rag.py +329 -0
- sphynx/rag_diagnostics.py +309 -0
- sphynx/rag_papers/__init__.py +29 -0
- sphynx/rag_papers/__main__.py +45 -0
- sphynx/rag_papers/candidates.py +19 -0
- sphynx/rag_papers/contracts.py +282 -0
- sphynx/rag_papers/datasets.py +291 -0
- sphynx/rag_papers/experiments.py +116 -0
- sphynx/rag_papers/judges.py +308 -0
- sphynx/rag_papers/registry.py +33 -0
- sphynx/rag_papers/runner.py +176 -0
- sphynx/rag_papers/sable.py +135 -0
- sphynx/rag_papers/schemas.py +12 -0
- sphynx/rag_papers/scoring.py +249 -0
- sphynx/rag_papers/semantic.py +288 -0
- sphynx/rag_papers/trec.py +80 -0
- sphynx/recorded_scores.py +352 -0
- sphynx/records.py +124 -0
- sphynx/reflex_traces.py +193 -0
- sphynx/report_rendering.py +71 -0
- sphynx/rerank_family_contracts.py +150 -0
- sphynx/rerank_workflow.py +424 -0
- sphynx/research_candidate.py +190 -0
- sphynx/results.py +852 -0
- sphynx/retrieval.py +750 -0
- sphynx/retrieval_candidates.py +295 -0
- sphynx/retrieval_catalog_data.py +19 -0
- sphynx/retrieval_recordings.py +125 -0
- sphynx/reviewer_audit.py +266 -0
- sphynx/routing_policies.py +375 -0
- sphynx/runner.py +544 -0
- sphynx/sable_rerank.py +489 -0
- sphynx/score_reliability.py +393 -0
- sphynx/speech_benchmarks.py +204 -0
- sphynx/speech_campaigns.py +378 -0
- sphynx/speech_complete.py +236 -0
- sphynx/speech_contracts.py +225 -0
- sphynx/speech_evidence.py +429 -0
- sphynx/speech_listening.py +115 -0
- sphynx/speech_multilingual.py +71 -0
- sphynx/speech_tts.py +479 -0
- sphynx/sql/001_sync_v1.sql +30 -0
- sphynx/stage_contracts.py +199 -0
- sphynx/statistical_catalog_data.py +53 -0
- sphynx/statistical_rerank.py +232 -0
- sphynx/sync.py +797 -0
- sphynx/workbench.py +378 -0
- sphynx/workbench_catalog_data.py +13 -0
- sphynx/workbench_tracks.py +78 -0
- sphynx/workflow_async.py +197 -0
- sphynx/workflow_contracts.py +594 -0
- sphynx/workflow_decisions.py +211 -0
- sphynx/workflow_defaults.py +16 -0
- sphynx/workflow_extensions.py +212 -0
- sphynx/workflow_registry.py +513 -0
- sphynx/workflows.py +792 -0
|
@@ -0,0 +1,515 @@
|
|
|
1
|
+
"""Typed permitted analytical snapshots. The service owns access and policy."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
from collections.abc import Mapping
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
from datetime import datetime
|
|
7
|
+
import html
|
|
8
|
+
import json
|
|
9
|
+
import re
|
|
10
|
+
from types import MappingProxyType
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
from ._analytical_values import (
|
|
14
|
+
MetricProjectionSummary, ProjectionSummary, ReportProjectionSummary, _Values,
|
|
15
|
+
count, digest, frame, identifier, identity, keys, parse_json, require, sha, summary, text,
|
|
16
|
+
)
|
|
17
|
+
from .audit_contracts import BankRelease
|
|
18
|
+
from .evaluation_protocols import VersionedIdentity
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _display(value: object) -> str:
|
|
22
|
+
result = ''.join(char if char.isprintable() else ' ' for char in str(value))
|
|
23
|
+
return result if len(result) <= 256 else result[:256] + '… (full value in to_dict())'
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _cell(value: object) -> str:
|
|
27
|
+
return re.sub(r'([\\`*_{}\[\]()#+.!|>~\-])', r'\\\1', html.escape(_display(value)))
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _details(value: Mapping[str, Any]) -> str:
|
|
31
|
+
raw = json.dumps(value, ensure_ascii=False, indent=2, allow_nan=False)
|
|
32
|
+
return '\n\n<details><summary>Permitted record details</summary>\n\n<pre>' + html.escape(raw) + '</pre>\n</details>'
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@dataclass(frozen=True)
|
|
36
|
+
class ProjectionRow(_Values):
|
|
37
|
+
"""One permitted summary, checked against its exact stored hash frames."""
|
|
38
|
+
team_id: str
|
|
39
|
+
event_id: str
|
|
40
|
+
row_id: str
|
|
41
|
+
row_hash: str
|
|
42
|
+
row_kind: str
|
|
43
|
+
record_id: str
|
|
44
|
+
bank_id: str
|
|
45
|
+
release_id: str
|
|
46
|
+
cohort_id: str
|
|
47
|
+
source_version: int
|
|
48
|
+
summary_json: str = field(repr=False)
|
|
49
|
+
source_digest: str
|
|
50
|
+
plan_digest: str
|
|
51
|
+
manifest_digest: str
|
|
52
|
+
row_count: int
|
|
53
|
+
commit_hash: str
|
|
54
|
+
|
|
55
|
+
def __post_init__(self) -> None:
|
|
56
|
+
for name in ('team_id', 'event_id', 'record_id', 'bank_id', 'release_id'):
|
|
57
|
+
identifier(getattr(self, name))
|
|
58
|
+
identifier(self.cohort_id, empty=True)
|
|
59
|
+
require(count(self.source_version, nullable=False) != 0)
|
|
60
|
+
require(count(self.row_count, nullable=False, maximum=4096) != 0)
|
|
61
|
+
for name in ('row_id', 'row_hash', 'source_digest', 'plan_digest', 'manifest_digest', 'commit_hash'):
|
|
62
|
+
digest(getattr(self, name))
|
|
63
|
+
value = self.summary
|
|
64
|
+
body = json.dumps({'schema_version': 'sphynx.evaluation.' + self.row_kind + '-summary.v2',
|
|
65
|
+
'summary': value.to_dict()}, sort_keys=True, ensure_ascii=False,
|
|
66
|
+
allow_nan=False, separators=(',', ':'))
|
|
67
|
+
# Only integer counts and exact score bits occur in this schema. Never
|
|
68
|
+
# canonicalize native floating point payloads with this JSON profile.
|
|
69
|
+
canonical_summary = json.dumps(value.to_dict(), sort_keys=True, ensure_ascii=False,
|
|
70
|
+
allow_nan=False, separators=(',', ':'))
|
|
71
|
+
require(canonical_summary == self.summary_json, 'analytical summary bytes differ')
|
|
72
|
+
expected_id = sha(frame('sphynx.evaluation.row-id.v2', self.row_kind, self.record_id,
|
|
73
|
+
self.bank_id, self.release_id, self.cohort_id, self.source_version))
|
|
74
|
+
require(self.row_id == expected_id, 'analytical row identity differs')
|
|
75
|
+
expected_hash = sha(frame('sphynx.evaluation.row.v2', self.team_id, self.event_id,
|
|
76
|
+
self.row_id, self.row_kind, self.record_id, self.bank_id,
|
|
77
|
+
self.release_id, self.cohort_id, self.source_version, body))
|
|
78
|
+
require(self.row_hash == expected_hash, 'analytical row hash differs')
|
|
79
|
+
plan = sha(frame('sphynx.evaluation.plan.v2', self.team_id, self.event_id,
|
|
80
|
+
self.source_digest, self.manifest_digest, self.row_count))
|
|
81
|
+
commit = sha(frame('sphynx.evaluation.commit.v2', plan, self.source_digest,
|
|
82
|
+
self.manifest_digest, self.row_count))
|
|
83
|
+
require(self.plan_digest == plan and self.commit_hash == commit,
|
|
84
|
+
'analytical commit hash differs')
|
|
85
|
+
|
|
86
|
+
@property
|
|
87
|
+
def summary(self) -> ProjectionSummary:
|
|
88
|
+
require(type(self.row_kind) is str and self.row_kind in ('report', 'metric', 'comparison', 'audit'))
|
|
89
|
+
return summary(self.row_kind, parse_json(self.summary_json))
|
|
90
|
+
|
|
91
|
+
@classmethod
|
|
92
|
+
def from_dict(cls, value: Mapping[str, Any]) -> ProjectionRow:
|
|
93
|
+
return cls(**keys(value, {'team_id', 'event_id', 'row_id', 'row_hash', 'row_kind',
|
|
94
|
+
'record_id', 'bank_id', 'release_id', 'cohort_id', 'source_version', 'summary_json',
|
|
95
|
+
'source_digest', 'plan_digest', 'manifest_digest', 'row_count', 'commit_hash'}))
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
@dataclass(frozen=True)
|
|
99
|
+
class ProjectionEventRef(_Values):
|
|
100
|
+
"""A known source reference; it does not grant read access."""
|
|
101
|
+
event_id: str
|
|
102
|
+
source_digest: str
|
|
103
|
+
|
|
104
|
+
def __post_init__(self) -> None:
|
|
105
|
+
identifier(self.event_id)
|
|
106
|
+
digest(self.source_digest)
|
|
107
|
+
|
|
108
|
+
@classmethod
|
|
109
|
+
def from_dict(cls, value: Mapping[str, Any]) -> ProjectionEventRef:
|
|
110
|
+
return cls(**keys(value, {'event_id', 'source_digest'}))
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
@dataclass(frozen=True)
|
|
114
|
+
class ProjectionResult(_Values):
|
|
115
|
+
"""A complete committed inventory, or an explicit unavailable result."""
|
|
116
|
+
event_id: str
|
|
117
|
+
source_digest: str
|
|
118
|
+
availability: str
|
|
119
|
+
reason: str | None
|
|
120
|
+
rows: tuple[ProjectionRow, ...] = field(repr=False)
|
|
121
|
+
|
|
122
|
+
def __post_init__(self) -> None:
|
|
123
|
+
ProjectionEventRef(self.event_id, self.source_digest)
|
|
124
|
+
require(isinstance(self.rows, (tuple, list)) and len(self.rows) <= 4096)
|
|
125
|
+
require(all(isinstance(row, ProjectionRow) for row in self.rows))
|
|
126
|
+
rows = tuple(sorted(self.rows, key=lambda row: row.row_id))
|
|
127
|
+
object.__setattr__(self, 'rows', rows)
|
|
128
|
+
require(self.availability in ('available', 'unavailable'))
|
|
129
|
+
if self.availability == 'unavailable':
|
|
130
|
+
require(not rows and self.reason == 'projection_not_committed')
|
|
131
|
+
return
|
|
132
|
+
require(bool(rows) and self.reason is None)
|
|
133
|
+
first = rows[0]
|
|
134
|
+
require(len(rows) == first.row_count and len({row.row_id for row in rows}) == len(rows),
|
|
135
|
+
'analytical inventory is incomplete')
|
|
136
|
+
for row in rows:
|
|
137
|
+
require(row.event_id == self.event_id and row.source_digest == self.source_digest)
|
|
138
|
+
require(all(getattr(row, name) == getattr(first, name) for name in (
|
|
139
|
+
'team_id', 'bank_id', 'release_id', 'cohort_id', 'row_count',
|
|
140
|
+
'plan_digest', 'manifest_digest', 'commit_hash')))
|
|
141
|
+
manifest = sha(frame('sphynx.evaluation.manifest.v2',
|
|
142
|
+
'\n'.join(row.row_id + '\t' + row.row_hash for row in rows)))
|
|
143
|
+
require(manifest == first.manifest_digest, 'analytical inventory hash differs')
|
|
144
|
+
|
|
145
|
+
@classmethod
|
|
146
|
+
def from_dict(cls, value: Mapping[str, Any]) -> ProjectionResult:
|
|
147
|
+
values = keys(value, {'event_id', 'source_digest', 'availability', 'reason', 'rows'})
|
|
148
|
+
require(isinstance(values['rows'], (tuple, list)))
|
|
149
|
+
values['rows'] = tuple(ProjectionRow.from_dict(row) for row in values['rows'])
|
|
150
|
+
return cls(**values)
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
@dataclass(frozen=True)
|
|
154
|
+
class ProjectionSelection(_Values):
|
|
155
|
+
board_id: str
|
|
156
|
+
board_version: str
|
|
157
|
+
entry_id: str
|
|
158
|
+
definition_digest: str
|
|
159
|
+
|
|
160
|
+
def __post_init__(self) -> None:
|
|
161
|
+
for name in ('board_id', 'entry_id'):
|
|
162
|
+
identifier(getattr(self, name))
|
|
163
|
+
text(self.board_version)
|
|
164
|
+
digest(self.definition_digest)
|
|
165
|
+
|
|
166
|
+
@classmethod
|
|
167
|
+
def from_dict(cls, value: Mapping[str, Any]) -> ProjectionSelection:
|
|
168
|
+
return cls(**keys(value, {'board_id', 'board_version', 'entry_id', 'definition_digest'}))
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
@dataclass(frozen=True)
|
|
172
|
+
class ProjectionPage(_Values):
|
|
173
|
+
"""Historical committed data; current policy requires a service check."""
|
|
174
|
+
team_id: str
|
|
175
|
+
bank_id: str
|
|
176
|
+
release_id: str
|
|
177
|
+
cohort_id: str
|
|
178
|
+
authority: str
|
|
179
|
+
items: tuple[ProjectionResult, ...] = field(repr=False)
|
|
180
|
+
selection: ProjectionSelection | None = None
|
|
181
|
+
|
|
182
|
+
def __post_init__(self) -> None:
|
|
183
|
+
for name in ('team_id', 'bank_id', 'release_id', 'cohort_id'):
|
|
184
|
+
identifier(getattr(self, name))
|
|
185
|
+
require(self.authority == 'committed_snapshot_requires_current_service_check')
|
|
186
|
+
require(isinstance(self.items, (tuple, list)) and 1 <= len(self.items) <= 25)
|
|
187
|
+
require(all(isinstance(item, ProjectionResult) for item in self.items))
|
|
188
|
+
require(len({item.event_id for item in self.items}) == len(self.items))
|
|
189
|
+
require(sum(len(item.rows) for item in self.items) <= 4096)
|
|
190
|
+
require(self.selection is None or isinstance(self.selection, ProjectionSelection))
|
|
191
|
+
object.__setattr__(self, 'items', tuple(self.items))
|
|
192
|
+
for row in self.rows:
|
|
193
|
+
require(row.team_id == self.team_id and row.bank_id == self.bank_id
|
|
194
|
+
and row.release_id == self.release_id
|
|
195
|
+
and (row.cohort_id == self.cohort_id or row.row_kind == 'audit' and row.cohort_id == ''),
|
|
196
|
+
'analytical page scope differs')
|
|
197
|
+
|
|
198
|
+
@property
|
|
199
|
+
def rows(self) -> tuple[ProjectionRow, ...]:
|
|
200
|
+
return tuple(row for item in self.items for row in item.rows)
|
|
201
|
+
|
|
202
|
+
@property
|
|
203
|
+
def source_refs(self) -> tuple[ProjectionEventRef, ...]:
|
|
204
|
+
return tuple(ProjectionEventRef(item.event_id, item.source_digest) for item in self.items)
|
|
205
|
+
|
|
206
|
+
def to_text(self) -> str:
|
|
207
|
+
available = sum(item.availability == 'available' for item in self.items)
|
|
208
|
+
return (f'Projection snapshot: {available}/{len(self.items)} committed events; {len(self.rows)} rows. '
|
|
209
|
+
'Current ranking and acceptance require a service check.')
|
|
210
|
+
|
|
211
|
+
def to_markdown(self, *, include_details: bool = False) -> str:
|
|
212
|
+
require(type(include_details) is bool)
|
|
213
|
+
lines = [self.to_text(), '', '| Event | Availability | Rows | Reason |', '| --- | --- | --- | --- |']
|
|
214
|
+
for item in self.items:
|
|
215
|
+
lines.append('| ' + ' | '.join(_cell(value) for value in (
|
|
216
|
+
item.event_id, item.availability, len(item.rows), item.reason or '—')) + ' |')
|
|
217
|
+
result = '\n'.join(lines)
|
|
218
|
+
return result + _details(self.to_dict()) if include_details else result
|
|
219
|
+
|
|
220
|
+
def to_dict(self) -> dict[str, Any]:
|
|
221
|
+
result = {'schema_version': 'sphynx.evaluation.projection-page.v2',
|
|
222
|
+
**super().to_dict()}
|
|
223
|
+
if self.selection is None:
|
|
224
|
+
del result['selection']
|
|
225
|
+
return result
|
|
226
|
+
|
|
227
|
+
@classmethod
|
|
228
|
+
def from_dict(cls, value: Mapping[str, Any]) -> ProjectionPage:
|
|
229
|
+
names = {'schema_version', 'team_id', 'bank_id', 'release_id', 'cohort_id', 'authority', 'items'}
|
|
230
|
+
require(isinstance(value, Mapping))
|
|
231
|
+
if 'selection' in value:
|
|
232
|
+
names.add('selection')
|
|
233
|
+
values = keys(value, names)
|
|
234
|
+
require(values.pop('schema_version') == 'sphynx.evaluation.projection-page.v2')
|
|
235
|
+
require(isinstance(values['items'], (tuple, list)))
|
|
236
|
+
values['items'] = tuple(ProjectionResult.from_dict(item) for item in values['items'])
|
|
237
|
+
if 'selection' in values:
|
|
238
|
+
values['selection'] = ProjectionSelection.from_dict(values['selection'])
|
|
239
|
+
return cls(**values)
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
@dataclass(frozen=True)
|
|
243
|
+
class ProjectionProof(_Values):
|
|
244
|
+
"""Source evidence retained on a service board entry, without raw rows."""
|
|
245
|
+
event_id: str
|
|
246
|
+
source_digest: str
|
|
247
|
+
plan_digest: str
|
|
248
|
+
manifest_digest: str
|
|
249
|
+
row_count: int
|
|
250
|
+
commit_hash: str
|
|
251
|
+
|
|
252
|
+
def __post_init__(self) -> None:
|
|
253
|
+
identifier(self.event_id)
|
|
254
|
+
for name in ('source_digest', 'plan_digest', 'manifest_digest', 'commit_hash'):
|
|
255
|
+
digest(getattr(self, name))
|
|
256
|
+
require(count(self.row_count, nullable=False, maximum=4096) != 0)
|
|
257
|
+
require(self.commit_hash == sha(frame('sphynx.evaluation.commit.v2', self.plan_digest,
|
|
258
|
+
self.source_digest, self.manifest_digest, self.row_count)))
|
|
259
|
+
|
|
260
|
+
@classmethod
|
|
261
|
+
def from_dict(cls, value: Mapping[str, Any]) -> ProjectionProof:
|
|
262
|
+
return cls(**keys(value, {'event_id', 'source_digest', 'plan_digest',
|
|
263
|
+
'manifest_digest', 'row_count', 'commit_hash'}))
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
@dataclass(frozen=True)
|
|
267
|
+
class BoardRanking(_Values):
|
|
268
|
+
"""Declared metric meaning. This value contains no private rank policy."""
|
|
269
|
+
metric_id: str
|
|
270
|
+
unit: str
|
|
271
|
+
direction: str
|
|
272
|
+
ties: str
|
|
273
|
+
population_id: str
|
|
274
|
+
method_id: str
|
|
275
|
+
uncertainty_id: str | None
|
|
276
|
+
|
|
277
|
+
def __post_init__(self) -> None:
|
|
278
|
+
for name in ('metric_id', 'unit', 'population_id', 'method_id'):
|
|
279
|
+
text(getattr(self, name))
|
|
280
|
+
require(self.direction in ('higher', 'lower') and self.ties == 'competition')
|
|
281
|
+
if self.uncertainty_id is not None:
|
|
282
|
+
text(self.uncertainty_id)
|
|
283
|
+
|
|
284
|
+
@classmethod
|
|
285
|
+
def from_dict(cls, value: Mapping[str, Any]) -> BoardRanking:
|
|
286
|
+
return cls(**keys(value, {'metric_id', 'unit', 'direction', 'ties', 'population_id',
|
|
287
|
+
'method_id', 'uncertainty_id'}))
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
@dataclass(frozen=True)
|
|
291
|
+
class BoardCoverage(_Values):
|
|
292
|
+
selected: int
|
|
293
|
+
projected: int
|
|
294
|
+
pending: int
|
|
295
|
+
rank_eligible: int
|
|
296
|
+
|
|
297
|
+
def __post_init__(self) -> None:
|
|
298
|
+
for name in ('selected', 'projected', 'pending', 'rank_eligible'):
|
|
299
|
+
count(getattr(self, name), nullable=False, maximum=1000)
|
|
300
|
+
# These are board source states, not native metric population counts.
|
|
301
|
+
require(self.projected + self.pending == self.selected and self.rank_eligible <= self.projected)
|
|
302
|
+
|
|
303
|
+
@classmethod
|
|
304
|
+
def from_dict(cls, value: Mapping[str, Any]) -> BoardCoverage:
|
|
305
|
+
return cls(**keys(value, {'selected', 'projected', 'pending', 'rank_eligible'}))
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
@dataclass(frozen=True)
|
|
309
|
+
class BoardEntry(_Values):
|
|
310
|
+
entry_id: str
|
|
311
|
+
candidate: VersionedIdentity = field(repr=False)
|
|
312
|
+
run_id: str = field(repr=False)
|
|
313
|
+
grading_id: str = field(repr=False)
|
|
314
|
+
state: str
|
|
315
|
+
rank_eligible: bool
|
|
316
|
+
provisional: bool
|
|
317
|
+
rank: int | None
|
|
318
|
+
reasons: tuple[str, ...]
|
|
319
|
+
report: ReportProjectionSummary | None = field(repr=False)
|
|
320
|
+
metric: MetricProjectionSummary | None = field(repr=False)
|
|
321
|
+
projection: ProjectionProof | None
|
|
322
|
+
|
|
323
|
+
def __post_init__(self) -> None:
|
|
324
|
+
identifier(self.entry_id)
|
|
325
|
+
text(self.run_id); text(self.grading_id)
|
|
326
|
+
require(isinstance(self.candidate, VersionedIdentity) and self.candidate.digest is not None)
|
|
327
|
+
require(type(self.rank_eligible) is bool and type(self.provisional) is bool)
|
|
328
|
+
require(self.state in ('pending', 'unranked', 'provisional', 'ranked'))
|
|
329
|
+
require(isinstance(self.reasons, (tuple, list)) and len(self.reasons) <= 32)
|
|
330
|
+
for reason in self.reasons:
|
|
331
|
+
text(reason)
|
|
332
|
+
object.__setattr__(self, 'reasons', tuple(self.reasons))
|
|
333
|
+
count(self.rank, maximum=1000)
|
|
334
|
+
require(self.rank != 0 and (self.rank is not None) == self.rank_eligible)
|
|
335
|
+
require(self.report is None or isinstance(self.report, ReportProjectionSummary))
|
|
336
|
+
require(self.metric is None or isinstance(self.metric, MetricProjectionSummary))
|
|
337
|
+
require(self.projection is None or isinstance(self.projection, ProjectionProof))
|
|
338
|
+
if self.state == 'pending':
|
|
339
|
+
require(not self.rank_eligible and self.projection is None and self.report is None
|
|
340
|
+
and self.metric is None and bool(self.reasons))
|
|
341
|
+
else:
|
|
342
|
+
require(self.projection is not None)
|
|
343
|
+
if self.rank_eligible:
|
|
344
|
+
require(not self.reasons and self.state == ('provisional' if self.provisional else 'ranked'))
|
|
345
|
+
require(self.report is not None and self.metric is not None)
|
|
346
|
+
assert self.metric is not None
|
|
347
|
+
require(self.metric.availability == 'available' and self.metric.value_kind == 'score')
|
|
348
|
+
else:
|
|
349
|
+
require(bool(self.reasons) and self.state != 'ranked')
|
|
350
|
+
if self.report is not None:
|
|
351
|
+
require(self.report.grading_id == self.grading_id)
|
|
352
|
+
if self.metric is not None:
|
|
353
|
+
require(self.metric.grading_id == self.grading_id)
|
|
354
|
+
|
|
355
|
+
@classmethod
|
|
356
|
+
def from_dict(cls, value: Mapping[str, Any]) -> BoardEntry:
|
|
357
|
+
values = keys(value, {'entry_id', 'candidate', 'run_id', 'grading_id', 'state',
|
|
358
|
+
'rank_eligible', 'provisional', 'rank', 'reasons', 'report', 'metric', 'projection'})
|
|
359
|
+
values['candidate'] = identity(values['candidate'])
|
|
360
|
+
for name, reader in (('report', ReportProjectionSummary), ('metric', MetricProjectionSummary),
|
|
361
|
+
('projection', ProjectionProof)):
|
|
362
|
+
if values[name] is not None:
|
|
363
|
+
values[name] = reader.from_dict(values[name])
|
|
364
|
+
return cls(**values)
|
|
365
|
+
|
|
366
|
+
|
|
367
|
+
def board_cursor(value: Any, *, snapshot: str | None = None, after: str | None = None) -> str:
|
|
368
|
+
require(type(value) is str and value.startswith('v2:') and len(value) <= 2048,
|
|
369
|
+
'invalid board cursor; retain the original request')
|
|
370
|
+
try:
|
|
371
|
+
raw = bytes.fromhex(value[3:]).decode('utf-8')
|
|
372
|
+
except (ValueError, UnicodeError):
|
|
373
|
+
require(False, 'invalid board cursor; retain the original request')
|
|
374
|
+
raise AssertionError('unreachable')
|
|
375
|
+
parsed = keys(parse_json(raw), {'schema_version', 'snapshot_digest', 'after_entry_id'})
|
|
376
|
+
require(parsed['schema_version'] == 'sphynx.evaluation-board-cursor.v2')
|
|
377
|
+
digest(parsed['snapshot_digest']); identifier(parsed['after_entry_id'])
|
|
378
|
+
require(snapshot is None or snapshot == parsed['snapshot_digest'], 'board cursor snapshot differs')
|
|
379
|
+
require(after is None or after == parsed['after_entry_id'], 'board cursor entry differs')
|
|
380
|
+
return value
|
|
381
|
+
|
|
382
|
+
|
|
383
|
+
@dataclass(frozen=True)
|
|
384
|
+
class EvaluationBoardPage(_Values):
|
|
385
|
+
"""A bounded page from a current authorized service check, not PR acceptance."""
|
|
386
|
+
authority: str
|
|
387
|
+
team_id: str
|
|
388
|
+
bank_id: str
|
|
389
|
+
board_id: str
|
|
390
|
+
board_version: str
|
|
391
|
+
definition_digest: str
|
|
392
|
+
cohort: VersionedIdentity = field(repr=False)
|
|
393
|
+
conditions: Mapping[str, VersionedIdentity] = field(repr=False)
|
|
394
|
+
scope: str
|
|
395
|
+
mode: str
|
|
396
|
+
ranking: BoardRanking
|
|
397
|
+
ranking_scope: str
|
|
398
|
+
bank_release: BankRelease = field(repr=False)
|
|
399
|
+
coverage: BoardCoverage
|
|
400
|
+
projection_coverage_complete: bool
|
|
401
|
+
snapshot_digest: str
|
|
402
|
+
checked_at: str
|
|
403
|
+
total_entries: int
|
|
404
|
+
limit: int
|
|
405
|
+
entries: tuple[BoardEntry, ...] = field(repr=False)
|
|
406
|
+
next_cursor: str | None = field(repr=False)
|
|
407
|
+
|
|
408
|
+
def __post_init__(self) -> None:
|
|
409
|
+
require(self.authority == 'current_service_check')
|
|
410
|
+
for name in ('team_id', 'bank_id', 'board_id'):
|
|
411
|
+
identifier(getattr(self, name))
|
|
412
|
+
text(self.board_version); digest(self.definition_digest); digest(self.snapshot_digest)
|
|
413
|
+
require(isinstance(self.cohort, VersionedIdentity) and self.cohort.digest is not None)
|
|
414
|
+
identifier(self.cohort.identity_id)
|
|
415
|
+
require(isinstance(self.conditions, Mapping) and {'protocol', 'bank', 'evaluator', 'runtime'} <= set(self.conditions)
|
|
416
|
+
and set(self.conditions) <= {'protocol', 'bank', 'evaluator', 'runtime', 'labels', 'references', 'hardware'})
|
|
417
|
+
require(all(isinstance(value, VersionedIdentity) and value.digest is not None for value in self.conditions.values()))
|
|
418
|
+
object.__setattr__(self, 'conditions', MappingProxyType(dict(self.conditions)))
|
|
419
|
+
require(self.scope in ('component', 'subsystem', 'system') and self.mode in ('local', 'hosted'))
|
|
420
|
+
require(isinstance(self.ranking, BoardRanking) and self.ranking_scope == 'verified_available_entries')
|
|
421
|
+
require(isinstance(self.bank_release, BankRelease) and self.bank_release.bank_id == self.bank_id
|
|
422
|
+
and self.bank_release.team_id == self.team_id)
|
|
423
|
+
require(all(self.conditions[key] == self.bank_release.bindings.get(key)
|
|
424
|
+
for key in ('bank', 'labels', 'references') if key in self.conditions))
|
|
425
|
+
require(isinstance(self.coverage, BoardCoverage))
|
|
426
|
+
require(type(self.projection_coverage_complete) is bool
|
|
427
|
+
and self.projection_coverage_complete == (self.coverage.pending == 0))
|
|
428
|
+
count(self.total_entries, nullable=False, maximum=1000)
|
|
429
|
+
require(self.total_entries == self.coverage.selected)
|
|
430
|
+
require(count(self.limit, nullable=False, maximum=100) != 0)
|
|
431
|
+
require(isinstance(self.entries, (tuple, list)) and len(self.entries) <= min(self.limit, self.total_entries)
|
|
432
|
+
and all(isinstance(entry, BoardEntry) for entry in self.entries))
|
|
433
|
+
object.__setattr__(self, 'entries', tuple(self.entries))
|
|
434
|
+
require(len({entry.entry_id for entry in self.entries}) == len(self.entries))
|
|
435
|
+
require(len({(entry.candidate.identity_id, entry.candidate.version, entry.candidate.digest)
|
|
436
|
+
for entry in self.entries}) == len(self.entries))
|
|
437
|
+
try:
|
|
438
|
+
checked = datetime.fromisoformat(self.checked_at)
|
|
439
|
+
require(checked.tzinfo is not None)
|
|
440
|
+
except (ValueError, TypeError, OverflowError):
|
|
441
|
+
require(False, 'invalid board check timestamp')
|
|
442
|
+
for entry in self.entries:
|
|
443
|
+
if entry.projection is not None:
|
|
444
|
+
proof = entry.projection
|
|
445
|
+
expected = sha(frame('sphynx.evaluation.plan.v2', self.team_id, proof.event_id,
|
|
446
|
+
proof.source_digest, proof.manifest_digest, proof.row_count))
|
|
447
|
+
require(proof.plan_digest == expected)
|
|
448
|
+
if entry.rank_eligible:
|
|
449
|
+
require(bool(self.bank_release.eligibility['ranking']['eligible']))
|
|
450
|
+
require(entry.report is not None and entry.metric is not None)
|
|
451
|
+
assert entry.report is not None and entry.metric is not None
|
|
452
|
+
require(entry.report.bindings == {**self.conditions, 'candidate': entry.candidate}
|
|
453
|
+
and entry.report.run_id == entry.run_id and entry.metric.run_id == entry.run_id
|
|
454
|
+
and entry.report.scope == self.scope and entry.report.mode == self.mode)
|
|
455
|
+
require(all(getattr(entry.metric, name) == getattr(self.ranking, name) for name in (
|
|
456
|
+
'metric_id', 'unit', 'direction', 'population_id', 'method_id', 'uncertainty_id')))
|
|
457
|
+
require(entry.rank is not None and entry.rank <= self.coverage.rank_eligible)
|
|
458
|
+
if self.next_cursor is not None:
|
|
459
|
+
require(bool(self.entries) and len(self.entries) == self.limit)
|
|
460
|
+
board_cursor(self.next_cursor, snapshot=self.snapshot_digest, after=self.entries[-1].entry_id)
|
|
461
|
+
|
|
462
|
+
def to_dict(self) -> dict[str, Any]:
|
|
463
|
+
result = {'schema_version': 'sphynx.evaluation-board.v2', **super().to_dict()}
|
|
464
|
+
result['bank_release'] = self.bank_release.to_dict()
|
|
465
|
+
return result
|
|
466
|
+
|
|
467
|
+
def to_text(self) -> str:
|
|
468
|
+
heading = (f'Board {self.board_id}: {len(self.entries)}/{self.total_entries} entries shown; '
|
|
469
|
+
f'{self.coverage.pending} pending projections; {self.coverage.rank_eligible} rank eligible. '
|
|
470
|
+
f'Checked {self.checked_at}. Rank is separate from PR acceptance.')
|
|
471
|
+
return '\n'.join([heading, *(f'{entry.rank if entry.rank is not None else "—"}. '
|
|
472
|
+
+ _display(entry.candidate.identity_id) + ': ' + _display(_entry_value(entry))
|
|
473
|
+
+ ' [' + entry.state + ']' + (': ' + ', '.join(_display(reason) for reason in entry.reasons) if entry.reasons else '')
|
|
474
|
+
for entry in self.entries)])
|
|
475
|
+
|
|
476
|
+
def to_markdown(self, *, include_details: bool = False) -> str:
|
|
477
|
+
require(type(include_details) is bool)
|
|
478
|
+
lines = [f'Board **{_cell(self.board_id)}**: {len(self.entries)}/{self.total_entries} entries shown; '
|
|
479
|
+
f'{self.coverage.pending} pending projections; {self.coverage.rank_eligible} rank eligible.',
|
|
480
|
+
'', 'Rank is separate from PR acceptance. The service checked this snapshot at ' + _cell(self.checked_at) + '.',
|
|
481
|
+
'', '| Rank | Candidate | Value | State | Reasons |', '| --- | --- | --- | --- | --- |']
|
|
482
|
+
for entry in self.entries:
|
|
483
|
+
lines.append('| ' + ' | '.join(_cell(value) for value in (
|
|
484
|
+
entry.rank if entry.rank is not None else '—', entry.candidate.identity_id,
|
|
485
|
+
_entry_value(entry), entry.state, ', '.join(entry.reasons) or '—')) + ' |')
|
|
486
|
+
result = '\n'.join(lines)
|
|
487
|
+
return result + _details(self.to_dict()) if include_details else result
|
|
488
|
+
|
|
489
|
+
@classmethod
|
|
490
|
+
def from_dict(cls, value: Mapping[str, Any]) -> EvaluationBoardPage:
|
|
491
|
+
values = keys(value, {'schema_version', 'authority', 'team_id', 'bank_id', 'board_id',
|
|
492
|
+
'board_version', 'definition_digest', 'cohort', 'conditions', 'scope', 'mode', 'ranking',
|
|
493
|
+
'ranking_scope', 'bank_release', 'coverage', 'projection_coverage_complete', 'snapshot_digest',
|
|
494
|
+
'checked_at', 'total_entries', 'limit', 'entries', 'next_cursor'})
|
|
495
|
+
require(values.pop('schema_version') == 'sphynx.evaluation-board.v2')
|
|
496
|
+
values['cohort'] = identity(values['cohort'])
|
|
497
|
+
require(isinstance(values['conditions'], Mapping))
|
|
498
|
+
values['conditions'] = {name: identity(item) for name, item in values['conditions'].items()}
|
|
499
|
+
values['ranking'] = BoardRanking.from_dict(values['ranking'])
|
|
500
|
+
values['bank_release'] = BankRelease.from_dict(values['bank_release'])
|
|
501
|
+
values['coverage'] = BoardCoverage.from_dict(values['coverage'])
|
|
502
|
+
require(isinstance(values['entries'], (tuple, list)))
|
|
503
|
+
values['entries'] = tuple(BoardEntry.from_dict(entry) for entry in values['entries'])
|
|
504
|
+
return cls(**values)
|
|
505
|
+
|
|
506
|
+
|
|
507
|
+
def _entry_value(entry: BoardEntry) -> str:
|
|
508
|
+
metric = entry.metric
|
|
509
|
+
if metric is None:
|
|
510
|
+
return 'pending projection' if entry.state == 'pending' else 'metric unavailable'
|
|
511
|
+
if metric.availability != 'available':
|
|
512
|
+
return metric.availability
|
|
513
|
+
if metric.value_kind == 'custom':
|
|
514
|
+
return 'custom value in native report'
|
|
515
|
+
return f'{metric.value} {metric.unit}'
|