sable-sphynx 0.7.0.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sable_sphynx-0.7.0.dev0.dist-info/METADATA +152 -0
- sable_sphynx-0.7.0.dev0.dist-info/RECORD +174 -0
- sable_sphynx-0.7.0.dev0.dist-info/WHEEL +5 -0
- sable_sphynx-0.7.0.dev0.dist-info/top_level.txt +1 -0
- sphynx/__init__.py +469 -0
- sphynx/__init__.pyi +376 -0
- sphynx/_analytical_client.py +77 -0
- sphynx/_analytical_values.py +252 -0
- sphynx/_audit_client.py +173 -0
- sphynx/_comparison_adapters.py +113 -0
- sphynx/_componentized_full_rag.py +34 -0
- sphynx/_composed_pipeline.py +109 -0
- sphynx/_full_rag_grading.py +218 -0
- sphynx/_full_rag_trace.py +221 -0
- sphynx/_local_eval.py +268 -0
- sphynx/_pipeline_core.py +243 -0
- sphynx/_pr_client.py +237 -0
- sphynx/_rag_diagnostics_calibration.py +285 -0
- sphynx/_rag_diagnostics_catalog.py +39 -0
- sphynx/_rag_diagnostics_common.py +26 -0
- sphynx/_rag_diagnostics_contracts.py +186 -0
- sphynx/_rag_diagnostics_engine.py +243 -0
- sphynx/_rag_diagnostics_importers.py +242 -0
- sphynx/_rag_diagnostics_ranking.py +81 -0
- sphynx/_rag_diagnostics_semantic.py +325 -0
- sphynx/_rag_ranking.py +149 -0
- sphynx/_speech_benchmark_scoring.py +140 -0
- sphynx/_speech_benchmark_sources.py +275 -0
- sphynx/_speech_complete_scoring.py +218 -0
- sphynx/_speech_complete_source.py +118 -0
- sphynx/_speech_kido.py +294 -0
- sphynx/_speech_scoring.py +77 -0
- sphynx/_track_capture.py +213 -0
- sphynx/analytical_views.py +515 -0
- sphynx/assessment_contracts.py +378 -0
- sphynx/assessment_selection.py +57 -0
- sphynx/audit_access.py +245 -0
- sphynx/audit_contracts.py +204 -0
- sphynx/audit_requests.py +180 -0
- sphynx/authoring.py +188 -0
- sphynx/benchmarks.py +568 -0
- sphynx/candidates.py +62 -0
- sphynx/catalog.py +272 -0
- sphynx/chunking.py +372 -0
- sphynx/clickhouse_reader.py +169 -0
- sphynx/client.py +486 -0
- sphynx/comparison_core.py +149 -0
- sphynx/comparison_views.py +223 -0
- sphynx/components.py +1661 -0
- sphynx/contracts.py +514 -0
- sphynx/customer_rerank.py +299 -0
- sphynx/customer_retrieval.py +146 -0
- sphynx/data/benchmark-releases.v1.json +164 -0
- sphynx/data/public-benchmark-release.v1.json +354 -0
- sphynx/data/rag-diagnostics-synthetic.v1.json +6982 -0
- sphynx/data/rag-paper-references.v1.json +176 -0
- sphynx/data/rag-paper-schemas.v1.json +1069 -0
- sphynx/data/rag-rerank-v1.json +507 -0
- sphynx/dataset_catalog_data.py +1214 -0
- sphynx/dataset_readiness.py +138 -0
- sphynx/dataset_v3.py +999 -0
- sphynx/demo.py +130 -0
- sphynx/demo_candidates/__init__.py +1 -0
- sphynx/demo_candidates/dense.py +11 -0
- sphynx/demo_candidates/lexical.py +21 -0
- sphynx/diagnostics.py +207 -0
- sphynx/documentation.py +737 -0
- sphynx/documentation_candidate.py +939 -0
- sphynx/evals.py +538 -0
- sphynx/evaluation_protocols.py +124 -0
- sphynx/evaluator_bindings.py +370 -0
- sphynx/evidence.py +352 -0
- sphynx/execution_capabilities.py +109 -0
- sphynx/fast_search.py +285 -0
- sphynx/full_rag.py +341 -0
- sphynx/hosted/__init__.py +18 -0
- sphynx/hosted/benchmark_intake.py +218 -0
- sphynx/hosted/contracts.py +361 -0
- sphynx/hosted/model_admission.py +130 -0
- sphynx/hosted/model_artifacts.py +298 -0
- sphynx/hosted/model_contracts.py +63 -0
- sphynx/hosted/model_gcs.py +181 -0
- sphynx/hosted/model_history.py +84 -0
- sphynx/hosted/model_session.py +233 -0
- sphynx/hosted/public_suite.py +74 -0
- sphynx/hosted/registry.py +97 -0
- sphynx/hosted/revisions.py +126 -0
- sphynx/hosted/sandbox.py +105 -0
- sphynx/hosted/submission.py +287 -0
- sphynx/hosted_workflow.py +451 -0
- sphynx/http_transport.py +86 -0
- sphynx/integrated_rag.py +352 -0
- sphynx/integrated_workflow.py +552 -0
- sphynx/integration_contracts.py +474 -0
- sphynx/invocation.py +791 -0
- sphynx/invocation_candidates.py +392 -0
- sphynx/invocation_catalog_data.py +40 -0
- sphynx/learned_catalog_data.py +30 -0
- sphynx/learned_rerank.py +267 -0
- sphynx/legacy_full_rag.py +67 -0
- sphynx/lifecycle.py +885 -0
- sphynx/lifecycle_workflow.py +490 -0
- sphynx/metric_views.py +474 -0
- sphynx/metrics.py +282 -0
- sphynx/model_adapters.py +438 -0
- sphynx/model_catalog_data.py +30 -0
- sphynx/official_benchmarks.py +385 -0
- sphynx/packs.py +258 -0
- sphynx/pipeline.py +313 -0
- sphynx/pipeline_workflow.py +551 -0
- sphynx/pr_evaluations.py +560 -0
- sphynx/progress.py +89 -0
- sphynx/public_bundle.py +173 -0
- sphynx/public_eval.py +62 -0
- sphynx/public_rerank.py +355 -0
- sphynx/py.typed +0 -0
- sphynx/pytest_plugin.py +62 -0
- sphynx/question_robustness.py +437 -0
- sphynx/rag.py +329 -0
- sphynx/rag_diagnostics.py +309 -0
- sphynx/rag_papers/__init__.py +29 -0
- sphynx/rag_papers/__main__.py +45 -0
- sphynx/rag_papers/candidates.py +19 -0
- sphynx/rag_papers/contracts.py +282 -0
- sphynx/rag_papers/datasets.py +291 -0
- sphynx/rag_papers/experiments.py +116 -0
- sphynx/rag_papers/judges.py +308 -0
- sphynx/rag_papers/registry.py +33 -0
- sphynx/rag_papers/runner.py +176 -0
- sphynx/rag_papers/sable.py +135 -0
- sphynx/rag_papers/schemas.py +12 -0
- sphynx/rag_papers/scoring.py +249 -0
- sphynx/rag_papers/semantic.py +288 -0
- sphynx/rag_papers/trec.py +80 -0
- sphynx/recorded_scores.py +352 -0
- sphynx/records.py +124 -0
- sphynx/reflex_traces.py +193 -0
- sphynx/report_rendering.py +71 -0
- sphynx/rerank_family_contracts.py +150 -0
- sphynx/rerank_workflow.py +424 -0
- sphynx/research_candidate.py +190 -0
- sphynx/results.py +852 -0
- sphynx/retrieval.py +750 -0
- sphynx/retrieval_candidates.py +295 -0
- sphynx/retrieval_catalog_data.py +19 -0
- sphynx/retrieval_recordings.py +125 -0
- sphynx/reviewer_audit.py +266 -0
- sphynx/routing_policies.py +375 -0
- sphynx/runner.py +544 -0
- sphynx/sable_rerank.py +489 -0
- sphynx/score_reliability.py +393 -0
- sphynx/speech_benchmarks.py +204 -0
- sphynx/speech_campaigns.py +378 -0
- sphynx/speech_complete.py +236 -0
- sphynx/speech_contracts.py +225 -0
- sphynx/speech_evidence.py +429 -0
- sphynx/speech_listening.py +115 -0
- sphynx/speech_multilingual.py +71 -0
- sphynx/speech_tts.py +479 -0
- sphynx/sql/001_sync_v1.sql +30 -0
- sphynx/stage_contracts.py +199 -0
- sphynx/statistical_catalog_data.py +53 -0
- sphynx/statistical_rerank.py +232 -0
- sphynx/sync.py +797 -0
- sphynx/workbench.py +378 -0
- sphynx/workbench_catalog_data.py +13 -0
- sphynx/workbench_tracks.py +78 -0
- sphynx/workflow_async.py +197 -0
- sphynx/workflow_contracts.py +594 -0
- sphynx/workflow_decisions.py +211 -0
- sphynx/workflow_defaults.py +16 -0
- sphynx/workflow_extensions.py +212 -0
- sphynx/workflow_registry.py +513 -0
- sphynx/workflows.py +792 -0
sphynx/reflex_traces.py
ADDED
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
"""Offline ingestion of opt-in Reflex quality traces; no service or model calls."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import copy
|
|
5
|
+
import hashlib
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
from ._rag_diagnostics_contracts import ContractError, validate_dataset
|
|
9
|
+
from ._rag_diagnostics_importers import _json
|
|
10
|
+
|
|
11
|
+
SCHEMA = "reflex.query-quality.v1"
|
|
12
|
+
MAX_FILE_BYTES = 100_000_000
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _need(condition, message):
|
|
16
|
+
if not condition:
|
|
17
|
+
raise ContractError(message)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _hash(value):
|
|
21
|
+
return isinstance(value, str) and len(value) == 64 and all(c in '0123456789abcdef' for c in value)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def load_reflex_traces(path: str | Path) -> dict:
|
|
25
|
+
"""Load finalized, unsampled, loss-free Reflex JSONL quality evidence.
|
|
26
|
+
|
|
27
|
+
Health requests are retained separately. Failed search records are explicit;
|
|
28
|
+
they cannot be substituted with successful empty rankings.
|
|
29
|
+
"""
|
|
30
|
+
path = Path(path)
|
|
31
|
+
_need(path.is_file() and not path.is_symlink() and path.stat().st_size <= MAX_FILE_BYTES,
|
|
32
|
+
'Expected a regular Reflex trace file no larger than 100MB')
|
|
33
|
+
raw = path.read_bytes()
|
|
34
|
+
_need(len(raw) <= MAX_FILE_BYTES, 'Reflex trace file exceeds 100MB')
|
|
35
|
+
rows = [_json(line) for line in raw.splitlines() if line.strip()]
|
|
36
|
+
_need(len(rows) >= 2 and all(type(row) is dict for row in rows), 'Expected finalized JSONL records')
|
|
37
|
+
session, summary = rows[0], rows[-1]
|
|
38
|
+
_need(session.get('type') == 'session' and summary.get('type') == 'summary', 'Missing session or final summary')
|
|
39
|
+
_need(session.get('version') == 1 and summary.get('version') == 1, 'Unsupported Reflex timing schema')
|
|
40
|
+
_need(type(session.get('sample_every')) is int and session['sample_every'] == 1,
|
|
41
|
+
'Complete quality capture requires sample_every=1')
|
|
42
|
+
counters = ('observed', 'sampled', 'enqueued', 'written', 'dropped', 'unfinished')
|
|
43
|
+
_need(all(type(summary.get(key)) is int and summary[key] >= 0 for key in counters), 'Invalid trace counters')
|
|
44
|
+
requests = rows[1:-1]
|
|
45
|
+
_need(summary['written'] == len(requests) == summary['enqueued'], 'Written trace counts do not reconcile')
|
|
46
|
+
_need(summary['sampled'] == summary['enqueued'] + summary['dropped'] + summary['unfinished'] == summary['observed'],
|
|
47
|
+
'Sampled/observed trace counts do not reconcile')
|
|
48
|
+
_need(summary['dropped'] == summary['unfinished'] == 0, 'Dropped or unfinished evidence cannot claim complete capture')
|
|
49
|
+
ids, searches, other = set(), {}, 0
|
|
50
|
+
for row in requests:
|
|
51
|
+
rid = row.get('request_id')
|
|
52
|
+
_need(row.get('type') == 'request' and row.get('version') == 1 and type(rid) is str and rid and rid not in ids,
|
|
53
|
+
'Request IDs must be unique and nonempty')
|
|
54
|
+
ids.add(rid)
|
|
55
|
+
_need(type(row.get('elapsed_ns')) is int and row['elapsed_ns'] >= 0, 'Invalid elapsed time')
|
|
56
|
+
end = 0
|
|
57
|
+
_need(type(row.get('spans')) is list and all(type(s) is dict for s in row['spans']), 'Trace spans are required')
|
|
58
|
+
_need(row.get('outcome') in ('response_prepared', 'abandoned'), 'Invalid request outcome')
|
|
59
|
+
_need(row.get('mode') in (None, 'normal', 'speculative'), 'Unknown Reflex execution mode')
|
|
60
|
+
_need(row.get('http_status') is None or type(row['http_status']) is int and 100 <= row['http_status'] <= 599,
|
|
61
|
+
'Invalid HTTP status')
|
|
62
|
+
for span in sorted(row['spans'], key=lambda s: s.get('start_ns', -1)):
|
|
63
|
+
_need(type(span.get('start_ns')) is int and type(span.get('duration_ns')) is int
|
|
64
|
+
and span['start_ns'] >= end and span['duration_ns'] >= 0, 'Invalid or overlapping timing spans')
|
|
65
|
+
end = span['start_ns'] + span['duration_ns']
|
|
66
|
+
_need(end <= row['elapsed_ns'], 'Timing span exceeds request duration')
|
|
67
|
+
if row.get('mode') not in ('normal', 'speculative'):
|
|
68
|
+
other += 1
|
|
69
|
+
continue
|
|
70
|
+
quality = row.get('quality')
|
|
71
|
+
if quality is None and row.get('http_status') != 200:
|
|
72
|
+
# Decode/admission failures have no executed query pipeline. Keep the
|
|
73
|
+
# request in the census; pairing cannot invent its missing query.
|
|
74
|
+
searches[rid] = copy.deepcopy(row)
|
|
75
|
+
continue
|
|
76
|
+
_need(type(quality) is dict and quality.get('schema_version') == SCHEMA,
|
|
77
|
+
'Search is missing opt-in Reflex quality evidence')
|
|
78
|
+
_need(type(quality.get('query')) is str and 0 < len(quality['query']) <= 20000, 'Missing or oversized exact query')
|
|
79
|
+
_need(_hash(quality.get('snapshot_sha256')), 'Exact served snapshot identity is required')
|
|
80
|
+
_need(type(quality.get('indexed_passages')) is int and quality['indexed_passages'] >= 0, 'Invalid index population')
|
|
81
|
+
_need(type(quality.get('top_k')) is int and 1 <= quality['top_k'] <= 100, 'Invalid requested result limit')
|
|
82
|
+
_need(type(quality.get('reranking_applied')) is bool, 'Reranking execution must be explicit')
|
|
83
|
+
_need(quality.get('capture_status') in ('complete', 'incomplete', 'content_limit'), 'Invalid capture status')
|
|
84
|
+
_need(quality['capture_status'] != 'content_limit', 'Quality content limit reached; capture is incomplete')
|
|
85
|
+
stages = quality.get('stages')
|
|
86
|
+
_need(type(stages) is dict and set(stages) == {'retrieved', 'reranked', 'selected'}, 'Explicit stage records required')
|
|
87
|
+
documents, previous = {}, None
|
|
88
|
+
for stage in ('retrieved', 'reranked', 'selected'):
|
|
89
|
+
ranking = stages[stage]
|
|
90
|
+
if ranking is None:
|
|
91
|
+
continue
|
|
92
|
+
_need(type(ranking) is list and len(ranking) <= 10000, 'Invalid stage ranking')
|
|
93
|
+
current = set()
|
|
94
|
+
for hit in ranking:
|
|
95
|
+
_need(type(hit) is dict and _hash(hit.get('chunk_sha256')) and type(hit.get('position')) is int
|
|
96
|
+
and 0 <= hit['position'] < quality['indexed_passages'], 'Invalid passage identity')
|
|
97
|
+
_need(_hash(hit.get('artifact_sha256')) and type(hit.get('chunk_index')) is int
|
|
98
|
+
and hit['chunk_index'] >= 0, 'Original artifact/chunk coordinates required')
|
|
99
|
+
_need(type(hit.get('text')) is str and type(hit.get('searchable_text')) is str, 'Exact passage texts required')
|
|
100
|
+
score = hit.get('score')
|
|
101
|
+
_need(type(score) in (int, float), 'Finite stage score required')
|
|
102
|
+
identity = f"{quality['snapshot_sha256']}:{hit['position']}"
|
|
103
|
+
_need(identity not in current, 'Duplicate stage passage identity')
|
|
104
|
+
current.add(identity)
|
|
105
|
+
document = {'id': identity, 'text': hit['text'], 'searchable_text': hit['searchable_text'],
|
|
106
|
+
'chunk_sha256': hit['chunk_sha256'], 'artifact_sha256': hit['artifact_sha256'],
|
|
107
|
+
'chunk_index': hit['chunk_index']}
|
|
108
|
+
_need(identity not in documents or documents[identity] == document, 'Passage content changed between stages')
|
|
109
|
+
documents[identity] = document
|
|
110
|
+
_need(previous is None or current <= previous, 'A later stage invents a passage')
|
|
111
|
+
previous = current
|
|
112
|
+
_need(stages['reranked'] is None or quality['reranking_applied'], 'Unexecuted reranking cannot claim a ranking')
|
|
113
|
+
if row.get('outcome') == 'response_prepared' and row.get('http_status') == 200:
|
|
114
|
+
_need(quality['capture_status'] == 'complete' and stages['retrieved'] is not None and stages['selected'] is not None,
|
|
115
|
+
'Successful search is missing stage evidence')
|
|
116
|
+
_need(not quality['reranking_applied'] or stages['reranked'] is not None, 'Missing reranked stage')
|
|
117
|
+
_need(len(stages['selected']) <= quality['top_k'], 'Selected count exceeds requested limit')
|
|
118
|
+
upstream = stages['reranked'] if quality['reranking_applied'] else stages['retrieved']
|
|
119
|
+
_need(stages['selected'] == upstream[:quality['top_k']], 'Selected output is not the exact final ranking prefix')
|
|
120
|
+
searches[rid] = copy.deepcopy(row)
|
|
121
|
+
return {'schema_version': 'sphynx.reflex-trace-file.v1', 'requests': searches,
|
|
122
|
+
'nonsearch_requests': other, 'source_sha256': hashlib.sha256(raw).hexdigest(),
|
|
123
|
+
'source_bytes': len(raw), 'summary': summary}
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def pair_reflex_traces(baseline: dict, candidate: dict, case_requests: dict, *, release: str,
|
|
127
|
+
execution_identities: dict) -> dict:
|
|
128
|
+
"""Pair exact request IDs, without deriving labels or matching by query text.
|
|
129
|
+
|
|
130
|
+
Each case mapping supplies baseline/candidate request IDs and a cluster ID.
|
|
131
|
+
Cluster IDs describe grouping; they never authorize population inference.
|
|
132
|
+
"""
|
|
133
|
+
_need(type(release) is str and release and type(case_requests) is dict and case_requests, 'Release and explicit case map required')
|
|
134
|
+
_need(type(execution_identities) is dict and set(execution_identities) == {'baseline', 'candidate'},
|
|
135
|
+
'Both execution identity manifests are required')
|
|
136
|
+
for identity in execution_identities.values():
|
|
137
|
+
_need(type(identity) is dict and type(identity.get('source_commit')) is str
|
|
138
|
+
and len(identity['source_commit']) == 40 and all(c in '0123456789abcdef' for c in identity['source_commit'])
|
|
139
|
+
and _hash(identity.get('binary_sha256')) and _hash(identity.get('query_plan_sha256')),
|
|
140
|
+
'Pin exact source commit, executable bytes, and query plan')
|
|
141
|
+
_need(type(identity.get('model_artifacts_sha256')) is dict
|
|
142
|
+
and all(type(k) is str and k and _hash(v) for k, v in identity['model_artifacts_sha256'].items()),
|
|
143
|
+
'Explicit model artifact hashes required (empty only for model-free execution)')
|
|
144
|
+
cases, used = [], {'baseline': set(), 'candidate': set()}
|
|
145
|
+
for case_id, binding in case_requests.items():
|
|
146
|
+
_need(type(binding) is dict and set(binding) == {'baseline', 'candidate', 'cluster_id'}, 'Case map requires both request IDs and a cluster ID')
|
|
147
|
+
records = {}
|
|
148
|
+
for variant, source in (('baseline', baseline), ('candidate', candidate)):
|
|
149
|
+
rid = binding[variant]
|
|
150
|
+
_need(rid in source['requests'] and rid not in used[variant], 'Unknown or reused request ID')
|
|
151
|
+
used[variant].add(rid)
|
|
152
|
+
records[variant] = source['requests'][rid]
|
|
153
|
+
_need(all(type(r.get('quality')) is dict for r in records.values()),
|
|
154
|
+
'Pre-execution failure has no query to pair; retain it in the request census and resolve explicitly')
|
|
155
|
+
left, right = (records[v]['quality'] for v in ('baseline', 'candidate'))
|
|
156
|
+
_need(left['query'] == right['query'], 'Paired requests have different exact queries')
|
|
157
|
+
_need(left['snapshot_sha256'] == right['snapshot_sha256'], 'Paired requests must use the same frozen corpus snapshot')
|
|
158
|
+
documents, traces = {}, {}
|
|
159
|
+
for variant, record in records.items():
|
|
160
|
+
quality = record['quality']
|
|
161
|
+
stages = {'indexed': None, 'retrieved': None, 'reranked': None, 'selected': None, 'prompt': None}
|
|
162
|
+
for stage, ranking in quality['stages'].items():
|
|
163
|
+
if ranking is None:
|
|
164
|
+
continue
|
|
165
|
+
stages[stage] = []
|
|
166
|
+
for hit in ranking:
|
|
167
|
+
did = f"{quality['snapshot_sha256']}:{hit['position']}"
|
|
168
|
+
stages[stage].append(did)
|
|
169
|
+
document = {key: hit[key] for key in ('text', 'searchable_text', 'chunk_sha256', 'artifact_sha256', 'chunk_index')}
|
|
170
|
+
document['id'] = did
|
|
171
|
+
_need(did not in documents or documents[did] == document, 'Paired document bytes differ')
|
|
172
|
+
documents[did] = document
|
|
173
|
+
ok = record.get('outcome') == 'response_prepared' and record.get('http_status') == 200
|
|
174
|
+
traces[variant] = {'id': record['request_id'], 'execution_mode': 'recorded_replay', 'stages': stages,
|
|
175
|
+
'response': {'status': 'ok' if ok else 'error', 'text': '', 'abstained': None},
|
|
176
|
+
'operational': {'origin': 'component_reported', 'latency_ms': None,
|
|
177
|
+
'tokens': None, 'cost_usd': None},
|
|
178
|
+
'reflex_server_timing': {'elapsed_ns': record['elapsed_ns'], 'spans': copy.deepcopy(record['spans']),
|
|
179
|
+
'scope': 'Reflex server request with content-capture overhead; excludes generation, TTS and playout',
|
|
180
|
+
'performance_comparison_eligible': False},
|
|
181
|
+
'reflex_quality': copy.deepcopy(quality), 'answer_generation_observed': False}
|
|
182
|
+
cases.append({'id': case_id, 'suite_id': 'reflex-search', 'cluster_id': binding['cluster_id'],
|
|
183
|
+
'question': left['query'], 'documents': list(documents.values()), 'traces': traces,
|
|
184
|
+
'references': {'qrels': None, 'corpus_answerable': None, 'required_atoms': None, 'atom_support': None,
|
|
185
|
+
'required_relations': None, 'relation_support': None, 'reviews': {'baseline': None, 'candidate': None}}})
|
|
186
|
+
_need(all(used[v] == set(source['requests']) for v, source in (('baseline', baseline), ('candidate', candidate))),
|
|
187
|
+
'Explicit case mapping must account for every search request; no silent exclusions')
|
|
188
|
+
return validate_dataset({'schema_version': 'sphynx.rag-diagnostics.dataset.v1', 'release': release,
|
|
189
|
+
'suites': [{'id': 'reflex-search', 'title': 'Recorded Reflex retrieval pipeline'}], 'cases': cases,
|
|
190
|
+
'provenance': {'baseline_sha256': baseline['source_sha256'], 'candidate_sha256': candidate['source_sha256'],
|
|
191
|
+
'independent_clusters_verified': False, 'human_review_status': 'KIV',
|
|
192
|
+
'execution_identities': copy.deepcopy(execution_identities),
|
|
193
|
+
'scope': 'Reflex-owned stages only; no final prompt/answer or population claim'}})
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
"""Readable lifecycle projections without changing saved report payloads."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
from collections.abc import Mapping
|
|
5
|
+
from html import escape
|
|
6
|
+
import json
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def _cell(value):
|
|
10
|
+
if value is None:
|
|
11
|
+
return 'unavailable'
|
|
12
|
+
if isinstance(value, bool):
|
|
13
|
+
return str(value).lower()
|
|
14
|
+
return escape(str(value)).replace('|', '\\|').replace('\r', ' ').replace('\n', ' ')
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _facts(value, prefix=''):
|
|
18
|
+
if isinstance(value, Mapping):
|
|
19
|
+
for key, item in value.items():
|
|
20
|
+
yield from _facts(item, prefix + str(key) + '.')
|
|
21
|
+
elif isinstance(value, (list, tuple)):
|
|
22
|
+
yield prefix.rstrip('.'), _cell(json.dumps(value, ensure_ascii=False, allow_nan=False))
|
|
23
|
+
elif isinstance(value, (str, int, float, bool)) or value is None:
|
|
24
|
+
yield prefix.rstrip('.'), _cell(value)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def lifecycle_markdown(value, context=None):
|
|
28
|
+
"""Display supplied facts; unknown units/coverage remain undeclared."""
|
|
29
|
+
context = context or {}
|
|
30
|
+
lines = ['# Engineering evaluation report', '']
|
|
31
|
+
identity_keys = ('suite_id', 'suite_version', 'run_id', 'grading_id', 'scope', 'mode', 'status', 'trust_tier')
|
|
32
|
+
for key in identity_keys:
|
|
33
|
+
item = context.get(key, value.get(key))
|
|
34
|
+
lines.append(_cell(key.replace('_', ' ').capitalize()) + ': ' + _cell(item))
|
|
35
|
+
for name in ('candidate', 'evaluator', 'execution_policy'):
|
|
36
|
+
supplied = context.get(name, value.get(name))
|
|
37
|
+
if isinstance(supplied, Mapping):
|
|
38
|
+
lines += ['', _cell(name.replace('_', ' ').capitalize()) + ':']
|
|
39
|
+
lines.extend('- ' + _cell(key) + ': ' + item for key, item in _facts(supplied))
|
|
40
|
+
coverage = context.get('coverage')
|
|
41
|
+
if isinstance(coverage, Mapping):
|
|
42
|
+
lines += ['', 'Grading coverage (case/trial units):']
|
|
43
|
+
lines.extend('- ' + _cell(key.replace('_', ' ')) + ': ' + item for key, item in _facts(coverage))
|
|
44
|
+
else:
|
|
45
|
+
lines += ['', 'Required/observed/missing coverage: not declared by this projection.']
|
|
46
|
+
counts = value.get('counts')
|
|
47
|
+
if isinstance(counts, Mapping):
|
|
48
|
+
lines += ['', '| Report count | Value |', '| --- | ---: |']
|
|
49
|
+
lines.extend('| ' + _cell(key) + ' | ' + item + ' |' for key, item in _facts(counts))
|
|
50
|
+
metrics = value.get('metrics')
|
|
51
|
+
if isinstance(metrics, Mapping) and metrics:
|
|
52
|
+
lines += ['', '| Metric | Value | Unit | Direction |', '| --- | ---: | --- | --- |']
|
|
53
|
+
definitions = context.get('metric_definitions', {})
|
|
54
|
+
for name, metric in metrics.items():
|
|
55
|
+
definition = definitions.get(name, {})
|
|
56
|
+
item = metric if isinstance(metric, Mapping) else {'value': metric}
|
|
57
|
+
number = item.get('value')
|
|
58
|
+
lines.append('| ' + ' | '.join(_cell(x) for x in (name,
|
|
59
|
+
number if not isinstance(number, (Mapping, list, tuple)) else 'see native details',
|
|
60
|
+
item.get('unit', definition.get('unit', 'not declared')),
|
|
61
|
+
item.get('direction', definition.get('direction', 'not declared')))) + ' |')
|
|
62
|
+
else:
|
|
63
|
+
lines += ['', 'No common numeric metric is declared. Inspect the permitted assessments and native details.']
|
|
64
|
+
for key in ('reason', 'outcome'):
|
|
65
|
+
if isinstance(value.get(key), (str, bool, int, float)):
|
|
66
|
+
lines += ['', key.capitalize() + ': ' + _cell(value[key])]
|
|
67
|
+
lines += ['', 'Execution/grading completion does not supply a ranking or acceptance verdict.', '',
|
|
68
|
+
'<details><summary>Complete permitted native projection</summary>', '', '<pre>' +
|
|
69
|
+
escape(json.dumps(value, ensure_ascii=False, allow_nan=False, sort_keys=True, indent=2)) +
|
|
70
|
+
'</pre>', '', '</details>', '']
|
|
71
|
+
return '\n'.join(lines)
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
"""Frozen, label-free ranking-family inputs and provenance.
|
|
2
|
+
|
|
3
|
+
These objects declare algorithm and asset identity. They do not grant benchmark
|
|
4
|
+
authority, execute models, fit coefficients, or attest to honest local execution.
|
|
5
|
+
"""
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import hashlib
|
|
9
|
+
import json
|
|
10
|
+
import math
|
|
11
|
+
import re
|
|
12
|
+
from collections.abc import Mapping
|
|
13
|
+
from dataclasses import dataclass
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from types import MappingProxyType
|
|
16
|
+
|
|
17
|
+
from .contracts import ContractError, canonical_json
|
|
18
|
+
from .integration_contracts import CandidateConfiguration, _freeze, _wire, require_digest, value_digest
|
|
19
|
+
|
|
20
|
+
FAMILY_INTERFACE = "sphynx.rerank-family.v1"
|
|
21
|
+
FITTED_SCHEMA = "sphynx.frozen-rerank-state.v1"
|
|
22
|
+
SOURCE_PINS = MappingProxyType({
|
|
23
|
+
"statistical-rerankers.ts": "sha256:28fd9a5e8c6a654fd42db96f3b898b2e1b4e3b8797e39bad41601a7c3edd2130",
|
|
24
|
+
"minilm-treatments.ts": "sha256:a78fbfa25f7884babc623486b3130244e4e888e110bcacae01ae5711f5a2830e",
|
|
25
|
+
"nextgen-rerankers.ts": "sha256:53f4d9e2052309e13678889db28d72a77beae6710a885d49f7b14ed97e0c0189",
|
|
26
|
+
"nextgen-evaluation.ts": "sha256:472424150ecff491a8d710688204f68e3e7c6aa1d4cdde05fdac402dac1e6e5c",
|
|
27
|
+
"metrics.ts": "sha256:57c3207e0bf521ae177f25702603e8556292bb9d818dd2dc2c598878f36d964e",
|
|
28
|
+
"calibrator-fit.ts": "sha256:68f3c33850e6d364b559a5a65bf2656b7c981b6366b16fb2cb9c9e2b8ad823e7",
|
|
29
|
+
"cherry/evaluate_training_free.py": "sha256:c3e9342df622c7860c711afe43336f4318543ac3adaff6cbcbd0dc06e944a805",
|
|
30
|
+
})
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def finite(value, name="signal"):
|
|
34
|
+
if type(value) not in (int, float) or not math.isfinite(value):
|
|
35
|
+
raise ContractError(f"{name} must be finite numeric data")
|
|
36
|
+
return float(value)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def validate_signals(signals, required=()):
|
|
40
|
+
"""Check complete aligned source signals; no relevance labels are accepted."""
|
|
41
|
+
if not isinstance(signals, (list, tuple)) or not 1 <= len(signals) <= 100:
|
|
42
|
+
raise ContractError("ranking families require 1..100 signals")
|
|
43
|
+
forbidden = {"grade", "label", "relevance_grade", "evaluation_grade", "training_grade", "references", "gold"}
|
|
44
|
+
allowed = {"candidate_id", "dense_score", "dense_rank", "minilm_logit", "jina_logit", "bm25_score", "query_text", "passage_text", "feature_values"}
|
|
45
|
+
rows, ids, ranks = [], set(), set()
|
|
46
|
+
for signal in signals:
|
|
47
|
+
if not isinstance(signal, Mapping) or set(signal) & forbidden or set(signal) - allowed:
|
|
48
|
+
raise ContractError("signals must contain only declared label-free fields")
|
|
49
|
+
row = dict(signal)
|
|
50
|
+
identity = row.get("candidate_id")
|
|
51
|
+
if not isinstance(identity, str) or not identity or len(identity) > 256 or identity in ids:
|
|
52
|
+
raise ContractError("candidate IDs must be nonempty and unique")
|
|
53
|
+
ids.add(identity)
|
|
54
|
+
rank = row.get("dense_rank")
|
|
55
|
+
if type(rank) is not int or not 1 <= rank <= len(signals) or rank in ranks:
|
|
56
|
+
raise ContractError("dense ranks must be a permutation of 1..pool size")
|
|
57
|
+
ranks.add(rank)
|
|
58
|
+
for field in ("dense_score", *required):
|
|
59
|
+
if field not in row:
|
|
60
|
+
raise ContractError(f"missing required source signal: {field}")
|
|
61
|
+
row[field] = finite(row[field], field)
|
|
62
|
+
for field in ("minilm_logit", "jina_logit", "bm25_score"):
|
|
63
|
+
if field in row:
|
|
64
|
+
row[field] = finite(row[field], field)
|
|
65
|
+
rows.append(row)
|
|
66
|
+
return rows
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
@dataclass(frozen=True)
|
|
70
|
+
class FrozenFittedState:
|
|
71
|
+
"""An exact external fitted artifact plus truthful caller-supplied lineage.
|
|
72
|
+
|
|
73
|
+
The payload remains external. Binding IDs does not independently prove that
|
|
74
|
+
the declared training/calibration partitions were used or reviewed.
|
|
75
|
+
"""
|
|
76
|
+
method: str
|
|
77
|
+
payload: Mapping
|
|
78
|
+
artifact_digest: str
|
|
79
|
+
provenance: Mapping
|
|
80
|
+
schema_version: str = FITTED_SCHEMA
|
|
81
|
+
|
|
82
|
+
def __post_init__(self):
|
|
83
|
+
if self.schema_version != FITTED_SCHEMA or self.method not in ("T2", "T7", "T14", "T17", "T18", "T19", "T20", "T21", "T22"):
|
|
84
|
+
raise ContractError("unsupported frozen fitted-state schema or method")
|
|
85
|
+
require_digest(self.artifact_digest, "artifact_digest")
|
|
86
|
+
if not isinstance(self.payload, Mapping) or not isinstance(self.provenance, Mapping):
|
|
87
|
+
raise ContractError("fitted payload/provenance must be objects")
|
|
88
|
+
fields = {"source_digest", "fit_cohort_digest", "calibration_cohort_digest", "excluded_evaluation_cohort_digest", "family_digest", "label_mode", "authority", "original_artifact_digest", "payload_digest"}
|
|
89
|
+
if set(self.provenance) != fields:
|
|
90
|
+
raise ContractError("fitted provenance differs from the exact schema")
|
|
91
|
+
for key in fields - {"label_mode", "authority"}:
|
|
92
|
+
require_digest(self.provenance[key], key)
|
|
93
|
+
if self.provenance["label_mode"] not in ("fully_labeled", "known_judgments_only", "positive_unlabeled_diagnostic"):
|
|
94
|
+
raise ContractError("fitted label policy is unknown")
|
|
95
|
+
if self.provenance["authority"] != "caller_declared_unverified":
|
|
96
|
+
raise ContractError("local fitted lineage cannot claim verified authority")
|
|
97
|
+
if self.provenance["payload_digest"] != value_digest(self.payload):
|
|
98
|
+
raise ContractError("fitted payload differs from its binding")
|
|
99
|
+
expected_source=SOURCE_PINS['minilm-treatments.ts' if self.method in ('T2','T7','T14','T17','T18') else 'nextgen-rerankers.ts']
|
|
100
|
+
if self.provenance['source_digest']!=expected_source:
|
|
101
|
+
raise ContractError("fitted source implementation differs from the admitted family")
|
|
102
|
+
object.__setattr__(self, "payload", _freeze(self.payload,reject_secrets=True))
|
|
103
|
+
object.__setattr__(self, "provenance", _freeze(self.provenance,reject_secrets=True))
|
|
104
|
+
|
|
105
|
+
def ensure_evaluation_cohort(self, digest):
|
|
106
|
+
require_digest(digest, "evaluation cohort")
|
|
107
|
+
if digest != self.provenance["excluded_evaluation_cohort_digest"] or digest in (self.provenance["fit_cohort_digest"], self.provenance["calibration_cohort_digest"]):
|
|
108
|
+
raise ContractError("evaluation cohort differs from the declared excluded partition")
|
|
109
|
+
|
|
110
|
+
def to_dict(self):
|
|
111
|
+
return {"schema_version": self.schema_version, "method": self.method, "payload": _wire(self.payload), "artifact_digest": self.artifact_digest, "provenance": _wire(self.provenance)}
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def load_fitted_state(path, expected_sha256):
|
|
115
|
+
"""Read a bounded frozen wrapper; never refit or import artifact code."""
|
|
116
|
+
require_digest(expected_sha256, "fitted state digest")
|
|
117
|
+
raw = Path(path).read_bytes()
|
|
118
|
+
if len(raw) > 8 * 1024 * 1024 or "sha256:" + hashlib.sha256(raw).hexdigest() != expected_sha256:
|
|
119
|
+
raise ContractError("fitted state bytes differ from the supplied digest")
|
|
120
|
+
def unique(pairs):
|
|
121
|
+
result = {}
|
|
122
|
+
for key, value in pairs:
|
|
123
|
+
if key in result: raise ContractError("duplicate fitted-state JSON key")
|
|
124
|
+
result[key] = value
|
|
125
|
+
return result
|
|
126
|
+
try:
|
|
127
|
+
value = json.loads(raw, object_pairs_hook=unique, parse_constant=lambda _: (_ for _ in ()).throw(ContractError("nonfinite fitted-state JSON")))
|
|
128
|
+
except ContractError:
|
|
129
|
+
raise
|
|
130
|
+
except (ValueError, UnicodeError):
|
|
131
|
+
raise ContractError("invalid fitted-state JSON") from None
|
|
132
|
+
if not isinstance(value, dict) or set(value) != {"schema_version", "method", "payload", "provenance"}:
|
|
133
|
+
raise ContractError("fitted-state wrapper fields differ")
|
|
134
|
+
return FrozenFittedState(value["method"], value["payload"], expected_sha256, value["provenance"], value["schema_version"])
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def family_configuration(method, implementation_digest, *, parameters, scorers=None, fitted_state=None):
|
|
138
|
+
dependencies = {}
|
|
139
|
+
for name, scorer in (scorers or {}).items():
|
|
140
|
+
config = getattr(scorer, "configuration", None)
|
|
141
|
+
if not isinstance(config, CandidateConfiguration):
|
|
142
|
+
raise ContractError("every source scorer must expose its immutable CandidateConfiguration")
|
|
143
|
+
dependencies[name] = config.to_dict()
|
|
144
|
+
# Original arm names contain '/' (for example MiniLM-W90/10). Keep that
|
|
145
|
+
# exact name in parameters; the SDK's stable identifier grammar uses '-'.
|
|
146
|
+
return CandidateConfiguration("rerank-family:" + method.replace("/", "-"), FAMILY_INTERFACE, implementation_digest,
|
|
147
|
+
{"method": method, "method_version": "v1", "reference_sources": dict(SOURCE_PINS),
|
|
148
|
+
"parameters": parameters, "source_scorers": dependencies,
|
|
149
|
+
"fitted_state": None if fitted_state is None else {"artifact_digest": fitted_state.artifact_digest, "provenance": _wire(fitted_state.provenance)}},
|
|
150
|
+
"recorded" if dependencies and all(value["execution_mode"] == "recorded" for value in dependencies.values()) else "trusted_local")
|