sable-sphynx 0.7.0.dev0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (174) hide show
  1. sable_sphynx-0.7.0.dev0.dist-info/METADATA +152 -0
  2. sable_sphynx-0.7.0.dev0.dist-info/RECORD +174 -0
  3. sable_sphynx-0.7.0.dev0.dist-info/WHEEL +5 -0
  4. sable_sphynx-0.7.0.dev0.dist-info/top_level.txt +1 -0
  5. sphynx/__init__.py +469 -0
  6. sphynx/__init__.pyi +376 -0
  7. sphynx/_analytical_client.py +77 -0
  8. sphynx/_analytical_values.py +252 -0
  9. sphynx/_audit_client.py +173 -0
  10. sphynx/_comparison_adapters.py +113 -0
  11. sphynx/_componentized_full_rag.py +34 -0
  12. sphynx/_composed_pipeline.py +109 -0
  13. sphynx/_full_rag_grading.py +218 -0
  14. sphynx/_full_rag_trace.py +221 -0
  15. sphynx/_local_eval.py +268 -0
  16. sphynx/_pipeline_core.py +243 -0
  17. sphynx/_pr_client.py +237 -0
  18. sphynx/_rag_diagnostics_calibration.py +285 -0
  19. sphynx/_rag_diagnostics_catalog.py +39 -0
  20. sphynx/_rag_diagnostics_common.py +26 -0
  21. sphynx/_rag_diagnostics_contracts.py +186 -0
  22. sphynx/_rag_diagnostics_engine.py +243 -0
  23. sphynx/_rag_diagnostics_importers.py +242 -0
  24. sphynx/_rag_diagnostics_ranking.py +81 -0
  25. sphynx/_rag_diagnostics_semantic.py +325 -0
  26. sphynx/_rag_ranking.py +149 -0
  27. sphynx/_speech_benchmark_scoring.py +140 -0
  28. sphynx/_speech_benchmark_sources.py +275 -0
  29. sphynx/_speech_complete_scoring.py +218 -0
  30. sphynx/_speech_complete_source.py +118 -0
  31. sphynx/_speech_kido.py +294 -0
  32. sphynx/_speech_scoring.py +77 -0
  33. sphynx/_track_capture.py +213 -0
  34. sphynx/analytical_views.py +515 -0
  35. sphynx/assessment_contracts.py +378 -0
  36. sphynx/assessment_selection.py +57 -0
  37. sphynx/audit_access.py +245 -0
  38. sphynx/audit_contracts.py +204 -0
  39. sphynx/audit_requests.py +180 -0
  40. sphynx/authoring.py +188 -0
  41. sphynx/benchmarks.py +568 -0
  42. sphynx/candidates.py +62 -0
  43. sphynx/catalog.py +272 -0
  44. sphynx/chunking.py +372 -0
  45. sphynx/clickhouse_reader.py +169 -0
  46. sphynx/client.py +486 -0
  47. sphynx/comparison_core.py +149 -0
  48. sphynx/comparison_views.py +223 -0
  49. sphynx/components.py +1661 -0
  50. sphynx/contracts.py +514 -0
  51. sphynx/customer_rerank.py +299 -0
  52. sphynx/customer_retrieval.py +146 -0
  53. sphynx/data/benchmark-releases.v1.json +164 -0
  54. sphynx/data/public-benchmark-release.v1.json +354 -0
  55. sphynx/data/rag-diagnostics-synthetic.v1.json +6982 -0
  56. sphynx/data/rag-paper-references.v1.json +176 -0
  57. sphynx/data/rag-paper-schemas.v1.json +1069 -0
  58. sphynx/data/rag-rerank-v1.json +507 -0
  59. sphynx/dataset_catalog_data.py +1214 -0
  60. sphynx/dataset_readiness.py +138 -0
  61. sphynx/dataset_v3.py +999 -0
  62. sphynx/demo.py +130 -0
  63. sphynx/demo_candidates/__init__.py +1 -0
  64. sphynx/demo_candidates/dense.py +11 -0
  65. sphynx/demo_candidates/lexical.py +21 -0
  66. sphynx/diagnostics.py +207 -0
  67. sphynx/documentation.py +737 -0
  68. sphynx/documentation_candidate.py +939 -0
  69. sphynx/evals.py +538 -0
  70. sphynx/evaluation_protocols.py +124 -0
  71. sphynx/evaluator_bindings.py +370 -0
  72. sphynx/evidence.py +352 -0
  73. sphynx/execution_capabilities.py +109 -0
  74. sphynx/fast_search.py +285 -0
  75. sphynx/full_rag.py +341 -0
  76. sphynx/hosted/__init__.py +18 -0
  77. sphynx/hosted/benchmark_intake.py +218 -0
  78. sphynx/hosted/contracts.py +361 -0
  79. sphynx/hosted/model_admission.py +130 -0
  80. sphynx/hosted/model_artifacts.py +298 -0
  81. sphynx/hosted/model_contracts.py +63 -0
  82. sphynx/hosted/model_gcs.py +181 -0
  83. sphynx/hosted/model_history.py +84 -0
  84. sphynx/hosted/model_session.py +233 -0
  85. sphynx/hosted/public_suite.py +74 -0
  86. sphynx/hosted/registry.py +97 -0
  87. sphynx/hosted/revisions.py +126 -0
  88. sphynx/hosted/sandbox.py +105 -0
  89. sphynx/hosted/submission.py +287 -0
  90. sphynx/hosted_workflow.py +451 -0
  91. sphynx/http_transport.py +86 -0
  92. sphynx/integrated_rag.py +352 -0
  93. sphynx/integrated_workflow.py +552 -0
  94. sphynx/integration_contracts.py +474 -0
  95. sphynx/invocation.py +791 -0
  96. sphynx/invocation_candidates.py +392 -0
  97. sphynx/invocation_catalog_data.py +40 -0
  98. sphynx/learned_catalog_data.py +30 -0
  99. sphynx/learned_rerank.py +267 -0
  100. sphynx/legacy_full_rag.py +67 -0
  101. sphynx/lifecycle.py +885 -0
  102. sphynx/lifecycle_workflow.py +490 -0
  103. sphynx/metric_views.py +474 -0
  104. sphynx/metrics.py +282 -0
  105. sphynx/model_adapters.py +438 -0
  106. sphynx/model_catalog_data.py +30 -0
  107. sphynx/official_benchmarks.py +385 -0
  108. sphynx/packs.py +258 -0
  109. sphynx/pipeline.py +313 -0
  110. sphynx/pipeline_workflow.py +551 -0
  111. sphynx/pr_evaluations.py +560 -0
  112. sphynx/progress.py +89 -0
  113. sphynx/public_bundle.py +173 -0
  114. sphynx/public_eval.py +62 -0
  115. sphynx/public_rerank.py +355 -0
  116. sphynx/py.typed +0 -0
  117. sphynx/pytest_plugin.py +62 -0
  118. sphynx/question_robustness.py +437 -0
  119. sphynx/rag.py +329 -0
  120. sphynx/rag_diagnostics.py +309 -0
  121. sphynx/rag_papers/__init__.py +29 -0
  122. sphynx/rag_papers/__main__.py +45 -0
  123. sphynx/rag_papers/candidates.py +19 -0
  124. sphynx/rag_papers/contracts.py +282 -0
  125. sphynx/rag_papers/datasets.py +291 -0
  126. sphynx/rag_papers/experiments.py +116 -0
  127. sphynx/rag_papers/judges.py +308 -0
  128. sphynx/rag_papers/registry.py +33 -0
  129. sphynx/rag_papers/runner.py +176 -0
  130. sphynx/rag_papers/sable.py +135 -0
  131. sphynx/rag_papers/schemas.py +12 -0
  132. sphynx/rag_papers/scoring.py +249 -0
  133. sphynx/rag_papers/semantic.py +288 -0
  134. sphynx/rag_papers/trec.py +80 -0
  135. sphynx/recorded_scores.py +352 -0
  136. sphynx/records.py +124 -0
  137. sphynx/reflex_traces.py +193 -0
  138. sphynx/report_rendering.py +71 -0
  139. sphynx/rerank_family_contracts.py +150 -0
  140. sphynx/rerank_workflow.py +424 -0
  141. sphynx/research_candidate.py +190 -0
  142. sphynx/results.py +852 -0
  143. sphynx/retrieval.py +750 -0
  144. sphynx/retrieval_candidates.py +295 -0
  145. sphynx/retrieval_catalog_data.py +19 -0
  146. sphynx/retrieval_recordings.py +125 -0
  147. sphynx/reviewer_audit.py +266 -0
  148. sphynx/routing_policies.py +375 -0
  149. sphynx/runner.py +544 -0
  150. sphynx/sable_rerank.py +489 -0
  151. sphynx/score_reliability.py +393 -0
  152. sphynx/speech_benchmarks.py +204 -0
  153. sphynx/speech_campaigns.py +378 -0
  154. sphynx/speech_complete.py +236 -0
  155. sphynx/speech_contracts.py +225 -0
  156. sphynx/speech_evidence.py +429 -0
  157. sphynx/speech_listening.py +115 -0
  158. sphynx/speech_multilingual.py +71 -0
  159. sphynx/speech_tts.py +479 -0
  160. sphynx/sql/001_sync_v1.sql +30 -0
  161. sphynx/stage_contracts.py +199 -0
  162. sphynx/statistical_catalog_data.py +53 -0
  163. sphynx/statistical_rerank.py +232 -0
  164. sphynx/sync.py +797 -0
  165. sphynx/workbench.py +378 -0
  166. sphynx/workbench_catalog_data.py +13 -0
  167. sphynx/workbench_tracks.py +78 -0
  168. sphynx/workflow_async.py +197 -0
  169. sphynx/workflow_contracts.py +594 -0
  170. sphynx/workflow_decisions.py +211 -0
  171. sphynx/workflow_defaults.py +16 -0
  172. sphynx/workflow_extensions.py +212 -0
  173. sphynx/workflow_registry.py +513 -0
  174. sphynx/workflows.py +792 -0
sphynx/__init__.pyi ADDED
@@ -0,0 +1,376 @@
1
+ # Generated by tools/check_public_types.py; keep runtime imports lazy.
2
+ # Explicit aliases expose the same objects to static editors.
3
+
4
+ from .workflows import Admission as Admission
5
+ from .workflow_contracts import AdmissionBinding as AdmissionBinding
6
+ from ._analytical_client import AnalyticalReadError as AnalyticalReadError
7
+ from .evals import ArtifactBinding as ArtifactBinding
8
+ from .evals import ArtifactReference as ArtifactReference
9
+ from .assessment_contracts import Assessment as Assessment
10
+ from .assessment_contracts import AssessmentContractError as AssessmentContractError
11
+ from .assessment_selection import AssessmentSelector as AssessmentSelector
12
+ from .benchmarks import AssetRef as AssetRef
13
+ from .workflow_async import AsyncWorkflowBackend as AsyncWorkflowBackend
14
+ from .lifecycle import AttemptOutput as AttemptOutput
15
+ from ._analytical_values import AuditProjectionSummary as AuditProjectionSummary
16
+ from ._audit_client import AuditRequestError as AuditRequestError
17
+ from .workflow_contracts import BackendDescriptor as BackendDescriptor
18
+ from .audit_contracts import BankRelease as BankRelease
19
+ from .audit_requests import BankReleaseDeclaration as BankReleaseDeclaration
20
+ from .audit_requests import BankReleaseTransition as BankReleaseTransition
21
+ from .components import BatchAlternative as BatchAlternative
22
+ from .components import BatchConfiguration as BatchConfiguration
23
+ from .components import BatchPlan as BatchPlan
24
+ from .benchmarks import BenchmarkCase as BenchmarkCase
25
+ from .benchmarks import BenchmarkContractError as BenchmarkContractError
26
+ from .benchmarks import BenchmarkEligibility as BenchmarkEligibility
27
+ from .benchmarks import BenchmarkRegistry as BenchmarkRegistry
28
+ from .benchmarks import BenchmarkRelease as BenchmarkRelease
29
+ from .analytical_views import BoardCoverage as BoardCoverage
30
+ from .analytical_views import BoardEntry as BoardEntry
31
+ from .analytical_views import BoardRanking as BoardRanking
32
+ from .authoring import CallableCandidate as CallableCandidate
33
+ from .lifecycle import CancellationToken as CancellationToken
34
+ from .packs import CandidateCheck as CandidateCheck
35
+ from .integration_contracts import CandidateConfiguration as CandidateConfiguration
36
+ from .contracts import CandidateRequest as CandidateRequest
37
+ from .contracts import CandidateResponse as CandidateResponse
38
+ from .contracts import CandidateScore as CandidateScore
39
+ from .contracts import CandidateSpec as CandidateSpec
40
+ from .records import CaptureIdentity as CaptureIdentity
41
+ from .workflow_contracts import CaseProvider as CaseProvider
42
+ from .contracts import CaseReferences as CaseReferences
43
+ from .assessment_contracts import CaseView as CaseView
44
+ from .clickhouse_reader import ClickHouseReader as ClickHouseReader
45
+ from .client import Client as Client
46
+ from .results import Comparison as Comparison
47
+ from .integration_contracts import ComparisonIdentity as ComparisonIdentity
48
+ from .comparison_views import ComparisonMetric as ComparisonMetric
49
+ from .comparison_views import ComparisonPair as ComparisonPair
50
+ from ._analytical_values import ComparisonProjectionSummary as ComparisonProjectionSummary
51
+ from .client import ComparisonRef as ComparisonRef
52
+ from .comparison_views import ComparisonView as ComparisonView
53
+ from .contracts import ContractError as ContractError
54
+ from .contracts import CorpusVersions as CorpusVersions
55
+ from .workflow_contracts import Criterion as Criterion
56
+ from .workflow_contracts import Dataset as Dataset
57
+ from .workflow_contracts import DatasetDefinition as DatasetDefinition
58
+ from .workflow_decisions import DecisionContext as DecisionContext
59
+ from .workflow_decisions import DecisionOutcome as DecisionOutcome
60
+ from .workflow_decisions import DecisionPolicyError as DecisionPolicyError
61
+ from .diagnostics import Diagnostic as Diagnostic
62
+ from .lifecycle import DriverContext as DriverContext
63
+ from .lifecycle import DriverDescriptor as DriverDescriptor
64
+ from .lifecycle import DriverFailure as DriverFailure
65
+ from .records import EVALUATION_SCHEMA_VERSION as EVALUATION_SCHEMA_VERSION
66
+ from .records import EVIDENCE_SCHEMA_VERSION as EVIDENCE_SCHEMA_VERSION
67
+ from .analytical_views import EvaluationBoardPage as EvaluationBoardPage
68
+ from .evals import EvaluationContractError as EvaluationContractError
69
+ from .evals import EvaluationDefinition as EvaluationDefinition
70
+ from .records import EvaluationIdentity as EvaluationIdentity
71
+ from .evaluation_protocols import EvaluationProtocol as EvaluationProtocol
72
+ from .results import EvaluationRef as EvaluationRef
73
+ from .evals import EvaluationRegistry as EvaluationRegistry
74
+ from .evaluator_bindings import EvaluatorBinding as EvaluatorBinding
75
+ from .assessment_contracts import EvaluatorContext as EvaluatorContext
76
+ from .workflow_contracts import EvaluatorDefinition as EvaluatorDefinition
77
+ from .records import EvidenceCapability as EvidenceCapability
78
+ from .contracts import EvidenceGroup as EvidenceGroup
79
+ from .assessment_contracts import EvidenceView as EvidenceView
80
+ from .execution_capabilities import ExecutionPath as ExecutionPath
81
+ from .packs import ExecutionPolicy as ExecutionPolicy
82
+ from .client import Experiment as Experiment
83
+ from .workflow_extensions import ExtensionError as ExtensionError
84
+ from .contracts import FrozenCandidate as FrozenCandidate
85
+ from .rerank_family_contracts import FrozenFittedState as FrozenFittedState
86
+ from .contracts import FrozenPoolCase as FrozenPoolCase
87
+ from .lifecycle import GradingContext as GradingContext
88
+ from .workflows import GradingPass as GradingPass
89
+ from .results import EvaluationRef as GradingPassRef
90
+ from .lifecycle import GradingPolicy as GradingPolicy
91
+ from .workflows import GradingRequestError as GradingRequestError
92
+ from .workflow_async import GradingSnapshot as GradingSnapshot
93
+ from .client import HostedClientError as HostedClientError
94
+ from .hosted_workflow import HostedComparison as HostedComparison
95
+ from .client import HostedEvaluation as HostedEvaluation
96
+ from .client import HostedGradingPass as HostedGradingPass
97
+ from .client import HostedReport as HostedReport
98
+ from .client import HostedResult as HostedResult
99
+ from .client import HostedRun as HostedRun
100
+ from .client import HostedSubmission as HostedSubmission
101
+ from .hosted_workflow import HostedWorkflowBackend as HostedWorkflowBackend
102
+ from .contracts import INTERFACE_VERSION as INTERFACE_VERSION
103
+ from .evaluator_bindings import InputRequirement as InputRequirement
104
+ from .integrated_workflow import IntegratedBackend as IntegratedBackend
105
+ from .integrated_rag import IntegratedCandidate as IntegratedCandidate
106
+ from .integrated_rag import IntegratedDriver as IntegratedDriver
107
+ from .invocation import InvocationBank as InvocationBank
108
+ from .invocation import InvocationComparison as InvocationComparison
109
+ from .invocation import InvocationError as InvocationError
110
+ from .invocation import InvocationEvaluationRef as InvocationEvaluationRef
111
+ from .invocation import InvocationEvaluationRef as InvocationGradingPassRef
112
+ from .invocation import InvocationPack as InvocationPack
113
+ from .invocation import InvocationRunRef as InvocationRunRef
114
+ from .invocation import InvocationScorecard as InvocationScorecard
115
+ from .benchmarks import LabelCase as LabelCase
116
+ from .benchmarks import LabelLineage as LabelLineage
117
+ from .benchmarks import LabelSetRelease as LabelSetRelease
118
+ from .lifecycle_workflow import LifecycleBackend as LifecycleBackend
119
+ from .lifecycle import LifecycleDriver as LifecycleDriver
120
+ from .lifecycle import LifecycleError as LifecycleError
121
+ from .lifecycle import LifecycleGradingRef as LifecycleGradingRef
122
+ from .lifecycle_workflow import LifecycleProjection as LifecycleProjection
123
+ from .lifecycle import LifecycleRunRef as LifecycleRunRef
124
+ from ._local_eval import LocalEvaluationRef as LocalEvaluationRef
125
+ from ._local_eval import LocalEvaluationRef as LocalGradingPassRef
126
+ from ._local_eval import LocalRunRef as LocalRunRef
127
+ from ._local_eval import LocalScorecard as LocalScorecard
128
+ from .records import METRIC_DEFINITION_SCHEMA_VERSION as METRIC_DEFINITION_SCHEMA_VERSION
129
+ from .metric_views import MetricCard as MetricCard
130
+ from .records import MetricDefinition as MetricDefinition
131
+ from ._analytical_values import MetricProjectionSummary as MetricProjectionSummary
132
+ from .workflow_contracts import MetricSpec as MetricSpec
133
+ from .metric_views import MetricSummary as MetricSummary
134
+ from .question_robustness import ObservedSearch as ObservedSearch
135
+ from .question_robustness import PILOT_MANIFEST_SHA256 as PILOT_MANIFEST_SHA256
136
+ from .public_rerank import PUBLIC_POOL_SHA256 as PUBLIC_POOL_SHA256
137
+ from .public_rerank import PUBLIC_RERANK_PACK_ID as PUBLIC_RERANK_PACK_ID
138
+ from .packs import PackDescriptor as PackDescriptor
139
+ from .workflow_extensions import PackagedExtension as PackagedExtension
140
+ from .workflow_contracts import PayloadSchema as PayloadSchema
141
+ from .workflows import PendingWorkError as PendingWorkError
142
+ from .pipeline_workflow import PipelineBackend as PipelineBackend
143
+ from .pipeline_workflow import PipelineCandidate as PipelineCandidate
144
+ from .pipeline_workflow import PipelineSource as PipelineSource
145
+ from .workflows import Plan as Plan
146
+ from .workflow_contracts import PlanBinding as PlanBinding
147
+ from ._pr_client import PrEvaluation as PrEvaluation
148
+ from ._pr_client import PrEvaluationError as PrEvaluationError
149
+ from .pr_evaluations import PrEvaluationRequest as PrEvaluationRequest
150
+ from .pr_evaluations import PrEvaluationResult as PrEvaluationResult
151
+ from .sable_rerank import PrivateEvaluation as PrivateEvaluation
152
+ from .sable_rerank import PrivatePack as PrivatePack
153
+ from .progress import ProgressView as ProgressView
154
+ from ._analytical_values import ProjectionCoverage as ProjectionCoverage
155
+ from .analytical_views import ProjectionEventRef as ProjectionEventRef
156
+ from .analytical_views import ProjectionPage as ProjectionPage
157
+ from .analytical_views import ProjectionProof as ProjectionProof
158
+ from .analytical_views import ProjectionResult as ProjectionResult
159
+ from .analytical_views import ProjectionRow as ProjectionRow
160
+ from .analytical_views import ProjectionSelection as ProjectionSelection
161
+ from .public_bundle import PublicRerankBundle as PublicRerankBundle
162
+ from .public_rerank import PublicRerankPack as PublicRerankPack
163
+ from .hosted_workflow import PullRequestCandidate as PullRequestCandidate
164
+ from .question_robustness import QUESTION_ROBUSTNESS_INTERFACE as QUESTION_ROBUSTNESS_INTERFACE
165
+ from .question_robustness import QUESTION_ROBUSTNESS_PACK_ID as QUESTION_ROBUSTNESS_PACK_ID
166
+ from .question_robustness import QuestionRobustnessPack as QuestionRobustnessPack
167
+ from .recorded_scores import RECORDED_SCORE_SCHEMA as RECORDED_SCORE_SCHEMA
168
+ from .contracts import REQUEST_SCHEMA_VERSION as REQUEST_SCHEMA_VERSION
169
+ from .contracts import RESPONSE_SCHEMA_VERSION as RESPONSE_SCHEMA_VERSION
170
+ from .retrieval import RETRIEVAL_INTERFACE_VERSION as RETRIEVAL_INTERFACE_VERSION
171
+ from .retrieval import RETRIEVAL_PACK_ID as RETRIEVAL_PACK_ID
172
+ from .clickhouse_reader import ReadOnlyClickHouseConfig as ReadOnlyClickHouseConfig
173
+ from .clickhouse_reader import ReadOnlyClickHouseError as ReadOnlyClickHouseError
174
+ from .retrieval_candidates import RecordedHybridSignals as RecordedHybridSignals
175
+ from .workflow_contracts import ReferenceBinding as ReferenceBinding
176
+ from .contracts import ReferenceCandidate as ReferenceCandidate
177
+ from .score_reliability import ReliabilityReference as ReliabilityReference
178
+ from .workflows import Report as Report
179
+ from .evaluator_bindings import ReportAssessment as ReportAssessment
180
+ from .workflow_contracts import ReportProjection as ReportProjection
181
+ from ._analytical_values import ReportProjectionSummary as ReportProjectionSummary
182
+ from .workflow_decisions import ReportSnapshot as ReportSnapshot
183
+ from .workflow_contracts import ResolvedBinding as ResolvedBinding
184
+ from .evals import ResolvedInvocation as ResolvedInvocation
185
+ from .workflows import ResolvedPlan as ResolvedPlan
186
+ from .retrieval import RetrievalEvaluation as RetrievalEvaluation
187
+ from .retrieval import RetrievalPack as RetrievalPack
188
+ from .retrieval import RetrievalRun as RetrievalRun
189
+ from .audit_requests import ReviewAdjudication as ReviewAdjudication
190
+ from .audit_contracts import ReviewContribution as ReviewContribution
191
+ from .audit_contracts import ReviewEvent as ReviewEvent
192
+ from .audit_access import ReviewHistoryPage as ReviewHistoryPage
193
+ from .audit_access import ReviewOperationReceipt as ReviewOperationReceipt
194
+ from .benchmarks import ReviewReceipt as ReviewReceipt
195
+ from .audit_contracts import ReviewTarget as ReviewTarget
196
+ from .audit_access import ReviewTargetPage as ReviewTargetPage
197
+ from .reviewer_audit import ReviewerAudit as ReviewerAudit
198
+ from .workflows import Run as Run
199
+ from .lifecycle import RunLimits as RunLimits
200
+ from .lifecycle import RunPolicy as RunPolicy
201
+ from .evidence import RunRef as RunRef
202
+ from .sable_rerank import SABLE_PACK_ID as SABLE_PACK_ID
203
+ from .integration_contracts import ScoreField as ScoreField
204
+ from .integration_contracts import ScoreSemantics as ScoreSemantics
205
+ from .results import Scorecard as Scorecard
206
+ from .benchmarks import SliceParameters as SliceParameters
207
+ from .stage_contracts import StageContractError as StageContractError
208
+ from .stage_contracts import StageDefinition as StageDefinition
209
+ from .stage_contracts import StageEvidence as StageEvidence
210
+ from .stage_contracts import StageEvidenceBackend as StageEvidenceBackend
211
+ from .stage_contracts import StageReplacement as StageReplacement
212
+ from .stage_contracts import StageSelector as StageSelector
213
+ from .sync import StorageConfig as StorageConfig
214
+ from .workflows import Submission as Submission
215
+ from .workflow_async import SubmissionError as SubmissionError
216
+ from .workflow_async import SubmissionIntent as SubmissionIntent
217
+ from .workflow_async import SubmissionSnapshot as SubmissionSnapshot
218
+ from .workflows import Suite as Suite
219
+ from .workflow_contracts import SuiteDefinition as SuiteDefinition
220
+ from .sync import SyncResult as SyncResult
221
+ from .audit_access import TargetReviewPage as TargetReviewPage
222
+ from .integration_contracts import TrackEvaluationRef as TrackGradingPassRef
223
+ from .workflow_decisions import Verdict as Verdict
224
+ from .evaluation_protocols import VersionedIdentity as VersionedIdentity
225
+ from .workflow_contracts import WorkflowBackend as WorkflowBackend
226
+ from .workflow_contracts import WorkflowContractError as WorkflowContractError
227
+ from .workflows import WorkflowError as WorkflowError
228
+ from .workflow_registry import WorkflowRegistry as WorkflowRegistry
229
+ from .workflow_registry import WorkflowRegistryError as WorkflowRegistryError
230
+ from .workflows import WorkflowTimeoutError as WorkflowTimeoutError
231
+ from .lifecycle import assess_lifecycle as assess_lifecycle
232
+ from .reviewer_audit import audit_invocation_reviewer as audit_invocation_reviewer
233
+ from .legacy_full_rag import audit_legacy_cherry_full_rag as audit_legacy_cherry_full_rag
234
+ from .rag_diagnostics import audit_rag_judge as audit_rag_judge
235
+ from .contracts import build_candidate_request as build_candidate_request
236
+ from .sable_rerank import build_cherry_development_dataset as build_cherry_development_dataset
237
+ from .customer_rerank import build_customer_development_dataset as build_customer_development_dataset
238
+ from .customer_retrieval import build_customer_retrieval_dataset as build_customer_retrieval_dataset
239
+ from .sable_rerank import build_exact_sable_dataset as build_exact_sable_dataset
240
+ from .hosted_workflow import build_hosted_registry as build_hosted_registry
241
+ from .integrated_workflow import build_integrated_registry as build_integrated_registry
242
+ from .pipeline_workflow import build_pipeline_registry as build_pipeline_registry
243
+ from .public_rerank import build_public_rerank_request as build_public_rerank_request
244
+ from .question_robustness import build_question_robustness_request as build_question_robustness_request
245
+ from .question_robustness import build_question_robustness_response as build_question_robustness_response
246
+ from .retrieval import build_retrieval_request as build_retrieval_request
247
+ from .evals import canonical_contract_json as canonical_contract_json
248
+ from .contracts import canonical_json as canonical_json
249
+ from .retrieval import capture_retrieval as capture_retrieval
250
+ from .packs import check_candidate as check_candidate
251
+ from .results import compare as compare
252
+ from .full_rag import compare_full_rag as compare_full_rag
253
+ from .full_rag import compare_full_rag_contexts as compare_full_rag_contexts
254
+ from .invocation import compare_invocation as compare_invocation
255
+ from .pipeline import compare_pipeline as compare_pipeline
256
+ from .sable_rerank import compare_private_rerank as compare_private_rerank
257
+ from .public_rerank import compare_public_rerank as compare_public_rerank
258
+ from .question_robustness import compare_question_robustness as compare_question_robustness
259
+ from .retrieval import compare_retrieval as compare_retrieval
260
+ from .score_reliability import compare_score_reliability as compare_score_reliability
261
+ from .workflow_contracts import contract_digest as contract_digest
262
+ from .full_rag import create_full_rag_review_template as create_full_rag_review_template
263
+ from .learned_rerank import create_learned_scorer as create_learned_scorer
264
+ from .model_adapters import create_model_scorer as create_model_scorer
265
+ from .question_robustness import create_query_review_template as create_query_review_template
266
+ from .rag_diagnostics import create_rag_diagnostics_review_template as create_rag_diagnostics_review_template
267
+ from .recorded_scores import create_recorded_scorer as create_recorded_scorer
268
+ from .statistical_rerank import create_statistical_scorer as create_statistical_scorer
269
+ from .workflow_decisions import decide_report as decide_report
270
+ from .benchmarks import default_registry as default_benchmark_registry
271
+ from .evals import default_evaluation_registry as default_evaluation_registry
272
+ from .workflow_defaults import default_workflow_registry as default_workflow_registry
273
+ from .evals import describe_eval as describe_eval
274
+ from .workflow_extensions import discover_extensions as discover_extensions
275
+ from .workflow_extensions import enable_extension as enable_extension
276
+ from .rag_diagnostics import estimate_rag_ppi_mean as estimate_rag_ppi_mean
277
+ from .score_reliability import evaluate_fitted_calibrator as evaluate_fitted_calibrator
278
+ from .authoring import evaluation_suite as evaluation_suite
279
+ from .lifecycle import execute_lifecycle as execute_lifecycle
280
+ from .diagnostics import explain_error as explain_error
281
+ from .catalog import get_dataset as get_dataset
282
+ from .model_adapters import get_model_spec as get_model_spec
283
+ from .packs import get_pack as get_pack
284
+ from .results import grade as grade
285
+ from .full_rag import grade_full_rag as grade_full_rag
286
+ from .invocation import grade_invocation as grade_invocation
287
+ from .pipeline import grade_pipeline as grade_pipeline
288
+ from .sable_rerank import grade_private_rerank as grade_private_rerank
289
+ from .public_rerank import grade_public_rerank as grade_public_rerank
290
+ from .question_robustness import grade_question_robustness as grade_question_robustness
291
+ from .rag_diagnostics import grade_rag_diagnostics as grade_rag_diagnostics
292
+ from .retrieval import grade_retrieval as grade_retrieval
293
+ from .score_reliability import grade_score_reliability as grade_score_reliability
294
+ from .retrieval_recordings import import_hybrid_recordings as import_hybrid_recordings
295
+ from .rag_diagnostics import import_rag_diagnostics_beir as import_rag_diagnostics_beir
296
+ from .recorded_scores import import_recorded_scores as import_recorded_scores
297
+ from .catalog import list_candidates as list_candidates
298
+ from .catalog import list_datasets as list_datasets
299
+ from .evals import list_evals as list_evals
300
+ from .invocation import list_invocation_banks as list_invocation_banks
301
+ from .metric_views import list_metric_tracks as list_metric_tracks
302
+ from .metric_views import list_metrics as list_metrics
303
+ from .packs import list_packs as list_packs
304
+ from .public_rerank import list_public_pool_releases as list_public_pool_releases
305
+ from .rag_diagnostics import list_rag_diagnostics_metrics as list_rag_diagnostics_metrics
306
+ from .components import load_batch_manifest as load_batch_manifest
307
+ from .rerank_family_contracts import load_fitted_state as load_fitted_state
308
+ from .invocation import load_invocation_pack as load_invocation_pack
309
+ from .sable_rerank import load_private_pack as load_private_pack
310
+ from .public_rerank import load_public_pool_release as load_public_pool_release
311
+ from .public_bundle import load_public_rerank_bundle as load_public_rerank_bundle
312
+ from .public_rerank import load_public_rerank_pack as load_public_rerank_pack
313
+ from .question_robustness import load_question_robustness_pack as load_question_robustness_pack
314
+ from .rag_diagnostics import load_rag_diagnostics_dataset as load_rag_diagnostics_dataset
315
+ from .rag_diagnostics import load_rag_diagnostics_demo as load_rag_diagnostics_demo
316
+ from .rag_diagnostics import load_rag_diagnostics_report as load_rag_diagnostics_report
317
+ from .score_reliability import load_reliability_reference as load_reliability_reference
318
+ from .research_candidate import load_research_candidate as load_research_candidate
319
+ from .retrieval import load_retrieval_pack as load_retrieval_pack
320
+ from .research_candidate import make_research_candidate as make_research_candidate
321
+ from .retrieval_candidates import make_retrieval_candidate as make_retrieval_candidate
322
+ from .results import open_evaluation as open_evaluation
323
+ from .lifecycle import open_lifecycle_run as open_lifecycle_run
324
+ from ._local_eval import open_local_evaluation as open_local_evaluation
325
+ from ._local_eval import open_local_run as open_local_run
326
+ from .evidence import open_run as open_run
327
+ from .contracts import pool_digest as pool_digest
328
+ from .model_adapters import preflight_model as preflight_model
329
+ from .public_bundle import prepare_public_rerank_bundle as prepare_public_rerank_bundle
330
+ from .rag_diagnostics import rag_candidate_view as rag_candidate_view
331
+ from .rag_diagnostics import rag_diagnostics_capabilities as rag_diagnostics_capabilities
332
+ from .contracts import ranked_candidate_ids as ranked_candidate_ids
333
+ from .sync import read_committed_projection as read_committed_projection
334
+ from .pipeline_workflow import register_pipeline_backend as register_pipeline_backend
335
+ from .rag_diagnostics import render_rag_diagnostics_report as render_rag_diagnostics_report
336
+ from .recorded_scores import replay_rerank_scores as replay_rerank_scores
337
+ from .results import report as report
338
+ from .score_reliability import report_fitted_calibrator as report_fitted_calibrator
339
+ from .full_rag import report_full_rag as report_full_rag
340
+ from .invocation import report_invocation as report_invocation
341
+ from .pipeline import report_pipeline as report_pipeline
342
+ from .public_rerank import report_public_rerank as report_public_rerank
343
+ from .question_robustness import report_question_robustness as report_question_robustness
344
+ from .evals import resolve_eval as resolve_eval
345
+ from .retrieval_candidates import retrieval_configuration as retrieval_configuration
346
+ from .runner import run as run
347
+ from .workbench import run_batch as run_batch
348
+ from .full_rag import run_full_rag as run_full_rag
349
+ from .integrated_rag import run_integrated as run_integrated
350
+ from .invocation import run_invocation as run_invocation
351
+ from .pipeline import run_pipeline as run_pipeline
352
+ from .sable_rerank import run_private_rerank as run_private_rerank
353
+ from .public_rerank import run_public_rerank as run_public_rerank
354
+ from .question_robustness import run_question_robustness as run_question_robustness
355
+ from .retrieval import run_retrieval as run_retrieval
356
+ from .rag_diagnostics import save_rag_diagnostics_report as save_rag_diagnostics_report
357
+ from .learned_rerank import score_learned_signals as score_learned_signals
358
+ from .statistical_rerank import score_signals as score_signals
359
+ from .customer_rerank import summarize_customer_rerank as summarize_customer_rerank
360
+ from .metric_views import summarize_metrics as summarize_metrics
361
+ from .sync import sync as sync
362
+ from .workbench import validate_batch as validate_batch
363
+ from .components import validate_batch_manifest as validate_batch_manifest
364
+ from .contracts import validate_candidate_response as validate_candidate_response
365
+ from .execution_capabilities import validate_execution_requirement as validate_execution_requirement
366
+ from .rag_diagnostics import validate_rag_diagnostics_dataset as validate_rag_diagnostics_dataset
367
+ from .score_reliability import validate_reliability_reference as validate_reliability_reference
368
+ from .results import verify_evaluation as verify_evaluation
369
+ from .lifecycle import verify_lifecycle_grading as verify_lifecycle_grading
370
+ from .lifecycle import verify_lifecycle_run as verify_lifecycle_run
371
+ from .retrieval import verify_retrieval_run as verify_retrieval_run
372
+ from .evidence import verify_run as verify_run
373
+ from .workbench_tracks import workbench_track_runners as workbench_track_runners
374
+
375
+ __version__: str
376
+ __all__: list[str]
@@ -0,0 +1,77 @@
1
+ """Selected-bank analytical reads over the existing Client transport."""
2
+ from __future__ import annotations
3
+ from dataclasses import replace
4
+ from typing import TYPE_CHECKING
5
+ from urllib.parse import quote, urlencode
6
+
7
+ from ._analytical_values import identifier, require
8
+ from .analytical_views import EvaluationBoardPage, ProjectionPage, board_cursor
9
+ from .client import HostedClientError
10
+ from .workflow_contracts import WorkflowContractError
11
+
12
+ if TYPE_CHECKING:
13
+ from .client import Client
14
+ from .diagnostics import Diagnostic
15
+
16
+
17
+ class AnalyticalReadError(HostedClientError):
18
+ """Retain the original selected request without printing its cursor."""
19
+ def __init__(self, cause: HostedClientError, *, bank_id: str, board_id: str,
20
+ limit: int | None = None, cursor: str | None = None, entry_id: str | None = None):
21
+ self.bank_id, self.board_id = bank_id, board_id
22
+ self.limit, self.cursor, self.entry_id = limit, cursor, entry_id
23
+ super().__init__(cause.code, 'analytical read did not return a verified page; retain its original selectors and cursor')
24
+
25
+ @property
26
+ def diagnostic(self) -> Diagnostic:
27
+ view = super().diagnostic
28
+ if self.code == 'evaluation_board_snapshot_changed':
29
+ return replace(view, summary='The board changed while this snapshot was read.',
30
+ next_actions=('Explicitly request the first page for a new snapshot; keep the old cursor with its original page.',),
31
+ retryability='fix_required')
32
+ return view
33
+
34
+
35
+ def _path(bank: str, board: str) -> str:
36
+ return '/v1/banks/' + quote(identifier(bank), safe='') + '/boards/' + quote(identifier(board), safe='')
37
+
38
+
39
+ def _invalid() -> HostedClientError:
40
+ return HostedClientError('invalid_server_response', 'the selected analytical response could not be verified')
41
+
42
+
43
+ def board(client: Client, bank_id: str, board_id: str, *, limit: int = 20,
44
+ cursor: str | None = None) -> EvaluationBoardPage:
45
+ path = _path(bank_id, board_id)
46
+ require(type(limit) is int and 1 <= limit <= 100, 'board limit must be from 1 to 100')
47
+ query: dict[str, object] = {'limit': limit}
48
+ if cursor is not None:
49
+ query['cursor'] = board_cursor(cursor)
50
+ try:
51
+ raw = client._request('GET', path + '?' + urlencode(query))
52
+ try:
53
+ page = EvaluationBoardPage.from_dict(raw)
54
+ require((page.bank_id, page.board_id, page.limit) == (bank_id, board_id, limit))
55
+ if cursor is not None:
56
+ board_cursor(cursor, snapshot=page.snapshot_digest)
57
+ return page
58
+ except (WorkflowContractError, ValueError, TypeError, KeyError, AttributeError, RecursionError, UnicodeError):
59
+ raise _invalid() from None
60
+ except HostedClientError as error:
61
+ raise AnalyticalReadError(error, bank_id=bank_id, board_id=board_id, limit=limit, cursor=cursor) from None
62
+
63
+
64
+ def projection(client: Client, bank_id: str, board_id: str, entry_id: str) -> ProjectionPage:
65
+ path = _path(bank_id, board_id) + '/projections?' + urlencode({'entry_id': identifier(entry_id)})
66
+ try:
67
+ raw = client._request('GET', path)
68
+ try:
69
+ page = ProjectionPage.from_dict(raw)
70
+ require(page.bank_id == bank_id and page.selection is not None and len(page.items) == 1)
71
+ assert page.selection is not None
72
+ require((page.selection.board_id, page.selection.entry_id) == (board_id, entry_id))
73
+ return page
74
+ except (WorkflowContractError, ValueError, TypeError, KeyError, AttributeError, RecursionError, UnicodeError):
75
+ raise _invalid() from None
76
+ except HostedClientError as error:
77
+ raise AnalyticalReadError(error, bank_id=bank_id, board_id=board_id, entry_id=entry_id) from None