sable-sphynx 0.7.0.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sable_sphynx-0.7.0.dev0.dist-info/METADATA +152 -0
- sable_sphynx-0.7.0.dev0.dist-info/RECORD +174 -0
- sable_sphynx-0.7.0.dev0.dist-info/WHEEL +5 -0
- sable_sphynx-0.7.0.dev0.dist-info/top_level.txt +1 -0
- sphynx/__init__.py +469 -0
- sphynx/__init__.pyi +376 -0
- sphynx/_analytical_client.py +77 -0
- sphynx/_analytical_values.py +252 -0
- sphynx/_audit_client.py +173 -0
- sphynx/_comparison_adapters.py +113 -0
- sphynx/_componentized_full_rag.py +34 -0
- sphynx/_composed_pipeline.py +109 -0
- sphynx/_full_rag_grading.py +218 -0
- sphynx/_full_rag_trace.py +221 -0
- sphynx/_local_eval.py +268 -0
- sphynx/_pipeline_core.py +243 -0
- sphynx/_pr_client.py +237 -0
- sphynx/_rag_diagnostics_calibration.py +285 -0
- sphynx/_rag_diagnostics_catalog.py +39 -0
- sphynx/_rag_diagnostics_common.py +26 -0
- sphynx/_rag_diagnostics_contracts.py +186 -0
- sphynx/_rag_diagnostics_engine.py +243 -0
- sphynx/_rag_diagnostics_importers.py +242 -0
- sphynx/_rag_diagnostics_ranking.py +81 -0
- sphynx/_rag_diagnostics_semantic.py +325 -0
- sphynx/_rag_ranking.py +149 -0
- sphynx/_speech_benchmark_scoring.py +140 -0
- sphynx/_speech_benchmark_sources.py +275 -0
- sphynx/_speech_complete_scoring.py +218 -0
- sphynx/_speech_complete_source.py +118 -0
- sphynx/_speech_kido.py +294 -0
- sphynx/_speech_scoring.py +77 -0
- sphynx/_track_capture.py +213 -0
- sphynx/analytical_views.py +515 -0
- sphynx/assessment_contracts.py +378 -0
- sphynx/assessment_selection.py +57 -0
- sphynx/audit_access.py +245 -0
- sphynx/audit_contracts.py +204 -0
- sphynx/audit_requests.py +180 -0
- sphynx/authoring.py +188 -0
- sphynx/benchmarks.py +568 -0
- sphynx/candidates.py +62 -0
- sphynx/catalog.py +272 -0
- sphynx/chunking.py +372 -0
- sphynx/clickhouse_reader.py +169 -0
- sphynx/client.py +486 -0
- sphynx/comparison_core.py +149 -0
- sphynx/comparison_views.py +223 -0
- sphynx/components.py +1661 -0
- sphynx/contracts.py +514 -0
- sphynx/customer_rerank.py +299 -0
- sphynx/customer_retrieval.py +146 -0
- sphynx/data/benchmark-releases.v1.json +164 -0
- sphynx/data/public-benchmark-release.v1.json +354 -0
- sphynx/data/rag-diagnostics-synthetic.v1.json +6982 -0
- sphynx/data/rag-paper-references.v1.json +176 -0
- sphynx/data/rag-paper-schemas.v1.json +1069 -0
- sphynx/data/rag-rerank-v1.json +507 -0
- sphynx/dataset_catalog_data.py +1214 -0
- sphynx/dataset_readiness.py +138 -0
- sphynx/dataset_v3.py +999 -0
- sphynx/demo.py +130 -0
- sphynx/demo_candidates/__init__.py +1 -0
- sphynx/demo_candidates/dense.py +11 -0
- sphynx/demo_candidates/lexical.py +21 -0
- sphynx/diagnostics.py +207 -0
- sphynx/documentation.py +737 -0
- sphynx/documentation_candidate.py +939 -0
- sphynx/evals.py +538 -0
- sphynx/evaluation_protocols.py +124 -0
- sphynx/evaluator_bindings.py +370 -0
- sphynx/evidence.py +352 -0
- sphynx/execution_capabilities.py +109 -0
- sphynx/fast_search.py +285 -0
- sphynx/full_rag.py +341 -0
- sphynx/hosted/__init__.py +18 -0
- sphynx/hosted/benchmark_intake.py +218 -0
- sphynx/hosted/contracts.py +361 -0
- sphynx/hosted/model_admission.py +130 -0
- sphynx/hosted/model_artifacts.py +298 -0
- sphynx/hosted/model_contracts.py +63 -0
- sphynx/hosted/model_gcs.py +181 -0
- sphynx/hosted/model_history.py +84 -0
- sphynx/hosted/model_session.py +233 -0
- sphynx/hosted/public_suite.py +74 -0
- sphynx/hosted/registry.py +97 -0
- sphynx/hosted/revisions.py +126 -0
- sphynx/hosted/sandbox.py +105 -0
- sphynx/hosted/submission.py +287 -0
- sphynx/hosted_workflow.py +451 -0
- sphynx/http_transport.py +86 -0
- sphynx/integrated_rag.py +352 -0
- sphynx/integrated_workflow.py +552 -0
- sphynx/integration_contracts.py +474 -0
- sphynx/invocation.py +791 -0
- sphynx/invocation_candidates.py +392 -0
- sphynx/invocation_catalog_data.py +40 -0
- sphynx/learned_catalog_data.py +30 -0
- sphynx/learned_rerank.py +267 -0
- sphynx/legacy_full_rag.py +67 -0
- sphynx/lifecycle.py +885 -0
- sphynx/lifecycle_workflow.py +490 -0
- sphynx/metric_views.py +474 -0
- sphynx/metrics.py +282 -0
- sphynx/model_adapters.py +438 -0
- sphynx/model_catalog_data.py +30 -0
- sphynx/official_benchmarks.py +385 -0
- sphynx/packs.py +258 -0
- sphynx/pipeline.py +313 -0
- sphynx/pipeline_workflow.py +551 -0
- sphynx/pr_evaluations.py +560 -0
- sphynx/progress.py +89 -0
- sphynx/public_bundle.py +173 -0
- sphynx/public_eval.py +62 -0
- sphynx/public_rerank.py +355 -0
- sphynx/py.typed +0 -0
- sphynx/pytest_plugin.py +62 -0
- sphynx/question_robustness.py +437 -0
- sphynx/rag.py +329 -0
- sphynx/rag_diagnostics.py +309 -0
- sphynx/rag_papers/__init__.py +29 -0
- sphynx/rag_papers/__main__.py +45 -0
- sphynx/rag_papers/candidates.py +19 -0
- sphynx/rag_papers/contracts.py +282 -0
- sphynx/rag_papers/datasets.py +291 -0
- sphynx/rag_papers/experiments.py +116 -0
- sphynx/rag_papers/judges.py +308 -0
- sphynx/rag_papers/registry.py +33 -0
- sphynx/rag_papers/runner.py +176 -0
- sphynx/rag_papers/sable.py +135 -0
- sphynx/rag_papers/schemas.py +12 -0
- sphynx/rag_papers/scoring.py +249 -0
- sphynx/rag_papers/semantic.py +288 -0
- sphynx/rag_papers/trec.py +80 -0
- sphynx/recorded_scores.py +352 -0
- sphynx/records.py +124 -0
- sphynx/reflex_traces.py +193 -0
- sphynx/report_rendering.py +71 -0
- sphynx/rerank_family_contracts.py +150 -0
- sphynx/rerank_workflow.py +424 -0
- sphynx/research_candidate.py +190 -0
- sphynx/results.py +852 -0
- sphynx/retrieval.py +750 -0
- sphynx/retrieval_candidates.py +295 -0
- sphynx/retrieval_catalog_data.py +19 -0
- sphynx/retrieval_recordings.py +125 -0
- sphynx/reviewer_audit.py +266 -0
- sphynx/routing_policies.py +375 -0
- sphynx/runner.py +544 -0
- sphynx/sable_rerank.py +489 -0
- sphynx/score_reliability.py +393 -0
- sphynx/speech_benchmarks.py +204 -0
- sphynx/speech_campaigns.py +378 -0
- sphynx/speech_complete.py +236 -0
- sphynx/speech_contracts.py +225 -0
- sphynx/speech_evidence.py +429 -0
- sphynx/speech_listening.py +115 -0
- sphynx/speech_multilingual.py +71 -0
- sphynx/speech_tts.py +479 -0
- sphynx/sql/001_sync_v1.sql +30 -0
- sphynx/stage_contracts.py +199 -0
- sphynx/statistical_catalog_data.py +53 -0
- sphynx/statistical_rerank.py +232 -0
- sphynx/sync.py +797 -0
- sphynx/workbench.py +378 -0
- sphynx/workbench_catalog_data.py +13 -0
- sphynx/workbench_tracks.py +78 -0
- sphynx/workflow_async.py +197 -0
- sphynx/workflow_contracts.py +594 -0
- sphynx/workflow_decisions.py +211 -0
- sphynx/workflow_defaults.py +16 -0
- sphynx/workflow_extensions.py +212 -0
- sphynx/workflow_registry.py +513 -0
- sphynx/workflows.py +792 -0
sphynx/__init__.pyi
ADDED
|
@@ -0,0 +1,376 @@
|
|
|
1
|
+
# Generated by tools/check_public_types.py; keep runtime imports lazy.
|
|
2
|
+
# Explicit aliases expose the same objects to static editors.
|
|
3
|
+
|
|
4
|
+
from .workflows import Admission as Admission
|
|
5
|
+
from .workflow_contracts import AdmissionBinding as AdmissionBinding
|
|
6
|
+
from ._analytical_client import AnalyticalReadError as AnalyticalReadError
|
|
7
|
+
from .evals import ArtifactBinding as ArtifactBinding
|
|
8
|
+
from .evals import ArtifactReference as ArtifactReference
|
|
9
|
+
from .assessment_contracts import Assessment as Assessment
|
|
10
|
+
from .assessment_contracts import AssessmentContractError as AssessmentContractError
|
|
11
|
+
from .assessment_selection import AssessmentSelector as AssessmentSelector
|
|
12
|
+
from .benchmarks import AssetRef as AssetRef
|
|
13
|
+
from .workflow_async import AsyncWorkflowBackend as AsyncWorkflowBackend
|
|
14
|
+
from .lifecycle import AttemptOutput as AttemptOutput
|
|
15
|
+
from ._analytical_values import AuditProjectionSummary as AuditProjectionSummary
|
|
16
|
+
from ._audit_client import AuditRequestError as AuditRequestError
|
|
17
|
+
from .workflow_contracts import BackendDescriptor as BackendDescriptor
|
|
18
|
+
from .audit_contracts import BankRelease as BankRelease
|
|
19
|
+
from .audit_requests import BankReleaseDeclaration as BankReleaseDeclaration
|
|
20
|
+
from .audit_requests import BankReleaseTransition as BankReleaseTransition
|
|
21
|
+
from .components import BatchAlternative as BatchAlternative
|
|
22
|
+
from .components import BatchConfiguration as BatchConfiguration
|
|
23
|
+
from .components import BatchPlan as BatchPlan
|
|
24
|
+
from .benchmarks import BenchmarkCase as BenchmarkCase
|
|
25
|
+
from .benchmarks import BenchmarkContractError as BenchmarkContractError
|
|
26
|
+
from .benchmarks import BenchmarkEligibility as BenchmarkEligibility
|
|
27
|
+
from .benchmarks import BenchmarkRegistry as BenchmarkRegistry
|
|
28
|
+
from .benchmarks import BenchmarkRelease as BenchmarkRelease
|
|
29
|
+
from .analytical_views import BoardCoverage as BoardCoverage
|
|
30
|
+
from .analytical_views import BoardEntry as BoardEntry
|
|
31
|
+
from .analytical_views import BoardRanking as BoardRanking
|
|
32
|
+
from .authoring import CallableCandidate as CallableCandidate
|
|
33
|
+
from .lifecycle import CancellationToken as CancellationToken
|
|
34
|
+
from .packs import CandidateCheck as CandidateCheck
|
|
35
|
+
from .integration_contracts import CandidateConfiguration as CandidateConfiguration
|
|
36
|
+
from .contracts import CandidateRequest as CandidateRequest
|
|
37
|
+
from .contracts import CandidateResponse as CandidateResponse
|
|
38
|
+
from .contracts import CandidateScore as CandidateScore
|
|
39
|
+
from .contracts import CandidateSpec as CandidateSpec
|
|
40
|
+
from .records import CaptureIdentity as CaptureIdentity
|
|
41
|
+
from .workflow_contracts import CaseProvider as CaseProvider
|
|
42
|
+
from .contracts import CaseReferences as CaseReferences
|
|
43
|
+
from .assessment_contracts import CaseView as CaseView
|
|
44
|
+
from .clickhouse_reader import ClickHouseReader as ClickHouseReader
|
|
45
|
+
from .client import Client as Client
|
|
46
|
+
from .results import Comparison as Comparison
|
|
47
|
+
from .integration_contracts import ComparisonIdentity as ComparisonIdentity
|
|
48
|
+
from .comparison_views import ComparisonMetric as ComparisonMetric
|
|
49
|
+
from .comparison_views import ComparisonPair as ComparisonPair
|
|
50
|
+
from ._analytical_values import ComparisonProjectionSummary as ComparisonProjectionSummary
|
|
51
|
+
from .client import ComparisonRef as ComparisonRef
|
|
52
|
+
from .comparison_views import ComparisonView as ComparisonView
|
|
53
|
+
from .contracts import ContractError as ContractError
|
|
54
|
+
from .contracts import CorpusVersions as CorpusVersions
|
|
55
|
+
from .workflow_contracts import Criterion as Criterion
|
|
56
|
+
from .workflow_contracts import Dataset as Dataset
|
|
57
|
+
from .workflow_contracts import DatasetDefinition as DatasetDefinition
|
|
58
|
+
from .workflow_decisions import DecisionContext as DecisionContext
|
|
59
|
+
from .workflow_decisions import DecisionOutcome as DecisionOutcome
|
|
60
|
+
from .workflow_decisions import DecisionPolicyError as DecisionPolicyError
|
|
61
|
+
from .diagnostics import Diagnostic as Diagnostic
|
|
62
|
+
from .lifecycle import DriverContext as DriverContext
|
|
63
|
+
from .lifecycle import DriverDescriptor as DriverDescriptor
|
|
64
|
+
from .lifecycle import DriverFailure as DriverFailure
|
|
65
|
+
from .records import EVALUATION_SCHEMA_VERSION as EVALUATION_SCHEMA_VERSION
|
|
66
|
+
from .records import EVIDENCE_SCHEMA_VERSION as EVIDENCE_SCHEMA_VERSION
|
|
67
|
+
from .analytical_views import EvaluationBoardPage as EvaluationBoardPage
|
|
68
|
+
from .evals import EvaluationContractError as EvaluationContractError
|
|
69
|
+
from .evals import EvaluationDefinition as EvaluationDefinition
|
|
70
|
+
from .records import EvaluationIdentity as EvaluationIdentity
|
|
71
|
+
from .evaluation_protocols import EvaluationProtocol as EvaluationProtocol
|
|
72
|
+
from .results import EvaluationRef as EvaluationRef
|
|
73
|
+
from .evals import EvaluationRegistry as EvaluationRegistry
|
|
74
|
+
from .evaluator_bindings import EvaluatorBinding as EvaluatorBinding
|
|
75
|
+
from .assessment_contracts import EvaluatorContext as EvaluatorContext
|
|
76
|
+
from .workflow_contracts import EvaluatorDefinition as EvaluatorDefinition
|
|
77
|
+
from .records import EvidenceCapability as EvidenceCapability
|
|
78
|
+
from .contracts import EvidenceGroup as EvidenceGroup
|
|
79
|
+
from .assessment_contracts import EvidenceView as EvidenceView
|
|
80
|
+
from .execution_capabilities import ExecutionPath as ExecutionPath
|
|
81
|
+
from .packs import ExecutionPolicy as ExecutionPolicy
|
|
82
|
+
from .client import Experiment as Experiment
|
|
83
|
+
from .workflow_extensions import ExtensionError as ExtensionError
|
|
84
|
+
from .contracts import FrozenCandidate as FrozenCandidate
|
|
85
|
+
from .rerank_family_contracts import FrozenFittedState as FrozenFittedState
|
|
86
|
+
from .contracts import FrozenPoolCase as FrozenPoolCase
|
|
87
|
+
from .lifecycle import GradingContext as GradingContext
|
|
88
|
+
from .workflows import GradingPass as GradingPass
|
|
89
|
+
from .results import EvaluationRef as GradingPassRef
|
|
90
|
+
from .lifecycle import GradingPolicy as GradingPolicy
|
|
91
|
+
from .workflows import GradingRequestError as GradingRequestError
|
|
92
|
+
from .workflow_async import GradingSnapshot as GradingSnapshot
|
|
93
|
+
from .client import HostedClientError as HostedClientError
|
|
94
|
+
from .hosted_workflow import HostedComparison as HostedComparison
|
|
95
|
+
from .client import HostedEvaluation as HostedEvaluation
|
|
96
|
+
from .client import HostedGradingPass as HostedGradingPass
|
|
97
|
+
from .client import HostedReport as HostedReport
|
|
98
|
+
from .client import HostedResult as HostedResult
|
|
99
|
+
from .client import HostedRun as HostedRun
|
|
100
|
+
from .client import HostedSubmission as HostedSubmission
|
|
101
|
+
from .hosted_workflow import HostedWorkflowBackend as HostedWorkflowBackend
|
|
102
|
+
from .contracts import INTERFACE_VERSION as INTERFACE_VERSION
|
|
103
|
+
from .evaluator_bindings import InputRequirement as InputRequirement
|
|
104
|
+
from .integrated_workflow import IntegratedBackend as IntegratedBackend
|
|
105
|
+
from .integrated_rag import IntegratedCandidate as IntegratedCandidate
|
|
106
|
+
from .integrated_rag import IntegratedDriver as IntegratedDriver
|
|
107
|
+
from .invocation import InvocationBank as InvocationBank
|
|
108
|
+
from .invocation import InvocationComparison as InvocationComparison
|
|
109
|
+
from .invocation import InvocationError as InvocationError
|
|
110
|
+
from .invocation import InvocationEvaluationRef as InvocationEvaluationRef
|
|
111
|
+
from .invocation import InvocationEvaluationRef as InvocationGradingPassRef
|
|
112
|
+
from .invocation import InvocationPack as InvocationPack
|
|
113
|
+
from .invocation import InvocationRunRef as InvocationRunRef
|
|
114
|
+
from .invocation import InvocationScorecard as InvocationScorecard
|
|
115
|
+
from .benchmarks import LabelCase as LabelCase
|
|
116
|
+
from .benchmarks import LabelLineage as LabelLineage
|
|
117
|
+
from .benchmarks import LabelSetRelease as LabelSetRelease
|
|
118
|
+
from .lifecycle_workflow import LifecycleBackend as LifecycleBackend
|
|
119
|
+
from .lifecycle import LifecycleDriver as LifecycleDriver
|
|
120
|
+
from .lifecycle import LifecycleError as LifecycleError
|
|
121
|
+
from .lifecycle import LifecycleGradingRef as LifecycleGradingRef
|
|
122
|
+
from .lifecycle_workflow import LifecycleProjection as LifecycleProjection
|
|
123
|
+
from .lifecycle import LifecycleRunRef as LifecycleRunRef
|
|
124
|
+
from ._local_eval import LocalEvaluationRef as LocalEvaluationRef
|
|
125
|
+
from ._local_eval import LocalEvaluationRef as LocalGradingPassRef
|
|
126
|
+
from ._local_eval import LocalRunRef as LocalRunRef
|
|
127
|
+
from ._local_eval import LocalScorecard as LocalScorecard
|
|
128
|
+
from .records import METRIC_DEFINITION_SCHEMA_VERSION as METRIC_DEFINITION_SCHEMA_VERSION
|
|
129
|
+
from .metric_views import MetricCard as MetricCard
|
|
130
|
+
from .records import MetricDefinition as MetricDefinition
|
|
131
|
+
from ._analytical_values import MetricProjectionSummary as MetricProjectionSummary
|
|
132
|
+
from .workflow_contracts import MetricSpec as MetricSpec
|
|
133
|
+
from .metric_views import MetricSummary as MetricSummary
|
|
134
|
+
from .question_robustness import ObservedSearch as ObservedSearch
|
|
135
|
+
from .question_robustness import PILOT_MANIFEST_SHA256 as PILOT_MANIFEST_SHA256
|
|
136
|
+
from .public_rerank import PUBLIC_POOL_SHA256 as PUBLIC_POOL_SHA256
|
|
137
|
+
from .public_rerank import PUBLIC_RERANK_PACK_ID as PUBLIC_RERANK_PACK_ID
|
|
138
|
+
from .packs import PackDescriptor as PackDescriptor
|
|
139
|
+
from .workflow_extensions import PackagedExtension as PackagedExtension
|
|
140
|
+
from .workflow_contracts import PayloadSchema as PayloadSchema
|
|
141
|
+
from .workflows import PendingWorkError as PendingWorkError
|
|
142
|
+
from .pipeline_workflow import PipelineBackend as PipelineBackend
|
|
143
|
+
from .pipeline_workflow import PipelineCandidate as PipelineCandidate
|
|
144
|
+
from .pipeline_workflow import PipelineSource as PipelineSource
|
|
145
|
+
from .workflows import Plan as Plan
|
|
146
|
+
from .workflow_contracts import PlanBinding as PlanBinding
|
|
147
|
+
from ._pr_client import PrEvaluation as PrEvaluation
|
|
148
|
+
from ._pr_client import PrEvaluationError as PrEvaluationError
|
|
149
|
+
from .pr_evaluations import PrEvaluationRequest as PrEvaluationRequest
|
|
150
|
+
from .pr_evaluations import PrEvaluationResult as PrEvaluationResult
|
|
151
|
+
from .sable_rerank import PrivateEvaluation as PrivateEvaluation
|
|
152
|
+
from .sable_rerank import PrivatePack as PrivatePack
|
|
153
|
+
from .progress import ProgressView as ProgressView
|
|
154
|
+
from ._analytical_values import ProjectionCoverage as ProjectionCoverage
|
|
155
|
+
from .analytical_views import ProjectionEventRef as ProjectionEventRef
|
|
156
|
+
from .analytical_views import ProjectionPage as ProjectionPage
|
|
157
|
+
from .analytical_views import ProjectionProof as ProjectionProof
|
|
158
|
+
from .analytical_views import ProjectionResult as ProjectionResult
|
|
159
|
+
from .analytical_views import ProjectionRow as ProjectionRow
|
|
160
|
+
from .analytical_views import ProjectionSelection as ProjectionSelection
|
|
161
|
+
from .public_bundle import PublicRerankBundle as PublicRerankBundle
|
|
162
|
+
from .public_rerank import PublicRerankPack as PublicRerankPack
|
|
163
|
+
from .hosted_workflow import PullRequestCandidate as PullRequestCandidate
|
|
164
|
+
from .question_robustness import QUESTION_ROBUSTNESS_INTERFACE as QUESTION_ROBUSTNESS_INTERFACE
|
|
165
|
+
from .question_robustness import QUESTION_ROBUSTNESS_PACK_ID as QUESTION_ROBUSTNESS_PACK_ID
|
|
166
|
+
from .question_robustness import QuestionRobustnessPack as QuestionRobustnessPack
|
|
167
|
+
from .recorded_scores import RECORDED_SCORE_SCHEMA as RECORDED_SCORE_SCHEMA
|
|
168
|
+
from .contracts import REQUEST_SCHEMA_VERSION as REQUEST_SCHEMA_VERSION
|
|
169
|
+
from .contracts import RESPONSE_SCHEMA_VERSION as RESPONSE_SCHEMA_VERSION
|
|
170
|
+
from .retrieval import RETRIEVAL_INTERFACE_VERSION as RETRIEVAL_INTERFACE_VERSION
|
|
171
|
+
from .retrieval import RETRIEVAL_PACK_ID as RETRIEVAL_PACK_ID
|
|
172
|
+
from .clickhouse_reader import ReadOnlyClickHouseConfig as ReadOnlyClickHouseConfig
|
|
173
|
+
from .clickhouse_reader import ReadOnlyClickHouseError as ReadOnlyClickHouseError
|
|
174
|
+
from .retrieval_candidates import RecordedHybridSignals as RecordedHybridSignals
|
|
175
|
+
from .workflow_contracts import ReferenceBinding as ReferenceBinding
|
|
176
|
+
from .contracts import ReferenceCandidate as ReferenceCandidate
|
|
177
|
+
from .score_reliability import ReliabilityReference as ReliabilityReference
|
|
178
|
+
from .workflows import Report as Report
|
|
179
|
+
from .evaluator_bindings import ReportAssessment as ReportAssessment
|
|
180
|
+
from .workflow_contracts import ReportProjection as ReportProjection
|
|
181
|
+
from ._analytical_values import ReportProjectionSummary as ReportProjectionSummary
|
|
182
|
+
from .workflow_decisions import ReportSnapshot as ReportSnapshot
|
|
183
|
+
from .workflow_contracts import ResolvedBinding as ResolvedBinding
|
|
184
|
+
from .evals import ResolvedInvocation as ResolvedInvocation
|
|
185
|
+
from .workflows import ResolvedPlan as ResolvedPlan
|
|
186
|
+
from .retrieval import RetrievalEvaluation as RetrievalEvaluation
|
|
187
|
+
from .retrieval import RetrievalPack as RetrievalPack
|
|
188
|
+
from .retrieval import RetrievalRun as RetrievalRun
|
|
189
|
+
from .audit_requests import ReviewAdjudication as ReviewAdjudication
|
|
190
|
+
from .audit_contracts import ReviewContribution as ReviewContribution
|
|
191
|
+
from .audit_contracts import ReviewEvent as ReviewEvent
|
|
192
|
+
from .audit_access import ReviewHistoryPage as ReviewHistoryPage
|
|
193
|
+
from .audit_access import ReviewOperationReceipt as ReviewOperationReceipt
|
|
194
|
+
from .benchmarks import ReviewReceipt as ReviewReceipt
|
|
195
|
+
from .audit_contracts import ReviewTarget as ReviewTarget
|
|
196
|
+
from .audit_access import ReviewTargetPage as ReviewTargetPage
|
|
197
|
+
from .reviewer_audit import ReviewerAudit as ReviewerAudit
|
|
198
|
+
from .workflows import Run as Run
|
|
199
|
+
from .lifecycle import RunLimits as RunLimits
|
|
200
|
+
from .lifecycle import RunPolicy as RunPolicy
|
|
201
|
+
from .evidence import RunRef as RunRef
|
|
202
|
+
from .sable_rerank import SABLE_PACK_ID as SABLE_PACK_ID
|
|
203
|
+
from .integration_contracts import ScoreField as ScoreField
|
|
204
|
+
from .integration_contracts import ScoreSemantics as ScoreSemantics
|
|
205
|
+
from .results import Scorecard as Scorecard
|
|
206
|
+
from .benchmarks import SliceParameters as SliceParameters
|
|
207
|
+
from .stage_contracts import StageContractError as StageContractError
|
|
208
|
+
from .stage_contracts import StageDefinition as StageDefinition
|
|
209
|
+
from .stage_contracts import StageEvidence as StageEvidence
|
|
210
|
+
from .stage_contracts import StageEvidenceBackend as StageEvidenceBackend
|
|
211
|
+
from .stage_contracts import StageReplacement as StageReplacement
|
|
212
|
+
from .stage_contracts import StageSelector as StageSelector
|
|
213
|
+
from .sync import StorageConfig as StorageConfig
|
|
214
|
+
from .workflows import Submission as Submission
|
|
215
|
+
from .workflow_async import SubmissionError as SubmissionError
|
|
216
|
+
from .workflow_async import SubmissionIntent as SubmissionIntent
|
|
217
|
+
from .workflow_async import SubmissionSnapshot as SubmissionSnapshot
|
|
218
|
+
from .workflows import Suite as Suite
|
|
219
|
+
from .workflow_contracts import SuiteDefinition as SuiteDefinition
|
|
220
|
+
from .sync import SyncResult as SyncResult
|
|
221
|
+
from .audit_access import TargetReviewPage as TargetReviewPage
|
|
222
|
+
from .integration_contracts import TrackEvaluationRef as TrackGradingPassRef
|
|
223
|
+
from .workflow_decisions import Verdict as Verdict
|
|
224
|
+
from .evaluation_protocols import VersionedIdentity as VersionedIdentity
|
|
225
|
+
from .workflow_contracts import WorkflowBackend as WorkflowBackend
|
|
226
|
+
from .workflow_contracts import WorkflowContractError as WorkflowContractError
|
|
227
|
+
from .workflows import WorkflowError as WorkflowError
|
|
228
|
+
from .workflow_registry import WorkflowRegistry as WorkflowRegistry
|
|
229
|
+
from .workflow_registry import WorkflowRegistryError as WorkflowRegistryError
|
|
230
|
+
from .workflows import WorkflowTimeoutError as WorkflowTimeoutError
|
|
231
|
+
from .lifecycle import assess_lifecycle as assess_lifecycle
|
|
232
|
+
from .reviewer_audit import audit_invocation_reviewer as audit_invocation_reviewer
|
|
233
|
+
from .legacy_full_rag import audit_legacy_cherry_full_rag as audit_legacy_cherry_full_rag
|
|
234
|
+
from .rag_diagnostics import audit_rag_judge as audit_rag_judge
|
|
235
|
+
from .contracts import build_candidate_request as build_candidate_request
|
|
236
|
+
from .sable_rerank import build_cherry_development_dataset as build_cherry_development_dataset
|
|
237
|
+
from .customer_rerank import build_customer_development_dataset as build_customer_development_dataset
|
|
238
|
+
from .customer_retrieval import build_customer_retrieval_dataset as build_customer_retrieval_dataset
|
|
239
|
+
from .sable_rerank import build_exact_sable_dataset as build_exact_sable_dataset
|
|
240
|
+
from .hosted_workflow import build_hosted_registry as build_hosted_registry
|
|
241
|
+
from .integrated_workflow import build_integrated_registry as build_integrated_registry
|
|
242
|
+
from .pipeline_workflow import build_pipeline_registry as build_pipeline_registry
|
|
243
|
+
from .public_rerank import build_public_rerank_request as build_public_rerank_request
|
|
244
|
+
from .question_robustness import build_question_robustness_request as build_question_robustness_request
|
|
245
|
+
from .question_robustness import build_question_robustness_response as build_question_robustness_response
|
|
246
|
+
from .retrieval import build_retrieval_request as build_retrieval_request
|
|
247
|
+
from .evals import canonical_contract_json as canonical_contract_json
|
|
248
|
+
from .contracts import canonical_json as canonical_json
|
|
249
|
+
from .retrieval import capture_retrieval as capture_retrieval
|
|
250
|
+
from .packs import check_candidate as check_candidate
|
|
251
|
+
from .results import compare as compare
|
|
252
|
+
from .full_rag import compare_full_rag as compare_full_rag
|
|
253
|
+
from .full_rag import compare_full_rag_contexts as compare_full_rag_contexts
|
|
254
|
+
from .invocation import compare_invocation as compare_invocation
|
|
255
|
+
from .pipeline import compare_pipeline as compare_pipeline
|
|
256
|
+
from .sable_rerank import compare_private_rerank as compare_private_rerank
|
|
257
|
+
from .public_rerank import compare_public_rerank as compare_public_rerank
|
|
258
|
+
from .question_robustness import compare_question_robustness as compare_question_robustness
|
|
259
|
+
from .retrieval import compare_retrieval as compare_retrieval
|
|
260
|
+
from .score_reliability import compare_score_reliability as compare_score_reliability
|
|
261
|
+
from .workflow_contracts import contract_digest as contract_digest
|
|
262
|
+
from .full_rag import create_full_rag_review_template as create_full_rag_review_template
|
|
263
|
+
from .learned_rerank import create_learned_scorer as create_learned_scorer
|
|
264
|
+
from .model_adapters import create_model_scorer as create_model_scorer
|
|
265
|
+
from .question_robustness import create_query_review_template as create_query_review_template
|
|
266
|
+
from .rag_diagnostics import create_rag_diagnostics_review_template as create_rag_diagnostics_review_template
|
|
267
|
+
from .recorded_scores import create_recorded_scorer as create_recorded_scorer
|
|
268
|
+
from .statistical_rerank import create_statistical_scorer as create_statistical_scorer
|
|
269
|
+
from .workflow_decisions import decide_report as decide_report
|
|
270
|
+
from .benchmarks import default_registry as default_benchmark_registry
|
|
271
|
+
from .evals import default_evaluation_registry as default_evaluation_registry
|
|
272
|
+
from .workflow_defaults import default_workflow_registry as default_workflow_registry
|
|
273
|
+
from .evals import describe_eval as describe_eval
|
|
274
|
+
from .workflow_extensions import discover_extensions as discover_extensions
|
|
275
|
+
from .workflow_extensions import enable_extension as enable_extension
|
|
276
|
+
from .rag_diagnostics import estimate_rag_ppi_mean as estimate_rag_ppi_mean
|
|
277
|
+
from .score_reliability import evaluate_fitted_calibrator as evaluate_fitted_calibrator
|
|
278
|
+
from .authoring import evaluation_suite as evaluation_suite
|
|
279
|
+
from .lifecycle import execute_lifecycle as execute_lifecycle
|
|
280
|
+
from .diagnostics import explain_error as explain_error
|
|
281
|
+
from .catalog import get_dataset as get_dataset
|
|
282
|
+
from .model_adapters import get_model_spec as get_model_spec
|
|
283
|
+
from .packs import get_pack as get_pack
|
|
284
|
+
from .results import grade as grade
|
|
285
|
+
from .full_rag import grade_full_rag as grade_full_rag
|
|
286
|
+
from .invocation import grade_invocation as grade_invocation
|
|
287
|
+
from .pipeline import grade_pipeline as grade_pipeline
|
|
288
|
+
from .sable_rerank import grade_private_rerank as grade_private_rerank
|
|
289
|
+
from .public_rerank import grade_public_rerank as grade_public_rerank
|
|
290
|
+
from .question_robustness import grade_question_robustness as grade_question_robustness
|
|
291
|
+
from .rag_diagnostics import grade_rag_diagnostics as grade_rag_diagnostics
|
|
292
|
+
from .retrieval import grade_retrieval as grade_retrieval
|
|
293
|
+
from .score_reliability import grade_score_reliability as grade_score_reliability
|
|
294
|
+
from .retrieval_recordings import import_hybrid_recordings as import_hybrid_recordings
|
|
295
|
+
from .rag_diagnostics import import_rag_diagnostics_beir as import_rag_diagnostics_beir
|
|
296
|
+
from .recorded_scores import import_recorded_scores as import_recorded_scores
|
|
297
|
+
from .catalog import list_candidates as list_candidates
|
|
298
|
+
from .catalog import list_datasets as list_datasets
|
|
299
|
+
from .evals import list_evals as list_evals
|
|
300
|
+
from .invocation import list_invocation_banks as list_invocation_banks
|
|
301
|
+
from .metric_views import list_metric_tracks as list_metric_tracks
|
|
302
|
+
from .metric_views import list_metrics as list_metrics
|
|
303
|
+
from .packs import list_packs as list_packs
|
|
304
|
+
from .public_rerank import list_public_pool_releases as list_public_pool_releases
|
|
305
|
+
from .rag_diagnostics import list_rag_diagnostics_metrics as list_rag_diagnostics_metrics
|
|
306
|
+
from .components import load_batch_manifest as load_batch_manifest
|
|
307
|
+
from .rerank_family_contracts import load_fitted_state as load_fitted_state
|
|
308
|
+
from .invocation import load_invocation_pack as load_invocation_pack
|
|
309
|
+
from .sable_rerank import load_private_pack as load_private_pack
|
|
310
|
+
from .public_rerank import load_public_pool_release as load_public_pool_release
|
|
311
|
+
from .public_bundle import load_public_rerank_bundle as load_public_rerank_bundle
|
|
312
|
+
from .public_rerank import load_public_rerank_pack as load_public_rerank_pack
|
|
313
|
+
from .question_robustness import load_question_robustness_pack as load_question_robustness_pack
|
|
314
|
+
from .rag_diagnostics import load_rag_diagnostics_dataset as load_rag_diagnostics_dataset
|
|
315
|
+
from .rag_diagnostics import load_rag_diagnostics_demo as load_rag_diagnostics_demo
|
|
316
|
+
from .rag_diagnostics import load_rag_diagnostics_report as load_rag_diagnostics_report
|
|
317
|
+
from .score_reliability import load_reliability_reference as load_reliability_reference
|
|
318
|
+
from .research_candidate import load_research_candidate as load_research_candidate
|
|
319
|
+
from .retrieval import load_retrieval_pack as load_retrieval_pack
|
|
320
|
+
from .research_candidate import make_research_candidate as make_research_candidate
|
|
321
|
+
from .retrieval_candidates import make_retrieval_candidate as make_retrieval_candidate
|
|
322
|
+
from .results import open_evaluation as open_evaluation
|
|
323
|
+
from .lifecycle import open_lifecycle_run as open_lifecycle_run
|
|
324
|
+
from ._local_eval import open_local_evaluation as open_local_evaluation
|
|
325
|
+
from ._local_eval import open_local_run as open_local_run
|
|
326
|
+
from .evidence import open_run as open_run
|
|
327
|
+
from .contracts import pool_digest as pool_digest
|
|
328
|
+
from .model_adapters import preflight_model as preflight_model
|
|
329
|
+
from .public_bundle import prepare_public_rerank_bundle as prepare_public_rerank_bundle
|
|
330
|
+
from .rag_diagnostics import rag_candidate_view as rag_candidate_view
|
|
331
|
+
from .rag_diagnostics import rag_diagnostics_capabilities as rag_diagnostics_capabilities
|
|
332
|
+
from .contracts import ranked_candidate_ids as ranked_candidate_ids
|
|
333
|
+
from .sync import read_committed_projection as read_committed_projection
|
|
334
|
+
from .pipeline_workflow import register_pipeline_backend as register_pipeline_backend
|
|
335
|
+
from .rag_diagnostics import render_rag_diagnostics_report as render_rag_diagnostics_report
|
|
336
|
+
from .recorded_scores import replay_rerank_scores as replay_rerank_scores
|
|
337
|
+
from .results import report as report
|
|
338
|
+
from .score_reliability import report_fitted_calibrator as report_fitted_calibrator
|
|
339
|
+
from .full_rag import report_full_rag as report_full_rag
|
|
340
|
+
from .invocation import report_invocation as report_invocation
|
|
341
|
+
from .pipeline import report_pipeline as report_pipeline
|
|
342
|
+
from .public_rerank import report_public_rerank as report_public_rerank
|
|
343
|
+
from .question_robustness import report_question_robustness as report_question_robustness
|
|
344
|
+
from .evals import resolve_eval as resolve_eval
|
|
345
|
+
from .retrieval_candidates import retrieval_configuration as retrieval_configuration
|
|
346
|
+
from .runner import run as run
|
|
347
|
+
from .workbench import run_batch as run_batch
|
|
348
|
+
from .full_rag import run_full_rag as run_full_rag
|
|
349
|
+
from .integrated_rag import run_integrated as run_integrated
|
|
350
|
+
from .invocation import run_invocation as run_invocation
|
|
351
|
+
from .pipeline import run_pipeline as run_pipeline
|
|
352
|
+
from .sable_rerank import run_private_rerank as run_private_rerank
|
|
353
|
+
from .public_rerank import run_public_rerank as run_public_rerank
|
|
354
|
+
from .question_robustness import run_question_robustness as run_question_robustness
|
|
355
|
+
from .retrieval import run_retrieval as run_retrieval
|
|
356
|
+
from .rag_diagnostics import save_rag_diagnostics_report as save_rag_diagnostics_report
|
|
357
|
+
from .learned_rerank import score_learned_signals as score_learned_signals
|
|
358
|
+
from .statistical_rerank import score_signals as score_signals
|
|
359
|
+
from .customer_rerank import summarize_customer_rerank as summarize_customer_rerank
|
|
360
|
+
from .metric_views import summarize_metrics as summarize_metrics
|
|
361
|
+
from .sync import sync as sync
|
|
362
|
+
from .workbench import validate_batch as validate_batch
|
|
363
|
+
from .components import validate_batch_manifest as validate_batch_manifest
|
|
364
|
+
from .contracts import validate_candidate_response as validate_candidate_response
|
|
365
|
+
from .execution_capabilities import validate_execution_requirement as validate_execution_requirement
|
|
366
|
+
from .rag_diagnostics import validate_rag_diagnostics_dataset as validate_rag_diagnostics_dataset
|
|
367
|
+
from .score_reliability import validate_reliability_reference as validate_reliability_reference
|
|
368
|
+
from .results import verify_evaluation as verify_evaluation
|
|
369
|
+
from .lifecycle import verify_lifecycle_grading as verify_lifecycle_grading
|
|
370
|
+
from .lifecycle import verify_lifecycle_run as verify_lifecycle_run
|
|
371
|
+
from .retrieval import verify_retrieval_run as verify_retrieval_run
|
|
372
|
+
from .evidence import verify_run as verify_run
|
|
373
|
+
from .workbench_tracks import workbench_track_runners as workbench_track_runners
|
|
374
|
+
|
|
375
|
+
__version__: str
|
|
376
|
+
__all__: list[str]
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
"""Selected-bank analytical reads over the existing Client transport."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
from dataclasses import replace
|
|
4
|
+
from typing import TYPE_CHECKING
|
|
5
|
+
from urllib.parse import quote, urlencode
|
|
6
|
+
|
|
7
|
+
from ._analytical_values import identifier, require
|
|
8
|
+
from .analytical_views import EvaluationBoardPage, ProjectionPage, board_cursor
|
|
9
|
+
from .client import HostedClientError
|
|
10
|
+
from .workflow_contracts import WorkflowContractError
|
|
11
|
+
|
|
12
|
+
if TYPE_CHECKING:
|
|
13
|
+
from .client import Client
|
|
14
|
+
from .diagnostics import Diagnostic
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class AnalyticalReadError(HostedClientError):
|
|
18
|
+
"""Retain the original selected request without printing its cursor."""
|
|
19
|
+
def __init__(self, cause: HostedClientError, *, bank_id: str, board_id: str,
|
|
20
|
+
limit: int | None = None, cursor: str | None = None, entry_id: str | None = None):
|
|
21
|
+
self.bank_id, self.board_id = bank_id, board_id
|
|
22
|
+
self.limit, self.cursor, self.entry_id = limit, cursor, entry_id
|
|
23
|
+
super().__init__(cause.code, 'analytical read did not return a verified page; retain its original selectors and cursor')
|
|
24
|
+
|
|
25
|
+
@property
|
|
26
|
+
def diagnostic(self) -> Diagnostic:
|
|
27
|
+
view = super().diagnostic
|
|
28
|
+
if self.code == 'evaluation_board_snapshot_changed':
|
|
29
|
+
return replace(view, summary='The board changed while this snapshot was read.',
|
|
30
|
+
next_actions=('Explicitly request the first page for a new snapshot; keep the old cursor with its original page.',),
|
|
31
|
+
retryability='fix_required')
|
|
32
|
+
return view
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _path(bank: str, board: str) -> str:
|
|
36
|
+
return '/v1/banks/' + quote(identifier(bank), safe='') + '/boards/' + quote(identifier(board), safe='')
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _invalid() -> HostedClientError:
|
|
40
|
+
return HostedClientError('invalid_server_response', 'the selected analytical response could not be verified')
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def board(client: Client, bank_id: str, board_id: str, *, limit: int = 20,
|
|
44
|
+
cursor: str | None = None) -> EvaluationBoardPage:
|
|
45
|
+
path = _path(bank_id, board_id)
|
|
46
|
+
require(type(limit) is int and 1 <= limit <= 100, 'board limit must be from 1 to 100')
|
|
47
|
+
query: dict[str, object] = {'limit': limit}
|
|
48
|
+
if cursor is not None:
|
|
49
|
+
query['cursor'] = board_cursor(cursor)
|
|
50
|
+
try:
|
|
51
|
+
raw = client._request('GET', path + '?' + urlencode(query))
|
|
52
|
+
try:
|
|
53
|
+
page = EvaluationBoardPage.from_dict(raw)
|
|
54
|
+
require((page.bank_id, page.board_id, page.limit) == (bank_id, board_id, limit))
|
|
55
|
+
if cursor is not None:
|
|
56
|
+
board_cursor(cursor, snapshot=page.snapshot_digest)
|
|
57
|
+
return page
|
|
58
|
+
except (WorkflowContractError, ValueError, TypeError, KeyError, AttributeError, RecursionError, UnicodeError):
|
|
59
|
+
raise _invalid() from None
|
|
60
|
+
except HostedClientError as error:
|
|
61
|
+
raise AnalyticalReadError(error, bank_id=bank_id, board_id=board_id, limit=limit, cursor=cursor) from None
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def projection(client: Client, bank_id: str, board_id: str, entry_id: str) -> ProjectionPage:
|
|
65
|
+
path = _path(bank_id, board_id) + '/projections?' + urlencode({'entry_id': identifier(entry_id)})
|
|
66
|
+
try:
|
|
67
|
+
raw = client._request('GET', path)
|
|
68
|
+
try:
|
|
69
|
+
page = ProjectionPage.from_dict(raw)
|
|
70
|
+
require(page.bank_id == bank_id and page.selection is not None and len(page.items) == 1)
|
|
71
|
+
assert page.selection is not None
|
|
72
|
+
require((page.selection.board_id, page.selection.entry_id) == (board_id, entry_id))
|
|
73
|
+
return page
|
|
74
|
+
except (WorkflowContractError, ValueError, TypeError, KeyError, AttributeError, RecursionError, UnicodeError):
|
|
75
|
+
raise _invalid() from None
|
|
76
|
+
except HostedClientError as error:
|
|
77
|
+
raise AnalyticalReadError(error, bank_id=bank_id, board_id=board_id, entry_id=entry_id) from None
|