sable-sphynx 0.7.0.dev0__py3-none-any.whl

sphynx/__init__.py

@@ -0,0 +1,469 @@
+"""Public entry points for Sphynx suites, native APIs, and extensions."""
+
+from .contracts import (
+    INTERFACE_VERSION,
+    REQUEST_SCHEMA_VERSION,
+    RESPONSE_SCHEMA_VERSION,
+    CandidateRequest,
+    CandidateResponse,
+    CandidateScore,
+    CandidateSpec,
+    CaseReferences,
+    ContractError,
+    CorpusVersions,
+    EvidenceGroup,
+    FrozenCandidate,
+    FrozenPoolCase,
+    ReferenceCandidate,
+    build_candidate_request,
+    canonical_json,
+    pool_digest,
+    ranked_candidate_ids,
+    validate_candidate_response,
+)
+from .packs import CandidateCheck, ExecutionPolicy, PackDescriptor, check_candidate, get_pack, list_packs
+from .records import (
+    EVALUATION_SCHEMA_VERSION,
+    EVIDENCE_SCHEMA_VERSION,
+    METRIC_DEFINITION_SCHEMA_VERSION,
+    CaptureIdentity,
+    EvaluationIdentity,
+    EvidenceCapability,
+    MetricDefinition,
+)
+from .evidence import RunRef, open_run, verify_run
+from .runner import run
+from .results import (
+    Comparison,
+    EvaluationRef,
+    Scorecard,
+    compare,
+    grade,
+    open_evaluation,
+    report,
+    verify_evaluation,
+)
+from .sync import StorageConfig, SyncResult, read_committed_projection, sync
+from .invocation import (
+    InvocationBank, InvocationComparison, InvocationError, InvocationEvaluationRef, InvocationPack, InvocationRunRef,
+    InvocationScorecard, compare_invocation, grade_invocation,
+    list_invocation_banks, load_invocation_pack, report_invocation, run_invocation,
+)
+from .sable_rerank import (
+    SABLE_PACK_ID,
+    PrivateEvaluation,
+    PrivatePack,
+    build_cherry_development_dataset,
+    build_exact_sable_dataset,
+    compare_private_rerank,
+    grade_private_rerank,
+    load_private_pack,
+    run_private_rerank,
+)
+
+__version__ = "0.7.0.dev0"
+
+__all__ = [
+    "INTERFACE_VERSION",
+    "REQUEST_SCHEMA_VERSION",
+    "RESPONSE_SCHEMA_VERSION",
+    "CandidateRequest",
+    "CandidateResponse",
+    "CandidateScore",
+    "CandidateSpec",
+    "CaseReferences",
+    "ContractError",
+    "CorpusVersions",
+    "EvidenceGroup",
+    "FrozenCandidate",
+    "FrozenPoolCase",
+    "ReferenceCandidate",
+    "build_candidate_request",
+    "canonical_json",
+    "pool_digest",
+    "ranked_candidate_ids",
+    "validate_candidate_response",
+    "CandidateCheck",
+    "ExecutionPolicy",
+    "PackDescriptor",
+    "check_candidate",
+    "get_pack",
+    "list_packs",
+    "EVALUATION_SCHEMA_VERSION",
+    "EVIDENCE_SCHEMA_VERSION",
+    "METRIC_DEFINITION_SCHEMA_VERSION",
+    "CaptureIdentity",
+    "EvaluationIdentity",
+    "EvidenceCapability",
+    "MetricDefinition",
+    "RunRef",
+    "open_run",
+    "verify_run",
+    "run",
+    "EvaluationRef",
+    "open_evaluation",
+    "verify_evaluation",
+    "grade",
+    "Scorecard",
+    "report",
+    "Comparison",
+    "compare",
+    "StorageConfig",
+    "SyncResult",
+    "sync",
+    "read_committed_projection",
+    "InvocationError",
+    "InvocationBank",
+    "list_invocation_banks",
+    "InvocationComparison",
+    "InvocationEvaluationRef",
+    "InvocationPack",
+    "InvocationRunRef",
+    "InvocationScorecard",
+    "load_invocation_pack",
+    "run_invocation",
+    "grade_invocation",
+    "report_invocation",
+    "compare_invocation",
+    "SABLE_PACK_ID",
+    "PrivateEvaluation",
+    "PrivatePack",
+    "build_cherry_development_dataset",
+    "build_exact_sable_dataset",
+    "compare_private_rerank",
+    "grade_private_rerank",
+    "load_private_pack",
+    "run_private_rerank",
+]
+
+from .metric_views import MetricCard, MetricSummary, list_metric_tracks, list_metrics, summarize_metrics
+from .reviewer_audit import ReviewerAudit, audit_invocation_reviewer
+from .recorded_scores import RECORDED_SCORE_SCHEMA, replay_rerank_scores
+
+__all__ += ["MetricCard", "MetricSummary", "list_metric_tracks", "list_metrics", "summarize_metrics", "ReviewerAudit", "audit_invocation_reviewer", "RECORDED_SCORE_SCHEMA", "replay_rerank_scores"]
+
+from .customer_rerank import build_customer_development_dataset, summarize_customer_rerank
+__all__ += ["build_customer_development_dataset", "summarize_customer_rerank"]
+
+from .retrieval import (
+    RETRIEVAL_PACK_ID, RETRIEVAL_INTERFACE_VERSION, RetrievalPack,
+    RetrievalRun, RetrievalEvaluation, load_retrieval_pack,
+    build_retrieval_request, capture_retrieval, run_retrieval,
+    verify_retrieval_run, grade_retrieval, compare_retrieval,
+)
+from .customer_retrieval import build_customer_retrieval_dataset
+
+__all__ += [
+    "RETRIEVAL_PACK_ID", "RETRIEVAL_INTERFACE_VERSION", "RetrievalPack",
+    "RetrievalRun", "RetrievalEvaluation", "load_retrieval_pack",
+    "build_retrieval_request", "capture_retrieval", "run_retrieval",
+    "verify_retrieval_run", "grade_retrieval", "compare_retrieval",
+    "build_customer_retrieval_dataset",
+]
+
+
+from ._local_eval import LocalRunRef, LocalEvaluationRef, LocalScorecard, open_local_run, open_local_evaluation
+from .question_robustness import (
+    QUESTION_ROBUSTNESS_PACK_ID, QUESTION_ROBUSTNESS_INTERFACE, PILOT_MANIFEST_SHA256,
+    QuestionRobustnessPack, ObservedSearch, load_question_robustness_pack,
+    build_question_robustness_request, build_question_robustness_response, run_question_robustness, grade_question_robustness,
+    report_question_robustness, compare_question_robustness, create_query_review_template,
+)
+from .public_rerank import (
+    PUBLIC_RERANK_PACK_ID, PUBLIC_POOL_SHA256, PublicRerankPack, load_public_rerank_pack,
+    build_public_rerank_request, run_public_rerank, grade_public_rerank,
+    report_public_rerank, compare_public_rerank,
+)
+__all__ += [
+    "LocalRunRef", "LocalEvaluationRef", "LocalScorecard", "QUESTION_ROBUSTNESS_PACK_ID",
+    "QUESTION_ROBUSTNESS_INTERFACE", "PILOT_MANIFEST_SHA256", "QuestionRobustnessPack",
+    "ObservedSearch", "open_local_run", "open_local_evaluation", "build_question_robustness_response", "load_question_robustness_pack", "build_question_robustness_request",
+    "run_question_robustness", "grade_question_robustness", "report_question_robustness",
+    "compare_question_robustness", "create_query_review_template",
+    "PUBLIC_RERANK_PACK_ID", "PUBLIC_POOL_SHA256", "PublicRerankPack", "load_public_rerank_pack",
+    "build_public_rerank_request", "run_public_rerank", "grade_public_rerank",
+    "report_public_rerank", "compare_public_rerank",
+]
+
+from .public_bundle import PublicRerankBundle, prepare_public_rerank_bundle, load_public_rerank_bundle
+__all__ += ["PublicRerankBundle", "prepare_public_rerank_bundle", "load_public_rerank_bundle"]
+
+# Hosted client is additive; existing local evaluation APIs remain available.
+from .client import Client, ComparisonRef, Experiment, HostedClientError, HostedEvaluation, HostedResult, HostedRun
+__all__ += ["Client", "ComparisonRef", "Experiment", "HostedClientError", "HostedEvaluation", "HostedResult", "HostedRun"]
+
+from .evals import (
+    ArtifactBinding, ArtifactReference, EvaluationContractError,
+    EvaluationDefinition, EvaluationRegistry, ResolvedInvocation,
+    canonical_contract_json, default_evaluation_registry,
+    describe_eval, list_evals, resolve_eval,
+)
+__all__ += [
+    "ArtifactBinding", "ArtifactReference", "EvaluationContractError",
+    "EvaluationDefinition", "EvaluationRegistry", "ResolvedInvocation",
+    "canonical_contract_json", "default_evaluation_registry",
+    "describe_eval", "list_evals", "resolve_eval",
+]
+
+# Metadata-only benchmark contracts preserve existing local/hosted evaluation APIs.
+from .benchmarks import (
+    AssetRef, BenchmarkCase, BenchmarkContractError, BenchmarkEligibility,
+    BenchmarkRegistry, BenchmarkRelease, LabelCase, LabelLineage,
+    LabelSetRelease, ReviewReceipt, SliceParameters,
+    default_registry as default_benchmark_registry,
+)
+__all__ += [
+    "AssetRef", "BenchmarkCase", "BenchmarkContractError", "BenchmarkEligibility",
+    "BenchmarkRegistry", "BenchmarkRelease", "LabelCase", "LabelLineage",
+    "LabelSetRelease", "ReviewReceipt", "SliceParameters", "default_benchmark_registry",
+]
+
+# Lazy research exports keep metadata discovery free of adapter imports.
+_RESEARCH_EXPORTS = {
+    'load_rag_diagnostics_dataset': 'rag_diagnostics',
+    'validate_rag_diagnostics_dataset': 'rag_diagnostics',
+    'rag_candidate_view': 'rag_diagnostics',
+    'grade_rag_diagnostics': 'rag_diagnostics',
+    'load_rag_diagnostics_demo': 'rag_diagnostics',
+    'list_rag_diagnostics_metrics': 'rag_diagnostics',
+    'rag_diagnostics_capabilities': 'rag_diagnostics',
+    'save_rag_diagnostics_report': 'rag_diagnostics',
+    'load_rag_diagnostics_report': 'rag_diagnostics',
+    'render_rag_diagnostics_report': 'rag_diagnostics',
+    'import_rag_diagnostics_beir': 'rag_diagnostics',
+    'audit_rag_judge': 'rag_diagnostics',
+    'estimate_rag_ppi_mean': 'rag_diagnostics',
+    'create_rag_diagnostics_review_template': 'rag_diagnostics',
+    'ReliabilityReference': "score_reliability",
+    'validate_reliability_reference': "score_reliability",
+    'load_reliability_reference': "score_reliability",
+    'grade_score_reliability': "score_reliability",
+    'compare_score_reliability': "score_reliability",
+    'evaluate_fitted_calibrator': "score_reliability",
+    'report_fitted_calibrator': "score_reliability",
+    'make_research_candidate': 'research_candidate',
+    'load_research_candidate': 'research_candidate',
+    'list_candidates': 'catalog',
+    'list_datasets': 'catalog',
+    'get_dataset': 'catalog',
+    'CandidateConfiguration': 'integration_contracts',
+    'ComparisonIdentity': 'integration_contracts',
+    'ScoreField': 'integration_contracts',
+    'ScoreSemantics': 'integration_contracts',
+    'create_model_scorer': 'model_adapters',
+    'get_model_spec': 'model_adapters',
+    'preflight_model': 'model_adapters',
+    'create_recorded_scorer': 'recorded_scores',
+    'import_recorded_scores': 'recorded_scores',
+    'FrozenFittedState': 'rerank_family_contracts',
+    'load_fitted_state': 'rerank_family_contracts',
+    'create_statistical_scorer': 'statistical_rerank',
+    'score_signals': 'statistical_rerank',
+    'create_learned_scorer': 'learned_rerank',
+    'score_learned_signals': 'learned_rerank',
+    'RecordedHybridSignals': 'retrieval_candidates',
+    'make_retrieval_candidate': 'retrieval_candidates',
+    'retrieval_configuration': 'retrieval_candidates',
+    'import_hybrid_recordings': 'retrieval_recordings',
+    'list_public_pool_releases': 'public_rerank',
+    'load_public_pool_release': 'public_rerank',
+    'run_pipeline': 'pipeline',
+    'grade_pipeline': 'pipeline',
+    'report_pipeline': 'pipeline',
+    'compare_pipeline': 'pipeline',
+    'run_full_rag': 'full_rag',
+    'grade_full_rag': 'full_rag',
+    'report_full_rag': 'full_rag',
+    'compare_full_rag': 'full_rag',
+    'compare_full_rag_contexts': 'full_rag',
+    'create_full_rag_review_template': 'full_rag',
+    'workbench_track_runners': 'workbench_tracks',
+    'audit_legacy_cherry_full_rag': 'legacy_full_rag',
+}
+__all__ += list(_RESEARCH_EXPORTS)
+
+# These native modules are already loaded above. The aliases preserve their
+# exact grading-pass classes; a grading pass is not an individual assessment.
+GradingPassRef = EvaluationRef
+LocalGradingPassRef = LocalEvaluationRef
+InvocationGradingPassRef = InvocationEvaluationRef
+__all__ += ["GradingPassRef", "LocalGradingPassRef", "InvocationGradingPassRef"]
+
+# Explicit lazy aliases keep discovery independent of extension, backend, and
+# optional adapter imports. Each tuple names the authoritative module/object;
+# resolving an export never registers or executes a candidate or evaluator.
+_WORKFLOW_EXPORTS = {
+    "ReportSnapshot": ("workflow_decisions", "ReportSnapshot"),
+    "DecisionContext": ("workflow_decisions", "DecisionContext"),
+    "DecisionOutcome": ("workflow_decisions", "DecisionOutcome"),
+    "DecisionPolicyError": ("workflow_decisions", "DecisionPolicyError"),
+    "Verdict": ("workflow_decisions", "Verdict"),
+    "decide_report": ("workflow_decisions", "decide_report"),
+    "PrEvaluationRequest": ("pr_evaluations", "PrEvaluationRequest"),
+    "PrEvaluationResult": ("pr_evaluations", "PrEvaluationResult"),
+    "PrEvaluation": ("_pr_client", "PrEvaluation"),
+    "PrEvaluationError": ("_pr_client", "PrEvaluationError"),
+    "ProjectionPage": ("analytical_views", "ProjectionPage"),
+    "ProjectionRow": ("analytical_views", "ProjectionRow"),
+    "ProjectionEventRef": ("analytical_views", "ProjectionEventRef"),
+    "ProjectionResult": ("analytical_views", "ProjectionResult"),
+    "ProjectionSelection": ("analytical_views", "ProjectionSelection"),
+    "ProjectionProof": ("analytical_views", "ProjectionProof"),
+    "EvaluationBoardPage": ("analytical_views", "EvaluationBoardPage"),
+    "BoardEntry": ("analytical_views", "BoardEntry"),
+    "BoardRanking": ("analytical_views", "BoardRanking"),
+    "BoardCoverage": ("analytical_views", "BoardCoverage"),
+    "ProjectionCoverage": ("_analytical_values", "ProjectionCoverage"),
+    "ReportProjectionSummary": ("_analytical_values", "ReportProjectionSummary"),
+    "MetricProjectionSummary": ("_analytical_values", "MetricProjectionSummary"),
+    "ComparisonProjectionSummary": ("_analytical_values", "ComparisonProjectionSummary"),
+    "AuditProjectionSummary": ("_analytical_values", "AuditProjectionSummary"),
+    "ClickHouseReader": ("clickhouse_reader", "ClickHouseReader"),
+    "ReadOnlyClickHouseConfig": ("clickhouse_reader", "ReadOnlyClickHouseConfig"),
+    "ReadOnlyClickHouseError": ("clickhouse_reader", "ReadOnlyClickHouseError"),
+    "AnalyticalReadError": ("_analytical_client", "AnalyticalReadError"),
+    "BankReleaseDeclaration": ("audit_requests", "BankReleaseDeclaration"),
+    "ReviewAdjudication": ("audit_requests", "ReviewAdjudication"),
+    "BankReleaseTransition": ("audit_requests", "BankReleaseTransition"),
+    "ReviewTargetPage": ("audit_access", "ReviewTargetPage"),
+    "ReviewHistoryPage": ("audit_access", "ReviewHistoryPage"),
+    "TargetReviewPage": ("audit_access", "TargetReviewPage"),
+    "ReviewOperationReceipt": ("audit_access", "ReviewOperationReceipt"),
+    "AuditRequestError": ("_audit_client", "AuditRequestError"),
+    "ProgressView": ("progress", "ProgressView"),
+    "Diagnostic": ("diagnostics", "Diagnostic"),
+    "explain_error": ("diagnostics", "explain_error"),
+    "CallableCandidate": ("authoring", "CallableCandidate"),
+    "evaluation_suite": ("authoring", "evaluation_suite"),
+    "ComparisonPair": ("comparison_views", "ComparisonPair"),
+    "ComparisonMetric": ("comparison_views", "ComparisonMetric"),
+    "ComparisonView": ("comparison_views", "ComparisonView"),
+    "BankRelease": ("audit_contracts", "BankRelease"),
+    "ReviewTarget": ("audit_contracts", "ReviewTarget"),
+    "ReviewContribution": ("audit_contracts", "ReviewContribution"),
+    "ReviewEvent": ("audit_contracts", "ReviewEvent"),
+    "InputRequirement": ("evaluator_bindings", "InputRequirement"),
+    "EvaluatorBinding": ("evaluator_bindings", "EvaluatorBinding"),
+    "ReportAssessment": ("evaluator_bindings", "ReportAssessment"),
+    "VersionedIdentity": ("evaluation_protocols", "VersionedIdentity"),
+    "EvaluationProtocol": ("evaluation_protocols", "EvaluationProtocol"),
+    "ExecutionPath": ("execution_capabilities", "ExecutionPath"),
+    "validate_execution_requirement": ("execution_capabilities", "validate_execution_requirement"),
+    "AssessmentSelector": ("assessment_selection", "AssessmentSelector"),
+    "Suite": ("workflows", "Suite"),
+    "Plan": ("workflows", "Plan"),
+    "ResolvedPlan": ("workflows", "ResolvedPlan"),
+    "Admission": ("workflows", "Admission"),
+    "Run": ("workflows", "Run"),
+    "Report": ("workflows", "Report"),
+    "Submission": ("workflows", "Submission"),
+    "GradingPass": ("workflows", "GradingPass"),
+    "WorkflowTimeoutError": ("workflows", "WorkflowTimeoutError"),
+    "PendingWorkError": ("workflows", "PendingWorkError"),
+    "GradingRequestError": ("workflows", "GradingRequestError"),
+    "SubmissionIntent": ("workflow_async", "SubmissionIntent"),
+    "SubmissionSnapshot": ("workflow_async", "SubmissionSnapshot"),
+    "GradingSnapshot": ("workflow_async", "GradingSnapshot"),
+    "SubmissionError": ("workflow_async", "SubmissionError"),
+    "AsyncWorkflowBackend": ("workflow_async", "AsyncWorkflowBackend"),
+    "StageDefinition": ("stage_contracts", "StageDefinition"),
+    "StageReplacement": ("stage_contracts", "StageReplacement"),
+    "StageSelector": ("stage_contracts", "StageSelector"),
+    "StageEvidence": ("stage_contracts", "StageEvidence"),
+    "StageContractError": ("stage_contracts", "StageContractError"),
+    "StageEvidenceBackend": ("stage_contracts", "StageEvidenceBackend"),
+    "PackagedExtension": ("workflow_extensions", "PackagedExtension"),
+    "ExtensionError": ("workflow_extensions", "ExtensionError"),
+    "discover_extensions": ("workflow_extensions", "discover_extensions"),
+    "enable_extension": ("workflow_extensions", "enable_extension"),
+    "PullRequestCandidate": ("hosted_workflow", "PullRequestCandidate"),
+    "HostedWorkflowBackend": ("hosted_workflow", "HostedWorkflowBackend"),
+    "HostedComparison": ("hosted_workflow", "HostedComparison"),
+    "build_hosted_registry": ("hosted_workflow", "build_hosted_registry"),
+    "PipelineSource": ("pipeline_workflow", "PipelineSource"),
+    "PipelineCandidate": ("pipeline_workflow", "PipelineCandidate"),
+    "PipelineBackend": ("pipeline_workflow", "PipelineBackend"),
+    "build_pipeline_registry": ("pipeline_workflow", "build_pipeline_registry"),
+    "register_pipeline_backend": ("pipeline_workflow", "register_pipeline_backend"),
+    "RunLimits": ("lifecycle", "RunLimits"),
+    "RunPolicy": ("lifecycle", "RunPolicy"),
+    "GradingPolicy": ("lifecycle", "GradingPolicy"),
+    "CancellationToken": ("lifecycle", "CancellationToken"),
+    "DriverDescriptor": ("lifecycle", "DriverDescriptor"),
+    "DriverContext": ("lifecycle", "DriverContext"),
+    "AttemptOutput": ("lifecycle", "AttemptOutput"),
+    "DriverFailure": ("lifecycle", "DriverFailure"),
+    "LifecycleDriver": ("lifecycle", "LifecycleDriver"),
+    "LifecycleRunRef": ("lifecycle", "LifecycleRunRef"),
+    "LifecycleGradingRef": ("lifecycle", "LifecycleGradingRef"),
+    "GradingContext": ("lifecycle", "GradingContext"),
+    "LifecycleError": ("lifecycle", "LifecycleError"),
+    "execute_lifecycle": ("lifecycle", "execute_lifecycle"),
+    "assess_lifecycle": ("lifecycle", "assess_lifecycle"),
+    "open_lifecycle_run": ("lifecycle", "open_lifecycle_run"),
+    "verify_lifecycle_run": ("lifecycle", "verify_lifecycle_run"),
+    "verify_lifecycle_grading": ("lifecycle", "verify_lifecycle_grading"),
+    "LifecycleBackend": ("lifecycle_workflow", "LifecycleBackend"),
+    "LifecycleProjection": ("lifecycle_workflow", "LifecycleProjection"),
+    "IntegratedCandidate": ("integrated_rag", "IntegratedCandidate"),
+    "IntegratedDriver": ("integrated_rag", "IntegratedDriver"),
+    "run_integrated": ("integrated_rag", "run_integrated"),
+    "IntegratedBackend": ("integrated_workflow", "IntegratedBackend"),
+    "build_integrated_registry": ("integrated_workflow", "build_integrated_registry"),
+    "WorkflowError": ("workflows", "WorkflowError"),
+    "CaseView": ("assessment_contracts", "CaseView"),
+    "EvidenceView": ("assessment_contracts", "EvidenceView"),
+    "Assessment": ("assessment_contracts", "Assessment"),
+    "EvaluatorContext": ("assessment_contracts", "EvaluatorContext"),
+    "AssessmentContractError": ("assessment_contracts", "AssessmentContractError"),
+    "PayloadSchema": ("workflow_contracts", "PayloadSchema"),
+    "DatasetDefinition": ("workflow_contracts", "DatasetDefinition"),
+    "CaseProvider": ("workflow_contracts", "CaseProvider"),
+    "Dataset": ("workflow_contracts", "Dataset"),
+    "Criterion": ("workflow_contracts", "Criterion"),
+    "EvaluatorDefinition": ("workflow_contracts", "EvaluatorDefinition"),
+    "MetricSpec": ("workflow_contracts", "MetricSpec"),
+    "BackendDescriptor": ("workflow_contracts", "BackendDescriptor"),
+    "SuiteDefinition": ("workflow_contracts", "SuiteDefinition"),
+    "PlanBinding": ("workflow_contracts", "PlanBinding"),
+    "ResolvedBinding": ("workflow_contracts", "ResolvedBinding"),
+    "AdmissionBinding": ("workflow_contracts", "AdmissionBinding"),
+    "ReferenceBinding": ("workflow_contracts", "ReferenceBinding"),
+    "WorkflowBackend": ("workflow_contracts", "WorkflowBackend"),
+    "WorkflowContractError": ("workflow_contracts", "WorkflowContractError"),
+    "ReportProjection": ("workflow_contracts", "ReportProjection"),
+    "contract_digest": ("workflow_contracts", "contract_digest"),
+    "WorkflowRegistry": ("workflow_registry", "WorkflowRegistry"),
+    "WorkflowRegistryError": ("workflow_registry", "WorkflowRegistryError"),
+    "default_workflow_registry": ("workflow_defaults", "default_workflow_registry"),
+    "HostedSubmission": ("client", "HostedSubmission"),
+    "HostedGradingPass": ("client", "HostedGradingPass"),
+    "HostedReport": ("client", "HostedReport"),
+    "BatchConfiguration": ("components", "BatchConfiguration"),
+    "BatchAlternative": ("components", "BatchAlternative"),
+    "BatchPlan": ("components", "BatchPlan"),
+    "validate_batch_manifest": ("components", "validate_batch_manifest"),
+    "load_batch_manifest": ("components", "load_batch_manifest"),
+    "validate_batch": ("workbench", "validate_batch"),
+    "run_batch": ("workbench", "run_batch"),
+    "TrackGradingPassRef": ("integration_contracts", "TrackEvaluationRef"),
+}
+__all__ += list(_WORKFLOW_EXPORTS)
+
+
+def __getattr__(name):
+    binding = _WORKFLOW_EXPORTS.get(name)
+    if binding is None:
+        module_name = _RESEARCH_EXPORTS.get(name)
+        if module_name is None:
+            raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+        binding = (module_name, name)
+    from importlib import import_module
+    module_name, object_name = binding
+    value = getattr(import_module("." + module_name, __name__), object_name)
+    globals()[name] = value
+    return value
+
+
+def __dir__():
+    return sorted(set(globals()) | set(_RESEARCH_EXPORTS) | set(_WORKFLOW_EXPORTS))