experiential 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- experiential-0.4.0/.gitignore +42 -0
- experiential-0.4.0/PKG-INFO +132 -0
- experiential-0.4.0/README.md +101 -0
- experiential-0.4.0/assets/experiential-workflow.png +0 -0
- experiential-0.4.0/conftest.py +11 -0
- experiential-0.4.0/docs/reference/gateway-architecture.md +187 -0
- experiential-0.4.0/docs/release-scope.md +86 -0
- experiential-0.4.0/docs/usage.md +124 -0
- experiential-0.4.0/exp/__init__.py +262 -0
- experiential-0.4.0/exp/__main__.py +11 -0
- experiential-0.4.0/exp/cli/__init__.py +5 -0
- experiential-0.4.0/exp/cli/app.py +50 -0
- experiential-0.4.0/exp/cli/app_test.py +31 -0
- experiential-0.4.0/exp/cli/build/__init__.py +1 -0
- experiential-0.4.0/exp/cli/build/app.py +966 -0
- experiential-0.4.0/exp/cli/build/app_test.py +1607 -0
- experiential-0.4.0/exp/cli/build/cost.py +77 -0
- experiential-0.4.0/exp/cli/build/cost_test.py +29 -0
- experiential-0.4.0/exp/cli/build/wizard.py +893 -0
- experiential-0.4.0/exp/cli/build/wizard_screens.py +142 -0
- experiential-0.4.0/exp/cli/build/wizard_screens_test.py +89 -0
- experiential-0.4.0/exp/cli/build/wizard_test.py +1286 -0
- experiential-0.4.0/exp/cli/config/__init__.py +1 -0
- experiential-0.4.0/exp/cli/config/app.py +167 -0
- experiential-0.4.0/exp/cli/config/app_test.py +45 -0
- experiential-0.4.0/exp/cli/gateway/__init__.py +1 -0
- experiential-0.4.0/exp/cli/gateway/alias.py +348 -0
- experiential-0.4.0/exp/cli/gateway/app.py +498 -0
- experiential-0.4.0/exp/cli/gateway/app_test.py +1329 -0
- experiential-0.4.0/exp/cli/gateway/budget.py +185 -0
- experiential-0.4.0/exp/cli/gateway/budget_test.py +153 -0
- experiential-0.4.0/exp/cli/gateway/compatibility.py +121 -0
- experiential-0.4.0/exp/cli/gateway/key_output.py +532 -0
- experiential-0.4.0/exp/cli/gateway/pool.py +197 -0
- experiential-0.4.0/exp/cli/gateway/provider.py +166 -0
- experiential-0.4.0/exp/cli/gateway/receipts.py +74 -0
- experiential-0.4.0/exp/cli/gateway/setup.py +114 -0
- experiential-0.4.0/exp/cli/judge/__init__.py +1 -0
- experiential-0.4.0/exp/cli/judge/app.py +545 -0
- experiential-0.4.0/exp/cli/judge/app_test.py +399 -0
- experiential-0.4.0/exp/cli/judge/review.py +675 -0
- experiential-0.4.0/exp/cli/judge/review_test.py +777 -0
- experiential-0.4.0/exp/cli/judge/rubric.py +261 -0
- experiential-0.4.0/exp/cli/judge/rubric_test.py +130 -0
- experiential-0.4.0/exp/cli/judge/trace_viewer.py +581 -0
- experiential-0.4.0/exp/cli/judge/trace_viewer_test.py +190 -0
- experiential-0.4.0/exp/cli/judge/transcript.py +298 -0
- experiential-0.4.0/exp/cli/judge/transcript_test.py +1 -0
- experiential-0.4.0/exp/cli/optimize/__init__.py +1 -0
- experiential-0.4.0/exp/cli/optimize/app.py +28 -0
- experiential-0.4.0/exp/cli/optimize/model.py +613 -0
- experiential-0.4.0/exp/cli/optimize/model_test.py +1587 -0
- experiential-0.4.0/exp/cli/optimize/router.py +219 -0
- experiential-0.4.0/exp/cli/optimize/router_candidates.py +524 -0
- experiential-0.4.0/exp/cli/optimize/router_candidates_test.py +453 -0
- experiential-0.4.0/exp/cli/optimize/router_test.py +192 -0
- experiential-0.4.0/exp/cli/providers/__init__.py +1 -0
- experiential-0.4.0/exp/cli/providers/model_picker.py +830 -0
- experiential-0.4.0/exp/cli/providers/model_picker_test.py +524 -0
- experiential-0.4.0/exp/cli/providers/provider_picker.py +718 -0
- experiential-0.4.0/exp/cli/providers/provider_picker_test.py +681 -0
- experiential-0.4.0/exp/cli/providers/setup.py +780 -0
- experiential-0.4.0/exp/cli/providers/setup_test.py +1174 -0
- experiential-0.4.0/exp/cli/run/__init__.py +1 -0
- experiential-0.4.0/exp/cli/run/app.py +236 -0
- experiential-0.4.0/exp/cli/run/app_test.py +153 -0
- experiential-0.4.0/exp/cli/shared/__init__.py +1 -0
- experiential-0.4.0/exp/cli/shared/consent.py +299 -0
- experiential-0.4.0/exp/cli/shared/consent_test.py +446 -0
- experiential-0.4.0/exp/cli/shared/defer.py +101 -0
- experiential-0.4.0/exp/cli/shared/options.py +22 -0
- experiential-0.4.0/exp/cli/shared/options_test.py +2 -0
- experiential-0.4.0/exp/cli/shared/picker.py +814 -0
- experiential-0.4.0/exp/cli/shared/picker_test.py +919 -0
- experiential-0.4.0/exp/cli/shared/picker_view.py +277 -0
- experiential-0.4.0/exp/cli/shared/picker_view_test.py +268 -0
- experiential-0.4.0/exp/cli/shared/progress.py +237 -0
- experiential-0.4.0/exp/cli/shared/progress_test.py +146 -0
- experiential-0.4.0/exp/cli/shared/theme.py +31 -0
- experiential-0.4.0/exp/cli/shared/theme_test.py +53 -0
- experiential-0.4.0/exp/cli/tests/cli_help_test.py +64 -0
- experiential-0.4.0/exp/cli/tests/package_layout_test.py +125 -0
- experiential-0.4.0/exp/cli/tests/startup_test.py +147 -0
- experiential-0.4.0/exp/cli/tests/terminal_tasks_happy_path_test.py +516 -0
- experiential-0.4.0/exp/cli/tests/terminal_tasks_live_pipeline_test.py +447 -0
- experiential-0.4.0/exp/common/__init__.py +1 -0
- experiential-0.4.0/exp/common/config/__init__.py +23 -0
- experiential-0.4.0/exp/common/config/dotenv.py +31 -0
- experiential-0.4.0/exp/common/config/dotenv_test.py +25 -0
- experiential-0.4.0/exp/common/config/paths.py +5 -0
- experiential-0.4.0/exp/common/config/settings.py +192 -0
- experiential-0.4.0/exp/common/config/settings_test.py +201 -0
- experiential-0.4.0/exp/common/core/artifacts.py +435 -0
- experiential-0.4.0/exp/common/core/artifacts_test.py +239 -0
- experiential-0.4.0/exp/common/core/files.py +181 -0
- experiential-0.4.0/exp/common/core/files_test.py +375 -0
- experiential-0.4.0/exp/common/core/hashing.py +35 -0
- experiential-0.4.0/exp/common/core/hashing_test.py +26 -0
- experiential-0.4.0/exp/common/core/locks.py +97 -0
- experiential-0.4.0/exp/common/core/locks_test.py +130 -0
- experiential-0.4.0/exp/common/core/money.py +84 -0
- experiential-0.4.0/exp/common/core/text.py +32 -0
- experiential-0.4.0/exp/common/core/union_find.py +59 -0
- experiential-0.4.0/exp/common/core/union_find_test.py +17 -0
- experiential-0.4.0/exp/common/evaluations/__init__.py +46 -0
- experiential-0.4.0/exp/common/evaluations/build.py +532 -0
- experiential-0.4.0/exp/common/evaluations/build_test.py +852 -0
- experiential-0.4.0/exp/common/evaluations/dataset.py +291 -0
- experiential-0.4.0/exp/common/evaluations/dataset_test.py +227 -0
- experiential-0.4.0/exp/common/evaluations/evidence.py +229 -0
- experiential-0.4.0/exp/common/evaluations/fidelity.py +322 -0
- experiential-0.4.0/exp/common/evaluations/plan.py +144 -0
- experiential-0.4.0/exp/common/evaluations/plan_test.py +147 -0
- experiential-0.4.0/exp/common/evaluations/planning.py +487 -0
- experiential-0.4.0/exp/common/judging/__init__.py +111 -0
- experiential-0.4.0/exp/common/judging/calibration.py +961 -0
- experiential-0.4.0/exp/common/judging/calibration_assembly.py +65 -0
- experiential-0.4.0/exp/common/judging/calibration_contracts.py +170 -0
- experiential-0.4.0/exp/common/judging/calibration_metrics.py +387 -0
- experiential-0.4.0/exp/common/judging/calibration_metrics_test.py +66 -0
- experiential-0.4.0/exp/common/judging/calibration_provenance.py +103 -0
- experiential-0.4.0/exp/common/judging/calibration_test.py +954 -0
- experiential-0.4.0/exp/common/judging/display.py +95 -0
- experiential-0.4.0/exp/common/judging/display_test.py +50 -0
- experiential-0.4.0/exp/common/judging/evidence.py +141 -0
- experiential-0.4.0/exp/common/judging/evidence_test.py +234 -0
- experiential-0.4.0/exp/common/judging/interface.py +35 -0
- experiential-0.4.0/exp/common/judging/judgment.py +89 -0
- experiential-0.4.0/exp/common/judging/judgment_test.py +127 -0
- experiential-0.4.0/exp/common/judging/labels.py +581 -0
- experiential-0.4.0/exp/common/judging/labels_test.py +228 -0
- experiential-0.4.0/exp/common/judging/lineage.py +149 -0
- experiential-0.4.0/exp/common/judging/lm.py +530 -0
- experiential-0.4.0/exp/common/judging/lm_test.py +543 -0
- experiential-0.4.0/exp/common/judging/prompts.py +41 -0
- experiential-0.4.0/exp/common/judging/proposal.py +215 -0
- experiential-0.4.0/exp/common/judging/proposal_test.py +1 -0
- experiential-0.4.0/exp/common/judging/provenance.py +137 -0
- experiential-0.4.0/exp/common/judging/review.py +692 -0
- experiential-0.4.0/exp/common/judging/review_test.py +270 -0
- experiential-0.4.0/exp/common/judging/risk_acceptance.py +222 -0
- experiential-0.4.0/exp/common/judging/rubric.py +373 -0
- experiential-0.4.0/exp/common/judging/rubric_test.py +190 -0
- experiential-0.4.0/exp/common/judging/tests/calibration_persistence_test.py +52 -0
- experiential-0.4.0/exp/common/judging/tests/lm_authority_test.py +66 -0
- experiential-0.4.0/exp/common/models/__init__.py +185 -0
- experiential-0.4.0/exp/common/models/catalog.py +551 -0
- experiential-0.4.0/exp/common/models/catalog_test.py +528 -0
- experiential-0.4.0/exp/common/models/client.py +58 -0
- experiential-0.4.0/exp/common/models/client_test.py +1 -0
- experiential-0.4.0/exp/common/models/connection_authoring.py +86 -0
- experiential-0.4.0/exp/common/models/connection_authoring_test.py +108 -0
- experiential-0.4.0/exp/common/models/discovery.py +267 -0
- experiential-0.4.0/exp/common/models/discovery_test.py +186 -0
- experiential-0.4.0/exp/common/models/gateway_catalog.py +225 -0
- experiential-0.4.0/exp/common/models/gateway_catalog_test.py +326 -0
- experiential-0.4.0/exp/common/models/known_models.py +573 -0
- experiential-0.4.0/exp/common/models/known_models_test.py +256 -0
- experiential-0.4.0/exp/common/models/model.py +461 -0
- experiential-0.4.0/exp/common/models/model_test.py +279 -0
- experiential-0.4.0/exp/common/models/pricing.py +548 -0
- experiential-0.4.0/exp/common/models/pricing_test.py +291 -0
- experiential-0.4.0/exp/common/models/router_candidates.py +380 -0
- experiential-0.4.0/exp/common/models/router_candidates_test.py +340 -0
- experiential-0.4.0/exp/common/models/setup.py +352 -0
- experiential-0.4.0/exp/common/models/setup_test.py +297 -0
- experiential-0.4.0/exp/common/models/structured.py +36 -0
- experiential-0.4.0/exp/common/models/structured_test.py +59 -0
- experiential-0.4.0/exp/common/observability/__init__.py +15 -0
- experiential-0.4.0/exp/common/observability/telemetry.py +427 -0
- experiential-0.4.0/exp/common/observability/telemetry_test.py +403 -0
- experiential-0.4.0/exp/common/progress.py +66 -0
- experiential-0.4.0/exp/common/progress_test.py +42 -0
- experiential-0.4.0/exp/common/project/__init__.py +142 -0
- experiential-0.4.0/exp/common/project/bundle.py +932 -0
- experiential-0.4.0/exp/common/project/bundle_test.py +772 -0
- experiential-0.4.0/exp/common/project/catalog.py +141 -0
- experiential-0.4.0/exp/common/project/catalog_test.py +104 -0
- experiential-0.4.0/exp/common/project/events.py +94 -0
- experiential-0.4.0/exp/common/project/events_test.py +102 -0
- experiential-0.4.0/exp/common/project/hosted_state.py +308 -0
- experiential-0.4.0/exp/common/project/manifests.py +73 -0
- experiential-0.4.0/exp/common/project/paths.py +119 -0
- experiential-0.4.0/exp/common/project/paths_test.py +43 -0
- experiential-0.4.0/exp/common/project/project.py +383 -0
- experiential-0.4.0/exp/common/project/project_test.py +158 -0
- experiential-0.4.0/exp/common/project/store.py +990 -0
- experiential-0.4.0/exp/common/project/store_test.py +435 -0
- experiential-0.4.0/exp/common/release_revision.py +48 -0
- experiential-0.4.0/exp/common/release_revision_test.py +69 -0
- experiential-0.4.0/exp/common/rollouts/__init__.py +47 -0
- experiential-0.4.0/exp/common/rollouts/artifact.py +370 -0
- experiential-0.4.0/exp/common/rollouts/artifact_test.py +300 -0
- experiential-0.4.0/exp/common/rollouts/dispatch_failures.py +87 -0
- experiential-0.4.0/exp/common/rollouts/dispatch_failures_test.py +147 -0
- experiential-0.4.0/exp/common/rollouts/otel.py +82 -0
- experiential-0.4.0/exp/common/routing/__init__.py +40 -0
- experiential-0.4.0/exp/common/routing/bank.py +374 -0
- experiential-0.4.0/exp/common/routing/decision.py +371 -0
- experiential-0.4.0/exp/common/routing/embeddings.py +559 -0
- experiential-0.4.0/exp/common/routing/embeddings_test.py +625 -0
- experiential-0.4.0/exp/common/routing/features.py +120 -0
- experiential-0.4.0/exp/common/routing/features_test.py +68 -0
- experiential-0.4.0/exp/common/routing/policy.py +133 -0
- experiential-0.4.0/exp/common/routing/policy_test.py +151 -0
- experiential-0.4.0/exp/common/tasks/__init__.py +12 -0
- experiential-0.4.0/exp/common/tasks/store.py +68 -0
- experiential-0.4.0/exp/common/tasks/store_test.py +92 -0
- experiential-0.4.0/exp/common/tasks/task.py +90 -0
- experiential-0.4.0/exp/common/tasks/task_test.py +55 -0
- experiential-0.4.0/exp/common/tests/dependency_surface_test.py +79 -0
- experiential-0.4.0/exp/common/traces/__init__.py +14 -0
- experiential-0.4.0/exp/common/traces/store.py +134 -0
- experiential-0.4.0/exp/common/traces/store_test.py +178 -0
- experiential-0.4.0/exp/common/traces/trace.py +135 -0
- experiential-0.4.0/exp/common/traces/trace_test.py +96 -0
- experiential-0.4.0/exp/conftest.py +302 -0
- experiential-0.4.0/exp/optimize/__init__.py +1 -0
- experiential-0.4.0/exp/optimize/model/__init__.py +1 -0
- experiential-0.4.0/exp/optimize/model/sft/__init__.py +145 -0
- experiential-0.4.0/exp/optimize/model/sft/automatic.py +797 -0
- experiential-0.4.0/exp/optimize/model/sft/automatic_test.py +1106 -0
- experiential-0.4.0/exp/optimize/model/sft/builder.py +868 -0
- experiential-0.4.0/exp/optimize/model/sft/builder_test.py +1010 -0
- experiential-0.4.0/exp/optimize/model/sft/composition.py +948 -0
- experiential-0.4.0/exp/optimize/model/sft/composition_test.py +497 -0
- experiential-0.4.0/exp/optimize/model/sft/contracts.py +511 -0
- experiential-0.4.0/exp/optimize/model/sft/provider_resources.py +39 -0
- experiential-0.4.0/exp/optimize/model/sft/rendering.py +89 -0
- experiential-0.4.0/exp/optimize/model/sft/rendering_test.py +120 -0
- experiential-0.4.0/exp/optimize/model/sft/run_manifest.py +814 -0
- experiential-0.4.0/exp/optimize/model/sft/run_manifest_test.py +1 -0
- experiential-0.4.0/exp/optimize/model/sft/runtime_source.py +276 -0
- experiential-0.4.0/exp/optimize/model/sft/runtime_source_test.py +354 -0
- experiential-0.4.0/exp/optimize/model/sft/selection.py +386 -0
- experiential-0.4.0/exp/optimize/model/sft/selection_test.py +1 -0
- experiential-0.4.0/exp/optimize/model/sft/sources.py +568 -0
- experiential-0.4.0/exp/optimize/model/sft/tinker.py +407 -0
- experiential-0.4.0/exp/optimize/model/sft/tinker_test.py +371 -0
- experiential-0.4.0/exp/optimize/model/sft/training.py +865 -0
- experiential-0.4.0/exp/optimize/model/sft/training_contracts.py +414 -0
- experiential-0.4.0/exp/optimize/model/sft/training_runtime.py +221 -0
- experiential-0.4.0/exp/optimize/model/sft/training_test.py +622 -0
- experiential-0.4.0/exp/optimize/router/__init__.py +150 -0
- experiential-0.4.0/exp/optimize/router/activation.py +159 -0
- experiential-0.4.0/exp/optimize/router/activation_test.py +98 -0
- experiential-0.4.0/exp/optimize/router/attempt_authority.py +729 -0
- experiential-0.4.0/exp/optimize/router/attempt_authority_test.py +85 -0
- experiential-0.4.0/exp/optimize/router/automatic/__init__.py +1 -0
- experiential-0.4.0/exp/optimize/router/automatic/artifacts.py +340 -0
- experiential-0.4.0/exp/optimize/router/automatic/artifacts_test.py +1 -0
- experiential-0.4.0/exp/optimize/router/automatic/attribution.py +578 -0
- experiential-0.4.0/exp/optimize/router/automatic/attribution_test.py +426 -0
- experiential-0.4.0/exp/optimize/router/automatic/execution_contract.py +415 -0
- experiential-0.4.0/exp/optimize/router/automatic/execution_contract_test.py +1 -0
- experiential-0.4.0/exp/optimize/router/automatic/judge.py +350 -0
- experiential-0.4.0/exp/optimize/router/automatic/judge_provenance.py +325 -0
- experiential-0.4.0/exp/optimize/router/automatic/judge_provenance_test.py +1 -0
- experiential-0.4.0/exp/optimize/router/automatic/judge_test.py +274 -0
- experiential-0.4.0/exp/optimize/router/automatic/preflight.py +793 -0
- experiential-0.4.0/exp/optimize/router/automatic/preflight_test.py +157 -0
- experiential-0.4.0/exp/optimize/router/automatic/provisional.py +471 -0
- experiential-0.4.0/exp/optimize/router/automatic/replay.py +646 -0
- experiential-0.4.0/exp/optimize/router/automatic/replay_test.py +1 -0
- experiential-0.4.0/exp/optimize/router/automatic/reservations.py +751 -0
- experiential-0.4.0/exp/optimize/router/automatic/reservations_test.py +495 -0
- experiential-0.4.0/exp/optimize/router/automatic/service.py +738 -0
- experiential-0.4.0/exp/optimize/router/automatic/service_test.py +2050 -0
- experiential-0.4.0/exp/optimize/router/composition.py +759 -0
- experiential-0.4.0/exp/optimize/router/composition_test.py +2011 -0
- experiential-0.4.0/exp/optimize/router/errors.py +32 -0
- experiential-0.4.0/exp/optimize/router/errors_test.py +0 -0
- experiential-0.4.0/exp/optimize/router/evaluation/__init__.py +1 -0
- experiential-0.4.0/exp/optimize/router/evaluation/build.py +123 -0
- experiential-0.4.0/exp/optimize/router/evaluation/build_test.py +1 -0
- experiential-0.4.0/exp/optimize/router/evaluation/setup.py +50 -0
- experiential-0.4.0/exp/optimize/router/evaluation/setup_test.py +1 -0
- experiential-0.4.0/exp/optimize/router/evaluation/simulation_spec.py +91 -0
- experiential-0.4.0/exp/optimize/router/evaluation/simulation_spec_test.py +129 -0
- experiential-0.4.0/exp/optimize/router/evaluation/spend.py +199 -0
- experiential-0.4.0/exp/optimize/router/evaluation/spend_test.py +243 -0
- experiential-0.4.0/exp/optimize/router/fit/__init__.py +1 -0
- experiential-0.4.0/exp/optimize/router/fit/optimizer.py +910 -0
- experiential-0.4.0/exp/optimize/router/fit/optimizer_test.py +1142 -0
- experiential-0.4.0/exp/optimize/router/fit/report.py +640 -0
- experiential-0.4.0/exp/optimize/router/fit/report_test.py +1 -0
- experiential-0.4.0/exp/optimize/router/fit/spec.py +81 -0
- experiential-0.4.0/exp/optimize/router/fit/spec_test.py +1 -0
- experiential-0.4.0/exp/optimize/router/fit/tests/bank_test.py +313 -0
- experiential-0.4.0/exp/optimize/router/fit/tests/decision_test.py +294 -0
- experiential-0.4.0/exp/optimize/router/fit/workflow.py +320 -0
- experiential-0.4.0/exp/optimize/router/fit/workflow_test.py +310 -0
- experiential-0.4.0/exp/optimize/router/hosted.py +944 -0
- experiential-0.4.0/exp/optimize/router/hosted_preflight.py +638 -0
- experiential-0.4.0/exp/optimize/router/hosted_spend.py +285 -0
- experiential-0.4.0/exp/optimize/router/hosted_stages.py +149 -0
- experiential-0.4.0/exp/optimize/router/hosted_test.py +2009 -0
- experiential-0.4.0/exp/optimize/router/hosted_verification.py +719 -0
- experiential-0.4.0/exp/optimize/router/judging/__init__.py +1 -0
- experiential-0.4.0/exp/optimize/router/judging/artifacts.py +940 -0
- experiential-0.4.0/exp/optimize/router/judging/artifacts_test.py +137 -0
- experiential-0.4.0/exp/optimize/router/judging/contracts.py +932 -0
- experiential-0.4.0/exp/optimize/router/judging/contracts_test.py +102 -0
- experiential-0.4.0/exp/optimize/router/judging/labels.py +207 -0
- experiential-0.4.0/exp/optimize/router/judging/labels_test.py +130 -0
- experiential-0.4.0/exp/optimize/router/judging/pricing.py +103 -0
- experiential-0.4.0/exp/optimize/router/judging/pricing_test.py +226 -0
- experiential-0.4.0/exp/optimize/router/judging/protocol.py +777 -0
- experiential-0.4.0/exp/optimize/router/judging/protocol_test.py +445 -0
- experiential-0.4.0/exp/optimize/router/judging/provisional.py +108 -0
- experiential-0.4.0/exp/optimize/router/judging/provisional_test.py +1 -0
- experiential-0.4.0/exp/optimize/router/judging/review.py +954 -0
- experiential-0.4.0/exp/optimize/router/judging/review_test.py +476 -0
- experiential-0.4.0/exp/optimize/router/judging/selection.py +179 -0
- experiential-0.4.0/exp/optimize/router/judging/selection_test.py +1 -0
- experiential-0.4.0/exp/optimize/router/judging/service.py +920 -0
- experiential-0.4.0/exp/optimize/router/judging/service_test.py +1619 -0
- experiential-0.4.0/exp/optimize/router/judging/setup_store.py +83 -0
- experiential-0.4.0/exp/optimize/router/judging/setup_store_test.py +5 -0
- experiential-0.4.0/exp/optimize/router/judging/template_bind.py +112 -0
- experiential-0.4.0/exp/optimize/router/judging/template_bind_test.py +53 -0
- experiential-0.4.0/exp/optimize/router/judgment_budget.py +866 -0
- experiential-0.4.0/exp/optimize/router/judgment_budget_test.py +140 -0
- experiential-0.4.0/exp/optimize/router/spend.py +353 -0
- experiential-0.4.0/exp/optimize/router/spend_test.py +53 -0
- experiential-0.4.0/exp/optimize/router/tests/composition_evidence_test.py +668 -0
- experiential-0.4.0/exp/runtime/agents/__init__.py +83 -0
- experiential-0.4.0/exp/runtime/agents/chat.py +252 -0
- experiential-0.4.0/exp/runtime/agents/chat_test.py +246 -0
- experiential-0.4.0/exp/runtime/agents/factory.py +145 -0
- experiential-0.4.0/exp/runtime/agents/factory_test.py +85 -0
- experiential-0.4.0/exp/runtime/agents/interface.py +97 -0
- experiential-0.4.0/exp/runtime/agents/interface_test.py +76 -0
- experiential-0.4.0/exp/runtime/agents/lifecycle.py +230 -0
- experiential-0.4.0/exp/runtime/agents/lifecycle_test.py +465 -0
- experiential-0.4.0/exp/runtime/agents/pi.py +841 -0
- experiential-0.4.0/exp/runtime/agents/pi_test.py +610 -0
- experiential-0.4.0/exp/runtime/environments/__init__.py +49 -0
- experiential-0.4.0/exp/runtime/environments/harbor.py +562 -0
- experiential-0.4.0/exp/runtime/environments/harbor_test.py +978 -0
- experiential-0.4.0/exp/runtime/environments/interface.py +54 -0
- experiential-0.4.0/exp/runtime/environments/interface_test.py +1 -0
- experiential-0.4.0/exp/runtime/environments/local.py +998 -0
- experiential-0.4.0/exp/runtime/environments/local_exec_gate.py +27 -0
- experiential-0.4.0/exp/runtime/environments/local_test.py +466 -0
- experiential-0.4.0/exp/runtime/environments/sandbox_ledger.py +272 -0
- experiential-0.4.0/exp/runtime/environments/sandbox_ledger_test.py +105 -0
- experiential-0.4.0/exp/runtime/gateway/__init__.py +203 -0
- experiential-0.4.0/exp/runtime/gateway/aggregation.py +49 -0
- experiential-0.4.0/exp/runtime/gateway/auth.py +241 -0
- experiential-0.4.0/exp/runtime/gateway/auth_test.py +58 -0
- experiential-0.4.0/exp/runtime/gateway/budgets.py +664 -0
- experiential-0.4.0/exp/runtime/gateway/budgets_test.py +518 -0
- experiential-0.4.0/exp/runtime/gateway/catalog_authority.py +446 -0
- experiential-0.4.0/exp/runtime/gateway/composition.py +235 -0
- experiential-0.4.0/exp/runtime/gateway/composition_test.py +347 -0
- experiential-0.4.0/exp/runtime/gateway/contracts.py +341 -0
- experiential-0.4.0/exp/runtime/gateway/contracts_test.py +148 -0
- experiential-0.4.0/exp/runtime/gateway/execution.py +891 -0
- experiential-0.4.0/exp/runtime/gateway/health.py +143 -0
- experiential-0.4.0/exp/runtime/gateway/interfaces.py +145 -0
- experiential-0.4.0/exp/runtime/gateway/ledger.py +806 -0
- experiential-0.4.0/exp/runtime/gateway/ledger_test.py +854 -0
- experiential-0.4.0/exp/runtime/gateway/lifecycle.py +554 -0
- experiential-0.4.0/exp/runtime/gateway/lifecycle_test.py +787 -0
- experiential-0.4.0/exp/runtime/gateway/management.py +791 -0
- experiential-0.4.0/exp/runtime/gateway/platform.py +846 -0
- experiential-0.4.0/exp/runtime/gateway/platform_test.py +554 -0
- experiential-0.4.0/exp/runtime/gateway/project_activation.py +310 -0
- experiential-0.4.0/exp/runtime/gateway/project_activation_test.py +94 -0
- experiential-0.4.0/exp/runtime/gateway/project_alias.py +168 -0
- experiential-0.4.0/exp/runtime/gateway/project_alias_test.py +142 -0
- experiential-0.4.0/exp/runtime/gateway/provider_certification.py +277 -0
- experiential-0.4.0/exp/runtime/gateway/provider_certification_test.py +87 -0
- experiential-0.4.0/exp/runtime/gateway/routing.py +342 -0
- experiential-0.4.0/exp/runtime/gateway/secrets.py +45 -0
- experiential-0.4.0/exp/runtime/gateway/secrets_test.py +18 -0
- experiential-0.4.0/exp/runtime/gateway/service.py +911 -0
- experiential-0.4.0/exp/runtime/gateway/sqlite/__init__.py +5 -0
- experiential-0.4.0/exp/runtime/gateway/sqlite/alias_activation.py +166 -0
- experiential-0.4.0/exp/runtime/gateway/sqlite/key_delivery.py +158 -0
- experiential-0.4.0/exp/runtime/gateway/sqlite/migrations.py +694 -0
- experiential-0.4.0/exp/runtime/gateway/sqlite/migrations_test.py +604 -0
- experiential-0.4.0/exp/runtime/gateway/sqlite/platform.py +988 -0
- experiential-0.4.0/exp/runtime/gateway/sqlite/platform_records.py +224 -0
- experiential-0.4.0/exp/runtime/gateway/sqlite/platform_records_test.py +1 -0
- experiential-0.4.0/exp/runtime/gateway/sqlite/platform_test.py +689 -0
- experiential-0.4.0/exp/runtime/gateway/sqlite/provider_authority.py +327 -0
- experiential-0.4.0/exp/runtime/gateway/sqlite/provider_store.py +161 -0
- experiential-0.4.0/exp/runtime/gateway/sqlite/store.py +975 -0
- experiential-0.4.0/exp/runtime/gateway/sqlite/store_test.py +1191 -0
- experiential-0.4.0/exp/runtime/gateway/tests/data_plane_test.py +1333 -0
- experiential-0.4.0/exp/runtime/gateway/tests/launch_test.py +642 -0
- experiential-0.4.0/exp/runtime/gateway/tests/waterfall_test.py +1148 -0
- experiential-0.4.0/exp/runtime/gateway/usage.py +175 -0
- experiential-0.4.0/exp/runtime/models/__init__.py +18 -0
- experiential-0.4.0/exp/runtime/models/credentials.py +42 -0
- experiential-0.4.0/exp/runtime/models/preflight.py +83 -0
- experiential-0.4.0/exp/runtime/models/providers/__init__.py +72 -0
- experiential-0.4.0/exp/runtime/models/providers/anthropic.py +308 -0
- experiential-0.4.0/exp/runtime/models/providers/async_transport.py +674 -0
- experiential-0.4.0/exp/runtime/models/providers/async_transport_test.py +162 -0
- experiential-0.4.0/exp/runtime/models/providers/azure.py +179 -0
- experiential-0.4.0/exp/runtime/models/providers/azure_test.py +265 -0
- experiential-0.4.0/exp/runtime/models/providers/base.py +304 -0
- experiential-0.4.0/exp/runtime/models/providers/base_test.py +210 -0
- experiential-0.4.0/exp/runtime/models/providers/bedrock.py +737 -0
- experiential-0.4.0/exp/runtime/models/providers/bedrock_streaming.py +461 -0
- experiential-0.4.0/exp/runtime/models/providers/bedrock_streaming_test.py +350 -0
- experiential-0.4.0/exp/runtime/models/providers/bedrock_test.py +613 -0
- experiential-0.4.0/exp/runtime/models/providers/errors.py +249 -0
- experiential-0.4.0/exp/runtime/models/providers/errors_test.py +99 -0
- experiential-0.4.0/exp/runtime/models/providers/gemini.py +354 -0
- experiential-0.4.0/exp/runtime/models/providers/gemini_streaming.py +241 -0
- experiential-0.4.0/exp/runtime/models/providers/gemini_streaming_test.py +247 -0
- experiential-0.4.0/exp/runtime/models/providers/listing.py +323 -0
- experiential-0.4.0/exp/runtime/models/providers/listing_test.py +261 -0
- experiential-0.4.0/exp/runtime/models/providers/openai.py +364 -0
- experiential-0.4.0/exp/runtime/models/providers/openai_compatible.py +395 -0
- experiential-0.4.0/exp/runtime/models/providers/openai_compatible_test.py +348 -0
- experiential-0.4.0/exp/runtime/models/providers/protocol.py +221 -0
- experiential-0.4.0/exp/runtime/models/providers/protocol_test.py +158 -0
- experiential-0.4.0/exp/runtime/models/providers/stream_attempts.py +128 -0
- experiential-0.4.0/exp/runtime/models/providers/streaming.py +970 -0
- experiential-0.4.0/exp/runtime/models/providers/streaming_requests.py +310 -0
- experiential-0.4.0/exp/runtime/models/providers/streaming_requests_test.py +118 -0
- experiential-0.4.0/exp/runtime/models/providers/streaming_test.py +1208 -0
- experiential-0.4.0/exp/runtime/models/providers/streaming_usage.py +85 -0
- experiential-0.4.0/exp/runtime/models/providers/tests/later_provider_streaming_test.py +164 -0
- experiential-0.4.0/exp/runtime/models/providers/tests/native_test.py +608 -0
- experiential-0.4.0/exp/runtime/models/providers/tinker_sampling.py +334 -0
- experiential-0.4.0/exp/runtime/models/providers/transport.py +434 -0
- experiential-0.4.0/exp/runtime/models/providers/transport_test.py +116 -0
- experiential-0.4.0/exp/runtime/models/registry.py +415 -0
- experiential-0.4.0/exp/runtime/models/registry_test.py +425 -0
- experiential-0.4.0/exp/runtime/openai_protocol/__init__.py +21 -0
- experiential-0.4.0/exp/runtime/openai_protocol/errors.py +166 -0
- experiential-0.4.0/exp/runtime/openai_protocol/errors_test.py +26 -0
- experiential-0.4.0/exp/runtime/openai_protocol/headers.py +127 -0
- experiential-0.4.0/exp/runtime/openai_protocol/headers_test.py +48 -0
- experiential-0.4.0/exp/runtime/openai_protocol/manifest.py +147 -0
- experiential-0.4.0/exp/runtime/openai_protocol/manifest_test.py +42 -0
- experiential-0.4.0/exp/runtime/openai_protocol/model_adapter.py +137 -0
- experiential-0.4.0/exp/runtime/openai_protocol/model_adapter_test.py +101 -0
- experiential-0.4.0/exp/runtime/openai_protocol/requests.py +559 -0
- experiential-0.4.0/exp/runtime/openai_protocol/requests_test.py +263 -0
- experiential-0.4.0/exp/runtime/openai_protocol/response.py +191 -0
- experiential-0.4.0/exp/runtime/openai_protocol/state.py +533 -0
- experiential-0.4.0/exp/runtime/openai_protocol/state_test.py +267 -0
- experiential-0.4.0/exp/runtime/openai_protocol/streaming.py +727 -0
- experiential-0.4.0/exp/runtime/openai_protocol/streaming_test.py +206 -0
- experiential-0.4.0/exp/runtime/openai_protocol/tests/sdk_harness_test.py +246 -0
- experiential-0.4.0/exp/runtime/router/__init__.py +123 -0
- experiential-0.4.0/exp/runtime/router/application.py +170 -0
- experiential-0.4.0/exp/runtime/router/application_test.py +118 -0
- experiential-0.4.0/exp/runtime/router/capability.py +219 -0
- experiential-0.4.0/exp/runtime/router/capability_test.py +170 -0
- experiential-0.4.0/exp/runtime/router/completion.py +149 -0
- experiential-0.4.0/exp/runtime/router/completion_test.py +139 -0
- experiential-0.4.0/exp/runtime/router/economics.py +217 -0
- experiential-0.4.0/exp/runtime/router/errors.py +5 -0
- experiential-0.4.0/exp/runtime/router/journal.py +977 -0
- experiential-0.4.0/exp/runtime/router/journal_handoff.py +47 -0
- experiential-0.4.0/exp/runtime/router/journal_handoff_test.py +1 -0
- experiential-0.4.0/exp/runtime/router/journal_io.py +77 -0
- experiential-0.4.0/exp/runtime/router/journal_service.py +379 -0
- experiential-0.4.0/exp/runtime/router/journal_spend.py +350 -0
- experiential-0.4.0/exp/runtime/router/journal_test.py +1424 -0
- experiential-0.4.0/exp/runtime/router/journal_validation.py +417 -0
- experiential-0.4.0/exp/runtime/router/runtime.py +926 -0
- experiential-0.4.0/exp/runtime/router/runtime_selection.py +133 -0
- experiential-0.4.0/exp/runtime/router/runtime_support.py +453 -0
- experiential-0.4.0/exp/runtime/router/runtime_test.py +1327 -0
- experiential-0.4.0/exp/runtime/router/snapshot.py +889 -0
- experiential-0.4.0/exp/runtime/router/snapshot_test.py +983 -0
- experiential-0.4.0/exp/runtime/router/tests/runtime_capability_test.py +198 -0
- experiential-0.4.0/exp/runtime/router/tests/runtime_idempotency_test.py +73 -0
- experiential-0.4.0/exp/simulation/__init__.py +39 -0
- experiential-0.4.0/exp/simulation/build.py +670 -0
- experiential-0.4.0/exp/simulation/build_test.py +501 -0
- experiential-0.4.0/exp/simulation/comparison.py +782 -0
- experiential-0.4.0/exp/simulation/comparison_test.py +768 -0
- experiential-0.4.0/exp/simulation/engines/__init__.py +11 -0
- experiential-0.4.0/exp/simulation/engines/clock.py +32 -0
- experiential-0.4.0/exp/simulation/engines/clock_test.py +29 -0
- experiential-0.4.0/exp/simulation/engines/sandbox.py +916 -0
- experiential-0.4.0/exp/simulation/engines/sandbox_bindings.py +243 -0
- experiential-0.4.0/exp/simulation/engines/sandbox_recording.py +748 -0
- experiential-0.4.0/exp/simulation/engines/sandbox_test.py +996 -0
- experiential-0.4.0/exp/simulation/engines/tests/sandbox_regression_test.py +271 -0
- experiential-0.4.0/exp/simulation/engines/text/__init__.py +29 -0
- experiential-0.4.0/exp/simulation/engines/text/artifact_set.py +119 -0
- experiential-0.4.0/exp/simulation/engines/text/artifact_set_test.py +0 -0
- experiential-0.4.0/exp/simulation/engines/text/bindings.py +260 -0
- experiential-0.4.0/exp/simulation/engines/text/cell_progress.py +54 -0
- experiential-0.4.0/exp/simulation/engines/text/cell_progress_test.py +52 -0
- experiential-0.4.0/exp/simulation/engines/text/environment.py +66 -0
- experiential-0.4.0/exp/simulation/engines/text/episode_loop.py +111 -0
- experiential-0.4.0/exp/simulation/engines/text/errors.py +13 -0
- experiential-0.4.0/exp/simulation/engines/text/errors_test.py +0 -0
- experiential-0.4.0/exp/simulation/engines/text/grounded_rollout.py +141 -0
- experiential-0.4.0/exp/simulation/engines/text/grounded_rollout_test.py +0 -0
- experiential-0.4.0/exp/simulation/engines/text/grounding.py +336 -0
- experiential-0.4.0/exp/simulation/engines/text/grounding_test.py +0 -0
- experiential-0.4.0/exp/simulation/engines/text/leases.py +701 -0
- experiential-0.4.0/exp/simulation/engines/text/leases_test.py +487 -0
- experiential-0.4.0/exp/simulation/engines/text/prompt.py +172 -0
- experiential-0.4.0/exp/simulation/engines/text/prompt_test.py +84 -0
- experiential-0.4.0/exp/simulation/engines/text/recording.py +890 -0
- experiential-0.4.0/exp/simulation/engines/text/recording_test.py +730 -0
- experiential-0.4.0/exp/simulation/engines/text/redaction.py +261 -0
- experiential-0.4.0/exp/simulation/engines/text/redaction_test.py +242 -0
- experiential-0.4.0/exp/simulation/engines/text/resume.py +229 -0
- experiential-0.4.0/exp/simulation/engines/text/resume_test.py +1 -0
- experiential-0.4.0/exp/simulation/engines/text/rollout_support.py +170 -0
- experiential-0.4.0/exp/simulation/engines/text/simulator.py +977 -0
- experiential-0.4.0/exp/simulation/engines/text/simulator_test.py +2173 -0
- experiential-0.4.0/exp/simulation/engines/text/spec_persistence.py +60 -0
- experiential-0.4.0/exp/simulation/engines/text/spec_persistence_test.py +5 -0
- experiential-0.4.0/exp/simulation/ingest/__init__.py +64 -0
- experiential-0.4.0/exp/simulation/ingest/braintrust.py +260 -0
- experiential-0.4.0/exp/simulation/ingest/braintrust_test.py +69 -0
- experiential-0.4.0/exp/simulation/ingest/chat_json.py +318 -0
- experiential-0.4.0/exp/simulation/ingest/chat_json_test.py +124 -0
- experiential-0.4.0/exp/simulation/ingest/dataset.py +376 -0
- experiential-0.4.0/exp/simulation/ingest/dataset_test.py +477 -0
- experiential-0.4.0/exp/simulation/ingest/environment_capture.py +415 -0
- experiential-0.4.0/exp/simulation/ingest/environment_capture_test.py +321 -0
- experiential-0.4.0/exp/simulation/ingest/json_strict.py +31 -0
- experiential-0.4.0/exp/simulation/ingest/json_strict_test.py +25 -0
- experiential-0.4.0/exp/simulation/ingest/langfuse.py +355 -0
- experiential-0.4.0/exp/simulation/ingest/langfuse_test.py +147 -0
- experiential-0.4.0/exp/simulation/ingest/langsmith.py +233 -0
- experiential-0.4.0/exp/simulation/ingest/langsmith_test.py +77 -0
- experiential-0.4.0/exp/simulation/ingest/mastra.py +237 -0
- experiential-0.4.0/exp/simulation/ingest/mastra_test.py +83 -0
- experiential-0.4.0/exp/simulation/ingest/model_identity.py +279 -0
- experiential-0.4.0/exp/simulation/ingest/model_identity_test.py +1 -0
- experiential-0.4.0/exp/simulation/ingest/otel_genai.py +297 -0
- experiential-0.4.0/exp/simulation/ingest/otel_genai_test.py +152 -0
- experiential-0.4.0/exp/simulation/ingest/otlp.py +740 -0
- experiential-0.4.0/exp/simulation/ingest/otlp_test.py +569 -0
- experiential-0.4.0/exp/simulation/ingest/phoenix.py +574 -0
- experiential-0.4.0/exp/simulation/ingest/phoenix_test.py +151 -0
- experiential-0.4.0/exp/simulation/ingest/posthog.py +16 -0
- experiential-0.4.0/exp/simulation/ingest/posthog_canonical.py +620 -0
- experiential-0.4.0/exp/simulation/ingest/posthog_canonical_test.py +686 -0
- experiential-0.4.0/exp/simulation/ingest/posthog_pull.py +214 -0
- experiential-0.4.0/exp/simulation/ingest/sources.py +98 -0
- experiential-0.4.0/exp/simulation/ingest/sources_test.py +217 -0
- experiential-0.4.0/exp/simulation/ingest/testdata/sample_otlp.json +60 -0
- experiential-0.4.0/exp/simulation/ingest/testdata/sample_spans.jsonl +3 -0
- experiential-0.4.0/exp/simulation/ingest/tests/loader_contract_test.py +560 -0
- experiential-0.4.0/exp/simulation/ingest/trace_extensions.py +355 -0
- experiential-0.4.0/exp/simulation/ingest/trace_extensions_test.py +39 -0
- experiential-0.4.0/exp/simulation/ingest/vendor_observations.py +390 -0
- experiential-0.4.0/exp/simulation/ingest/vendor_observations_test.py +85 -0
- experiential-0.4.0/exp/simulation/ingest/vendor_records.py +434 -0
- experiential-0.4.0/exp/simulation/ingest/vendor_records_test.py +122 -0
- experiential-0.4.0/exp/simulation/ingest/vendor_source.py +155 -0
- experiential-0.4.0/exp/simulation/ingest/vendor_source_test.py +82 -0
- experiential-0.4.0/exp/simulation/ingest/vendor_trace.py +669 -0
- experiential-0.4.0/exp/simulation/ingest/vendor_trace_test.py +184 -0
- experiential-0.4.0/exp/simulation/mining/__init__.py +1 -0
- experiential-0.4.0/exp/simulation/mining/bindings.py +411 -0
- experiential-0.4.0/exp/simulation/mining/bindings_test.py +1 -0
- experiential-0.4.0/exp/simulation/mining/cleanup.py +323 -0
- experiential-0.4.0/exp/simulation/mining/coverage.py +359 -0
- experiential-0.4.0/exp/simulation/mining/deduplicate.py +500 -0
- experiential-0.4.0/exp/simulation/mining/descriptors.py +315 -0
- experiential-0.4.0/exp/simulation/mining/lineage.py +126 -0
- experiential-0.4.0/exp/simulation/mining/select.py +490 -0
- experiential-0.4.0/exp/simulation/mining/service.py +385 -0
- experiential-0.4.0/exp/simulation/mining/service_test.py +513 -0
- experiential-0.4.0/exp/simulation/orchestration/__init__.py +9 -0
- experiential-0.4.0/exp/simulation/orchestration/interface.py +45 -0
- experiential-0.4.0/exp/simulation/retrieval/__init__.py +119 -0
- experiential-0.4.0/exp/simulation/retrieval/build.py +265 -0
- experiential-0.4.0/exp/simulation/retrieval/build_inputs.py +62 -0
- experiential-0.4.0/exp/simulation/retrieval/build_inputs_test.py +813 -0
- experiential-0.4.0/exp/simulation/retrieval/contracts.py +254 -0
- experiential-0.4.0/exp/simulation/retrieval/embedding.py +130 -0
- experiential-0.4.0/exp/simulation/retrieval/refresh.py +794 -0
- experiential-0.4.0/exp/simulation/retrieval/refresh_dataset.py +211 -0
- experiential-0.4.0/exp/simulation/retrieval/refresh_dataset_test.py +0 -0
- experiential-0.4.0/exp/simulation/retrieval/refresh_test.py +1342 -0
- experiential-0.4.0/exp/simulation/retrieval/retriever.py +177 -0
- experiential-0.4.0/exp/simulation/retrieval/runtime_stitching.py +399 -0
- experiential-0.4.0/exp/simulation/retrieval/runtime_stitching_test.py +0 -0
- experiential-0.4.0/exp/simulation/retrieval/store.py +212 -0
- experiential-0.4.0/exp/simulation/retrieval/tests/retrieval_test.py +670 -0
- experiential-0.4.0/exp/simulation/retrieval/transitions.py +305 -0
- experiential-0.4.0/exp/simulation/specs/__init__.py +27 -0
- experiential-0.4.0/exp/simulation/specs/completion.py +221 -0
- experiential-0.4.0/exp/simulation/specs/completion_test.py +115 -0
- experiential-0.4.0/exp/simulation/specs/simulation.py +145 -0
- experiential-0.4.0/exp/simulation/specs/simulation_test.py +129 -0
- experiential-0.4.0/exp/simulation/tests/comparison_evidence_test.py +578 -0
- experiential-0.4.0/exp/simulation/world_model/__init__.py +91 -0
- experiential-0.4.0/exp/simulation/world_model/__init___test.py +0 -0
- experiential-0.4.0/exp/simulation/world_model/application.py +475 -0
- experiential-0.4.0/exp/simulation/world_model/application_test.py +572 -0
- experiential-0.4.0/exp/simulation/world_model/artifact.py +193 -0
- experiential-0.4.0/exp/simulation/world_model/artifact_test.py +0 -0
- experiential-0.4.0/exp/simulation/world_model/runtime.py +400 -0
- experiential-0.4.0/exp/simulation/world_model/runtime_test.py +315 -0
- experiential-0.4.0/exp/tests/api_test.py +231 -0
- experiential-0.4.0/exp/tests/release_revision_test.py +184 -0
- experiential-0.4.0/exp/tests/release_test.py +3184 -0
- experiential-0.4.0/exp/tests/repo_import_boundaries_test.py +235 -0
- experiential-0.4.0/exp/tests/repo_layout_test.py +147 -0
- experiential-0.4.0/pyproject.toml +87 -0
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
__pycache__/
|
|
2
|
+
*.py[cod]
|
|
3
|
+
.venv/
|
|
4
|
+
.env
|
|
5
|
+
.env.*
|
|
6
|
+
!.env.example
|
|
7
|
+
.exp/
|
|
8
|
+
dist/
|
|
9
|
+
*.egg-info/
|
|
10
|
+
.pytest_cache/
|
|
11
|
+
.ruff_cache/
|
|
12
|
+
.mypy_cache/
|
|
13
|
+
.ty_cache/
|
|
14
|
+
.DS_Store
|
|
15
|
+
|
|
16
|
+
# Local-only design notes; not committed.
|
|
17
|
+
DESIGN.md
|
|
18
|
+
|
|
19
|
+
demos/
|
|
20
|
+
|
|
21
|
+
# baseline-grid run scratch (not artifacts)
|
|
22
|
+
.exp-rebuild/
|
|
23
|
+
logs/
|
|
24
|
+
|
|
25
|
+
.exp-fixed/
|
|
26
|
+
|
|
27
|
+
# Local per-project settings (telemetry ids etc.) — never committed, wherever the root is.
|
|
28
|
+
settings.toml
|
|
29
|
+
|
|
30
|
+
# Claude Code local state (skills ARE committed; local settings/locks are not).
|
|
31
|
+
.claude/settings.local.json
|
|
32
|
+
.claude/scheduled_tasks.lock
|
|
33
|
+
.claude/projects/
|
|
34
|
+
|
|
35
|
+
# wandb run directories (local mirrors of what is already on the server)
|
|
36
|
+
wandb/
|
|
37
|
+
|
|
38
|
+
# External clones land in the working directory, never in git.
|
|
39
|
+
.tau2-bench/
|
|
40
|
+
|
|
41
|
+
# Local agent scratchpads: never tracked, never in PRs (AGENTS.md, Repository layout).
|
|
42
|
+
.agents/
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: experiential
|
|
3
|
+
Version: 0.4.0
|
|
4
|
+
Summary: Build simulations and optimize model routing from agent traces.
|
|
5
|
+
Requires-Python: >=3.12
|
|
6
|
+
Requires-Dist: boto3<2,>=1.35
|
|
7
|
+
Requires-Dist: botocore<2,>=1.35
|
|
8
|
+
Requires-Dist: click>=8.2
|
|
9
|
+
Requires-Dist: fastapi>=0.128
|
|
10
|
+
Requires-Dist: filelock>=3.12
|
|
11
|
+
Requires-Dist: httpx>=0.27
|
|
12
|
+
Requires-Dist: numpy>=1.26
|
|
13
|
+
Requires-Dist: openai<4,>=3.0
|
|
14
|
+
Requires-Dist: posthog>=7.0
|
|
15
|
+
Requires-Dist: pydantic>=2.6
|
|
16
|
+
Requires-Dist: rich>=14.1
|
|
17
|
+
Requires-Dist: tomli-w>=1.0
|
|
18
|
+
Requires-Dist: typer>=0.16
|
|
19
|
+
Requires-Dist: uvicorn>=0.38
|
|
20
|
+
Provides-Extra: dev
|
|
21
|
+
Requires-Dist: pytest-xdist>=3.8; extra == 'dev'
|
|
22
|
+
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
23
|
+
Requires-Dist: ruff>=0.5; extra == 'dev'
|
|
24
|
+
Requires-Dist: tinker-cookbook<0.5,>=0.4.3; extra == 'dev'
|
|
25
|
+
Requires-Dist: tinker<0.24,>=0.23; extra == 'dev'
|
|
26
|
+
Requires-Dist: ty>=0.0.1a1; extra == 'dev'
|
|
27
|
+
Provides-Extra: sft
|
|
28
|
+
Requires-Dist: tinker-cookbook<0.5,>=0.4.3; extra == 'sft'
|
|
29
|
+
Requires-Dist: tinker<0.24,>=0.23; extra == 'sft'
|
|
30
|
+
Description-Content-Type: text/markdown
|
|
31
|
+
|
|
32
|
+
# Experiential
|
|
33
|
+
|
|
34
|
+
Experiential is an open source gateway and router for agent workflows:
|
|
35
|
+
|
|
36
|
+
1. Use hosted, BYOK, and local models through one OpenAI-compatible API.
|
|
37
|
+
2. Control which users and agents can use which models, for which use cases, and how much they can spend.
|
|
38
|
+
3. Turn production traffic into a custom router or model optimized for quality, speed, and cost.
|
|
39
|
+
|
|
40
|
+

|
|
41
|
+
|
|
42
|
+
<p align="center">
|
|
43
|
+
🌐 <a href="https://platform.experientiallabs.ai">Platform</a> |
|
|
44
|
+
📚 <a href="https://github.com/experientiallabs/experiential/tree/main/docs">Docs</a> |
|
|
45
|
+
<a href="https://discord.gg/B6sM8xTVwU"><img src="https://cdn.simpleicons.org/discord/5865F2" alt="" width="16" height="16"> Discord</a>
|
|
46
|
+
</p>
|
|
47
|
+
|
|
48
|
+
## Getting Started
|
|
49
|
+
|
|
50
|
+
Start a local OpenAI-compatible gateway. On first run, the setup wizard asks for a provider,
|
|
51
|
+
model, and public alias, then prints a one-time virtual key:
|
|
52
|
+
|
|
53
|
+
```bash
|
|
54
|
+
pip install experiential
|
|
55
|
+
exp run
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
Choose a public alias such as `support-agent`, capture the issued key, and send a request:
|
|
59
|
+
|
|
60
|
+
```bash
|
|
61
|
+
export EXP_GATEWAY_KEY=...
|
|
62
|
+
curl http://127.0.0.1:8000/v1/chat/completions \
|
|
63
|
+
-H "Authorization: Bearer $EXP_GATEWAY_KEY" \
|
|
64
|
+
-H 'Content-Type: application/json' \
|
|
65
|
+
-d '{"model":"support-agent","messages":[{"role":"user","content":"Help me"}]}'
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
## Using the API
|
|
69
|
+
|
|
70
|
+
Create a local gateway programmatically:
|
|
71
|
+
|
|
72
|
+
```python
|
|
73
|
+
import uvicorn
|
|
74
|
+
|
|
75
|
+
from exp.runtime.gateway.lifecycle import load_local_gateway
|
|
76
|
+
|
|
77
|
+
gateway = load_local_gateway()
|
|
78
|
+
uvicorn.run(gateway.app, lifespan="on")
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
For hosted workers with their own storage and provider services, use the lower-level
|
|
82
|
+
`exp.create_gateway_runtime(...)` composition API.
|
|
83
|
+
|
|
84
|
+
## Optimize from Traffic
|
|
85
|
+
|
|
86
|
+
First, collect OpenTelemetry traces from your current agent. If you just want to try it out, grab
|
|
87
|
+
the public [terminal-tasks OTLP dataset](https://huggingface.co/datasets/experiential-labs/wmo-terminal-tasks-traces):
|
|
88
|
+
|
|
89
|
+
```bash
|
|
90
|
+
curl -L -o traces.otel.jsonl \
|
|
91
|
+
https://huggingface.co/datasets/experiential-labs/wmo-terminal-tasks-traces/resolve/540883e451dc13d34fb50fdd36b143cb0f1fb0db/traces.otel.jsonl
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
Then build a project. The build command walks you through providers,
|
|
95
|
+
models, and budget, and asks for your trace file:
|
|
96
|
+
|
|
97
|
+
```bash
|
|
98
|
+
# Build simulation from your agent traces and optimize a router against it
|
|
99
|
+
exp build support-agent
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
After collecting traces from your router, fine-tune an open source model you own using
|
|
103
|
+
[Tinker](https://tinker.thinkingmachines.ai/).
|
|
104
|
+
|
|
105
|
+
```bash
|
|
106
|
+
exp optimize model support-agent
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
## Telemetry
|
|
110
|
+
|
|
111
|
+
Anonymous aggregate PostHog product telemetry is enabled by default. It never includes prompts,
|
|
112
|
+
traces, actions, observations, paths, model names, credentials, or raw customer content.
|
|
113
|
+
|
|
114
|
+
```bash
|
|
115
|
+
exp config telemetry status
|
|
116
|
+
exp config telemetry disable
|
|
117
|
+
exp config telemetry enable
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
The preference is stored locally in `.exp/settings.toml`.
|
|
121
|
+
|
|
122
|
+
## Development
|
|
123
|
+
|
|
124
|
+
```bash
|
|
125
|
+
uv sync --extra dev
|
|
126
|
+
uv run ruff format --check .
|
|
127
|
+
uv run ruff check .
|
|
128
|
+
uv run ty check
|
|
129
|
+
uv run pytest -q
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
Repository and documentation conventions live in [AGENTS.md](./AGENTS.md).
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
# Experiential
|
|
2
|
+
|
|
3
|
+
Experiential is an open source gateway and router for agent workflows:
|
|
4
|
+
|
|
5
|
+
1. Use hosted, BYOK, and local models through one OpenAI-compatible API.
|
|
6
|
+
2. Control which users and agents can use which models, for which use cases, and how much they can spend.
|
|
7
|
+
3. Turn production traffic into a custom router or model optimized for quality, speed, and cost.
|
|
8
|
+
|
|
9
|
+

|
|
10
|
+
|
|
11
|
+
<p align="center">
|
|
12
|
+
🌐 <a href="https://platform.experientiallabs.ai">Platform</a> |
|
|
13
|
+
📚 <a href="https://github.com/experientiallabs/experiential/tree/main/docs">Docs</a> |
|
|
14
|
+
<a href="https://discord.gg/B6sM8xTVwU"><img src="https://cdn.simpleicons.org/discord/5865F2" alt="" width="16" height="16"> Discord</a>
|
|
15
|
+
</p>
|
|
16
|
+
|
|
17
|
+
## Getting Started
|
|
18
|
+
|
|
19
|
+
Start a local OpenAI-compatible gateway. On first run, the setup wizard asks for a provider,
|
|
20
|
+
model, and public alias, then prints a one-time virtual key:
|
|
21
|
+
|
|
22
|
+
```bash
|
|
23
|
+
pip install experiential
|
|
24
|
+
exp run
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
Choose a public alias such as `support-agent`, capture the issued key, and send a request:
|
|
28
|
+
|
|
29
|
+
```bash
|
|
30
|
+
export EXP_GATEWAY_KEY=...
|
|
31
|
+
curl http://127.0.0.1:8000/v1/chat/completions \
|
|
32
|
+
-H "Authorization: Bearer $EXP_GATEWAY_KEY" \
|
|
33
|
+
-H 'Content-Type: application/json' \
|
|
34
|
+
-d '{"model":"support-agent","messages":[{"role":"user","content":"Help me"}]}'
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
## Using the API
|
|
38
|
+
|
|
39
|
+
Create a local gateway programmatically:
|
|
40
|
+
|
|
41
|
+
```python
|
|
42
|
+
import uvicorn
|
|
43
|
+
|
|
44
|
+
from exp.runtime.gateway.lifecycle import load_local_gateway
|
|
45
|
+
|
|
46
|
+
gateway = load_local_gateway()
|
|
47
|
+
uvicorn.run(gateway.app, lifespan="on")
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
For hosted workers with their own storage and provider services, use the lower-level
|
|
51
|
+
`exp.create_gateway_runtime(...)` composition API.
|
|
52
|
+
|
|
53
|
+
## Optimize from Traffic
|
|
54
|
+
|
|
55
|
+
First, collect OpenTelemetry traces from your current agent. If you just want to try it out, grab
|
|
56
|
+
the public [terminal-tasks OTLP dataset](https://huggingface.co/datasets/experiential-labs/wmo-terminal-tasks-traces):
|
|
57
|
+
|
|
58
|
+
```bash
|
|
59
|
+
curl -L -o traces.otel.jsonl \
|
|
60
|
+
https://huggingface.co/datasets/experiential-labs/wmo-terminal-tasks-traces/resolve/540883e451dc13d34fb50fdd36b143cb0f1fb0db/traces.otel.jsonl
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
Then build a project. The build command walks you through providers,
|
|
64
|
+
models, and budget, and asks for your trace file:
|
|
65
|
+
|
|
66
|
+
```bash
|
|
67
|
+
# Build simulation from your agent traces and optimize a router against it
|
|
68
|
+
exp build support-agent
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
After collecting traces from your router, fine-tune an open source model you own using
|
|
72
|
+
[Tinker](https://tinker.thinkingmachines.ai/).
|
|
73
|
+
|
|
74
|
+
```bash
|
|
75
|
+
exp optimize model support-agent
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
## Telemetry
|
|
79
|
+
|
|
80
|
+
Anonymous aggregate PostHog product telemetry is enabled by default. It never includes prompts,
|
|
81
|
+
traces, actions, observations, paths, model names, credentials, or raw customer content.
|
|
82
|
+
|
|
83
|
+
```bash
|
|
84
|
+
exp config telemetry status
|
|
85
|
+
exp config telemetry disable
|
|
86
|
+
exp config telemetry enable
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
The preference is stored locally in `.exp/settings.toml`.
|
|
90
|
+
|
|
91
|
+
## Development
|
|
92
|
+
|
|
93
|
+
```bash
|
|
94
|
+
uv sync --extra dev
|
|
95
|
+
uv run ruff format --check .
|
|
96
|
+
uv run ruff check .
|
|
97
|
+
uv run ty check
|
|
98
|
+
uv run pytest -q
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
Repository and documentation conventions live in [AGENTS.md](./AGENTS.md).
|
|
Binary file
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"""Repo-wide pytest configuration."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
|
|
7
|
+
# Rich consoles snapshot color support when constructed, and `exp.cli.app` builds its console at
|
|
8
|
+
# import time — so color-forcing vars must go before any test module imports it, or a dev shell
|
|
9
|
+
# exporting FORCE_COLOR would inject ANSI codes into CliRunner captures and fail assertions.
|
|
10
|
+
os.environ.pop("FORCE_COLOR", None)
|
|
11
|
+
os.environ.pop("CLICOLOR_FORCE", None)
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
# Local gateway architecture
|
|
2
|
+
|
|
3
|
+
## Supported surface
|
|
4
|
+
|
|
5
|
+
No-argument `exp run` starts an authenticated multi-alias gateway on `127.0.0.1`.
|
|
6
|
+
It serves:
|
|
7
|
+
|
|
8
|
+
- `GET /v1/models`
|
|
9
|
+
- `POST /v1/chat/completions`
|
|
10
|
+
- `POST /v1/responses`
|
|
11
|
+
- `GET /health/live` and `GET /health/ready`
|
|
12
|
+
- `GET /usage` and `GET /usage.json`
|
|
13
|
+
|
|
14
|
+
`exp run PROJECT` is compatibility sugar that activates one project-backed alias and launches this
|
|
15
|
+
same gateway application. It does not create a router HTTP server. Gateway startup and readiness
|
|
16
|
+
perform no provider request. Only an authorized model request may cross the provider boundary.
|
|
17
|
+
|
|
18
|
+
## Embeddable worker composition
|
|
19
|
+
|
|
20
|
+
Platform workers use the public `create_gateway_runtime` seam. The worker supplies storage,
|
|
21
|
+
provider, secret, project-selection, clock, readiness, and usage implementations, then owns the
|
|
22
|
+
returned lifecycle handle:
|
|
23
|
+
|
|
24
|
+
```python
|
|
25
|
+
runtime = exp.create_gateway_runtime(
|
|
26
|
+
config=exp.GatewayRuntimeConfig(graceful_timeout_seconds=10),
|
|
27
|
+
authority=authority,
|
|
28
|
+
ledger=ledger,
|
|
29
|
+
routes=routes,
|
|
30
|
+
executor=executor,
|
|
31
|
+
clock=clock,
|
|
32
|
+
readiness=readiness,
|
|
33
|
+
usage=usage,
|
|
34
|
+
replay=replay,
|
|
35
|
+
continuations=continuations,
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
app = runtime.app
|
|
39
|
+
# Serve app with the platform's ASGI worker, including ASGI lifespan.
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
The factory performs no filesystem, SQLite, environment, lock, or server access. A worker may use
|
|
43
|
+
any `SecretResolver` while constructing its provider executor and any `ProjectTargetResolver`
|
|
44
|
+
while constructing its catalog route resolver. The composed application always mounts the same
|
|
45
|
+
`create_gateway_app` data plane used by the local CLI. The lifecycle exposes explicit preflight,
|
|
46
|
+
readiness, bounded drain, and shutdown operations in addition to its ASGI lifespan.
|
|
47
|
+
|
|
48
|
+
An ASGI host that drives application lifespan owns preflight and shutdown automatically. A host
|
|
49
|
+
that does not drive lifespan calls `runtime.preflight()` before admission and
|
|
50
|
+
`runtime.shutdown()` during teardown. Shutdown is idempotent across those paths: one runtime starts
|
|
51
|
+
one bounded drain and one terminal flush, then remains permanently not ready.
|
|
52
|
+
|
|
53
|
+
## Authority and management
|
|
54
|
+
|
|
55
|
+
`exp config gateway` owns explicit local setup. Its provider, identity, key, grant, alias, pool, and
|
|
56
|
+
monthly budget commands produce versioned receipts suitable for interactive or non-interactive
|
|
57
|
+
callers. There are no runtime seeds. A usable installation requires an organization, active
|
|
58
|
+
identity, active virtual key, explicit identity-to-alias grant, active alias revision, immutable
|
|
59
|
+
catalog snapshot, and a resolvable provider credential reference.
|
|
60
|
+
|
|
61
|
+
Private serving authority lives in `ROOT/gateway/gateway.db`, including identities, keys, grants,
|
|
62
|
+
provider connections and revisions, aliases and revisions, attempts, and usage. SQLite uses WAL
|
|
63
|
+
mode, versioned forward
|
|
64
|
+
migrations, private backups before migration, newer-schema refusal, and serialized initialization.
|
|
65
|
+
Virtual keys are stored only as peppered fingerprints. Key material is delivered once in a JSON
|
|
66
|
+
receipt or to a new mode-`0600` file, and commit ambiguity preserves recoverability. Provider
|
|
67
|
+
configuration stores an environment variable name, never its value. The local pepper is mode
|
|
68
|
+
`0600` and is not exported.
|
|
69
|
+
|
|
70
|
+
Every data-plane request is authenticated and authorized before request decoding, routing,
|
|
71
|
+
continuation lookup, or provider work. Authorization freezes organization, identity, API surface,
|
|
72
|
+
alias revision, target, catalog digest, request digest, optional hashed operation identity, and one
|
|
73
|
+
monotonic deadline. Identity disable, key revocation or expiry, grant removal, and alias revision
|
|
74
|
+
changes fail closed.
|
|
75
|
+
|
|
76
|
+
## Catalog, aliases, and exact-model pools
|
|
77
|
+
|
|
78
|
+
The gateway database owns current provider connection state. Existing model metadata remains the
|
|
79
|
+
authoring input for builds, policies, evaluations, and datasets; it is not consulted as mutable
|
|
80
|
+
serving authority after an alias revision binds exact connection revisions. Gateway snapshots under
|
|
81
|
+
`ROOT/gateway/catalog-snapshots/` are immutable, secret-free artifacts for an exact catalog digest.
|
|
82
|
+
An advertised alias is executable only when readiness holds for the exact tuple of alias name,
|
|
83
|
+
alias revision, and catalog digest used by authorization.
|
|
84
|
+
|
|
85
|
+
An alias targets either:
|
|
86
|
+
|
|
87
|
+
1. a direct exact-model pool, or
|
|
88
|
+
2. one immutable project activation.
|
|
89
|
+
|
|
90
|
+
A singleton alias creates a one-deployment pool. `exp config gateway pool certify` can replace that
|
|
91
|
+
with an ordered pool only when every member has the same exact logical model identity and an
|
|
92
|
+
operator-supplied equivalence certification. The certification records an ID, provenance, evidence
|
|
93
|
+
digest, time, and exact deployment order. Project policy selection still chooses one exact logical
|
|
94
|
+
model; operational fallback can only move among certified deployments for that model.
|
|
95
|
+
|
|
96
|
+
## Request, route, and provider attempts
|
|
97
|
+
|
|
98
|
+
The content-free ledger accepts the logical request before learned project selection. Selection or
|
|
99
|
+
direct resolution then produces an execution snapshot containing the exact model, pool, and ordered
|
|
100
|
+
deployment IDs. Each physical provider dispatch gets its own durable attempt row immediately before
|
|
101
|
+
network work. Attempt ordinal counts all physical dispatches; route depth identifies the selected
|
|
102
|
+
deployment position.
|
|
103
|
+
|
|
104
|
+
Provider execution is always internally streaming. Bounded same-deployment retries and ordered
|
|
105
|
+
deployment fallback are allowed only for typed precommit failures. The first outward text, refusal,
|
|
106
|
+
or tool-call semantic event commits the deployment, after which the gateway never switches
|
|
107
|
+
providers. Typed refusal fallback is disabled unless the active alias revision explicitly enables
|
|
108
|
+
it. Opted-in refusal deltas are withheld only in a bounded in-memory buffer: a refusal-only terminal
|
|
109
|
+
result can advance to the next certified deployment, while mixed semantic output or buffer overflow
|
|
110
|
+
commits and flushes the original route. Provider-internal retry layers are disabled so every
|
|
111
|
+
possible billable dispatch is visible to the gateway ledger.
|
|
112
|
+
|
|
113
|
+
Before each physical dispatch, the same immediate SQLite transaction reserves the request's
|
|
114
|
+
conservative maximum integer micro-USD cost and inserts its attempt row. Applicable hard limits can
|
|
115
|
+
cover the local team, one identity, one alias pool, and each provider deployment within that pool.
|
|
116
|
+
An exhausted deployment allocation removes only that route from the current certified waterfall.
|
|
117
|
+
If no route can fit the shared team, identity, or total pool allocation, the neutral protocol
|
|
118
|
+
returns HTTP 429 with OpenAI `insufficient_quota` semantics before provider work. Any required
|
|
119
|
+
unknown price makes that route ineligible while a hard limit applies.
|
|
120
|
+
|
|
121
|
+
Settlement replaces the reservation with observed integer micro-USD usage. A dispatched failure,
|
|
122
|
+
cancellation, or crash without trustworthy usage retains its conservative reservation because it
|
|
123
|
+
may be billable. Retries and fallbacks therefore consume one allocation entry per physical attempt,
|
|
124
|
+
while keyed replay creates no new reservation. A period is the immutable UTC bucket beginning at
|
|
125
|
+
`YYYY-MM-01T00:00:00+00:00`; rollover selects a new bucket and never clears or rewrites an earlier
|
|
126
|
+
month. Management and remaining-allocation reports are CLI surfaces only. There is no budgets
|
|
127
|
+
dashboard.
|
|
128
|
+
|
|
129
|
+
Each physical attempt records its own provider, model, usage, latency, terminal state, estimated
|
|
130
|
+
cost attribution, and frozen credential-ownership billing source. Later catalog activation and
|
|
131
|
+
process restart never rewrite that source. Schema-v1/v2 attempt rows migrate explicitly as
|
|
132
|
+
`customer_managed`; current dispatches persist either `host_managed` or `customer_managed` before
|
|
133
|
+
network work. The public usage report conserves physical attempt, token, cost, unknown-cost, and
|
|
134
|
+
terminal totals across those source buckets without partitioning logical request counts. The parent
|
|
135
|
+
request terminalizes once after success, final failure, cancellation, disconnect, or crash
|
|
136
|
+
reconciliation. Unknown prices remain unknown instead of being treated as zero or copied across
|
|
137
|
+
deployments.
|
|
138
|
+
|
|
139
|
+
## OpenAI-compatible protocol
|
|
140
|
+
|
|
141
|
+
`exp/runtime/openai_protocol` is the only OpenAI wire implementation. Chat Completions and
|
|
142
|
+
Responses have separate allowlist decoders and field-specific OpenAI error responses, but both
|
|
143
|
+
convert to one canonical gateway request without conflating their wire contracts. The package also
|
|
144
|
+
owns headers, response assembly, SSE framing, tool-call reconstruction, and official SDK
|
|
145
|
+
compatibility.
|
|
146
|
+
Chat streaming emits valid completion chunks and one `[DONE]`. Responses streaming emits the
|
|
147
|
+
created, in-progress, output, and exactly one terminal lifecycle. Provider tool-argument fragments
|
|
148
|
+
are accumulated in original order and validated only at the complete-call boundary.
|
|
149
|
+
|
|
150
|
+
Commit-independent headers are available before streaming begins. Route-dependent headers are
|
|
151
|
+
emitted only after an execution snapshot exists. Stable public IDs do not expose raw key,
|
|
152
|
+
idempotency, request, or provider values.
|
|
153
|
+
|
|
154
|
+
OpenAI `3.0.0` `OpenAI` and `AsyncOpenAI` clients are release-certified for Chat Completions and
|
|
155
|
+
Responses in synchronous and asynchronous, streaming and non-streaming forms. Responses
|
|
156
|
+
continuation and duplicate replay retain content only in bounded, process-local, tenant and
|
|
157
|
+
alias-revision-scoped stores. Replay is opt-in through an idempotency or client request key. Restart
|
|
158
|
+
or eviction returns an explicit unavailable error and never reconstructs content from SQLite.
|
|
159
|
+
|
|
160
|
+
## Content-free observability and lifecycle
|
|
161
|
+
|
|
162
|
+
SQLite stores hashes, frozen authority, route identity, state transitions, token counts, latency,
|
|
163
|
+
and estimated cost. It never stores prompts, responses, raw tool arguments, raw virtual keys, or
|
|
164
|
+
provider secrets. `GET /usage` and `GET /usage.json` are two renderings of the same schema-v2 report
|
|
165
|
+
and expose only aggregate, per-identity, and physical-attempt `by_billing_source` accounting.
|
|
166
|
+
Source buckets conserve attempt, token, known-cost, unknown-cost, and terminal-state totals but do
|
|
167
|
+
not partition logical request counts. Estimated cost is attribution, not a provider invoice.
|
|
168
|
+
|
|
169
|
+
The process owns readiness from preflight through bounded drain. New work is rejected after drain
|
|
170
|
+
starts. Admitted tasks, upstream streams, disconnect cleanup, replay ownership, continuation state,
|
|
171
|
+
and final ledger settlement are process-owned and bounded. A stuck cancellation cannot prevent the
|
|
172
|
+
terminal flusher from attempting content-free settlement.
|
|
173
|
+
|
|
174
|
+
## Certification boundary
|
|
175
|
+
|
|
176
|
+
Deterministic release evidence uses a built and freshly installed wheel, real SQLite, a real
|
|
177
|
+
subprocess-bound loopback gateway, a real loopback upstream, and the official SDK clients. One
|
|
178
|
+
scanner checks database, WAL, backups when present, catalog snapshots, stdout, stderr, logs, usage
|
|
179
|
+
responses, and error bodies for raw content and secret canaries.
|
|
180
|
+
|
|
181
|
+
`exp/runtime/gateway/provider_certification.py` is the dated provider capability matrix. Each cell
|
|
182
|
+
names the official client SDK, public gateway surfaces, provider wire surface, fixture result, and
|
|
183
|
+
credential-gated live status. OpenAI and Anthropic have native fixtures; generic OpenAI-compatible,
|
|
184
|
+
Azure, and OpenRouter share compatible-stream coverage; Gemini and Bedrock have native deterministic
|
|
185
|
+
fixtures. Live provider cells remain explicitly `not_run_requires_credentials` until a separately
|
|
186
|
+
authorized run supplies dated evidence. Deterministic fixtures do not imply hosted-provider
|
|
187
|
+
availability, billing, or account-specific behavior.
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
# Release scope
|
|
2
|
+
|
|
3
|
+
This release supports the current source and one wheel with either core dependencies or the
|
|
4
|
+
optional `sft` dependency extra on their documented local paths. It claims only behavior exercised
|
|
5
|
+
on the exact release checkout.
|
|
6
|
+
|
|
7
|
+
## Supported and verified
|
|
8
|
+
|
|
9
|
+
- Root CLI commands are exactly `build`, `config`, `optimize`, and `run`; optimizer commands are
|
|
10
|
+
exactly `router` and `model`.
|
|
11
|
+
- The local gateway supports explicit provider references, identities, virtual keys, grants,
|
|
12
|
+
singleton and certified ordered exact-model pools, frozen-project aliases, bounded precommit
|
|
13
|
+
provider fallback, Chat Completions, Responses, bounded in-memory continuation and replay,
|
|
14
|
+
content-free SQLite accounting, monthly integer micro-USD enforcement, and loopback-only health
|
|
15
|
+
and usage views.
|
|
16
|
+
- Both no-argument gateway launch and the retained `exp run PROJECT [--ghost]` compatibility form
|
|
17
|
+
are installed-wheel surfaces. Gateway startup is provider-idle and requires explicit authority.
|
|
18
|
+
- The installed-wheel gateway lane uses real SQLite, a real subprocess listener, a real loopback
|
|
19
|
+
upstream, and OpenAI `3.0.0`. It covers `OpenAI` and `AsyncOpenAI` across Chat Completions and
|
|
20
|
+
Responses, with both stream and non-stream requests. HTML and JSON usage are checked for the same
|
|
21
|
+
per-identity accounting values. The same lane measures default SDK retries against physical
|
|
22
|
+
attempts, provider-authentication fallback, refusal policy, post-commit no-switch, restart and
|
|
23
|
+
replay behavior, revocation, cancellation, WAL mode, and mixed `host_managed`/
|
|
24
|
+
`customer_managed` cost attribution that remains frozen across restart and catalog replacement.
|
|
25
|
+
- Real-SQLite monthly budget evidence covers atomic concurrent reservations, provider-only route
|
|
26
|
+
exhaustion, shared identity quota errors, billable failure and crash retention, retry and
|
|
27
|
+
fallback accounting, keyed replay without duplicate spend, explicit schema migration, and UTC
|
|
28
|
+
month rollover without a reset job. The real loopback waterfall is configured through the
|
|
29
|
+
interactive-capable CLI and returns OpenAI `insufficient_quota` after shared exhaustion.
|
|
30
|
+
- Deterministic source-level certification additionally covers selection-only project routing,
|
|
31
|
+
authentication-circuit open/skip/recovery, concurrent multi-identity key revocation and alias
|
|
32
|
+
revision activation, and atomic rollback of a failed legacy SQLite migration.
|
|
33
|
+
- One content and secret canary scanner covers the gateway database, live WAL, migration backups
|
|
34
|
+
when present, catalog snapshots, stdout, stderr, logs, usage responses, and HTTP error bodies.
|
|
35
|
+
- Public Python exposes provider-free build, explicit router composition, frozen selection-only
|
|
36
|
+
router load through the normal gateway application, structural text-versus-sandbox comparison,
|
|
37
|
+
and managed SFT composition. No separate router HTTP or SSE implementation is shipped.
|
|
38
|
+
- W16 router evidence uses 100 normalized traces, 50 fit tasks, 20 held-out tasks, 140 planned
|
|
39
|
+
cells, 130 deterministic text simulations, and 140 deterministic judgments under one finite
|
|
40
|
+
simulation and judgment budget. Observed hosted-service spend is exactly $0.00.
|
|
41
|
+
- W16 sandbox evidence compares two exact post-lock text and Darwin local-process pairs. It retains
|
|
42
|
+
one malformed sandbox failure in the denominator and claims structural terminal agreement only.
|
|
43
|
+
- Exact-checkout CI supplies the full 40-hex Git revision, recursively verifies every evidence
|
|
44
|
+
artifact and manifest input, and publishes machine-readable JSON plus JUnit evidence.
|
|
45
|
+
|
|
46
|
+
## Gateway provider evidence matrix
|
|
47
|
+
|
|
48
|
+
This table separates deterministic protocol evidence from hosted calls that need account
|
|
49
|
+
credentials. `Not run` means exactly that; it is not inferred from fixture coverage.
|
|
50
|
+
|
|
51
|
+
| Provider surface | Deterministic evidence in this release | Credential-gated live evidence |
|
|
52
|
+
|---|---|---|
|
|
53
|
+
| OpenAI | Native Responses fixtures for text, tool arguments, usage, cancellation, and refusal; all eight official SDK quadrants run against the installed local gateway | Not run; requires an OpenAI credential |
|
|
54
|
+
| Anthropic | Native Messages fixtures for text, tool arguments, usage, cancellation, and refusal through both public gateway surfaces | Not run; requires an Anthropic credential |
|
|
55
|
+
| Generic OpenAI-compatible | Real loopback upstream through the installed gateway; text, tool arguments, usage, cancellation, and refusal contracts | Not run; requires a compatible hosted endpoint and credential |
|
|
56
|
+
| Azure OpenAI | Compatible-adapter fixtures for text, tool arguments, usage, cancellation, and refusal | Not run; requires Azure endpoint and credential |
|
|
57
|
+
| OpenRouter | Compatible-adapter fixtures for text, tool arguments, usage, cancellation, and refusal | Not run; requires OpenRouter credential |
|
|
58
|
+
| Gemini | Native fixtures for text, structured complete function arguments, usage, cancellation, and refusal | Not run; requires Gemini credential |
|
|
59
|
+
| Amazon Bedrock | Native EventStream fixtures for text, incremental tool arguments, usage, bounded cancellation, refusal, and single dispatch | Not run; requires an authorized AWS account and region |
|
|
60
|
+
|
|
61
|
+
The machine-readable dated matrix is
|
|
62
|
+
`exp/runtime/gateway/provider_certification.py`. Its live cells are
|
|
63
|
+
`not_run_requires_credentials`. Gemini complete structured function arguments are explicitly not
|
|
64
|
+
labeled as provider-byte incremental tool-argument streaming.
|
|
65
|
+
|
|
66
|
+
## Explicitly excluded
|
|
67
|
+
|
|
68
|
+
- There is no budgets dashboard. Monthly allocation management and remaining-allocation reporting
|
|
69
|
+
are explicit interactive or non-interactive CLI operations.
|
|
70
|
+
|
|
71
|
+
- No paid E2B or Harbor cloud smoke ran. The repository verifies the optional `bounded-close-v1`
|
|
72
|
+
Harbor lifecycle and ledger with injected fakes, but makes no cloud cleanup, provider-quality, or
|
|
73
|
+
environment-parity claim.
|
|
74
|
+
- No real Tinker training ran. Managed SFT remains fail-closed unless its immutable configuration
|
|
75
|
+
has a finite positive ceiling and its backend supplies a conservative full-schedule estimate.
|
|
76
|
+
- No trained-versus-base behavioral comparison ran because this release produced no paid training
|
|
77
|
+
artifact. It makes no trained-model quality-improvement claim.
|
|
78
|
+
- No hosted model, judge, embedding, telemetry, environment, credential, or `.env` path was used by
|
|
79
|
+
release evidence. The deterministic W16 evidence reports exactly $0.00 observed service spend.
|
|
80
|
+
- Deterministic gateway certification uses a real loopback upstream and local SQLite. No live
|
|
81
|
+
provider matrix cell ran, so hosted availability, account limits, billing, and service-specific
|
|
82
|
+
behavior are not claimed.
|
|
83
|
+
|
|
84
|
+
These exclusions are product boundaries, not evidence that the corresponding hosted services are
|
|
85
|
+
unsafe or unsupported forever. Any future claim requires separately authorized, finite-budget,
|
|
86
|
+
denominator-preserving evidence.
|