experiential 0.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (612) hide show
  1. experiential-0.4.0/.gitignore +42 -0
  2. experiential-0.4.0/PKG-INFO +132 -0
  3. experiential-0.4.0/README.md +101 -0
  4. experiential-0.4.0/assets/experiential-workflow.png +0 -0
  5. experiential-0.4.0/conftest.py +11 -0
  6. experiential-0.4.0/docs/reference/gateway-architecture.md +187 -0
  7. experiential-0.4.0/docs/release-scope.md +86 -0
  8. experiential-0.4.0/docs/usage.md +124 -0
  9. experiential-0.4.0/exp/__init__.py +262 -0
  10. experiential-0.4.0/exp/__main__.py +11 -0
  11. experiential-0.4.0/exp/cli/__init__.py +5 -0
  12. experiential-0.4.0/exp/cli/app.py +50 -0
  13. experiential-0.4.0/exp/cli/app_test.py +31 -0
  14. experiential-0.4.0/exp/cli/build/__init__.py +1 -0
  15. experiential-0.4.0/exp/cli/build/app.py +966 -0
  16. experiential-0.4.0/exp/cli/build/app_test.py +1607 -0
  17. experiential-0.4.0/exp/cli/build/cost.py +77 -0
  18. experiential-0.4.0/exp/cli/build/cost_test.py +29 -0
  19. experiential-0.4.0/exp/cli/build/wizard.py +893 -0
  20. experiential-0.4.0/exp/cli/build/wizard_screens.py +142 -0
  21. experiential-0.4.0/exp/cli/build/wizard_screens_test.py +89 -0
  22. experiential-0.4.0/exp/cli/build/wizard_test.py +1286 -0
  23. experiential-0.4.0/exp/cli/config/__init__.py +1 -0
  24. experiential-0.4.0/exp/cli/config/app.py +167 -0
  25. experiential-0.4.0/exp/cli/config/app_test.py +45 -0
  26. experiential-0.4.0/exp/cli/gateway/__init__.py +1 -0
  27. experiential-0.4.0/exp/cli/gateway/alias.py +348 -0
  28. experiential-0.4.0/exp/cli/gateway/app.py +498 -0
  29. experiential-0.4.0/exp/cli/gateway/app_test.py +1329 -0
  30. experiential-0.4.0/exp/cli/gateway/budget.py +185 -0
  31. experiential-0.4.0/exp/cli/gateway/budget_test.py +153 -0
  32. experiential-0.4.0/exp/cli/gateway/compatibility.py +121 -0
  33. experiential-0.4.0/exp/cli/gateway/key_output.py +532 -0
  34. experiential-0.4.0/exp/cli/gateway/pool.py +197 -0
  35. experiential-0.4.0/exp/cli/gateway/provider.py +166 -0
  36. experiential-0.4.0/exp/cli/gateway/receipts.py +74 -0
  37. experiential-0.4.0/exp/cli/gateway/setup.py +114 -0
  38. experiential-0.4.0/exp/cli/judge/__init__.py +1 -0
  39. experiential-0.4.0/exp/cli/judge/app.py +545 -0
  40. experiential-0.4.0/exp/cli/judge/app_test.py +399 -0
  41. experiential-0.4.0/exp/cli/judge/review.py +675 -0
  42. experiential-0.4.0/exp/cli/judge/review_test.py +777 -0
  43. experiential-0.4.0/exp/cli/judge/rubric.py +261 -0
  44. experiential-0.4.0/exp/cli/judge/rubric_test.py +130 -0
  45. experiential-0.4.0/exp/cli/judge/trace_viewer.py +581 -0
  46. experiential-0.4.0/exp/cli/judge/trace_viewer_test.py +190 -0
  47. experiential-0.4.0/exp/cli/judge/transcript.py +298 -0
  48. experiential-0.4.0/exp/cli/judge/transcript_test.py +1 -0
  49. experiential-0.4.0/exp/cli/optimize/__init__.py +1 -0
  50. experiential-0.4.0/exp/cli/optimize/app.py +28 -0
  51. experiential-0.4.0/exp/cli/optimize/model.py +613 -0
  52. experiential-0.4.0/exp/cli/optimize/model_test.py +1587 -0
  53. experiential-0.4.0/exp/cli/optimize/router.py +219 -0
  54. experiential-0.4.0/exp/cli/optimize/router_candidates.py +524 -0
  55. experiential-0.4.0/exp/cli/optimize/router_candidates_test.py +453 -0
  56. experiential-0.4.0/exp/cli/optimize/router_test.py +192 -0
  57. experiential-0.4.0/exp/cli/providers/__init__.py +1 -0
  58. experiential-0.4.0/exp/cli/providers/model_picker.py +830 -0
  59. experiential-0.4.0/exp/cli/providers/model_picker_test.py +524 -0
  60. experiential-0.4.0/exp/cli/providers/provider_picker.py +718 -0
  61. experiential-0.4.0/exp/cli/providers/provider_picker_test.py +681 -0
  62. experiential-0.4.0/exp/cli/providers/setup.py +780 -0
  63. experiential-0.4.0/exp/cli/providers/setup_test.py +1174 -0
  64. experiential-0.4.0/exp/cli/run/__init__.py +1 -0
  65. experiential-0.4.0/exp/cli/run/app.py +236 -0
  66. experiential-0.4.0/exp/cli/run/app_test.py +153 -0
  67. experiential-0.4.0/exp/cli/shared/__init__.py +1 -0
  68. experiential-0.4.0/exp/cli/shared/consent.py +299 -0
  69. experiential-0.4.0/exp/cli/shared/consent_test.py +446 -0
  70. experiential-0.4.0/exp/cli/shared/defer.py +101 -0
  71. experiential-0.4.0/exp/cli/shared/options.py +22 -0
  72. experiential-0.4.0/exp/cli/shared/options_test.py +2 -0
  73. experiential-0.4.0/exp/cli/shared/picker.py +814 -0
  74. experiential-0.4.0/exp/cli/shared/picker_test.py +919 -0
  75. experiential-0.4.0/exp/cli/shared/picker_view.py +277 -0
  76. experiential-0.4.0/exp/cli/shared/picker_view_test.py +268 -0
  77. experiential-0.4.0/exp/cli/shared/progress.py +237 -0
  78. experiential-0.4.0/exp/cli/shared/progress_test.py +146 -0
  79. experiential-0.4.0/exp/cli/shared/theme.py +31 -0
  80. experiential-0.4.0/exp/cli/shared/theme_test.py +53 -0
  81. experiential-0.4.0/exp/cli/tests/cli_help_test.py +64 -0
  82. experiential-0.4.0/exp/cli/tests/package_layout_test.py +125 -0
  83. experiential-0.4.0/exp/cli/tests/startup_test.py +147 -0
  84. experiential-0.4.0/exp/cli/tests/terminal_tasks_happy_path_test.py +516 -0
  85. experiential-0.4.0/exp/cli/tests/terminal_tasks_live_pipeline_test.py +447 -0
  86. experiential-0.4.0/exp/common/__init__.py +1 -0
  87. experiential-0.4.0/exp/common/config/__init__.py +23 -0
  88. experiential-0.4.0/exp/common/config/dotenv.py +31 -0
  89. experiential-0.4.0/exp/common/config/dotenv_test.py +25 -0
  90. experiential-0.4.0/exp/common/config/paths.py +5 -0
  91. experiential-0.4.0/exp/common/config/settings.py +192 -0
  92. experiential-0.4.0/exp/common/config/settings_test.py +201 -0
  93. experiential-0.4.0/exp/common/core/artifacts.py +435 -0
  94. experiential-0.4.0/exp/common/core/artifacts_test.py +239 -0
  95. experiential-0.4.0/exp/common/core/files.py +181 -0
  96. experiential-0.4.0/exp/common/core/files_test.py +375 -0
  97. experiential-0.4.0/exp/common/core/hashing.py +35 -0
  98. experiential-0.4.0/exp/common/core/hashing_test.py +26 -0
  99. experiential-0.4.0/exp/common/core/locks.py +97 -0
  100. experiential-0.4.0/exp/common/core/locks_test.py +130 -0
  101. experiential-0.4.0/exp/common/core/money.py +84 -0
  102. experiential-0.4.0/exp/common/core/text.py +32 -0
  103. experiential-0.4.0/exp/common/core/union_find.py +59 -0
  104. experiential-0.4.0/exp/common/core/union_find_test.py +17 -0
  105. experiential-0.4.0/exp/common/evaluations/__init__.py +46 -0
  106. experiential-0.4.0/exp/common/evaluations/build.py +532 -0
  107. experiential-0.4.0/exp/common/evaluations/build_test.py +852 -0
  108. experiential-0.4.0/exp/common/evaluations/dataset.py +291 -0
  109. experiential-0.4.0/exp/common/evaluations/dataset_test.py +227 -0
  110. experiential-0.4.0/exp/common/evaluations/evidence.py +229 -0
  111. experiential-0.4.0/exp/common/evaluations/fidelity.py +322 -0
  112. experiential-0.4.0/exp/common/evaluations/plan.py +144 -0
  113. experiential-0.4.0/exp/common/evaluations/plan_test.py +147 -0
  114. experiential-0.4.0/exp/common/evaluations/planning.py +487 -0
  115. experiential-0.4.0/exp/common/judging/__init__.py +111 -0
  116. experiential-0.4.0/exp/common/judging/calibration.py +961 -0
  117. experiential-0.4.0/exp/common/judging/calibration_assembly.py +65 -0
  118. experiential-0.4.0/exp/common/judging/calibration_contracts.py +170 -0
  119. experiential-0.4.0/exp/common/judging/calibration_metrics.py +387 -0
  120. experiential-0.4.0/exp/common/judging/calibration_metrics_test.py +66 -0
  121. experiential-0.4.0/exp/common/judging/calibration_provenance.py +103 -0
  122. experiential-0.4.0/exp/common/judging/calibration_test.py +954 -0
  123. experiential-0.4.0/exp/common/judging/display.py +95 -0
  124. experiential-0.4.0/exp/common/judging/display_test.py +50 -0
  125. experiential-0.4.0/exp/common/judging/evidence.py +141 -0
  126. experiential-0.4.0/exp/common/judging/evidence_test.py +234 -0
  127. experiential-0.4.0/exp/common/judging/interface.py +35 -0
  128. experiential-0.4.0/exp/common/judging/judgment.py +89 -0
  129. experiential-0.4.0/exp/common/judging/judgment_test.py +127 -0
  130. experiential-0.4.0/exp/common/judging/labels.py +581 -0
  131. experiential-0.4.0/exp/common/judging/labels_test.py +228 -0
  132. experiential-0.4.0/exp/common/judging/lineage.py +149 -0
  133. experiential-0.4.0/exp/common/judging/lm.py +530 -0
  134. experiential-0.4.0/exp/common/judging/lm_test.py +543 -0
  135. experiential-0.4.0/exp/common/judging/prompts.py +41 -0
  136. experiential-0.4.0/exp/common/judging/proposal.py +215 -0
  137. experiential-0.4.0/exp/common/judging/proposal_test.py +1 -0
  138. experiential-0.4.0/exp/common/judging/provenance.py +137 -0
  139. experiential-0.4.0/exp/common/judging/review.py +692 -0
  140. experiential-0.4.0/exp/common/judging/review_test.py +270 -0
  141. experiential-0.4.0/exp/common/judging/risk_acceptance.py +222 -0
  142. experiential-0.4.0/exp/common/judging/rubric.py +373 -0
  143. experiential-0.4.0/exp/common/judging/rubric_test.py +190 -0
  144. experiential-0.4.0/exp/common/judging/tests/calibration_persistence_test.py +52 -0
  145. experiential-0.4.0/exp/common/judging/tests/lm_authority_test.py +66 -0
  146. experiential-0.4.0/exp/common/models/__init__.py +185 -0
  147. experiential-0.4.0/exp/common/models/catalog.py +551 -0
  148. experiential-0.4.0/exp/common/models/catalog_test.py +528 -0
  149. experiential-0.4.0/exp/common/models/client.py +58 -0
  150. experiential-0.4.0/exp/common/models/client_test.py +1 -0
  151. experiential-0.4.0/exp/common/models/connection_authoring.py +86 -0
  152. experiential-0.4.0/exp/common/models/connection_authoring_test.py +108 -0
  153. experiential-0.4.0/exp/common/models/discovery.py +267 -0
  154. experiential-0.4.0/exp/common/models/discovery_test.py +186 -0
  155. experiential-0.4.0/exp/common/models/gateway_catalog.py +225 -0
  156. experiential-0.4.0/exp/common/models/gateway_catalog_test.py +326 -0
  157. experiential-0.4.0/exp/common/models/known_models.py +573 -0
  158. experiential-0.4.0/exp/common/models/known_models_test.py +256 -0
  159. experiential-0.4.0/exp/common/models/model.py +461 -0
  160. experiential-0.4.0/exp/common/models/model_test.py +279 -0
  161. experiential-0.4.0/exp/common/models/pricing.py +548 -0
  162. experiential-0.4.0/exp/common/models/pricing_test.py +291 -0
  163. experiential-0.4.0/exp/common/models/router_candidates.py +380 -0
  164. experiential-0.4.0/exp/common/models/router_candidates_test.py +340 -0
  165. experiential-0.4.0/exp/common/models/setup.py +352 -0
  166. experiential-0.4.0/exp/common/models/setup_test.py +297 -0
  167. experiential-0.4.0/exp/common/models/structured.py +36 -0
  168. experiential-0.4.0/exp/common/models/structured_test.py +59 -0
  169. experiential-0.4.0/exp/common/observability/__init__.py +15 -0
  170. experiential-0.4.0/exp/common/observability/telemetry.py +427 -0
  171. experiential-0.4.0/exp/common/observability/telemetry_test.py +403 -0
  172. experiential-0.4.0/exp/common/progress.py +66 -0
  173. experiential-0.4.0/exp/common/progress_test.py +42 -0
  174. experiential-0.4.0/exp/common/project/__init__.py +142 -0
  175. experiential-0.4.0/exp/common/project/bundle.py +932 -0
  176. experiential-0.4.0/exp/common/project/bundle_test.py +772 -0
  177. experiential-0.4.0/exp/common/project/catalog.py +141 -0
  178. experiential-0.4.0/exp/common/project/catalog_test.py +104 -0
  179. experiential-0.4.0/exp/common/project/events.py +94 -0
  180. experiential-0.4.0/exp/common/project/events_test.py +102 -0
  181. experiential-0.4.0/exp/common/project/hosted_state.py +308 -0
  182. experiential-0.4.0/exp/common/project/manifests.py +73 -0
  183. experiential-0.4.0/exp/common/project/paths.py +119 -0
  184. experiential-0.4.0/exp/common/project/paths_test.py +43 -0
  185. experiential-0.4.0/exp/common/project/project.py +383 -0
  186. experiential-0.4.0/exp/common/project/project_test.py +158 -0
  187. experiential-0.4.0/exp/common/project/store.py +990 -0
  188. experiential-0.4.0/exp/common/project/store_test.py +435 -0
  189. experiential-0.4.0/exp/common/release_revision.py +48 -0
  190. experiential-0.4.0/exp/common/release_revision_test.py +69 -0
  191. experiential-0.4.0/exp/common/rollouts/__init__.py +47 -0
  192. experiential-0.4.0/exp/common/rollouts/artifact.py +370 -0
  193. experiential-0.4.0/exp/common/rollouts/artifact_test.py +300 -0
  194. experiential-0.4.0/exp/common/rollouts/dispatch_failures.py +87 -0
  195. experiential-0.4.0/exp/common/rollouts/dispatch_failures_test.py +147 -0
  196. experiential-0.4.0/exp/common/rollouts/otel.py +82 -0
  197. experiential-0.4.0/exp/common/routing/__init__.py +40 -0
  198. experiential-0.4.0/exp/common/routing/bank.py +374 -0
  199. experiential-0.4.0/exp/common/routing/decision.py +371 -0
  200. experiential-0.4.0/exp/common/routing/embeddings.py +559 -0
  201. experiential-0.4.0/exp/common/routing/embeddings_test.py +625 -0
  202. experiential-0.4.0/exp/common/routing/features.py +120 -0
  203. experiential-0.4.0/exp/common/routing/features_test.py +68 -0
  204. experiential-0.4.0/exp/common/routing/policy.py +133 -0
  205. experiential-0.4.0/exp/common/routing/policy_test.py +151 -0
  206. experiential-0.4.0/exp/common/tasks/__init__.py +12 -0
  207. experiential-0.4.0/exp/common/tasks/store.py +68 -0
  208. experiential-0.4.0/exp/common/tasks/store_test.py +92 -0
  209. experiential-0.4.0/exp/common/tasks/task.py +90 -0
  210. experiential-0.4.0/exp/common/tasks/task_test.py +55 -0
  211. experiential-0.4.0/exp/common/tests/dependency_surface_test.py +79 -0
  212. experiential-0.4.0/exp/common/traces/__init__.py +14 -0
  213. experiential-0.4.0/exp/common/traces/store.py +134 -0
  214. experiential-0.4.0/exp/common/traces/store_test.py +178 -0
  215. experiential-0.4.0/exp/common/traces/trace.py +135 -0
  216. experiential-0.4.0/exp/common/traces/trace_test.py +96 -0
  217. experiential-0.4.0/exp/conftest.py +302 -0
  218. experiential-0.4.0/exp/optimize/__init__.py +1 -0
  219. experiential-0.4.0/exp/optimize/model/__init__.py +1 -0
  220. experiential-0.4.0/exp/optimize/model/sft/__init__.py +145 -0
  221. experiential-0.4.0/exp/optimize/model/sft/automatic.py +797 -0
  222. experiential-0.4.0/exp/optimize/model/sft/automatic_test.py +1106 -0
  223. experiential-0.4.0/exp/optimize/model/sft/builder.py +868 -0
  224. experiential-0.4.0/exp/optimize/model/sft/builder_test.py +1010 -0
  225. experiential-0.4.0/exp/optimize/model/sft/composition.py +948 -0
  226. experiential-0.4.0/exp/optimize/model/sft/composition_test.py +497 -0
  227. experiential-0.4.0/exp/optimize/model/sft/contracts.py +511 -0
  228. experiential-0.4.0/exp/optimize/model/sft/provider_resources.py +39 -0
  229. experiential-0.4.0/exp/optimize/model/sft/rendering.py +89 -0
  230. experiential-0.4.0/exp/optimize/model/sft/rendering_test.py +120 -0
  231. experiential-0.4.0/exp/optimize/model/sft/run_manifest.py +814 -0
  232. experiential-0.4.0/exp/optimize/model/sft/run_manifest_test.py +1 -0
  233. experiential-0.4.0/exp/optimize/model/sft/runtime_source.py +276 -0
  234. experiential-0.4.0/exp/optimize/model/sft/runtime_source_test.py +354 -0
  235. experiential-0.4.0/exp/optimize/model/sft/selection.py +386 -0
  236. experiential-0.4.0/exp/optimize/model/sft/selection_test.py +1 -0
  237. experiential-0.4.0/exp/optimize/model/sft/sources.py +568 -0
  238. experiential-0.4.0/exp/optimize/model/sft/tinker.py +407 -0
  239. experiential-0.4.0/exp/optimize/model/sft/tinker_test.py +371 -0
  240. experiential-0.4.0/exp/optimize/model/sft/training.py +865 -0
  241. experiential-0.4.0/exp/optimize/model/sft/training_contracts.py +414 -0
  242. experiential-0.4.0/exp/optimize/model/sft/training_runtime.py +221 -0
  243. experiential-0.4.0/exp/optimize/model/sft/training_test.py +622 -0
  244. experiential-0.4.0/exp/optimize/router/__init__.py +150 -0
  245. experiential-0.4.0/exp/optimize/router/activation.py +159 -0
  246. experiential-0.4.0/exp/optimize/router/activation_test.py +98 -0
  247. experiential-0.4.0/exp/optimize/router/attempt_authority.py +729 -0
  248. experiential-0.4.0/exp/optimize/router/attempt_authority_test.py +85 -0
  249. experiential-0.4.0/exp/optimize/router/automatic/__init__.py +1 -0
  250. experiential-0.4.0/exp/optimize/router/automatic/artifacts.py +340 -0
  251. experiential-0.4.0/exp/optimize/router/automatic/artifacts_test.py +1 -0
  252. experiential-0.4.0/exp/optimize/router/automatic/attribution.py +578 -0
  253. experiential-0.4.0/exp/optimize/router/automatic/attribution_test.py +426 -0
  254. experiential-0.4.0/exp/optimize/router/automatic/execution_contract.py +415 -0
  255. experiential-0.4.0/exp/optimize/router/automatic/execution_contract_test.py +1 -0
  256. experiential-0.4.0/exp/optimize/router/automatic/judge.py +350 -0
  257. experiential-0.4.0/exp/optimize/router/automatic/judge_provenance.py +325 -0
  258. experiential-0.4.0/exp/optimize/router/automatic/judge_provenance_test.py +1 -0
  259. experiential-0.4.0/exp/optimize/router/automatic/judge_test.py +274 -0
  260. experiential-0.4.0/exp/optimize/router/automatic/preflight.py +793 -0
  261. experiential-0.4.0/exp/optimize/router/automatic/preflight_test.py +157 -0
  262. experiential-0.4.0/exp/optimize/router/automatic/provisional.py +471 -0
  263. experiential-0.4.0/exp/optimize/router/automatic/replay.py +646 -0
  264. experiential-0.4.0/exp/optimize/router/automatic/replay_test.py +1 -0
  265. experiential-0.4.0/exp/optimize/router/automatic/reservations.py +751 -0
  266. experiential-0.4.0/exp/optimize/router/automatic/reservations_test.py +495 -0
  267. experiential-0.4.0/exp/optimize/router/automatic/service.py +738 -0
  268. experiential-0.4.0/exp/optimize/router/automatic/service_test.py +2050 -0
  269. experiential-0.4.0/exp/optimize/router/composition.py +759 -0
  270. experiential-0.4.0/exp/optimize/router/composition_test.py +2011 -0
  271. experiential-0.4.0/exp/optimize/router/errors.py +32 -0
  272. experiential-0.4.0/exp/optimize/router/errors_test.py +0 -0
  273. experiential-0.4.0/exp/optimize/router/evaluation/__init__.py +1 -0
  274. experiential-0.4.0/exp/optimize/router/evaluation/build.py +123 -0
  275. experiential-0.4.0/exp/optimize/router/evaluation/build_test.py +1 -0
  276. experiential-0.4.0/exp/optimize/router/evaluation/setup.py +50 -0
  277. experiential-0.4.0/exp/optimize/router/evaluation/setup_test.py +1 -0
  278. experiential-0.4.0/exp/optimize/router/evaluation/simulation_spec.py +91 -0
  279. experiential-0.4.0/exp/optimize/router/evaluation/simulation_spec_test.py +129 -0
  280. experiential-0.4.0/exp/optimize/router/evaluation/spend.py +199 -0
  281. experiential-0.4.0/exp/optimize/router/evaluation/spend_test.py +243 -0
  282. experiential-0.4.0/exp/optimize/router/fit/__init__.py +1 -0
  283. experiential-0.4.0/exp/optimize/router/fit/optimizer.py +910 -0
  284. experiential-0.4.0/exp/optimize/router/fit/optimizer_test.py +1142 -0
  285. experiential-0.4.0/exp/optimize/router/fit/report.py +640 -0
  286. experiential-0.4.0/exp/optimize/router/fit/report_test.py +1 -0
  287. experiential-0.4.0/exp/optimize/router/fit/spec.py +81 -0
  288. experiential-0.4.0/exp/optimize/router/fit/spec_test.py +1 -0
  289. experiential-0.4.0/exp/optimize/router/fit/tests/bank_test.py +313 -0
  290. experiential-0.4.0/exp/optimize/router/fit/tests/decision_test.py +294 -0
  291. experiential-0.4.0/exp/optimize/router/fit/workflow.py +320 -0
  292. experiential-0.4.0/exp/optimize/router/fit/workflow_test.py +310 -0
  293. experiential-0.4.0/exp/optimize/router/hosted.py +944 -0
  294. experiential-0.4.0/exp/optimize/router/hosted_preflight.py +638 -0
  295. experiential-0.4.0/exp/optimize/router/hosted_spend.py +285 -0
  296. experiential-0.4.0/exp/optimize/router/hosted_stages.py +149 -0
  297. experiential-0.4.0/exp/optimize/router/hosted_test.py +2009 -0
  298. experiential-0.4.0/exp/optimize/router/hosted_verification.py +719 -0
  299. experiential-0.4.0/exp/optimize/router/judging/__init__.py +1 -0
  300. experiential-0.4.0/exp/optimize/router/judging/artifacts.py +940 -0
  301. experiential-0.4.0/exp/optimize/router/judging/artifacts_test.py +137 -0
  302. experiential-0.4.0/exp/optimize/router/judging/contracts.py +932 -0
  303. experiential-0.4.0/exp/optimize/router/judging/contracts_test.py +102 -0
  304. experiential-0.4.0/exp/optimize/router/judging/labels.py +207 -0
  305. experiential-0.4.0/exp/optimize/router/judging/labels_test.py +130 -0
  306. experiential-0.4.0/exp/optimize/router/judging/pricing.py +103 -0
  307. experiential-0.4.0/exp/optimize/router/judging/pricing_test.py +226 -0
  308. experiential-0.4.0/exp/optimize/router/judging/protocol.py +777 -0
  309. experiential-0.4.0/exp/optimize/router/judging/protocol_test.py +445 -0
  310. experiential-0.4.0/exp/optimize/router/judging/provisional.py +108 -0
  311. experiential-0.4.0/exp/optimize/router/judging/provisional_test.py +1 -0
  312. experiential-0.4.0/exp/optimize/router/judging/review.py +954 -0
  313. experiential-0.4.0/exp/optimize/router/judging/review_test.py +476 -0
  314. experiential-0.4.0/exp/optimize/router/judging/selection.py +179 -0
  315. experiential-0.4.0/exp/optimize/router/judging/selection_test.py +1 -0
  316. experiential-0.4.0/exp/optimize/router/judging/service.py +920 -0
  317. experiential-0.4.0/exp/optimize/router/judging/service_test.py +1619 -0
  318. experiential-0.4.0/exp/optimize/router/judging/setup_store.py +83 -0
  319. experiential-0.4.0/exp/optimize/router/judging/setup_store_test.py +5 -0
  320. experiential-0.4.0/exp/optimize/router/judging/template_bind.py +112 -0
  321. experiential-0.4.0/exp/optimize/router/judging/template_bind_test.py +53 -0
  322. experiential-0.4.0/exp/optimize/router/judgment_budget.py +866 -0
  323. experiential-0.4.0/exp/optimize/router/judgment_budget_test.py +140 -0
  324. experiential-0.4.0/exp/optimize/router/spend.py +353 -0
  325. experiential-0.4.0/exp/optimize/router/spend_test.py +53 -0
  326. experiential-0.4.0/exp/optimize/router/tests/composition_evidence_test.py +668 -0
  327. experiential-0.4.0/exp/runtime/agents/__init__.py +83 -0
  328. experiential-0.4.0/exp/runtime/agents/chat.py +252 -0
  329. experiential-0.4.0/exp/runtime/agents/chat_test.py +246 -0
  330. experiential-0.4.0/exp/runtime/agents/factory.py +145 -0
  331. experiential-0.4.0/exp/runtime/agents/factory_test.py +85 -0
  332. experiential-0.4.0/exp/runtime/agents/interface.py +97 -0
  333. experiential-0.4.0/exp/runtime/agents/interface_test.py +76 -0
  334. experiential-0.4.0/exp/runtime/agents/lifecycle.py +230 -0
  335. experiential-0.4.0/exp/runtime/agents/lifecycle_test.py +465 -0
  336. experiential-0.4.0/exp/runtime/agents/pi.py +841 -0
  337. experiential-0.4.0/exp/runtime/agents/pi_test.py +610 -0
  338. experiential-0.4.0/exp/runtime/environments/__init__.py +49 -0
  339. experiential-0.4.0/exp/runtime/environments/harbor.py +562 -0
  340. experiential-0.4.0/exp/runtime/environments/harbor_test.py +978 -0
  341. experiential-0.4.0/exp/runtime/environments/interface.py +54 -0
  342. experiential-0.4.0/exp/runtime/environments/interface_test.py +1 -0
  343. experiential-0.4.0/exp/runtime/environments/local.py +998 -0
  344. experiential-0.4.0/exp/runtime/environments/local_exec_gate.py +27 -0
  345. experiential-0.4.0/exp/runtime/environments/local_test.py +466 -0
  346. experiential-0.4.0/exp/runtime/environments/sandbox_ledger.py +272 -0
  347. experiential-0.4.0/exp/runtime/environments/sandbox_ledger_test.py +105 -0
  348. experiential-0.4.0/exp/runtime/gateway/__init__.py +203 -0
  349. experiential-0.4.0/exp/runtime/gateway/aggregation.py +49 -0
  350. experiential-0.4.0/exp/runtime/gateway/auth.py +241 -0
  351. experiential-0.4.0/exp/runtime/gateway/auth_test.py +58 -0
  352. experiential-0.4.0/exp/runtime/gateway/budgets.py +664 -0
  353. experiential-0.4.0/exp/runtime/gateway/budgets_test.py +518 -0
  354. experiential-0.4.0/exp/runtime/gateway/catalog_authority.py +446 -0
  355. experiential-0.4.0/exp/runtime/gateway/composition.py +235 -0
  356. experiential-0.4.0/exp/runtime/gateway/composition_test.py +347 -0
  357. experiential-0.4.0/exp/runtime/gateway/contracts.py +341 -0
  358. experiential-0.4.0/exp/runtime/gateway/contracts_test.py +148 -0
  359. experiential-0.4.0/exp/runtime/gateway/execution.py +891 -0
  360. experiential-0.4.0/exp/runtime/gateway/health.py +143 -0
  361. experiential-0.4.0/exp/runtime/gateway/interfaces.py +145 -0
  362. experiential-0.4.0/exp/runtime/gateway/ledger.py +806 -0
  363. experiential-0.4.0/exp/runtime/gateway/ledger_test.py +854 -0
  364. experiential-0.4.0/exp/runtime/gateway/lifecycle.py +554 -0
  365. experiential-0.4.0/exp/runtime/gateway/lifecycle_test.py +787 -0
  366. experiential-0.4.0/exp/runtime/gateway/management.py +791 -0
  367. experiential-0.4.0/exp/runtime/gateway/platform.py +846 -0
  368. experiential-0.4.0/exp/runtime/gateway/platform_test.py +554 -0
  369. experiential-0.4.0/exp/runtime/gateway/project_activation.py +310 -0
  370. experiential-0.4.0/exp/runtime/gateway/project_activation_test.py +94 -0
  371. experiential-0.4.0/exp/runtime/gateway/project_alias.py +168 -0
  372. experiential-0.4.0/exp/runtime/gateway/project_alias_test.py +142 -0
  373. experiential-0.4.0/exp/runtime/gateway/provider_certification.py +277 -0
  374. experiential-0.4.0/exp/runtime/gateway/provider_certification_test.py +87 -0
  375. experiential-0.4.0/exp/runtime/gateway/routing.py +342 -0
  376. experiential-0.4.0/exp/runtime/gateway/secrets.py +45 -0
  377. experiential-0.4.0/exp/runtime/gateway/secrets_test.py +18 -0
  378. experiential-0.4.0/exp/runtime/gateway/service.py +911 -0
  379. experiential-0.4.0/exp/runtime/gateway/sqlite/__init__.py +5 -0
  380. experiential-0.4.0/exp/runtime/gateway/sqlite/alias_activation.py +166 -0
  381. experiential-0.4.0/exp/runtime/gateway/sqlite/key_delivery.py +158 -0
  382. experiential-0.4.0/exp/runtime/gateway/sqlite/migrations.py +694 -0
  383. experiential-0.4.0/exp/runtime/gateway/sqlite/migrations_test.py +604 -0
  384. experiential-0.4.0/exp/runtime/gateway/sqlite/platform.py +988 -0
  385. experiential-0.4.0/exp/runtime/gateway/sqlite/platform_records.py +224 -0
  386. experiential-0.4.0/exp/runtime/gateway/sqlite/platform_records_test.py +1 -0
  387. experiential-0.4.0/exp/runtime/gateway/sqlite/platform_test.py +689 -0
  388. experiential-0.4.0/exp/runtime/gateway/sqlite/provider_authority.py +327 -0
  389. experiential-0.4.0/exp/runtime/gateway/sqlite/provider_store.py +161 -0
  390. experiential-0.4.0/exp/runtime/gateway/sqlite/store.py +975 -0
  391. experiential-0.4.0/exp/runtime/gateway/sqlite/store_test.py +1191 -0
  392. experiential-0.4.0/exp/runtime/gateway/tests/data_plane_test.py +1333 -0
  393. experiential-0.4.0/exp/runtime/gateway/tests/launch_test.py +642 -0
  394. experiential-0.4.0/exp/runtime/gateway/tests/waterfall_test.py +1148 -0
  395. experiential-0.4.0/exp/runtime/gateway/usage.py +175 -0
  396. experiential-0.4.0/exp/runtime/models/__init__.py +18 -0
  397. experiential-0.4.0/exp/runtime/models/credentials.py +42 -0
  398. experiential-0.4.0/exp/runtime/models/preflight.py +83 -0
  399. experiential-0.4.0/exp/runtime/models/providers/__init__.py +72 -0
  400. experiential-0.4.0/exp/runtime/models/providers/anthropic.py +308 -0
  401. experiential-0.4.0/exp/runtime/models/providers/async_transport.py +674 -0
  402. experiential-0.4.0/exp/runtime/models/providers/async_transport_test.py +162 -0
  403. experiential-0.4.0/exp/runtime/models/providers/azure.py +179 -0
  404. experiential-0.4.0/exp/runtime/models/providers/azure_test.py +265 -0
  405. experiential-0.4.0/exp/runtime/models/providers/base.py +304 -0
  406. experiential-0.4.0/exp/runtime/models/providers/base_test.py +210 -0
  407. experiential-0.4.0/exp/runtime/models/providers/bedrock.py +737 -0
  408. experiential-0.4.0/exp/runtime/models/providers/bedrock_streaming.py +461 -0
  409. experiential-0.4.0/exp/runtime/models/providers/bedrock_streaming_test.py +350 -0
  410. experiential-0.4.0/exp/runtime/models/providers/bedrock_test.py +613 -0
  411. experiential-0.4.0/exp/runtime/models/providers/errors.py +249 -0
  412. experiential-0.4.0/exp/runtime/models/providers/errors_test.py +99 -0
  413. experiential-0.4.0/exp/runtime/models/providers/gemini.py +354 -0
  414. experiential-0.4.0/exp/runtime/models/providers/gemini_streaming.py +241 -0
  415. experiential-0.4.0/exp/runtime/models/providers/gemini_streaming_test.py +247 -0
  416. experiential-0.4.0/exp/runtime/models/providers/listing.py +323 -0
  417. experiential-0.4.0/exp/runtime/models/providers/listing_test.py +261 -0
  418. experiential-0.4.0/exp/runtime/models/providers/openai.py +364 -0
  419. experiential-0.4.0/exp/runtime/models/providers/openai_compatible.py +395 -0
  420. experiential-0.4.0/exp/runtime/models/providers/openai_compatible_test.py +348 -0
  421. experiential-0.4.0/exp/runtime/models/providers/protocol.py +221 -0
  422. experiential-0.4.0/exp/runtime/models/providers/protocol_test.py +158 -0
  423. experiential-0.4.0/exp/runtime/models/providers/stream_attempts.py +128 -0
  424. experiential-0.4.0/exp/runtime/models/providers/streaming.py +970 -0
  425. experiential-0.4.0/exp/runtime/models/providers/streaming_requests.py +310 -0
  426. experiential-0.4.0/exp/runtime/models/providers/streaming_requests_test.py +118 -0
  427. experiential-0.4.0/exp/runtime/models/providers/streaming_test.py +1208 -0
  428. experiential-0.4.0/exp/runtime/models/providers/streaming_usage.py +85 -0
  429. experiential-0.4.0/exp/runtime/models/providers/tests/later_provider_streaming_test.py +164 -0
  430. experiential-0.4.0/exp/runtime/models/providers/tests/native_test.py +608 -0
  431. experiential-0.4.0/exp/runtime/models/providers/tinker_sampling.py +334 -0
  432. experiential-0.4.0/exp/runtime/models/providers/transport.py +434 -0
  433. experiential-0.4.0/exp/runtime/models/providers/transport_test.py +116 -0
  434. experiential-0.4.0/exp/runtime/models/registry.py +415 -0
  435. experiential-0.4.0/exp/runtime/models/registry_test.py +425 -0
  436. experiential-0.4.0/exp/runtime/openai_protocol/__init__.py +21 -0
  437. experiential-0.4.0/exp/runtime/openai_protocol/errors.py +166 -0
  438. experiential-0.4.0/exp/runtime/openai_protocol/errors_test.py +26 -0
  439. experiential-0.4.0/exp/runtime/openai_protocol/headers.py +127 -0
  440. experiential-0.4.0/exp/runtime/openai_protocol/headers_test.py +48 -0
  441. experiential-0.4.0/exp/runtime/openai_protocol/manifest.py +147 -0
  442. experiential-0.4.0/exp/runtime/openai_protocol/manifest_test.py +42 -0
  443. experiential-0.4.0/exp/runtime/openai_protocol/model_adapter.py +137 -0
  444. experiential-0.4.0/exp/runtime/openai_protocol/model_adapter_test.py +101 -0
  445. experiential-0.4.0/exp/runtime/openai_protocol/requests.py +559 -0
  446. experiential-0.4.0/exp/runtime/openai_protocol/requests_test.py +263 -0
  447. experiential-0.4.0/exp/runtime/openai_protocol/response.py +191 -0
  448. experiential-0.4.0/exp/runtime/openai_protocol/state.py +533 -0
  449. experiential-0.4.0/exp/runtime/openai_protocol/state_test.py +267 -0
  450. experiential-0.4.0/exp/runtime/openai_protocol/streaming.py +727 -0
  451. experiential-0.4.0/exp/runtime/openai_protocol/streaming_test.py +206 -0
  452. experiential-0.4.0/exp/runtime/openai_protocol/tests/sdk_harness_test.py +246 -0
  453. experiential-0.4.0/exp/runtime/router/__init__.py +123 -0
  454. experiential-0.4.0/exp/runtime/router/application.py +170 -0
  455. experiential-0.4.0/exp/runtime/router/application_test.py +118 -0
  456. experiential-0.4.0/exp/runtime/router/capability.py +219 -0
  457. experiential-0.4.0/exp/runtime/router/capability_test.py +170 -0
  458. experiential-0.4.0/exp/runtime/router/completion.py +149 -0
  459. experiential-0.4.0/exp/runtime/router/completion_test.py +139 -0
  460. experiential-0.4.0/exp/runtime/router/economics.py +217 -0
  461. experiential-0.4.0/exp/runtime/router/errors.py +5 -0
  462. experiential-0.4.0/exp/runtime/router/journal.py +977 -0
  463. experiential-0.4.0/exp/runtime/router/journal_handoff.py +47 -0
  464. experiential-0.4.0/exp/runtime/router/journal_handoff_test.py +1 -0
  465. experiential-0.4.0/exp/runtime/router/journal_io.py +77 -0
  466. experiential-0.4.0/exp/runtime/router/journal_service.py +379 -0
  467. experiential-0.4.0/exp/runtime/router/journal_spend.py +350 -0
  468. experiential-0.4.0/exp/runtime/router/journal_test.py +1424 -0
  469. experiential-0.4.0/exp/runtime/router/journal_validation.py +417 -0
  470. experiential-0.4.0/exp/runtime/router/runtime.py +926 -0
  471. experiential-0.4.0/exp/runtime/router/runtime_selection.py +133 -0
  472. experiential-0.4.0/exp/runtime/router/runtime_support.py +453 -0
  473. experiential-0.4.0/exp/runtime/router/runtime_test.py +1327 -0
  474. experiential-0.4.0/exp/runtime/router/snapshot.py +889 -0
  475. experiential-0.4.0/exp/runtime/router/snapshot_test.py +983 -0
  476. experiential-0.4.0/exp/runtime/router/tests/runtime_capability_test.py +198 -0
  477. experiential-0.4.0/exp/runtime/router/tests/runtime_idempotency_test.py +73 -0
  478. experiential-0.4.0/exp/simulation/__init__.py +39 -0
  479. experiential-0.4.0/exp/simulation/build.py +670 -0
  480. experiential-0.4.0/exp/simulation/build_test.py +501 -0
  481. experiential-0.4.0/exp/simulation/comparison.py +782 -0
  482. experiential-0.4.0/exp/simulation/comparison_test.py +768 -0
  483. experiential-0.4.0/exp/simulation/engines/__init__.py +11 -0
  484. experiential-0.4.0/exp/simulation/engines/clock.py +32 -0
  485. experiential-0.4.0/exp/simulation/engines/clock_test.py +29 -0
  486. experiential-0.4.0/exp/simulation/engines/sandbox.py +916 -0
  487. experiential-0.4.0/exp/simulation/engines/sandbox_bindings.py +243 -0
  488. experiential-0.4.0/exp/simulation/engines/sandbox_recording.py +748 -0
  489. experiential-0.4.0/exp/simulation/engines/sandbox_test.py +996 -0
  490. experiential-0.4.0/exp/simulation/engines/tests/sandbox_regression_test.py +271 -0
  491. experiential-0.4.0/exp/simulation/engines/text/__init__.py +29 -0
  492. experiential-0.4.0/exp/simulation/engines/text/artifact_set.py +119 -0
  493. experiential-0.4.0/exp/simulation/engines/text/artifact_set_test.py +0 -0
  494. experiential-0.4.0/exp/simulation/engines/text/bindings.py +260 -0
  495. experiential-0.4.0/exp/simulation/engines/text/cell_progress.py +54 -0
  496. experiential-0.4.0/exp/simulation/engines/text/cell_progress_test.py +52 -0
  497. experiential-0.4.0/exp/simulation/engines/text/environment.py +66 -0
  498. experiential-0.4.0/exp/simulation/engines/text/episode_loop.py +111 -0
  499. experiential-0.4.0/exp/simulation/engines/text/errors.py +13 -0
  500. experiential-0.4.0/exp/simulation/engines/text/errors_test.py +0 -0
  501. experiential-0.4.0/exp/simulation/engines/text/grounded_rollout.py +141 -0
  502. experiential-0.4.0/exp/simulation/engines/text/grounded_rollout_test.py +0 -0
  503. experiential-0.4.0/exp/simulation/engines/text/grounding.py +336 -0
  504. experiential-0.4.0/exp/simulation/engines/text/grounding_test.py +0 -0
  505. experiential-0.4.0/exp/simulation/engines/text/leases.py +701 -0
  506. experiential-0.4.0/exp/simulation/engines/text/leases_test.py +487 -0
  507. experiential-0.4.0/exp/simulation/engines/text/prompt.py +172 -0
  508. experiential-0.4.0/exp/simulation/engines/text/prompt_test.py +84 -0
  509. experiential-0.4.0/exp/simulation/engines/text/recording.py +890 -0
  510. experiential-0.4.0/exp/simulation/engines/text/recording_test.py +730 -0
  511. experiential-0.4.0/exp/simulation/engines/text/redaction.py +261 -0
  512. experiential-0.4.0/exp/simulation/engines/text/redaction_test.py +242 -0
  513. experiential-0.4.0/exp/simulation/engines/text/resume.py +229 -0
  514. experiential-0.4.0/exp/simulation/engines/text/resume_test.py +1 -0
  515. experiential-0.4.0/exp/simulation/engines/text/rollout_support.py +170 -0
  516. experiential-0.4.0/exp/simulation/engines/text/simulator.py +977 -0
  517. experiential-0.4.0/exp/simulation/engines/text/simulator_test.py +2173 -0
  518. experiential-0.4.0/exp/simulation/engines/text/spec_persistence.py +60 -0
  519. experiential-0.4.0/exp/simulation/engines/text/spec_persistence_test.py +5 -0
  520. experiential-0.4.0/exp/simulation/ingest/__init__.py +64 -0
  521. experiential-0.4.0/exp/simulation/ingest/braintrust.py +260 -0
  522. experiential-0.4.0/exp/simulation/ingest/braintrust_test.py +69 -0
  523. experiential-0.4.0/exp/simulation/ingest/chat_json.py +318 -0
  524. experiential-0.4.0/exp/simulation/ingest/chat_json_test.py +124 -0
  525. experiential-0.4.0/exp/simulation/ingest/dataset.py +376 -0
  526. experiential-0.4.0/exp/simulation/ingest/dataset_test.py +477 -0
  527. experiential-0.4.0/exp/simulation/ingest/environment_capture.py +415 -0
  528. experiential-0.4.0/exp/simulation/ingest/environment_capture_test.py +321 -0
  529. experiential-0.4.0/exp/simulation/ingest/json_strict.py +31 -0
  530. experiential-0.4.0/exp/simulation/ingest/json_strict_test.py +25 -0
  531. experiential-0.4.0/exp/simulation/ingest/langfuse.py +355 -0
  532. experiential-0.4.0/exp/simulation/ingest/langfuse_test.py +147 -0
  533. experiential-0.4.0/exp/simulation/ingest/langsmith.py +233 -0
  534. experiential-0.4.0/exp/simulation/ingest/langsmith_test.py +77 -0
  535. experiential-0.4.0/exp/simulation/ingest/mastra.py +237 -0
  536. experiential-0.4.0/exp/simulation/ingest/mastra_test.py +83 -0
  537. experiential-0.4.0/exp/simulation/ingest/model_identity.py +279 -0
  538. experiential-0.4.0/exp/simulation/ingest/model_identity_test.py +1 -0
  539. experiential-0.4.0/exp/simulation/ingest/otel_genai.py +297 -0
  540. experiential-0.4.0/exp/simulation/ingest/otel_genai_test.py +152 -0
  541. experiential-0.4.0/exp/simulation/ingest/otlp.py +740 -0
  542. experiential-0.4.0/exp/simulation/ingest/otlp_test.py +569 -0
  543. experiential-0.4.0/exp/simulation/ingest/phoenix.py +574 -0
  544. experiential-0.4.0/exp/simulation/ingest/phoenix_test.py +151 -0
  545. experiential-0.4.0/exp/simulation/ingest/posthog.py +16 -0
  546. experiential-0.4.0/exp/simulation/ingest/posthog_canonical.py +620 -0
  547. experiential-0.4.0/exp/simulation/ingest/posthog_canonical_test.py +686 -0
  548. experiential-0.4.0/exp/simulation/ingest/posthog_pull.py +214 -0
  549. experiential-0.4.0/exp/simulation/ingest/sources.py +98 -0
  550. experiential-0.4.0/exp/simulation/ingest/sources_test.py +217 -0
  551. experiential-0.4.0/exp/simulation/ingest/testdata/sample_otlp.json +60 -0
  552. experiential-0.4.0/exp/simulation/ingest/testdata/sample_spans.jsonl +3 -0
  553. experiential-0.4.0/exp/simulation/ingest/tests/loader_contract_test.py +560 -0
  554. experiential-0.4.0/exp/simulation/ingest/trace_extensions.py +355 -0
  555. experiential-0.4.0/exp/simulation/ingest/trace_extensions_test.py +39 -0
  556. experiential-0.4.0/exp/simulation/ingest/vendor_observations.py +390 -0
  557. experiential-0.4.0/exp/simulation/ingest/vendor_observations_test.py +85 -0
  558. experiential-0.4.0/exp/simulation/ingest/vendor_records.py +434 -0
  559. experiential-0.4.0/exp/simulation/ingest/vendor_records_test.py +122 -0
  560. experiential-0.4.0/exp/simulation/ingest/vendor_source.py +155 -0
  561. experiential-0.4.0/exp/simulation/ingest/vendor_source_test.py +82 -0
  562. experiential-0.4.0/exp/simulation/ingest/vendor_trace.py +669 -0
  563. experiential-0.4.0/exp/simulation/ingest/vendor_trace_test.py +184 -0
  564. experiential-0.4.0/exp/simulation/mining/__init__.py +1 -0
  565. experiential-0.4.0/exp/simulation/mining/bindings.py +411 -0
  566. experiential-0.4.0/exp/simulation/mining/bindings_test.py +1 -0
  567. experiential-0.4.0/exp/simulation/mining/cleanup.py +323 -0
  568. experiential-0.4.0/exp/simulation/mining/coverage.py +359 -0
  569. experiential-0.4.0/exp/simulation/mining/deduplicate.py +500 -0
  570. experiential-0.4.0/exp/simulation/mining/descriptors.py +315 -0
  571. experiential-0.4.0/exp/simulation/mining/lineage.py +126 -0
  572. experiential-0.4.0/exp/simulation/mining/select.py +490 -0
  573. experiential-0.4.0/exp/simulation/mining/service.py +385 -0
  574. experiential-0.4.0/exp/simulation/mining/service_test.py +513 -0
  575. experiential-0.4.0/exp/simulation/orchestration/__init__.py +9 -0
  576. experiential-0.4.0/exp/simulation/orchestration/interface.py +45 -0
  577. experiential-0.4.0/exp/simulation/retrieval/__init__.py +119 -0
  578. experiential-0.4.0/exp/simulation/retrieval/build.py +265 -0
  579. experiential-0.4.0/exp/simulation/retrieval/build_inputs.py +62 -0
  580. experiential-0.4.0/exp/simulation/retrieval/build_inputs_test.py +813 -0
  581. experiential-0.4.0/exp/simulation/retrieval/contracts.py +254 -0
  582. experiential-0.4.0/exp/simulation/retrieval/embedding.py +130 -0
  583. experiential-0.4.0/exp/simulation/retrieval/refresh.py +794 -0
  584. experiential-0.4.0/exp/simulation/retrieval/refresh_dataset.py +211 -0
  585. experiential-0.4.0/exp/simulation/retrieval/refresh_dataset_test.py +0 -0
  586. experiential-0.4.0/exp/simulation/retrieval/refresh_test.py +1342 -0
  587. experiential-0.4.0/exp/simulation/retrieval/retriever.py +177 -0
  588. experiential-0.4.0/exp/simulation/retrieval/runtime_stitching.py +399 -0
  589. experiential-0.4.0/exp/simulation/retrieval/runtime_stitching_test.py +0 -0
  590. experiential-0.4.0/exp/simulation/retrieval/store.py +212 -0
  591. experiential-0.4.0/exp/simulation/retrieval/tests/retrieval_test.py +670 -0
  592. experiential-0.4.0/exp/simulation/retrieval/transitions.py +305 -0
  593. experiential-0.4.0/exp/simulation/specs/__init__.py +27 -0
  594. experiential-0.4.0/exp/simulation/specs/completion.py +221 -0
  595. experiential-0.4.0/exp/simulation/specs/completion_test.py +115 -0
  596. experiential-0.4.0/exp/simulation/specs/simulation.py +145 -0
  597. experiential-0.4.0/exp/simulation/specs/simulation_test.py +129 -0
  598. experiential-0.4.0/exp/simulation/tests/comparison_evidence_test.py +578 -0
  599. experiential-0.4.0/exp/simulation/world_model/__init__.py +91 -0
  600. experiential-0.4.0/exp/simulation/world_model/__init___test.py +0 -0
  601. experiential-0.4.0/exp/simulation/world_model/application.py +475 -0
  602. experiential-0.4.0/exp/simulation/world_model/application_test.py +572 -0
  603. experiential-0.4.0/exp/simulation/world_model/artifact.py +193 -0
  604. experiential-0.4.0/exp/simulation/world_model/artifact_test.py +0 -0
  605. experiential-0.4.0/exp/simulation/world_model/runtime.py +400 -0
  606. experiential-0.4.0/exp/simulation/world_model/runtime_test.py +315 -0
  607. experiential-0.4.0/exp/tests/api_test.py +231 -0
  608. experiential-0.4.0/exp/tests/release_revision_test.py +184 -0
  609. experiential-0.4.0/exp/tests/release_test.py +3184 -0
  610. experiential-0.4.0/exp/tests/repo_import_boundaries_test.py +235 -0
  611. experiential-0.4.0/exp/tests/repo_layout_test.py +147 -0
  612. experiential-0.4.0/pyproject.toml +87 -0
@@ -0,0 +1,42 @@
1
+ __pycache__/
2
+ *.py[cod]
3
+ .venv/
4
+ .env
5
+ .env.*
6
+ !.env.example
7
+ .exp/
8
+ dist/
9
+ *.egg-info/
10
+ .pytest_cache/
11
+ .ruff_cache/
12
+ .mypy_cache/
13
+ .ty_cache/
14
+ .DS_Store
15
+
16
+ # Local-only design notes; not committed.
17
+ DESIGN.md
18
+
19
+ demos/
20
+
21
+ # baseline-grid run scratch (not artifacts)
22
+ .exp-rebuild/
23
+ logs/
24
+
25
+ .exp-fixed/
26
+
27
+ # Local per-project settings (telemetry ids etc.) — never committed, wherever the root is.
28
+ settings.toml
29
+
30
+ # Claude Code local state (skills ARE committed; local settings/locks are not).
31
+ .claude/settings.local.json
32
+ .claude/scheduled_tasks.lock
33
+ .claude/projects/
34
+
35
+ # wandb run directories (local mirrors of what is already on the server)
36
+ wandb/
37
+
38
+ # External clones land in the working directory, never in git.
39
+ .tau2-bench/
40
+
41
+ # Local agent scratchpads: never tracked, never in PRs (AGENTS.md, Repository layout).
42
+ .agents/
@@ -0,0 +1,132 @@
1
+ Metadata-Version: 2.5
2
+ Name: experiential
3
+ Version: 0.4.0
4
+ Summary: Build simulations and optimize model routing from agent traces.
5
+ Requires-Python: >=3.12
6
+ Requires-Dist: boto3<2,>=1.35
7
+ Requires-Dist: botocore<2,>=1.35
8
+ Requires-Dist: click>=8.2
9
+ Requires-Dist: fastapi>=0.128
10
+ Requires-Dist: filelock>=3.12
11
+ Requires-Dist: httpx>=0.27
12
+ Requires-Dist: numpy>=1.26
13
+ Requires-Dist: openai<4,>=3.0
14
+ Requires-Dist: posthog>=7.0
15
+ Requires-Dist: pydantic>=2.6
16
+ Requires-Dist: rich>=14.1
17
+ Requires-Dist: tomli-w>=1.0
18
+ Requires-Dist: typer>=0.16
19
+ Requires-Dist: uvicorn>=0.38
20
+ Provides-Extra: dev
21
+ Requires-Dist: pytest-xdist>=3.8; extra == 'dev'
22
+ Requires-Dist: pytest>=8.0; extra == 'dev'
23
+ Requires-Dist: ruff>=0.5; extra == 'dev'
24
+ Requires-Dist: tinker-cookbook<0.5,>=0.4.3; extra == 'dev'
25
+ Requires-Dist: tinker<0.24,>=0.23; extra == 'dev'
26
+ Requires-Dist: ty>=0.0.1a1; extra == 'dev'
27
+ Provides-Extra: sft
28
+ Requires-Dist: tinker-cookbook<0.5,>=0.4.3; extra == 'sft'
29
+ Requires-Dist: tinker<0.24,>=0.23; extra == 'sft'
30
+ Description-Content-Type: text/markdown
31
+
32
+ # Experiential
33
+
34
+ Experiential is an open source gateway and router for agent workflows:
35
+
36
+ 1. Use hosted, BYOK, and local models through one OpenAI-compatible API.
37
+ 2. Control which users and agents can use which models, for which use cases, and how much they can spend.
38
+ 3. Turn production traffic into a custom router or model optimized for quality, speed, and cost.
39
+
40
+ ![Your traces flow through simulation into routing and training optimization](https://raw.githubusercontent.com/experientiallabs/experiential/main/assets/experiential-workflow.png)
41
+
42
+ <p align="center">
43
+ 🌐 <a href="https://platform.experientiallabs.ai">Platform</a> |
44
+ 📚 <a href="https://github.com/experientiallabs/experiential/tree/main/docs">Docs</a> |
45
+ <a href="https://discord.gg/B6sM8xTVwU"><img src="https://cdn.simpleicons.org/discord/5865F2" alt="" width="16" height="16"> Discord</a>
46
+ </p>
47
+
48
+ ## Getting Started
49
+
50
+ Start a local OpenAI-compatible gateway. On first run, the setup wizard asks for a provider,
51
+ model, and public alias, then prints a one-time virtual key:
52
+
53
+ ```bash
54
+ pip install experiential
55
+ exp run
56
+ ```
57
+
58
+ Choose a public alias such as `support-agent`, capture the issued key, and send a request:
59
+
60
+ ```bash
61
+ export EXP_GATEWAY_KEY=...
62
+ curl http://127.0.0.1:8000/v1/chat/completions \
63
+ -H "Authorization: Bearer $EXP_GATEWAY_KEY" \
64
+ -H 'Content-Type: application/json' \
65
+ -d '{"model":"support-agent","messages":[{"role":"user","content":"Help me"}]}'
66
+ ```
67
+
68
+ ## Using the API
69
+
70
+ Create a local gateway programmatically:
71
+
72
+ ```python
73
+ import uvicorn
74
+
75
+ from exp.runtime.gateway.lifecycle import load_local_gateway
76
+
77
+ gateway = load_local_gateway()
78
+ uvicorn.run(gateway.app, lifespan="on")
79
+ ```
80
+
81
+ For hosted workers with their own storage and provider services, use the lower-level
82
+ `exp.create_gateway_runtime(...)` composition API.
83
+
84
+ ## Optimize from Traffic
85
+
86
+ First, collect OpenTelemetry traces from your current agent. If you just want to try it out, grab
87
+ the public [terminal-tasks OTLP dataset](https://huggingface.co/datasets/experiential-labs/wmo-terminal-tasks-traces):
88
+
89
+ ```bash
90
+ curl -L -o traces.otel.jsonl \
91
+ https://huggingface.co/datasets/experiential-labs/wmo-terminal-tasks-traces/resolve/540883e451dc13d34fb50fdd36b143cb0f1fb0db/traces.otel.jsonl
92
+ ```
93
+
94
+ Then build a project. The build command walks you through providers,
95
+ models, and budget, and asks for your trace file:
96
+
97
+ ```bash
98
+ # Build simulation from your agent traces and optimize a router against it
99
+ exp build support-agent
100
+ ```
101
+
102
+ After collecting traces from your router, fine-tune an open source model you own using
103
+ [Tinker](https://tinker.thinkingmachines.ai/).
104
+
105
+ ```bash
106
+ exp optimize model support-agent
107
+ ```
108
+
109
+ ## Telemetry
110
+
111
+ Anonymous aggregate PostHog product telemetry is enabled by default. It never includes prompts,
112
+ traces, actions, observations, paths, model names, credentials, or raw customer content.
113
+
114
+ ```bash
115
+ exp config telemetry status
116
+ exp config telemetry disable
117
+ exp config telemetry enable
118
+ ```
119
+
120
+ The preference is stored locally in `.exp/settings.toml`.
121
+
122
+ ## Development
123
+
124
+ ```bash
125
+ uv sync --extra dev
126
+ uv run ruff format --check .
127
+ uv run ruff check .
128
+ uv run ty check
129
+ uv run pytest -q
130
+ ```
131
+
132
+ Repository and documentation conventions live in [AGENTS.md](./AGENTS.md).
@@ -0,0 +1,101 @@
1
+ # Experiential
2
+
3
+ Experiential is an open source gateway and router for agent workflows:
4
+
5
+ 1. Use hosted, BYOK, and local models through one OpenAI-compatible API.
6
+ 2. Control which users and agents can use which models, for which use cases, and how much they can spend.
7
+ 3. Turn production traffic into a custom router or model optimized for quality, speed, and cost.
8
+
9
+ ![Your traces flow through simulation into routing and training optimization](https://raw.githubusercontent.com/experientiallabs/experiential/main/assets/experiential-workflow.png)
10
+
11
+ <p align="center">
12
+ 🌐 <a href="https://platform.experientiallabs.ai">Platform</a> |
13
+ 📚 <a href="https://github.com/experientiallabs/experiential/tree/main/docs">Docs</a> |
14
+ <a href="https://discord.gg/B6sM8xTVwU"><img src="https://cdn.simpleicons.org/discord/5865F2" alt="" width="16" height="16"> Discord</a>
15
+ </p>
16
+
17
+ ## Getting Started
18
+
19
+ Start a local OpenAI-compatible gateway. On first run, the setup wizard asks for a provider,
20
+ model, and public alias, then prints a one-time virtual key:
21
+
22
+ ```bash
23
+ pip install experiential
24
+ exp run
25
+ ```
26
+
27
+ Choose a public alias such as `support-agent`, capture the issued key, and send a request:
28
+
29
+ ```bash
30
+ export EXP_GATEWAY_KEY=...
31
+ curl http://127.0.0.1:8000/v1/chat/completions \
32
+ -H "Authorization: Bearer $EXP_GATEWAY_KEY" \
33
+ -H 'Content-Type: application/json' \
34
+ -d '{"model":"support-agent","messages":[{"role":"user","content":"Help me"}]}'
35
+ ```
36
+
37
+ ## Using the API
38
+
39
+ Create a local gateway programmatically:
40
+
41
+ ```python
42
+ import uvicorn
43
+
44
+ from exp.runtime.gateway.lifecycle import load_local_gateway
45
+
46
+ gateway = load_local_gateway()
47
+ uvicorn.run(gateway.app, lifespan="on")
48
+ ```
49
+
50
+ For hosted workers with their own storage and provider services, use the lower-level
51
+ `exp.create_gateway_runtime(...)` composition API.
52
+
53
+ ## Optimize from Traffic
54
+
55
+ First, collect OpenTelemetry traces from your current agent. If you just want to try it out, grab
56
+ the public [terminal-tasks OTLP dataset](https://huggingface.co/datasets/experiential-labs/wmo-terminal-tasks-traces):
57
+
58
+ ```bash
59
+ curl -L -o traces.otel.jsonl \
60
+ https://huggingface.co/datasets/experiential-labs/wmo-terminal-tasks-traces/resolve/540883e451dc13d34fb50fdd36b143cb0f1fb0db/traces.otel.jsonl
61
+ ```
62
+
63
+ Then build a project. The build command walks you through providers,
64
+ models, and budget, and asks for your trace file:
65
+
66
+ ```bash
67
+ # Build simulation from your agent traces and optimize a router against it
68
+ exp build support-agent
69
+ ```
70
+
71
+ After collecting traces from your router, fine-tune an open source model you own using
72
+ [Tinker](https://tinker.thinkingmachines.ai/).
73
+
74
+ ```bash
75
+ exp optimize model support-agent
76
+ ```
77
+
78
+ ## Telemetry
79
+
80
+ Anonymous aggregate PostHog product telemetry is enabled by default. It never includes prompts,
81
+ traces, actions, observations, paths, model names, credentials, or raw customer content.
82
+
83
+ ```bash
84
+ exp config telemetry status
85
+ exp config telemetry disable
86
+ exp config telemetry enable
87
+ ```
88
+
89
+ The preference is stored locally in `.exp/settings.toml`.
90
+
91
+ ## Development
92
+
93
+ ```bash
94
+ uv sync --extra dev
95
+ uv run ruff format --check .
96
+ uv run ruff check .
97
+ uv run ty check
98
+ uv run pytest -q
99
+ ```
100
+
101
+ Repository and documentation conventions live in [AGENTS.md](./AGENTS.md).
@@ -0,0 +1,11 @@
1
+ """Repo-wide pytest configuration."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import os
6
+
7
+ # Rich consoles snapshot color support when constructed, and `exp.cli.app` builds its console at
8
+ # import time — so color-forcing vars must go before any test module imports it, or a dev shell
9
+ # exporting FORCE_COLOR would inject ANSI codes into CliRunner captures and fail assertions.
10
+ os.environ.pop("FORCE_COLOR", None)
11
+ os.environ.pop("CLICOLOR_FORCE", None)
@@ -0,0 +1,187 @@
1
+ # Local gateway architecture
2
+
3
+ ## Supported surface
4
+
5
+ No-argument `exp run` starts an authenticated multi-alias gateway on `127.0.0.1`.
6
+ It serves:
7
+
8
+ - `GET /v1/models`
9
+ - `POST /v1/chat/completions`
10
+ - `POST /v1/responses`
11
+ - `GET /health/live` and `GET /health/ready`
12
+ - `GET /usage` and `GET /usage.json`
13
+
14
+ `exp run PROJECT` is compatibility sugar that activates one project-backed alias and launches this
15
+ same gateway application. It does not create a router HTTP server. Gateway startup and readiness
16
+ perform no provider request. Only an authorized model request may cross the provider boundary.
17
+
18
+ ## Embeddable worker composition
19
+
20
+ Platform workers use the public `create_gateway_runtime` seam. The worker supplies storage,
21
+ provider, secret, project-selection, clock, readiness, and usage implementations, then owns the
22
+ returned lifecycle handle:
23
+
24
+ ```python
25
+ runtime = exp.create_gateway_runtime(
26
+ config=exp.GatewayRuntimeConfig(graceful_timeout_seconds=10),
27
+ authority=authority,
28
+ ledger=ledger,
29
+ routes=routes,
30
+ executor=executor,
31
+ clock=clock,
32
+ readiness=readiness,
33
+ usage=usage,
34
+ replay=replay,
35
+ continuations=continuations,
36
+ )
37
+
38
+ app = runtime.app
39
+ # Serve app with the platform's ASGI worker, including ASGI lifespan.
40
+ ```
41
+
42
+ The factory performs no filesystem, SQLite, environment, lock, or server access. A worker may use
43
+ any `SecretResolver` while constructing its provider executor and any `ProjectTargetResolver`
44
+ while constructing its catalog route resolver. The composed application always mounts the same
45
+ `create_gateway_app` data plane used by the local CLI. The lifecycle exposes explicit preflight,
46
+ readiness, bounded drain, and shutdown operations in addition to its ASGI lifespan.
47
+
48
+ An ASGI host that drives application lifespan owns preflight and shutdown automatically. A host
49
+ that does not drive lifespan calls `runtime.preflight()` before admission and
50
+ `runtime.shutdown()` during teardown. Shutdown is idempotent across those paths: one runtime starts
51
+ one bounded drain and one terminal flush, then remains permanently not ready.
52
+
53
+ ## Authority and management
54
+
55
+ `exp config gateway` owns explicit local setup. Its provider, identity, key, grant, alias, pool, and
56
+ monthly budget commands produce versioned receipts suitable for interactive or non-interactive
57
+ callers. There are no runtime seeds. A usable installation requires an organization, active
58
+ identity, active virtual key, explicit identity-to-alias grant, active alias revision, immutable
59
+ catalog snapshot, and a resolvable provider credential reference.
60
+
61
+ Private serving authority lives in `ROOT/gateway/gateway.db`, including identities, keys, grants,
62
+ provider connections and revisions, aliases and revisions, attempts, and usage. SQLite uses WAL
63
+ mode, versioned forward
64
+ migrations, private backups before migration, newer-schema refusal, and serialized initialization.
65
+ Virtual keys are stored only as peppered fingerprints. Key material is delivered once in a JSON
66
+ receipt or to a new mode-`0600` file, and commit ambiguity preserves recoverability. Provider
67
+ configuration stores an environment variable name, never its value. The local pepper is mode
68
+ `0600` and is not exported.
69
+
70
+ Every data-plane request is authenticated and authorized before request decoding, routing,
71
+ continuation lookup, or provider work. Authorization freezes organization, identity, API surface,
72
+ alias revision, target, catalog digest, request digest, optional hashed operation identity, and one
73
+ monotonic deadline. Identity disable, key revocation or expiry, grant removal, and alias revision
74
+ changes fail closed.
75
+
76
+ ## Catalog, aliases, and exact-model pools
77
+
78
+ The gateway database owns current provider connection state. Existing model metadata remains the
79
+ authoring input for builds, policies, evaluations, and datasets; it is not consulted as mutable
80
+ serving authority after an alias revision binds exact connection revisions. Gateway snapshots under
81
+ `ROOT/gateway/catalog-snapshots/` are immutable, secret-free artifacts for an exact catalog digest.
82
+ An advertised alias is executable only when readiness holds for the exact tuple of alias name,
83
+ alias revision, and catalog digest used by authorization.
84
+
85
+ An alias targets either:
86
+
87
+ 1. a direct exact-model pool, or
88
+ 2. one immutable project activation.
89
+
90
+ A singleton alias creates a one-deployment pool. `exp config gateway pool certify` can replace that
91
+ with an ordered pool only when every member has the same exact logical model identity and an
92
+ operator-supplied equivalence certification. The certification records an ID, provenance, evidence
93
+ digest, time, and exact deployment order. Project policy selection still chooses one exact logical
94
+ model; operational fallback can only move among certified deployments for that model.
95
+
96
+ ## Request, route, and provider attempts
97
+
98
+ The content-free ledger accepts the logical request before learned project selection. Selection or
99
+ direct resolution then produces an execution snapshot containing the exact model, pool, and ordered
100
+ deployment IDs. Each physical provider dispatch gets its own durable attempt row immediately before
101
+ network work. Attempt ordinal counts all physical dispatches; route depth identifies the selected
102
+ deployment position.
103
+
104
+ Provider execution is always internally streaming. Bounded same-deployment retries and ordered
105
+ deployment fallback are allowed only for typed precommit failures. The first outward text, refusal,
106
+ or tool-call semantic event commits the deployment, after which the gateway never switches
107
+ providers. Typed refusal fallback is disabled unless the active alias revision explicitly enables
108
+ it. Opted-in refusal deltas are withheld only in a bounded in-memory buffer: a refusal-only terminal
109
+ result can advance to the next certified deployment, while mixed semantic output or buffer overflow
110
+ commits and flushes the original route. Provider-internal retry layers are disabled so every
111
+ possible billable dispatch is visible to the gateway ledger.
112
+
113
+ Before each physical dispatch, the same immediate SQLite transaction reserves the request's
114
+ conservative maximum integer micro-USD cost and inserts its attempt row. Applicable hard limits can
115
+ cover the local team, one identity, one alias pool, and each provider deployment within that pool.
116
+ An exhausted deployment allocation removes only that route from the current certified waterfall.
117
+ If no route can fit the shared team, identity, or total pool allocation, the neutral protocol
118
+ returns HTTP 429 with OpenAI `insufficient_quota` semantics before provider work. Any required
119
+ unknown price makes that route ineligible while a hard limit applies.
120
+
121
+ Settlement replaces the reservation with observed integer micro-USD usage. A dispatched failure,
122
+ cancellation, or crash without trustworthy usage retains its conservative reservation because it
123
+ may be billable. Retries and fallbacks therefore consume one allocation entry per physical attempt,
124
+ while keyed replay creates no new reservation. A period is the immutable UTC bucket beginning at
125
+ `YYYY-MM-01T00:00:00+00:00`; rollover selects a new bucket and never clears or rewrites an earlier
126
+ month. Management and remaining-allocation reports are CLI surfaces only. There is no budgets
127
+ dashboard.
128
+
129
+ Each physical attempt records its own provider, model, usage, latency, terminal state, estimated
130
+ cost attribution, and frozen credential-ownership billing source. Later catalog activation and
131
+ process restart never rewrite that source. Schema-v1/v2 attempt rows migrate explicitly as
132
+ `customer_managed`; current dispatches persist either `host_managed` or `customer_managed` before
133
+ network work. The public usage report conserves physical attempt, token, cost, unknown-cost, and
134
+ terminal totals across those source buckets without partitioning logical request counts. The parent
135
+ request terminalizes once after success, final failure, cancellation, disconnect, or crash
136
+ reconciliation. Unknown prices remain unknown instead of being treated as zero or copied across
137
+ deployments.
138
+
139
+ ## OpenAI-compatible protocol
140
+
141
+ `exp/runtime/openai_protocol` is the only OpenAI wire implementation. Chat Completions and
142
+ Responses have separate allowlist decoders and field-specific OpenAI error responses, but both
143
+ convert to one canonical gateway request without conflating their wire contracts. The package also
144
+ owns headers, response assembly, SSE framing, tool-call reconstruction, and official SDK
145
+ compatibility.
146
+ Chat streaming emits valid completion chunks and one `[DONE]`. Responses streaming emits the
147
+ created, in-progress, output, and exactly one terminal lifecycle. Provider tool-argument fragments
148
+ are accumulated in original order and validated only at the complete-call boundary.
149
+
150
+ Commit-independent headers are available before streaming begins. Route-dependent headers are
151
+ emitted only after an execution snapshot exists. Stable public IDs do not expose raw key,
152
+ idempotency, request, or provider values.
153
+
154
+ OpenAI `3.0.0` `OpenAI` and `AsyncOpenAI` clients are release-certified for Chat Completions and
155
+ Responses in synchronous and asynchronous, streaming and non-streaming forms. Responses
156
+ continuation and duplicate replay retain content only in bounded, process-local, tenant and
157
+ alias-revision-scoped stores. Replay is opt-in through an idempotency or client request key. Restart
158
+ or eviction returns an explicit unavailable error and never reconstructs content from SQLite.
159
+
160
+ ## Content-free observability and lifecycle
161
+
162
+ SQLite stores hashes, frozen authority, route identity, state transitions, token counts, latency,
163
+ and estimated cost. It never stores prompts, responses, raw tool arguments, raw virtual keys, or
164
+ provider secrets. `GET /usage` and `GET /usage.json` are two renderings of the same schema-v2 report
165
+ and expose only aggregate, per-identity, and physical-attempt `by_billing_source` accounting.
166
+ Source buckets conserve attempt, token, known-cost, unknown-cost, and terminal-state totals but do
167
+ not partition logical request counts. Estimated cost is attribution, not a provider invoice.
168
+
169
+ The process owns readiness from preflight through bounded drain. New work is rejected after drain
170
+ starts. Admitted tasks, upstream streams, disconnect cleanup, replay ownership, continuation state,
171
+ and final ledger settlement are process-owned and bounded. A stuck cancellation cannot prevent the
172
+ terminal flusher from attempting content-free settlement.
173
+
174
+ ## Certification boundary
175
+
176
+ Deterministic release evidence uses a built and freshly installed wheel, real SQLite, a real
177
+ subprocess-bound loopback gateway, a real loopback upstream, and the official SDK clients. One
178
+ scanner checks database, WAL, backups when present, catalog snapshots, stdout, stderr, logs, usage
179
+ responses, and error bodies for raw content and secret canaries.
180
+
181
+ `exp/runtime/gateway/provider_certification.py` is the dated provider capability matrix. Each cell
182
+ names the official client SDK, public gateway surfaces, provider wire surface, fixture result, and
183
+ credential-gated live status. OpenAI and Anthropic have native fixtures; generic OpenAI-compatible,
184
+ Azure, and OpenRouter share compatible-stream coverage; Gemini and Bedrock have native deterministic
185
+ fixtures. Live provider cells remain explicitly `not_run_requires_credentials` until a separately
186
+ authorized run supplies dated evidence. Deterministic fixtures do not imply hosted-provider
187
+ availability, billing, or account-specific behavior.
@@ -0,0 +1,86 @@
1
+ # Release scope
2
+
3
+ This release supports the current source and one wheel with either core dependencies or the
4
+ optional `sft` dependency extra on their documented local paths. It claims only behavior exercised
5
+ on the exact release checkout.
6
+
7
+ ## Supported and verified
8
+
9
+ - Root CLI commands are exactly `build`, `config`, `optimize`, and `run`; optimizer commands are
10
+ exactly `router` and `model`.
11
+ - The local gateway supports explicit provider references, identities, virtual keys, grants,
12
+ singleton and certified ordered exact-model pools, frozen-project aliases, bounded precommit
13
+ provider fallback, Chat Completions, Responses, bounded in-memory continuation and replay,
14
+ content-free SQLite accounting, monthly integer micro-USD enforcement, and loopback-only health
15
+ and usage views.
16
+ - Both no-argument gateway launch and the retained `exp run PROJECT [--ghost]` compatibility form
17
+ are installed-wheel surfaces. Gateway startup is provider-idle and requires explicit authority.
18
+ - The installed-wheel gateway lane uses real SQLite, a real subprocess listener, a real loopback
19
+ upstream, and OpenAI `3.0.0`. It covers `OpenAI` and `AsyncOpenAI` across Chat Completions and
20
+ Responses, with both stream and non-stream requests. HTML and JSON usage are checked for the same
21
+ per-identity accounting values. The same lane measures default SDK retries against physical
22
+ attempts, provider-authentication fallback, refusal policy, post-commit no-switch, restart and
23
+ replay behavior, revocation, cancellation, WAL mode, and mixed `host_managed`/
24
+ `customer_managed` cost attribution that remains frozen across restart and catalog replacement.
25
+ - Real-SQLite monthly budget evidence covers atomic concurrent reservations, provider-only route
26
+ exhaustion, shared identity quota errors, billable failure and crash retention, retry and
27
+ fallback accounting, keyed replay without duplicate spend, explicit schema migration, and UTC
28
+ month rollover without a reset job. The real loopback waterfall is configured through the
29
+ interactive-capable CLI and returns OpenAI `insufficient_quota` after shared exhaustion.
30
+ - Deterministic source-level certification additionally covers selection-only project routing,
31
+ authentication-circuit open/skip/recovery, concurrent multi-identity key revocation and alias
32
+ revision activation, and atomic rollback of a failed legacy SQLite migration.
33
+ - One content and secret canary scanner covers the gateway database, live WAL, migration backups
34
+ when present, catalog snapshots, stdout, stderr, logs, usage responses, and HTTP error bodies.
35
+ - Public Python exposes provider-free build, explicit router composition, frozen selection-only
36
+ router load through the normal gateway application, structural text-versus-sandbox comparison,
37
+ and managed SFT composition. No separate router HTTP or SSE implementation is shipped.
38
+ - W16 router evidence uses 100 normalized traces, 50 fit tasks, 20 held-out tasks, 140 planned
39
+ cells, 130 deterministic text simulations, and 140 deterministic judgments under one finite
40
+ simulation and judgment budget. Observed hosted-service spend is exactly $0.00.
41
+ - W16 sandbox evidence compares two exact post-lock text and Darwin local-process pairs. It retains
42
+ one malformed sandbox failure in the denominator and claims structural terminal agreement only.
43
+ - Exact-checkout CI supplies the full 40-hex Git revision, recursively verifies every evidence
44
+ artifact and manifest input, and publishes machine-readable JSON plus JUnit evidence.
45
+
46
+ ## Gateway provider evidence matrix
47
+
48
+ This table separates deterministic protocol evidence from hosted calls that need account
49
+ credentials. `Not run` means exactly that; it is not inferred from fixture coverage.
50
+
51
+ | Provider surface | Deterministic evidence in this release | Credential-gated live evidence |
52
+ |---|---|---|
53
+ | OpenAI | Native Responses fixtures for text, tool arguments, usage, cancellation, and refusal; all eight official SDK quadrants run against the installed local gateway | Not run; requires an OpenAI credential |
54
+ | Anthropic | Native Messages fixtures for text, tool arguments, usage, cancellation, and refusal through both public gateway surfaces | Not run; requires an Anthropic credential |
55
+ | Generic OpenAI-compatible | Real loopback upstream through the installed gateway; text, tool arguments, usage, cancellation, and refusal contracts | Not run; requires a compatible hosted endpoint and credential |
56
+ | Azure OpenAI | Compatible-adapter fixtures for text, tool arguments, usage, cancellation, and refusal | Not run; requires Azure endpoint and credential |
57
+ | OpenRouter | Compatible-adapter fixtures for text, tool arguments, usage, cancellation, and refusal | Not run; requires OpenRouter credential |
58
+ | Gemini | Native fixtures for text, structured complete function arguments, usage, cancellation, and refusal | Not run; requires Gemini credential |
59
+ | Amazon Bedrock | Native EventStream fixtures for text, incremental tool arguments, usage, bounded cancellation, refusal, and single dispatch | Not run; requires an authorized AWS account and region |
60
+
61
+ The machine-readable dated matrix is
62
+ `exp/runtime/gateway/provider_certification.py`. Its live cells are
63
+ `not_run_requires_credentials`. Gemini complete structured function arguments are explicitly not
64
+ labeled as provider-byte incremental tool-argument streaming.
65
+
66
+ ## Explicitly excluded
67
+
68
+ - There is no budgets dashboard. Monthly allocation management and remaining-allocation reporting
69
+ are explicit interactive or non-interactive CLI operations.
70
+
71
+ - No paid E2B or Harbor cloud smoke ran. The repository verifies the optional `bounded-close-v1`
72
+ Harbor lifecycle and ledger with injected fakes, but makes no cloud cleanup, provider-quality, or
73
+ environment-parity claim.
74
+ - No real Tinker training ran. Managed SFT remains fail-closed unless its immutable configuration
75
+ has a finite positive ceiling and its backend supplies a conservative full-schedule estimate.
76
+ - No trained-versus-base behavioral comparison ran because this release produced no paid training
77
+ artifact. It makes no trained-model quality-improvement claim.
78
+ - No hosted model, judge, embedding, telemetry, environment, credential, or `.env` path was used by
79
+ release evidence. The deterministic W16 evidence reports exactly $0.00 observed service spend.
80
+ - Deterministic gateway certification uses a real loopback upstream and local SQLite. No live
81
+ provider matrix cell ran, so hosted availability, account limits, billing, and service-specific
82
+ behavior are not claimed.
83
+
84
+ These exclusions are product boundaries, not evidence that the corresponding hosted services are
85
+ unsafe or unsupported forever. Any future claim requires separately authorized, finite-budget,
86
+ denominator-preserving evidence.