agent-learning-kit 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (642) hide show
  1. agent_learning_kit-0.1.0.dist-info/METADATA +381 -0
  2. agent_learning_kit-0.1.0.dist-info/RECORD +642 -0
  3. agent_learning_kit-0.1.0.dist-info/WHEEL +4 -0
  4. agent_learning_kit-0.1.0.dist-info/entry_points.txt +5 -0
  5. agent_learning_kit-0.1.0.dist-info/licenses/LICENSE +173 -0
  6. agent_learning_kit-0.1.0.dist-info/licenses/NOTICE +7 -0
  7. fi/__init__.py +5 -0
  8. fi/alk/__init__.py +57 -0
  9. fi/alk/_facade.py +31 -0
  10. fi/alk/_module_alias.py +68 -0
  11. fi/alk/_paths.py +14 -0
  12. fi/alk/_schema.py +522 -0
  13. fi/alk/actions.py +727 -0
  14. fi/alk/bench/__init__.py +517 -0
  15. fi/alk/bench/_codeexec.py +213 -0
  16. fi/alk/bench/_coding.py +215 -0
  17. fi/alk/bench/_docker.py +237 -0
  18. fi/alk/bench/_grader.py +286 -0
  19. fi/alk/bench/_pull.py +212 -0
  20. fi/alk/bench/_voice.py +147 -0
  21. fi/alk/capabilities.py +627 -0
  22. fi/alk/cli.py +6396 -0
  23. fi/alk/config.py +130 -0
  24. fi/alk/cua_loop.py +562 -0
  25. fi/alk/evals.py +2351 -0
  26. fi/alk/extensions.py +163 -0
  27. fi/alk/harness/ARCHITECTURE.md +231 -0
  28. fi/alk/harness/DESIGN.md +246 -0
  29. fi/alk/harness/ENVIRONMENT_CONFORMANCE.md +127 -0
  30. fi/alk/harness/HOW-IT-WORKS.md +297 -0
  31. fi/alk/harness/IMPLEMENTATION_AND_VALIDATION_STATUS.md +229 -0
  32. fi/alk/harness/README.md +417 -0
  33. fi/alk/harness/__init__.py +77 -0
  34. fi/alk/harness/__main__.py +3 -0
  35. fi/alk/harness/amend.py +312 -0
  36. fi/alk/harness/artifacts.py +319 -0
  37. fi/alk/harness/authoring_entrypoint.py +189 -0
  38. fi/alk/harness/authoring_runtime_validation.py +267 -0
  39. fi/alk/harness/backends/README.md +43 -0
  40. fi/alk/harness/backends/__init__.py +122 -0
  41. fi/alk/harness/backends/base.py +241 -0
  42. fi/alk/harness/backends/claude.py +211 -0
  43. fi/alk/harness/backends/files.py +182 -0
  44. fi/alk/harness/backends/vertex_gemini.py +457 -0
  45. fi/alk/harness/background_noise.py +95 -0
  46. fi/alk/harness/build.py +385 -0
  47. fi/alk/harness/bundle.py +593 -0
  48. fi/alk/harness/bundle_author_v2.py +1831 -0
  49. fi/alk/harness/bundle_v2.py +719 -0
  50. fi/alk/harness/call_runner.py +1440 -0
  51. fi/alk/harness/callback_http_adapter.py +111 -0
  52. fi/alk/harness/catalogue.py +287 -0
  53. fi/alk/harness/chat.py +428 -0
  54. fi/alk/harness/chat_call_runner.py +506 -0
  55. fi/alk/harness/checks.py +136 -0
  56. fi/alk/harness/cli.py +1354 -0
  57. fi/alk/harness/config.py +338 -0
  58. fi/alk/harness/contract.py +718 -0
  59. fi/alk/harness/credentials.py +674 -0
  60. fi/alk/harness/data/persona_vocabulary.json +111 -0
  61. fi/alk/harness/environment.py +99 -0
  62. fi/alk/harness/environment_plan.py +168 -0
  63. fi/alk/harness/events.py +125 -0
  64. fi/alk/harness/executor.py +304 -0
  65. fi/alk/harness/folder.py +234 -0
  66. fi/alk/harness/generated_runtime.py +815 -0
  67. fi/alk/harness/github.py +72 -0
  68. fi/alk/harness/hosted_authoring_entrypoint.py +183 -0
  69. fi/alk/harness/hosted_entrypoint.py +2402 -0
  70. fi/alk/harness/hosted_scheduler.py +2218 -0
  71. fi/alk/harness/job.py +426 -0
  72. fi/alk/harness/judge.py +184 -0
  73. fi/alk/harness/livekit_source.py +50 -0
  74. fi/alk/harness/livekit_tool_trace_bootstrap.py +71 -0
  75. fi/alk/harness/observability.py +208 -0
  76. fi/alk/harness/outbound.py +3252 -0
  77. fi/alk/harness/packaging.py +515 -0
  78. fi/alk/harness/persona_guides.py +157 -0
  79. fi/alk/harness/platform.py +692 -0
  80. fi/alk/harness/process_preflight.py +764 -0
  81. fi/alk/harness/process_runtime.py +5670 -0
  82. fi/alk/harness/prove.py +425 -0
  83. fi/alk/harness/provider_import.py +703 -0
  84. fi/alk/harness/provider_lifecycle.py +392 -0
  85. fi/alk/harness/provision.py +2896 -0
  86. fi/alk/harness/reception.py +147 -0
  87. fi/alk/harness/retell_chat_call_runner.py +373 -0
  88. fi/alk/harness/run/__init__.py +296 -0
  89. fi/alk/harness/run/alk.py +184 -0
  90. fi/alk/harness/run/call.py +162 -0
  91. fi/alk/harness/run/conversation.py +264 -0
  92. fi/alk/harness/run/data/voices_by_language_and_gender.json +693 -0
  93. fi/alk/harness/run/evidence.py +195 -0
  94. fi/alk/harness/run/grade.py +598 -0
  95. fi/alk/harness/run/live.py +297 -0
  96. fi/alk/harness/run/models.py +56 -0
  97. fi/alk/harness/run/platform_evals.py +227 -0
  98. fi/alk/harness/run/sdk_voice.py +130 -0
  99. fi/alk/harness/run/simulation.py +1209 -0
  100. fi/alk/harness/run/stage.py +91 -0
  101. fi/alk/harness/run/targets.py +508 -0
  102. fi/alk/harness/run/tools.py +601 -0
  103. fi/alk/harness/run/voice.py +340 -0
  104. fi/alk/harness/runtime.py +172 -0
  105. fi/alk/harness/sandbox_server.py +2011 -0
  106. fi/alk/harness/sandbox_worker.py +44 -0
  107. fi/alk/harness/scenario.py +1048 -0
  108. fi/alk/harness/scenario_source.py +879 -0
  109. fi/alk/harness/scenario_tools.py +1143 -0
  110. fi/alk/harness/scenarios.py +915 -0
  111. fi/alk/harness/secrets.py +168 -0
  112. fi/alk/harness/service_catalog.py +97 -0
  113. fi/alk/harness/session.py +391 -0
  114. fi/alk/harness/sessions.py +372 -0
  115. fi/alk/harness/simulator.py +76 -0
  116. fi/alk/harness/simulator_voice.py +928 -0
  117. fi/alk/harness/skills/build-environment/SKILL.md +538 -0
  118. fi/alk/harness/skills/harness.md +131 -0
  119. fi/alk/harness/skills/kinds/chat.md +48 -0
  120. fi/alk/harness/skills/kinds/voice-voicemail.md +63 -0
  121. fi/alk/harness/skills/kinds/voice.md +59 -0
  122. fi/alk/harness/skills/plan-suite/SKILL.md +103 -0
  123. fi/alk/harness/skills/provision-environment/SKILL.md +136 -0
  124. fi/alk/harness/skills/run-scenarios/SKILL.md +112 -0
  125. fi/alk/harness/skills/understand-agent/SKILL.md +251 -0
  126. fi/alk/harness/skills/write-scenarios/SKILL.md +606 -0
  127. fi/alk/harness/skills/write-scenarios/references/refusals.md +28 -0
  128. fi/alk/harness/skills/write-scenarios/references/world-api.md +92 -0
  129. fi/alk/harness/source_data_invariants.py +444 -0
  130. fi/alk/harness/source_tool_evidence.py +79 -0
  131. fi/alk/harness/sources.py +253 -0
  132. fi/alk/harness/spend.py +140 -0
  133. fi/alk/harness/tool_trace_proxy.py +104 -0
  134. fi/alk/harness/tools.py +1018 -0
  135. fi/alk/harness/understand.py +169 -0
  136. fi/alk/harness/voicemail_audio.py +74 -0
  137. fi/alk/harness/world/__init__.py +33 -0
  138. fi/alk/harness/world/errors.py +68 -0
  139. fi/alk/harness/world/expectations.py +91 -0
  140. fi/alk/harness/world/handle.py +538 -0
  141. fi/alk/harness/world/kinds.py +196 -0
  142. fi/alk/harness/world/mutate.py +186 -0
  143. fi/alk/harness/world/probe.py +413 -0
  144. fi/alk/harness/world/provision.py +511 -0
  145. fi/alk/harness/world/provisioned.py +191 -0
  146. fi/alk/harness/world/runtime.py +616 -0
  147. fi/alk/harness/world/snapshot.py +288 -0
  148. fi/alk/harness/world/stores/__init__.py +305 -0
  149. fi/alk/harness/world/stores/container.py +215 -0
  150. fi/alk/harness/world/stores/inprocess.py +346 -0
  151. fi/alk/harness/world/stores/postgres.py +481 -0
  152. fi/alk/harness/world/stores/prove.py +202 -0
  153. fi/alk/harness/world/stores/sqlite.py +245 -0
  154. fi/alk/harness/world/stores/written.py +182 -0
  155. fi/alk/harness/world/tools.py +1516 -0
  156. fi/alk/harness/world/workspace.py +144 -0
  157. fi/alk/image_loop.py +453 -0
  158. fi/alk/image_perturb.py +241 -0
  159. fi/alk/improve.py +274 -0
  160. fi/alk/live/__init__.py +154 -0
  161. fi/alk/live/_attribution.py +184 -0
  162. fi/alk/live/_capture.py +264 -0
  163. fi/alk/live/_codec.py +391 -0
  164. fi/alk/live/_contract.py +134 -0
  165. fi/alk/live/_loopback.py +316 -0
  166. fi/alk/live/_perturb.py +449 -0
  167. fi/alk/live/_runner.py +386 -0
  168. fi/alk/live/_stats.py +561 -0
  169. fi/alk/live/_transcript.py +240 -0
  170. fi/alk/live/_workers/__init__.py +9 -0
  171. fi/alk/live/_workers/a2a_worker.py +316 -0
  172. fi/alk/live/_workers/langgraph_worker.py +217 -0
  173. fi/alk/live/_workers/livekit_worker.py +207 -0
  174. fi/alk/live/_workers/mcp_loopback_server.py +46 -0
  175. fi/alk/live/_workers/mcp_worker.py +158 -0
  176. fi/alk/live/_workers/pipecat_worker.py +189 -0
  177. fi/alk/live/a2a_lane.py +138 -0
  178. fi/alk/live/langgraph_lane.py +339 -0
  179. fi/alk/live/livekit_lane.py +376 -0
  180. fi/alk/live/mcp_lane.py +172 -0
  181. fi/alk/live/pipecat_lane.py +341 -0
  182. fi/alk/live/voice_redteam.py +494 -0
  183. fi/alk/loss.py +306 -0
  184. fi/alk/optimize.py +36260 -0
  185. fi/alk/practice/__init__.py +51 -0
  186. fi/alk/practice/_assess.py +103 -0
  187. fi/alk/practice/_budget.py +81 -0
  188. fi/alk/practice/_calibrate.py +69 -0
  189. fi/alk/practice/_capstone.py +86 -0
  190. fi/alk/practice/_contract.py +91 -0
  191. fi/alk/practice/_diagnose.py +79 -0
  192. fi/alk/practice/_drill.py +196 -0
  193. fi/alk/practice/_experiment.py +720 -0
  194. fi/alk/practice/_schedule.py +102 -0
  195. fi/alk/practice/_store.py +194 -0
  196. fi/alk/practice/_trainer.py +245 -0
  197. fi/alk/practice/_update.py +125 -0
  198. fi/alk/redteam.py +2621 -0
  199. fi/alk/rewardhack.py +237 -0
  200. fi/alk/simulate.py +10351 -0
  201. fi/alk/studio/__init__.py +82 -0
  202. fi/alk/studio/_bias.py +314 -0
  203. fi/alk/studio/_calibration.py +522 -0
  204. fi/alk/studio/_coverage.py +262 -0
  205. fi/alk/studio/_download.py +665 -0
  206. fi/alk/studio/_fidelity_attack.py +114 -0
  207. fi/alk/studio/_generate.py +652 -0
  208. fi/alk/studio/_library.py +370 -0
  209. fi/alk/studio/_scan.py +134 -0
  210. fi/alk/studio/_upgrade.py +42 -0
  211. fi/alk/studio/_vendor.py +172 -0
  212. fi/alk/suite.py +4200 -0
  213. fi/alk/tasks.py +828 -0
  214. fi/alk/telemetry/__init__.py +149 -0
  215. fi/alk/telemetry/_contract.py +141 -0
  216. fi/alk/telemetry/_emit.py +182 -0
  217. fi/alk/telemetry/_ledger.py +296 -0
  218. fi/alk/telemetry/_queue.py +127 -0
  219. fi/alk/telemetry/_row.py +294 -0
  220. fi/alk/telemetry/_run.py +233 -0
  221. fi/alk/telemetry/_sync.py +193 -0
  222. fi/alk/telemetry/_url.py +119 -0
  223. fi/alk/trinity.py +49397 -0
  224. fi/alk/voice_loop.py +174 -0
  225. fi/api/__init__.py +1 -0
  226. fi/api/auth.py +137 -0
  227. fi/api/types.py +29 -0
  228. fi/cli/__init__.py +9 -0
  229. fi/cli/assertions/__init__.py +25 -0
  230. fi/cli/assertions/conditions.py +76 -0
  231. fi/cli/assertions/evaluator.py +286 -0
  232. fi/cli/assertions/exit_codes.py +20 -0
  233. fi/cli/assertions/parser.py +131 -0
  234. fi/cli/assertions/reporter.py +194 -0
  235. fi/cli/commands/__init__.py +9 -0
  236. fi/cli/commands/config.py +165 -0
  237. fi/cli/commands/export.py +208 -0
  238. fi/cli/commands/init.py +112 -0
  239. fi/cli/commands/list_cmd.py +213 -0
  240. fi/cli/commands/run.py +486 -0
  241. fi/cli/commands/validate.py +173 -0
  242. fi/cli/commands/view.py +424 -0
  243. fi/cli/config/__init__.py +6 -0
  244. fi/cli/config/defaults.py +206 -0
  245. fi/cli/config/loader.py +155 -0
  246. fi/cli/config/schema.py +174 -0
  247. fi/cli/main.py +78 -0
  248. fi/cli/output/__init__.py +6 -0
  249. fi/cli/output/formatters.py +106 -0
  250. fi/cli/output/reporters.py +46 -0
  251. fi/cli/storage/__init__.py +5 -0
  252. fi/cli/storage/run_history.py +249 -0
  253. fi/cli/utils/__init__.py +5 -0
  254. fi/cli/utils/console.py +44 -0
  255. fi/evals/__init__.py +131 -0
  256. fi/evals/autoeval/__init__.py +137 -0
  257. fi/evals/autoeval/analyzer.py +211 -0
  258. fi/evals/autoeval/config.py +244 -0
  259. fi/evals/autoeval/export.py +213 -0
  260. fi/evals/autoeval/interactive.py +283 -0
  261. fi/evals/autoeval/pipeline.py +625 -0
  262. fi/evals/autoeval/prompts.py +139 -0
  263. fi/evals/autoeval/recommender.py +242 -0
  264. fi/evals/autoeval/rules.py +589 -0
  265. fi/evals/autoeval/templates.py +299 -0
  266. fi/evals/autoeval/types.py +232 -0
  267. fi/evals/core/__init__.py +16 -0
  268. fi/evals/core/cloud_registry.py +184 -0
  269. fi/evals/core/engines.py +368 -0
  270. fi/evals/core/evaluate.py +319 -0
  271. fi/evals/core/judge_prompt.py +90 -0
  272. fi/evals/core/prompt_generator.py +83 -0
  273. fi/evals/core/registry.py +57 -0
  274. fi/evals/core/result.py +55 -0
  275. fi/evals/evaluator.py +721 -0
  276. fi/evals/execution.py +168 -0
  277. fi/evals/feedback/__init__.py +32 -0
  278. fi/evals/feedback/calibrator.py +160 -0
  279. fi/evals/feedback/collector.py +214 -0
  280. fi/evals/feedback/hooks.py +81 -0
  281. fi/evals/feedback/retriever.py +128 -0
  282. fi/evals/feedback/store.py +272 -0
  283. fi/evals/feedback/types.py +99 -0
  284. fi/evals/framework/README.md +79 -0
  285. fi/evals/framework/__init__.py +267 -0
  286. fi/evals/framework/backends/Dockerfile.eval-runner +33 -0
  287. fi/evals/framework/backends/__init__.py +99 -0
  288. fi/evals/framework/backends/_container.py +141 -0
  289. fi/evals/framework/backends/_utils.py +145 -0
  290. fi/evals/framework/backends/base.py +223 -0
  291. fi/evals/framework/backends/celery_backend.py +417 -0
  292. fi/evals/framework/backends/celery_worker.py +78 -0
  293. fi/evals/framework/backends/kubernetes_backend.py +665 -0
  294. fi/evals/framework/backends/ray_backend.py +521 -0
  295. fi/evals/framework/backends/temporal.py +350 -0
  296. fi/evals/framework/backends/temporal_worker.py +126 -0
  297. fi/evals/framework/backends/thread_pool.py +286 -0
  298. fi/evals/framework/context.py +258 -0
  299. fi/evals/framework/enrichment.py +306 -0
  300. fi/evals/framework/evals/__init__.py +68 -0
  301. fi/evals/framework/evals/agentic.py +399 -0
  302. fi/evals/framework/evals/builder.py +609 -0
  303. fi/evals/framework/evals/semantic.py +142 -0
  304. fi/evals/framework/evaluator.py +647 -0
  305. fi/evals/framework/evaluators/__init__.py +22 -0
  306. fi/evals/framework/evaluators/blocking.py +347 -0
  307. fi/evals/framework/evaluators/non_blocking.py +577 -0
  308. fi/evals/framework/propagation.py +421 -0
  309. fi/evals/framework/protocols.py +385 -0
  310. fi/evals/framework/registry.py +370 -0
  311. fi/evals/framework/resilience/__init__.py +150 -0
  312. fi/evals/framework/resilience/circuit_breaker.py +309 -0
  313. fi/evals/framework/resilience/degradation.py +355 -0
  314. fi/evals/framework/resilience/health.py +505 -0
  315. fi/evals/framework/resilience/rate_limiter.py +228 -0
  316. fi/evals/framework/resilience/retry.py +274 -0
  317. fi/evals/framework/resilience/types.py +288 -0
  318. fi/evals/framework/resilience/wrapper.py +433 -0
  319. fi/evals/framework/types.py +218 -0
  320. fi/evals/guardrails/README.md +915 -0
  321. fi/evals/guardrails/__init__.py +96 -0
  322. fi/evals/guardrails/backends/__init__.py +43 -0
  323. fi/evals/guardrails/backends/azure.py +361 -0
  324. fi/evals/guardrails/backends/base.py +88 -0
  325. fi/evals/guardrails/backends/generic_llm.py +163 -0
  326. fi/evals/guardrails/backends/granite.py +216 -0
  327. fi/evals/guardrails/backends/llamaguard.py +221 -0
  328. fi/evals/guardrails/backends/local_base.py +479 -0
  329. fi/evals/guardrails/backends/openai.py +365 -0
  330. fi/evals/guardrails/backends/qwen.py +170 -0
  331. fi/evals/guardrails/backends/shieldgemma.py +154 -0
  332. fi/evals/guardrails/backends/turing.py +235 -0
  333. fi/evals/guardrails/backends/vllm_client.py +321 -0
  334. fi/evals/guardrails/backends/wildguard.py +188 -0
  335. fi/evals/guardrails/base.py +888 -0
  336. fi/evals/guardrails/config.py +221 -0
  337. fi/evals/guardrails/discovery.py +243 -0
  338. fi/evals/guardrails/gateway.py +437 -0
  339. fi/evals/guardrails/registry.py +231 -0
  340. fi/evals/guardrails/scanners/__init__.py +127 -0
  341. fi/evals/guardrails/scanners/base.py +191 -0
  342. fi/evals/guardrails/scanners/code_injection.py +243 -0
  343. fi/evals/guardrails/scanners/eval_delegate.py +574 -0
  344. fi/evals/guardrails/scanners/invisible_chars.py +351 -0
  345. fi/evals/guardrails/scanners/jailbreak.py +412 -0
  346. fi/evals/guardrails/scanners/language.py +288 -0
  347. fi/evals/guardrails/scanners/pipeline.py +260 -0
  348. fi/evals/guardrails/scanners/regex.py +311 -0
  349. fi/evals/guardrails/scanners/secrets.py +274 -0
  350. fi/evals/guardrails/scanners/topics.py +649 -0
  351. fi/evals/guardrails/scanners/urls.py +341 -0
  352. fi/evals/guardrails/types.py +96 -0
  353. fi/evals/llm/__init__.py +3 -0
  354. fi/evals/llm/base_llm_provider.py +35 -0
  355. fi/evals/llm/providers/litellm.py +70 -0
  356. fi/evals/local/__init__.py +90 -0
  357. fi/evals/local/evaluator.py +690 -0
  358. fi/evals/local/execution_mode.py +121 -0
  359. fi/evals/local/llm.py +489 -0
  360. fi/evals/local/metrics/__init__.py +19 -0
  361. fi/evals/local/registry.py +360 -0
  362. fi/evals/manager.py +1018 -0
  363. fi/evals/manager_types.py +362 -0
  364. fi/evals/metrics/__init__.py +185 -0
  365. fi/evals/metrics/agents/__init__.py +74 -0
  366. fi/evals/metrics/agents/metrics.py +693 -0
  367. fi/evals/metrics/agents/report.py +36463 -0
  368. fi/evals/metrics/agents/types.py +160 -0
  369. fi/evals/metrics/base_llm_metric.py +111 -0
  370. fi/evals/metrics/base_metric.py +138 -0
  371. fi/evals/metrics/code_security/__init__.py +305 -0
  372. fi/evals/metrics/code_security/analyzer.py +985 -0
  373. fi/evals/metrics/code_security/benchmarks/__init__.py +73 -0
  374. fi/evals/metrics/code_security/benchmarks/builtin.py +750 -0
  375. fi/evals/metrics/code_security/benchmarks/loader.py +580 -0
  376. fi/evals/metrics/code_security/benchmarks/types.py +308 -0
  377. fi/evals/metrics/code_security/detectors/__init__.py +186 -0
  378. fi/evals/metrics/code_security/detectors/base.py +394 -0
  379. fi/evals/metrics/code_security/detectors/cryptography.py +345 -0
  380. fi/evals/metrics/code_security/detectors/injection.py +744 -0
  381. fi/evals/metrics/code_security/detectors/secrets.py +287 -0
  382. fi/evals/metrics/code_security/detectors/serialization.py +192 -0
  383. fi/evals/metrics/code_security/joint_metrics.py +588 -0
  384. fi/evals/metrics/code_security/judges/__init__.py +83 -0
  385. fi/evals/metrics/code_security/judges/base.py +238 -0
  386. fi/evals/metrics/code_security/judges/dual_judge.py +534 -0
  387. fi/evals/metrics/code_security/judges/llm_judge.py +301 -0
  388. fi/evals/metrics/code_security/judges/pattern_judge.py +515 -0
  389. fi/evals/metrics/code_security/metrics.py +388 -0
  390. fi/evals/metrics/code_security/modes/__init__.py +63 -0
  391. fi/evals/metrics/code_security/modes/adversarial.py +284 -0
  392. fi/evals/metrics/code_security/modes/autocomplete.py +198 -0
  393. fi/evals/metrics/code_security/modes/base.py +283 -0
  394. fi/evals/metrics/code_security/modes/instruct.py +253 -0
  395. fi/evals/metrics/code_security/modes/repair.py +230 -0
  396. fi/evals/metrics/code_security/reports/__init__.py +57 -0
  397. fi/evals/metrics/code_security/reports/generator.py +404 -0
  398. fi/evals/metrics/code_security/reports/leaderboard.py +509 -0
  399. fi/evals/metrics/code_security/types.py +534 -0
  400. fi/evals/metrics/function_calling/__init__.py +34 -0
  401. fi/evals/metrics/function_calling/metrics.py +573 -0
  402. fi/evals/metrics/function_calling/types.py +87 -0
  403. fi/evals/metrics/hallucination/__init__.py +54 -0
  404. fi/evals/metrics/hallucination/detector.py +149 -0
  405. fi/evals/metrics/hallucination/metrics.py +390 -0
  406. fi/evals/metrics/hallucination/nli.py +253 -0
  407. fi/evals/metrics/hallucination/sentinel.py +106 -0
  408. fi/evals/metrics/hallucination/types.py +132 -0
  409. fi/evals/metrics/heuristics/aggregation_metrics.py +85 -0
  410. fi/evals/metrics/heuristics/json_metrics.py +87 -0
  411. fi/evals/metrics/heuristics/similarity_metrics.py +375 -0
  412. fi/evals/metrics/heuristics/string_metrics.py +391 -0
  413. fi/evals/metrics/llm_as_judges/__init__.py +17 -0
  414. fi/evals/metrics/llm_as_judges/custom_judge/metric.py +112 -0
  415. fi/evals/metrics/llm_as_judges/custom_judge/prompts.py +26 -0
  416. fi/evals/metrics/llm_as_judges/types.py +48 -0
  417. fi/evals/metrics/rag/__init__.py +111 -0
  418. fi/evals/metrics/rag/advanced/__init__.py +14 -0
  419. fi/evals/metrics/rag/advanced/multi_hop.py +283 -0
  420. fi/evals/metrics/rag/advanced/source_attribution.py +344 -0
  421. fi/evals/metrics/rag/generation/__init__.py +17 -0
  422. fi/evals/metrics/rag/generation/answer_relevancy.py +176 -0
  423. fi/evals/metrics/rag/generation/context_utilization.py +245 -0
  424. fi/evals/metrics/rag/generation/faithfulness.py +241 -0
  425. fi/evals/metrics/rag/generation/groundedness.py +131 -0
  426. fi/evals/metrics/rag/rag_score.py +277 -0
  427. fi/evals/metrics/rag/retrieval/__init__.py +20 -0
  428. fi/evals/metrics/rag/retrieval/context_entity_recall.py +124 -0
  429. fi/evals/metrics/rag/retrieval/context_precision.py +158 -0
  430. fi/evals/metrics/rag/retrieval/context_recall.py +106 -0
  431. fi/evals/metrics/rag/retrieval/noise_sensitivity.py +163 -0
  432. fi/evals/metrics/rag/retrieval/ranking.py +261 -0
  433. fi/evals/metrics/rag/types.py +100 -0
  434. fi/evals/metrics/rag/utils/__init__.py +62 -0
  435. fi/evals/metrics/rag/utils/claims.py +189 -0
  436. fi/evals/metrics/rag/utils/entities.py +244 -0
  437. fi/evals/metrics/rag/utils/nli.py +92 -0
  438. fi/evals/metrics/rag/utils/similarity.py +345 -0
  439. fi/evals/metrics/structured/__init__.py +114 -0
  440. fi/evals/metrics/structured/field_completeness.py +313 -0
  441. fi/evals/metrics/structured/hierarchy_score.py +366 -0
  442. fi/evals/metrics/structured/json_validation.py +190 -0
  443. fi/evals/metrics/structured/schema_compliance.py +280 -0
  444. fi/evals/metrics/structured/structured_output_score.py +298 -0
  445. fi/evals/metrics/structured/types.py +108 -0
  446. fi/evals/metrics/structured/validators/__init__.py +30 -0
  447. fi/evals/metrics/structured/validators/base.py +189 -0
  448. fi/evals/metrics/structured/validators/json_validator.py +196 -0
  449. fi/evals/metrics/structured/validators/pydantic_validator.py +178 -0
  450. fi/evals/metrics/structured/validators/yaml_validator.py +248 -0
  451. fi/evals/otel/__init__.py +266 -0
  452. fi/evals/otel/config.py +400 -0
  453. fi/evals/otel/conventions.py +463 -0
  454. fi/evals/otel/enrichment.py +371 -0
  455. fi/evals/otel/instrumentors/__init__.py +140 -0
  456. fi/evals/otel/instrumentors/anthropic.py +517 -0
  457. fi/evals/otel/instrumentors/base.py +382 -0
  458. fi/evals/otel/instrumentors/openai.py +673 -0
  459. fi/evals/otel/processors/__init__.py +36 -0
  460. fi/evals/otel/processors/base.py +473 -0
  461. fi/evals/otel/processors/cost.py +445 -0
  462. fi/evals/otel/processors/evaluation.py +559 -0
  463. fi/evals/otel/processors/llm.py +462 -0
  464. fi/evals/otel/tracer.py +506 -0
  465. fi/evals/otel/types.py +232 -0
  466. fi/evals/otel_utils.py +23 -0
  467. fi/evals/protect.py +671 -0
  468. fi/evals/protect_input_adapter.py +154 -0
  469. fi/evals/streaming/__init__.py +88 -0
  470. fi/evals/streaming/buffer.py +213 -0
  471. fi/evals/streaming/evaluator.py +551 -0
  472. fi/evals/streaming/policy.py +307 -0
  473. fi/evals/streaming/scorers.py +368 -0
  474. fi/evals/streaming/types.py +238 -0
  475. fi/evals/templates.py +472 -0
  476. fi/evals/types.py +156 -0
  477. fi/opt/__init__.py +221 -0
  478. fi/opt/_objective_scoring.py +85 -0
  479. fi/opt/base/__init__.py +11 -0
  480. fi/opt/base/base_generator.py +33 -0
  481. fi/opt/base/base_mapper.py +26 -0
  482. fi/opt/base/base_optimizer.py +45 -0
  483. fi/opt/base/evaluator.py +211 -0
  484. fi/opt/components.py +3095 -0
  485. fi/opt/datamappers/__init__.py +3 -0
  486. fi/opt/datamappers/basic_mapper.py +40 -0
  487. fi/opt/deployment.py +1021 -0
  488. fi/opt/evidence.py +4332 -0
  489. fi/opt/generators/__init__.py +3 -0
  490. fi/opt/generators/litellm.py +66 -0
  491. fi/opt/integrations/__init__.py +23 -0
  492. fi/opt/integrations/generative_suite.py +410 -0
  493. fi/opt/integrations/simulate.py +1313 -0
  494. fi/opt/mutations.py +771 -0
  495. fi/opt/observability.py +4639 -0
  496. fi/opt/optimizer_trace.py +889 -0
  497. fi/opt/optimizers/__init__.py +80 -0
  498. fi/opt/optimizers/agent.py +331 -0
  499. fi/opt/optimizers/agent_bandit.py +392 -0
  500. fi/opt/optimizers/agent_curriculum.py +635 -0
  501. fi/opt/optimizers/agent_evolution.py +894 -0
  502. fi/opt/optimizers/agent_feedback.py +1863 -0
  503. fi/opt/optimizers/agent_pareto.py +547 -0
  504. fi/opt/optimizers/agent_social_memory.py +1113 -0
  505. fi/opt/optimizers/agent_tpe.py +321 -0
  506. fi/opt/optimizers/bayesian_search.py +449 -0
  507. fi/opt/optimizers/council.py +2075 -0
  508. fi/opt/optimizers/futureagi_replay.py +799 -0
  509. fi/opt/optimizers/gepa.py +322 -0
  510. fi/opt/optimizers/metaprompt.py +243 -0
  511. fi/opt/optimizers/promptwizard.py +417 -0
  512. fi/opt/optimizers/protegi.py +329 -0
  513. fi/opt/optimizers/random_search.py +224 -0
  514. fi/opt/research.py +518 -0
  515. fi/opt/simulation.py +260 -0
  516. fi/opt/targets.py +232 -0
  517. fi/opt/types.py +66 -0
  518. fi/opt/utils/__init__.py +4 -0
  519. fi/opt/utils/early_stopping.py +266 -0
  520. fi/opt/utils/setup_logging.py +82 -0
  521. fi/simulate/__init__.py +540 -0
  522. fi/simulate/_hashing.py +35 -0
  523. fi/simulate/_logging.py +10 -0
  524. fi/simulate/adapters.py +87 -0
  525. fi/simulate/agent/__init__.py +120 -0
  526. fi/simulate/agent/browser.py +658 -0
  527. fi/simulate/agent/definition.py +587 -0
  528. fi/simulate/agent/frameworks.py +3528 -0
  529. fi/simulate/agent/generic.py +8286 -0
  530. fi/simulate/agent/import_probe.py +227 -0
  531. fi/simulate/agent/memory.py +905 -0
  532. fi/simulate/agent/mocks.py +101 -0
  533. fi/simulate/agent/multi_agent.py +361 -0
  534. fi/simulate/agent/orchestration.py +903 -0
  535. fi/simulate/agent/realtime.py +665 -0
  536. fi/simulate/agent/wrapper.py +99 -0
  537. fi/simulate/agent/wrappers/__init__.py +18 -0
  538. fi/simulate/agent/wrappers/anthropic.py +62 -0
  539. fi/simulate/agent/wrappers/gemini.py +65 -0
  540. fi/simulate/agent/wrappers/http.py +404 -0
  541. fi/simulate/agent/wrappers/langchain.py +80 -0
  542. fi/simulate/agent/wrappers/openai.py +75 -0
  543. fi/simulate/agent/wrappers/websocket.py +326 -0
  544. fi/simulate/artifacts/__init__.py +11 -0
  545. fi/simulate/artifacts/manifest.py +62 -0
  546. fi/simulate/cli.py +20560 -0
  547. fi/simulate/endpoints/__init__.py +45 -0
  548. fi/simulate/endpoints/_http_actor.py +73 -0
  549. fi/simulate/endpoints/actor_sources.py +243 -0
  550. fi/simulate/endpoints/base.py +107 -0
  551. fi/simulate/endpoints/builtins.py +10 -0
  552. fi/simulate/endpoints/callable.py +95 -0
  553. fi/simulate/endpoints/http.py +76 -0
  554. fi/simulate/endpoints/livekit.py +138 -0
  555. fi/simulate/endpoints/originators.py +132 -0
  556. fi/simulate/endpoints/profiles.py +348 -0
  557. fi/simulate/endpoints/retell.py +633 -0
  558. fi/simulate/endpoints/vapi.py +205 -0
  559. fi/simulate/endpoints/websocket.py +76 -0
  560. fi/simulate/environment.py +33026 -0
  561. fi/simulate/environments/__init__.py +11 -0
  562. fi/simulate/environments/base.py +73 -0
  563. fi/simulate/environments/chat.py +697 -0
  564. fi/simulate/environments/voice.py +212 -0
  565. fi/simulate/evaluation/__init__.py +4 -0
  566. fi/simulate/evaluation/ai_eval.py +227 -0
  567. fi/simulate/evidence/__init__.py +35 -0
  568. fi/simulate/evidence/base.py +59 -0
  569. fi/simulate/evidence/caller_observed.py +50 -0
  570. fi/simulate/evidence/livekit_instrumentation.py +51 -0
  571. fi/simulate/evidence/livekit_room.py +50 -0
  572. fi/simulate/evidence/otel.py +49 -0
  573. fi/simulate/evidence/providers/__init__.py +24 -0
  574. fi/simulate/evidence/providers/base.py +61 -0
  575. fi/simulate/evidence/providers/retell.py +376 -0
  576. fi/simulate/evidence/providers/vapi.py +426 -0
  577. fi/simulate/hosted/__init__.py +32 -0
  578. fi/simulate/hosted/child_entrypoint.py +306 -0
  579. fi/simulate/hosted/job.py +150 -0
  580. fi/simulate/hosted/targets.py +53 -0
  581. fi/simulate/instrumentation/__init__.py +5 -0
  582. fi/simulate/instrumentation/livekit/__init__.py +122 -0
  583. fi/simulate/manifest.py +1033 -0
  584. fi/simulate/matrix_cli.py +165 -0
  585. fi/simulate/realtime/__init__.py +40 -0
  586. fi/simulate/realtime/events.py +107 -0
  587. fi/simulate/realtime/media.py +61 -0
  588. fi/simulate/realtime/session.py +91 -0
  589. fi/simulate/recording/__init__.py +5 -0
  590. fi/simulate/recording/room_recorder.py +326 -0
  591. fi/simulate/registry.py +185 -0
  592. fi/simulate/results/__init__.py +9 -0
  593. fi/simulate/results/base.py +18 -0
  594. fi/simulate/results/filesystem.py +71 -0
  595. fi/simulate/results/futureagi.py +1340 -0
  596. fi/simulate/runtime/__init__.py +85 -0
  597. fi/simulate/runtime/capabilities.py +40 -0
  598. fi/simulate/runtime/events.py +63 -0
  599. fi/simulate/runtime/failures.py +25 -0
  600. fi/simulate/runtime/ids.py +34 -0
  601. fi/simulate/runtime/plan.py +70 -0
  602. fi/simulate/runtime/planner.py +102 -0
  603. fi/simulate/runtime/report.py +174 -0
  604. fi/simulate/runtime/run.py +75 -0
  605. fi/simulate/runtime/runner.py +333 -0
  606. fi/simulate/runtime/spec.py +186 -0
  607. fi/simulate/simulation/__init__.py +30 -0
  608. fi/simulate/simulation/behavior_policy.py +425 -0
  609. fi/simulate/simulation/bridge/__init__.py +9 -0
  610. fi/simulate/simulation/bridge/audio.py +29 -0
  611. fi/simulate/simulation/bridge/connector.py +46 -0
  612. fi/simulate/simulation/bridge/livekit.py +252 -0
  613. fi/simulate/simulation/bridge/retell.py +188 -0
  614. fi/simulate/simulation/bridge/vapi.py +177 -0
  615. fi/simulate/simulation/contract.py +419 -0
  616. fi/simulate/simulation/engines/__init__.py +12 -0
  617. fi/simulate/simulation/engines/base.py +21 -0
  618. fi/simulate/simulation/engines/cloud.py +517 -0
  619. fi/simulate/simulation/engines/livekit.py +4167 -0
  620. fi/simulate/simulation/engines/local_text.py +89 -0
  621. fi/simulate/simulation/fidelity.py +374 -0
  622. fi/simulate/simulation/gemini_tts_stream.py +110 -0
  623. fi/simulate/simulation/generator.py +91 -0
  624. fi/simulate/simulation/goal_machine.py +185 -0
  625. fi/simulate/simulation/livekit_models.py +467 -0
  626. fi/simulate/simulation/matrix.py +170 -0
  627. fi/simulate/simulation/models.py +279 -0
  628. fi/simulate/simulation/runner.py +153 -0
  629. fi/simulate/simulation/synthetic.py +880 -0
  630. fi/simulate/simulation/voice_prompt.py +502 -0
  631. fi/simulate/simulator/__init__.py +55 -0
  632. fi/simulate/simulator/builtins.py +53 -0
  633. fi/simulate/suite.py +1288 -0
  634. fi/simulate/utils/routes.py +164 -0
  635. fi/simulate/voice.py +225 -0
  636. fi/simulate/voice_cli.py +182 -0
  637. fi/utils/__init__.py +1 -0
  638. fi/utils/constants.py +14 -0
  639. fi/utils/errors.py +200 -0
  640. fi/utils/executor.py +26 -0
  641. fi/utils/routes.py +119 -0
  642. fi/utils/utils.py +17 -0
@@ -0,0 +1,703 @@
1
+ """Built-in, ownership-safe import adapters for provider-hosted voice targets.
2
+
3
+ The adapter copies configuration; it never fabricates tool implementations. Custom HTTP tools are
4
+ rewired to an already-running world endpoint. Provider-native tools are preserved. Every created
5
+ resource is returned in a validated lifecycle receipt so cleanup is exact and retryable.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import json
11
+ from collections.abc import Callable, Mapping
12
+ from dataclasses import dataclass
13
+ from typing import Any, Literal
14
+ from urllib.parse import urlsplit, urlunsplit
15
+ from urllib.error import HTTPError, URLError
16
+ from urllib.request import Request, urlopen
17
+
18
+ from pydantic import BaseModel, ConfigDict, Field
19
+ from pydantic import model_validator
20
+
21
+ from .provider_lifecycle import (
22
+ ProviderCleanupReceipt,
23
+ ProviderContext,
24
+ ProviderProvisionReceipt,
25
+ ProviderResource,
26
+ ProviderTarget,
27
+ ProviderType,
28
+ )
29
+
30
+
31
+ class ProviderImportError(RuntimeError):
32
+ """The provider definition could not be copied without guessing."""
33
+
34
+
35
+ class ProviderImportSpec(BaseModel):
36
+ model_config = ConfigDict(extra="forbid")
37
+
38
+ type: ProviderType
39
+ source_target_id: str = Field(min_length=1)
40
+ public_capability: str = Field(pattern=r"^[a-z][a-z0-9_]*$")
41
+ environment_tools: list[str] = Field(default_factory=list)
42
+ event_path: str = "/provider/events"
43
+ tool_path: str = "/provider/tools"
44
+ api_base_url: str | None = None
45
+ target_modality: Literal["voice", "chat"] = "voice"
46
+
47
+ @model_validator(mode="after")
48
+ def _provider_api_origin_is_fixed(self) -> "ProviderImportSpec":
49
+ if self.api_base_url:
50
+ expected = {
51
+ ProviderType.VAPI: "https://api.vapi.ai",
52
+ ProviderType.RETELL: "https://api.retellai.com",
53
+ }[self.type]
54
+ if self.api_base_url.rstrip("/") != expected:
55
+ raise ValueError("provider_import_api_base_url_not_allowed")
56
+ for name, value in (
57
+ ("event_path", self.event_path),
58
+ ("tool_path", self.tool_path),
59
+ ):
60
+ if (
61
+ not value.startswith("/")
62
+ or value.startswith("//")
63
+ or ".." in value.split("/")
64
+ ):
65
+ raise ValueError(f"provider_import_{name}_invalid")
66
+ return self
67
+
68
+ @model_validator(mode="after")
69
+ def _environment_tool_names_are_unambiguous(self) -> "ProviderImportSpec":
70
+ normalized = [name.strip() for name in self.environment_tools]
71
+ if any(not name for name in normalized) or len(set(normalized)) != len(
72
+ normalized
73
+ ):
74
+ raise ValueError("provider_import_environment_tools_invalid")
75
+ self.environment_tools = normalized
76
+ return self
77
+
78
+
79
+ JsonRequest = Callable[[str, str, str, Mapping[str, Any] | None], dict[str, Any]]
80
+
81
+
82
+ def _safe_profile_value(value: Any, key: str = "") -> Any:
83
+ """Return provider configuration that is safe to give the authoring model.
84
+
85
+ Imported assistant definitions are part of the source of truth for contract and scenario
86
+ authoring, but provider responses may also contain credentials or signed callback URLs. Keep
87
+ behavioral configuration while removing secrets and URL userinfo/query fragments.
88
+ """
89
+ lowered = key.lower()
90
+ if any(
91
+ marker in lowered
92
+ for marker in (
93
+ "secret",
94
+ "token",
95
+ "api_key",
96
+ "apikey",
97
+ "credential",
98
+ "authorization",
99
+ )
100
+ ):
101
+ return "[redacted]"
102
+ if isinstance(value, Mapping):
103
+ return {
104
+ str(item_key): _safe_profile_value(item, str(item_key))
105
+ for item_key, item in value.items()
106
+ }
107
+ if isinstance(value, list):
108
+ return [_safe_profile_value(item, key) for item in value]
109
+ if isinstance(value, str) and "://" in value:
110
+ parsed = urlsplit(value)
111
+ if parsed.scheme and parsed.netloc:
112
+ hostname = parsed.hostname or ""
113
+ port = f":{parsed.port}" if parsed.port else ""
114
+ return urlunsplit((parsed.scheme, hostname + port, parsed.path, "", ""))
115
+ return value
116
+
117
+
118
+ def inspect_provider_target(
119
+ provider: ProviderType | str,
120
+ *,
121
+ source_target_id: str,
122
+ api_key: str,
123
+ api_base_url: str | None = None,
124
+ target_modality: Literal["voice", "chat"] = "voice",
125
+ request: JsonRequest | None = None,
126
+ ) -> dict[str, Any]:
127
+ """Fetch a sanitized, read-only behavioral profile for hosted authoring.
128
+
129
+ This deliberately performs no create/update/delete operation. The profile lets contract and
130
+ scenario generation see the externally hosted prompt, model/voice configuration and exact
131
+ tool schemas instead of guessing solely from a submitted webhook implementation.
132
+ """
133
+ provider = ProviderType(provider)
134
+ if not api_key:
135
+ raise ProviderImportError(f"{provider.value}_api_key_missing")
136
+ if not source_target_id:
137
+ raise ProviderImportError("provider_import_source_target_id_missing")
138
+ request = request or _request_json
139
+ if provider is ProviderType.VAPI:
140
+ base = (api_base_url or "https://api.vapi.ai").rstrip("/")
141
+ assistant = request(
142
+ "GET", f"{base}/assistant/{source_target_id}", api_key, None
143
+ )
144
+ model = assistant.get("model")
145
+ reusable_tools: list[dict[str, Any]] = []
146
+ if isinstance(model, Mapping):
147
+ for tool_id in model.get("toolIds") or []:
148
+ reusable_tools.append(
149
+ request("GET", f"{base}/tool/{tool_id}", api_key, None)
150
+ )
151
+ profile = {
152
+ "provider": "vapi",
153
+ "source_target_id": source_target_id,
154
+ "name": assistant.get("name"),
155
+ "first_message": assistant.get("firstMessage"),
156
+ "first_message_mode": assistant.get("firstMessageMode"),
157
+ "model": model,
158
+ "voice": assistant.get("voice"),
159
+ "transcriber": assistant.get("transcriber"),
160
+ "end_call_message": assistant.get("endCallMessage"),
161
+ "end_call_phrases": assistant.get("endCallPhrases"),
162
+ "reusable_tools": reusable_tools,
163
+ }
164
+ else:
165
+ base = (api_base_url or "https://api.retellai.com").rstrip("/")
166
+ agent_path = "get-chat-agent" if target_modality == "chat" else "get-agent"
167
+ agent = request("GET", f"{base}/{agent_path}/{source_target_id}", api_key, None)
168
+ engine = agent.get("response_engine")
169
+ if not isinstance(engine, Mapping):
170
+ raise ProviderImportError(
171
+ "retell_response_engine_unsupported: response_engine must be an object"
172
+ )
173
+ engine_type = str(engine.get("type") or "").strip()
174
+ profile: dict[str, Any] = {
175
+ "provider": "retell",
176
+ "modality": target_modality,
177
+ "source_target_id": source_target_id,
178
+ "name": agent.get("agent_name"),
179
+ "voice_id": agent.get("voice_id"),
180
+ "language": agent.get("language"),
181
+ "responsiveness": agent.get("responsiveness"),
182
+ "interruption_sensitivity": agent.get("interruption_sensitivity"),
183
+ "enable_backchannel": agent.get("enable_backchannel"),
184
+ "response_engine_type": engine_type,
185
+ }
186
+ if engine_type == "retell-llm":
187
+ llm_id = str(engine.get("llm_id") or "").strip()
188
+ if not llm_id:
189
+ raise ProviderImportError("retell_response_engine_missing_llm_id")
190
+ llm = request("GET", f"{base}/get-retell-llm/{llm_id}", api_key, None)
191
+ profile.update(
192
+ {
193
+ "begin_message": llm.get("begin_message"),
194
+ "general_prompt": llm.get("general_prompt"),
195
+ "general_tools": llm.get("general_tools"),
196
+ "states": llm.get("states"),
197
+ "model": llm.get("model"),
198
+ }
199
+ )
200
+ elif engine_type == "conversation-flow":
201
+ flow_id = str(engine.get("conversation_flow_id") or "").strip()
202
+ if not flow_id:
203
+ raise ProviderImportError(
204
+ "retell_response_engine_missing_conversation_flow_id"
205
+ )
206
+ flow = request(
207
+ "GET", f"{base}/get-conversation-flow/{flow_id}", api_key, None
208
+ )
209
+ profile["conversation_flow"] = flow
210
+ else:
211
+ raise ProviderImportError(
212
+ "retell_response_engine_unsupported: provider_import supports "
213
+ "retell-llm and conversation-flow"
214
+ )
215
+ return _safe_profile_value(profile)
216
+
217
+
218
+ def _request_json(
219
+ method: str, url: str, api_key: str, body: Mapping[str, Any] | None
220
+ ) -> dict[str, Any]:
221
+ data = None if body is None else json.dumps(body).encode("utf-8")
222
+ request = Request(
223
+ url,
224
+ data=data,
225
+ method=method,
226
+ headers={
227
+ "Authorization": f"Bearer {api_key}",
228
+ "Content-Type": "application/json",
229
+ "Accept": "application/json",
230
+ # Vapi's Cloudflare policy rejects urllib's default Python user
231
+ # agent as Error 1010 from hosted sandboxes. Use a stable product
232
+ # identity for provider API traffic instead.
233
+ "User-Agent": "FutureAGI-ALK/1.0",
234
+ },
235
+ )
236
+ try:
237
+ with urlopen(request, timeout=30) as response: # noqa: S310 - fixed provider hosts
238
+ raw = response.read()
239
+ except HTTPError as exc:
240
+ # Provider APIs commonly return a safe machine-readable explanation (for
241
+ # example an account/IP policy or an invalid target id). Preserve a
242
+ # bounded, single-line excerpt so hosted failures are actionable. Never
243
+ # include request headers or the caller's credential.
244
+ try:
245
+ detail = exc.read(512).decode("utf-8", errors="replace")
246
+ except (AttributeError, OSError):
247
+ detail = ""
248
+ detail = " ".join(detail.split())
249
+ suffix = f": {detail}" if detail else ""
250
+ raise ProviderImportError(
251
+ f"provider_api_error: {method} returned HTTP {exc.code}{suffix}"
252
+ ) from exc
253
+ except (URLError, TimeoutError) as exc:
254
+ raise ProviderImportError(
255
+ f"provider_api_unavailable: {method} {type(exc).__name__}"
256
+ ) from exc
257
+ if not raw:
258
+ return {}
259
+ try:
260
+ value = json.loads(raw)
261
+ except ValueError as exc:
262
+ raise ProviderImportError("provider_api_response_invalid_json") from exc
263
+ if not isinstance(value, dict):
264
+ raise ProviderImportError("provider_api_response_must_be_object")
265
+ return value
266
+
267
+
268
+ def _without(body: Mapping[str, Any], names: set[str]) -> dict[str, Any]:
269
+ return {key: value for key, value in body.items() if key not in names}
270
+
271
+
272
+ def _join(base: str, original: Any) -> str:
273
+ """Preserve a declared route while moving only its origin into the world."""
274
+ from urllib.parse import urlsplit
275
+
276
+ path = urlsplit(str(original or "")).path
277
+ if not path or path == "/":
278
+ return base
279
+ base_path = urlsplit(base).path.rstrip("/")
280
+ # A common source definition already uses the same logical mount
281
+ # (`/provider/tools/...`) as ALK's signed world endpoint. Do not append the
282
+ # mount twice when only the origin is changing.
283
+ if base_path and (path == base_path or path.startswith(base_path + "/")):
284
+ path = path[len(base_path) :]
285
+ return base.rstrip("/") + "/" + path.lstrip("/")
286
+
287
+
288
+ def _rewire_vapi_tool(tool: Mapping[str, Any], tool_base_url: str) -> dict[str, Any]:
289
+ copied = dict(tool)
290
+ kind = str(copied.get("type") or "")
291
+ if kind == "apiRequest" and copied.get("url"):
292
+ copied["url"] = _join(tool_base_url, copied["url"])
293
+ elif kind == "function":
294
+ server = copied.get("server")
295
+ if isinstance(server, Mapping) and server.get("url"):
296
+ copied["server"] = {**server, "url": _join(tool_base_url, server["url"])}
297
+ return copied
298
+
299
+
300
+ def _vapi_custom_tool_name(tool: Mapping[str, Any]) -> str | None:
301
+ kind = str(tool.get("type") or "")
302
+ if kind == "function":
303
+ function = tool.get("function")
304
+ if isinstance(function, Mapping):
305
+ return str(function.get("name") or "").strip() or None
306
+ if kind == "apiRequest":
307
+ return str(tool.get("name") or "").strip() or None
308
+ return None
309
+
310
+
311
+ def _require_environment_tool(
312
+ *, provider: str, name: str | None, available: set[str]
313
+ ) -> None:
314
+ if not name:
315
+ raise ProviderImportError(f"{provider}_custom_tool_missing_name")
316
+ if name not in available:
317
+ raise ProviderImportError(
318
+ f"provider_tool_implementation_missing: {provider} tool {name!r} "
319
+ "is not declared by the submitted environment"
320
+ )
321
+
322
+
323
+ def _rewire_retell_tools(value: Any, tool_base_url: str) -> Any:
324
+ if isinstance(value, list):
325
+ return [_rewire_retell_tools(item, tool_base_url) for item in value]
326
+ if not isinstance(value, Mapping):
327
+ return value
328
+ copied = {
329
+ key: _rewire_retell_tools(item, tool_base_url) for key, item in value.items()
330
+ }
331
+ if str(copied.get("type") or "") == "custom" and copied.get("url"):
332
+ copied["url"] = _join(tool_base_url, copied["url"])
333
+ return copied
334
+
335
+
336
+ def _validate_retell_custom_tools(value: Any, environment_tools: set[str]) -> None:
337
+ """Validate every custom tool wherever Retell nests it in an engine graph."""
338
+ if isinstance(value, list):
339
+ for item in value:
340
+ _validate_retell_custom_tools(item, environment_tools)
341
+ return
342
+ if not isinstance(value, Mapping):
343
+ return
344
+ if str(value.get("type") or "") == "custom":
345
+ _require_environment_tool(
346
+ provider="retell",
347
+ name=str(value.get("name") or "").strip() or None,
348
+ available=environment_tools,
349
+ )
350
+ for item in value.values():
351
+ _validate_retell_custom_tools(item, environment_tools)
352
+
353
+
354
+ _RETELL_AGENT_READ_ONLY_FIELDS = {
355
+ "agent_id",
356
+ "last_modification_timestamp",
357
+ "version",
358
+ "base_version",
359
+ "is_published",
360
+ }
361
+
362
+ _RETELL_ENGINE_READ_ONLY_FIELDS = {
363
+ "llm_id",
364
+ "conversation_flow_id",
365
+ "last_modification_timestamp",
366
+ "version",
367
+ "is_published",
368
+ }
369
+
370
+
371
+ @dataclass(frozen=True)
372
+ class _CloneResult:
373
+ target_id: str
374
+ target_kind: str
375
+ resources: tuple[ProviderResource, ...]
376
+ metadata: dict[str, Any]
377
+
378
+
379
+ def _clone_vapi(
380
+ spec: ProviderImportSpec,
381
+ context: ProviderContext,
382
+ api_key: str,
383
+ request: JsonRequest,
384
+ ) -> _CloneResult:
385
+ base = (spec.api_base_url or "https://api.vapi.ai").rstrip("/")
386
+ original = request(
387
+ "GET", f"{base}/assistant/{spec.source_target_id}", api_key, None
388
+ )
389
+ assistant = _without(
390
+ original,
391
+ {
392
+ "id",
393
+ "orgId",
394
+ "createdAt",
395
+ "updatedAt",
396
+ "latestVersion",
397
+ "isServerUrlSecretSet",
398
+ },
399
+ )
400
+ assistant["name"] = context.provider_resource_prefix[:40]
401
+ assistant["metadata"] = {
402
+ **(
403
+ assistant.get("metadata")
404
+ if isinstance(assistant.get("metadata"), dict)
405
+ else {}
406
+ ),
407
+ "alk_attempt_id": context.attempt_id,
408
+ "alk_world_id": context.world_id,
409
+ }
410
+ assistant["server"] = {"url": context.event_url}
411
+ resources: list[ProviderResource] = []
412
+ environment_tools = set(spec.environment_tools)
413
+ try:
414
+ model = assistant.get("model")
415
+ if isinstance(model, dict):
416
+ model = dict(model)
417
+ inline = model.get("tools")
418
+ if isinstance(inline, list):
419
+ rewritten_inline: list[Any] = []
420
+ for item in inline:
421
+ if isinstance(item, Mapping) and str(item.get("type") or "") in {
422
+ "function",
423
+ "apiRequest",
424
+ }:
425
+ _require_environment_tool(
426
+ provider="vapi",
427
+ name=_vapi_custom_tool_name(item),
428
+ available=environment_tools,
429
+ )
430
+ item = _rewire_vapi_tool(item, context.tool_base_url)
431
+ rewritten_inline.append(item)
432
+ model["tools"] = rewritten_inline
433
+ tool_ids = model.get("toolIds")
434
+ if isinstance(tool_ids, list):
435
+ cloned_ids: list[str] = []
436
+ for source_id in tool_ids:
437
+ source = request("GET", f"{base}/tool/{source_id}", api_key, None)
438
+ if str(source.get("type") or "") in {"function", "apiRequest"}:
439
+ _require_environment_tool(
440
+ provider="vapi",
441
+ name=_vapi_custom_tool_name(source),
442
+ available=environment_tools,
443
+ )
444
+ create = _rewire_vapi_tool(
445
+ _without(source, {"id", "orgId", "createdAt", "updatedAt"}),
446
+ context.tool_base_url,
447
+ )
448
+ created = request("POST", f"{base}/tool", api_key, create)
449
+ target_id = str(created.get("id") or "").strip()
450
+ if not target_id:
451
+ raise ProviderImportError("vapi_tool_create_missing_id")
452
+ cloned_ids.append(target_id)
453
+ resources.append(
454
+ ProviderResource(kind="tool", id=target_id, owned=True)
455
+ )
456
+ model["toolIds"] = cloned_ids
457
+ assistant["model"] = model
458
+ created = request("POST", f"{base}/assistant", api_key, assistant)
459
+ except ProviderImportError:
460
+ for resource in reversed(resources):
461
+ try:
462
+ request("DELETE", f"{base}/tool/{resource.id}", api_key, None)
463
+ except ProviderImportError:
464
+ pass
465
+ raise
466
+ target_id = str(created.get("id") or "").strip()
467
+ if not target_id:
468
+ for resource in reversed(resources):
469
+ try:
470
+ request("DELETE", f"{base}/tool/{resource.id}", api_key, None)
471
+ except ProviderImportError:
472
+ pass
473
+ raise ProviderImportError("vapi_assistant_create_missing_id")
474
+ resources.append(ProviderResource(kind="assistant", id=target_id, owned=True))
475
+ return _CloneResult(
476
+ target_id=target_id,
477
+ target_kind="assistant",
478
+ resources=tuple(resources),
479
+ metadata={
480
+ "source_target_id": spec.source_target_id,
481
+ "clone_kind": "provider_import",
482
+ },
483
+ )
484
+
485
+
486
+ def _clone_retell(
487
+ spec: ProviderImportSpec,
488
+ context: ProviderContext,
489
+ api_key: str,
490
+ request: JsonRequest,
491
+ ) -> _CloneResult:
492
+ base = (spec.api_base_url or "https://api.retellai.com").rstrip("/")
493
+ agent_path = "get-chat-agent" if spec.target_modality == "chat" else "get-agent"
494
+ original = request(
495
+ "GET", f"{base}/{agent_path}/{spec.source_target_id}", api_key, None
496
+ )
497
+ engine = original.get("response_engine")
498
+ if not isinstance(engine, Mapping):
499
+ raise ProviderImportError(
500
+ "retell_response_engine_unsupported: response_engine must be an object"
501
+ )
502
+ environment_tools = set(spec.environment_tools)
503
+ engine_type = str(engine.get("type") or "").strip()
504
+ engine_resource: ProviderResource
505
+ cloned_engine: dict[str, Any]
506
+ source_engine_id: str
507
+ delete_engine_url: str
508
+
509
+ if engine_type == "retell-llm":
510
+ source_engine_id = str(engine.get("llm_id") or "").strip()
511
+ if not source_engine_id:
512
+ raise ProviderImportError("retell_response_engine_missing_llm_id")
513
+ source_engine = request(
514
+ "GET", f"{base}/get-retell-llm/{source_engine_id}", api_key, None
515
+ )
516
+ _validate_retell_custom_tools(source_engine, environment_tools)
517
+ engine_create = _rewire_retell_tools(
518
+ _without(source_engine, _RETELL_ENGINE_READ_ONLY_FIELDS),
519
+ context.tool_base_url,
520
+ )
521
+ created_engine = request(
522
+ "POST", f"{base}/create-retell-llm", api_key, engine_create
523
+ )
524
+ cloned_engine_id = str(created_engine.get("llm_id") or "").strip()
525
+ if not cloned_engine_id:
526
+ raise ProviderImportError("retell_llm_create_missing_id")
527
+ engine_resource = ProviderResource(
528
+ kind="retell_llm", id=cloned_engine_id, owned=True
529
+ )
530
+ cloned_engine = {"type": "retell-llm", "llm_id": cloned_engine_id}
531
+ delete_engine_url = f"{base}/delete-retell-llm/{cloned_engine_id}"
532
+ elif engine_type == "conversation-flow":
533
+ source_engine_id = str(engine.get("conversation_flow_id") or "").strip()
534
+ if not source_engine_id:
535
+ raise ProviderImportError(
536
+ "retell_response_engine_missing_conversation_flow_id"
537
+ )
538
+ source_engine = request(
539
+ "GET",
540
+ f"{base}/get-conversation-flow/{source_engine_id}",
541
+ api_key,
542
+ None,
543
+ )
544
+ _validate_retell_custom_tools(source_engine, environment_tools)
545
+ engine_create = _rewire_retell_tools(
546
+ _without(source_engine, _RETELL_ENGINE_READ_ONLY_FIELDS),
547
+ context.tool_base_url,
548
+ )
549
+ created_engine = request(
550
+ "POST", f"{base}/create-conversation-flow", api_key, engine_create
551
+ )
552
+ cloned_engine_id = str(created_engine.get("conversation_flow_id") or "").strip()
553
+ if not cloned_engine_id:
554
+ raise ProviderImportError("retell_conversation_flow_create_missing_id")
555
+ engine_resource = ProviderResource(
556
+ kind="conversation_flow", id=cloned_engine_id, owned=True
557
+ )
558
+ cloned_engine = {
559
+ "type": "conversation-flow",
560
+ "conversation_flow_id": cloned_engine_id,
561
+ }
562
+ delete_engine_url = f"{base}/delete-conversation-flow/{cloned_engine_id}"
563
+ else:
564
+ raise ProviderImportError(
565
+ "retell_response_engine_unsupported: provider_import supports "
566
+ "retell-llm and conversation-flow"
567
+ )
568
+
569
+ agent_create = _without(
570
+ original,
571
+ _RETELL_AGENT_READ_ONLY_FIELDS,
572
+ )
573
+ agent_create["agent_name"] = context.provider_resource_prefix
574
+ agent_create["response_engine"] = cloned_engine
575
+ agent_create["webhook_url"] = context.event_url
576
+ create_path = (
577
+ "create-chat-agent" if spec.target_modality == "chat" else "create-agent"
578
+ )
579
+ try:
580
+ created_agent = request("POST", f"{base}/{create_path}", api_key, agent_create)
581
+ except ProviderImportError:
582
+ try:
583
+ request("DELETE", delete_engine_url, api_key, None)
584
+ except ProviderImportError:
585
+ pass
586
+ raise
587
+ target_id = str(created_agent.get("agent_id") or "").strip()
588
+ if not target_id:
589
+ try:
590
+ request("DELETE", delete_engine_url, api_key, None)
591
+ except ProviderImportError:
592
+ pass
593
+ raise ProviderImportError("retell_agent_create_missing_id")
594
+ return _CloneResult(
595
+ target_id=target_id,
596
+ target_kind="chat_agent" if spec.target_modality == "chat" else "voice_agent",
597
+ resources=(
598
+ engine_resource,
599
+ ProviderResource(
600
+ kind="chat_agent" if spec.target_modality == "chat" else "voice_agent",
601
+ id=target_id,
602
+ owned=True,
603
+ ),
604
+ ),
605
+ metadata={
606
+ "source_target_id": spec.source_target_id,
607
+ "source_response_engine_id": source_engine_id,
608
+ "source_response_engine_type": engine_type,
609
+ "target_modality": spec.target_modality,
610
+ "clone_kind": "provider_import",
611
+ },
612
+ )
613
+
614
+
615
+ def clone_provider_target(
616
+ spec: ProviderImportSpec,
617
+ *,
618
+ context: ProviderContext,
619
+ api_key: str,
620
+ request: JsonRequest = _request_json,
621
+ ) -> ProviderProvisionReceipt:
622
+ if not api_key:
623
+ raise ProviderImportError(f"{spec.type.value}_api_key_missing")
624
+ result = (
625
+ _clone_vapi(spec, context, api_key, request)
626
+ if spec.type is ProviderType.VAPI
627
+ else _clone_retell(spec, context, api_key, request)
628
+ )
629
+ return ProviderProvisionReceipt(
630
+ schema_version="1",
631
+ provider=spec.type,
632
+ attempt_id=context.attempt_id,
633
+ world_id=context.world_id,
634
+ target=ProviderTarget(kind=result.target_kind, id=result.target_id),
635
+ resources=list(result.resources),
636
+ cleanup=ProviderCleanupReceipt(idempotency_key=context.idempotency_key),
637
+ metadata=result.metadata,
638
+ )
639
+
640
+
641
+ def destroy_imported_target(
642
+ spec: ProviderImportSpec,
643
+ *,
644
+ receipt: ProviderProvisionReceipt,
645
+ api_key: str,
646
+ request: JsonRequest = _request_json,
647
+ ) -> None:
648
+ base = (
649
+ spec.api_base_url
650
+ or (
651
+ "https://api.vapi.ai"
652
+ if spec.type is ProviderType.VAPI
653
+ else "https://api.retellai.com"
654
+ )
655
+ ).rstrip("/")
656
+ # Reverse dependency order: target first, then tools/response engine.
657
+ for resource in reversed(receipt.resources):
658
+ if not resource.owned:
659
+ continue
660
+ if spec.type is ProviderType.VAPI:
661
+ path = "assistant" if resource.kind == "assistant" else "tool"
662
+ try:
663
+ request("DELETE", f"{base}/{path}/{resource.id}", api_key, None)
664
+ except ProviderImportError as exc:
665
+ if "HTTP 404" not in str(exc):
666
+ raise
667
+ elif resource.kind in {"voice_agent", "chat_agent"}:
668
+ delete_path = (
669
+ "delete-chat-agent" if resource.kind == "chat_agent" else "delete-agent"
670
+ )
671
+ try:
672
+ request("DELETE", f"{base}/{delete_path}/{resource.id}", api_key, None)
673
+ except ProviderImportError as exc:
674
+ if "HTTP 404" not in str(exc):
675
+ raise
676
+ elif resource.kind == "retell_llm":
677
+ try:
678
+ request(
679
+ "DELETE", f"{base}/delete-retell-llm/{resource.id}", api_key, None
680
+ )
681
+ except ProviderImportError as exc:
682
+ if "HTTP 404" not in str(exc):
683
+ raise
684
+ elif resource.kind == "conversation_flow":
685
+ try:
686
+ request(
687
+ "DELETE",
688
+ f"{base}/delete-conversation-flow/{resource.id}",
689
+ api_key,
690
+ None,
691
+ )
692
+ except ProviderImportError as exc:
693
+ if "HTTP 404" not in str(exc):
694
+ raise
695
+
696
+
697
+ __all__ = [
698
+ "ProviderImportError",
699
+ "ProviderImportSpec",
700
+ "clone_provider_target",
701
+ "destroy_imported_target",
702
+ "inspect_provider_target",
703
+ ]