agent-learning-kit 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (642) hide show
  1. agent_learning_kit-0.1.0.dist-info/METADATA +381 -0
  2. agent_learning_kit-0.1.0.dist-info/RECORD +642 -0
  3. agent_learning_kit-0.1.0.dist-info/WHEEL +4 -0
  4. agent_learning_kit-0.1.0.dist-info/entry_points.txt +5 -0
  5. agent_learning_kit-0.1.0.dist-info/licenses/LICENSE +173 -0
  6. agent_learning_kit-0.1.0.dist-info/licenses/NOTICE +7 -0
  7. fi/__init__.py +5 -0
  8. fi/alk/__init__.py +57 -0
  9. fi/alk/_facade.py +31 -0
  10. fi/alk/_module_alias.py +68 -0
  11. fi/alk/_paths.py +14 -0
  12. fi/alk/_schema.py +522 -0
  13. fi/alk/actions.py +727 -0
  14. fi/alk/bench/__init__.py +517 -0
  15. fi/alk/bench/_codeexec.py +213 -0
  16. fi/alk/bench/_coding.py +215 -0
  17. fi/alk/bench/_docker.py +237 -0
  18. fi/alk/bench/_grader.py +286 -0
  19. fi/alk/bench/_pull.py +212 -0
  20. fi/alk/bench/_voice.py +147 -0
  21. fi/alk/capabilities.py +627 -0
  22. fi/alk/cli.py +6396 -0
  23. fi/alk/config.py +130 -0
  24. fi/alk/cua_loop.py +562 -0
  25. fi/alk/evals.py +2351 -0
  26. fi/alk/extensions.py +163 -0
  27. fi/alk/harness/ARCHITECTURE.md +231 -0
  28. fi/alk/harness/DESIGN.md +246 -0
  29. fi/alk/harness/ENVIRONMENT_CONFORMANCE.md +127 -0
  30. fi/alk/harness/HOW-IT-WORKS.md +297 -0
  31. fi/alk/harness/IMPLEMENTATION_AND_VALIDATION_STATUS.md +229 -0
  32. fi/alk/harness/README.md +417 -0
  33. fi/alk/harness/__init__.py +77 -0
  34. fi/alk/harness/__main__.py +3 -0
  35. fi/alk/harness/amend.py +312 -0
  36. fi/alk/harness/artifacts.py +319 -0
  37. fi/alk/harness/authoring_entrypoint.py +189 -0
  38. fi/alk/harness/authoring_runtime_validation.py +267 -0
  39. fi/alk/harness/backends/README.md +43 -0
  40. fi/alk/harness/backends/__init__.py +122 -0
  41. fi/alk/harness/backends/base.py +241 -0
  42. fi/alk/harness/backends/claude.py +211 -0
  43. fi/alk/harness/backends/files.py +182 -0
  44. fi/alk/harness/backends/vertex_gemini.py +457 -0
  45. fi/alk/harness/background_noise.py +95 -0
  46. fi/alk/harness/build.py +385 -0
  47. fi/alk/harness/bundle.py +593 -0
  48. fi/alk/harness/bundle_author_v2.py +1831 -0
  49. fi/alk/harness/bundle_v2.py +719 -0
  50. fi/alk/harness/call_runner.py +1440 -0
  51. fi/alk/harness/callback_http_adapter.py +111 -0
  52. fi/alk/harness/catalogue.py +287 -0
  53. fi/alk/harness/chat.py +428 -0
  54. fi/alk/harness/chat_call_runner.py +506 -0
  55. fi/alk/harness/checks.py +136 -0
  56. fi/alk/harness/cli.py +1354 -0
  57. fi/alk/harness/config.py +338 -0
  58. fi/alk/harness/contract.py +718 -0
  59. fi/alk/harness/credentials.py +674 -0
  60. fi/alk/harness/data/persona_vocabulary.json +111 -0
  61. fi/alk/harness/environment.py +99 -0
  62. fi/alk/harness/environment_plan.py +168 -0
  63. fi/alk/harness/events.py +125 -0
  64. fi/alk/harness/executor.py +304 -0
  65. fi/alk/harness/folder.py +234 -0
  66. fi/alk/harness/generated_runtime.py +815 -0
  67. fi/alk/harness/github.py +72 -0
  68. fi/alk/harness/hosted_authoring_entrypoint.py +183 -0
  69. fi/alk/harness/hosted_entrypoint.py +2402 -0
  70. fi/alk/harness/hosted_scheduler.py +2218 -0
  71. fi/alk/harness/job.py +426 -0
  72. fi/alk/harness/judge.py +184 -0
  73. fi/alk/harness/livekit_source.py +50 -0
  74. fi/alk/harness/livekit_tool_trace_bootstrap.py +71 -0
  75. fi/alk/harness/observability.py +208 -0
  76. fi/alk/harness/outbound.py +3252 -0
  77. fi/alk/harness/packaging.py +515 -0
  78. fi/alk/harness/persona_guides.py +157 -0
  79. fi/alk/harness/platform.py +692 -0
  80. fi/alk/harness/process_preflight.py +764 -0
  81. fi/alk/harness/process_runtime.py +5670 -0
  82. fi/alk/harness/prove.py +425 -0
  83. fi/alk/harness/provider_import.py +703 -0
  84. fi/alk/harness/provider_lifecycle.py +392 -0
  85. fi/alk/harness/provision.py +2896 -0
  86. fi/alk/harness/reception.py +147 -0
  87. fi/alk/harness/retell_chat_call_runner.py +373 -0
  88. fi/alk/harness/run/__init__.py +296 -0
  89. fi/alk/harness/run/alk.py +184 -0
  90. fi/alk/harness/run/call.py +162 -0
  91. fi/alk/harness/run/conversation.py +264 -0
  92. fi/alk/harness/run/data/voices_by_language_and_gender.json +693 -0
  93. fi/alk/harness/run/evidence.py +195 -0
  94. fi/alk/harness/run/grade.py +598 -0
  95. fi/alk/harness/run/live.py +297 -0
  96. fi/alk/harness/run/models.py +56 -0
  97. fi/alk/harness/run/platform_evals.py +227 -0
  98. fi/alk/harness/run/sdk_voice.py +130 -0
  99. fi/alk/harness/run/simulation.py +1209 -0
  100. fi/alk/harness/run/stage.py +91 -0
  101. fi/alk/harness/run/targets.py +508 -0
  102. fi/alk/harness/run/tools.py +601 -0
  103. fi/alk/harness/run/voice.py +340 -0
  104. fi/alk/harness/runtime.py +172 -0
  105. fi/alk/harness/sandbox_server.py +2011 -0
  106. fi/alk/harness/sandbox_worker.py +44 -0
  107. fi/alk/harness/scenario.py +1048 -0
  108. fi/alk/harness/scenario_source.py +879 -0
  109. fi/alk/harness/scenario_tools.py +1143 -0
  110. fi/alk/harness/scenarios.py +915 -0
  111. fi/alk/harness/secrets.py +168 -0
  112. fi/alk/harness/service_catalog.py +97 -0
  113. fi/alk/harness/session.py +391 -0
  114. fi/alk/harness/sessions.py +372 -0
  115. fi/alk/harness/simulator.py +76 -0
  116. fi/alk/harness/simulator_voice.py +928 -0
  117. fi/alk/harness/skills/build-environment/SKILL.md +538 -0
  118. fi/alk/harness/skills/harness.md +131 -0
  119. fi/alk/harness/skills/kinds/chat.md +48 -0
  120. fi/alk/harness/skills/kinds/voice-voicemail.md +63 -0
  121. fi/alk/harness/skills/kinds/voice.md +59 -0
  122. fi/alk/harness/skills/plan-suite/SKILL.md +103 -0
  123. fi/alk/harness/skills/provision-environment/SKILL.md +136 -0
  124. fi/alk/harness/skills/run-scenarios/SKILL.md +112 -0
  125. fi/alk/harness/skills/understand-agent/SKILL.md +251 -0
  126. fi/alk/harness/skills/write-scenarios/SKILL.md +606 -0
  127. fi/alk/harness/skills/write-scenarios/references/refusals.md +28 -0
  128. fi/alk/harness/skills/write-scenarios/references/world-api.md +92 -0
  129. fi/alk/harness/source_data_invariants.py +444 -0
  130. fi/alk/harness/source_tool_evidence.py +79 -0
  131. fi/alk/harness/sources.py +253 -0
  132. fi/alk/harness/spend.py +140 -0
  133. fi/alk/harness/tool_trace_proxy.py +104 -0
  134. fi/alk/harness/tools.py +1018 -0
  135. fi/alk/harness/understand.py +169 -0
  136. fi/alk/harness/voicemail_audio.py +74 -0
  137. fi/alk/harness/world/__init__.py +33 -0
  138. fi/alk/harness/world/errors.py +68 -0
  139. fi/alk/harness/world/expectations.py +91 -0
  140. fi/alk/harness/world/handle.py +538 -0
  141. fi/alk/harness/world/kinds.py +196 -0
  142. fi/alk/harness/world/mutate.py +186 -0
  143. fi/alk/harness/world/probe.py +413 -0
  144. fi/alk/harness/world/provision.py +511 -0
  145. fi/alk/harness/world/provisioned.py +191 -0
  146. fi/alk/harness/world/runtime.py +616 -0
  147. fi/alk/harness/world/snapshot.py +288 -0
  148. fi/alk/harness/world/stores/__init__.py +305 -0
  149. fi/alk/harness/world/stores/container.py +215 -0
  150. fi/alk/harness/world/stores/inprocess.py +346 -0
  151. fi/alk/harness/world/stores/postgres.py +481 -0
  152. fi/alk/harness/world/stores/prove.py +202 -0
  153. fi/alk/harness/world/stores/sqlite.py +245 -0
  154. fi/alk/harness/world/stores/written.py +182 -0
  155. fi/alk/harness/world/tools.py +1516 -0
  156. fi/alk/harness/world/workspace.py +144 -0
  157. fi/alk/image_loop.py +453 -0
  158. fi/alk/image_perturb.py +241 -0
  159. fi/alk/improve.py +274 -0
  160. fi/alk/live/__init__.py +154 -0
  161. fi/alk/live/_attribution.py +184 -0
  162. fi/alk/live/_capture.py +264 -0
  163. fi/alk/live/_codec.py +391 -0
  164. fi/alk/live/_contract.py +134 -0
  165. fi/alk/live/_loopback.py +316 -0
  166. fi/alk/live/_perturb.py +449 -0
  167. fi/alk/live/_runner.py +386 -0
  168. fi/alk/live/_stats.py +561 -0
  169. fi/alk/live/_transcript.py +240 -0
  170. fi/alk/live/_workers/__init__.py +9 -0
  171. fi/alk/live/_workers/a2a_worker.py +316 -0
  172. fi/alk/live/_workers/langgraph_worker.py +217 -0
  173. fi/alk/live/_workers/livekit_worker.py +207 -0
  174. fi/alk/live/_workers/mcp_loopback_server.py +46 -0
  175. fi/alk/live/_workers/mcp_worker.py +158 -0
  176. fi/alk/live/_workers/pipecat_worker.py +189 -0
  177. fi/alk/live/a2a_lane.py +138 -0
  178. fi/alk/live/langgraph_lane.py +339 -0
  179. fi/alk/live/livekit_lane.py +376 -0
  180. fi/alk/live/mcp_lane.py +172 -0
  181. fi/alk/live/pipecat_lane.py +341 -0
  182. fi/alk/live/voice_redteam.py +494 -0
  183. fi/alk/loss.py +306 -0
  184. fi/alk/optimize.py +36260 -0
  185. fi/alk/practice/__init__.py +51 -0
  186. fi/alk/practice/_assess.py +103 -0
  187. fi/alk/practice/_budget.py +81 -0
  188. fi/alk/practice/_calibrate.py +69 -0
  189. fi/alk/practice/_capstone.py +86 -0
  190. fi/alk/practice/_contract.py +91 -0
  191. fi/alk/practice/_diagnose.py +79 -0
  192. fi/alk/practice/_drill.py +196 -0
  193. fi/alk/practice/_experiment.py +720 -0
  194. fi/alk/practice/_schedule.py +102 -0
  195. fi/alk/practice/_store.py +194 -0
  196. fi/alk/practice/_trainer.py +245 -0
  197. fi/alk/practice/_update.py +125 -0
  198. fi/alk/redteam.py +2621 -0
  199. fi/alk/rewardhack.py +237 -0
  200. fi/alk/simulate.py +10351 -0
  201. fi/alk/studio/__init__.py +82 -0
  202. fi/alk/studio/_bias.py +314 -0
  203. fi/alk/studio/_calibration.py +522 -0
  204. fi/alk/studio/_coverage.py +262 -0
  205. fi/alk/studio/_download.py +665 -0
  206. fi/alk/studio/_fidelity_attack.py +114 -0
  207. fi/alk/studio/_generate.py +652 -0
  208. fi/alk/studio/_library.py +370 -0
  209. fi/alk/studio/_scan.py +134 -0
  210. fi/alk/studio/_upgrade.py +42 -0
  211. fi/alk/studio/_vendor.py +172 -0
  212. fi/alk/suite.py +4200 -0
  213. fi/alk/tasks.py +828 -0
  214. fi/alk/telemetry/__init__.py +149 -0
  215. fi/alk/telemetry/_contract.py +141 -0
  216. fi/alk/telemetry/_emit.py +182 -0
  217. fi/alk/telemetry/_ledger.py +296 -0
  218. fi/alk/telemetry/_queue.py +127 -0
  219. fi/alk/telemetry/_row.py +294 -0
  220. fi/alk/telemetry/_run.py +233 -0
  221. fi/alk/telemetry/_sync.py +193 -0
  222. fi/alk/telemetry/_url.py +119 -0
  223. fi/alk/trinity.py +49397 -0
  224. fi/alk/voice_loop.py +174 -0
  225. fi/api/__init__.py +1 -0
  226. fi/api/auth.py +137 -0
  227. fi/api/types.py +29 -0
  228. fi/cli/__init__.py +9 -0
  229. fi/cli/assertions/__init__.py +25 -0
  230. fi/cli/assertions/conditions.py +76 -0
  231. fi/cli/assertions/evaluator.py +286 -0
  232. fi/cli/assertions/exit_codes.py +20 -0
  233. fi/cli/assertions/parser.py +131 -0
  234. fi/cli/assertions/reporter.py +194 -0
  235. fi/cli/commands/__init__.py +9 -0
  236. fi/cli/commands/config.py +165 -0
  237. fi/cli/commands/export.py +208 -0
  238. fi/cli/commands/init.py +112 -0
  239. fi/cli/commands/list_cmd.py +213 -0
  240. fi/cli/commands/run.py +486 -0
  241. fi/cli/commands/validate.py +173 -0
  242. fi/cli/commands/view.py +424 -0
  243. fi/cli/config/__init__.py +6 -0
  244. fi/cli/config/defaults.py +206 -0
  245. fi/cli/config/loader.py +155 -0
  246. fi/cli/config/schema.py +174 -0
  247. fi/cli/main.py +78 -0
  248. fi/cli/output/__init__.py +6 -0
  249. fi/cli/output/formatters.py +106 -0
  250. fi/cli/output/reporters.py +46 -0
  251. fi/cli/storage/__init__.py +5 -0
  252. fi/cli/storage/run_history.py +249 -0
  253. fi/cli/utils/__init__.py +5 -0
  254. fi/cli/utils/console.py +44 -0
  255. fi/evals/__init__.py +131 -0
  256. fi/evals/autoeval/__init__.py +137 -0
  257. fi/evals/autoeval/analyzer.py +211 -0
  258. fi/evals/autoeval/config.py +244 -0
  259. fi/evals/autoeval/export.py +213 -0
  260. fi/evals/autoeval/interactive.py +283 -0
  261. fi/evals/autoeval/pipeline.py +625 -0
  262. fi/evals/autoeval/prompts.py +139 -0
  263. fi/evals/autoeval/recommender.py +242 -0
  264. fi/evals/autoeval/rules.py +589 -0
  265. fi/evals/autoeval/templates.py +299 -0
  266. fi/evals/autoeval/types.py +232 -0
  267. fi/evals/core/__init__.py +16 -0
  268. fi/evals/core/cloud_registry.py +184 -0
  269. fi/evals/core/engines.py +368 -0
  270. fi/evals/core/evaluate.py +319 -0
  271. fi/evals/core/judge_prompt.py +90 -0
  272. fi/evals/core/prompt_generator.py +83 -0
  273. fi/evals/core/registry.py +57 -0
  274. fi/evals/core/result.py +55 -0
  275. fi/evals/evaluator.py +721 -0
  276. fi/evals/execution.py +168 -0
  277. fi/evals/feedback/__init__.py +32 -0
  278. fi/evals/feedback/calibrator.py +160 -0
  279. fi/evals/feedback/collector.py +214 -0
  280. fi/evals/feedback/hooks.py +81 -0
  281. fi/evals/feedback/retriever.py +128 -0
  282. fi/evals/feedback/store.py +272 -0
  283. fi/evals/feedback/types.py +99 -0
  284. fi/evals/framework/README.md +79 -0
  285. fi/evals/framework/__init__.py +267 -0
  286. fi/evals/framework/backends/Dockerfile.eval-runner +33 -0
  287. fi/evals/framework/backends/__init__.py +99 -0
  288. fi/evals/framework/backends/_container.py +141 -0
  289. fi/evals/framework/backends/_utils.py +145 -0
  290. fi/evals/framework/backends/base.py +223 -0
  291. fi/evals/framework/backends/celery_backend.py +417 -0
  292. fi/evals/framework/backends/celery_worker.py +78 -0
  293. fi/evals/framework/backends/kubernetes_backend.py +665 -0
  294. fi/evals/framework/backends/ray_backend.py +521 -0
  295. fi/evals/framework/backends/temporal.py +350 -0
  296. fi/evals/framework/backends/temporal_worker.py +126 -0
  297. fi/evals/framework/backends/thread_pool.py +286 -0
  298. fi/evals/framework/context.py +258 -0
  299. fi/evals/framework/enrichment.py +306 -0
  300. fi/evals/framework/evals/__init__.py +68 -0
  301. fi/evals/framework/evals/agentic.py +399 -0
  302. fi/evals/framework/evals/builder.py +609 -0
  303. fi/evals/framework/evals/semantic.py +142 -0
  304. fi/evals/framework/evaluator.py +647 -0
  305. fi/evals/framework/evaluators/__init__.py +22 -0
  306. fi/evals/framework/evaluators/blocking.py +347 -0
  307. fi/evals/framework/evaluators/non_blocking.py +577 -0
  308. fi/evals/framework/propagation.py +421 -0
  309. fi/evals/framework/protocols.py +385 -0
  310. fi/evals/framework/registry.py +370 -0
  311. fi/evals/framework/resilience/__init__.py +150 -0
  312. fi/evals/framework/resilience/circuit_breaker.py +309 -0
  313. fi/evals/framework/resilience/degradation.py +355 -0
  314. fi/evals/framework/resilience/health.py +505 -0
  315. fi/evals/framework/resilience/rate_limiter.py +228 -0
  316. fi/evals/framework/resilience/retry.py +274 -0
  317. fi/evals/framework/resilience/types.py +288 -0
  318. fi/evals/framework/resilience/wrapper.py +433 -0
  319. fi/evals/framework/types.py +218 -0
  320. fi/evals/guardrails/README.md +915 -0
  321. fi/evals/guardrails/__init__.py +96 -0
  322. fi/evals/guardrails/backends/__init__.py +43 -0
  323. fi/evals/guardrails/backends/azure.py +361 -0
  324. fi/evals/guardrails/backends/base.py +88 -0
  325. fi/evals/guardrails/backends/generic_llm.py +163 -0
  326. fi/evals/guardrails/backends/granite.py +216 -0
  327. fi/evals/guardrails/backends/llamaguard.py +221 -0
  328. fi/evals/guardrails/backends/local_base.py +479 -0
  329. fi/evals/guardrails/backends/openai.py +365 -0
  330. fi/evals/guardrails/backends/qwen.py +170 -0
  331. fi/evals/guardrails/backends/shieldgemma.py +154 -0
  332. fi/evals/guardrails/backends/turing.py +235 -0
  333. fi/evals/guardrails/backends/vllm_client.py +321 -0
  334. fi/evals/guardrails/backends/wildguard.py +188 -0
  335. fi/evals/guardrails/base.py +888 -0
  336. fi/evals/guardrails/config.py +221 -0
  337. fi/evals/guardrails/discovery.py +243 -0
  338. fi/evals/guardrails/gateway.py +437 -0
  339. fi/evals/guardrails/registry.py +231 -0
  340. fi/evals/guardrails/scanners/__init__.py +127 -0
  341. fi/evals/guardrails/scanners/base.py +191 -0
  342. fi/evals/guardrails/scanners/code_injection.py +243 -0
  343. fi/evals/guardrails/scanners/eval_delegate.py +574 -0
  344. fi/evals/guardrails/scanners/invisible_chars.py +351 -0
  345. fi/evals/guardrails/scanners/jailbreak.py +412 -0
  346. fi/evals/guardrails/scanners/language.py +288 -0
  347. fi/evals/guardrails/scanners/pipeline.py +260 -0
  348. fi/evals/guardrails/scanners/regex.py +311 -0
  349. fi/evals/guardrails/scanners/secrets.py +274 -0
  350. fi/evals/guardrails/scanners/topics.py +649 -0
  351. fi/evals/guardrails/scanners/urls.py +341 -0
  352. fi/evals/guardrails/types.py +96 -0
  353. fi/evals/llm/__init__.py +3 -0
  354. fi/evals/llm/base_llm_provider.py +35 -0
  355. fi/evals/llm/providers/litellm.py +70 -0
  356. fi/evals/local/__init__.py +90 -0
  357. fi/evals/local/evaluator.py +690 -0
  358. fi/evals/local/execution_mode.py +121 -0
  359. fi/evals/local/llm.py +489 -0
  360. fi/evals/local/metrics/__init__.py +19 -0
  361. fi/evals/local/registry.py +360 -0
  362. fi/evals/manager.py +1018 -0
  363. fi/evals/manager_types.py +362 -0
  364. fi/evals/metrics/__init__.py +185 -0
  365. fi/evals/metrics/agents/__init__.py +74 -0
  366. fi/evals/metrics/agents/metrics.py +693 -0
  367. fi/evals/metrics/agents/report.py +36463 -0
  368. fi/evals/metrics/agents/types.py +160 -0
  369. fi/evals/metrics/base_llm_metric.py +111 -0
  370. fi/evals/metrics/base_metric.py +138 -0
  371. fi/evals/metrics/code_security/__init__.py +305 -0
  372. fi/evals/metrics/code_security/analyzer.py +985 -0
  373. fi/evals/metrics/code_security/benchmarks/__init__.py +73 -0
  374. fi/evals/metrics/code_security/benchmarks/builtin.py +750 -0
  375. fi/evals/metrics/code_security/benchmarks/loader.py +580 -0
  376. fi/evals/metrics/code_security/benchmarks/types.py +308 -0
  377. fi/evals/metrics/code_security/detectors/__init__.py +186 -0
  378. fi/evals/metrics/code_security/detectors/base.py +394 -0
  379. fi/evals/metrics/code_security/detectors/cryptography.py +345 -0
  380. fi/evals/metrics/code_security/detectors/injection.py +744 -0
  381. fi/evals/metrics/code_security/detectors/secrets.py +287 -0
  382. fi/evals/metrics/code_security/detectors/serialization.py +192 -0
  383. fi/evals/metrics/code_security/joint_metrics.py +588 -0
  384. fi/evals/metrics/code_security/judges/__init__.py +83 -0
  385. fi/evals/metrics/code_security/judges/base.py +238 -0
  386. fi/evals/metrics/code_security/judges/dual_judge.py +534 -0
  387. fi/evals/metrics/code_security/judges/llm_judge.py +301 -0
  388. fi/evals/metrics/code_security/judges/pattern_judge.py +515 -0
  389. fi/evals/metrics/code_security/metrics.py +388 -0
  390. fi/evals/metrics/code_security/modes/__init__.py +63 -0
  391. fi/evals/metrics/code_security/modes/adversarial.py +284 -0
  392. fi/evals/metrics/code_security/modes/autocomplete.py +198 -0
  393. fi/evals/metrics/code_security/modes/base.py +283 -0
  394. fi/evals/metrics/code_security/modes/instruct.py +253 -0
  395. fi/evals/metrics/code_security/modes/repair.py +230 -0
  396. fi/evals/metrics/code_security/reports/__init__.py +57 -0
  397. fi/evals/metrics/code_security/reports/generator.py +404 -0
  398. fi/evals/metrics/code_security/reports/leaderboard.py +509 -0
  399. fi/evals/metrics/code_security/types.py +534 -0
  400. fi/evals/metrics/function_calling/__init__.py +34 -0
  401. fi/evals/metrics/function_calling/metrics.py +573 -0
  402. fi/evals/metrics/function_calling/types.py +87 -0
  403. fi/evals/metrics/hallucination/__init__.py +54 -0
  404. fi/evals/metrics/hallucination/detector.py +149 -0
  405. fi/evals/metrics/hallucination/metrics.py +390 -0
  406. fi/evals/metrics/hallucination/nli.py +253 -0
  407. fi/evals/metrics/hallucination/sentinel.py +106 -0
  408. fi/evals/metrics/hallucination/types.py +132 -0
  409. fi/evals/metrics/heuristics/aggregation_metrics.py +85 -0
  410. fi/evals/metrics/heuristics/json_metrics.py +87 -0
  411. fi/evals/metrics/heuristics/similarity_metrics.py +375 -0
  412. fi/evals/metrics/heuristics/string_metrics.py +391 -0
  413. fi/evals/metrics/llm_as_judges/__init__.py +17 -0
  414. fi/evals/metrics/llm_as_judges/custom_judge/metric.py +112 -0
  415. fi/evals/metrics/llm_as_judges/custom_judge/prompts.py +26 -0
  416. fi/evals/metrics/llm_as_judges/types.py +48 -0
  417. fi/evals/metrics/rag/__init__.py +111 -0
  418. fi/evals/metrics/rag/advanced/__init__.py +14 -0
  419. fi/evals/metrics/rag/advanced/multi_hop.py +283 -0
  420. fi/evals/metrics/rag/advanced/source_attribution.py +344 -0
  421. fi/evals/metrics/rag/generation/__init__.py +17 -0
  422. fi/evals/metrics/rag/generation/answer_relevancy.py +176 -0
  423. fi/evals/metrics/rag/generation/context_utilization.py +245 -0
  424. fi/evals/metrics/rag/generation/faithfulness.py +241 -0
  425. fi/evals/metrics/rag/generation/groundedness.py +131 -0
  426. fi/evals/metrics/rag/rag_score.py +277 -0
  427. fi/evals/metrics/rag/retrieval/__init__.py +20 -0
  428. fi/evals/metrics/rag/retrieval/context_entity_recall.py +124 -0
  429. fi/evals/metrics/rag/retrieval/context_precision.py +158 -0
  430. fi/evals/metrics/rag/retrieval/context_recall.py +106 -0
  431. fi/evals/metrics/rag/retrieval/noise_sensitivity.py +163 -0
  432. fi/evals/metrics/rag/retrieval/ranking.py +261 -0
  433. fi/evals/metrics/rag/types.py +100 -0
  434. fi/evals/metrics/rag/utils/__init__.py +62 -0
  435. fi/evals/metrics/rag/utils/claims.py +189 -0
  436. fi/evals/metrics/rag/utils/entities.py +244 -0
  437. fi/evals/metrics/rag/utils/nli.py +92 -0
  438. fi/evals/metrics/rag/utils/similarity.py +345 -0
  439. fi/evals/metrics/structured/__init__.py +114 -0
  440. fi/evals/metrics/structured/field_completeness.py +313 -0
  441. fi/evals/metrics/structured/hierarchy_score.py +366 -0
  442. fi/evals/metrics/structured/json_validation.py +190 -0
  443. fi/evals/metrics/structured/schema_compliance.py +280 -0
  444. fi/evals/metrics/structured/structured_output_score.py +298 -0
  445. fi/evals/metrics/structured/types.py +108 -0
  446. fi/evals/metrics/structured/validators/__init__.py +30 -0
  447. fi/evals/metrics/structured/validators/base.py +189 -0
  448. fi/evals/metrics/structured/validators/json_validator.py +196 -0
  449. fi/evals/metrics/structured/validators/pydantic_validator.py +178 -0
  450. fi/evals/metrics/structured/validators/yaml_validator.py +248 -0
  451. fi/evals/otel/__init__.py +266 -0
  452. fi/evals/otel/config.py +400 -0
  453. fi/evals/otel/conventions.py +463 -0
  454. fi/evals/otel/enrichment.py +371 -0
  455. fi/evals/otel/instrumentors/__init__.py +140 -0
  456. fi/evals/otel/instrumentors/anthropic.py +517 -0
  457. fi/evals/otel/instrumentors/base.py +382 -0
  458. fi/evals/otel/instrumentors/openai.py +673 -0
  459. fi/evals/otel/processors/__init__.py +36 -0
  460. fi/evals/otel/processors/base.py +473 -0
  461. fi/evals/otel/processors/cost.py +445 -0
  462. fi/evals/otel/processors/evaluation.py +559 -0
  463. fi/evals/otel/processors/llm.py +462 -0
  464. fi/evals/otel/tracer.py +506 -0
  465. fi/evals/otel/types.py +232 -0
  466. fi/evals/otel_utils.py +23 -0
  467. fi/evals/protect.py +671 -0
  468. fi/evals/protect_input_adapter.py +154 -0
  469. fi/evals/streaming/__init__.py +88 -0
  470. fi/evals/streaming/buffer.py +213 -0
  471. fi/evals/streaming/evaluator.py +551 -0
  472. fi/evals/streaming/policy.py +307 -0
  473. fi/evals/streaming/scorers.py +368 -0
  474. fi/evals/streaming/types.py +238 -0
  475. fi/evals/templates.py +472 -0
  476. fi/evals/types.py +156 -0
  477. fi/opt/__init__.py +221 -0
  478. fi/opt/_objective_scoring.py +85 -0
  479. fi/opt/base/__init__.py +11 -0
  480. fi/opt/base/base_generator.py +33 -0
  481. fi/opt/base/base_mapper.py +26 -0
  482. fi/opt/base/base_optimizer.py +45 -0
  483. fi/opt/base/evaluator.py +211 -0
  484. fi/opt/components.py +3095 -0
  485. fi/opt/datamappers/__init__.py +3 -0
  486. fi/opt/datamappers/basic_mapper.py +40 -0
  487. fi/opt/deployment.py +1021 -0
  488. fi/opt/evidence.py +4332 -0
  489. fi/opt/generators/__init__.py +3 -0
  490. fi/opt/generators/litellm.py +66 -0
  491. fi/opt/integrations/__init__.py +23 -0
  492. fi/opt/integrations/generative_suite.py +410 -0
  493. fi/opt/integrations/simulate.py +1313 -0
  494. fi/opt/mutations.py +771 -0
  495. fi/opt/observability.py +4639 -0
  496. fi/opt/optimizer_trace.py +889 -0
  497. fi/opt/optimizers/__init__.py +80 -0
  498. fi/opt/optimizers/agent.py +331 -0
  499. fi/opt/optimizers/agent_bandit.py +392 -0
  500. fi/opt/optimizers/agent_curriculum.py +635 -0
  501. fi/opt/optimizers/agent_evolution.py +894 -0
  502. fi/opt/optimizers/agent_feedback.py +1863 -0
  503. fi/opt/optimizers/agent_pareto.py +547 -0
  504. fi/opt/optimizers/agent_social_memory.py +1113 -0
  505. fi/opt/optimizers/agent_tpe.py +321 -0
  506. fi/opt/optimizers/bayesian_search.py +449 -0
  507. fi/opt/optimizers/council.py +2075 -0
  508. fi/opt/optimizers/futureagi_replay.py +799 -0
  509. fi/opt/optimizers/gepa.py +322 -0
  510. fi/opt/optimizers/metaprompt.py +243 -0
  511. fi/opt/optimizers/promptwizard.py +417 -0
  512. fi/opt/optimizers/protegi.py +329 -0
  513. fi/opt/optimizers/random_search.py +224 -0
  514. fi/opt/research.py +518 -0
  515. fi/opt/simulation.py +260 -0
  516. fi/opt/targets.py +232 -0
  517. fi/opt/types.py +66 -0
  518. fi/opt/utils/__init__.py +4 -0
  519. fi/opt/utils/early_stopping.py +266 -0
  520. fi/opt/utils/setup_logging.py +82 -0
  521. fi/simulate/__init__.py +540 -0
  522. fi/simulate/_hashing.py +35 -0
  523. fi/simulate/_logging.py +10 -0
  524. fi/simulate/adapters.py +87 -0
  525. fi/simulate/agent/__init__.py +120 -0
  526. fi/simulate/agent/browser.py +658 -0
  527. fi/simulate/agent/definition.py +587 -0
  528. fi/simulate/agent/frameworks.py +3528 -0
  529. fi/simulate/agent/generic.py +8286 -0
  530. fi/simulate/agent/import_probe.py +227 -0
  531. fi/simulate/agent/memory.py +905 -0
  532. fi/simulate/agent/mocks.py +101 -0
  533. fi/simulate/agent/multi_agent.py +361 -0
  534. fi/simulate/agent/orchestration.py +903 -0
  535. fi/simulate/agent/realtime.py +665 -0
  536. fi/simulate/agent/wrapper.py +99 -0
  537. fi/simulate/agent/wrappers/__init__.py +18 -0
  538. fi/simulate/agent/wrappers/anthropic.py +62 -0
  539. fi/simulate/agent/wrappers/gemini.py +65 -0
  540. fi/simulate/agent/wrappers/http.py +404 -0
  541. fi/simulate/agent/wrappers/langchain.py +80 -0
  542. fi/simulate/agent/wrappers/openai.py +75 -0
  543. fi/simulate/agent/wrappers/websocket.py +326 -0
  544. fi/simulate/artifacts/__init__.py +11 -0
  545. fi/simulate/artifacts/manifest.py +62 -0
  546. fi/simulate/cli.py +20560 -0
  547. fi/simulate/endpoints/__init__.py +45 -0
  548. fi/simulate/endpoints/_http_actor.py +73 -0
  549. fi/simulate/endpoints/actor_sources.py +243 -0
  550. fi/simulate/endpoints/base.py +107 -0
  551. fi/simulate/endpoints/builtins.py +10 -0
  552. fi/simulate/endpoints/callable.py +95 -0
  553. fi/simulate/endpoints/http.py +76 -0
  554. fi/simulate/endpoints/livekit.py +138 -0
  555. fi/simulate/endpoints/originators.py +132 -0
  556. fi/simulate/endpoints/profiles.py +348 -0
  557. fi/simulate/endpoints/retell.py +633 -0
  558. fi/simulate/endpoints/vapi.py +205 -0
  559. fi/simulate/endpoints/websocket.py +76 -0
  560. fi/simulate/environment.py +33026 -0
  561. fi/simulate/environments/__init__.py +11 -0
  562. fi/simulate/environments/base.py +73 -0
  563. fi/simulate/environments/chat.py +697 -0
  564. fi/simulate/environments/voice.py +212 -0
  565. fi/simulate/evaluation/__init__.py +4 -0
  566. fi/simulate/evaluation/ai_eval.py +227 -0
  567. fi/simulate/evidence/__init__.py +35 -0
  568. fi/simulate/evidence/base.py +59 -0
  569. fi/simulate/evidence/caller_observed.py +50 -0
  570. fi/simulate/evidence/livekit_instrumentation.py +51 -0
  571. fi/simulate/evidence/livekit_room.py +50 -0
  572. fi/simulate/evidence/otel.py +49 -0
  573. fi/simulate/evidence/providers/__init__.py +24 -0
  574. fi/simulate/evidence/providers/base.py +61 -0
  575. fi/simulate/evidence/providers/retell.py +376 -0
  576. fi/simulate/evidence/providers/vapi.py +426 -0
  577. fi/simulate/hosted/__init__.py +32 -0
  578. fi/simulate/hosted/child_entrypoint.py +306 -0
  579. fi/simulate/hosted/job.py +150 -0
  580. fi/simulate/hosted/targets.py +53 -0
  581. fi/simulate/instrumentation/__init__.py +5 -0
  582. fi/simulate/instrumentation/livekit/__init__.py +122 -0
  583. fi/simulate/manifest.py +1033 -0
  584. fi/simulate/matrix_cli.py +165 -0
  585. fi/simulate/realtime/__init__.py +40 -0
  586. fi/simulate/realtime/events.py +107 -0
  587. fi/simulate/realtime/media.py +61 -0
  588. fi/simulate/realtime/session.py +91 -0
  589. fi/simulate/recording/__init__.py +5 -0
  590. fi/simulate/recording/room_recorder.py +326 -0
  591. fi/simulate/registry.py +185 -0
  592. fi/simulate/results/__init__.py +9 -0
  593. fi/simulate/results/base.py +18 -0
  594. fi/simulate/results/filesystem.py +71 -0
  595. fi/simulate/results/futureagi.py +1340 -0
  596. fi/simulate/runtime/__init__.py +85 -0
  597. fi/simulate/runtime/capabilities.py +40 -0
  598. fi/simulate/runtime/events.py +63 -0
  599. fi/simulate/runtime/failures.py +25 -0
  600. fi/simulate/runtime/ids.py +34 -0
  601. fi/simulate/runtime/plan.py +70 -0
  602. fi/simulate/runtime/planner.py +102 -0
  603. fi/simulate/runtime/report.py +174 -0
  604. fi/simulate/runtime/run.py +75 -0
  605. fi/simulate/runtime/runner.py +333 -0
  606. fi/simulate/runtime/spec.py +186 -0
  607. fi/simulate/simulation/__init__.py +30 -0
  608. fi/simulate/simulation/behavior_policy.py +425 -0
  609. fi/simulate/simulation/bridge/__init__.py +9 -0
  610. fi/simulate/simulation/bridge/audio.py +29 -0
  611. fi/simulate/simulation/bridge/connector.py +46 -0
  612. fi/simulate/simulation/bridge/livekit.py +252 -0
  613. fi/simulate/simulation/bridge/retell.py +188 -0
  614. fi/simulate/simulation/bridge/vapi.py +177 -0
  615. fi/simulate/simulation/contract.py +419 -0
  616. fi/simulate/simulation/engines/__init__.py +12 -0
  617. fi/simulate/simulation/engines/base.py +21 -0
  618. fi/simulate/simulation/engines/cloud.py +517 -0
  619. fi/simulate/simulation/engines/livekit.py +4167 -0
  620. fi/simulate/simulation/engines/local_text.py +89 -0
  621. fi/simulate/simulation/fidelity.py +374 -0
  622. fi/simulate/simulation/gemini_tts_stream.py +110 -0
  623. fi/simulate/simulation/generator.py +91 -0
  624. fi/simulate/simulation/goal_machine.py +185 -0
  625. fi/simulate/simulation/livekit_models.py +467 -0
  626. fi/simulate/simulation/matrix.py +170 -0
  627. fi/simulate/simulation/models.py +279 -0
  628. fi/simulate/simulation/runner.py +153 -0
  629. fi/simulate/simulation/synthetic.py +880 -0
  630. fi/simulate/simulation/voice_prompt.py +502 -0
  631. fi/simulate/simulator/__init__.py +55 -0
  632. fi/simulate/simulator/builtins.py +53 -0
  633. fi/simulate/suite.py +1288 -0
  634. fi/simulate/utils/routes.py +164 -0
  635. fi/simulate/voice.py +225 -0
  636. fi/simulate/voice_cli.py +182 -0
  637. fi/utils/__init__.py +1 -0
  638. fi/utils/constants.py +14 -0
  639. fi/utils/errors.py +200 -0
  640. fi/utils/executor.py +26 -0
  641. fi/utils/routes.py +119 -0
  642. fi/utils/utils.py +17 -0
@@ -0,0 +1,540 @@
1
+ from .agent import (
2
+ AgentDefinition,
3
+ LiveKitSimulatorRuntime,
4
+ RetellTargetConfig,
5
+ SimulatorAgentDefinition,
6
+ VapiTargetConfig,
7
+ LLMConfig,
8
+ TTSConfig,
9
+ STTConfig,
10
+ VADConfig,
11
+ AgentInput,
12
+ AgentResponse,
13
+ AgentWrapper,
14
+ SimulationArtifact,
15
+ SimulationEvent,
16
+ GenericAgentWrapper,
17
+ FrameworkAdapterSpec,
18
+ browser_cua_contract,
19
+ discover_framework_adapter,
20
+ framework_adapter_capability_profile,
21
+ framework_adapter_capability_profiles,
22
+ framework_adapter_contract,
23
+ framework_adapter_contract_matrix,
24
+ memory_layer_contract,
25
+ multi_agent_room_contract,
26
+ orchestration_stack_contract,
27
+ probe_browser_cua,
28
+ probe_framework_adapter,
29
+ probe_memory_layer,
30
+ probe_multi_agent_room,
31
+ probe_orchestration_stack,
32
+ probe_realtime_stack,
33
+ realtime_stack_contract,
34
+ run_browser_cua_probe,
35
+ run_framework_adapter_probe,
36
+ run_memory_layer_probe,
37
+ run_multi_agent_room_probe,
38
+ run_orchestration_stack_probe,
39
+ run_realtime_stack_probe,
40
+ supported_frameworks,
41
+ probe_framework_imports,
42
+ wrap_agent,
43
+ wrap_framework,
44
+ EchoAgentWrapper,
45
+ RuleBasedAgentWrapper,
46
+ ScriptedAgentWrapper,
47
+ make_tool_response,
48
+ OpenAIAgentWrapper,
49
+ LangChainAgentWrapper,
50
+ GeminiAgentWrapper,
51
+ AnthropicAgentWrapper,
52
+ HTTPAgentWrapper,
53
+ OpenAICompatibleHTTPAgentWrapper,
54
+ WebSocketAgentWrapper,
55
+ )
56
+ from .environment import (
57
+ AdversarialEnvironmentPack,
58
+ AgentControlPlaneEnvironment,
59
+ AgentIntegrationEnvironment,
60
+ AgentMemoryLineageEnvironment,
61
+ AgentTrustBoundaryEnvironment,
62
+ AutonomyLoopEnvironment,
63
+ BrowserEnvironment,
64
+ DomainPackageEnvironment,
65
+ EnvironmentAdapter,
66
+ EnvironmentSnapshot,
67
+ FileEnvironment,
68
+ FrameworkCapabilityEnvironment,
69
+ FrameworkImportManifestEnvironment,
70
+ FrameworkLifecycleEnvironment,
71
+ FrameworkPortabilityEnvironment,
72
+ FrameworkProbeEnvironment,
73
+ FrameworkTraceEnvironment,
74
+ HarnessTrajectoryReplayEnvironment,
75
+ ImageEnvironment,
76
+ MultiAgentRoomEnvironment,
77
+ ObservabilityReplayEnvironment,
78
+ EnvironmentReplayEnvironment,
79
+ OpenEnvEnvironment,
80
+ OptimizerPortfolioEnvironment,
81
+ OptimizerTraceEnvironment,
82
+ OrchestrationTraceEnvironment,
83
+ PersistentStateRedTeamEnvironment,
84
+ RedTeamAttackEvolutionEnvironment,
85
+ RetrievalHookEnvironment,
86
+ RetrievalMemoryEnvironment,
87
+ RedTeamCampaignEnvironment,
88
+ RedTeamReadinessEnvironment,
89
+ StatefulToolWorldEnvironment,
90
+ StreamingTraceEnvironment,
91
+ StructuredArtifactEnvironment,
92
+ ToolExecutionResult,
93
+ ToolFaultInjectionEnvironment,
94
+ ToolMockEnvironment,
95
+ VoiceEnvironment,
96
+ WorkflowHookEnvironment,
97
+ WorkflowTraceEnvironment,
98
+ WorldAttackReplayEnvironment,
99
+ WorldContractEnvironment,
100
+ WorldOrchestrationReplayEnvironment,
101
+ WorkspaceRunEnvironment,
102
+ load_adversarial_attack_pack,
103
+ load_agent_integration_manifest,
104
+ load_agent_memory_lineage_manifest,
105
+ load_browser_mutation_pack,
106
+ load_browser_trace_export,
107
+ load_voice_export,
108
+ load_world_attack_replay,
109
+ load_world_orchestration_replay,
110
+ load_workspace_run_manifest,
111
+ load_pipecat_frame_log,
112
+ normalize_harness_trajectory_replay,
113
+ load_world_contract,
114
+ load_playwright_trace_export,
115
+ load_red_team_attack_evolution_manifest,
116
+ load_red_team_campaign_manifest,
117
+ load_red_team_readiness_manifest,
118
+ load_framework_trace_export,
119
+ load_framework_import_manifest,
120
+ load_mcp_tool_session_export,
121
+ load_observability_replay_pack,
122
+ load_environment_replay_manifest,
123
+ load_openenv_manifest,
124
+ load_optimizer_backend_portfolio,
125
+ load_persistent_state_attack_manifest,
126
+ load_framework_multi_agent_transcript,
127
+ load_orchestration_trace_export,
128
+ load_streaming_trace_export,
129
+ load_autogen_groupchat_transcript,
130
+ load_crewai_event_log,
131
+ load_openai_agents_trace,
132
+ load_openai_responses_trace,
133
+ load_langchain_event_stream,
134
+ load_langgraph_event_stream,
135
+ normalize_voice_timing_distribution,
136
+ normalize_pipecat_frame_log,
137
+ normalize_orchestration_trace_events,
138
+ normalize_orchestration_trace_export,
139
+ normalize_streaming_trace_events,
140
+ normalize_streaming_trace_export,
141
+ normalize_framework_lifecycle_trace,
142
+ normalize_framework_import_manifest,
143
+ normalize_framework_capability_matrix,
144
+ normalize_agent_control_plane,
145
+ normalize_agent_memory_lineage_manifest,
146
+ normalize_agent_trust_boundary_model,
147
+ normalize_framework_portability_matrix,
148
+ normalize_framework_trace_events,
149
+ normalize_framework_probe_suite,
150
+ normalize_framework_adapter_conformance,
151
+ normalize_observability_replay_pack,
152
+ normalize_environment_replay_manifest,
153
+ normalize_openenv_manifest,
154
+ normalize_optimizer_backend_portfolio,
155
+ normalize_optimizer_society_trace,
156
+ normalize_persistent_state_attack_manifest,
157
+ normalize_framework_trace_export,
158
+ normalize_mcp_tool_session_export,
159
+ normalize_openai_responses_trace,
160
+ normalize_browser_trace_export,
161
+ normalize_browser_mutation_pack,
162
+ normalize_voice_export,
163
+ normalize_adversarial_attack_pack,
164
+ normalize_agent_integration_manifest,
165
+ normalize_workspace_run_manifest,
166
+ normalize_world_attack_replay,
167
+ normalize_world_orchestration_replay,
168
+ normalize_world_contract,
169
+ normalize_stateful_tool_world_manifest,
170
+ normalize_playwright_trace_export,
171
+ normalize_red_team_attack_evolution_manifest,
172
+ normalize_red_team_campaign_manifest,
173
+ normalize_red_team_readiness_manifest,
174
+ )
175
+ from .simulation import (
176
+ AttackDefinition,
177
+ AttackVector,
178
+ Persona,
179
+ Scenario,
180
+ TestReport,
181
+ TestCaseResult,
182
+ TestRunner,
183
+ ScenarioGenerator,
184
+ SyntheticDataGenerator,
185
+ SyntheticScenarioConfig,
186
+ SyntheticTrajectoryTemplateBundle,
187
+ SyntheticTrajectoryTemplateConfig,
188
+ SyntheticToolTaskBundle,
189
+ SyntheticToolTaskConfig,
190
+ )
191
+ from .evaluation import evaluate_agent_report, evaluate_report
192
+ from .voice import (
193
+ build_voice_run_manifest,
194
+ generate_platform_voice_scenario,
195
+ run_voice_simulation,
196
+ )
197
+ from .manifest import (
198
+ MANIFEST_SCHEMA_VERSION,
199
+ ManifestError,
200
+ ManifestOptimizationOptions,
201
+ ManifestRunOptions,
202
+ apply_manifest_env,
203
+ build_manifest_agent_callback,
204
+ build_manifest_environments,
205
+ build_manifest_optimization_problem,
206
+ compare_result_files,
207
+ compare_results,
208
+ create_baseline,
209
+ create_baseline_file,
210
+ detect_manifest_command,
211
+ evaluate_manifest_report,
212
+ load_manifest,
213
+ load_manifest_file,
214
+ write_manifest_file,
215
+ missing_manifest_env,
216
+ optimize_manifest,
217
+ optimize_manifest_file,
218
+ prepare_redteam_manifest,
219
+ promote_to_regression,
220
+ promote_to_regression_file,
221
+ public_result,
222
+ redteam_manifest,
223
+ redteam_manifest_file,
224
+ required_manifest_env,
225
+ render_junit,
226
+ render_markdown,
227
+ render_report,
228
+ render_report_file,
229
+ render_sarif,
230
+ replay_manifests,
231
+ run_local_text_manifest,
232
+ run_manifest,
233
+ run_manifest_file,
234
+ run_redteam_manifest,
235
+ run_redteam_manifest_file,
236
+ shrink_attack_evolution,
237
+ shrink_attack_evolution_file,
238
+ supported_manifest_environment_types,
239
+ validate_manifest_env,
240
+ )
241
+ from .suite import (
242
+ EVAL_SUITE_SCHEMA_VERSION,
243
+ EvalSuiteOptions,
244
+ evaluate_assertions,
245
+ load_eval_suite_file,
246
+ run_eval_suite,
247
+ run_eval_suite_file,
248
+ )
249
+ from .endpoints import (
250
+ AgentEndpoint,
251
+ CallableAgentEndpoint,
252
+ HttpAgentEndpoint,
253
+ LiveKitAgentEndpoint,
254
+ RetellAgentEndpoint,
255
+ VapiAgentEndpoint,
256
+ WebSocketAgentEndpoint,
257
+ )
258
+ from .realtime import (
259
+ AudioFrame,
260
+ CANONICAL_EVENT_TYPES,
261
+ RealtimeBridgeSession,
262
+ RealtimeEndpoint,
263
+ RealtimeEvent,
264
+ )
265
+ from .simulator import (
266
+ PolicyContext,
267
+ PolicyState,
268
+ PolicySummary,
269
+ SimulatorPolicy,
270
+ )
271
+ from .results import FutureAGIResultSink
272
+ from .instrumentation.livekit import FutureAGIObserver
273
+ from .adapters import (
274
+ EnvironmentAdapters,
275
+ SimulatorAdapters,
276
+ TargetAdapters,
277
+ WorldKinds,
278
+ )
279
+
280
+ __all__ = [
281
+ "AgentDefinition",
282
+ "EnvironmentAdapters",
283
+ "SimulatorAdapters",
284
+ "TargetAdapters",
285
+ "WorldKinds",
286
+ "LiveKitSimulatorRuntime",
287
+ "VapiTargetConfig",
288
+ "RetellTargetConfig",
289
+ "SimulatorAgentDefinition",
290
+ "LLMConfig",
291
+ "TTSConfig",
292
+ "STTConfig",
293
+ "VADConfig",
294
+ "AgentInput",
295
+ "AgentResponse",
296
+ "AgentWrapper",
297
+ "SimulationArtifact",
298
+ "SimulationEvent",
299
+ "GenericAgentWrapper",
300
+ "FrameworkAdapterSpec",
301
+ "browser_cua_contract",
302
+ "discover_framework_adapter",
303
+ "framework_adapter_capability_profile",
304
+ "framework_adapter_capability_profiles",
305
+ "framework_adapter_contract",
306
+ "framework_adapter_contract_matrix",
307
+ "memory_layer_contract",
308
+ "multi_agent_room_contract",
309
+ "orchestration_stack_contract",
310
+ "probe_browser_cua",
311
+ "probe_framework_adapter",
312
+ "probe_memory_layer",
313
+ "probe_multi_agent_room",
314
+ "probe_orchestration_stack",
315
+ "probe_realtime_stack",
316
+ "realtime_stack_contract",
317
+ "run_browser_cua_probe",
318
+ "run_framework_adapter_probe",
319
+ "run_memory_layer_probe",
320
+ "run_multi_agent_room_probe",
321
+ "run_orchestration_stack_probe",
322
+ "run_realtime_stack_probe",
323
+ "supported_frameworks",
324
+ "probe_framework_imports",
325
+ "wrap_agent",
326
+ "wrap_framework",
327
+ "EchoAgentWrapper",
328
+ "RuleBasedAgentWrapper",
329
+ "ScriptedAgentWrapper",
330
+ "make_tool_response",
331
+ "OpenAIAgentWrapper",
332
+ "LangChainAgentWrapper",
333
+ "GeminiAgentWrapper",
334
+ "AnthropicAgentWrapper",
335
+ "HTTPAgentWrapper",
336
+ "OpenAICompatibleHTTPAgentWrapper",
337
+ "WebSocketAgentWrapper",
338
+ "AdversarialEnvironmentPack",
339
+ "AgentControlPlaneEnvironment",
340
+ "AgentIntegrationEnvironment",
341
+ "AgentMemoryLineageEnvironment",
342
+ "AgentTrustBoundaryEnvironment",
343
+ "AutonomyLoopEnvironment",
344
+ "BrowserEnvironment",
345
+ "DomainPackageEnvironment",
346
+ "EnvironmentAdapter",
347
+ "EnvironmentSnapshot",
348
+ "FileEnvironment",
349
+ "FrameworkCapabilityEnvironment",
350
+ "FrameworkImportManifestEnvironment",
351
+ "FrameworkLifecycleEnvironment",
352
+ "FrameworkPortabilityEnvironment",
353
+ "FrameworkProbeEnvironment",
354
+ "FrameworkTraceEnvironment",
355
+ "HarnessTrajectoryReplayEnvironment",
356
+ "ImageEnvironment",
357
+ "MultiAgentRoomEnvironment",
358
+ "ObservabilityReplayEnvironment",
359
+ "EnvironmentReplayEnvironment",
360
+ "OpenEnvEnvironment",
361
+ "OptimizerPortfolioEnvironment",
362
+ "OptimizerTraceEnvironment",
363
+ "OrchestrationTraceEnvironment",
364
+ "PersistentStateRedTeamEnvironment",
365
+ "RedTeamAttackEvolutionEnvironment",
366
+ "RetrievalHookEnvironment",
367
+ "RetrievalMemoryEnvironment",
368
+ "RedTeamCampaignEnvironment",
369
+ "RedTeamReadinessEnvironment",
370
+ "StatefulToolWorldEnvironment",
371
+ "StreamingTraceEnvironment",
372
+ "StructuredArtifactEnvironment",
373
+ "ToolExecutionResult",
374
+ "ToolFaultInjectionEnvironment",
375
+ "ToolMockEnvironment",
376
+ "VoiceEnvironment",
377
+ "WorkflowHookEnvironment",
378
+ "WorkflowTraceEnvironment",
379
+ "WorldAttackReplayEnvironment",
380
+ "WorldContractEnvironment",
381
+ "WorldOrchestrationReplayEnvironment",
382
+ "WorkspaceRunEnvironment",
383
+ "load_adversarial_attack_pack",
384
+ "load_agent_integration_manifest",
385
+ "load_agent_memory_lineage_manifest",
386
+ "load_browser_mutation_pack",
387
+ "load_browser_trace_export",
388
+ "load_voice_export",
389
+ "load_world_attack_replay",
390
+ "load_world_orchestration_replay",
391
+ "load_workspace_run_manifest",
392
+ "load_pipecat_frame_log",
393
+ "normalize_harness_trajectory_replay",
394
+ "load_world_contract",
395
+ "load_playwright_trace_export",
396
+ "load_red_team_attack_evolution_manifest",
397
+ "load_red_team_campaign_manifest",
398
+ "load_red_team_readiness_manifest",
399
+ "load_framework_trace_export",
400
+ "load_framework_import_manifest",
401
+ "load_mcp_tool_session_export",
402
+ "load_observability_replay_pack",
403
+ "load_environment_replay_manifest",
404
+ "load_openenv_manifest",
405
+ "load_optimizer_backend_portfolio",
406
+ "load_persistent_state_attack_manifest",
407
+ "load_framework_multi_agent_transcript",
408
+ "load_orchestration_trace_export",
409
+ "load_streaming_trace_export",
410
+ "load_autogen_groupchat_transcript",
411
+ "load_crewai_event_log",
412
+ "load_openai_agents_trace",
413
+ "load_openai_responses_trace",
414
+ "load_langchain_event_stream",
415
+ "load_langgraph_event_stream",
416
+ "normalize_voice_timing_distribution",
417
+ "normalize_pipecat_frame_log",
418
+ "normalize_orchestration_trace_events",
419
+ "normalize_orchestration_trace_export",
420
+ "normalize_streaming_trace_events",
421
+ "normalize_streaming_trace_export",
422
+ "normalize_framework_lifecycle_trace",
423
+ "normalize_framework_import_manifest",
424
+ "normalize_framework_capability_matrix",
425
+ "normalize_agent_control_plane",
426
+ "normalize_agent_memory_lineage_manifest",
427
+ "normalize_agent_trust_boundary_model",
428
+ "normalize_framework_portability_matrix",
429
+ "normalize_framework_trace_events",
430
+ "normalize_framework_probe_suite",
431
+ "normalize_framework_adapter_conformance",
432
+ "normalize_observability_replay_pack",
433
+ "normalize_environment_replay_manifest",
434
+ "normalize_openenv_manifest",
435
+ "normalize_optimizer_backend_portfolio",
436
+ "normalize_optimizer_society_trace",
437
+ "normalize_persistent_state_attack_manifest",
438
+ "normalize_framework_trace_export",
439
+ "normalize_mcp_tool_session_export",
440
+ "normalize_openai_responses_trace",
441
+ "normalize_browser_trace_export",
442
+ "normalize_browser_mutation_pack",
443
+ "normalize_voice_export",
444
+ "normalize_adversarial_attack_pack",
445
+ "normalize_agent_integration_manifest",
446
+ "normalize_workspace_run_manifest",
447
+ "normalize_world_attack_replay",
448
+ "normalize_world_orchestration_replay",
449
+ "normalize_world_contract",
450
+ "normalize_stateful_tool_world_manifest",
451
+ "normalize_playwright_trace_export",
452
+ "normalize_red_team_attack_evolution_manifest",
453
+ "normalize_red_team_campaign_manifest",
454
+ "normalize_red_team_readiness_manifest",
455
+ "AttackDefinition",
456
+ "AttackVector",
457
+ "Persona",
458
+ "Scenario",
459
+ "TestReport",
460
+ "TestCaseResult",
461
+ "TestRunner",
462
+ "ScenarioGenerator",
463
+ "SyntheticDataGenerator",
464
+ "SyntheticScenarioConfig",
465
+ "SyntheticTrajectoryTemplateBundle",
466
+ "SyntheticTrajectoryTemplateConfig",
467
+ "SyntheticToolTaskBundle",
468
+ "SyntheticToolTaskConfig",
469
+ "evaluate_report",
470
+ "evaluate_agent_report",
471
+ "MANIFEST_SCHEMA_VERSION",
472
+ "ManifestError",
473
+ "ManifestOptimizationOptions",
474
+ "ManifestRunOptions",
475
+ "EVAL_SUITE_SCHEMA_VERSION",
476
+ "EvalSuiteOptions",
477
+ "apply_manifest_env",
478
+ "build_manifest_agent_callback",
479
+ "build_manifest_environments",
480
+ "build_manifest_optimization_problem",
481
+ "compare_result_files",
482
+ "compare_results",
483
+ "create_baseline",
484
+ "create_baseline_file",
485
+ "detect_manifest_command",
486
+ "evaluate_manifest_report",
487
+ "load_manifest",
488
+ "load_manifest_file",
489
+ "write_manifest_file",
490
+ "load_eval_suite_file",
491
+ "missing_manifest_env",
492
+ "optimize_manifest",
493
+ "optimize_manifest_file",
494
+ "prepare_redteam_manifest",
495
+ "promote_to_regression",
496
+ "promote_to_regression_file",
497
+ "public_result",
498
+ "redteam_manifest",
499
+ "redteam_manifest_file",
500
+ "required_manifest_env",
501
+ "render_junit",
502
+ "render_markdown",
503
+ "render_report",
504
+ "render_report_file",
505
+ "render_sarif",
506
+ "replay_manifests",
507
+ "run_eval_suite",
508
+ "run_eval_suite_file",
509
+ "evaluate_assertions",
510
+ "run_local_text_manifest",
511
+ "run_voice_simulation",
512
+ "generate_platform_voice_scenario",
513
+ "build_voice_run_manifest",
514
+ "run_manifest",
515
+ "run_manifest_file",
516
+ "run_redteam_manifest",
517
+ "run_redteam_manifest_file",
518
+ "shrink_attack_evolution",
519
+ "shrink_attack_evolution_file",
520
+ "supported_manifest_environment_types",
521
+ "validate_manifest_env",
522
+ "AgentEndpoint",
523
+ "AudioFrame",
524
+ "CANONICAL_EVENT_TYPES",
525
+ "CallableAgentEndpoint",
526
+ "FutureAGIObserver",
527
+ "FutureAGIResultSink",
528
+ "HttpAgentEndpoint",
529
+ "LiveKitAgentEndpoint",
530
+ "PolicyContext",
531
+ "PolicyState",
532
+ "PolicySummary",
533
+ "RealtimeBridgeSession",
534
+ "RealtimeEndpoint",
535
+ "RealtimeEvent",
536
+ "RetellAgentEndpoint",
537
+ "SimulatorPolicy",
538
+ "VapiAgentEndpoint",
539
+ "WebSocketAgentEndpoint",
540
+ ]
@@ -0,0 +1,35 @@
1
+ from __future__ import annotations
2
+
3
+ import hashlib
4
+ import json
5
+ from collections.abc import Mapping
6
+ from datetime import date, datetime
7
+ from enum import Enum
8
+ from typing import Any
9
+
10
+ from pydantic import BaseModel
11
+
12
+
13
+ def content_hash(value: BaseModel | Mapping[str, Any]) -> str:
14
+ payload = (
15
+ value.model_dump(mode="json", exclude_none=True)
16
+ if isinstance(value, BaseModel)
17
+ else dict(value)
18
+ )
19
+ encoded = json.dumps(
20
+ payload,
21
+ default=_json_default,
22
+ separators=(",", ":"),
23
+ sort_keys=True,
24
+ ).encode("utf-8")
25
+ return f"sha256:{hashlib.sha256(encoded).hexdigest()}"
26
+
27
+
28
+ def _json_default(value: Any) -> Any:
29
+ if isinstance(value, Enum):
30
+ return value.value
31
+ if isinstance(value, (date, datetime)):
32
+ return value.isoformat()
33
+ if isinstance(value, BaseModel):
34
+ return value.model_dump(mode="json", exclude_none=True)
35
+ raise TypeError(f"Object of type {type(value).__name__} is not JSON serializable")
@@ -0,0 +1,10 @@
1
+ from __future__ import annotations
2
+
3
+ from types import TracebackType
4
+
5
+
6
+ def redacted_exc_info(
7
+ exc: BaseException,
8
+ ) -> tuple[type[RuntimeError], RuntimeError, TracebackType | None]:
9
+ redacted = RuntimeError(f"{type(exc).__name__}: details redacted")
10
+ return RuntimeError, redacted, exc.__traceback__
@@ -0,0 +1,87 @@
1
+ """Named constants for the built-in adapter strings used across a ``SimulationSpec``.
2
+
3
+ These are **ergonomic sugar**, not a closed vocabulary. Every adapter slot on the
4
+ spec stays a plain ``str`` so third-party plugins registered by name (decorator or
5
+ ``entry_points``) keep working. Each enum subclasses ``str``, so a member is the
6
+ string — ``EnvironmentSpec(adapter=EnvironmentAdapters.CHAT)`` is identical to
7
+ ``adapter="chat"`` (same value, same ``spec_hash``). Use them for autocomplete,
8
+ typo-safety, and discoverability; drop to a raw string for anything custom.
9
+
10
+ ``(str, Enum)`` rather than ``enum.StrEnum`` because the SDK floor is Python 3.10.
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ from enum import Enum
16
+
17
+
18
+ class EnvironmentAdapters(str, Enum):
19
+ """Built-in ``environment.adapter`` values — the **interaction loop** (how turns
20
+ happen), not the world's contents. Tool mocking / stateful tools are a property
21
+ of the *world object* (an ``EnvironmentAdapter``) that any loop can drive, not a
22
+ loop of their own."""
23
+
24
+ CHAT = "chat"
25
+ VOICE = "voice"
26
+
27
+
28
+ class TargetAdapters(str, Enum):
29
+ """Built-in ``target.adapter`` values (the agent-under-test / ActorSource)."""
30
+
31
+ SYSTEM_PROMPT = "system_prompt"
32
+ FACTORY = "factory"
33
+ IMPORT_OBJECT = "import_object"
34
+ FRAMEWORK = "framework"
35
+ CALLABLE = "callable"
36
+ PYTHON_CALLABLE = "python_callable"
37
+ HTTP = "http"
38
+ WEBSOCKET = "websocket"
39
+ WEBRTC = "webrtc"
40
+ LIVEKIT = "livekit"
41
+ VAPI_WEBSOCKET = "vapi_websocket"
42
+ RETELL_WEBCALL = "retell_webcall"
43
+ SIP_INBOUND = "sip_inbound"
44
+ SIP_OUTBOUND = "sip_outbound"
45
+
46
+
47
+ class SimulatorAdapters(str, Enum):
48
+ """Built-in ``simulator.adapter`` values (the synthetic-user policy)."""
49
+
50
+ SYNTHETIC_USER = "synthetic_user"
51
+ LIVEKIT_SIMULATOR = "livekit_simulator"
52
+
53
+
54
+ class WorldKinds(str, Enum):
55
+ """Built-in ``environment.world_kind`` labels — a **faithful mirror** of
56
+ ``fi.simulate.simulation.contract.SIMULATION_WORLD_KINDS`` (the frozen
57
+ single-home; ``test_worldkinds_mirror_contract`` byte-compares the two).
58
+
59
+ A ``world_kind`` names the **primary surface / modality** of an episode —
60
+ what is being exercised — not the engine and not "can a tool be called". It
61
+ is an *admission + benchmark label*, never an executor selector: executable
62
+ kinds (``conversation``, ``tool_api``) run contract-native rung-1 on the
63
+ shared text loop; ``browser`` / ``voice_telephony`` run derived-legacy;
64
+ ``computer_use`` / ``code_exec`` refuse (typed-only, engine staged).
65
+
66
+ Tool *use* is orthogonal — a capability (``contract.ToolBinding`` +
67
+ ``TOOL_MOCK_LEVELS``) attachable to ANY kind via ``WorldSpec.tools``; a
68
+ ``conversation`` world may call tools too. ``tool_api`` is the kind where the
69
+ tool surface *is* the world (scored as the tool-use modality), not "a world
70
+ that happens to call tools". Legacy runtime aliases ``"chat"`` / ``"text"``
71
+ (folded into ``conversation``) and ``"voice"`` (canonical:
72
+ ``voice_telephony``) remain valid as raw strings; the enum carries canon."""
73
+
74
+ CONVERSATION = "conversation"
75
+ TOOL_API = "tool_api"
76
+ BROWSER = "browser"
77
+ COMPUTER_USE = "computer_use"
78
+ CODE_EXEC = "code_exec"
79
+ VOICE_TELEPHONY = "voice_telephony"
80
+
81
+
82
+ __all__ = [
83
+ "EnvironmentAdapters",
84
+ "TargetAdapters",
85
+ "SimulatorAdapters",
86
+ "WorldKinds",
87
+ ]