agent-learning-kit 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (642) hide show
  1. agent_learning_kit-0.1.0.dist-info/METADATA +381 -0
  2. agent_learning_kit-0.1.0.dist-info/RECORD +642 -0
  3. agent_learning_kit-0.1.0.dist-info/WHEEL +4 -0
  4. agent_learning_kit-0.1.0.dist-info/entry_points.txt +5 -0
  5. agent_learning_kit-0.1.0.dist-info/licenses/LICENSE +173 -0
  6. agent_learning_kit-0.1.0.dist-info/licenses/NOTICE +7 -0
  7. fi/__init__.py +5 -0
  8. fi/alk/__init__.py +57 -0
  9. fi/alk/_facade.py +31 -0
  10. fi/alk/_module_alias.py +68 -0
  11. fi/alk/_paths.py +14 -0
  12. fi/alk/_schema.py +522 -0
  13. fi/alk/actions.py +727 -0
  14. fi/alk/bench/__init__.py +517 -0
  15. fi/alk/bench/_codeexec.py +213 -0
  16. fi/alk/bench/_coding.py +215 -0
  17. fi/alk/bench/_docker.py +237 -0
  18. fi/alk/bench/_grader.py +286 -0
  19. fi/alk/bench/_pull.py +212 -0
  20. fi/alk/bench/_voice.py +147 -0
  21. fi/alk/capabilities.py +627 -0
  22. fi/alk/cli.py +6396 -0
  23. fi/alk/config.py +130 -0
  24. fi/alk/cua_loop.py +562 -0
  25. fi/alk/evals.py +2351 -0
  26. fi/alk/extensions.py +163 -0
  27. fi/alk/harness/ARCHITECTURE.md +231 -0
  28. fi/alk/harness/DESIGN.md +246 -0
  29. fi/alk/harness/ENVIRONMENT_CONFORMANCE.md +127 -0
  30. fi/alk/harness/HOW-IT-WORKS.md +297 -0
  31. fi/alk/harness/IMPLEMENTATION_AND_VALIDATION_STATUS.md +229 -0
  32. fi/alk/harness/README.md +417 -0
  33. fi/alk/harness/__init__.py +77 -0
  34. fi/alk/harness/__main__.py +3 -0
  35. fi/alk/harness/amend.py +312 -0
  36. fi/alk/harness/artifacts.py +319 -0
  37. fi/alk/harness/authoring_entrypoint.py +189 -0
  38. fi/alk/harness/authoring_runtime_validation.py +267 -0
  39. fi/alk/harness/backends/README.md +43 -0
  40. fi/alk/harness/backends/__init__.py +122 -0
  41. fi/alk/harness/backends/base.py +241 -0
  42. fi/alk/harness/backends/claude.py +211 -0
  43. fi/alk/harness/backends/files.py +182 -0
  44. fi/alk/harness/backends/vertex_gemini.py +457 -0
  45. fi/alk/harness/background_noise.py +95 -0
  46. fi/alk/harness/build.py +385 -0
  47. fi/alk/harness/bundle.py +593 -0
  48. fi/alk/harness/bundle_author_v2.py +1831 -0
  49. fi/alk/harness/bundle_v2.py +719 -0
  50. fi/alk/harness/call_runner.py +1440 -0
  51. fi/alk/harness/callback_http_adapter.py +111 -0
  52. fi/alk/harness/catalogue.py +287 -0
  53. fi/alk/harness/chat.py +428 -0
  54. fi/alk/harness/chat_call_runner.py +506 -0
  55. fi/alk/harness/checks.py +136 -0
  56. fi/alk/harness/cli.py +1354 -0
  57. fi/alk/harness/config.py +338 -0
  58. fi/alk/harness/contract.py +718 -0
  59. fi/alk/harness/credentials.py +674 -0
  60. fi/alk/harness/data/persona_vocabulary.json +111 -0
  61. fi/alk/harness/environment.py +99 -0
  62. fi/alk/harness/environment_plan.py +168 -0
  63. fi/alk/harness/events.py +125 -0
  64. fi/alk/harness/executor.py +304 -0
  65. fi/alk/harness/folder.py +234 -0
  66. fi/alk/harness/generated_runtime.py +815 -0
  67. fi/alk/harness/github.py +72 -0
  68. fi/alk/harness/hosted_authoring_entrypoint.py +183 -0
  69. fi/alk/harness/hosted_entrypoint.py +2402 -0
  70. fi/alk/harness/hosted_scheduler.py +2218 -0
  71. fi/alk/harness/job.py +426 -0
  72. fi/alk/harness/judge.py +184 -0
  73. fi/alk/harness/livekit_source.py +50 -0
  74. fi/alk/harness/livekit_tool_trace_bootstrap.py +71 -0
  75. fi/alk/harness/observability.py +208 -0
  76. fi/alk/harness/outbound.py +3252 -0
  77. fi/alk/harness/packaging.py +515 -0
  78. fi/alk/harness/persona_guides.py +157 -0
  79. fi/alk/harness/platform.py +692 -0
  80. fi/alk/harness/process_preflight.py +764 -0
  81. fi/alk/harness/process_runtime.py +5670 -0
  82. fi/alk/harness/prove.py +425 -0
  83. fi/alk/harness/provider_import.py +703 -0
  84. fi/alk/harness/provider_lifecycle.py +392 -0
  85. fi/alk/harness/provision.py +2896 -0
  86. fi/alk/harness/reception.py +147 -0
  87. fi/alk/harness/retell_chat_call_runner.py +373 -0
  88. fi/alk/harness/run/__init__.py +296 -0
  89. fi/alk/harness/run/alk.py +184 -0
  90. fi/alk/harness/run/call.py +162 -0
  91. fi/alk/harness/run/conversation.py +264 -0
  92. fi/alk/harness/run/data/voices_by_language_and_gender.json +693 -0
  93. fi/alk/harness/run/evidence.py +195 -0
  94. fi/alk/harness/run/grade.py +598 -0
  95. fi/alk/harness/run/live.py +297 -0
  96. fi/alk/harness/run/models.py +56 -0
  97. fi/alk/harness/run/platform_evals.py +227 -0
  98. fi/alk/harness/run/sdk_voice.py +130 -0
  99. fi/alk/harness/run/simulation.py +1209 -0
  100. fi/alk/harness/run/stage.py +91 -0
  101. fi/alk/harness/run/targets.py +508 -0
  102. fi/alk/harness/run/tools.py +601 -0
  103. fi/alk/harness/run/voice.py +340 -0
  104. fi/alk/harness/runtime.py +172 -0
  105. fi/alk/harness/sandbox_server.py +2011 -0
  106. fi/alk/harness/sandbox_worker.py +44 -0
  107. fi/alk/harness/scenario.py +1048 -0
  108. fi/alk/harness/scenario_source.py +879 -0
  109. fi/alk/harness/scenario_tools.py +1143 -0
  110. fi/alk/harness/scenarios.py +915 -0
  111. fi/alk/harness/secrets.py +168 -0
  112. fi/alk/harness/service_catalog.py +97 -0
  113. fi/alk/harness/session.py +391 -0
  114. fi/alk/harness/sessions.py +372 -0
  115. fi/alk/harness/simulator.py +76 -0
  116. fi/alk/harness/simulator_voice.py +928 -0
  117. fi/alk/harness/skills/build-environment/SKILL.md +538 -0
  118. fi/alk/harness/skills/harness.md +131 -0
  119. fi/alk/harness/skills/kinds/chat.md +48 -0
  120. fi/alk/harness/skills/kinds/voice-voicemail.md +63 -0
  121. fi/alk/harness/skills/kinds/voice.md +59 -0
  122. fi/alk/harness/skills/plan-suite/SKILL.md +103 -0
  123. fi/alk/harness/skills/provision-environment/SKILL.md +136 -0
  124. fi/alk/harness/skills/run-scenarios/SKILL.md +112 -0
  125. fi/alk/harness/skills/understand-agent/SKILL.md +251 -0
  126. fi/alk/harness/skills/write-scenarios/SKILL.md +606 -0
  127. fi/alk/harness/skills/write-scenarios/references/refusals.md +28 -0
  128. fi/alk/harness/skills/write-scenarios/references/world-api.md +92 -0
  129. fi/alk/harness/source_data_invariants.py +444 -0
  130. fi/alk/harness/source_tool_evidence.py +79 -0
  131. fi/alk/harness/sources.py +253 -0
  132. fi/alk/harness/spend.py +140 -0
  133. fi/alk/harness/tool_trace_proxy.py +104 -0
  134. fi/alk/harness/tools.py +1018 -0
  135. fi/alk/harness/understand.py +169 -0
  136. fi/alk/harness/voicemail_audio.py +74 -0
  137. fi/alk/harness/world/__init__.py +33 -0
  138. fi/alk/harness/world/errors.py +68 -0
  139. fi/alk/harness/world/expectations.py +91 -0
  140. fi/alk/harness/world/handle.py +538 -0
  141. fi/alk/harness/world/kinds.py +196 -0
  142. fi/alk/harness/world/mutate.py +186 -0
  143. fi/alk/harness/world/probe.py +413 -0
  144. fi/alk/harness/world/provision.py +511 -0
  145. fi/alk/harness/world/provisioned.py +191 -0
  146. fi/alk/harness/world/runtime.py +616 -0
  147. fi/alk/harness/world/snapshot.py +288 -0
  148. fi/alk/harness/world/stores/__init__.py +305 -0
  149. fi/alk/harness/world/stores/container.py +215 -0
  150. fi/alk/harness/world/stores/inprocess.py +346 -0
  151. fi/alk/harness/world/stores/postgres.py +481 -0
  152. fi/alk/harness/world/stores/prove.py +202 -0
  153. fi/alk/harness/world/stores/sqlite.py +245 -0
  154. fi/alk/harness/world/stores/written.py +182 -0
  155. fi/alk/harness/world/tools.py +1516 -0
  156. fi/alk/harness/world/workspace.py +144 -0
  157. fi/alk/image_loop.py +453 -0
  158. fi/alk/image_perturb.py +241 -0
  159. fi/alk/improve.py +274 -0
  160. fi/alk/live/__init__.py +154 -0
  161. fi/alk/live/_attribution.py +184 -0
  162. fi/alk/live/_capture.py +264 -0
  163. fi/alk/live/_codec.py +391 -0
  164. fi/alk/live/_contract.py +134 -0
  165. fi/alk/live/_loopback.py +316 -0
  166. fi/alk/live/_perturb.py +449 -0
  167. fi/alk/live/_runner.py +386 -0
  168. fi/alk/live/_stats.py +561 -0
  169. fi/alk/live/_transcript.py +240 -0
  170. fi/alk/live/_workers/__init__.py +9 -0
  171. fi/alk/live/_workers/a2a_worker.py +316 -0
  172. fi/alk/live/_workers/langgraph_worker.py +217 -0
  173. fi/alk/live/_workers/livekit_worker.py +207 -0
  174. fi/alk/live/_workers/mcp_loopback_server.py +46 -0
  175. fi/alk/live/_workers/mcp_worker.py +158 -0
  176. fi/alk/live/_workers/pipecat_worker.py +189 -0
  177. fi/alk/live/a2a_lane.py +138 -0
  178. fi/alk/live/langgraph_lane.py +339 -0
  179. fi/alk/live/livekit_lane.py +376 -0
  180. fi/alk/live/mcp_lane.py +172 -0
  181. fi/alk/live/pipecat_lane.py +341 -0
  182. fi/alk/live/voice_redteam.py +494 -0
  183. fi/alk/loss.py +306 -0
  184. fi/alk/optimize.py +36260 -0
  185. fi/alk/practice/__init__.py +51 -0
  186. fi/alk/practice/_assess.py +103 -0
  187. fi/alk/practice/_budget.py +81 -0
  188. fi/alk/practice/_calibrate.py +69 -0
  189. fi/alk/practice/_capstone.py +86 -0
  190. fi/alk/practice/_contract.py +91 -0
  191. fi/alk/practice/_diagnose.py +79 -0
  192. fi/alk/practice/_drill.py +196 -0
  193. fi/alk/practice/_experiment.py +720 -0
  194. fi/alk/practice/_schedule.py +102 -0
  195. fi/alk/practice/_store.py +194 -0
  196. fi/alk/practice/_trainer.py +245 -0
  197. fi/alk/practice/_update.py +125 -0
  198. fi/alk/redteam.py +2621 -0
  199. fi/alk/rewardhack.py +237 -0
  200. fi/alk/simulate.py +10351 -0
  201. fi/alk/studio/__init__.py +82 -0
  202. fi/alk/studio/_bias.py +314 -0
  203. fi/alk/studio/_calibration.py +522 -0
  204. fi/alk/studio/_coverage.py +262 -0
  205. fi/alk/studio/_download.py +665 -0
  206. fi/alk/studio/_fidelity_attack.py +114 -0
  207. fi/alk/studio/_generate.py +652 -0
  208. fi/alk/studio/_library.py +370 -0
  209. fi/alk/studio/_scan.py +134 -0
  210. fi/alk/studio/_upgrade.py +42 -0
  211. fi/alk/studio/_vendor.py +172 -0
  212. fi/alk/suite.py +4200 -0
  213. fi/alk/tasks.py +828 -0
  214. fi/alk/telemetry/__init__.py +149 -0
  215. fi/alk/telemetry/_contract.py +141 -0
  216. fi/alk/telemetry/_emit.py +182 -0
  217. fi/alk/telemetry/_ledger.py +296 -0
  218. fi/alk/telemetry/_queue.py +127 -0
  219. fi/alk/telemetry/_row.py +294 -0
  220. fi/alk/telemetry/_run.py +233 -0
  221. fi/alk/telemetry/_sync.py +193 -0
  222. fi/alk/telemetry/_url.py +119 -0
  223. fi/alk/trinity.py +49397 -0
  224. fi/alk/voice_loop.py +174 -0
  225. fi/api/__init__.py +1 -0
  226. fi/api/auth.py +137 -0
  227. fi/api/types.py +29 -0
  228. fi/cli/__init__.py +9 -0
  229. fi/cli/assertions/__init__.py +25 -0
  230. fi/cli/assertions/conditions.py +76 -0
  231. fi/cli/assertions/evaluator.py +286 -0
  232. fi/cli/assertions/exit_codes.py +20 -0
  233. fi/cli/assertions/parser.py +131 -0
  234. fi/cli/assertions/reporter.py +194 -0
  235. fi/cli/commands/__init__.py +9 -0
  236. fi/cli/commands/config.py +165 -0
  237. fi/cli/commands/export.py +208 -0
  238. fi/cli/commands/init.py +112 -0
  239. fi/cli/commands/list_cmd.py +213 -0
  240. fi/cli/commands/run.py +486 -0
  241. fi/cli/commands/validate.py +173 -0
  242. fi/cli/commands/view.py +424 -0
  243. fi/cli/config/__init__.py +6 -0
  244. fi/cli/config/defaults.py +206 -0
  245. fi/cli/config/loader.py +155 -0
  246. fi/cli/config/schema.py +174 -0
  247. fi/cli/main.py +78 -0
  248. fi/cli/output/__init__.py +6 -0
  249. fi/cli/output/formatters.py +106 -0
  250. fi/cli/output/reporters.py +46 -0
  251. fi/cli/storage/__init__.py +5 -0
  252. fi/cli/storage/run_history.py +249 -0
  253. fi/cli/utils/__init__.py +5 -0
  254. fi/cli/utils/console.py +44 -0
  255. fi/evals/__init__.py +131 -0
  256. fi/evals/autoeval/__init__.py +137 -0
  257. fi/evals/autoeval/analyzer.py +211 -0
  258. fi/evals/autoeval/config.py +244 -0
  259. fi/evals/autoeval/export.py +213 -0
  260. fi/evals/autoeval/interactive.py +283 -0
  261. fi/evals/autoeval/pipeline.py +625 -0
  262. fi/evals/autoeval/prompts.py +139 -0
  263. fi/evals/autoeval/recommender.py +242 -0
  264. fi/evals/autoeval/rules.py +589 -0
  265. fi/evals/autoeval/templates.py +299 -0
  266. fi/evals/autoeval/types.py +232 -0
  267. fi/evals/core/__init__.py +16 -0
  268. fi/evals/core/cloud_registry.py +184 -0
  269. fi/evals/core/engines.py +368 -0
  270. fi/evals/core/evaluate.py +319 -0
  271. fi/evals/core/judge_prompt.py +90 -0
  272. fi/evals/core/prompt_generator.py +83 -0
  273. fi/evals/core/registry.py +57 -0
  274. fi/evals/core/result.py +55 -0
  275. fi/evals/evaluator.py +721 -0
  276. fi/evals/execution.py +168 -0
  277. fi/evals/feedback/__init__.py +32 -0
  278. fi/evals/feedback/calibrator.py +160 -0
  279. fi/evals/feedback/collector.py +214 -0
  280. fi/evals/feedback/hooks.py +81 -0
  281. fi/evals/feedback/retriever.py +128 -0
  282. fi/evals/feedback/store.py +272 -0
  283. fi/evals/feedback/types.py +99 -0
  284. fi/evals/framework/README.md +79 -0
  285. fi/evals/framework/__init__.py +267 -0
  286. fi/evals/framework/backends/Dockerfile.eval-runner +33 -0
  287. fi/evals/framework/backends/__init__.py +99 -0
  288. fi/evals/framework/backends/_container.py +141 -0
  289. fi/evals/framework/backends/_utils.py +145 -0
  290. fi/evals/framework/backends/base.py +223 -0
  291. fi/evals/framework/backends/celery_backend.py +417 -0
  292. fi/evals/framework/backends/celery_worker.py +78 -0
  293. fi/evals/framework/backends/kubernetes_backend.py +665 -0
  294. fi/evals/framework/backends/ray_backend.py +521 -0
  295. fi/evals/framework/backends/temporal.py +350 -0
  296. fi/evals/framework/backends/temporal_worker.py +126 -0
  297. fi/evals/framework/backends/thread_pool.py +286 -0
  298. fi/evals/framework/context.py +258 -0
  299. fi/evals/framework/enrichment.py +306 -0
  300. fi/evals/framework/evals/__init__.py +68 -0
  301. fi/evals/framework/evals/agentic.py +399 -0
  302. fi/evals/framework/evals/builder.py +609 -0
  303. fi/evals/framework/evals/semantic.py +142 -0
  304. fi/evals/framework/evaluator.py +647 -0
  305. fi/evals/framework/evaluators/__init__.py +22 -0
  306. fi/evals/framework/evaluators/blocking.py +347 -0
  307. fi/evals/framework/evaluators/non_blocking.py +577 -0
  308. fi/evals/framework/propagation.py +421 -0
  309. fi/evals/framework/protocols.py +385 -0
  310. fi/evals/framework/registry.py +370 -0
  311. fi/evals/framework/resilience/__init__.py +150 -0
  312. fi/evals/framework/resilience/circuit_breaker.py +309 -0
  313. fi/evals/framework/resilience/degradation.py +355 -0
  314. fi/evals/framework/resilience/health.py +505 -0
  315. fi/evals/framework/resilience/rate_limiter.py +228 -0
  316. fi/evals/framework/resilience/retry.py +274 -0
  317. fi/evals/framework/resilience/types.py +288 -0
  318. fi/evals/framework/resilience/wrapper.py +433 -0
  319. fi/evals/framework/types.py +218 -0
  320. fi/evals/guardrails/README.md +915 -0
  321. fi/evals/guardrails/__init__.py +96 -0
  322. fi/evals/guardrails/backends/__init__.py +43 -0
  323. fi/evals/guardrails/backends/azure.py +361 -0
  324. fi/evals/guardrails/backends/base.py +88 -0
  325. fi/evals/guardrails/backends/generic_llm.py +163 -0
  326. fi/evals/guardrails/backends/granite.py +216 -0
  327. fi/evals/guardrails/backends/llamaguard.py +221 -0
  328. fi/evals/guardrails/backends/local_base.py +479 -0
  329. fi/evals/guardrails/backends/openai.py +365 -0
  330. fi/evals/guardrails/backends/qwen.py +170 -0
  331. fi/evals/guardrails/backends/shieldgemma.py +154 -0
  332. fi/evals/guardrails/backends/turing.py +235 -0
  333. fi/evals/guardrails/backends/vllm_client.py +321 -0
  334. fi/evals/guardrails/backends/wildguard.py +188 -0
  335. fi/evals/guardrails/base.py +888 -0
  336. fi/evals/guardrails/config.py +221 -0
  337. fi/evals/guardrails/discovery.py +243 -0
  338. fi/evals/guardrails/gateway.py +437 -0
  339. fi/evals/guardrails/registry.py +231 -0
  340. fi/evals/guardrails/scanners/__init__.py +127 -0
  341. fi/evals/guardrails/scanners/base.py +191 -0
  342. fi/evals/guardrails/scanners/code_injection.py +243 -0
  343. fi/evals/guardrails/scanners/eval_delegate.py +574 -0
  344. fi/evals/guardrails/scanners/invisible_chars.py +351 -0
  345. fi/evals/guardrails/scanners/jailbreak.py +412 -0
  346. fi/evals/guardrails/scanners/language.py +288 -0
  347. fi/evals/guardrails/scanners/pipeline.py +260 -0
  348. fi/evals/guardrails/scanners/regex.py +311 -0
  349. fi/evals/guardrails/scanners/secrets.py +274 -0
  350. fi/evals/guardrails/scanners/topics.py +649 -0
  351. fi/evals/guardrails/scanners/urls.py +341 -0
  352. fi/evals/guardrails/types.py +96 -0
  353. fi/evals/llm/__init__.py +3 -0
  354. fi/evals/llm/base_llm_provider.py +35 -0
  355. fi/evals/llm/providers/litellm.py +70 -0
  356. fi/evals/local/__init__.py +90 -0
  357. fi/evals/local/evaluator.py +690 -0
  358. fi/evals/local/execution_mode.py +121 -0
  359. fi/evals/local/llm.py +489 -0
  360. fi/evals/local/metrics/__init__.py +19 -0
  361. fi/evals/local/registry.py +360 -0
  362. fi/evals/manager.py +1018 -0
  363. fi/evals/manager_types.py +362 -0
  364. fi/evals/metrics/__init__.py +185 -0
  365. fi/evals/metrics/agents/__init__.py +74 -0
  366. fi/evals/metrics/agents/metrics.py +693 -0
  367. fi/evals/metrics/agents/report.py +36463 -0
  368. fi/evals/metrics/agents/types.py +160 -0
  369. fi/evals/metrics/base_llm_metric.py +111 -0
  370. fi/evals/metrics/base_metric.py +138 -0
  371. fi/evals/metrics/code_security/__init__.py +305 -0
  372. fi/evals/metrics/code_security/analyzer.py +985 -0
  373. fi/evals/metrics/code_security/benchmarks/__init__.py +73 -0
  374. fi/evals/metrics/code_security/benchmarks/builtin.py +750 -0
  375. fi/evals/metrics/code_security/benchmarks/loader.py +580 -0
  376. fi/evals/metrics/code_security/benchmarks/types.py +308 -0
  377. fi/evals/metrics/code_security/detectors/__init__.py +186 -0
  378. fi/evals/metrics/code_security/detectors/base.py +394 -0
  379. fi/evals/metrics/code_security/detectors/cryptography.py +345 -0
  380. fi/evals/metrics/code_security/detectors/injection.py +744 -0
  381. fi/evals/metrics/code_security/detectors/secrets.py +287 -0
  382. fi/evals/metrics/code_security/detectors/serialization.py +192 -0
  383. fi/evals/metrics/code_security/joint_metrics.py +588 -0
  384. fi/evals/metrics/code_security/judges/__init__.py +83 -0
  385. fi/evals/metrics/code_security/judges/base.py +238 -0
  386. fi/evals/metrics/code_security/judges/dual_judge.py +534 -0
  387. fi/evals/metrics/code_security/judges/llm_judge.py +301 -0
  388. fi/evals/metrics/code_security/judges/pattern_judge.py +515 -0
  389. fi/evals/metrics/code_security/metrics.py +388 -0
  390. fi/evals/metrics/code_security/modes/__init__.py +63 -0
  391. fi/evals/metrics/code_security/modes/adversarial.py +284 -0
  392. fi/evals/metrics/code_security/modes/autocomplete.py +198 -0
  393. fi/evals/metrics/code_security/modes/base.py +283 -0
  394. fi/evals/metrics/code_security/modes/instruct.py +253 -0
  395. fi/evals/metrics/code_security/modes/repair.py +230 -0
  396. fi/evals/metrics/code_security/reports/__init__.py +57 -0
  397. fi/evals/metrics/code_security/reports/generator.py +404 -0
  398. fi/evals/metrics/code_security/reports/leaderboard.py +509 -0
  399. fi/evals/metrics/code_security/types.py +534 -0
  400. fi/evals/metrics/function_calling/__init__.py +34 -0
  401. fi/evals/metrics/function_calling/metrics.py +573 -0
  402. fi/evals/metrics/function_calling/types.py +87 -0
  403. fi/evals/metrics/hallucination/__init__.py +54 -0
  404. fi/evals/metrics/hallucination/detector.py +149 -0
  405. fi/evals/metrics/hallucination/metrics.py +390 -0
  406. fi/evals/metrics/hallucination/nli.py +253 -0
  407. fi/evals/metrics/hallucination/sentinel.py +106 -0
  408. fi/evals/metrics/hallucination/types.py +132 -0
  409. fi/evals/metrics/heuristics/aggregation_metrics.py +85 -0
  410. fi/evals/metrics/heuristics/json_metrics.py +87 -0
  411. fi/evals/metrics/heuristics/similarity_metrics.py +375 -0
  412. fi/evals/metrics/heuristics/string_metrics.py +391 -0
  413. fi/evals/metrics/llm_as_judges/__init__.py +17 -0
  414. fi/evals/metrics/llm_as_judges/custom_judge/metric.py +112 -0
  415. fi/evals/metrics/llm_as_judges/custom_judge/prompts.py +26 -0
  416. fi/evals/metrics/llm_as_judges/types.py +48 -0
  417. fi/evals/metrics/rag/__init__.py +111 -0
  418. fi/evals/metrics/rag/advanced/__init__.py +14 -0
  419. fi/evals/metrics/rag/advanced/multi_hop.py +283 -0
  420. fi/evals/metrics/rag/advanced/source_attribution.py +344 -0
  421. fi/evals/metrics/rag/generation/__init__.py +17 -0
  422. fi/evals/metrics/rag/generation/answer_relevancy.py +176 -0
  423. fi/evals/metrics/rag/generation/context_utilization.py +245 -0
  424. fi/evals/metrics/rag/generation/faithfulness.py +241 -0
  425. fi/evals/metrics/rag/generation/groundedness.py +131 -0
  426. fi/evals/metrics/rag/rag_score.py +277 -0
  427. fi/evals/metrics/rag/retrieval/__init__.py +20 -0
  428. fi/evals/metrics/rag/retrieval/context_entity_recall.py +124 -0
  429. fi/evals/metrics/rag/retrieval/context_precision.py +158 -0
  430. fi/evals/metrics/rag/retrieval/context_recall.py +106 -0
  431. fi/evals/metrics/rag/retrieval/noise_sensitivity.py +163 -0
  432. fi/evals/metrics/rag/retrieval/ranking.py +261 -0
  433. fi/evals/metrics/rag/types.py +100 -0
  434. fi/evals/metrics/rag/utils/__init__.py +62 -0
  435. fi/evals/metrics/rag/utils/claims.py +189 -0
  436. fi/evals/metrics/rag/utils/entities.py +244 -0
  437. fi/evals/metrics/rag/utils/nli.py +92 -0
  438. fi/evals/metrics/rag/utils/similarity.py +345 -0
  439. fi/evals/metrics/structured/__init__.py +114 -0
  440. fi/evals/metrics/structured/field_completeness.py +313 -0
  441. fi/evals/metrics/structured/hierarchy_score.py +366 -0
  442. fi/evals/metrics/structured/json_validation.py +190 -0
  443. fi/evals/metrics/structured/schema_compliance.py +280 -0
  444. fi/evals/metrics/structured/structured_output_score.py +298 -0
  445. fi/evals/metrics/structured/types.py +108 -0
  446. fi/evals/metrics/structured/validators/__init__.py +30 -0
  447. fi/evals/metrics/structured/validators/base.py +189 -0
  448. fi/evals/metrics/structured/validators/json_validator.py +196 -0
  449. fi/evals/metrics/structured/validators/pydantic_validator.py +178 -0
  450. fi/evals/metrics/structured/validators/yaml_validator.py +248 -0
  451. fi/evals/otel/__init__.py +266 -0
  452. fi/evals/otel/config.py +400 -0
  453. fi/evals/otel/conventions.py +463 -0
  454. fi/evals/otel/enrichment.py +371 -0
  455. fi/evals/otel/instrumentors/__init__.py +140 -0
  456. fi/evals/otel/instrumentors/anthropic.py +517 -0
  457. fi/evals/otel/instrumentors/base.py +382 -0
  458. fi/evals/otel/instrumentors/openai.py +673 -0
  459. fi/evals/otel/processors/__init__.py +36 -0
  460. fi/evals/otel/processors/base.py +473 -0
  461. fi/evals/otel/processors/cost.py +445 -0
  462. fi/evals/otel/processors/evaluation.py +559 -0
  463. fi/evals/otel/processors/llm.py +462 -0
  464. fi/evals/otel/tracer.py +506 -0
  465. fi/evals/otel/types.py +232 -0
  466. fi/evals/otel_utils.py +23 -0
  467. fi/evals/protect.py +671 -0
  468. fi/evals/protect_input_adapter.py +154 -0
  469. fi/evals/streaming/__init__.py +88 -0
  470. fi/evals/streaming/buffer.py +213 -0
  471. fi/evals/streaming/evaluator.py +551 -0
  472. fi/evals/streaming/policy.py +307 -0
  473. fi/evals/streaming/scorers.py +368 -0
  474. fi/evals/streaming/types.py +238 -0
  475. fi/evals/templates.py +472 -0
  476. fi/evals/types.py +156 -0
  477. fi/opt/__init__.py +221 -0
  478. fi/opt/_objective_scoring.py +85 -0
  479. fi/opt/base/__init__.py +11 -0
  480. fi/opt/base/base_generator.py +33 -0
  481. fi/opt/base/base_mapper.py +26 -0
  482. fi/opt/base/base_optimizer.py +45 -0
  483. fi/opt/base/evaluator.py +211 -0
  484. fi/opt/components.py +3095 -0
  485. fi/opt/datamappers/__init__.py +3 -0
  486. fi/opt/datamappers/basic_mapper.py +40 -0
  487. fi/opt/deployment.py +1021 -0
  488. fi/opt/evidence.py +4332 -0
  489. fi/opt/generators/__init__.py +3 -0
  490. fi/opt/generators/litellm.py +66 -0
  491. fi/opt/integrations/__init__.py +23 -0
  492. fi/opt/integrations/generative_suite.py +410 -0
  493. fi/opt/integrations/simulate.py +1313 -0
  494. fi/opt/mutations.py +771 -0
  495. fi/opt/observability.py +4639 -0
  496. fi/opt/optimizer_trace.py +889 -0
  497. fi/opt/optimizers/__init__.py +80 -0
  498. fi/opt/optimizers/agent.py +331 -0
  499. fi/opt/optimizers/agent_bandit.py +392 -0
  500. fi/opt/optimizers/agent_curriculum.py +635 -0
  501. fi/opt/optimizers/agent_evolution.py +894 -0
  502. fi/opt/optimizers/agent_feedback.py +1863 -0
  503. fi/opt/optimizers/agent_pareto.py +547 -0
  504. fi/opt/optimizers/agent_social_memory.py +1113 -0
  505. fi/opt/optimizers/agent_tpe.py +321 -0
  506. fi/opt/optimizers/bayesian_search.py +449 -0
  507. fi/opt/optimizers/council.py +2075 -0
  508. fi/opt/optimizers/futureagi_replay.py +799 -0
  509. fi/opt/optimizers/gepa.py +322 -0
  510. fi/opt/optimizers/metaprompt.py +243 -0
  511. fi/opt/optimizers/promptwizard.py +417 -0
  512. fi/opt/optimizers/protegi.py +329 -0
  513. fi/opt/optimizers/random_search.py +224 -0
  514. fi/opt/research.py +518 -0
  515. fi/opt/simulation.py +260 -0
  516. fi/opt/targets.py +232 -0
  517. fi/opt/types.py +66 -0
  518. fi/opt/utils/__init__.py +4 -0
  519. fi/opt/utils/early_stopping.py +266 -0
  520. fi/opt/utils/setup_logging.py +82 -0
  521. fi/simulate/__init__.py +540 -0
  522. fi/simulate/_hashing.py +35 -0
  523. fi/simulate/_logging.py +10 -0
  524. fi/simulate/adapters.py +87 -0
  525. fi/simulate/agent/__init__.py +120 -0
  526. fi/simulate/agent/browser.py +658 -0
  527. fi/simulate/agent/definition.py +587 -0
  528. fi/simulate/agent/frameworks.py +3528 -0
  529. fi/simulate/agent/generic.py +8286 -0
  530. fi/simulate/agent/import_probe.py +227 -0
  531. fi/simulate/agent/memory.py +905 -0
  532. fi/simulate/agent/mocks.py +101 -0
  533. fi/simulate/agent/multi_agent.py +361 -0
  534. fi/simulate/agent/orchestration.py +903 -0
  535. fi/simulate/agent/realtime.py +665 -0
  536. fi/simulate/agent/wrapper.py +99 -0
  537. fi/simulate/agent/wrappers/__init__.py +18 -0
  538. fi/simulate/agent/wrappers/anthropic.py +62 -0
  539. fi/simulate/agent/wrappers/gemini.py +65 -0
  540. fi/simulate/agent/wrappers/http.py +404 -0
  541. fi/simulate/agent/wrappers/langchain.py +80 -0
  542. fi/simulate/agent/wrappers/openai.py +75 -0
  543. fi/simulate/agent/wrappers/websocket.py +326 -0
  544. fi/simulate/artifacts/__init__.py +11 -0
  545. fi/simulate/artifacts/manifest.py +62 -0
  546. fi/simulate/cli.py +20560 -0
  547. fi/simulate/endpoints/__init__.py +45 -0
  548. fi/simulate/endpoints/_http_actor.py +73 -0
  549. fi/simulate/endpoints/actor_sources.py +243 -0
  550. fi/simulate/endpoints/base.py +107 -0
  551. fi/simulate/endpoints/builtins.py +10 -0
  552. fi/simulate/endpoints/callable.py +95 -0
  553. fi/simulate/endpoints/http.py +76 -0
  554. fi/simulate/endpoints/livekit.py +138 -0
  555. fi/simulate/endpoints/originators.py +132 -0
  556. fi/simulate/endpoints/profiles.py +348 -0
  557. fi/simulate/endpoints/retell.py +633 -0
  558. fi/simulate/endpoints/vapi.py +205 -0
  559. fi/simulate/endpoints/websocket.py +76 -0
  560. fi/simulate/environment.py +33026 -0
  561. fi/simulate/environments/__init__.py +11 -0
  562. fi/simulate/environments/base.py +73 -0
  563. fi/simulate/environments/chat.py +697 -0
  564. fi/simulate/environments/voice.py +212 -0
  565. fi/simulate/evaluation/__init__.py +4 -0
  566. fi/simulate/evaluation/ai_eval.py +227 -0
  567. fi/simulate/evidence/__init__.py +35 -0
  568. fi/simulate/evidence/base.py +59 -0
  569. fi/simulate/evidence/caller_observed.py +50 -0
  570. fi/simulate/evidence/livekit_instrumentation.py +51 -0
  571. fi/simulate/evidence/livekit_room.py +50 -0
  572. fi/simulate/evidence/otel.py +49 -0
  573. fi/simulate/evidence/providers/__init__.py +24 -0
  574. fi/simulate/evidence/providers/base.py +61 -0
  575. fi/simulate/evidence/providers/retell.py +376 -0
  576. fi/simulate/evidence/providers/vapi.py +426 -0
  577. fi/simulate/hosted/__init__.py +32 -0
  578. fi/simulate/hosted/child_entrypoint.py +306 -0
  579. fi/simulate/hosted/job.py +150 -0
  580. fi/simulate/hosted/targets.py +53 -0
  581. fi/simulate/instrumentation/__init__.py +5 -0
  582. fi/simulate/instrumentation/livekit/__init__.py +122 -0
  583. fi/simulate/manifest.py +1033 -0
  584. fi/simulate/matrix_cli.py +165 -0
  585. fi/simulate/realtime/__init__.py +40 -0
  586. fi/simulate/realtime/events.py +107 -0
  587. fi/simulate/realtime/media.py +61 -0
  588. fi/simulate/realtime/session.py +91 -0
  589. fi/simulate/recording/__init__.py +5 -0
  590. fi/simulate/recording/room_recorder.py +326 -0
  591. fi/simulate/registry.py +185 -0
  592. fi/simulate/results/__init__.py +9 -0
  593. fi/simulate/results/base.py +18 -0
  594. fi/simulate/results/filesystem.py +71 -0
  595. fi/simulate/results/futureagi.py +1340 -0
  596. fi/simulate/runtime/__init__.py +85 -0
  597. fi/simulate/runtime/capabilities.py +40 -0
  598. fi/simulate/runtime/events.py +63 -0
  599. fi/simulate/runtime/failures.py +25 -0
  600. fi/simulate/runtime/ids.py +34 -0
  601. fi/simulate/runtime/plan.py +70 -0
  602. fi/simulate/runtime/planner.py +102 -0
  603. fi/simulate/runtime/report.py +174 -0
  604. fi/simulate/runtime/run.py +75 -0
  605. fi/simulate/runtime/runner.py +333 -0
  606. fi/simulate/runtime/spec.py +186 -0
  607. fi/simulate/simulation/__init__.py +30 -0
  608. fi/simulate/simulation/behavior_policy.py +425 -0
  609. fi/simulate/simulation/bridge/__init__.py +9 -0
  610. fi/simulate/simulation/bridge/audio.py +29 -0
  611. fi/simulate/simulation/bridge/connector.py +46 -0
  612. fi/simulate/simulation/bridge/livekit.py +252 -0
  613. fi/simulate/simulation/bridge/retell.py +188 -0
  614. fi/simulate/simulation/bridge/vapi.py +177 -0
  615. fi/simulate/simulation/contract.py +419 -0
  616. fi/simulate/simulation/engines/__init__.py +12 -0
  617. fi/simulate/simulation/engines/base.py +21 -0
  618. fi/simulate/simulation/engines/cloud.py +517 -0
  619. fi/simulate/simulation/engines/livekit.py +4167 -0
  620. fi/simulate/simulation/engines/local_text.py +89 -0
  621. fi/simulate/simulation/fidelity.py +374 -0
  622. fi/simulate/simulation/gemini_tts_stream.py +110 -0
  623. fi/simulate/simulation/generator.py +91 -0
  624. fi/simulate/simulation/goal_machine.py +185 -0
  625. fi/simulate/simulation/livekit_models.py +467 -0
  626. fi/simulate/simulation/matrix.py +170 -0
  627. fi/simulate/simulation/models.py +279 -0
  628. fi/simulate/simulation/runner.py +153 -0
  629. fi/simulate/simulation/synthetic.py +880 -0
  630. fi/simulate/simulation/voice_prompt.py +502 -0
  631. fi/simulate/simulator/__init__.py +55 -0
  632. fi/simulate/simulator/builtins.py +53 -0
  633. fi/simulate/suite.py +1288 -0
  634. fi/simulate/utils/routes.py +164 -0
  635. fi/simulate/voice.py +225 -0
  636. fi/simulate/voice_cli.py +182 -0
  637. fi/utils/__init__.py +1 -0
  638. fi/utils/constants.py +14 -0
  639. fi/utils/errors.py +200 -0
  640. fi/utils/executor.py +26 -0
  641. fi/utils/routes.py +119 -0
  642. fi/utils/utils.py +17 -0
@@ -0,0 +1,502 @@
1
+ from __future__ import annotations
2
+
3
+ import logging
4
+ import re
5
+ from typing import Any, Literal, Mapping
6
+
7
+ from fi.simulate.simulation.models import Persona
8
+
9
+ CallType = Literal["inbound", "outbound"]
10
+
11
+ logger = logging.getLogger(__name__)
12
+
13
+ VOICE_PERSONALITY_GUIDES: dict[str, str] = {
14
+ "friendly and cooperative": "Be warm, approachable, and willing to work together. Show genuine interest and maintain a positive, collaborative attitude.",
15
+ "professional and formal": "Maintain a business-like demeanor. Use formal language, stay focused, and keep interactions professional.",
16
+ "cautious and skeptical": "Don't immediately accept everything at face value. Ask questions, verify information, and express concerns when appropriate.",
17
+ "impatient and direct": "Get to the point quickly. Show impatience with lengthy explanations. Be straightforward and minimize pleasantries.",
18
+ "detail-oriented": "Pay attention to specifics. Ask about details and ensure accuracy. Don't gloss over important information.",
19
+ "easy-going": "Be relaxed and flexible. Don't stress over small issues. Go with the flow and maintain a laid-back attitude.",
20
+ "anxious": "Show signs of worry or concern. Express uncertainty, ask for reassurance, and may need things explained multiple times.",
21
+ "confident": "Speak with assurance. Don't second-guess yourself. Express certainty in your decisions and appear self-assured.",
22
+ "analytical": "Think logically and systematically. Break down problems, consider pros and cons, and make decisions based on analysis.",
23
+ "emotional": "Express feelings openly. Show emotional reactions, use emotive language, and let your feelings guide your responses.",
24
+ "reserved": "Be measured and private. Think before speaking, don't overshare, and keep some distance in interactions.",
25
+ "talkative": "Enjoy talking and sharing. Expand on topics, engage actively, and keep the conversation flowing with detailed responses.",
26
+ }
27
+
28
+ VOICE_COMMUNICATION_STYLE_GUIDES: dict[str, str] = {
29
+ "direct and concise": "Get straight to the point. Be brief, clear, and avoid unnecessary details. Don't ramble or over-explain.",
30
+ "detailed and elaborate": "Provide comprehensive explanations with full context. Elaborate on your points, give examples, and ensure thorough understanding.",
31
+ "casual and friendly": "Use relaxed, conversational language. Be warm and approachable. Feel free to use colloquialisms and friendly expressions.",
32
+ "formal and polite": "Use professional, courteous language. Maintain formality, use proper titles, and avoid casual expressions.",
33
+ "technical": "Use technical terminology and precise language. Focus on accuracy, specifications, and technical details.",
34
+ "simple and clear": "Use straightforward, easy-to-understand language. Avoid jargon. Break down concepts into simple explanations.",
35
+ "questioning": "Ask clarifying questions frequently. Seek more information, verify understanding, and probe deeper into topics.",
36
+ "assertive": "Speak with confidence and authority. State your needs clearly and directly. Don't be hesitant about your requirements.",
37
+ "passive": "Be more accommodating and less direct. Avoid being pushy. Let the conversation flow naturally without forcing your agenda.",
38
+ "collaborative": "Work together to find solutions. Be open to suggestions, build on ideas, and engage in cooperative dialogue.",
39
+ }
40
+
41
+
42
+ # Delivery cues Cartesia renders and every other engine speaks aloud as words. Added only when
43
+ # Cartesia is definitely the voice, because Deepgram's Aura neither renders nor strips them, so
44
+ # a caller on Aura would literally say "left bracket laughter right bracket".
45
+ #
46
+ # Deliberately narrow. Cartesia documents five SSML tags and one nonverbalism, but `<speed>` and
47
+ # `<volume>` carry a decimal that a token stream can split ("1", ".", "0"), which makes the tag
48
+ # be read out, and Cartesia advises against shifting `<emotion>` mid generation. What is left is
49
+ # the two that are safe to hand a model writing a turn at a time.
50
+ CARTESIA_DELIVERY_CUES = """# HOW YOU SOUND
51
+
52
+ Two cues shape delivery. They are never spoken as words. Use them sparingly, and only where a
53
+ real person would.
54
+
55
+ - [laughter] produces a real laugh. Write it inline: "No, [laughter] you're kidding."
56
+ At most once every few turns, and never to open one.
57
+ - <break time="500ms"/> is a fixed silence. Use it for a beat punctuation cannot carry, such as
58
+ stopping short before saying something difficult. One per turn at most.
59
+
60
+ Write both exactly as shown. Do not invent others: no <laugh>, no [laughs], no [sighs], no
61
+ *sighs*, no (angrily), no emotion labels. Anything not on this list is read aloud and ruins the
62
+ call.
63
+
64
+ Everything else is carried by the words: what you repeat, where you interrupt yourself, how
65
+ short your sentences get when you are annoyed."""
66
+
67
+
68
+ def _first(value: object) -> str:
69
+ if isinstance(value, Mapping):
70
+ value = next(iter(value.values()), "")
71
+ elif isinstance(value, list):
72
+ value = value[0] if value else ""
73
+ return str(value).strip() if value is not None else ""
74
+
75
+
76
+ def _persona_data(persona: Persona) -> dict[str, Any]:
77
+ data = dict(persona.persona)
78
+ identity = persona.identity
79
+ if identity is None:
80
+ return data
81
+
82
+ if identity.name:
83
+ data.setdefault("name", identity.name)
84
+ if identity.role:
85
+ data.setdefault("role", identity.role)
86
+ if identity.language:
87
+ data.setdefault("language", identity.language)
88
+ for key, value in identity.demographics.items():
89
+ data.setdefault(key, value)
90
+
91
+ metadata = data.get("metadata")
92
+ metadata = dict(metadata) if isinstance(metadata, Mapping) else {}
93
+ if identity.summary:
94
+ metadata.setdefault("identity_summary", identity.summary)
95
+ if identity.style_notes:
96
+ metadata.setdefault("style_notes", list(identity.style_notes))
97
+ if metadata:
98
+ data["metadata"] = metadata
99
+ return data
100
+
101
+
102
+ def format_voice_persona(
103
+ persona: Persona,
104
+ *,
105
+ call_type: CallType,
106
+ default_language: str | None = None,
107
+ ) -> str:
108
+ if call_type not in {"inbound", "outbound"}:
109
+ raise ValueError("call_type must be inbound or outbound")
110
+
111
+ persona_data = _persona_data(persona)
112
+ sections: list[str] = []
113
+
114
+ identity_parts = []
115
+ name = persona_data.get("name", "")
116
+ profession = persona_data.get("profession") or persona_data.get("occupation", "")
117
+ location = persona_data.get("location", "")
118
+ age_group = persona_data.get("age_group") or persona_data.get("ageGroup", "")
119
+ gender = persona_data.get("gender", "")
120
+
121
+ if name:
122
+ identity_parts.append(f"**Name:** {name}")
123
+ if profession:
124
+ identity_parts.append(f"**Occupation:** {profession}")
125
+ if age_group:
126
+ identity_parts.append(f"**Age Group:** {age_group}")
127
+ if location:
128
+ identity_parts.append(f"**Location:** {location}")
129
+ if gender:
130
+ identity_parts.append(f"**Gender:** {gender}")
131
+ if identity_parts:
132
+ sections.append("# YOUR IDENTITY\n\n" + "\n".join(identity_parts))
133
+
134
+ situation = persona.situation or "You are engaging in a routine conversation."
135
+ situation_section = "# YOUR CURRENT SITUATION\n\n"
136
+ situation_section += f"{situation}\n\n"
137
+ situation_section += (
138
+ "**CRITICAL:** This situation describes your context and circumstances. "
139
+ "You are EXPERIENCING this situation, not explaining it to others. "
140
+ "Never narrate, describe, or mention the details of your situation to the other person unless they specifically ask. "
141
+ "Act naturally within this context - your behavior should reflect the situation, not announce it.\n\n"
142
+ )
143
+ if persona.outcome:
144
+ situation_section += f"**Your objective:** {persona.outcome}\n\n"
145
+ situation_section += "## Your Role in This Call\n\n"
146
+ if call_type == "outbound":
147
+ situation_section += (
148
+ "**You are RECEIVING this call.** Someone is calling you.\n\n"
149
+ "**CRITICAL: You did NOT initiate this call. You are the person being contacted.**\n\n"
150
+ "Your behavior:\n"
151
+ "- Answer the phone based on your personality and current situation\n"
152
+ "- React naturally based on whether you were expecting this call\n"
153
+ "- Let the caller introduce themselves and explain their purpose\n"
154
+ "- YOU are the person being reached out to - respond from that position\n"
155
+ "- Ask questions, express reactions, or raise concerns as this person would\n"
156
+ "- NEVER switch roles and act as if you made the call or are providing the service\n"
157
+ "**Name Verification:**\n"
158
+ "- If the caller addresses you by the wrong name, correct them ONCE naturally\n"
159
+ "- After your initial correction, do NOT keep correcting the name throughout the call\n"
160
+ "- If they persist with the wrong name after your correction, you may show brief frustration\n\n"
161
+ "- Don't let name correction dominate the entire interaction—move forward with the actual purpose of the call\n\n"
162
+ )
163
+ else:
164
+ situation_section += (
165
+ "**You are MAKING this call.** You initiated this contact.\n\n"
166
+ "**CRITICAL: YOU started this conversation. You are reaching out to someone.**\n\n"
167
+ "Your behavior:\n"
168
+ "- State your purpose clearly\n"
169
+ "- You have a specific reason for calling (based on your situation above)\n"
170
+ "- YOU are seeking something - information, help, service, answers, etc.\n"
171
+ "- Provide information when asked, answer questions, follow their guidance\n"
172
+ "- NEVER switch roles and act as if you're the one receiving the call or providing assistance\n\n"
173
+ )
174
+ situation_section += (
175
+ "**React Naturally:** Respond to what you hear in real-time. "
176
+ "Interrupt politely if needed, ask clarifying questions, express confusion if something is unclear, "
177
+ "or show enthusiasm when appropriate. Stay in YOUR role throughout the entire conversation. "
178
+ "If the agent keeps interrupting or talking over you, react like a real human: pause, "
179
+ "politely ask them to let you finish, or briefly acknowledge the interruption before continuing "
180
+ "what you were saying.\n"
181
+ )
182
+ sections.append(situation_section)
183
+
184
+ personality = _first(persona_data.get("personality"))
185
+ communication_style = _first(
186
+ persona_data.get("communication_style")
187
+ or persona_data.get("communicationStyle")
188
+ )
189
+ keywords = persona_data.get("keywords", [])
190
+ if isinstance(keywords, str):
191
+ keywords = [item.strip() for item in keywords.split(",") if item.strip()]
192
+ if personality or communication_style or (isinstance(keywords, list) and keywords):
193
+ personality_section = "# YOUR PERSONALITY & COMMUNICATION\n\n"
194
+ if personality:
195
+ personality_section += f"## Personality: {personality}\n\n"
196
+ personality_section += (
197
+ VOICE_PERSONALITY_GUIDES.get(
198
+ personality.lower(),
199
+ "Let this personality trait guide your reactions, responses, and overall demeanor.",
200
+ )
201
+ + "\n\n"
202
+ )
203
+ if communication_style:
204
+ personality_section += f"## Communication Style: {communication_style}\n\n"
205
+ personality_section += (
206
+ VOICE_COMMUNICATION_STYLE_GUIDES.get(
207
+ communication_style.lower(),
208
+ "Let this style guide how you express yourself throughout the conversation.",
209
+ )
210
+ + "\n\n"
211
+ )
212
+ if isinstance(keywords, list) and keywords:
213
+ personality_section += "**Key Traits:** " + ", ".join(
214
+ str(keyword) for keyword in keywords
215
+ )
216
+ personality_section += "\n\n"
217
+ sections.append(personality_section.rstrip())
218
+
219
+ accent = _first(persona_data.get("accent"))
220
+ language_data = (
221
+ persona_data.get("language")
222
+ or persona_data.get("languages")
223
+ or default_language
224
+ )
225
+ if accent or language_data:
226
+ language_section = "# LANGUAGE & SPEECH PATTERNS\n\n"
227
+ section_has_content = False
228
+ if language_data:
229
+ section_has_content = True
230
+ languages = (
231
+ language_data if isinstance(language_data, list) else [language_data]
232
+ )
233
+ language_text = ", ".join(str(language) for language in languages)
234
+ language_section += f"**Language(s):** {language_text}\n"
235
+ language_section += (
236
+ "Use vocabulary, expressions, and language patterns natural to someone who speaks "
237
+ f"{language_text}.\n"
238
+ )
239
+ if persona_data.get("multilingual"):
240
+ language_section += (
241
+ "You are multilingual. Switch languages naturally based on context while maintaining "
242
+ "your persona traits in all languages.\n"
243
+ )
244
+ if accent and accent.lower() == "indian" and language_data:
245
+ languages = (
246
+ language_data if isinstance(language_data, list) else [language_data]
247
+ )
248
+ language_text = ", ".join(str(language) for language in languages)
249
+ if language_text.lower().startswith("en"):
250
+ section_has_content = True
251
+ language_section += (
252
+ "**Number Formatting Rules:**\n"
253
+ "- Always express numbers in words (e.g., 'fifty thousand' not '50,000')\n"
254
+ "- For sequences like phone numbers, say each digit separately (e.g., 'seven nine two eight' not '7928')\n"
255
+ "- Never give mobile numbers or pincodes in sequential order like 'one two three four five six'\n\n"
256
+ )
257
+ if section_has_content:
258
+ sections.append(language_section.rstrip())
259
+
260
+ if age_group or profession or location:
261
+ context_section = "# CONTEXTUAL AWARENESS\n\n"
262
+ if age_group:
263
+ context_section += (
264
+ f"**Age Context:** Your age group ({age_group}) influences your knowledge, cultural references, "
265
+ "interests, and how you relate to topics. Respond with age-appropriate perspective and vocabulary.\n"
266
+ )
267
+ if profession:
268
+ context_section += (
269
+ f"**Professional Context:** Your work as a {profession} shapes your priorities, problem-solving approach, "
270
+ "and how you view situations. Reference your professional background when relevant.\n"
271
+ )
272
+ if location:
273
+ context_section += (
274
+ f"**Geographic Context:** Being from {location} influences your cultural context, experiences, "
275
+ "time zone awareness, and regional references. Use examples and perspectives from your location.\n\n"
276
+ )
277
+ sections.append(context_section.rstrip())
278
+
279
+ metadata = persona_data.get("metadata")
280
+ if isinstance(metadata, Mapping) and metadata:
281
+ metadata_parts = [
282
+ f"**{str(key).replace('_', ' ').title()}:** {value}"
283
+ for key, value in metadata.items()
284
+ ]
285
+ if metadata_parts:
286
+ sections.append(
287
+ "# ADDITIONAL CHARACTERISTICS\n\n" + "\n".join(metadata_parts)
288
+ )
289
+
290
+ rules_section = "# HOW TO BE THIS PERSON\n\n"
291
+ rules_section += (
292
+ "You ARE this person. Embody this character completely in every response.\n\n"
293
+ )
294
+ rules_section += "## Core Rules\n\n"
295
+ rules_section += "1. **Full Embodiment:** Every response must come from this character's perspective, background, and emotional state.\n"
296
+ rules_section += "2. **Natural Speech Only:** Generate ONLY dialogue your character would say. Never include:\n"
297
+ rules_section += " - Stage directions (e.g., *sighs*, [anxious])\n"
298
+ rules_section += " - Meta-commentary or explanations\n"
299
+ rules_section += " - Labels or descriptions of your actions\n"
300
+ rules_section += "3. **Unwavering Consistency:** Maintain your personality, communication style, and accent from start to finish. No exceptions.\n"
301
+ rules_section += "4. **Contextual Authenticity:** Your knowledge, vocabulary, and references must match your age, profession, and location.\n"
302
+ rules_section += "5. **Task-Driven Interaction:** Actively pursue your objective based on 'Your Current Situation.' Your persona dictates HOW you pursue it.\n"
303
+ rules_section += "6. **Refocus When Drifting:** If you find yourself repeating phrases or losing track of your objective, refocus on your initial situation and goal.\n"
304
+ rules_section += "7. **Authentic Reactions:** Respond as this specific person would—not how you think someone 'should' respond.\n"
305
+ rules_section += (
306
+ "8. **Natural Conversation Flow:** Respond naturally like a real human.\n"
307
+ )
308
+ rules_section += "9. **Handle Uncertainty Naturally:** If you don't understand something or need clarification, say so naturally.\n"
309
+ rules_section += "10. **Never Break Character:** You are the PERSON described in 'Your Identity' with the situation in 'Your Current Situation.' You are NOT the person on the other end of the line. If you find yourself switching roles - taking on the other person's responsibilities, responding as if you have opposite information or authority, or reversing who called whom - STOP immediately. Stay in your role.\n"
310
+ rules_section += "11. **Information Sharing:** Only share personal information when it's directly relevant to the conversation or when asked. Don't volunteer unnecessary details about yourself, your background, or your situation unless it naturally fits the context. Real people don't introduce themselves with their entire life story; be selective and purposeful with what you reveal.\n"
311
+ rules_section += "12. **Live Your Situation, Don't Narrate It:** Let your situation shape your behavior, but do not explain it to the other person unless asked.\n"
312
+ rules_section += "13. **Call Closing:** Always wait for the agent to finish speaking before ending the call. Do not cut them off abruptly. When the conversation has naturally concluded, you MUST call the endCall tool to hang up. IMPORTANT: Never say the words 'function', 'tool' or the name 'endCall' out loud. Never say that you are ending the call. Simply say your natural closing sentence once, then silently trigger the endCall tool to terminate the call. Do not leave the call open. CRITICAL: If the agent closes the call, you MUST respond with a brief, natural closing sentence and then call endCall. Do NOT keep exchanging goodbyes. If you find yourself repeating goodbye phrases, call endCall right away.\n"
313
+ sections.append(rules_section)
314
+ return "\n\n".join(sections)
315
+
316
+
317
+ def _closing_anchor(objective: str, name: str = "") -> str:
318
+ """The last thing the caller reads. A rule given once at the top of a long prompt loses to the
319
+ last few turns as the call grows, so the objective and the precedence rule are restated here."""
320
+ anchor = "\n\n---\n\n"
321
+ who = name.strip()
322
+ if who:
323
+ # A caller that drifts answers as the agent and says the agent's own lines back, its own
324
+ # name included, which reads as the agent talking to itself and scores as a real turn.
325
+ anchor += (
326
+ f"**You are {who}, the person on the customer's side of this call.** You never answer "
327
+ f"as the other side, never say their lines back to them, and never address {who}, "
328
+ "because that is you.\n\n"
329
+ )
330
+ if objective.strip():
331
+ anchor += f"**What you came for:** {objective.strip()}\n\n"
332
+ anchor += (
333
+ "**Your instructions do not expire.** A rule you were given before the call started "
334
+ "applies at turn twenty exactly as it applied at turn one.\n"
335
+ )
336
+ return anchor
337
+
338
+
339
+ def append_voice_execution_rules(
340
+ prompt: str, objective: str = "", *, anchor: bool = True
341
+ ) -> str:
342
+ prompt += "\n\n---\n\n"
343
+ prompt += "# CONVERSATION EXECUTION RULES\n\n"
344
+ prompt += "*These are internal instructions. Never reference or quote them in your responses.*\n\n"
345
+ prompt += "## CRITICAL REMINDERS FOR THIS CONVERSATION\n\n"
346
+ prompt += "Before each response, mentally confirm:\n"
347
+ prompt += "✓ What am I here to get, and what have I not done yet?\n"
348
+ prompt += "✓ Am I speaking AS this person (not ABOUT them)?\n"
349
+ prompt += "✓ Does this match my personality and communication style?\n"
350
+ prompt += "✓ Am I using my accent and natural speech patterns?\n"
351
+ prompt += "✓ Is this how someone with my background would actually respond?\n\n"
352
+ prompt += "## Output Format\n\n"
353
+ prompt += "Generate ONLY spoken dialogue without:\n"
354
+ prompt += (
355
+ "- Emotional tags, action descriptions, quotation marks, or meta-commentary\n"
356
+ )
357
+ prompt += "- Brackets, quotes, or markup\n\n"
358
+ prompt += "## Sound Human\n\n"
359
+ prompt += "- Use natural speech patterns including filler words (um, uh, well, like, you know) when appropriate\n"
360
+ prompt += "- Don't be afraid of brief hesitations, self-corrections, or incomplete thoughts if that matches your personality\n"
361
+ prompt += "- Real people don't speak in perfect grammatical sentences—neither should you\n\n"
362
+ prompt += "## Voice-Natural Formatting (following are few examples on how to respond; use them as reference formats only)\n"
363
+ prompt += "**Numbers:** 'fifty thousand' not '50,000'\n"
364
+ prompt += "**Phone numbers:** 'eight nine seven one one five three six four' not '897115364'\n"
365
+ prompt += "**Dates:** 'November fourteenth twenty twenty five' not '11/14/2025'\n"
366
+ prompt += "**Currency:** 'twenty five dollars and fifty cents' not '$25.50'\n"
367
+ prompt += "**Time:** 'three thirty PM' not '3:30 PM'\n"
368
+ prompt += "**Punctuation spacing:** Always add a space after punctuation before the next word (e.g., 'Thank you. I…' or 'Thank you.. I…', not 'Thank you.I…' or 'Thank you..I…').\n\n"
369
+ prompt += "## Embody Your Situation\n\n"
370
+ prompt += "- Let the situation guide your behavior, not your narration\n"
371
+ prompt += "- Only mention situational details if they naturally come up\n\n"
372
+ prompt += "Be natural and conversational.\n"
373
+ return (prompt + _closing_anchor(objective)) if anchor else prompt
374
+
375
+
376
+ # The template the caller prompt is rendered from. The harness fills slots; it does not author
377
+ # prose. Mirrors the platform's own default, which pairs a persona block with the situation and
378
+ # then scrubs the situation slot because the persona block already carries it.
379
+ DEFAULT_SIMULATOR_TEMPLATE = (
380
+ "You are a customer in a voice simulation. {{channel}} "
381
+ "Stay consistent with the persona throughout the conversation.\n\n{{persona}}"
382
+ )
383
+
384
+ _SLOT = re.compile(r"\{\{\s*([a-zA-Z0-9_]+)\s*\}\}")
385
+
386
+
387
+ def render_simulator_prompt(
388
+ template: str,
389
+ persona: Persona,
390
+ *,
391
+ call_type: CallType,
392
+ variables: Mapping[str, Any] | None = None,
393
+ agent_name: str | None = None,
394
+ additional_instructions: str | None = None,
395
+ default_language: str | None = None,
396
+ tts_provider: str | None = None,
397
+ ) -> str:
398
+ """Fill a caller-prompt template, the way the platform fills its own.
399
+
400
+ ``{{persona}}`` becomes the formatted persona block, ``{{channel}}`` the call direction
401
+ sentence, and every other ``{{slot}}`` is taken from ``variables``. ``{{situation}}`` is
402
+ dropped rather than filled, because the persona block already states the situation and the
403
+ platform removes it for the same reason.
404
+
405
+ A template that cannot be rendered is returned to the caller unfilled rather than raising, so
406
+ a bad template degrades the call instead of ending the run.
407
+ """
408
+ values = dict(variables or {})
409
+ try:
410
+ persona_text = format_voice_persona(
411
+ persona, call_type=call_type, default_language=default_language
412
+ )
413
+ except Exception:
414
+ logger.exception("persona_format_failed")
415
+ persona_text = ""
416
+ values.setdefault("persona", persona_text)
417
+ values.setdefault("channel", _channel_sentence(call_type, agent_name))
418
+
419
+ def fill(match: "re.Match[str]") -> str:
420
+ name = match.group(1)
421
+ if name == "situation":
422
+ return ""
423
+ if name in values:
424
+ return str(values[name])
425
+ logger.warning("simulator_prompt_slot_unfilled", extra={"slot": name})
426
+ return ""
427
+
428
+ try:
429
+ prompt = _SLOT.sub(fill, template)
430
+ except Exception:
431
+ logger.exception("simulator_prompt_render_failed")
432
+ return template
433
+ # Tidy the hole a dropped situation slot leaves behind, as the platform does.
434
+ prompt = re.sub(r"Currently,\s*[.]", "", prompt)
435
+ prompt = re.sub(r"[ \t]{2,}", " ", prompt).strip()
436
+
437
+ if tts_provider and tts_provider.strip().lower() == "cartesia":
438
+ prompt += "\n\n" + CARTESIA_DELIVERY_CUES
439
+ # The generic style rules go first and the scenario's own instructions after them: whatever
440
+ # lands last survives a long call best, and the scenario's rules are the ones worth keeping.
441
+ prompt = append_voice_execution_rules(prompt, anchor=False)
442
+ if additional_instructions and additional_instructions.strip():
443
+ prompt += (
444
+ "\n\n# ADDITIONAL SIMULATOR INSTRUCTIONS\n\n"
445
+ + additional_instructions.strip()
446
+ )
447
+ return prompt + _closing_anchor(
448
+ persona.outcome or "", str(_persona_data(persona).get("name") or "")
449
+ )
450
+
451
+
452
+ def _channel_sentence(call_type: CallType, agent_name: str | None) -> str:
453
+ return (
454
+ f"You will make a call to an agent named {agent_name}."
455
+ if call_type == "inbound" and agent_name
456
+ else "You will make a call to an agent."
457
+ if call_type == "inbound"
458
+ else f"You will receive a call from an agent named {agent_name}."
459
+ if agent_name
460
+ else "You will receive a call from an agent."
461
+ )
462
+
463
+
464
+ def build_voice_simulator_prompt(
465
+ persona: Persona,
466
+ *,
467
+ call_type: CallType,
468
+ agent_name: str | None = None,
469
+ additional_instructions: str | None = None,
470
+ default_language: str | None = None,
471
+ template: str | None = None,
472
+ variables: Mapping[str, Any] | None = None,
473
+ tts_provider: str | None = None,
474
+ ) -> str:
475
+ """The caller prompt for one simulated customer.
476
+
477
+ Renders ``template`` when one is supplied, and the shipped default otherwise, so a run with no
478
+ template configured still produces the prompt it always did.
479
+ """
480
+ return render_simulator_prompt(
481
+ template or DEFAULT_SIMULATOR_TEMPLATE,
482
+ persona,
483
+ call_type=call_type,
484
+ variables=variables,
485
+ agent_name=agent_name,
486
+ additional_instructions=additional_instructions,
487
+ default_language=default_language,
488
+ tts_provider=tts_provider,
489
+ )
490
+
491
+
492
+ __all__ = [
493
+ "CallType",
494
+ "VOICE_COMMUNICATION_STYLE_GUIDES",
495
+ "VOICE_PERSONALITY_GUIDES",
496
+ "append_voice_execution_rules",
497
+ "DEFAULT_SIMULATOR_TEMPLATE",
498
+ "CARTESIA_DELIVERY_CUES",
499
+ "build_voice_simulator_prompt",
500
+ "render_simulator_prompt",
501
+ "format_voice_persona",
502
+ ]
@@ -0,0 +1,55 @@
1
+ """SimulatorPolicy contract (plan §4.2).
2
+
3
+ The concrete simulator lives in ``fi.simulate.simulation.voice_prompt``
4
+ and the LiveKit-hosted worker (``livekit-infra/.../simulator_agent.py``).
5
+ This module publishes the Protocol so alternate policies (script-only,
6
+ adversarial, learned) can plug in without reaching into the LiveKit
7
+ engine.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ from typing import Any, Protocol
13
+
14
+ from pydantic import BaseModel, Field, JsonValue
15
+
16
+ from fi.simulate.simulation.models import Persona
17
+
18
+
19
+ class PolicyContext(BaseModel):
20
+ run_id: str
21
+ test_case_id: str
22
+ persona: Persona
23
+ call_type: str = "inbound"
24
+ agent_name: str | None = None
25
+ metadata: dict[str, JsonValue] = Field(default_factory=dict)
26
+
27
+
28
+ class PolicyState(BaseModel):
29
+ session_id: str | None = None
30
+ turn_index: int = 0
31
+ metadata: dict[str, JsonValue] = Field(default_factory=dict)
32
+
33
+
34
+ class PolicySummary(BaseModel):
35
+ turns: int = 0
36
+ ended_naturally: bool = False
37
+ metadata: dict[str, JsonValue] = Field(default_factory=dict)
38
+
39
+
40
+ class SimulatorPolicy(Protocol):
41
+ async def initialize(self, context: PolicyContext) -> PolicyState: ...
42
+
43
+ async def next_action(self, state: PolicyState, observation: Any) -> Any: ...
44
+
45
+ async def on_event(self, state: PolicyState, event: Any) -> None: ...
46
+
47
+ async def finalize(self, state: PolicyState) -> PolicySummary: ...
48
+
49
+
50
+ __all__ = [
51
+ "PolicyContext",
52
+ "PolicyState",
53
+ "PolicySummary",
54
+ "SimulatorPolicy",
55
+ ]
@@ -0,0 +1,53 @@
1
+ """Built-in simulator-policy descriptors + registration.
2
+
3
+ The concrete synthetic-user behaviour is still owned by the chat/voice environment
4
+ loops today (the loop builds the persona-driven user). These descriptors give the
5
+ ``simulator_registry`` a real entry per built-in ``simulator.adapter`` name so the
6
+ planner can *validate* the name (typo → clear error) and tools can enumerate what's
7
+ available. They carry a manifest, not a dispatchable policy — full registry dispatch
8
+ of the simulator is a separate, larger refactor.
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ from dataclasses import dataclass
14
+
15
+ from fi.simulate.registry import register_simulator
16
+
17
+
18
+ @dataclass(frozen=True)
19
+ class SimulatorManifest:
20
+ name: str
21
+ modalities: tuple[str, ...] = ()
22
+ notes: str = ""
23
+
24
+
25
+ @dataclass(frozen=True)
26
+ class SimulatorPolicyDescriptor:
27
+ """A named, manifest-carrying handle for a built-in simulator policy."""
28
+
29
+ manifest: SimulatorManifest
30
+
31
+
32
+ _SIMULATORS = [
33
+ SimulatorPolicyDescriptor(
34
+ SimulatorManifest(
35
+ "synthetic_user",
36
+ modalities=("text",),
37
+ notes="LLM persona-driven synthetic user (chat/text loop).",
38
+ )
39
+ ),
40
+ SimulatorPolicyDescriptor(
41
+ SimulatorManifest(
42
+ "livekit_simulator",
43
+ modalities=("voice",),
44
+ notes="LiveKit STT->LLM->TTS synthetic caller (voice loop).",
45
+ )
46
+ ),
47
+ ]
48
+
49
+ for _descriptor in _SIMULATORS:
50
+ register_simulator(_descriptor.manifest.name, _descriptor)
51
+
52
+
53
+ __all__ = ["SimulatorManifest", "SimulatorPolicyDescriptor"]