agent-learning-kit 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (642) hide show
  1. agent_learning_kit-0.1.0.dist-info/METADATA +381 -0
  2. agent_learning_kit-0.1.0.dist-info/RECORD +642 -0
  3. agent_learning_kit-0.1.0.dist-info/WHEEL +4 -0
  4. agent_learning_kit-0.1.0.dist-info/entry_points.txt +5 -0
  5. agent_learning_kit-0.1.0.dist-info/licenses/LICENSE +173 -0
  6. agent_learning_kit-0.1.0.dist-info/licenses/NOTICE +7 -0
  7. fi/__init__.py +5 -0
  8. fi/alk/__init__.py +57 -0
  9. fi/alk/_facade.py +31 -0
  10. fi/alk/_module_alias.py +68 -0
  11. fi/alk/_paths.py +14 -0
  12. fi/alk/_schema.py +522 -0
  13. fi/alk/actions.py +727 -0
  14. fi/alk/bench/__init__.py +517 -0
  15. fi/alk/bench/_codeexec.py +213 -0
  16. fi/alk/bench/_coding.py +215 -0
  17. fi/alk/bench/_docker.py +237 -0
  18. fi/alk/bench/_grader.py +286 -0
  19. fi/alk/bench/_pull.py +212 -0
  20. fi/alk/bench/_voice.py +147 -0
  21. fi/alk/capabilities.py +627 -0
  22. fi/alk/cli.py +6396 -0
  23. fi/alk/config.py +130 -0
  24. fi/alk/cua_loop.py +562 -0
  25. fi/alk/evals.py +2351 -0
  26. fi/alk/extensions.py +163 -0
  27. fi/alk/harness/ARCHITECTURE.md +231 -0
  28. fi/alk/harness/DESIGN.md +246 -0
  29. fi/alk/harness/ENVIRONMENT_CONFORMANCE.md +127 -0
  30. fi/alk/harness/HOW-IT-WORKS.md +297 -0
  31. fi/alk/harness/IMPLEMENTATION_AND_VALIDATION_STATUS.md +229 -0
  32. fi/alk/harness/README.md +417 -0
  33. fi/alk/harness/__init__.py +77 -0
  34. fi/alk/harness/__main__.py +3 -0
  35. fi/alk/harness/amend.py +312 -0
  36. fi/alk/harness/artifacts.py +319 -0
  37. fi/alk/harness/authoring_entrypoint.py +189 -0
  38. fi/alk/harness/authoring_runtime_validation.py +267 -0
  39. fi/alk/harness/backends/README.md +43 -0
  40. fi/alk/harness/backends/__init__.py +122 -0
  41. fi/alk/harness/backends/base.py +241 -0
  42. fi/alk/harness/backends/claude.py +211 -0
  43. fi/alk/harness/backends/files.py +182 -0
  44. fi/alk/harness/backends/vertex_gemini.py +457 -0
  45. fi/alk/harness/background_noise.py +95 -0
  46. fi/alk/harness/build.py +385 -0
  47. fi/alk/harness/bundle.py +593 -0
  48. fi/alk/harness/bundle_author_v2.py +1831 -0
  49. fi/alk/harness/bundle_v2.py +719 -0
  50. fi/alk/harness/call_runner.py +1440 -0
  51. fi/alk/harness/callback_http_adapter.py +111 -0
  52. fi/alk/harness/catalogue.py +287 -0
  53. fi/alk/harness/chat.py +428 -0
  54. fi/alk/harness/chat_call_runner.py +506 -0
  55. fi/alk/harness/checks.py +136 -0
  56. fi/alk/harness/cli.py +1354 -0
  57. fi/alk/harness/config.py +338 -0
  58. fi/alk/harness/contract.py +718 -0
  59. fi/alk/harness/credentials.py +674 -0
  60. fi/alk/harness/data/persona_vocabulary.json +111 -0
  61. fi/alk/harness/environment.py +99 -0
  62. fi/alk/harness/environment_plan.py +168 -0
  63. fi/alk/harness/events.py +125 -0
  64. fi/alk/harness/executor.py +304 -0
  65. fi/alk/harness/folder.py +234 -0
  66. fi/alk/harness/generated_runtime.py +815 -0
  67. fi/alk/harness/github.py +72 -0
  68. fi/alk/harness/hosted_authoring_entrypoint.py +183 -0
  69. fi/alk/harness/hosted_entrypoint.py +2402 -0
  70. fi/alk/harness/hosted_scheduler.py +2218 -0
  71. fi/alk/harness/job.py +426 -0
  72. fi/alk/harness/judge.py +184 -0
  73. fi/alk/harness/livekit_source.py +50 -0
  74. fi/alk/harness/livekit_tool_trace_bootstrap.py +71 -0
  75. fi/alk/harness/observability.py +208 -0
  76. fi/alk/harness/outbound.py +3252 -0
  77. fi/alk/harness/packaging.py +515 -0
  78. fi/alk/harness/persona_guides.py +157 -0
  79. fi/alk/harness/platform.py +692 -0
  80. fi/alk/harness/process_preflight.py +764 -0
  81. fi/alk/harness/process_runtime.py +5670 -0
  82. fi/alk/harness/prove.py +425 -0
  83. fi/alk/harness/provider_import.py +703 -0
  84. fi/alk/harness/provider_lifecycle.py +392 -0
  85. fi/alk/harness/provision.py +2896 -0
  86. fi/alk/harness/reception.py +147 -0
  87. fi/alk/harness/retell_chat_call_runner.py +373 -0
  88. fi/alk/harness/run/__init__.py +296 -0
  89. fi/alk/harness/run/alk.py +184 -0
  90. fi/alk/harness/run/call.py +162 -0
  91. fi/alk/harness/run/conversation.py +264 -0
  92. fi/alk/harness/run/data/voices_by_language_and_gender.json +693 -0
  93. fi/alk/harness/run/evidence.py +195 -0
  94. fi/alk/harness/run/grade.py +598 -0
  95. fi/alk/harness/run/live.py +297 -0
  96. fi/alk/harness/run/models.py +56 -0
  97. fi/alk/harness/run/platform_evals.py +227 -0
  98. fi/alk/harness/run/sdk_voice.py +130 -0
  99. fi/alk/harness/run/simulation.py +1209 -0
  100. fi/alk/harness/run/stage.py +91 -0
  101. fi/alk/harness/run/targets.py +508 -0
  102. fi/alk/harness/run/tools.py +601 -0
  103. fi/alk/harness/run/voice.py +340 -0
  104. fi/alk/harness/runtime.py +172 -0
  105. fi/alk/harness/sandbox_server.py +2011 -0
  106. fi/alk/harness/sandbox_worker.py +44 -0
  107. fi/alk/harness/scenario.py +1048 -0
  108. fi/alk/harness/scenario_source.py +879 -0
  109. fi/alk/harness/scenario_tools.py +1143 -0
  110. fi/alk/harness/scenarios.py +915 -0
  111. fi/alk/harness/secrets.py +168 -0
  112. fi/alk/harness/service_catalog.py +97 -0
  113. fi/alk/harness/session.py +391 -0
  114. fi/alk/harness/sessions.py +372 -0
  115. fi/alk/harness/simulator.py +76 -0
  116. fi/alk/harness/simulator_voice.py +928 -0
  117. fi/alk/harness/skills/build-environment/SKILL.md +538 -0
  118. fi/alk/harness/skills/harness.md +131 -0
  119. fi/alk/harness/skills/kinds/chat.md +48 -0
  120. fi/alk/harness/skills/kinds/voice-voicemail.md +63 -0
  121. fi/alk/harness/skills/kinds/voice.md +59 -0
  122. fi/alk/harness/skills/plan-suite/SKILL.md +103 -0
  123. fi/alk/harness/skills/provision-environment/SKILL.md +136 -0
  124. fi/alk/harness/skills/run-scenarios/SKILL.md +112 -0
  125. fi/alk/harness/skills/understand-agent/SKILL.md +251 -0
  126. fi/alk/harness/skills/write-scenarios/SKILL.md +606 -0
  127. fi/alk/harness/skills/write-scenarios/references/refusals.md +28 -0
  128. fi/alk/harness/skills/write-scenarios/references/world-api.md +92 -0
  129. fi/alk/harness/source_data_invariants.py +444 -0
  130. fi/alk/harness/source_tool_evidence.py +79 -0
  131. fi/alk/harness/sources.py +253 -0
  132. fi/alk/harness/spend.py +140 -0
  133. fi/alk/harness/tool_trace_proxy.py +104 -0
  134. fi/alk/harness/tools.py +1018 -0
  135. fi/alk/harness/understand.py +169 -0
  136. fi/alk/harness/voicemail_audio.py +74 -0
  137. fi/alk/harness/world/__init__.py +33 -0
  138. fi/alk/harness/world/errors.py +68 -0
  139. fi/alk/harness/world/expectations.py +91 -0
  140. fi/alk/harness/world/handle.py +538 -0
  141. fi/alk/harness/world/kinds.py +196 -0
  142. fi/alk/harness/world/mutate.py +186 -0
  143. fi/alk/harness/world/probe.py +413 -0
  144. fi/alk/harness/world/provision.py +511 -0
  145. fi/alk/harness/world/provisioned.py +191 -0
  146. fi/alk/harness/world/runtime.py +616 -0
  147. fi/alk/harness/world/snapshot.py +288 -0
  148. fi/alk/harness/world/stores/__init__.py +305 -0
  149. fi/alk/harness/world/stores/container.py +215 -0
  150. fi/alk/harness/world/stores/inprocess.py +346 -0
  151. fi/alk/harness/world/stores/postgres.py +481 -0
  152. fi/alk/harness/world/stores/prove.py +202 -0
  153. fi/alk/harness/world/stores/sqlite.py +245 -0
  154. fi/alk/harness/world/stores/written.py +182 -0
  155. fi/alk/harness/world/tools.py +1516 -0
  156. fi/alk/harness/world/workspace.py +144 -0
  157. fi/alk/image_loop.py +453 -0
  158. fi/alk/image_perturb.py +241 -0
  159. fi/alk/improve.py +274 -0
  160. fi/alk/live/__init__.py +154 -0
  161. fi/alk/live/_attribution.py +184 -0
  162. fi/alk/live/_capture.py +264 -0
  163. fi/alk/live/_codec.py +391 -0
  164. fi/alk/live/_contract.py +134 -0
  165. fi/alk/live/_loopback.py +316 -0
  166. fi/alk/live/_perturb.py +449 -0
  167. fi/alk/live/_runner.py +386 -0
  168. fi/alk/live/_stats.py +561 -0
  169. fi/alk/live/_transcript.py +240 -0
  170. fi/alk/live/_workers/__init__.py +9 -0
  171. fi/alk/live/_workers/a2a_worker.py +316 -0
  172. fi/alk/live/_workers/langgraph_worker.py +217 -0
  173. fi/alk/live/_workers/livekit_worker.py +207 -0
  174. fi/alk/live/_workers/mcp_loopback_server.py +46 -0
  175. fi/alk/live/_workers/mcp_worker.py +158 -0
  176. fi/alk/live/_workers/pipecat_worker.py +189 -0
  177. fi/alk/live/a2a_lane.py +138 -0
  178. fi/alk/live/langgraph_lane.py +339 -0
  179. fi/alk/live/livekit_lane.py +376 -0
  180. fi/alk/live/mcp_lane.py +172 -0
  181. fi/alk/live/pipecat_lane.py +341 -0
  182. fi/alk/live/voice_redteam.py +494 -0
  183. fi/alk/loss.py +306 -0
  184. fi/alk/optimize.py +36260 -0
  185. fi/alk/practice/__init__.py +51 -0
  186. fi/alk/practice/_assess.py +103 -0
  187. fi/alk/practice/_budget.py +81 -0
  188. fi/alk/practice/_calibrate.py +69 -0
  189. fi/alk/practice/_capstone.py +86 -0
  190. fi/alk/practice/_contract.py +91 -0
  191. fi/alk/practice/_diagnose.py +79 -0
  192. fi/alk/practice/_drill.py +196 -0
  193. fi/alk/practice/_experiment.py +720 -0
  194. fi/alk/practice/_schedule.py +102 -0
  195. fi/alk/practice/_store.py +194 -0
  196. fi/alk/practice/_trainer.py +245 -0
  197. fi/alk/practice/_update.py +125 -0
  198. fi/alk/redteam.py +2621 -0
  199. fi/alk/rewardhack.py +237 -0
  200. fi/alk/simulate.py +10351 -0
  201. fi/alk/studio/__init__.py +82 -0
  202. fi/alk/studio/_bias.py +314 -0
  203. fi/alk/studio/_calibration.py +522 -0
  204. fi/alk/studio/_coverage.py +262 -0
  205. fi/alk/studio/_download.py +665 -0
  206. fi/alk/studio/_fidelity_attack.py +114 -0
  207. fi/alk/studio/_generate.py +652 -0
  208. fi/alk/studio/_library.py +370 -0
  209. fi/alk/studio/_scan.py +134 -0
  210. fi/alk/studio/_upgrade.py +42 -0
  211. fi/alk/studio/_vendor.py +172 -0
  212. fi/alk/suite.py +4200 -0
  213. fi/alk/tasks.py +828 -0
  214. fi/alk/telemetry/__init__.py +149 -0
  215. fi/alk/telemetry/_contract.py +141 -0
  216. fi/alk/telemetry/_emit.py +182 -0
  217. fi/alk/telemetry/_ledger.py +296 -0
  218. fi/alk/telemetry/_queue.py +127 -0
  219. fi/alk/telemetry/_row.py +294 -0
  220. fi/alk/telemetry/_run.py +233 -0
  221. fi/alk/telemetry/_sync.py +193 -0
  222. fi/alk/telemetry/_url.py +119 -0
  223. fi/alk/trinity.py +49397 -0
  224. fi/alk/voice_loop.py +174 -0
  225. fi/api/__init__.py +1 -0
  226. fi/api/auth.py +137 -0
  227. fi/api/types.py +29 -0
  228. fi/cli/__init__.py +9 -0
  229. fi/cli/assertions/__init__.py +25 -0
  230. fi/cli/assertions/conditions.py +76 -0
  231. fi/cli/assertions/evaluator.py +286 -0
  232. fi/cli/assertions/exit_codes.py +20 -0
  233. fi/cli/assertions/parser.py +131 -0
  234. fi/cli/assertions/reporter.py +194 -0
  235. fi/cli/commands/__init__.py +9 -0
  236. fi/cli/commands/config.py +165 -0
  237. fi/cli/commands/export.py +208 -0
  238. fi/cli/commands/init.py +112 -0
  239. fi/cli/commands/list_cmd.py +213 -0
  240. fi/cli/commands/run.py +486 -0
  241. fi/cli/commands/validate.py +173 -0
  242. fi/cli/commands/view.py +424 -0
  243. fi/cli/config/__init__.py +6 -0
  244. fi/cli/config/defaults.py +206 -0
  245. fi/cli/config/loader.py +155 -0
  246. fi/cli/config/schema.py +174 -0
  247. fi/cli/main.py +78 -0
  248. fi/cli/output/__init__.py +6 -0
  249. fi/cli/output/formatters.py +106 -0
  250. fi/cli/output/reporters.py +46 -0
  251. fi/cli/storage/__init__.py +5 -0
  252. fi/cli/storage/run_history.py +249 -0
  253. fi/cli/utils/__init__.py +5 -0
  254. fi/cli/utils/console.py +44 -0
  255. fi/evals/__init__.py +131 -0
  256. fi/evals/autoeval/__init__.py +137 -0
  257. fi/evals/autoeval/analyzer.py +211 -0
  258. fi/evals/autoeval/config.py +244 -0
  259. fi/evals/autoeval/export.py +213 -0
  260. fi/evals/autoeval/interactive.py +283 -0
  261. fi/evals/autoeval/pipeline.py +625 -0
  262. fi/evals/autoeval/prompts.py +139 -0
  263. fi/evals/autoeval/recommender.py +242 -0
  264. fi/evals/autoeval/rules.py +589 -0
  265. fi/evals/autoeval/templates.py +299 -0
  266. fi/evals/autoeval/types.py +232 -0
  267. fi/evals/core/__init__.py +16 -0
  268. fi/evals/core/cloud_registry.py +184 -0
  269. fi/evals/core/engines.py +368 -0
  270. fi/evals/core/evaluate.py +319 -0
  271. fi/evals/core/judge_prompt.py +90 -0
  272. fi/evals/core/prompt_generator.py +83 -0
  273. fi/evals/core/registry.py +57 -0
  274. fi/evals/core/result.py +55 -0
  275. fi/evals/evaluator.py +721 -0
  276. fi/evals/execution.py +168 -0
  277. fi/evals/feedback/__init__.py +32 -0
  278. fi/evals/feedback/calibrator.py +160 -0
  279. fi/evals/feedback/collector.py +214 -0
  280. fi/evals/feedback/hooks.py +81 -0
  281. fi/evals/feedback/retriever.py +128 -0
  282. fi/evals/feedback/store.py +272 -0
  283. fi/evals/feedback/types.py +99 -0
  284. fi/evals/framework/README.md +79 -0
  285. fi/evals/framework/__init__.py +267 -0
  286. fi/evals/framework/backends/Dockerfile.eval-runner +33 -0
  287. fi/evals/framework/backends/__init__.py +99 -0
  288. fi/evals/framework/backends/_container.py +141 -0
  289. fi/evals/framework/backends/_utils.py +145 -0
  290. fi/evals/framework/backends/base.py +223 -0
  291. fi/evals/framework/backends/celery_backend.py +417 -0
  292. fi/evals/framework/backends/celery_worker.py +78 -0
  293. fi/evals/framework/backends/kubernetes_backend.py +665 -0
  294. fi/evals/framework/backends/ray_backend.py +521 -0
  295. fi/evals/framework/backends/temporal.py +350 -0
  296. fi/evals/framework/backends/temporal_worker.py +126 -0
  297. fi/evals/framework/backends/thread_pool.py +286 -0
  298. fi/evals/framework/context.py +258 -0
  299. fi/evals/framework/enrichment.py +306 -0
  300. fi/evals/framework/evals/__init__.py +68 -0
  301. fi/evals/framework/evals/agentic.py +399 -0
  302. fi/evals/framework/evals/builder.py +609 -0
  303. fi/evals/framework/evals/semantic.py +142 -0
  304. fi/evals/framework/evaluator.py +647 -0
  305. fi/evals/framework/evaluators/__init__.py +22 -0
  306. fi/evals/framework/evaluators/blocking.py +347 -0
  307. fi/evals/framework/evaluators/non_blocking.py +577 -0
  308. fi/evals/framework/propagation.py +421 -0
  309. fi/evals/framework/protocols.py +385 -0
  310. fi/evals/framework/registry.py +370 -0
  311. fi/evals/framework/resilience/__init__.py +150 -0
  312. fi/evals/framework/resilience/circuit_breaker.py +309 -0
  313. fi/evals/framework/resilience/degradation.py +355 -0
  314. fi/evals/framework/resilience/health.py +505 -0
  315. fi/evals/framework/resilience/rate_limiter.py +228 -0
  316. fi/evals/framework/resilience/retry.py +274 -0
  317. fi/evals/framework/resilience/types.py +288 -0
  318. fi/evals/framework/resilience/wrapper.py +433 -0
  319. fi/evals/framework/types.py +218 -0
  320. fi/evals/guardrails/README.md +915 -0
  321. fi/evals/guardrails/__init__.py +96 -0
  322. fi/evals/guardrails/backends/__init__.py +43 -0
  323. fi/evals/guardrails/backends/azure.py +361 -0
  324. fi/evals/guardrails/backends/base.py +88 -0
  325. fi/evals/guardrails/backends/generic_llm.py +163 -0
  326. fi/evals/guardrails/backends/granite.py +216 -0
  327. fi/evals/guardrails/backends/llamaguard.py +221 -0
  328. fi/evals/guardrails/backends/local_base.py +479 -0
  329. fi/evals/guardrails/backends/openai.py +365 -0
  330. fi/evals/guardrails/backends/qwen.py +170 -0
  331. fi/evals/guardrails/backends/shieldgemma.py +154 -0
  332. fi/evals/guardrails/backends/turing.py +235 -0
  333. fi/evals/guardrails/backends/vllm_client.py +321 -0
  334. fi/evals/guardrails/backends/wildguard.py +188 -0
  335. fi/evals/guardrails/base.py +888 -0
  336. fi/evals/guardrails/config.py +221 -0
  337. fi/evals/guardrails/discovery.py +243 -0
  338. fi/evals/guardrails/gateway.py +437 -0
  339. fi/evals/guardrails/registry.py +231 -0
  340. fi/evals/guardrails/scanners/__init__.py +127 -0
  341. fi/evals/guardrails/scanners/base.py +191 -0
  342. fi/evals/guardrails/scanners/code_injection.py +243 -0
  343. fi/evals/guardrails/scanners/eval_delegate.py +574 -0
  344. fi/evals/guardrails/scanners/invisible_chars.py +351 -0
  345. fi/evals/guardrails/scanners/jailbreak.py +412 -0
  346. fi/evals/guardrails/scanners/language.py +288 -0
  347. fi/evals/guardrails/scanners/pipeline.py +260 -0
  348. fi/evals/guardrails/scanners/regex.py +311 -0
  349. fi/evals/guardrails/scanners/secrets.py +274 -0
  350. fi/evals/guardrails/scanners/topics.py +649 -0
  351. fi/evals/guardrails/scanners/urls.py +341 -0
  352. fi/evals/guardrails/types.py +96 -0
  353. fi/evals/llm/__init__.py +3 -0
  354. fi/evals/llm/base_llm_provider.py +35 -0
  355. fi/evals/llm/providers/litellm.py +70 -0
  356. fi/evals/local/__init__.py +90 -0
  357. fi/evals/local/evaluator.py +690 -0
  358. fi/evals/local/execution_mode.py +121 -0
  359. fi/evals/local/llm.py +489 -0
  360. fi/evals/local/metrics/__init__.py +19 -0
  361. fi/evals/local/registry.py +360 -0
  362. fi/evals/manager.py +1018 -0
  363. fi/evals/manager_types.py +362 -0
  364. fi/evals/metrics/__init__.py +185 -0
  365. fi/evals/metrics/agents/__init__.py +74 -0
  366. fi/evals/metrics/agents/metrics.py +693 -0
  367. fi/evals/metrics/agents/report.py +36463 -0
  368. fi/evals/metrics/agents/types.py +160 -0
  369. fi/evals/metrics/base_llm_metric.py +111 -0
  370. fi/evals/metrics/base_metric.py +138 -0
  371. fi/evals/metrics/code_security/__init__.py +305 -0
  372. fi/evals/metrics/code_security/analyzer.py +985 -0
  373. fi/evals/metrics/code_security/benchmarks/__init__.py +73 -0
  374. fi/evals/metrics/code_security/benchmarks/builtin.py +750 -0
  375. fi/evals/metrics/code_security/benchmarks/loader.py +580 -0
  376. fi/evals/metrics/code_security/benchmarks/types.py +308 -0
  377. fi/evals/metrics/code_security/detectors/__init__.py +186 -0
  378. fi/evals/metrics/code_security/detectors/base.py +394 -0
  379. fi/evals/metrics/code_security/detectors/cryptography.py +345 -0
  380. fi/evals/metrics/code_security/detectors/injection.py +744 -0
  381. fi/evals/metrics/code_security/detectors/secrets.py +287 -0
  382. fi/evals/metrics/code_security/detectors/serialization.py +192 -0
  383. fi/evals/metrics/code_security/joint_metrics.py +588 -0
  384. fi/evals/metrics/code_security/judges/__init__.py +83 -0
  385. fi/evals/metrics/code_security/judges/base.py +238 -0
  386. fi/evals/metrics/code_security/judges/dual_judge.py +534 -0
  387. fi/evals/metrics/code_security/judges/llm_judge.py +301 -0
  388. fi/evals/metrics/code_security/judges/pattern_judge.py +515 -0
  389. fi/evals/metrics/code_security/metrics.py +388 -0
  390. fi/evals/metrics/code_security/modes/__init__.py +63 -0
  391. fi/evals/metrics/code_security/modes/adversarial.py +284 -0
  392. fi/evals/metrics/code_security/modes/autocomplete.py +198 -0
  393. fi/evals/metrics/code_security/modes/base.py +283 -0
  394. fi/evals/metrics/code_security/modes/instruct.py +253 -0
  395. fi/evals/metrics/code_security/modes/repair.py +230 -0
  396. fi/evals/metrics/code_security/reports/__init__.py +57 -0
  397. fi/evals/metrics/code_security/reports/generator.py +404 -0
  398. fi/evals/metrics/code_security/reports/leaderboard.py +509 -0
  399. fi/evals/metrics/code_security/types.py +534 -0
  400. fi/evals/metrics/function_calling/__init__.py +34 -0
  401. fi/evals/metrics/function_calling/metrics.py +573 -0
  402. fi/evals/metrics/function_calling/types.py +87 -0
  403. fi/evals/metrics/hallucination/__init__.py +54 -0
  404. fi/evals/metrics/hallucination/detector.py +149 -0
  405. fi/evals/metrics/hallucination/metrics.py +390 -0
  406. fi/evals/metrics/hallucination/nli.py +253 -0
  407. fi/evals/metrics/hallucination/sentinel.py +106 -0
  408. fi/evals/metrics/hallucination/types.py +132 -0
  409. fi/evals/metrics/heuristics/aggregation_metrics.py +85 -0
  410. fi/evals/metrics/heuristics/json_metrics.py +87 -0
  411. fi/evals/metrics/heuristics/similarity_metrics.py +375 -0
  412. fi/evals/metrics/heuristics/string_metrics.py +391 -0
  413. fi/evals/metrics/llm_as_judges/__init__.py +17 -0
  414. fi/evals/metrics/llm_as_judges/custom_judge/metric.py +112 -0
  415. fi/evals/metrics/llm_as_judges/custom_judge/prompts.py +26 -0
  416. fi/evals/metrics/llm_as_judges/types.py +48 -0
  417. fi/evals/metrics/rag/__init__.py +111 -0
  418. fi/evals/metrics/rag/advanced/__init__.py +14 -0
  419. fi/evals/metrics/rag/advanced/multi_hop.py +283 -0
  420. fi/evals/metrics/rag/advanced/source_attribution.py +344 -0
  421. fi/evals/metrics/rag/generation/__init__.py +17 -0
  422. fi/evals/metrics/rag/generation/answer_relevancy.py +176 -0
  423. fi/evals/metrics/rag/generation/context_utilization.py +245 -0
  424. fi/evals/metrics/rag/generation/faithfulness.py +241 -0
  425. fi/evals/metrics/rag/generation/groundedness.py +131 -0
  426. fi/evals/metrics/rag/rag_score.py +277 -0
  427. fi/evals/metrics/rag/retrieval/__init__.py +20 -0
  428. fi/evals/metrics/rag/retrieval/context_entity_recall.py +124 -0
  429. fi/evals/metrics/rag/retrieval/context_precision.py +158 -0
  430. fi/evals/metrics/rag/retrieval/context_recall.py +106 -0
  431. fi/evals/metrics/rag/retrieval/noise_sensitivity.py +163 -0
  432. fi/evals/metrics/rag/retrieval/ranking.py +261 -0
  433. fi/evals/metrics/rag/types.py +100 -0
  434. fi/evals/metrics/rag/utils/__init__.py +62 -0
  435. fi/evals/metrics/rag/utils/claims.py +189 -0
  436. fi/evals/metrics/rag/utils/entities.py +244 -0
  437. fi/evals/metrics/rag/utils/nli.py +92 -0
  438. fi/evals/metrics/rag/utils/similarity.py +345 -0
  439. fi/evals/metrics/structured/__init__.py +114 -0
  440. fi/evals/metrics/structured/field_completeness.py +313 -0
  441. fi/evals/metrics/structured/hierarchy_score.py +366 -0
  442. fi/evals/metrics/structured/json_validation.py +190 -0
  443. fi/evals/metrics/structured/schema_compliance.py +280 -0
  444. fi/evals/metrics/structured/structured_output_score.py +298 -0
  445. fi/evals/metrics/structured/types.py +108 -0
  446. fi/evals/metrics/structured/validators/__init__.py +30 -0
  447. fi/evals/metrics/structured/validators/base.py +189 -0
  448. fi/evals/metrics/structured/validators/json_validator.py +196 -0
  449. fi/evals/metrics/structured/validators/pydantic_validator.py +178 -0
  450. fi/evals/metrics/structured/validators/yaml_validator.py +248 -0
  451. fi/evals/otel/__init__.py +266 -0
  452. fi/evals/otel/config.py +400 -0
  453. fi/evals/otel/conventions.py +463 -0
  454. fi/evals/otel/enrichment.py +371 -0
  455. fi/evals/otel/instrumentors/__init__.py +140 -0
  456. fi/evals/otel/instrumentors/anthropic.py +517 -0
  457. fi/evals/otel/instrumentors/base.py +382 -0
  458. fi/evals/otel/instrumentors/openai.py +673 -0
  459. fi/evals/otel/processors/__init__.py +36 -0
  460. fi/evals/otel/processors/base.py +473 -0
  461. fi/evals/otel/processors/cost.py +445 -0
  462. fi/evals/otel/processors/evaluation.py +559 -0
  463. fi/evals/otel/processors/llm.py +462 -0
  464. fi/evals/otel/tracer.py +506 -0
  465. fi/evals/otel/types.py +232 -0
  466. fi/evals/otel_utils.py +23 -0
  467. fi/evals/protect.py +671 -0
  468. fi/evals/protect_input_adapter.py +154 -0
  469. fi/evals/streaming/__init__.py +88 -0
  470. fi/evals/streaming/buffer.py +213 -0
  471. fi/evals/streaming/evaluator.py +551 -0
  472. fi/evals/streaming/policy.py +307 -0
  473. fi/evals/streaming/scorers.py +368 -0
  474. fi/evals/streaming/types.py +238 -0
  475. fi/evals/templates.py +472 -0
  476. fi/evals/types.py +156 -0
  477. fi/opt/__init__.py +221 -0
  478. fi/opt/_objective_scoring.py +85 -0
  479. fi/opt/base/__init__.py +11 -0
  480. fi/opt/base/base_generator.py +33 -0
  481. fi/opt/base/base_mapper.py +26 -0
  482. fi/opt/base/base_optimizer.py +45 -0
  483. fi/opt/base/evaluator.py +211 -0
  484. fi/opt/components.py +3095 -0
  485. fi/opt/datamappers/__init__.py +3 -0
  486. fi/opt/datamappers/basic_mapper.py +40 -0
  487. fi/opt/deployment.py +1021 -0
  488. fi/opt/evidence.py +4332 -0
  489. fi/opt/generators/__init__.py +3 -0
  490. fi/opt/generators/litellm.py +66 -0
  491. fi/opt/integrations/__init__.py +23 -0
  492. fi/opt/integrations/generative_suite.py +410 -0
  493. fi/opt/integrations/simulate.py +1313 -0
  494. fi/opt/mutations.py +771 -0
  495. fi/opt/observability.py +4639 -0
  496. fi/opt/optimizer_trace.py +889 -0
  497. fi/opt/optimizers/__init__.py +80 -0
  498. fi/opt/optimizers/agent.py +331 -0
  499. fi/opt/optimizers/agent_bandit.py +392 -0
  500. fi/opt/optimizers/agent_curriculum.py +635 -0
  501. fi/opt/optimizers/agent_evolution.py +894 -0
  502. fi/opt/optimizers/agent_feedback.py +1863 -0
  503. fi/opt/optimizers/agent_pareto.py +547 -0
  504. fi/opt/optimizers/agent_social_memory.py +1113 -0
  505. fi/opt/optimizers/agent_tpe.py +321 -0
  506. fi/opt/optimizers/bayesian_search.py +449 -0
  507. fi/opt/optimizers/council.py +2075 -0
  508. fi/opt/optimizers/futureagi_replay.py +799 -0
  509. fi/opt/optimizers/gepa.py +322 -0
  510. fi/opt/optimizers/metaprompt.py +243 -0
  511. fi/opt/optimizers/promptwizard.py +417 -0
  512. fi/opt/optimizers/protegi.py +329 -0
  513. fi/opt/optimizers/random_search.py +224 -0
  514. fi/opt/research.py +518 -0
  515. fi/opt/simulation.py +260 -0
  516. fi/opt/targets.py +232 -0
  517. fi/opt/types.py +66 -0
  518. fi/opt/utils/__init__.py +4 -0
  519. fi/opt/utils/early_stopping.py +266 -0
  520. fi/opt/utils/setup_logging.py +82 -0
  521. fi/simulate/__init__.py +540 -0
  522. fi/simulate/_hashing.py +35 -0
  523. fi/simulate/_logging.py +10 -0
  524. fi/simulate/adapters.py +87 -0
  525. fi/simulate/agent/__init__.py +120 -0
  526. fi/simulate/agent/browser.py +658 -0
  527. fi/simulate/agent/definition.py +587 -0
  528. fi/simulate/agent/frameworks.py +3528 -0
  529. fi/simulate/agent/generic.py +8286 -0
  530. fi/simulate/agent/import_probe.py +227 -0
  531. fi/simulate/agent/memory.py +905 -0
  532. fi/simulate/agent/mocks.py +101 -0
  533. fi/simulate/agent/multi_agent.py +361 -0
  534. fi/simulate/agent/orchestration.py +903 -0
  535. fi/simulate/agent/realtime.py +665 -0
  536. fi/simulate/agent/wrapper.py +99 -0
  537. fi/simulate/agent/wrappers/__init__.py +18 -0
  538. fi/simulate/agent/wrappers/anthropic.py +62 -0
  539. fi/simulate/agent/wrappers/gemini.py +65 -0
  540. fi/simulate/agent/wrappers/http.py +404 -0
  541. fi/simulate/agent/wrappers/langchain.py +80 -0
  542. fi/simulate/agent/wrappers/openai.py +75 -0
  543. fi/simulate/agent/wrappers/websocket.py +326 -0
  544. fi/simulate/artifacts/__init__.py +11 -0
  545. fi/simulate/artifacts/manifest.py +62 -0
  546. fi/simulate/cli.py +20560 -0
  547. fi/simulate/endpoints/__init__.py +45 -0
  548. fi/simulate/endpoints/_http_actor.py +73 -0
  549. fi/simulate/endpoints/actor_sources.py +243 -0
  550. fi/simulate/endpoints/base.py +107 -0
  551. fi/simulate/endpoints/builtins.py +10 -0
  552. fi/simulate/endpoints/callable.py +95 -0
  553. fi/simulate/endpoints/http.py +76 -0
  554. fi/simulate/endpoints/livekit.py +138 -0
  555. fi/simulate/endpoints/originators.py +132 -0
  556. fi/simulate/endpoints/profiles.py +348 -0
  557. fi/simulate/endpoints/retell.py +633 -0
  558. fi/simulate/endpoints/vapi.py +205 -0
  559. fi/simulate/endpoints/websocket.py +76 -0
  560. fi/simulate/environment.py +33026 -0
  561. fi/simulate/environments/__init__.py +11 -0
  562. fi/simulate/environments/base.py +73 -0
  563. fi/simulate/environments/chat.py +697 -0
  564. fi/simulate/environments/voice.py +212 -0
  565. fi/simulate/evaluation/__init__.py +4 -0
  566. fi/simulate/evaluation/ai_eval.py +227 -0
  567. fi/simulate/evidence/__init__.py +35 -0
  568. fi/simulate/evidence/base.py +59 -0
  569. fi/simulate/evidence/caller_observed.py +50 -0
  570. fi/simulate/evidence/livekit_instrumentation.py +51 -0
  571. fi/simulate/evidence/livekit_room.py +50 -0
  572. fi/simulate/evidence/otel.py +49 -0
  573. fi/simulate/evidence/providers/__init__.py +24 -0
  574. fi/simulate/evidence/providers/base.py +61 -0
  575. fi/simulate/evidence/providers/retell.py +376 -0
  576. fi/simulate/evidence/providers/vapi.py +426 -0
  577. fi/simulate/hosted/__init__.py +32 -0
  578. fi/simulate/hosted/child_entrypoint.py +306 -0
  579. fi/simulate/hosted/job.py +150 -0
  580. fi/simulate/hosted/targets.py +53 -0
  581. fi/simulate/instrumentation/__init__.py +5 -0
  582. fi/simulate/instrumentation/livekit/__init__.py +122 -0
  583. fi/simulate/manifest.py +1033 -0
  584. fi/simulate/matrix_cli.py +165 -0
  585. fi/simulate/realtime/__init__.py +40 -0
  586. fi/simulate/realtime/events.py +107 -0
  587. fi/simulate/realtime/media.py +61 -0
  588. fi/simulate/realtime/session.py +91 -0
  589. fi/simulate/recording/__init__.py +5 -0
  590. fi/simulate/recording/room_recorder.py +326 -0
  591. fi/simulate/registry.py +185 -0
  592. fi/simulate/results/__init__.py +9 -0
  593. fi/simulate/results/base.py +18 -0
  594. fi/simulate/results/filesystem.py +71 -0
  595. fi/simulate/results/futureagi.py +1340 -0
  596. fi/simulate/runtime/__init__.py +85 -0
  597. fi/simulate/runtime/capabilities.py +40 -0
  598. fi/simulate/runtime/events.py +63 -0
  599. fi/simulate/runtime/failures.py +25 -0
  600. fi/simulate/runtime/ids.py +34 -0
  601. fi/simulate/runtime/plan.py +70 -0
  602. fi/simulate/runtime/planner.py +102 -0
  603. fi/simulate/runtime/report.py +174 -0
  604. fi/simulate/runtime/run.py +75 -0
  605. fi/simulate/runtime/runner.py +333 -0
  606. fi/simulate/runtime/spec.py +186 -0
  607. fi/simulate/simulation/__init__.py +30 -0
  608. fi/simulate/simulation/behavior_policy.py +425 -0
  609. fi/simulate/simulation/bridge/__init__.py +9 -0
  610. fi/simulate/simulation/bridge/audio.py +29 -0
  611. fi/simulate/simulation/bridge/connector.py +46 -0
  612. fi/simulate/simulation/bridge/livekit.py +252 -0
  613. fi/simulate/simulation/bridge/retell.py +188 -0
  614. fi/simulate/simulation/bridge/vapi.py +177 -0
  615. fi/simulate/simulation/contract.py +419 -0
  616. fi/simulate/simulation/engines/__init__.py +12 -0
  617. fi/simulate/simulation/engines/base.py +21 -0
  618. fi/simulate/simulation/engines/cloud.py +517 -0
  619. fi/simulate/simulation/engines/livekit.py +4167 -0
  620. fi/simulate/simulation/engines/local_text.py +89 -0
  621. fi/simulate/simulation/fidelity.py +374 -0
  622. fi/simulate/simulation/gemini_tts_stream.py +110 -0
  623. fi/simulate/simulation/generator.py +91 -0
  624. fi/simulate/simulation/goal_machine.py +185 -0
  625. fi/simulate/simulation/livekit_models.py +467 -0
  626. fi/simulate/simulation/matrix.py +170 -0
  627. fi/simulate/simulation/models.py +279 -0
  628. fi/simulate/simulation/runner.py +153 -0
  629. fi/simulate/simulation/synthetic.py +880 -0
  630. fi/simulate/simulation/voice_prompt.py +502 -0
  631. fi/simulate/simulator/__init__.py +55 -0
  632. fi/simulate/simulator/builtins.py +53 -0
  633. fi/simulate/suite.py +1288 -0
  634. fi/simulate/utils/routes.py +164 -0
  635. fi/simulate/voice.py +225 -0
  636. fi/simulate/voice_cli.py +182 -0
  637. fi/utils/__init__.py +1 -0
  638. fi/utils/constants.py +14 -0
  639. fi/utils/errors.py +200 -0
  640. fi/utils/executor.py +26 -0
  641. fi/utils/routes.py +119 -0
  642. fi/utils/utils.py +17 -0
fi/opt/mutations.py ADDED
@@ -0,0 +1,771 @@
1
+ from __future__ import annotations
2
+
3
+ import json
4
+ from dataclasses import dataclass
5
+ from typing import Any, Iterable, Mapping, Optional, Sequence
6
+
7
+ from .components import ComponentDiagnosis
8
+ from .targets import OptimizationTarget
9
+
10
+
11
+ @dataclass(frozen=True)
12
+ class AgentMutationBundle:
13
+ """A coherent framework-aware config patch for agent optimization."""
14
+
15
+ name: str
16
+ framework: str
17
+ component: str
18
+ patch: dict[str, Any]
19
+ reason: str = ""
20
+ priority: float = 1.0
21
+ tags: tuple[str, ...] = ()
22
+
23
+
24
+ @dataclass(frozen=True)
25
+ class FrameworkMutationRule:
26
+ name: str
27
+ framework: str
28
+ components: tuple[str, ...]
29
+ path_groups: tuple[tuple[str, ...], ...]
30
+ reason: str
31
+ priority: float = 1.0
32
+ tags: tuple[str, ...] = ()
33
+
34
+
35
+ DEFAULT_FRAMEWORK_MUTATION_RULES: tuple[FrameworkMutationRule, ...] = (
36
+ FrameworkMutationRule(
37
+ name="langgraph_event_stream_checkpoint_bundle",
38
+ framework="langgraph",
39
+ components=("framework", "orchestration", "planner", "memory", "multi_agent"),
40
+ path_groups=(
41
+ ("langgraph", "stream_events", "framework.events.source", "framework.stream_events"),
42
+ ("langgraph.nodes", "policy_node", "node"),
43
+ ("planner.tool_sequence", "tool_sequence", "lookup", "tool"),
44
+ ("memory.state_persistence", "checkpoint", "checkpointer", "case", "state"),
45
+ ("langgraph.checkpointer", "framework.checkpoints", "checkpoint", "checkpoint_state"),
46
+ ("framework.sessions", "thread_id", "session", "configurable"),
47
+ ("framework.trace", "collector", "span", "event"),
48
+ ),
49
+ reason="Enable typed LangGraph event replay, graph nodes, tool order, and checkpoint/state capture together.",
50
+ priority=4.0,
51
+ tags=("stream_events", "checkpoint", "graph"),
52
+ ),
53
+ FrameworkMutationRule(
54
+ name="langchain_event_stream_tool_bundle",
55
+ framework="langchain",
56
+ components=("framework", "tools", "retrieval", "memory", "orchestration"),
57
+ path_groups=(
58
+ ("langchain", "stream_events", "framework.events.source", "framework.stream_events"),
59
+ ("callbacks", "tracing", "trace", "collector"),
60
+ ("tool", "tool_calls", "tool_sequence", "schema"),
61
+ ("retrieval", "retriever", "documents", "source"),
62
+ ("memory", "state", "checkpoint", "persistence"),
63
+ ),
64
+ reason="Enable LangChain event streams plus tool, retrieval, and memory evidence.",
65
+ priority=3.5,
66
+ tags=("stream_events", "tools", "retrieval"),
67
+ ),
68
+ FrameworkMutationRule(
69
+ name="openai_agents_trace_session_bundle",
70
+ framework="openai_agents",
71
+ components=("framework", "multi_agent", "tools", "policy", "memory"),
72
+ path_groups=(
73
+ ("openai_agents", "trace", "tracing", "span", "processor"),
74
+ ("openai_agents.sessions", "session", "memory", "checkpoint"),
75
+ ("handoff", "handoffs", "multi_agent"),
76
+ ("guardrail", "guardrails", "policy", "safety"),
77
+ ("tool", "tool_owner", "tool_output", "function_call"),
78
+ ),
79
+ reason="Capture OpenAI Agents traces, sessions, handoffs, guardrails, and tool ownership as one candidate.",
80
+ priority=3.5,
81
+ tags=("tracing", "sessions", "handoffs"),
82
+ ),
83
+ FrameworkMutationRule(
84
+ name="otel_genai_trace_bundle",
85
+ framework="opentelemetry",
86
+ components=("framework", "model", "retrieval", "tools", "implementation"),
87
+ path_groups=(
88
+ ("otel", "opentelemetry", "otlp", "gen_ai", "semantic"),
89
+ ("framework.trace", "span", "collector", "export"),
90
+ ("model", "llm", "chat", "inference"),
91
+ ("retrieval", "retriever", "data_source", "documents"),
92
+ ("tool", "execute_tool", "function_call"),
93
+ ),
94
+ reason="Enable OpenTelemetry GenAI-style spans for inference, retrieval, and tool execution.",
95
+ priority=3.2,
96
+ tags=("otel", "gen_ai", "spans"),
97
+ ),
98
+ FrameworkMutationRule(
99
+ name="mcp_tool_session_replay_bundle",
100
+ framework="mcp",
101
+ components=("framework", "tools", "implementation", "harness"),
102
+ path_groups=(
103
+ ("mcp", "tool_session", "session_export", "server"),
104
+ ("tools.list", "tools_list", "mcp.tools.list", "tool_discovery"),
105
+ ("schema", "input_schema", "tool_schema", "mcp.tools.schema"),
106
+ ("tools.call", "tool_call", "mcp.tools.call", "arguments"),
107
+ ("result", "tool_result", "mcp.tools.result", "output"),
108
+ ("error", "tool_error", "mcp.tools.error", "exception"),
109
+ ("resources", "resource_templates", "mcp.resources"),
110
+ ),
111
+ reason="Capture MCP tool discovery, schemas, calls, results, errors, and resources as one replayable session export.",
112
+ priority=3.3,
113
+ tags=("mcp", "tools", "schemas", "results"),
114
+ ),
115
+ FrameworkMutationRule(
116
+ name="livekit_session_voice_pipeline_bundle",
117
+ framework="livekit",
118
+ components=("voice", "streaming", "framework", "perception", "tools"),
119
+ path_groups=(
120
+ ("livekit", "session_events", "agent_session", "room_io"),
121
+ ("voice.trace", "trace", "timeline"),
122
+ ("voice.webrtc", "webrtc", "get_stats", "getstats", "stats_source", "rtc_stats"),
123
+ ("rtp", "rtp_counters", "packet", "packets", "packet_loss"),
124
+ ("track", "track_identifier", "track_stats"),
125
+ ("codec", "codec_id", "codec_stats"),
126
+ ("audio_level", "audiolevel", "level_stats"),
127
+ ("transport", "jitter", "packet_loss", "stable"),
128
+ ("transcript", "transcription", "stt", "llm", "tts"),
129
+ ("audio", "recording", "track", "webrtc", "media"),
130
+ ("vad", "turn", "endpointing", "interruption"),
131
+ ("routing", "route", "intent", "handoff"),
132
+ ("tool", "handoff", "function"),
133
+ ),
134
+ reason="Capture LiveKit session events, WebRTC getStats/RTP/track/codec evidence, voice pipeline nodes, transcripts, media, turn handling, and tools together.",
135
+ priority=3.4,
136
+ tags=("voice", "session_events", "media", "webrtc"),
137
+ ),
138
+ FrameworkMutationRule(
139
+ name="pipecat_frame_pipeline_bundle",
140
+ framework="pipecat",
141
+ components=("voice", "streaming", "framework", "perception"),
142
+ path_groups=(
143
+ ("pipecat", "frames", "frame_pipeline", "frame_source"),
144
+ ("transcription", "stt", "transcript"),
145
+ ("tts", "audio", "output_audio"),
146
+ ("raw_pcm", "audio_decode", "waveform", "media"),
147
+ ("interruption", "overlap", "barge"),
148
+ ("timing", "latency", "turn", "endpointing"),
149
+ ),
150
+ reason="Enable Pipecat frame, transcript, TTS/audio, interruption, and timing capture as a bundle.",
151
+ priority=3.4,
152
+ tags=("voice", "frames", "timing"),
153
+ ),
154
+ FrameworkMutationRule(
155
+ name="browser_cua_replay_bundle",
156
+ framework="browser_cua",
157
+ components=("browser", "cua", "perception", "action", "harness"),
158
+ path_groups=(
159
+ ("browser.trace", "playwright", "browser_use", "openai_cua", "trace"),
160
+ ("screenshot", "video", "artifact", "visual"),
161
+ ("har", "network", "resource_body"),
162
+ ("actionability", "selector", "fallback"),
163
+ ("storage", "cookies", "local_storage", "runtime"),
164
+ ("allow_cross_origin", "domain"),
165
+ ),
166
+ reason="Capture browser/CUA traces, screenshots/video, network, actionability, storage/runtime, and safe domain policy.",
167
+ priority=3.2,
168
+ tags=("browser", "cua", "replay"),
169
+ ),
170
+ FrameworkMutationRule(
171
+ name="redteam_campaign_matrix_evidence_bundle",
172
+ framework="red_team",
173
+ components=("security", "environment", "harness", "evaluator", "integration"),
174
+ path_groups=(
175
+ ("red_team.matrix_evidence", "coverage_matrix", "matrix_cells"),
176
+ ("red_team.scenarios.matrix_cell_ids", "red_team.scenarios", "scenario_matrix"),
177
+ ("red_team.runs.matrix_cell_ids", "red_team.runs", "run_artifact", "passed_run"),
178
+ ("red_team.artifacts.matrix_cell_ids", "red_team.artifacts", "artifact_matrix"),
179
+ ("red_team.artifacts.execution_evidence", "red_team.artifacts", "executed_evidence"),
180
+ ("red_team.findings.matrix_cell_ids", "red_team.findings", "finding_mapping"),
181
+ ("red_team.mitigations.matrix_cell_ids", "red_team.mitigations", "mitigation_mapping"),
182
+ (
183
+ "evaluation.red_team_campaign_quality.require_attack_surface_matrix",
184
+ "require_attack_surface_matrix",
185
+ ),
186
+ (
187
+ "evaluation.red_team_campaign_quality.require_run_artifacts",
188
+ "require_run_artifacts",
189
+ ),
190
+ (
191
+ "evaluation.red_team_campaign_quality.require_executed_run_evidence",
192
+ "require_executed_run_evidence",
193
+ ),
194
+ (
195
+ "evaluation.red_team_campaign_quality.require_finding_mapping",
196
+ "require_finding_mapping",
197
+ ),
198
+ (
199
+ "evaluation.red_team_campaign_quality.require_mitigation_mapping",
200
+ "require_mitigation_mapping",
201
+ ),
202
+ ),
203
+ reason=(
204
+ "Bind red-team campaign matrix cells to scenario, passed-run, "
205
+ "artifact, and mitigation evidence before relaxing quality gates."
206
+ ),
207
+ priority=3.8,
208
+ tags=("red_team", "campaign", "matrix", "artifact", "mitigation"),
209
+ ),
210
+ FrameworkMutationRule(
211
+ name="manifest_optimizer_trace_governance_bundle",
212
+ framework="optimizer",
213
+ components=("harness", "evaluator", "multi_agent", "planner", "integration"),
214
+ path_groups=(
215
+ ("optimizer.trace", "optimizer_trace", "trace", "trace_capture"),
216
+ ("optimizer.society_trace", "optimizer_society_trace", "society_trace"),
217
+ ("optimizer.roles", "optimizer.role", "roles", "role"),
218
+ ("optimizer.role_graph", "role_graph", "role_graph.strategy"),
219
+ ("optimizer.proposals", "optimizer.proposal", "proposals", "proposal"),
220
+ ("optimizer.credit", "credit", "role_credit", "proposal_credit"),
221
+ (
222
+ "optimizer.governance.checks",
223
+ "governance.checks",
224
+ "governance_checks",
225
+ ),
226
+ (
227
+ "optimizer.governance.contract_gate",
228
+ "contract_gate",
229
+ "contract",
230
+ ),
231
+ ("optimizer.governance.rollback", "rollback", "rollback_window"),
232
+ (
233
+ "optimizer.governance.search_locality",
234
+ "search_locality",
235
+ "locality",
236
+ ),
237
+ (
238
+ "evaluation.optimizer_trace_quality.required_governance_signals",
239
+ "required_governance_signals",
240
+ "governance_signals",
241
+ ),
242
+ (
243
+ "evaluation.optimizer_trace_quality.required_roles",
244
+ "required_roles",
245
+ "manifest_seed",
246
+ "deterministic_search",
247
+ "selection_steward",
248
+ ),
249
+ (
250
+ "evaluation.optimizer_trace_quality.required_search_paths",
251
+ "required_search_paths",
252
+ "search_paths",
253
+ ),
254
+ (
255
+ "evaluation.optimizer_trace_quality.min_role_count",
256
+ "min_role_count",
257
+ "role_count",
258
+ "threshold",
259
+ ),
260
+ (
261
+ "evaluation.optimizer_trace_quality.min_proposal_count",
262
+ "min_proposal_count",
263
+ "proposal_count",
264
+ "threshold",
265
+ ),
266
+ (
267
+ "evaluation.optimizer_trace_quality.min_governance_checks",
268
+ "min_governance_checks",
269
+ "governance_check_count",
270
+ "threshold",
271
+ ),
272
+ (
273
+ "evaluation.optimizer_trace_quality.min_credit_entries",
274
+ "min_credit_entries",
275
+ "role_credit",
276
+ ),
277
+ (
278
+ "evaluation.optimizer_trace_quality.min_governance_pass_rate",
279
+ "min_governance_pass_rate",
280
+ "pass_rate",
281
+ ),
282
+ (
283
+ "evaluation.optimizer_trace_quality.require_contract_gate",
284
+ "require_contract_gate",
285
+ "contract_gate",
286
+ ),
287
+ (
288
+ "evaluation.optimizer_trace_quality.require_rollback",
289
+ "require_rollback",
290
+ "rollback",
291
+ ),
292
+ (
293
+ "evaluation.optimizer_trace_quality.require_locality",
294
+ "require_locality",
295
+ "search_locality",
296
+ ),
297
+ (
298
+ "optimization.target.search_space",
299
+ "target.search_space",
300
+ "manifest_search_space",
301
+ ),
302
+ (
303
+ "optimization.optimizer.max_candidates",
304
+ "optimizer.max_candidates",
305
+ "max_candidates",
306
+ ),
307
+ ("optimization.threshold", "optimization.target_score", "threshold"),
308
+ (
309
+ "evaluation.manifest_optimization_quality.required_search_paths",
310
+ "manifest_optimization_quality.required_search_paths",
311
+ "required_search_paths",
312
+ ),
313
+ (
314
+ "evaluation.manifest_optimization_quality.required_metrics",
315
+ "manifest_optimization_quality.required_metrics",
316
+ "required_metrics",
317
+ ),
318
+ (
319
+ "evaluation.manifest_optimization_quality.min_history_count",
320
+ "min_history_count",
321
+ "history_count",
322
+ ),
323
+ (
324
+ "evaluation.manifest_optimization_quality.min_candidate_count",
325
+ "min_candidate_count",
326
+ "candidate_count",
327
+ ),
328
+ (
329
+ "evaluation.manifest_optimization_quality.min_patch_count",
330
+ "min_patch_count",
331
+ "patch_count",
332
+ ),
333
+ (
334
+ "evaluation.manifest_optimization_quality.min_metric_count",
335
+ "min_metric_count",
336
+ "metric_count",
337
+ ),
338
+ (
339
+ "evaluation.manifest_optimization_quality.min_final_score",
340
+ "min_final_score",
341
+ "manifest_quality_threshold",
342
+ ),
343
+ (
344
+ "evaluation.manifest_optimization_quality.require_passed",
345
+ "require_passed",
346
+ "passed",
347
+ ),
348
+ (
349
+ "evaluation.manifest_optimization_quality.require_best_candidate",
350
+ "require_best_candidate",
351
+ "best_candidate",
352
+ ),
353
+ (
354
+ "evaluation.manifest_optimization_quality.require_best_config",
355
+ "require_best_config",
356
+ "best_config",
357
+ ),
358
+ (
359
+ "evaluation.manifest_optimization_quality.require_history",
360
+ "require_history",
361
+ "history",
362
+ ),
363
+ (
364
+ "evaluation.manifest_optimization_quality.require_candidate_patches",
365
+ "require_candidate_patches",
366
+ "candidate_patches",
367
+ ),
368
+ (
369
+ "evaluation.manifest_optimization_quality.require_metrics",
370
+ "require_metrics",
371
+ "metrics",
372
+ ),
373
+ (
374
+ "evaluation.manifest_optimization_quality.require_search_paths",
375
+ "require_search_paths",
376
+ "search_paths",
377
+ ),
378
+ ),
379
+ reason=(
380
+ "Bind manifest optimization search space and optimizer society-trace "
381
+ "governance evidence before tightening trace and manifest quality gates."
382
+ ),
383
+ priority=3.9,
384
+ tags=("manifest_optimization", "trace", "governance", "society_trace"),
385
+ ),
386
+ FrameworkMutationRule(
387
+ name="rag_grounding_memory_bundle",
388
+ framework="rag",
389
+ components=("retrieval", "memory", "policy", "evaluator"),
390
+ path_groups=(
391
+ ("retrieval", "retriever", "source", "document"),
392
+ ("citation", "citations", "attribution"),
393
+ ("grounded", "grounded_only", "source_consistent"),
394
+ ("stale", "fresh", "current"),
395
+ ("memory", "memory_write", "write_resolution", "recall"),
396
+ ),
397
+ reason="Bundle current retrieval, citation/attribution, grounded generation, freshness, and memory-write settings.",
398
+ priority=3.0,
399
+ tags=("retrieval", "grounding", "memory"),
400
+ ),
401
+ FrameworkMutationRule(
402
+ name="multi_agent_handoff_review_bundle",
403
+ framework="multi_agent",
404
+ components=("multi_agent", "planner", "memory", "policy"),
405
+ path_groups=(
406
+ ("multi_agent", "role", "specialist"),
407
+ ("handoff", "contract", "context"),
408
+ ("review", "qa", "critic"),
409
+ ("memory", "shared", "case_summary"),
410
+ ("reconciliation", "accepted_source", "evidence"),
411
+ ),
412
+ reason="Bundle specialist routing, handoff contracts, review, shared memory, and evidence-weighted reconciliation.",
413
+ priority=3.0,
414
+ tags=("multi_agent", "handoff", "review"),
415
+ ),
416
+ )
417
+
418
+
419
+ class AgentMutationLibrary:
420
+ """
421
+ Proposes coherent config patches from framework and diagnosis evidence.
422
+
423
+ The library is intentionally value-aware: each proposed patch uses only
424
+ paths and values that already exist in `OptimizationTarget.search_space`.
425
+ """
426
+
427
+ def __init__(
428
+ self,
429
+ *,
430
+ rules: Optional[Iterable[FrameworkMutationRule]] = None,
431
+ bundles: Optional[Iterable[AgentMutationBundle]] = None,
432
+ name: str = "default_framework_mutations",
433
+ ) -> None:
434
+ self.rules = (
435
+ DEFAULT_FRAMEWORK_MUTATION_RULES
436
+ if rules is None
437
+ else tuple(rules)
438
+ )
439
+ self.bundles = tuple(bundles or ())
440
+ self.name = name
441
+
442
+ def propose(
443
+ self,
444
+ target: OptimizationTarget,
445
+ *,
446
+ diagnoses: Sequence[ComponentDiagnosis] = (),
447
+ search_paths: Optional[Sequence[str]] = None,
448
+ max_bundles: Optional[int] = None,
449
+ ) -> list[AgentMutationBundle]:
450
+ allowed_paths = [path for path in (search_paths or target.search_space) if path in target.search_space]
451
+ if not allowed_paths:
452
+ return []
453
+
454
+ proposed: list[AgentMutationBundle] = []
455
+ for bundle in self.bundles:
456
+ filtered = _filter_patch(bundle.patch, target=target, allowed_paths=allowed_paths)
457
+ if filtered:
458
+ proposed.append(
459
+ AgentMutationBundle(
460
+ name=bundle.name,
461
+ framework=bundle.framework,
462
+ component=bundle.component,
463
+ patch=filtered,
464
+ reason=bundle.reason,
465
+ priority=bundle.priority,
466
+ tags=bundle.tags,
467
+ )
468
+ )
469
+
470
+ hints = _target_hints(target)
471
+ for rule in self.rules:
472
+ rule_score = _rule_score(rule, target=target, diagnoses=diagnoses, hints=hints)
473
+ if rule_score <= 0:
474
+ continue
475
+ patch = _rule_patch(rule, target=target, allowed_paths=allowed_paths)
476
+ if not patch:
477
+ continue
478
+ proposed.append(
479
+ AgentMutationBundle(
480
+ name=rule.name,
481
+ framework=rule.framework,
482
+ component=_dominant_component(rule, diagnoses),
483
+ patch=patch,
484
+ reason=rule.reason,
485
+ priority=round(rule.priority + rule_score + len(patch) * 0.1, 4),
486
+ tags=rule.tags,
487
+ )
488
+ )
489
+
490
+ proposed = _dedupe_bundles(proposed)
491
+ proposed.sort(key=lambda item: (item.priority, len(item.patch), item.name), reverse=True)
492
+ if max_bundles is not None:
493
+ return proposed[:max(0, max_bundles)]
494
+ return proposed
495
+
496
+
497
+ DEFAULT_AGENT_MUTATION_LIBRARY = AgentMutationLibrary()
498
+
499
+
500
+ def resolve_agent_mutation_library(
501
+ value: Optional[AgentMutationLibrary | Iterable[AgentMutationBundle] | bool],
502
+ ) -> Optional[AgentMutationLibrary]:
503
+ if value is False:
504
+ return None
505
+ if value is None or value is True:
506
+ return DEFAULT_AGENT_MUTATION_LIBRARY
507
+ if isinstance(value, AgentMutationLibrary):
508
+ return value
509
+ return AgentMutationLibrary(bundles=value, rules=(), name="custom_mutation_bundles")
510
+
511
+
512
+ def dump_mutation_bundle(bundle: AgentMutationBundle) -> dict[str, Any]:
513
+ return {
514
+ "name": bundle.name,
515
+ "framework": bundle.framework,
516
+ "component": bundle.component,
517
+ "patch": dict(bundle.patch),
518
+ "reason": bundle.reason,
519
+ "priority": bundle.priority,
520
+ "tags": list(bundle.tags),
521
+ }
522
+
523
+
524
+ def _rule_score(
525
+ rule: FrameworkMutationRule,
526
+ *,
527
+ target: OptimizationTarget,
528
+ diagnoses: Sequence[ComponentDiagnosis],
529
+ hints: str,
530
+ ) -> float:
531
+ score = 0.0
532
+ framework_token = _normalize_token(rule.framework)
533
+ if framework_token and framework_token in hints:
534
+ score += 2.0
535
+ for path in target.search_space:
536
+ if _path_matches(path, (rule.framework, *rule.tags)):
537
+ score += 0.4
538
+
539
+ if not diagnoses:
540
+ return score
541
+
542
+ if score <= 0 and rule.framework not in {"rag", "multi_agent"}:
543
+ return 0.0
544
+
545
+ diagnosis_score = 0.0
546
+ for diagnosis in diagnoses:
547
+ if str(diagnosis.component) in rule.components:
548
+ diagnosis_score += 2.0 * diagnosis.confidence
549
+ for suggested_path in diagnosis.suggested_paths:
550
+ if any(_path_matches(suggested_path, group) for group in rule.path_groups):
551
+ diagnosis_score += 1.0 * diagnosis.confidence
552
+ if diagnosis_score <= 0:
553
+ return 0.0
554
+ return score + diagnosis_score
555
+
556
+
557
+ def _rule_patch(
558
+ rule: FrameworkMutationRule,
559
+ *,
560
+ target: OptimizationTarget,
561
+ allowed_paths: Sequence[str],
562
+ ) -> dict[str, Any]:
563
+ patch: dict[str, Any] = {}
564
+ base = target.seed_candidate()
565
+ for group in rule.path_groups:
566
+ path = _best_matching_path(group, target=target, allowed_paths=allowed_paths, used_paths=patch)
567
+ if path is None:
568
+ continue
569
+ value = _preferred_value(
570
+ path,
571
+ target.search_space[path],
572
+ tokens=(*group, rule.framework, *rule.tags),
573
+ )
574
+ if value != base.get_path(path):
575
+ patch[path] = value
576
+ return patch
577
+
578
+
579
+ def _best_matching_path(
580
+ tokens: Sequence[str],
581
+ *,
582
+ target: OptimizationTarget,
583
+ allowed_paths: Sequence[str],
584
+ used_paths: Mapping[str, Any],
585
+ ) -> Optional[str]:
586
+ candidates = [
587
+ path
588
+ for path in allowed_paths
589
+ if path not in used_paths and target.search_space.get(path) and _path_matches(path, tokens)
590
+ ]
591
+ if not candidates:
592
+ return None
593
+ candidates.sort(key=lambda path: (_path_match_score(path, tokens), -len(path), path), reverse=True)
594
+ return candidates[0]
595
+
596
+
597
+ def _preferred_value(path: str, values: Sequence[Any], *, tokens: Sequence[str]) -> Any:
598
+ if not values:
599
+ return None
600
+ ranked = sorted(
601
+ values,
602
+ key=lambda value: (_value_score(path, value, tokens), json.dumps(value, sort_keys=True, default=str)),
603
+ reverse=True,
604
+ )
605
+ return ranked[0]
606
+
607
+
608
+ def _value_score(path: str, value: Any, tokens: Sequence[str]) -> float:
609
+ normalized_path = _normalize_token(path)
610
+ if isinstance(value, bool):
611
+ if any(token in normalized_path for token in ("allow_cross_origin", "unsafe", "disable", "ignore")):
612
+ return 3.0 if value is False else 0.0
613
+ return 3.0 if value is True else 0.0
614
+ if isinstance(value, (int, float)) and not isinstance(value, bool):
615
+ return float(value)
616
+ if isinstance(value, (list, tuple, set)):
617
+ return len(value) + sum(_value_score(path, item, tokens) * 0.1 for item in value)
618
+ if isinstance(value, Mapping):
619
+ return len(value) + sum(1.0 for item in value.values() if item)
620
+
621
+ text = _normalize_token(value)
622
+ score = 0.0
623
+ for token in tokens:
624
+ normalized = _normalize_token(token)
625
+ if normalized and normalized in text:
626
+ score += 2.0
627
+ if "search_locality" in normalized_path:
628
+ for preferred in ("local", "bounded", "neighborhood", "nearby"):
629
+ if preferred in text:
630
+ score += 2.0
631
+ for avoided in ("global", "unbounded"):
632
+ if avoided in text:
633
+ score -= 1.0
634
+ for preferred in (
635
+ "enabled",
636
+ "enable",
637
+ "true",
638
+ "capture",
639
+ "captured",
640
+ "full",
641
+ "complete",
642
+ "stream_events",
643
+ "langgraph_stream_events",
644
+ "trace",
645
+ "tracing",
646
+ "otel",
647
+ "opentelemetry",
648
+ "checkpoint",
649
+ "checkpointer",
650
+ "persistent",
651
+ "thread_id",
652
+ "configurable",
653
+ "sqlite",
654
+ "store",
655
+ "case_status",
656
+ "lookup_then_refund",
657
+ "summary",
658
+ "current",
659
+ "fresh",
660
+ "grounded",
661
+ "citation",
662
+ "evidence",
663
+ "red_team",
664
+ "explicit",
665
+ "society_trace",
666
+ "role_graph",
667
+ "role_weighted",
668
+ "proposal",
669
+ "proposals",
670
+ "credit",
671
+ "governance",
672
+ "contract_gate",
673
+ "rollback",
674
+ "search_locality",
675
+ "bounded",
676
+ "steward",
677
+ "manifest",
678
+ "manifest_optimization",
679
+ "required",
680
+ "requirement",
681
+ "quality_gate",
682
+ ):
683
+ if preferred in text:
684
+ score += 1.0
685
+ for avoided in ("none", "disabled", "manual", "buffer", "refund_only", "stale", "unsafe"):
686
+ if avoided in text:
687
+ score -= 1.0
688
+ return score
689
+
690
+
691
+ def _filter_patch(
692
+ patch: Mapping[str, Any],
693
+ *,
694
+ target: OptimizationTarget,
695
+ allowed_paths: Sequence[str],
696
+ ) -> dict[str, Any]:
697
+ filtered: dict[str, Any] = {}
698
+ base = target.seed_candidate()
699
+ allowed = set(allowed_paths)
700
+ for path, value in patch.items():
701
+ values = target.search_space.get(path)
702
+ if path not in allowed or not values:
703
+ continue
704
+ if not any(_json_equal(value, candidate_value) for candidate_value in values):
705
+ continue
706
+ if not _json_equal(value, base.get_path(path)):
707
+ filtered[path] = value
708
+ return filtered
709
+
710
+
711
+ def _dedupe_bundles(bundles: Sequence[AgentMutationBundle]) -> list[AgentMutationBundle]:
712
+ seen: set[str] = set()
713
+ deduped: list[AgentMutationBundle] = []
714
+ for bundle in bundles:
715
+ key = json.dumps(bundle.patch, sort_keys=True, default=str)
716
+ if key in seen:
717
+ continue
718
+ seen.add(key)
719
+ deduped.append(bundle)
720
+ return deduped
721
+
722
+
723
+ def _dominant_component(
724
+ rule: FrameworkMutationRule,
725
+ diagnoses: Sequence[ComponentDiagnosis],
726
+ ) -> str:
727
+ for diagnosis in diagnoses:
728
+ if str(diagnosis.component) in rule.components:
729
+ return str(diagnosis.component)
730
+ return rule.components[0] if rule.components else "framework"
731
+
732
+
733
+ def _target_hints(target: OptimizationTarget) -> str:
734
+ payload = {
735
+ "name": target.name,
736
+ "layers": target.layers,
737
+ "metadata": target.metadata,
738
+ "paths": list(target.search_space),
739
+ }
740
+ return _normalize_token(payload)
741
+
742
+
743
+ def _path_matches(path: str, tokens: Sequence[str]) -> bool:
744
+ normalized_path = _normalize_token(path)
745
+ return any(_normalize_token(token) in normalized_path for token in tokens if _normalize_token(token))
746
+
747
+
748
+ def _path_match_score(path: str, tokens: Sequence[str]) -> float:
749
+ normalized_path = _normalize_token(path)
750
+ score = 0.0
751
+ for token in tokens:
752
+ normalized = _normalize_token(token)
753
+ if not normalized:
754
+ continue
755
+ if normalized_path == normalized:
756
+ score += 4.0
757
+ elif normalized_path.startswith(f"{normalized}.") or normalized_path.startswith(f"{normalized}_"):
758
+ score += 3.0
759
+ elif normalized in normalized_path:
760
+ score += 1.0
761
+ return score
762
+
763
+
764
+ def _json_equal(left: Any, right: Any) -> bool:
765
+ return json.dumps(left, sort_keys=True, default=str) == json.dumps(right, sort_keys=True, default=str)
766
+
767
+
768
+ def _normalize_token(value: Any) -> str:
769
+ if not isinstance(value, str):
770
+ value = json.dumps(value, sort_keys=True, default=str)
771
+ return value.lower().replace("-", "_").replace(" ", "_")