agent-learning-kit 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (642) hide show
  1. agent_learning_kit-0.1.0.dist-info/METADATA +381 -0
  2. agent_learning_kit-0.1.0.dist-info/RECORD +642 -0
  3. agent_learning_kit-0.1.0.dist-info/WHEEL +4 -0
  4. agent_learning_kit-0.1.0.dist-info/entry_points.txt +5 -0
  5. agent_learning_kit-0.1.0.dist-info/licenses/LICENSE +173 -0
  6. agent_learning_kit-0.1.0.dist-info/licenses/NOTICE +7 -0
  7. fi/__init__.py +5 -0
  8. fi/alk/__init__.py +57 -0
  9. fi/alk/_facade.py +31 -0
  10. fi/alk/_module_alias.py +68 -0
  11. fi/alk/_paths.py +14 -0
  12. fi/alk/_schema.py +522 -0
  13. fi/alk/actions.py +727 -0
  14. fi/alk/bench/__init__.py +517 -0
  15. fi/alk/bench/_codeexec.py +213 -0
  16. fi/alk/bench/_coding.py +215 -0
  17. fi/alk/bench/_docker.py +237 -0
  18. fi/alk/bench/_grader.py +286 -0
  19. fi/alk/bench/_pull.py +212 -0
  20. fi/alk/bench/_voice.py +147 -0
  21. fi/alk/capabilities.py +627 -0
  22. fi/alk/cli.py +6396 -0
  23. fi/alk/config.py +130 -0
  24. fi/alk/cua_loop.py +562 -0
  25. fi/alk/evals.py +2351 -0
  26. fi/alk/extensions.py +163 -0
  27. fi/alk/harness/ARCHITECTURE.md +231 -0
  28. fi/alk/harness/DESIGN.md +246 -0
  29. fi/alk/harness/ENVIRONMENT_CONFORMANCE.md +127 -0
  30. fi/alk/harness/HOW-IT-WORKS.md +297 -0
  31. fi/alk/harness/IMPLEMENTATION_AND_VALIDATION_STATUS.md +229 -0
  32. fi/alk/harness/README.md +417 -0
  33. fi/alk/harness/__init__.py +77 -0
  34. fi/alk/harness/__main__.py +3 -0
  35. fi/alk/harness/amend.py +312 -0
  36. fi/alk/harness/artifacts.py +319 -0
  37. fi/alk/harness/authoring_entrypoint.py +189 -0
  38. fi/alk/harness/authoring_runtime_validation.py +267 -0
  39. fi/alk/harness/backends/README.md +43 -0
  40. fi/alk/harness/backends/__init__.py +122 -0
  41. fi/alk/harness/backends/base.py +241 -0
  42. fi/alk/harness/backends/claude.py +211 -0
  43. fi/alk/harness/backends/files.py +182 -0
  44. fi/alk/harness/backends/vertex_gemini.py +457 -0
  45. fi/alk/harness/background_noise.py +95 -0
  46. fi/alk/harness/build.py +385 -0
  47. fi/alk/harness/bundle.py +593 -0
  48. fi/alk/harness/bundle_author_v2.py +1831 -0
  49. fi/alk/harness/bundle_v2.py +719 -0
  50. fi/alk/harness/call_runner.py +1440 -0
  51. fi/alk/harness/callback_http_adapter.py +111 -0
  52. fi/alk/harness/catalogue.py +287 -0
  53. fi/alk/harness/chat.py +428 -0
  54. fi/alk/harness/chat_call_runner.py +506 -0
  55. fi/alk/harness/checks.py +136 -0
  56. fi/alk/harness/cli.py +1354 -0
  57. fi/alk/harness/config.py +338 -0
  58. fi/alk/harness/contract.py +718 -0
  59. fi/alk/harness/credentials.py +674 -0
  60. fi/alk/harness/data/persona_vocabulary.json +111 -0
  61. fi/alk/harness/environment.py +99 -0
  62. fi/alk/harness/environment_plan.py +168 -0
  63. fi/alk/harness/events.py +125 -0
  64. fi/alk/harness/executor.py +304 -0
  65. fi/alk/harness/folder.py +234 -0
  66. fi/alk/harness/generated_runtime.py +815 -0
  67. fi/alk/harness/github.py +72 -0
  68. fi/alk/harness/hosted_authoring_entrypoint.py +183 -0
  69. fi/alk/harness/hosted_entrypoint.py +2402 -0
  70. fi/alk/harness/hosted_scheduler.py +2218 -0
  71. fi/alk/harness/job.py +426 -0
  72. fi/alk/harness/judge.py +184 -0
  73. fi/alk/harness/livekit_source.py +50 -0
  74. fi/alk/harness/livekit_tool_trace_bootstrap.py +71 -0
  75. fi/alk/harness/observability.py +208 -0
  76. fi/alk/harness/outbound.py +3252 -0
  77. fi/alk/harness/packaging.py +515 -0
  78. fi/alk/harness/persona_guides.py +157 -0
  79. fi/alk/harness/platform.py +692 -0
  80. fi/alk/harness/process_preflight.py +764 -0
  81. fi/alk/harness/process_runtime.py +5670 -0
  82. fi/alk/harness/prove.py +425 -0
  83. fi/alk/harness/provider_import.py +703 -0
  84. fi/alk/harness/provider_lifecycle.py +392 -0
  85. fi/alk/harness/provision.py +2896 -0
  86. fi/alk/harness/reception.py +147 -0
  87. fi/alk/harness/retell_chat_call_runner.py +373 -0
  88. fi/alk/harness/run/__init__.py +296 -0
  89. fi/alk/harness/run/alk.py +184 -0
  90. fi/alk/harness/run/call.py +162 -0
  91. fi/alk/harness/run/conversation.py +264 -0
  92. fi/alk/harness/run/data/voices_by_language_and_gender.json +693 -0
  93. fi/alk/harness/run/evidence.py +195 -0
  94. fi/alk/harness/run/grade.py +598 -0
  95. fi/alk/harness/run/live.py +297 -0
  96. fi/alk/harness/run/models.py +56 -0
  97. fi/alk/harness/run/platform_evals.py +227 -0
  98. fi/alk/harness/run/sdk_voice.py +130 -0
  99. fi/alk/harness/run/simulation.py +1209 -0
  100. fi/alk/harness/run/stage.py +91 -0
  101. fi/alk/harness/run/targets.py +508 -0
  102. fi/alk/harness/run/tools.py +601 -0
  103. fi/alk/harness/run/voice.py +340 -0
  104. fi/alk/harness/runtime.py +172 -0
  105. fi/alk/harness/sandbox_server.py +2011 -0
  106. fi/alk/harness/sandbox_worker.py +44 -0
  107. fi/alk/harness/scenario.py +1048 -0
  108. fi/alk/harness/scenario_source.py +879 -0
  109. fi/alk/harness/scenario_tools.py +1143 -0
  110. fi/alk/harness/scenarios.py +915 -0
  111. fi/alk/harness/secrets.py +168 -0
  112. fi/alk/harness/service_catalog.py +97 -0
  113. fi/alk/harness/session.py +391 -0
  114. fi/alk/harness/sessions.py +372 -0
  115. fi/alk/harness/simulator.py +76 -0
  116. fi/alk/harness/simulator_voice.py +928 -0
  117. fi/alk/harness/skills/build-environment/SKILL.md +538 -0
  118. fi/alk/harness/skills/harness.md +131 -0
  119. fi/alk/harness/skills/kinds/chat.md +48 -0
  120. fi/alk/harness/skills/kinds/voice-voicemail.md +63 -0
  121. fi/alk/harness/skills/kinds/voice.md +59 -0
  122. fi/alk/harness/skills/plan-suite/SKILL.md +103 -0
  123. fi/alk/harness/skills/provision-environment/SKILL.md +136 -0
  124. fi/alk/harness/skills/run-scenarios/SKILL.md +112 -0
  125. fi/alk/harness/skills/understand-agent/SKILL.md +251 -0
  126. fi/alk/harness/skills/write-scenarios/SKILL.md +606 -0
  127. fi/alk/harness/skills/write-scenarios/references/refusals.md +28 -0
  128. fi/alk/harness/skills/write-scenarios/references/world-api.md +92 -0
  129. fi/alk/harness/source_data_invariants.py +444 -0
  130. fi/alk/harness/source_tool_evidence.py +79 -0
  131. fi/alk/harness/sources.py +253 -0
  132. fi/alk/harness/spend.py +140 -0
  133. fi/alk/harness/tool_trace_proxy.py +104 -0
  134. fi/alk/harness/tools.py +1018 -0
  135. fi/alk/harness/understand.py +169 -0
  136. fi/alk/harness/voicemail_audio.py +74 -0
  137. fi/alk/harness/world/__init__.py +33 -0
  138. fi/alk/harness/world/errors.py +68 -0
  139. fi/alk/harness/world/expectations.py +91 -0
  140. fi/alk/harness/world/handle.py +538 -0
  141. fi/alk/harness/world/kinds.py +196 -0
  142. fi/alk/harness/world/mutate.py +186 -0
  143. fi/alk/harness/world/probe.py +413 -0
  144. fi/alk/harness/world/provision.py +511 -0
  145. fi/alk/harness/world/provisioned.py +191 -0
  146. fi/alk/harness/world/runtime.py +616 -0
  147. fi/alk/harness/world/snapshot.py +288 -0
  148. fi/alk/harness/world/stores/__init__.py +305 -0
  149. fi/alk/harness/world/stores/container.py +215 -0
  150. fi/alk/harness/world/stores/inprocess.py +346 -0
  151. fi/alk/harness/world/stores/postgres.py +481 -0
  152. fi/alk/harness/world/stores/prove.py +202 -0
  153. fi/alk/harness/world/stores/sqlite.py +245 -0
  154. fi/alk/harness/world/stores/written.py +182 -0
  155. fi/alk/harness/world/tools.py +1516 -0
  156. fi/alk/harness/world/workspace.py +144 -0
  157. fi/alk/image_loop.py +453 -0
  158. fi/alk/image_perturb.py +241 -0
  159. fi/alk/improve.py +274 -0
  160. fi/alk/live/__init__.py +154 -0
  161. fi/alk/live/_attribution.py +184 -0
  162. fi/alk/live/_capture.py +264 -0
  163. fi/alk/live/_codec.py +391 -0
  164. fi/alk/live/_contract.py +134 -0
  165. fi/alk/live/_loopback.py +316 -0
  166. fi/alk/live/_perturb.py +449 -0
  167. fi/alk/live/_runner.py +386 -0
  168. fi/alk/live/_stats.py +561 -0
  169. fi/alk/live/_transcript.py +240 -0
  170. fi/alk/live/_workers/__init__.py +9 -0
  171. fi/alk/live/_workers/a2a_worker.py +316 -0
  172. fi/alk/live/_workers/langgraph_worker.py +217 -0
  173. fi/alk/live/_workers/livekit_worker.py +207 -0
  174. fi/alk/live/_workers/mcp_loopback_server.py +46 -0
  175. fi/alk/live/_workers/mcp_worker.py +158 -0
  176. fi/alk/live/_workers/pipecat_worker.py +189 -0
  177. fi/alk/live/a2a_lane.py +138 -0
  178. fi/alk/live/langgraph_lane.py +339 -0
  179. fi/alk/live/livekit_lane.py +376 -0
  180. fi/alk/live/mcp_lane.py +172 -0
  181. fi/alk/live/pipecat_lane.py +341 -0
  182. fi/alk/live/voice_redteam.py +494 -0
  183. fi/alk/loss.py +306 -0
  184. fi/alk/optimize.py +36260 -0
  185. fi/alk/practice/__init__.py +51 -0
  186. fi/alk/practice/_assess.py +103 -0
  187. fi/alk/practice/_budget.py +81 -0
  188. fi/alk/practice/_calibrate.py +69 -0
  189. fi/alk/practice/_capstone.py +86 -0
  190. fi/alk/practice/_contract.py +91 -0
  191. fi/alk/practice/_diagnose.py +79 -0
  192. fi/alk/practice/_drill.py +196 -0
  193. fi/alk/practice/_experiment.py +720 -0
  194. fi/alk/practice/_schedule.py +102 -0
  195. fi/alk/practice/_store.py +194 -0
  196. fi/alk/practice/_trainer.py +245 -0
  197. fi/alk/practice/_update.py +125 -0
  198. fi/alk/redteam.py +2621 -0
  199. fi/alk/rewardhack.py +237 -0
  200. fi/alk/simulate.py +10351 -0
  201. fi/alk/studio/__init__.py +82 -0
  202. fi/alk/studio/_bias.py +314 -0
  203. fi/alk/studio/_calibration.py +522 -0
  204. fi/alk/studio/_coverage.py +262 -0
  205. fi/alk/studio/_download.py +665 -0
  206. fi/alk/studio/_fidelity_attack.py +114 -0
  207. fi/alk/studio/_generate.py +652 -0
  208. fi/alk/studio/_library.py +370 -0
  209. fi/alk/studio/_scan.py +134 -0
  210. fi/alk/studio/_upgrade.py +42 -0
  211. fi/alk/studio/_vendor.py +172 -0
  212. fi/alk/suite.py +4200 -0
  213. fi/alk/tasks.py +828 -0
  214. fi/alk/telemetry/__init__.py +149 -0
  215. fi/alk/telemetry/_contract.py +141 -0
  216. fi/alk/telemetry/_emit.py +182 -0
  217. fi/alk/telemetry/_ledger.py +296 -0
  218. fi/alk/telemetry/_queue.py +127 -0
  219. fi/alk/telemetry/_row.py +294 -0
  220. fi/alk/telemetry/_run.py +233 -0
  221. fi/alk/telemetry/_sync.py +193 -0
  222. fi/alk/telemetry/_url.py +119 -0
  223. fi/alk/trinity.py +49397 -0
  224. fi/alk/voice_loop.py +174 -0
  225. fi/api/__init__.py +1 -0
  226. fi/api/auth.py +137 -0
  227. fi/api/types.py +29 -0
  228. fi/cli/__init__.py +9 -0
  229. fi/cli/assertions/__init__.py +25 -0
  230. fi/cli/assertions/conditions.py +76 -0
  231. fi/cli/assertions/evaluator.py +286 -0
  232. fi/cli/assertions/exit_codes.py +20 -0
  233. fi/cli/assertions/parser.py +131 -0
  234. fi/cli/assertions/reporter.py +194 -0
  235. fi/cli/commands/__init__.py +9 -0
  236. fi/cli/commands/config.py +165 -0
  237. fi/cli/commands/export.py +208 -0
  238. fi/cli/commands/init.py +112 -0
  239. fi/cli/commands/list_cmd.py +213 -0
  240. fi/cli/commands/run.py +486 -0
  241. fi/cli/commands/validate.py +173 -0
  242. fi/cli/commands/view.py +424 -0
  243. fi/cli/config/__init__.py +6 -0
  244. fi/cli/config/defaults.py +206 -0
  245. fi/cli/config/loader.py +155 -0
  246. fi/cli/config/schema.py +174 -0
  247. fi/cli/main.py +78 -0
  248. fi/cli/output/__init__.py +6 -0
  249. fi/cli/output/formatters.py +106 -0
  250. fi/cli/output/reporters.py +46 -0
  251. fi/cli/storage/__init__.py +5 -0
  252. fi/cli/storage/run_history.py +249 -0
  253. fi/cli/utils/__init__.py +5 -0
  254. fi/cli/utils/console.py +44 -0
  255. fi/evals/__init__.py +131 -0
  256. fi/evals/autoeval/__init__.py +137 -0
  257. fi/evals/autoeval/analyzer.py +211 -0
  258. fi/evals/autoeval/config.py +244 -0
  259. fi/evals/autoeval/export.py +213 -0
  260. fi/evals/autoeval/interactive.py +283 -0
  261. fi/evals/autoeval/pipeline.py +625 -0
  262. fi/evals/autoeval/prompts.py +139 -0
  263. fi/evals/autoeval/recommender.py +242 -0
  264. fi/evals/autoeval/rules.py +589 -0
  265. fi/evals/autoeval/templates.py +299 -0
  266. fi/evals/autoeval/types.py +232 -0
  267. fi/evals/core/__init__.py +16 -0
  268. fi/evals/core/cloud_registry.py +184 -0
  269. fi/evals/core/engines.py +368 -0
  270. fi/evals/core/evaluate.py +319 -0
  271. fi/evals/core/judge_prompt.py +90 -0
  272. fi/evals/core/prompt_generator.py +83 -0
  273. fi/evals/core/registry.py +57 -0
  274. fi/evals/core/result.py +55 -0
  275. fi/evals/evaluator.py +721 -0
  276. fi/evals/execution.py +168 -0
  277. fi/evals/feedback/__init__.py +32 -0
  278. fi/evals/feedback/calibrator.py +160 -0
  279. fi/evals/feedback/collector.py +214 -0
  280. fi/evals/feedback/hooks.py +81 -0
  281. fi/evals/feedback/retriever.py +128 -0
  282. fi/evals/feedback/store.py +272 -0
  283. fi/evals/feedback/types.py +99 -0
  284. fi/evals/framework/README.md +79 -0
  285. fi/evals/framework/__init__.py +267 -0
  286. fi/evals/framework/backends/Dockerfile.eval-runner +33 -0
  287. fi/evals/framework/backends/__init__.py +99 -0
  288. fi/evals/framework/backends/_container.py +141 -0
  289. fi/evals/framework/backends/_utils.py +145 -0
  290. fi/evals/framework/backends/base.py +223 -0
  291. fi/evals/framework/backends/celery_backend.py +417 -0
  292. fi/evals/framework/backends/celery_worker.py +78 -0
  293. fi/evals/framework/backends/kubernetes_backend.py +665 -0
  294. fi/evals/framework/backends/ray_backend.py +521 -0
  295. fi/evals/framework/backends/temporal.py +350 -0
  296. fi/evals/framework/backends/temporal_worker.py +126 -0
  297. fi/evals/framework/backends/thread_pool.py +286 -0
  298. fi/evals/framework/context.py +258 -0
  299. fi/evals/framework/enrichment.py +306 -0
  300. fi/evals/framework/evals/__init__.py +68 -0
  301. fi/evals/framework/evals/agentic.py +399 -0
  302. fi/evals/framework/evals/builder.py +609 -0
  303. fi/evals/framework/evals/semantic.py +142 -0
  304. fi/evals/framework/evaluator.py +647 -0
  305. fi/evals/framework/evaluators/__init__.py +22 -0
  306. fi/evals/framework/evaluators/blocking.py +347 -0
  307. fi/evals/framework/evaluators/non_blocking.py +577 -0
  308. fi/evals/framework/propagation.py +421 -0
  309. fi/evals/framework/protocols.py +385 -0
  310. fi/evals/framework/registry.py +370 -0
  311. fi/evals/framework/resilience/__init__.py +150 -0
  312. fi/evals/framework/resilience/circuit_breaker.py +309 -0
  313. fi/evals/framework/resilience/degradation.py +355 -0
  314. fi/evals/framework/resilience/health.py +505 -0
  315. fi/evals/framework/resilience/rate_limiter.py +228 -0
  316. fi/evals/framework/resilience/retry.py +274 -0
  317. fi/evals/framework/resilience/types.py +288 -0
  318. fi/evals/framework/resilience/wrapper.py +433 -0
  319. fi/evals/framework/types.py +218 -0
  320. fi/evals/guardrails/README.md +915 -0
  321. fi/evals/guardrails/__init__.py +96 -0
  322. fi/evals/guardrails/backends/__init__.py +43 -0
  323. fi/evals/guardrails/backends/azure.py +361 -0
  324. fi/evals/guardrails/backends/base.py +88 -0
  325. fi/evals/guardrails/backends/generic_llm.py +163 -0
  326. fi/evals/guardrails/backends/granite.py +216 -0
  327. fi/evals/guardrails/backends/llamaguard.py +221 -0
  328. fi/evals/guardrails/backends/local_base.py +479 -0
  329. fi/evals/guardrails/backends/openai.py +365 -0
  330. fi/evals/guardrails/backends/qwen.py +170 -0
  331. fi/evals/guardrails/backends/shieldgemma.py +154 -0
  332. fi/evals/guardrails/backends/turing.py +235 -0
  333. fi/evals/guardrails/backends/vllm_client.py +321 -0
  334. fi/evals/guardrails/backends/wildguard.py +188 -0
  335. fi/evals/guardrails/base.py +888 -0
  336. fi/evals/guardrails/config.py +221 -0
  337. fi/evals/guardrails/discovery.py +243 -0
  338. fi/evals/guardrails/gateway.py +437 -0
  339. fi/evals/guardrails/registry.py +231 -0
  340. fi/evals/guardrails/scanners/__init__.py +127 -0
  341. fi/evals/guardrails/scanners/base.py +191 -0
  342. fi/evals/guardrails/scanners/code_injection.py +243 -0
  343. fi/evals/guardrails/scanners/eval_delegate.py +574 -0
  344. fi/evals/guardrails/scanners/invisible_chars.py +351 -0
  345. fi/evals/guardrails/scanners/jailbreak.py +412 -0
  346. fi/evals/guardrails/scanners/language.py +288 -0
  347. fi/evals/guardrails/scanners/pipeline.py +260 -0
  348. fi/evals/guardrails/scanners/regex.py +311 -0
  349. fi/evals/guardrails/scanners/secrets.py +274 -0
  350. fi/evals/guardrails/scanners/topics.py +649 -0
  351. fi/evals/guardrails/scanners/urls.py +341 -0
  352. fi/evals/guardrails/types.py +96 -0
  353. fi/evals/llm/__init__.py +3 -0
  354. fi/evals/llm/base_llm_provider.py +35 -0
  355. fi/evals/llm/providers/litellm.py +70 -0
  356. fi/evals/local/__init__.py +90 -0
  357. fi/evals/local/evaluator.py +690 -0
  358. fi/evals/local/execution_mode.py +121 -0
  359. fi/evals/local/llm.py +489 -0
  360. fi/evals/local/metrics/__init__.py +19 -0
  361. fi/evals/local/registry.py +360 -0
  362. fi/evals/manager.py +1018 -0
  363. fi/evals/manager_types.py +362 -0
  364. fi/evals/metrics/__init__.py +185 -0
  365. fi/evals/metrics/agents/__init__.py +74 -0
  366. fi/evals/metrics/agents/metrics.py +693 -0
  367. fi/evals/metrics/agents/report.py +36463 -0
  368. fi/evals/metrics/agents/types.py +160 -0
  369. fi/evals/metrics/base_llm_metric.py +111 -0
  370. fi/evals/metrics/base_metric.py +138 -0
  371. fi/evals/metrics/code_security/__init__.py +305 -0
  372. fi/evals/metrics/code_security/analyzer.py +985 -0
  373. fi/evals/metrics/code_security/benchmarks/__init__.py +73 -0
  374. fi/evals/metrics/code_security/benchmarks/builtin.py +750 -0
  375. fi/evals/metrics/code_security/benchmarks/loader.py +580 -0
  376. fi/evals/metrics/code_security/benchmarks/types.py +308 -0
  377. fi/evals/metrics/code_security/detectors/__init__.py +186 -0
  378. fi/evals/metrics/code_security/detectors/base.py +394 -0
  379. fi/evals/metrics/code_security/detectors/cryptography.py +345 -0
  380. fi/evals/metrics/code_security/detectors/injection.py +744 -0
  381. fi/evals/metrics/code_security/detectors/secrets.py +287 -0
  382. fi/evals/metrics/code_security/detectors/serialization.py +192 -0
  383. fi/evals/metrics/code_security/joint_metrics.py +588 -0
  384. fi/evals/metrics/code_security/judges/__init__.py +83 -0
  385. fi/evals/metrics/code_security/judges/base.py +238 -0
  386. fi/evals/metrics/code_security/judges/dual_judge.py +534 -0
  387. fi/evals/metrics/code_security/judges/llm_judge.py +301 -0
  388. fi/evals/metrics/code_security/judges/pattern_judge.py +515 -0
  389. fi/evals/metrics/code_security/metrics.py +388 -0
  390. fi/evals/metrics/code_security/modes/__init__.py +63 -0
  391. fi/evals/metrics/code_security/modes/adversarial.py +284 -0
  392. fi/evals/metrics/code_security/modes/autocomplete.py +198 -0
  393. fi/evals/metrics/code_security/modes/base.py +283 -0
  394. fi/evals/metrics/code_security/modes/instruct.py +253 -0
  395. fi/evals/metrics/code_security/modes/repair.py +230 -0
  396. fi/evals/metrics/code_security/reports/__init__.py +57 -0
  397. fi/evals/metrics/code_security/reports/generator.py +404 -0
  398. fi/evals/metrics/code_security/reports/leaderboard.py +509 -0
  399. fi/evals/metrics/code_security/types.py +534 -0
  400. fi/evals/metrics/function_calling/__init__.py +34 -0
  401. fi/evals/metrics/function_calling/metrics.py +573 -0
  402. fi/evals/metrics/function_calling/types.py +87 -0
  403. fi/evals/metrics/hallucination/__init__.py +54 -0
  404. fi/evals/metrics/hallucination/detector.py +149 -0
  405. fi/evals/metrics/hallucination/metrics.py +390 -0
  406. fi/evals/metrics/hallucination/nli.py +253 -0
  407. fi/evals/metrics/hallucination/sentinel.py +106 -0
  408. fi/evals/metrics/hallucination/types.py +132 -0
  409. fi/evals/metrics/heuristics/aggregation_metrics.py +85 -0
  410. fi/evals/metrics/heuristics/json_metrics.py +87 -0
  411. fi/evals/metrics/heuristics/similarity_metrics.py +375 -0
  412. fi/evals/metrics/heuristics/string_metrics.py +391 -0
  413. fi/evals/metrics/llm_as_judges/__init__.py +17 -0
  414. fi/evals/metrics/llm_as_judges/custom_judge/metric.py +112 -0
  415. fi/evals/metrics/llm_as_judges/custom_judge/prompts.py +26 -0
  416. fi/evals/metrics/llm_as_judges/types.py +48 -0
  417. fi/evals/metrics/rag/__init__.py +111 -0
  418. fi/evals/metrics/rag/advanced/__init__.py +14 -0
  419. fi/evals/metrics/rag/advanced/multi_hop.py +283 -0
  420. fi/evals/metrics/rag/advanced/source_attribution.py +344 -0
  421. fi/evals/metrics/rag/generation/__init__.py +17 -0
  422. fi/evals/metrics/rag/generation/answer_relevancy.py +176 -0
  423. fi/evals/metrics/rag/generation/context_utilization.py +245 -0
  424. fi/evals/metrics/rag/generation/faithfulness.py +241 -0
  425. fi/evals/metrics/rag/generation/groundedness.py +131 -0
  426. fi/evals/metrics/rag/rag_score.py +277 -0
  427. fi/evals/metrics/rag/retrieval/__init__.py +20 -0
  428. fi/evals/metrics/rag/retrieval/context_entity_recall.py +124 -0
  429. fi/evals/metrics/rag/retrieval/context_precision.py +158 -0
  430. fi/evals/metrics/rag/retrieval/context_recall.py +106 -0
  431. fi/evals/metrics/rag/retrieval/noise_sensitivity.py +163 -0
  432. fi/evals/metrics/rag/retrieval/ranking.py +261 -0
  433. fi/evals/metrics/rag/types.py +100 -0
  434. fi/evals/metrics/rag/utils/__init__.py +62 -0
  435. fi/evals/metrics/rag/utils/claims.py +189 -0
  436. fi/evals/metrics/rag/utils/entities.py +244 -0
  437. fi/evals/metrics/rag/utils/nli.py +92 -0
  438. fi/evals/metrics/rag/utils/similarity.py +345 -0
  439. fi/evals/metrics/structured/__init__.py +114 -0
  440. fi/evals/metrics/structured/field_completeness.py +313 -0
  441. fi/evals/metrics/structured/hierarchy_score.py +366 -0
  442. fi/evals/metrics/structured/json_validation.py +190 -0
  443. fi/evals/metrics/structured/schema_compliance.py +280 -0
  444. fi/evals/metrics/structured/structured_output_score.py +298 -0
  445. fi/evals/metrics/structured/types.py +108 -0
  446. fi/evals/metrics/structured/validators/__init__.py +30 -0
  447. fi/evals/metrics/structured/validators/base.py +189 -0
  448. fi/evals/metrics/structured/validators/json_validator.py +196 -0
  449. fi/evals/metrics/structured/validators/pydantic_validator.py +178 -0
  450. fi/evals/metrics/structured/validators/yaml_validator.py +248 -0
  451. fi/evals/otel/__init__.py +266 -0
  452. fi/evals/otel/config.py +400 -0
  453. fi/evals/otel/conventions.py +463 -0
  454. fi/evals/otel/enrichment.py +371 -0
  455. fi/evals/otel/instrumentors/__init__.py +140 -0
  456. fi/evals/otel/instrumentors/anthropic.py +517 -0
  457. fi/evals/otel/instrumentors/base.py +382 -0
  458. fi/evals/otel/instrumentors/openai.py +673 -0
  459. fi/evals/otel/processors/__init__.py +36 -0
  460. fi/evals/otel/processors/base.py +473 -0
  461. fi/evals/otel/processors/cost.py +445 -0
  462. fi/evals/otel/processors/evaluation.py +559 -0
  463. fi/evals/otel/processors/llm.py +462 -0
  464. fi/evals/otel/tracer.py +506 -0
  465. fi/evals/otel/types.py +232 -0
  466. fi/evals/otel_utils.py +23 -0
  467. fi/evals/protect.py +671 -0
  468. fi/evals/protect_input_adapter.py +154 -0
  469. fi/evals/streaming/__init__.py +88 -0
  470. fi/evals/streaming/buffer.py +213 -0
  471. fi/evals/streaming/evaluator.py +551 -0
  472. fi/evals/streaming/policy.py +307 -0
  473. fi/evals/streaming/scorers.py +368 -0
  474. fi/evals/streaming/types.py +238 -0
  475. fi/evals/templates.py +472 -0
  476. fi/evals/types.py +156 -0
  477. fi/opt/__init__.py +221 -0
  478. fi/opt/_objective_scoring.py +85 -0
  479. fi/opt/base/__init__.py +11 -0
  480. fi/opt/base/base_generator.py +33 -0
  481. fi/opt/base/base_mapper.py +26 -0
  482. fi/opt/base/base_optimizer.py +45 -0
  483. fi/opt/base/evaluator.py +211 -0
  484. fi/opt/components.py +3095 -0
  485. fi/opt/datamappers/__init__.py +3 -0
  486. fi/opt/datamappers/basic_mapper.py +40 -0
  487. fi/opt/deployment.py +1021 -0
  488. fi/opt/evidence.py +4332 -0
  489. fi/opt/generators/__init__.py +3 -0
  490. fi/opt/generators/litellm.py +66 -0
  491. fi/opt/integrations/__init__.py +23 -0
  492. fi/opt/integrations/generative_suite.py +410 -0
  493. fi/opt/integrations/simulate.py +1313 -0
  494. fi/opt/mutations.py +771 -0
  495. fi/opt/observability.py +4639 -0
  496. fi/opt/optimizer_trace.py +889 -0
  497. fi/opt/optimizers/__init__.py +80 -0
  498. fi/opt/optimizers/agent.py +331 -0
  499. fi/opt/optimizers/agent_bandit.py +392 -0
  500. fi/opt/optimizers/agent_curriculum.py +635 -0
  501. fi/opt/optimizers/agent_evolution.py +894 -0
  502. fi/opt/optimizers/agent_feedback.py +1863 -0
  503. fi/opt/optimizers/agent_pareto.py +547 -0
  504. fi/opt/optimizers/agent_social_memory.py +1113 -0
  505. fi/opt/optimizers/agent_tpe.py +321 -0
  506. fi/opt/optimizers/bayesian_search.py +449 -0
  507. fi/opt/optimizers/council.py +2075 -0
  508. fi/opt/optimizers/futureagi_replay.py +799 -0
  509. fi/opt/optimizers/gepa.py +322 -0
  510. fi/opt/optimizers/metaprompt.py +243 -0
  511. fi/opt/optimizers/promptwizard.py +417 -0
  512. fi/opt/optimizers/protegi.py +329 -0
  513. fi/opt/optimizers/random_search.py +224 -0
  514. fi/opt/research.py +518 -0
  515. fi/opt/simulation.py +260 -0
  516. fi/opt/targets.py +232 -0
  517. fi/opt/types.py +66 -0
  518. fi/opt/utils/__init__.py +4 -0
  519. fi/opt/utils/early_stopping.py +266 -0
  520. fi/opt/utils/setup_logging.py +82 -0
  521. fi/simulate/__init__.py +540 -0
  522. fi/simulate/_hashing.py +35 -0
  523. fi/simulate/_logging.py +10 -0
  524. fi/simulate/adapters.py +87 -0
  525. fi/simulate/agent/__init__.py +120 -0
  526. fi/simulate/agent/browser.py +658 -0
  527. fi/simulate/agent/definition.py +587 -0
  528. fi/simulate/agent/frameworks.py +3528 -0
  529. fi/simulate/agent/generic.py +8286 -0
  530. fi/simulate/agent/import_probe.py +227 -0
  531. fi/simulate/agent/memory.py +905 -0
  532. fi/simulate/agent/mocks.py +101 -0
  533. fi/simulate/agent/multi_agent.py +361 -0
  534. fi/simulate/agent/orchestration.py +903 -0
  535. fi/simulate/agent/realtime.py +665 -0
  536. fi/simulate/agent/wrapper.py +99 -0
  537. fi/simulate/agent/wrappers/__init__.py +18 -0
  538. fi/simulate/agent/wrappers/anthropic.py +62 -0
  539. fi/simulate/agent/wrappers/gemini.py +65 -0
  540. fi/simulate/agent/wrappers/http.py +404 -0
  541. fi/simulate/agent/wrappers/langchain.py +80 -0
  542. fi/simulate/agent/wrappers/openai.py +75 -0
  543. fi/simulate/agent/wrappers/websocket.py +326 -0
  544. fi/simulate/artifacts/__init__.py +11 -0
  545. fi/simulate/artifacts/manifest.py +62 -0
  546. fi/simulate/cli.py +20560 -0
  547. fi/simulate/endpoints/__init__.py +45 -0
  548. fi/simulate/endpoints/_http_actor.py +73 -0
  549. fi/simulate/endpoints/actor_sources.py +243 -0
  550. fi/simulate/endpoints/base.py +107 -0
  551. fi/simulate/endpoints/builtins.py +10 -0
  552. fi/simulate/endpoints/callable.py +95 -0
  553. fi/simulate/endpoints/http.py +76 -0
  554. fi/simulate/endpoints/livekit.py +138 -0
  555. fi/simulate/endpoints/originators.py +132 -0
  556. fi/simulate/endpoints/profiles.py +348 -0
  557. fi/simulate/endpoints/retell.py +633 -0
  558. fi/simulate/endpoints/vapi.py +205 -0
  559. fi/simulate/endpoints/websocket.py +76 -0
  560. fi/simulate/environment.py +33026 -0
  561. fi/simulate/environments/__init__.py +11 -0
  562. fi/simulate/environments/base.py +73 -0
  563. fi/simulate/environments/chat.py +697 -0
  564. fi/simulate/environments/voice.py +212 -0
  565. fi/simulate/evaluation/__init__.py +4 -0
  566. fi/simulate/evaluation/ai_eval.py +227 -0
  567. fi/simulate/evidence/__init__.py +35 -0
  568. fi/simulate/evidence/base.py +59 -0
  569. fi/simulate/evidence/caller_observed.py +50 -0
  570. fi/simulate/evidence/livekit_instrumentation.py +51 -0
  571. fi/simulate/evidence/livekit_room.py +50 -0
  572. fi/simulate/evidence/otel.py +49 -0
  573. fi/simulate/evidence/providers/__init__.py +24 -0
  574. fi/simulate/evidence/providers/base.py +61 -0
  575. fi/simulate/evidence/providers/retell.py +376 -0
  576. fi/simulate/evidence/providers/vapi.py +426 -0
  577. fi/simulate/hosted/__init__.py +32 -0
  578. fi/simulate/hosted/child_entrypoint.py +306 -0
  579. fi/simulate/hosted/job.py +150 -0
  580. fi/simulate/hosted/targets.py +53 -0
  581. fi/simulate/instrumentation/__init__.py +5 -0
  582. fi/simulate/instrumentation/livekit/__init__.py +122 -0
  583. fi/simulate/manifest.py +1033 -0
  584. fi/simulate/matrix_cli.py +165 -0
  585. fi/simulate/realtime/__init__.py +40 -0
  586. fi/simulate/realtime/events.py +107 -0
  587. fi/simulate/realtime/media.py +61 -0
  588. fi/simulate/realtime/session.py +91 -0
  589. fi/simulate/recording/__init__.py +5 -0
  590. fi/simulate/recording/room_recorder.py +326 -0
  591. fi/simulate/registry.py +185 -0
  592. fi/simulate/results/__init__.py +9 -0
  593. fi/simulate/results/base.py +18 -0
  594. fi/simulate/results/filesystem.py +71 -0
  595. fi/simulate/results/futureagi.py +1340 -0
  596. fi/simulate/runtime/__init__.py +85 -0
  597. fi/simulate/runtime/capabilities.py +40 -0
  598. fi/simulate/runtime/events.py +63 -0
  599. fi/simulate/runtime/failures.py +25 -0
  600. fi/simulate/runtime/ids.py +34 -0
  601. fi/simulate/runtime/plan.py +70 -0
  602. fi/simulate/runtime/planner.py +102 -0
  603. fi/simulate/runtime/report.py +174 -0
  604. fi/simulate/runtime/run.py +75 -0
  605. fi/simulate/runtime/runner.py +333 -0
  606. fi/simulate/runtime/spec.py +186 -0
  607. fi/simulate/simulation/__init__.py +30 -0
  608. fi/simulate/simulation/behavior_policy.py +425 -0
  609. fi/simulate/simulation/bridge/__init__.py +9 -0
  610. fi/simulate/simulation/bridge/audio.py +29 -0
  611. fi/simulate/simulation/bridge/connector.py +46 -0
  612. fi/simulate/simulation/bridge/livekit.py +252 -0
  613. fi/simulate/simulation/bridge/retell.py +188 -0
  614. fi/simulate/simulation/bridge/vapi.py +177 -0
  615. fi/simulate/simulation/contract.py +419 -0
  616. fi/simulate/simulation/engines/__init__.py +12 -0
  617. fi/simulate/simulation/engines/base.py +21 -0
  618. fi/simulate/simulation/engines/cloud.py +517 -0
  619. fi/simulate/simulation/engines/livekit.py +4167 -0
  620. fi/simulate/simulation/engines/local_text.py +89 -0
  621. fi/simulate/simulation/fidelity.py +374 -0
  622. fi/simulate/simulation/gemini_tts_stream.py +110 -0
  623. fi/simulate/simulation/generator.py +91 -0
  624. fi/simulate/simulation/goal_machine.py +185 -0
  625. fi/simulate/simulation/livekit_models.py +467 -0
  626. fi/simulate/simulation/matrix.py +170 -0
  627. fi/simulate/simulation/models.py +279 -0
  628. fi/simulate/simulation/runner.py +153 -0
  629. fi/simulate/simulation/synthetic.py +880 -0
  630. fi/simulate/simulation/voice_prompt.py +502 -0
  631. fi/simulate/simulator/__init__.py +55 -0
  632. fi/simulate/simulator/builtins.py +53 -0
  633. fi/simulate/suite.py +1288 -0
  634. fi/simulate/utils/routes.py +164 -0
  635. fi/simulate/voice.py +225 -0
  636. fi/simulate/voice_cli.py +182 -0
  637. fi/utils/__init__.py +1 -0
  638. fi/utils/constants.py +14 -0
  639. fi/utils/errors.py +200 -0
  640. fi/utils/executor.py +26 -0
  641. fi/utils/routes.py +119 -0
  642. fi/utils/utils.py +17 -0
fi/alk/actions.py ADDED
@@ -0,0 +1,727 @@
1
+ from __future__ import annotations
2
+
3
+ import contextlib
4
+ import copy
5
+ import importlib
6
+ import io
7
+ import json
8
+ import os
9
+ from pathlib import Path
10
+ from typing import Any, Iterable, Mapping, Optional
11
+
12
+ from ._schema import AGENT_LEARNING_CLI_SCHEMA_VERSION, normalize_public_payload
13
+
14
+ AGENT_LEARNING_ACTIONS_KIND = "agent-learning.actions.v1"
15
+ AGENT_LEARNING_ACTION_RUN_KIND = "agent-learning.action-run.v1"
16
+ _MISSING = object()
17
+
18
+
19
+ def load_artifact_file(path: str | Path) -> dict[str, Any]:
20
+ artifact_path = Path(path).expanduser().resolve()
21
+ loaded = _load_json_or_yaml(artifact_path)
22
+ if not isinstance(loaded, Mapping):
23
+ raise ValueError("action artifact root must be an object")
24
+ return dict(loaded)
25
+
26
+
27
+ def extract_actions(
28
+ artifact: Mapping[str, Any],
29
+ *,
30
+ action_id: Optional[str] = None,
31
+ ) -> list[dict[str, Any]]:
32
+ """Return embedded executable actions from an Agent Learning artifact/report."""
33
+
34
+ normalized = normalize_public_payload(artifact)
35
+ if not isinstance(normalized, Mapping):
36
+ return []
37
+ actions: list[dict[str, Any]] = []
38
+ seen: set[tuple[str, str, str, str, str]] = set()
39
+ for path, action in _walk_actions(normalized):
40
+ if action_id is not None and str(action.get("id") or "") != action_id:
41
+ continue
42
+ record = copy.deepcopy(dict(action))
43
+ record["path"] = path
44
+ record["source_card_path"] = _source_card_path(path)
45
+ key = (
46
+ str(record.get("id") or ""),
47
+ str(record.get("kind") or ""),
48
+ str(record.get("command") or ""),
49
+ str(record.get("artifact_ref") or ""),
50
+ str(record.get("path") or ""),
51
+ )
52
+ if key in seen:
53
+ continue
54
+ seen.add(key)
55
+ actions.append(record)
56
+ return actions
57
+
58
+
59
+ def get_action(
60
+ artifact: Mapping[str, Any],
61
+ action_id: str,
62
+ *,
63
+ source_path: str | Path = ".",
64
+ ) -> Optional[dict[str, Any]]:
65
+ actions = action_catalog(
66
+ artifact,
67
+ source_path=source_path,
68
+ action_id=action_id,
69
+ )["actions"]
70
+ return actions[0] if actions else None
71
+
72
+
73
+ def run_action(
74
+ artifact: Mapping[str, Any],
75
+ action_id: str,
76
+ *,
77
+ source_path: str | Path = ".",
78
+ inputs: Optional[Mapping[str, Any]] = None,
79
+ cwd: str | Path | None = None,
80
+ dry_run: bool = False,
81
+ name: Optional[str] = None,
82
+ artifact_output_path: str | Path | None = None,
83
+ ) -> dict[str, Any]:
84
+ action = get_action(artifact, action_id, source_path=source_path)
85
+ if action is None:
86
+ raise ValueError(f"action not found: {action_id}")
87
+ action_kind = str(action.get("kind") or "cli")
88
+ if action_kind == "download":
89
+ return _run_download_action(
90
+ artifact,
91
+ action_id,
92
+ action,
93
+ source_path=source_path,
94
+ inputs=inputs or {},
95
+ cwd=cwd,
96
+ dry_run=dry_run,
97
+ name=name,
98
+ artifact_output_path=artifact_output_path,
99
+ )
100
+ if action_kind != "cli":
101
+ raise ValueError(f"unsupported action kind: {action_kind}")
102
+
103
+ command_args = _resolved_command_args(action, inputs or {})
104
+ if not command_args:
105
+ raise ValueError(f"action {action_id!r} does not include command_args")
106
+ command_name = command_args[0]
107
+ if command_name != "agent-learn":
108
+ raise ValueError(
109
+ f"unsupported action command: {command_name}; use agent-learn"
110
+ )
111
+ if len(command_args) < 2:
112
+ raise ValueError(f"action {action_id!r} is missing a subcommand")
113
+ subcommand = command_args[1]
114
+ if subcommand in {"action-run", "run-action"}:
115
+ raise ValueError("action-run cannot recursively execute action-run")
116
+
117
+ run_cwd = Path(cwd).expanduser().resolve() if cwd is not None else Path.cwd()
118
+ command_args = _absolutize_output_args(command_args, run_cwd)
119
+ output_records = _command_output_records(command_args, run_cwd)
120
+ exit_code = 0
121
+ logs = {"stdout": "", "stderr": "", "stdout_bytes": 0, "stderr_bytes": 0}
122
+ if not dry_run:
123
+ dispatch = _dispatch_action_command(command_args, cwd=run_cwd)
124
+ exit_code = int(dispatch["exit_code"])
125
+ logs = {
126
+ "stdout": str(dispatch.get("stdout") or ""),
127
+ "stderr": str(dispatch.get("stderr") or ""),
128
+ "stdout_bytes": int(dispatch.get("stdout_bytes") or 0),
129
+ "stderr_bytes": int(dispatch.get("stderr_bytes") or 0),
130
+ }
131
+ output_records = _command_output_records(command_args, run_cwd)
132
+ status = "passed" if exit_code == 0 else "failed"
133
+ outputs_written_count = sum(
134
+ 1 for item in output_records if item.get("exists") is True
135
+ )
136
+ output_count = len(output_records)
137
+ output_completion_rate = (
138
+ round(outputs_written_count / output_count, 4)
139
+ if output_count
140
+ else 1.0
141
+ )
142
+ payload = {
143
+ "schema_version": AGENT_LEARNING_CLI_SCHEMA_VERSION,
144
+ "kind": AGENT_LEARNING_ACTION_RUN_KIND,
145
+ "name": str(name or f"{action_id}-action-run"),
146
+ "status": status,
147
+ "exit_code": exit_code,
148
+ "source_path": str(source_path),
149
+ "cwd": str(run_cwd),
150
+ "dry_run": bool(dry_run),
151
+ "action": action,
152
+ "command": " ".join(_shell_token(arg) for arg in command_args),
153
+ "command_args": command_args,
154
+ "logs": logs,
155
+ "outputs": output_records,
156
+ "outputs_written": [
157
+ str(item["path"])
158
+ for item in output_records
159
+ if item.get("exists") is True
160
+ ],
161
+ "summary": {
162
+ "action_id": str(action.get("id") or action_id),
163
+ "action_label": action.get("label"),
164
+ "action_kind": action_kind,
165
+ "source_card_path": action.get("source_card_path"),
166
+ "requires_input": bool(action.get("inputs")),
167
+ "command_exit_code": exit_code,
168
+ "stdout_bytes": logs["stdout_bytes"],
169
+ "stderr_bytes": logs["stderr_bytes"],
170
+ "output_count": output_count,
171
+ "outputs_written_count": outputs_written_count,
172
+ "output_completion_rate": output_completion_rate,
173
+ },
174
+ }
175
+ return payload
176
+
177
+
178
+ def _run_download_action(
179
+ artifact: Mapping[str, Any],
180
+ action_id: str,
181
+ action: Mapping[str, Any],
182
+ *,
183
+ source_path: str | Path,
184
+ inputs: Mapping[str, Any],
185
+ cwd: str | Path | None,
186
+ dry_run: bool,
187
+ name: Optional[str],
188
+ artifact_output_path: str | Path | None,
189
+ ) -> dict[str, Any]:
190
+ artifact_ref = str(action.get("artifact_ref") or "")
191
+ if not artifact_ref:
192
+ raise ValueError(f"download action {action_id!r} is missing artifact_ref")
193
+
194
+ value = _resolve_artifact_ref(
195
+ artifact,
196
+ artifact_ref,
197
+ source_path=source_path,
198
+ )
199
+ if value is _MISSING:
200
+ raise ValueError(
201
+ f"download action {action_id!r} artifact_ref not found: {artifact_ref}"
202
+ )
203
+
204
+ run_cwd = Path(cwd).expanduser().resolve() if cwd is not None else Path.cwd()
205
+ output_path = _download_output_path(
206
+ action,
207
+ inputs,
208
+ run_cwd,
209
+ artifact_output_path=artifact_output_path,
210
+ )
211
+ output_record = {
212
+ "flag": "--artifact-output",
213
+ "path": str(output_path),
214
+ "exists": output_path.exists(),
215
+ "artifact_ref": artifact_ref,
216
+ }
217
+ if not dry_run:
218
+ output_path.parent.mkdir(parents=True, exist_ok=True)
219
+ output_path.write_text(
220
+ _download_payload_text(value, output_path),
221
+ encoding="utf-8",
222
+ )
223
+ output_record["exists"] = output_path.exists()
224
+
225
+ outputs_written = [str(output_path)] if output_record["exists"] else []
226
+ output_count = 1
227
+ outputs_written_count = len(outputs_written)
228
+ payload = {
229
+ "schema_version": AGENT_LEARNING_CLI_SCHEMA_VERSION,
230
+ "kind": AGENT_LEARNING_ACTION_RUN_KIND,
231
+ "name": str(name or f"{action_id}-action-run"),
232
+ "status": "passed",
233
+ "exit_code": 0,
234
+ "source_path": str(source_path),
235
+ "cwd": str(run_cwd),
236
+ "dry_run": bool(dry_run),
237
+ "action": action,
238
+ "command": f"download {artifact_ref} -> {output_path}",
239
+ "command_args": [],
240
+ "artifact_ref": artifact_ref,
241
+ "artifact_output_path": str(output_path),
242
+ "logs": {"stdout": "", "stderr": "", "stdout_bytes": 0, "stderr_bytes": 0},
243
+ "outputs": [output_record],
244
+ "outputs_written": outputs_written,
245
+ "summary": {
246
+ "action_id": str(action.get("id") or action_id),
247
+ "action_label": action.get("label"),
248
+ "action_kind": "download",
249
+ "source_card_path": action.get("source_card_path"),
250
+ "requires_input": bool(action.get("inputs")),
251
+ "command_exit_code": 0,
252
+ "stdout_bytes": 0,
253
+ "stderr_bytes": 0,
254
+ "output_count": output_count,
255
+ "outputs_written_count": outputs_written_count,
256
+ "output_completion_rate": round(outputs_written_count / output_count, 4),
257
+ "artifact_ref": artifact_ref,
258
+ },
259
+ }
260
+ return payload
261
+
262
+
263
+ def action_catalog(
264
+ artifact: Mapping[str, Any],
265
+ *,
266
+ source_path: str | Path = ".",
267
+ action_id: Optional[str] = None,
268
+ name: Optional[str] = None,
269
+ ) -> dict[str, Any]:
270
+ artifacts = [artifact, _generic_report_action_artifact(source_path)]
271
+ synthesized = _synthesized_report_artifact(artifact, source_path=source_path)
272
+ if synthesized is not None:
273
+ artifacts.append(synthesized)
274
+ actions: list[dict[str, Any]] = []
275
+ seen: set[str] = set()
276
+ for item in artifacts:
277
+ for action in extract_actions(item, action_id=action_id):
278
+ key = str(action.get("id") or action.get("command") or action.get("path") or "")
279
+ if key in seen:
280
+ continue
281
+ seen.add(key)
282
+ action["requires_input"] = bool(action.get("inputs"))
283
+ actions.append(action)
284
+ source = normalize_public_payload(artifact)
285
+ source_kind = source.get("kind", source.get("schema_version")) if isinstance(source, Mapping) else None
286
+ source_name = source.get("name") if isinstance(source, Mapping) else None
287
+ cards = sorted({
288
+ str(item.get("source_card_path"))
289
+ for item in actions
290
+ if item.get("source_card_path")
291
+ })
292
+ payload = {
293
+ "schema_version": AGENT_LEARNING_CLI_SCHEMA_VERSION,
294
+ "kind": AGENT_LEARNING_ACTIONS_KIND,
295
+ "name": str(name or source_name or Path(source_path).stem),
296
+ "status": "passed",
297
+ "exit_code": 0,
298
+ "source_path": str(source_path),
299
+ "actions": actions,
300
+ "summary": {
301
+ "source_kind": source_kind,
302
+ "source_name": source_name,
303
+ "action_count": len(actions),
304
+ "action_ids": [str(item.get("id")) for item in actions if item.get("id")],
305
+ "source_card_paths": cards,
306
+ },
307
+ }
308
+ if action_id is not None:
309
+ payload["summary"]["filter_action_id"] = action_id
310
+ return payload
311
+
312
+
313
+ def _generic_report_action_artifact(source_path: str | Path) -> dict[str, Any]:
314
+ return {
315
+ "actions": [
316
+ {
317
+ "id": "report_artifact",
318
+ "kind": "cli",
319
+ "label": "Report artifact",
320
+ "command": "agent-learn report",
321
+ "command_args": [
322
+ "agent-learn",
323
+ "report",
324
+ str(Path(source_path).expanduser().resolve()),
325
+ "--output",
326
+ "report-artifact.json",
327
+ ],
328
+ "target_layers": ["reporting", "diagnosis"],
329
+ }
330
+ ]
331
+ }
332
+
333
+
334
+ def _synthesized_report_artifact(
335
+ artifact: Mapping[str, Any],
336
+ *,
337
+ source_path: str | Path,
338
+ ) -> Optional[dict[str, Any]]:
339
+ try:
340
+ cli = importlib.import_module("fi.alk.simulate.cli")
341
+ report = cli._report_result(
342
+ source=artifact,
343
+ source_path=Path(source_path),
344
+ name=None,
345
+ duration_seconds=0.0,
346
+ )
347
+ except Exception:
348
+ return None
349
+ return report if isinstance(report, dict) else None
350
+
351
+
352
+ def render_markdown(catalog: Mapping[str, Any]) -> str:
353
+ summary = dict(catalog.get("summary") or {})
354
+ rows = [
355
+ "| Action | Label | Source card | Status | Target layers | Command |",
356
+ "| --- | --- | --- | --- | --- | --- |",
357
+ ]
358
+ for action in catalog.get("actions") or []:
359
+ if not isinstance(action, Mapping):
360
+ continue
361
+ rows.append(
362
+ "| "
363
+ + " | ".join(
364
+ _md_cell(value)
365
+ for value in [
366
+ action.get("id"),
367
+ action.get("label"),
368
+ action.get("source_card_path"),
369
+ action.get("readiness_status")
370
+ or action.get("strategy_status")
371
+ or action.get("diagnosis_status")
372
+ or action.get("status"),
373
+ _join_values(action.get("target_layers")),
374
+ action.get("command") or action.get("artifact_ref"),
375
+ ]
376
+ )
377
+ + " |"
378
+ )
379
+ if len(rows) == 2:
380
+ rows.append("| No actions | | | | | |")
381
+ lines = [
382
+ f"# {_md_text(catalog.get('name') or 'artifact-actions')}",
383
+ "",
384
+ f"- Source: `{_md_code(catalog.get('source_path') or '.')}`",
385
+ f"- Source kind: {_md_text(summary.get('source_kind') or 'unknown')}",
386
+ f"- Actions: {_md_text(summary.get('action_count') or 0)}",
387
+ "",
388
+ "## Actions",
389
+ "",
390
+ *rows,
391
+ "",
392
+ ]
393
+ return "\n".join(lines)
394
+
395
+
396
+ def render_action_run_markdown(result: Mapping[str, Any]) -> str:
397
+ rows = [
398
+ "| Output | Exists |",
399
+ "| --- | --- |",
400
+ ]
401
+ for item in result.get("outputs") or []:
402
+ if not isinstance(item, Mapping):
403
+ continue
404
+ rows.append(
405
+ "| "
406
+ + " | ".join(_md_cell(value) for value in [item.get("path"), item.get("exists")])
407
+ + " |"
408
+ )
409
+ if len(rows) == 2:
410
+ rows.append("| No declared outputs | |")
411
+ summary = dict(result.get("summary") or {})
412
+ logs = dict(result.get("logs") or {})
413
+ lines = [
414
+ f"# {_md_text(result.get('name') or 'action-run')}",
415
+ "",
416
+ f"- Source: `{_md_code(result.get('source_path') or '.')}`",
417
+ f"- Action: {_md_text(summary.get('action_id') or 'unknown')}",
418
+ f"- Action kind: {_md_text(summary.get('action_kind') or 'cli')}",
419
+ f"- Status: {_md_text(result.get('status') or 'unknown')}",
420
+ f"- Exit code: {_md_text(result.get('exit_code'))}",
421
+ f"- Command: `{_md_code(result.get('command') or '')}`",
422
+ "",
423
+ "## Outputs",
424
+ "",
425
+ *rows,
426
+ "",
427
+ "## Logs",
428
+ "",
429
+ f"- Stdout bytes: {_md_text(summary.get('stdout_bytes') or 0)}",
430
+ f"- Stderr bytes: {_md_text(summary.get('stderr_bytes') or 0)}",
431
+ "",
432
+ ]
433
+ stdout = str(logs.get("stdout") or "")
434
+ stderr = str(logs.get("stderr") or "")
435
+ if stdout:
436
+ lines.extend(["### Stdout", "", "```text", stdout.rstrip(), "```", ""])
437
+ if stderr:
438
+ lines.extend(["### Stderr", "", "```text", stderr.rstrip(), "```", ""])
439
+ return "\n".join(lines)
440
+
441
+
442
+ def _walk_actions(value: Any, path: str = "") -> Iterable[tuple[str, Mapping[str, Any]]]:
443
+ if isinstance(value, Mapping):
444
+ if (
445
+ value.get("kind") == "cli"
446
+ and value.get("command_args") is not None
447
+ ) or (
448
+ value.get("kind") == "download"
449
+ and value.get("artifact_ref") is not None
450
+ ):
451
+ yield path, value
452
+ return
453
+ for key, item in value.items():
454
+ item_path = f"{path}.{key}" if path else str(key)
455
+ yield from _walk_actions(item, item_path)
456
+ elif isinstance(value, list):
457
+ for index, item in enumerate(value):
458
+ item_path = f"{path}.{index}" if path else str(index)
459
+ yield from _walk_actions(item, item_path)
460
+
461
+
462
+ def _source_card_path(action_path: str) -> str:
463
+ marker = ".actions."
464
+ if marker in action_path:
465
+ source = action_path.split(marker, 1)[0]
466
+ return source.removeprefix("report.")
467
+ if action_path.endswith(".actions"):
468
+ return action_path[: -len(".actions")].removeprefix("report.")
469
+ return action_path
470
+
471
+
472
+ def _resolved_command_args(
473
+ action: Mapping[str, Any],
474
+ inputs: Mapping[str, Any],
475
+ ) -> list[str]:
476
+ input_defaults = {
477
+ str(item.get("name")): item.get("default")
478
+ for item in action.get("inputs") or []
479
+ if isinstance(item, Mapping) and item.get("name") not in (None, "")
480
+ }
481
+ values = {**input_defaults, **{str(key): value for key, value in inputs.items()}}
482
+ resolved: list[str] = []
483
+ for raw_arg in action.get("command_args") or []:
484
+ arg = str(raw_arg)
485
+ for name, value in values.items():
486
+ arg = arg.replace("{{" + name + "}}", str(value))
487
+ if "{{" in arg or "}}" in arg:
488
+ raise ValueError(f"action {action.get('id')!r} requires input for {arg}")
489
+ resolved.append(arg)
490
+ return resolved
491
+
492
+
493
+ def _dispatch_action_command(command_args: list[str], *, cwd: Path) -> dict[str, Any]:
494
+ previous_cwd = Path.cwd()
495
+ cwd.mkdir(parents=True, exist_ok=True)
496
+ stdout = io.StringIO()
497
+ stderr = io.StringIO()
498
+ try:
499
+ os.chdir(cwd)
500
+ with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr):
501
+ cli = importlib.import_module("fi.alk.cli")
502
+ exit_code = int(cli.main(command_args[1:]))
503
+ finally:
504
+ os.chdir(previous_cwd)
505
+ stdout_text = stdout.getvalue()
506
+ stderr_text = stderr.getvalue()
507
+ return {
508
+ "exit_code": exit_code,
509
+ "stdout": stdout_text,
510
+ "stderr": stderr_text,
511
+ "stdout_bytes": len(stdout_text.encode("utf-8")),
512
+ "stderr_bytes": len(stderr_text.encode("utf-8")),
513
+ }
514
+
515
+
516
+ def _command_output_records(command_args: list[str], cwd: Path) -> list[dict[str, Any]]:
517
+ records: list[dict[str, Any]] = []
518
+ for flag, value in _command_output_values(command_args):
519
+ path = Path(str(value)).expanduser()
520
+ if not path.is_absolute():
521
+ path = cwd / path
522
+ path = path.resolve()
523
+ records.append({"flag": flag, "path": str(path), "exists": path.exists()})
524
+ return records
525
+
526
+
527
+ def _absolutize_output_args(command_args: list[str], cwd: Path) -> list[str]:
528
+ output_flags = {"-o", "--output", "--junit", "--sarif", "--markdown", "--md"}
529
+ resolved = list(command_args)
530
+ index = 0
531
+ while index < len(resolved):
532
+ arg = resolved[index]
533
+ if arg in output_flags and index + 1 < len(resolved):
534
+ resolved[index + 1] = str(_output_arg_path(resolved[index + 1], cwd))
535
+ index += 2
536
+ continue
537
+ replaced = False
538
+ for flag in output_flags:
539
+ prefix = flag + "="
540
+ if arg.startswith(prefix):
541
+ resolved[index] = prefix + str(_output_arg_path(arg[len(prefix):], cwd))
542
+ replaced = True
543
+ break
544
+ index += 1
545
+ if replaced:
546
+ continue
547
+ return resolved
548
+
549
+
550
+ def _command_output_values(command_args: list[str]) -> list[tuple[str, str]]:
551
+ output_flags = {"-o", "--output", "--junit", "--sarif", "--markdown", "--md"}
552
+ values: list[tuple[str, str]] = []
553
+ index = 0
554
+ while index < len(command_args):
555
+ arg = command_args[index]
556
+ flag: Optional[str] = None
557
+ value: Optional[str] = None
558
+ if arg in output_flags and index + 1 < len(command_args):
559
+ flag = arg
560
+ value = command_args[index + 1]
561
+ index += 2
562
+ else:
563
+ for candidate in output_flags:
564
+ prefix = candidate + "="
565
+ if arg.startswith(prefix):
566
+ flag = candidate
567
+ value = arg[len(prefix):]
568
+ break
569
+ index += 1
570
+ if flag is None or value in (None, ""):
571
+ continue
572
+ values.append((flag, str(value)))
573
+ return values
574
+
575
+
576
+ def _output_arg_path(value: str, cwd: Path) -> Path:
577
+ path = Path(str(value)).expanduser()
578
+ if path.is_absolute():
579
+ return path.resolve()
580
+ return (cwd / path).resolve()
581
+
582
+
583
+ def _resolve_artifact_ref(
584
+ artifact: Mapping[str, Any],
585
+ artifact_ref: str,
586
+ *,
587
+ source_path: str | Path = ".",
588
+ ) -> Any:
589
+ normalized = normalize_public_payload(artifact)
590
+ if not isinstance(normalized, Mapping):
591
+ return _MISSING
592
+ candidates = [artifact_ref]
593
+ if artifact_ref.startswith("report."):
594
+ candidates.append(artifact_ref.removeprefix("report."))
595
+ artifact_candidates: list[Mapping[str, Any] | None] = [normalized]
596
+ if artifact_ref.startswith("report."):
597
+ artifact_candidates.append(
598
+ _synthesized_report_artifact(
599
+ artifact,
600
+ source_path=source_path,
601
+ )
602
+ )
603
+ for artifact_candidate in artifact_candidates:
604
+ if not isinstance(artifact_candidate, Mapping):
605
+ continue
606
+ resolved = normalize_public_payload(artifact_candidate)
607
+ if not isinstance(resolved, Mapping):
608
+ continue
609
+ for candidate in candidates:
610
+ value = _resolve_path(resolved, candidate.split("."))
611
+ if value is not _MISSING:
612
+ return value
613
+ return _MISSING
614
+
615
+
616
+ def _resolve_path(value: Any, path: list[str]) -> Any:
617
+ current = value
618
+ for part in path:
619
+ if isinstance(current, Mapping):
620
+ if part not in current:
621
+ return _MISSING
622
+ current = current[part]
623
+ continue
624
+ if isinstance(current, list):
625
+ try:
626
+ index = int(part)
627
+ except ValueError:
628
+ return _MISSING
629
+ if index < 0 or index >= len(current):
630
+ return _MISSING
631
+ current = current[index]
632
+ continue
633
+ return _MISSING
634
+ return current
635
+
636
+
637
+ def _download_output_path(
638
+ action: Mapping[str, Any],
639
+ inputs: Mapping[str, Any],
640
+ cwd: Path,
641
+ *,
642
+ artifact_output_path: str | Path | None,
643
+ ) -> Path:
644
+ raw_value = artifact_output_path
645
+ if raw_value in (None, ""):
646
+ for name in ("artifact_output", "artifact_output_path", "output_path"):
647
+ if inputs.get(name) not in (None, ""):
648
+ raw_value = inputs[name]
649
+ break
650
+ if raw_value in (None, ""):
651
+ raw_value = action.get("default_filename")
652
+ if raw_value in (None, ""):
653
+ raw_value = f"{_slug(action.get('id') or 'artifact')}.json"
654
+
655
+ path = Path(str(raw_value)).expanduser()
656
+ if path.is_absolute():
657
+ return path.resolve()
658
+ return (cwd / path).resolve()
659
+
660
+
661
+ def _download_payload_text(value: Any, output_path: Path) -> str:
662
+ if isinstance(value, str) and output_path.suffix.lower() not in {".json", ".jsonl"}:
663
+ return value if value.endswith("\n") else value + "\n"
664
+ return json.dumps(value, indent=2, sort_keys=True, default=str) + "\n"
665
+
666
+
667
+ def _slug(value: Any, default: str = "artifact") -> str:
668
+ text = str(value or default).strip().lower()
669
+ chars = [char if char.isalnum() else "-" for char in text]
670
+ slug = "-".join(part for part in "".join(chars).split("-") if part)
671
+ return slug or default
672
+
673
+
674
+ def _shell_token(value: Any) -> str:
675
+ text = str(value)
676
+ if all(char.isalnum() or char in "-_./:=@" for char in text):
677
+ return text
678
+ return "'" + text.replace("'", "'\"'\"'") + "'"
679
+
680
+
681
+ def _load_json_or_yaml(path: Path) -> Any:
682
+ if not path.exists():
683
+ raise ValueError(f"action artifact file not found: {path}")
684
+ if path.suffix.lower() in {".yaml", ".yml"}:
685
+ try:
686
+ import yaml # type: ignore
687
+ except Exception as exc: # pragma: no cover - optional dependency clarity
688
+ raise ValueError("YAML artifacts require PyYAML; use JSON or install PyYAML.") from exc
689
+ with path.open("r", encoding="utf-8") as handle:
690
+ return yaml.safe_load(handle)
691
+ with path.open("r", encoding="utf-8") as handle:
692
+ return json.load(handle)
693
+
694
+
695
+ def _join_values(value: Any) -> Optional[str]:
696
+ if value in (None, "", [], {}):
697
+ return None
698
+ if isinstance(value, (list, tuple, set)):
699
+ values = [str(item) for item in value if item not in (None, "")]
700
+ return ", ".join(values) if values else None
701
+ return str(value)
702
+
703
+
704
+ def _md_text(value: Any) -> str:
705
+ return str(value).replace("\n", " ")
706
+
707
+
708
+ def _md_code(value: Any) -> str:
709
+ return str(value).replace("`", "\\`")
710
+
711
+
712
+ def _md_cell(value: Any) -> str:
713
+ text = _md_text(value if value is not None else "").replace("|", "\\|")
714
+ return text if len(text) <= 140 else f"{text[:137]}..."
715
+
716
+
717
+ __all__ = [
718
+ "AGENT_LEARNING_ACTIONS_KIND",
719
+ "AGENT_LEARNING_ACTION_RUN_KIND",
720
+ "action_catalog",
721
+ "extract_actions",
722
+ "get_action",
723
+ "load_artifact_file",
724
+ "render_action_run_markdown",
725
+ "render_markdown",
726
+ "run_action",
727
+ ]