agent-learning-kit 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (642) hide show
  1. agent_learning_kit-0.1.0.dist-info/METADATA +381 -0
  2. agent_learning_kit-0.1.0.dist-info/RECORD +642 -0
  3. agent_learning_kit-0.1.0.dist-info/WHEEL +4 -0
  4. agent_learning_kit-0.1.0.dist-info/entry_points.txt +5 -0
  5. agent_learning_kit-0.1.0.dist-info/licenses/LICENSE +173 -0
  6. agent_learning_kit-0.1.0.dist-info/licenses/NOTICE +7 -0
  7. fi/__init__.py +5 -0
  8. fi/alk/__init__.py +57 -0
  9. fi/alk/_facade.py +31 -0
  10. fi/alk/_module_alias.py +68 -0
  11. fi/alk/_paths.py +14 -0
  12. fi/alk/_schema.py +522 -0
  13. fi/alk/actions.py +727 -0
  14. fi/alk/bench/__init__.py +517 -0
  15. fi/alk/bench/_codeexec.py +213 -0
  16. fi/alk/bench/_coding.py +215 -0
  17. fi/alk/bench/_docker.py +237 -0
  18. fi/alk/bench/_grader.py +286 -0
  19. fi/alk/bench/_pull.py +212 -0
  20. fi/alk/bench/_voice.py +147 -0
  21. fi/alk/capabilities.py +627 -0
  22. fi/alk/cli.py +6396 -0
  23. fi/alk/config.py +130 -0
  24. fi/alk/cua_loop.py +562 -0
  25. fi/alk/evals.py +2351 -0
  26. fi/alk/extensions.py +163 -0
  27. fi/alk/harness/ARCHITECTURE.md +231 -0
  28. fi/alk/harness/DESIGN.md +246 -0
  29. fi/alk/harness/ENVIRONMENT_CONFORMANCE.md +127 -0
  30. fi/alk/harness/HOW-IT-WORKS.md +297 -0
  31. fi/alk/harness/IMPLEMENTATION_AND_VALIDATION_STATUS.md +229 -0
  32. fi/alk/harness/README.md +417 -0
  33. fi/alk/harness/__init__.py +77 -0
  34. fi/alk/harness/__main__.py +3 -0
  35. fi/alk/harness/amend.py +312 -0
  36. fi/alk/harness/artifacts.py +319 -0
  37. fi/alk/harness/authoring_entrypoint.py +189 -0
  38. fi/alk/harness/authoring_runtime_validation.py +267 -0
  39. fi/alk/harness/backends/README.md +43 -0
  40. fi/alk/harness/backends/__init__.py +122 -0
  41. fi/alk/harness/backends/base.py +241 -0
  42. fi/alk/harness/backends/claude.py +211 -0
  43. fi/alk/harness/backends/files.py +182 -0
  44. fi/alk/harness/backends/vertex_gemini.py +457 -0
  45. fi/alk/harness/background_noise.py +95 -0
  46. fi/alk/harness/build.py +385 -0
  47. fi/alk/harness/bundle.py +593 -0
  48. fi/alk/harness/bundle_author_v2.py +1831 -0
  49. fi/alk/harness/bundle_v2.py +719 -0
  50. fi/alk/harness/call_runner.py +1440 -0
  51. fi/alk/harness/callback_http_adapter.py +111 -0
  52. fi/alk/harness/catalogue.py +287 -0
  53. fi/alk/harness/chat.py +428 -0
  54. fi/alk/harness/chat_call_runner.py +506 -0
  55. fi/alk/harness/checks.py +136 -0
  56. fi/alk/harness/cli.py +1354 -0
  57. fi/alk/harness/config.py +338 -0
  58. fi/alk/harness/contract.py +718 -0
  59. fi/alk/harness/credentials.py +674 -0
  60. fi/alk/harness/data/persona_vocabulary.json +111 -0
  61. fi/alk/harness/environment.py +99 -0
  62. fi/alk/harness/environment_plan.py +168 -0
  63. fi/alk/harness/events.py +125 -0
  64. fi/alk/harness/executor.py +304 -0
  65. fi/alk/harness/folder.py +234 -0
  66. fi/alk/harness/generated_runtime.py +815 -0
  67. fi/alk/harness/github.py +72 -0
  68. fi/alk/harness/hosted_authoring_entrypoint.py +183 -0
  69. fi/alk/harness/hosted_entrypoint.py +2402 -0
  70. fi/alk/harness/hosted_scheduler.py +2218 -0
  71. fi/alk/harness/job.py +426 -0
  72. fi/alk/harness/judge.py +184 -0
  73. fi/alk/harness/livekit_source.py +50 -0
  74. fi/alk/harness/livekit_tool_trace_bootstrap.py +71 -0
  75. fi/alk/harness/observability.py +208 -0
  76. fi/alk/harness/outbound.py +3252 -0
  77. fi/alk/harness/packaging.py +515 -0
  78. fi/alk/harness/persona_guides.py +157 -0
  79. fi/alk/harness/platform.py +692 -0
  80. fi/alk/harness/process_preflight.py +764 -0
  81. fi/alk/harness/process_runtime.py +5670 -0
  82. fi/alk/harness/prove.py +425 -0
  83. fi/alk/harness/provider_import.py +703 -0
  84. fi/alk/harness/provider_lifecycle.py +392 -0
  85. fi/alk/harness/provision.py +2896 -0
  86. fi/alk/harness/reception.py +147 -0
  87. fi/alk/harness/retell_chat_call_runner.py +373 -0
  88. fi/alk/harness/run/__init__.py +296 -0
  89. fi/alk/harness/run/alk.py +184 -0
  90. fi/alk/harness/run/call.py +162 -0
  91. fi/alk/harness/run/conversation.py +264 -0
  92. fi/alk/harness/run/data/voices_by_language_and_gender.json +693 -0
  93. fi/alk/harness/run/evidence.py +195 -0
  94. fi/alk/harness/run/grade.py +598 -0
  95. fi/alk/harness/run/live.py +297 -0
  96. fi/alk/harness/run/models.py +56 -0
  97. fi/alk/harness/run/platform_evals.py +227 -0
  98. fi/alk/harness/run/sdk_voice.py +130 -0
  99. fi/alk/harness/run/simulation.py +1209 -0
  100. fi/alk/harness/run/stage.py +91 -0
  101. fi/alk/harness/run/targets.py +508 -0
  102. fi/alk/harness/run/tools.py +601 -0
  103. fi/alk/harness/run/voice.py +340 -0
  104. fi/alk/harness/runtime.py +172 -0
  105. fi/alk/harness/sandbox_server.py +2011 -0
  106. fi/alk/harness/sandbox_worker.py +44 -0
  107. fi/alk/harness/scenario.py +1048 -0
  108. fi/alk/harness/scenario_source.py +879 -0
  109. fi/alk/harness/scenario_tools.py +1143 -0
  110. fi/alk/harness/scenarios.py +915 -0
  111. fi/alk/harness/secrets.py +168 -0
  112. fi/alk/harness/service_catalog.py +97 -0
  113. fi/alk/harness/session.py +391 -0
  114. fi/alk/harness/sessions.py +372 -0
  115. fi/alk/harness/simulator.py +76 -0
  116. fi/alk/harness/simulator_voice.py +928 -0
  117. fi/alk/harness/skills/build-environment/SKILL.md +538 -0
  118. fi/alk/harness/skills/harness.md +131 -0
  119. fi/alk/harness/skills/kinds/chat.md +48 -0
  120. fi/alk/harness/skills/kinds/voice-voicemail.md +63 -0
  121. fi/alk/harness/skills/kinds/voice.md +59 -0
  122. fi/alk/harness/skills/plan-suite/SKILL.md +103 -0
  123. fi/alk/harness/skills/provision-environment/SKILL.md +136 -0
  124. fi/alk/harness/skills/run-scenarios/SKILL.md +112 -0
  125. fi/alk/harness/skills/understand-agent/SKILL.md +251 -0
  126. fi/alk/harness/skills/write-scenarios/SKILL.md +606 -0
  127. fi/alk/harness/skills/write-scenarios/references/refusals.md +28 -0
  128. fi/alk/harness/skills/write-scenarios/references/world-api.md +92 -0
  129. fi/alk/harness/source_data_invariants.py +444 -0
  130. fi/alk/harness/source_tool_evidence.py +79 -0
  131. fi/alk/harness/sources.py +253 -0
  132. fi/alk/harness/spend.py +140 -0
  133. fi/alk/harness/tool_trace_proxy.py +104 -0
  134. fi/alk/harness/tools.py +1018 -0
  135. fi/alk/harness/understand.py +169 -0
  136. fi/alk/harness/voicemail_audio.py +74 -0
  137. fi/alk/harness/world/__init__.py +33 -0
  138. fi/alk/harness/world/errors.py +68 -0
  139. fi/alk/harness/world/expectations.py +91 -0
  140. fi/alk/harness/world/handle.py +538 -0
  141. fi/alk/harness/world/kinds.py +196 -0
  142. fi/alk/harness/world/mutate.py +186 -0
  143. fi/alk/harness/world/probe.py +413 -0
  144. fi/alk/harness/world/provision.py +511 -0
  145. fi/alk/harness/world/provisioned.py +191 -0
  146. fi/alk/harness/world/runtime.py +616 -0
  147. fi/alk/harness/world/snapshot.py +288 -0
  148. fi/alk/harness/world/stores/__init__.py +305 -0
  149. fi/alk/harness/world/stores/container.py +215 -0
  150. fi/alk/harness/world/stores/inprocess.py +346 -0
  151. fi/alk/harness/world/stores/postgres.py +481 -0
  152. fi/alk/harness/world/stores/prove.py +202 -0
  153. fi/alk/harness/world/stores/sqlite.py +245 -0
  154. fi/alk/harness/world/stores/written.py +182 -0
  155. fi/alk/harness/world/tools.py +1516 -0
  156. fi/alk/harness/world/workspace.py +144 -0
  157. fi/alk/image_loop.py +453 -0
  158. fi/alk/image_perturb.py +241 -0
  159. fi/alk/improve.py +274 -0
  160. fi/alk/live/__init__.py +154 -0
  161. fi/alk/live/_attribution.py +184 -0
  162. fi/alk/live/_capture.py +264 -0
  163. fi/alk/live/_codec.py +391 -0
  164. fi/alk/live/_contract.py +134 -0
  165. fi/alk/live/_loopback.py +316 -0
  166. fi/alk/live/_perturb.py +449 -0
  167. fi/alk/live/_runner.py +386 -0
  168. fi/alk/live/_stats.py +561 -0
  169. fi/alk/live/_transcript.py +240 -0
  170. fi/alk/live/_workers/__init__.py +9 -0
  171. fi/alk/live/_workers/a2a_worker.py +316 -0
  172. fi/alk/live/_workers/langgraph_worker.py +217 -0
  173. fi/alk/live/_workers/livekit_worker.py +207 -0
  174. fi/alk/live/_workers/mcp_loopback_server.py +46 -0
  175. fi/alk/live/_workers/mcp_worker.py +158 -0
  176. fi/alk/live/_workers/pipecat_worker.py +189 -0
  177. fi/alk/live/a2a_lane.py +138 -0
  178. fi/alk/live/langgraph_lane.py +339 -0
  179. fi/alk/live/livekit_lane.py +376 -0
  180. fi/alk/live/mcp_lane.py +172 -0
  181. fi/alk/live/pipecat_lane.py +341 -0
  182. fi/alk/live/voice_redteam.py +494 -0
  183. fi/alk/loss.py +306 -0
  184. fi/alk/optimize.py +36260 -0
  185. fi/alk/practice/__init__.py +51 -0
  186. fi/alk/practice/_assess.py +103 -0
  187. fi/alk/practice/_budget.py +81 -0
  188. fi/alk/practice/_calibrate.py +69 -0
  189. fi/alk/practice/_capstone.py +86 -0
  190. fi/alk/practice/_contract.py +91 -0
  191. fi/alk/practice/_diagnose.py +79 -0
  192. fi/alk/practice/_drill.py +196 -0
  193. fi/alk/practice/_experiment.py +720 -0
  194. fi/alk/practice/_schedule.py +102 -0
  195. fi/alk/practice/_store.py +194 -0
  196. fi/alk/practice/_trainer.py +245 -0
  197. fi/alk/practice/_update.py +125 -0
  198. fi/alk/redteam.py +2621 -0
  199. fi/alk/rewardhack.py +237 -0
  200. fi/alk/simulate.py +10351 -0
  201. fi/alk/studio/__init__.py +82 -0
  202. fi/alk/studio/_bias.py +314 -0
  203. fi/alk/studio/_calibration.py +522 -0
  204. fi/alk/studio/_coverage.py +262 -0
  205. fi/alk/studio/_download.py +665 -0
  206. fi/alk/studio/_fidelity_attack.py +114 -0
  207. fi/alk/studio/_generate.py +652 -0
  208. fi/alk/studio/_library.py +370 -0
  209. fi/alk/studio/_scan.py +134 -0
  210. fi/alk/studio/_upgrade.py +42 -0
  211. fi/alk/studio/_vendor.py +172 -0
  212. fi/alk/suite.py +4200 -0
  213. fi/alk/tasks.py +828 -0
  214. fi/alk/telemetry/__init__.py +149 -0
  215. fi/alk/telemetry/_contract.py +141 -0
  216. fi/alk/telemetry/_emit.py +182 -0
  217. fi/alk/telemetry/_ledger.py +296 -0
  218. fi/alk/telemetry/_queue.py +127 -0
  219. fi/alk/telemetry/_row.py +294 -0
  220. fi/alk/telemetry/_run.py +233 -0
  221. fi/alk/telemetry/_sync.py +193 -0
  222. fi/alk/telemetry/_url.py +119 -0
  223. fi/alk/trinity.py +49397 -0
  224. fi/alk/voice_loop.py +174 -0
  225. fi/api/__init__.py +1 -0
  226. fi/api/auth.py +137 -0
  227. fi/api/types.py +29 -0
  228. fi/cli/__init__.py +9 -0
  229. fi/cli/assertions/__init__.py +25 -0
  230. fi/cli/assertions/conditions.py +76 -0
  231. fi/cli/assertions/evaluator.py +286 -0
  232. fi/cli/assertions/exit_codes.py +20 -0
  233. fi/cli/assertions/parser.py +131 -0
  234. fi/cli/assertions/reporter.py +194 -0
  235. fi/cli/commands/__init__.py +9 -0
  236. fi/cli/commands/config.py +165 -0
  237. fi/cli/commands/export.py +208 -0
  238. fi/cli/commands/init.py +112 -0
  239. fi/cli/commands/list_cmd.py +213 -0
  240. fi/cli/commands/run.py +486 -0
  241. fi/cli/commands/validate.py +173 -0
  242. fi/cli/commands/view.py +424 -0
  243. fi/cli/config/__init__.py +6 -0
  244. fi/cli/config/defaults.py +206 -0
  245. fi/cli/config/loader.py +155 -0
  246. fi/cli/config/schema.py +174 -0
  247. fi/cli/main.py +78 -0
  248. fi/cli/output/__init__.py +6 -0
  249. fi/cli/output/formatters.py +106 -0
  250. fi/cli/output/reporters.py +46 -0
  251. fi/cli/storage/__init__.py +5 -0
  252. fi/cli/storage/run_history.py +249 -0
  253. fi/cli/utils/__init__.py +5 -0
  254. fi/cli/utils/console.py +44 -0
  255. fi/evals/__init__.py +131 -0
  256. fi/evals/autoeval/__init__.py +137 -0
  257. fi/evals/autoeval/analyzer.py +211 -0
  258. fi/evals/autoeval/config.py +244 -0
  259. fi/evals/autoeval/export.py +213 -0
  260. fi/evals/autoeval/interactive.py +283 -0
  261. fi/evals/autoeval/pipeline.py +625 -0
  262. fi/evals/autoeval/prompts.py +139 -0
  263. fi/evals/autoeval/recommender.py +242 -0
  264. fi/evals/autoeval/rules.py +589 -0
  265. fi/evals/autoeval/templates.py +299 -0
  266. fi/evals/autoeval/types.py +232 -0
  267. fi/evals/core/__init__.py +16 -0
  268. fi/evals/core/cloud_registry.py +184 -0
  269. fi/evals/core/engines.py +368 -0
  270. fi/evals/core/evaluate.py +319 -0
  271. fi/evals/core/judge_prompt.py +90 -0
  272. fi/evals/core/prompt_generator.py +83 -0
  273. fi/evals/core/registry.py +57 -0
  274. fi/evals/core/result.py +55 -0
  275. fi/evals/evaluator.py +721 -0
  276. fi/evals/execution.py +168 -0
  277. fi/evals/feedback/__init__.py +32 -0
  278. fi/evals/feedback/calibrator.py +160 -0
  279. fi/evals/feedback/collector.py +214 -0
  280. fi/evals/feedback/hooks.py +81 -0
  281. fi/evals/feedback/retriever.py +128 -0
  282. fi/evals/feedback/store.py +272 -0
  283. fi/evals/feedback/types.py +99 -0
  284. fi/evals/framework/README.md +79 -0
  285. fi/evals/framework/__init__.py +267 -0
  286. fi/evals/framework/backends/Dockerfile.eval-runner +33 -0
  287. fi/evals/framework/backends/__init__.py +99 -0
  288. fi/evals/framework/backends/_container.py +141 -0
  289. fi/evals/framework/backends/_utils.py +145 -0
  290. fi/evals/framework/backends/base.py +223 -0
  291. fi/evals/framework/backends/celery_backend.py +417 -0
  292. fi/evals/framework/backends/celery_worker.py +78 -0
  293. fi/evals/framework/backends/kubernetes_backend.py +665 -0
  294. fi/evals/framework/backends/ray_backend.py +521 -0
  295. fi/evals/framework/backends/temporal.py +350 -0
  296. fi/evals/framework/backends/temporal_worker.py +126 -0
  297. fi/evals/framework/backends/thread_pool.py +286 -0
  298. fi/evals/framework/context.py +258 -0
  299. fi/evals/framework/enrichment.py +306 -0
  300. fi/evals/framework/evals/__init__.py +68 -0
  301. fi/evals/framework/evals/agentic.py +399 -0
  302. fi/evals/framework/evals/builder.py +609 -0
  303. fi/evals/framework/evals/semantic.py +142 -0
  304. fi/evals/framework/evaluator.py +647 -0
  305. fi/evals/framework/evaluators/__init__.py +22 -0
  306. fi/evals/framework/evaluators/blocking.py +347 -0
  307. fi/evals/framework/evaluators/non_blocking.py +577 -0
  308. fi/evals/framework/propagation.py +421 -0
  309. fi/evals/framework/protocols.py +385 -0
  310. fi/evals/framework/registry.py +370 -0
  311. fi/evals/framework/resilience/__init__.py +150 -0
  312. fi/evals/framework/resilience/circuit_breaker.py +309 -0
  313. fi/evals/framework/resilience/degradation.py +355 -0
  314. fi/evals/framework/resilience/health.py +505 -0
  315. fi/evals/framework/resilience/rate_limiter.py +228 -0
  316. fi/evals/framework/resilience/retry.py +274 -0
  317. fi/evals/framework/resilience/types.py +288 -0
  318. fi/evals/framework/resilience/wrapper.py +433 -0
  319. fi/evals/framework/types.py +218 -0
  320. fi/evals/guardrails/README.md +915 -0
  321. fi/evals/guardrails/__init__.py +96 -0
  322. fi/evals/guardrails/backends/__init__.py +43 -0
  323. fi/evals/guardrails/backends/azure.py +361 -0
  324. fi/evals/guardrails/backends/base.py +88 -0
  325. fi/evals/guardrails/backends/generic_llm.py +163 -0
  326. fi/evals/guardrails/backends/granite.py +216 -0
  327. fi/evals/guardrails/backends/llamaguard.py +221 -0
  328. fi/evals/guardrails/backends/local_base.py +479 -0
  329. fi/evals/guardrails/backends/openai.py +365 -0
  330. fi/evals/guardrails/backends/qwen.py +170 -0
  331. fi/evals/guardrails/backends/shieldgemma.py +154 -0
  332. fi/evals/guardrails/backends/turing.py +235 -0
  333. fi/evals/guardrails/backends/vllm_client.py +321 -0
  334. fi/evals/guardrails/backends/wildguard.py +188 -0
  335. fi/evals/guardrails/base.py +888 -0
  336. fi/evals/guardrails/config.py +221 -0
  337. fi/evals/guardrails/discovery.py +243 -0
  338. fi/evals/guardrails/gateway.py +437 -0
  339. fi/evals/guardrails/registry.py +231 -0
  340. fi/evals/guardrails/scanners/__init__.py +127 -0
  341. fi/evals/guardrails/scanners/base.py +191 -0
  342. fi/evals/guardrails/scanners/code_injection.py +243 -0
  343. fi/evals/guardrails/scanners/eval_delegate.py +574 -0
  344. fi/evals/guardrails/scanners/invisible_chars.py +351 -0
  345. fi/evals/guardrails/scanners/jailbreak.py +412 -0
  346. fi/evals/guardrails/scanners/language.py +288 -0
  347. fi/evals/guardrails/scanners/pipeline.py +260 -0
  348. fi/evals/guardrails/scanners/regex.py +311 -0
  349. fi/evals/guardrails/scanners/secrets.py +274 -0
  350. fi/evals/guardrails/scanners/topics.py +649 -0
  351. fi/evals/guardrails/scanners/urls.py +341 -0
  352. fi/evals/guardrails/types.py +96 -0
  353. fi/evals/llm/__init__.py +3 -0
  354. fi/evals/llm/base_llm_provider.py +35 -0
  355. fi/evals/llm/providers/litellm.py +70 -0
  356. fi/evals/local/__init__.py +90 -0
  357. fi/evals/local/evaluator.py +690 -0
  358. fi/evals/local/execution_mode.py +121 -0
  359. fi/evals/local/llm.py +489 -0
  360. fi/evals/local/metrics/__init__.py +19 -0
  361. fi/evals/local/registry.py +360 -0
  362. fi/evals/manager.py +1018 -0
  363. fi/evals/manager_types.py +362 -0
  364. fi/evals/metrics/__init__.py +185 -0
  365. fi/evals/metrics/agents/__init__.py +74 -0
  366. fi/evals/metrics/agents/metrics.py +693 -0
  367. fi/evals/metrics/agents/report.py +36463 -0
  368. fi/evals/metrics/agents/types.py +160 -0
  369. fi/evals/metrics/base_llm_metric.py +111 -0
  370. fi/evals/metrics/base_metric.py +138 -0
  371. fi/evals/metrics/code_security/__init__.py +305 -0
  372. fi/evals/metrics/code_security/analyzer.py +985 -0
  373. fi/evals/metrics/code_security/benchmarks/__init__.py +73 -0
  374. fi/evals/metrics/code_security/benchmarks/builtin.py +750 -0
  375. fi/evals/metrics/code_security/benchmarks/loader.py +580 -0
  376. fi/evals/metrics/code_security/benchmarks/types.py +308 -0
  377. fi/evals/metrics/code_security/detectors/__init__.py +186 -0
  378. fi/evals/metrics/code_security/detectors/base.py +394 -0
  379. fi/evals/metrics/code_security/detectors/cryptography.py +345 -0
  380. fi/evals/metrics/code_security/detectors/injection.py +744 -0
  381. fi/evals/metrics/code_security/detectors/secrets.py +287 -0
  382. fi/evals/metrics/code_security/detectors/serialization.py +192 -0
  383. fi/evals/metrics/code_security/joint_metrics.py +588 -0
  384. fi/evals/metrics/code_security/judges/__init__.py +83 -0
  385. fi/evals/metrics/code_security/judges/base.py +238 -0
  386. fi/evals/metrics/code_security/judges/dual_judge.py +534 -0
  387. fi/evals/metrics/code_security/judges/llm_judge.py +301 -0
  388. fi/evals/metrics/code_security/judges/pattern_judge.py +515 -0
  389. fi/evals/metrics/code_security/metrics.py +388 -0
  390. fi/evals/metrics/code_security/modes/__init__.py +63 -0
  391. fi/evals/metrics/code_security/modes/adversarial.py +284 -0
  392. fi/evals/metrics/code_security/modes/autocomplete.py +198 -0
  393. fi/evals/metrics/code_security/modes/base.py +283 -0
  394. fi/evals/metrics/code_security/modes/instruct.py +253 -0
  395. fi/evals/metrics/code_security/modes/repair.py +230 -0
  396. fi/evals/metrics/code_security/reports/__init__.py +57 -0
  397. fi/evals/metrics/code_security/reports/generator.py +404 -0
  398. fi/evals/metrics/code_security/reports/leaderboard.py +509 -0
  399. fi/evals/metrics/code_security/types.py +534 -0
  400. fi/evals/metrics/function_calling/__init__.py +34 -0
  401. fi/evals/metrics/function_calling/metrics.py +573 -0
  402. fi/evals/metrics/function_calling/types.py +87 -0
  403. fi/evals/metrics/hallucination/__init__.py +54 -0
  404. fi/evals/metrics/hallucination/detector.py +149 -0
  405. fi/evals/metrics/hallucination/metrics.py +390 -0
  406. fi/evals/metrics/hallucination/nli.py +253 -0
  407. fi/evals/metrics/hallucination/sentinel.py +106 -0
  408. fi/evals/metrics/hallucination/types.py +132 -0
  409. fi/evals/metrics/heuristics/aggregation_metrics.py +85 -0
  410. fi/evals/metrics/heuristics/json_metrics.py +87 -0
  411. fi/evals/metrics/heuristics/similarity_metrics.py +375 -0
  412. fi/evals/metrics/heuristics/string_metrics.py +391 -0
  413. fi/evals/metrics/llm_as_judges/__init__.py +17 -0
  414. fi/evals/metrics/llm_as_judges/custom_judge/metric.py +112 -0
  415. fi/evals/metrics/llm_as_judges/custom_judge/prompts.py +26 -0
  416. fi/evals/metrics/llm_as_judges/types.py +48 -0
  417. fi/evals/metrics/rag/__init__.py +111 -0
  418. fi/evals/metrics/rag/advanced/__init__.py +14 -0
  419. fi/evals/metrics/rag/advanced/multi_hop.py +283 -0
  420. fi/evals/metrics/rag/advanced/source_attribution.py +344 -0
  421. fi/evals/metrics/rag/generation/__init__.py +17 -0
  422. fi/evals/metrics/rag/generation/answer_relevancy.py +176 -0
  423. fi/evals/metrics/rag/generation/context_utilization.py +245 -0
  424. fi/evals/metrics/rag/generation/faithfulness.py +241 -0
  425. fi/evals/metrics/rag/generation/groundedness.py +131 -0
  426. fi/evals/metrics/rag/rag_score.py +277 -0
  427. fi/evals/metrics/rag/retrieval/__init__.py +20 -0
  428. fi/evals/metrics/rag/retrieval/context_entity_recall.py +124 -0
  429. fi/evals/metrics/rag/retrieval/context_precision.py +158 -0
  430. fi/evals/metrics/rag/retrieval/context_recall.py +106 -0
  431. fi/evals/metrics/rag/retrieval/noise_sensitivity.py +163 -0
  432. fi/evals/metrics/rag/retrieval/ranking.py +261 -0
  433. fi/evals/metrics/rag/types.py +100 -0
  434. fi/evals/metrics/rag/utils/__init__.py +62 -0
  435. fi/evals/metrics/rag/utils/claims.py +189 -0
  436. fi/evals/metrics/rag/utils/entities.py +244 -0
  437. fi/evals/metrics/rag/utils/nli.py +92 -0
  438. fi/evals/metrics/rag/utils/similarity.py +345 -0
  439. fi/evals/metrics/structured/__init__.py +114 -0
  440. fi/evals/metrics/structured/field_completeness.py +313 -0
  441. fi/evals/metrics/structured/hierarchy_score.py +366 -0
  442. fi/evals/metrics/structured/json_validation.py +190 -0
  443. fi/evals/metrics/structured/schema_compliance.py +280 -0
  444. fi/evals/metrics/structured/structured_output_score.py +298 -0
  445. fi/evals/metrics/structured/types.py +108 -0
  446. fi/evals/metrics/structured/validators/__init__.py +30 -0
  447. fi/evals/metrics/structured/validators/base.py +189 -0
  448. fi/evals/metrics/structured/validators/json_validator.py +196 -0
  449. fi/evals/metrics/structured/validators/pydantic_validator.py +178 -0
  450. fi/evals/metrics/structured/validators/yaml_validator.py +248 -0
  451. fi/evals/otel/__init__.py +266 -0
  452. fi/evals/otel/config.py +400 -0
  453. fi/evals/otel/conventions.py +463 -0
  454. fi/evals/otel/enrichment.py +371 -0
  455. fi/evals/otel/instrumentors/__init__.py +140 -0
  456. fi/evals/otel/instrumentors/anthropic.py +517 -0
  457. fi/evals/otel/instrumentors/base.py +382 -0
  458. fi/evals/otel/instrumentors/openai.py +673 -0
  459. fi/evals/otel/processors/__init__.py +36 -0
  460. fi/evals/otel/processors/base.py +473 -0
  461. fi/evals/otel/processors/cost.py +445 -0
  462. fi/evals/otel/processors/evaluation.py +559 -0
  463. fi/evals/otel/processors/llm.py +462 -0
  464. fi/evals/otel/tracer.py +506 -0
  465. fi/evals/otel/types.py +232 -0
  466. fi/evals/otel_utils.py +23 -0
  467. fi/evals/protect.py +671 -0
  468. fi/evals/protect_input_adapter.py +154 -0
  469. fi/evals/streaming/__init__.py +88 -0
  470. fi/evals/streaming/buffer.py +213 -0
  471. fi/evals/streaming/evaluator.py +551 -0
  472. fi/evals/streaming/policy.py +307 -0
  473. fi/evals/streaming/scorers.py +368 -0
  474. fi/evals/streaming/types.py +238 -0
  475. fi/evals/templates.py +472 -0
  476. fi/evals/types.py +156 -0
  477. fi/opt/__init__.py +221 -0
  478. fi/opt/_objective_scoring.py +85 -0
  479. fi/opt/base/__init__.py +11 -0
  480. fi/opt/base/base_generator.py +33 -0
  481. fi/opt/base/base_mapper.py +26 -0
  482. fi/opt/base/base_optimizer.py +45 -0
  483. fi/opt/base/evaluator.py +211 -0
  484. fi/opt/components.py +3095 -0
  485. fi/opt/datamappers/__init__.py +3 -0
  486. fi/opt/datamappers/basic_mapper.py +40 -0
  487. fi/opt/deployment.py +1021 -0
  488. fi/opt/evidence.py +4332 -0
  489. fi/opt/generators/__init__.py +3 -0
  490. fi/opt/generators/litellm.py +66 -0
  491. fi/opt/integrations/__init__.py +23 -0
  492. fi/opt/integrations/generative_suite.py +410 -0
  493. fi/opt/integrations/simulate.py +1313 -0
  494. fi/opt/mutations.py +771 -0
  495. fi/opt/observability.py +4639 -0
  496. fi/opt/optimizer_trace.py +889 -0
  497. fi/opt/optimizers/__init__.py +80 -0
  498. fi/opt/optimizers/agent.py +331 -0
  499. fi/opt/optimizers/agent_bandit.py +392 -0
  500. fi/opt/optimizers/agent_curriculum.py +635 -0
  501. fi/opt/optimizers/agent_evolution.py +894 -0
  502. fi/opt/optimizers/agent_feedback.py +1863 -0
  503. fi/opt/optimizers/agent_pareto.py +547 -0
  504. fi/opt/optimizers/agent_social_memory.py +1113 -0
  505. fi/opt/optimizers/agent_tpe.py +321 -0
  506. fi/opt/optimizers/bayesian_search.py +449 -0
  507. fi/opt/optimizers/council.py +2075 -0
  508. fi/opt/optimizers/futureagi_replay.py +799 -0
  509. fi/opt/optimizers/gepa.py +322 -0
  510. fi/opt/optimizers/metaprompt.py +243 -0
  511. fi/opt/optimizers/promptwizard.py +417 -0
  512. fi/opt/optimizers/protegi.py +329 -0
  513. fi/opt/optimizers/random_search.py +224 -0
  514. fi/opt/research.py +518 -0
  515. fi/opt/simulation.py +260 -0
  516. fi/opt/targets.py +232 -0
  517. fi/opt/types.py +66 -0
  518. fi/opt/utils/__init__.py +4 -0
  519. fi/opt/utils/early_stopping.py +266 -0
  520. fi/opt/utils/setup_logging.py +82 -0
  521. fi/simulate/__init__.py +540 -0
  522. fi/simulate/_hashing.py +35 -0
  523. fi/simulate/_logging.py +10 -0
  524. fi/simulate/adapters.py +87 -0
  525. fi/simulate/agent/__init__.py +120 -0
  526. fi/simulate/agent/browser.py +658 -0
  527. fi/simulate/agent/definition.py +587 -0
  528. fi/simulate/agent/frameworks.py +3528 -0
  529. fi/simulate/agent/generic.py +8286 -0
  530. fi/simulate/agent/import_probe.py +227 -0
  531. fi/simulate/agent/memory.py +905 -0
  532. fi/simulate/agent/mocks.py +101 -0
  533. fi/simulate/agent/multi_agent.py +361 -0
  534. fi/simulate/agent/orchestration.py +903 -0
  535. fi/simulate/agent/realtime.py +665 -0
  536. fi/simulate/agent/wrapper.py +99 -0
  537. fi/simulate/agent/wrappers/__init__.py +18 -0
  538. fi/simulate/agent/wrappers/anthropic.py +62 -0
  539. fi/simulate/agent/wrappers/gemini.py +65 -0
  540. fi/simulate/agent/wrappers/http.py +404 -0
  541. fi/simulate/agent/wrappers/langchain.py +80 -0
  542. fi/simulate/agent/wrappers/openai.py +75 -0
  543. fi/simulate/agent/wrappers/websocket.py +326 -0
  544. fi/simulate/artifacts/__init__.py +11 -0
  545. fi/simulate/artifacts/manifest.py +62 -0
  546. fi/simulate/cli.py +20560 -0
  547. fi/simulate/endpoints/__init__.py +45 -0
  548. fi/simulate/endpoints/_http_actor.py +73 -0
  549. fi/simulate/endpoints/actor_sources.py +243 -0
  550. fi/simulate/endpoints/base.py +107 -0
  551. fi/simulate/endpoints/builtins.py +10 -0
  552. fi/simulate/endpoints/callable.py +95 -0
  553. fi/simulate/endpoints/http.py +76 -0
  554. fi/simulate/endpoints/livekit.py +138 -0
  555. fi/simulate/endpoints/originators.py +132 -0
  556. fi/simulate/endpoints/profiles.py +348 -0
  557. fi/simulate/endpoints/retell.py +633 -0
  558. fi/simulate/endpoints/vapi.py +205 -0
  559. fi/simulate/endpoints/websocket.py +76 -0
  560. fi/simulate/environment.py +33026 -0
  561. fi/simulate/environments/__init__.py +11 -0
  562. fi/simulate/environments/base.py +73 -0
  563. fi/simulate/environments/chat.py +697 -0
  564. fi/simulate/environments/voice.py +212 -0
  565. fi/simulate/evaluation/__init__.py +4 -0
  566. fi/simulate/evaluation/ai_eval.py +227 -0
  567. fi/simulate/evidence/__init__.py +35 -0
  568. fi/simulate/evidence/base.py +59 -0
  569. fi/simulate/evidence/caller_observed.py +50 -0
  570. fi/simulate/evidence/livekit_instrumentation.py +51 -0
  571. fi/simulate/evidence/livekit_room.py +50 -0
  572. fi/simulate/evidence/otel.py +49 -0
  573. fi/simulate/evidence/providers/__init__.py +24 -0
  574. fi/simulate/evidence/providers/base.py +61 -0
  575. fi/simulate/evidence/providers/retell.py +376 -0
  576. fi/simulate/evidence/providers/vapi.py +426 -0
  577. fi/simulate/hosted/__init__.py +32 -0
  578. fi/simulate/hosted/child_entrypoint.py +306 -0
  579. fi/simulate/hosted/job.py +150 -0
  580. fi/simulate/hosted/targets.py +53 -0
  581. fi/simulate/instrumentation/__init__.py +5 -0
  582. fi/simulate/instrumentation/livekit/__init__.py +122 -0
  583. fi/simulate/manifest.py +1033 -0
  584. fi/simulate/matrix_cli.py +165 -0
  585. fi/simulate/realtime/__init__.py +40 -0
  586. fi/simulate/realtime/events.py +107 -0
  587. fi/simulate/realtime/media.py +61 -0
  588. fi/simulate/realtime/session.py +91 -0
  589. fi/simulate/recording/__init__.py +5 -0
  590. fi/simulate/recording/room_recorder.py +326 -0
  591. fi/simulate/registry.py +185 -0
  592. fi/simulate/results/__init__.py +9 -0
  593. fi/simulate/results/base.py +18 -0
  594. fi/simulate/results/filesystem.py +71 -0
  595. fi/simulate/results/futureagi.py +1340 -0
  596. fi/simulate/runtime/__init__.py +85 -0
  597. fi/simulate/runtime/capabilities.py +40 -0
  598. fi/simulate/runtime/events.py +63 -0
  599. fi/simulate/runtime/failures.py +25 -0
  600. fi/simulate/runtime/ids.py +34 -0
  601. fi/simulate/runtime/plan.py +70 -0
  602. fi/simulate/runtime/planner.py +102 -0
  603. fi/simulate/runtime/report.py +174 -0
  604. fi/simulate/runtime/run.py +75 -0
  605. fi/simulate/runtime/runner.py +333 -0
  606. fi/simulate/runtime/spec.py +186 -0
  607. fi/simulate/simulation/__init__.py +30 -0
  608. fi/simulate/simulation/behavior_policy.py +425 -0
  609. fi/simulate/simulation/bridge/__init__.py +9 -0
  610. fi/simulate/simulation/bridge/audio.py +29 -0
  611. fi/simulate/simulation/bridge/connector.py +46 -0
  612. fi/simulate/simulation/bridge/livekit.py +252 -0
  613. fi/simulate/simulation/bridge/retell.py +188 -0
  614. fi/simulate/simulation/bridge/vapi.py +177 -0
  615. fi/simulate/simulation/contract.py +419 -0
  616. fi/simulate/simulation/engines/__init__.py +12 -0
  617. fi/simulate/simulation/engines/base.py +21 -0
  618. fi/simulate/simulation/engines/cloud.py +517 -0
  619. fi/simulate/simulation/engines/livekit.py +4167 -0
  620. fi/simulate/simulation/engines/local_text.py +89 -0
  621. fi/simulate/simulation/fidelity.py +374 -0
  622. fi/simulate/simulation/gemini_tts_stream.py +110 -0
  623. fi/simulate/simulation/generator.py +91 -0
  624. fi/simulate/simulation/goal_machine.py +185 -0
  625. fi/simulate/simulation/livekit_models.py +467 -0
  626. fi/simulate/simulation/matrix.py +170 -0
  627. fi/simulate/simulation/models.py +279 -0
  628. fi/simulate/simulation/runner.py +153 -0
  629. fi/simulate/simulation/synthetic.py +880 -0
  630. fi/simulate/simulation/voice_prompt.py +502 -0
  631. fi/simulate/simulator/__init__.py +55 -0
  632. fi/simulate/simulator/builtins.py +53 -0
  633. fi/simulate/suite.py +1288 -0
  634. fi/simulate/utils/routes.py +164 -0
  635. fi/simulate/voice.py +225 -0
  636. fi/simulate/voice_cli.py +182 -0
  637. fi/utils/__init__.py +1 -0
  638. fi/utils/constants.py +14 -0
  639. fi/utils/errors.py +200 -0
  640. fi/utils/executor.py +26 -0
  641. fi/utils/routes.py +119 -0
  642. fi/utils/utils.py +17 -0
@@ -0,0 +1,674 @@
1
+ """Safe, deterministic credential discovery for submitted agent repositories.
2
+
3
+ The reasoning model may later explain a requirement, but it never needs to see a resolved
4
+ credential. This module performs the cheap preflight first and emits a structured manifest the
5
+ platform can use to ask for missing secrets before a build or provider call starts.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import ast
11
+ import hashlib
12
+ import json
13
+ import re
14
+ from enum import Enum
15
+ from pathlib import Path
16
+ from typing import Iterable
17
+
18
+ from pydantic import BaseModel, Field
19
+
20
+ from fi.simulate.runtime.spec import SecretRef
21
+
22
+ CREDENTIAL_MANIFEST_VERSION = "futureagi.credential-requirements.v1"
23
+
24
+
25
+ class RequirementKind(str, Enum):
26
+ SECRET = "secret"
27
+ CONFIGURATION = "configuration"
28
+ HARNESS_INFRASTRUCTURE = "harness_infrastructure"
29
+
30
+
31
+ class RequirementStatus(str, Enum):
32
+ MISSING = "missing"
33
+ CONFIGURED = "configured"
34
+ OPTIONAL = "optional"
35
+ HARNESS_PROVIDED = "harness_provided"
36
+
37
+
38
+ class CredentialRequirement(BaseModel):
39
+ id: str
40
+ environment_name: str
41
+ provider: str
42
+ purpose: str
43
+ kind: RequirementKind
44
+ required: bool = True
45
+ status: RequirementStatus
46
+ detected_from: list[str] = Field(default_factory=list)
47
+ accepted_secret_types: list[str] = Field(default_factory=list)
48
+
49
+
50
+ class CredentialChoice(BaseModel):
51
+ id: str
52
+ purpose: str
53
+ options: list[list[str]]
54
+ satisfied: bool = False
55
+
56
+
57
+ class CredentialManifest(BaseModel):
58
+ schema_version: str = CREDENTIAL_MANIFEST_VERSION
59
+ source_digest: str
60
+ detected_connectors: list[str] = Field(default_factory=list)
61
+ requirements: list[CredentialRequirement] = Field(default_factory=list)
62
+ credential_choices: list[CredentialChoice] = Field(default_factory=list)
63
+ scanned_files: int = Field(ge=0)
64
+ truncated: bool = False
65
+
66
+ @property
67
+ def missing_required(self) -> list[CredentialRequirement]:
68
+ choice_members = {
69
+ name
70
+ for choice in self.credential_choices
71
+ for option in choice.options
72
+ for name in option
73
+ }
74
+ return [
75
+ item
76
+ for item in self.requirements
77
+ if item.required
78
+ and item.status is RequirementStatus.MISSING
79
+ and item.environment_name not in choice_members
80
+ ]
81
+
82
+ @property
83
+ def ready(self) -> bool:
84
+ return not self.missing_required and all(
85
+ choice.satisfied for choice in self.credential_choices
86
+ )
87
+
88
+
89
+ _IGNORED_PARTS = {
90
+ ".git",
91
+ ".venv",
92
+ "venv",
93
+ ".tox",
94
+ ".nox",
95
+ "node_modules",
96
+ "test",
97
+ "tests",
98
+ "__pycache__",
99
+ "artifacts",
100
+ "build",
101
+ "dist",
102
+ "vendor",
103
+ }
104
+ _SOURCE_SUFFIXES = {
105
+ ".py",
106
+ ".js",
107
+ ".jsx",
108
+ ".ts",
109
+ ".tsx",
110
+ ".mjs",
111
+ ".cjs",
112
+ ".go",
113
+ ".rb",
114
+ ".toml",
115
+ ".yaml",
116
+ ".yml",
117
+ ".json",
118
+ }
119
+ _CONFIG_NAMES = {
120
+ ".env.example",
121
+ ".env.sample",
122
+ ".env.template",
123
+ "env.example",
124
+ "env.sample",
125
+ "env.template",
126
+ "docker-compose.yml",
127
+ "docker-compose.yaml",
128
+ "compose.yml",
129
+ "compose.yaml",
130
+ "pyproject.toml",
131
+ "package.json",
132
+ }
133
+ _MAX_FILES = 2_000
134
+ _MAX_FILE_BYTES = 1_000_000
135
+
136
+ _ENV_DECLARATION = re.compile(
137
+ r"(?m)^(?:export[ \t]+)?([A-Z][A-Z0-9_]{2,})[ \t]*=[ \t]*([^\r\n#]*)"
138
+ )
139
+ _ENV_TEMPLATE_NAMES = {
140
+ ".env.example",
141
+ ".env.sample",
142
+ ".env.template",
143
+ "env.example",
144
+ "env.sample",
145
+ "env.template",
146
+ }
147
+ _COMPOSE_NAMES = {
148
+ "docker-compose.yml",
149
+ "docker-compose.yaml",
150
+ "compose.yml",
151
+ "compose.yaml",
152
+ }
153
+ _PLACEHOLDER_VALUE = re.compile(
154
+ r"^(?:"
155
+ r"\[[^\]]*\]|<[^>]*>|\{[^}]*\}|"
156
+ r"(?:your|replace|insert|change)[-_ ]?(?:me|this|.*)|"
157
+ r"example|placeholder|todo|xxx+"
158
+ r")$",
159
+ re.IGNORECASE,
160
+ )
161
+ _COMPOSE_VARIABLE = re.compile(r"\$\{([A-Z][A-Z0-9_]{2,})(?:(:?[-?])([^}]*))?\}")
162
+ _PYTHON_REQUIRED = re.compile(r"os\.environ\s*\[\s*['\"]([A-Z][A-Z0-9_]{2,})['\"]\s*\]")
163
+ _PYTHON_GETENV = re.compile(
164
+ r"(?:os\.getenv|os\.environ\.get)\s*\(\s*['\"]([A-Z][A-Z0-9_]{2,})['\"](?:\s*,\s*([^\)]+))?"
165
+ )
166
+ _JS_ENV = re.compile(r"process\.env\.([A-Z][A-Z0-9_]{2,})")
167
+
168
+ _SECRET_NAME = re.compile(
169
+ r"(?:^|_)(?:API_?KEY|API_?SECRET|AUTHORIZATION|CREDENTIALS?|PASSWORD|PRIVATE_?KEY|SECRET|TOKEN)(?:_|$)"
170
+ )
171
+ _HARNESS_PROVIDED = {
172
+ "DATABASE_URL",
173
+ "POSTGRES_HOST",
174
+ "POSTGRES_PORT",
175
+ "POSTGRES_DB",
176
+ "POSTGRES_USER",
177
+ "POSTGRES_PASSWORD",
178
+ "REDIS_URL",
179
+ "TOOLS_API_URL",
180
+ "MCP_URL",
181
+ }
182
+ _NON_USER_CONFIGURATION = {
183
+ "PORT",
184
+ "HOST",
185
+ "LOG_LEVEL",
186
+ "ENVIRONMENT",
187
+ "NODE_ENV",
188
+ "PYTHONPATH",
189
+ }
190
+
191
+ _PROVIDERS: tuple[tuple[tuple[str, ...], str, str], ...] = (
192
+ (("LIVEKIT_", "ACCEPTANCE_LIVEKIT_"), "livekit", "connect to the voice room"),
193
+ (("DEEPGRAM_",), "deepgram", "speech-to-text"),
194
+ (("CARTESIA_",), "cartesia", "text-to-speech"),
195
+ (("ELEVENLABS_",), "elevenlabs", "text-to-speech"),
196
+ (("OPENAI_",), "openai", "language or realtime model"),
197
+ (("ANTHROPIC_",), "anthropic", "language model"),
198
+ (("GOOGLE_", "VERTEX_", "CLOUD_ML_"), "google_cloud", "model or cloud runtime"),
199
+ (("VAPI_",), "vapi", "hosted voice agent connection"),
200
+ (("RETELL_",), "retell", "hosted voice agent connection"),
201
+ (("TWILIO_",), "twilio", "telephony connection"),
202
+ (("AWS_",), "aws", "cloud service"),
203
+ (("AZURE_",), "azure", "cloud service"),
204
+ (("MONGODB_", "MONGO_"), "mongodb", "database connection"),
205
+ (("PINECONE_",), "pinecone", "vector database"),
206
+ )
207
+
208
+ _CONNECTOR_SIGNALS = {
209
+ "livekit": ("livekit", "@livekit/", "livekit-agents"),
210
+ "vapi": ("vapi", "@vapi-ai"),
211
+ "retell": ("retell", "retell-sdk"),
212
+ "twilio": ("twilio",),
213
+ "pipecat": ("pipecat",),
214
+ "mcp": ("mcp", "modelcontextprotocol"),
215
+ "http": ("fastapi", "flask", "express", "axios", "httpx"),
216
+ }
217
+
218
+ # Some SDKs consume credentials internally, so the submitted source never calls ``getenv``.
219
+ # Connector detection must still make those requirements visible before the expensive build or
220
+ # worker-start phase. Keep this list to credentials inherent to the connector itself; model and
221
+ # optional application integrations continue to be discovered from explicit source declarations.
222
+ _CONNECTOR_REQUIREMENTS: dict[str, tuple[str, ...]] = {
223
+ "livekit": ("LIVEKIT_URL", "LIVEKIT_API_KEY", "LIVEKIT_API_SECRET"),
224
+ }
225
+
226
+ _LIVEKIT_PROVIDER_CREDENTIALS = {
227
+ "deepgram": ("DEEPGRAM_API_KEY",),
228
+ "cartesia": ("CARTESIA_API_KEY",),
229
+ "elevenlabs": ("ELEVENLABS_API_KEY",),
230
+ "openai": ("OPENAI_API_KEY",),
231
+ }
232
+
233
+
234
+ def _dotted_name(node: ast.AST) -> str:
235
+ if isinstance(node, ast.Name):
236
+ return node.id
237
+ if isinstance(node, ast.Attribute):
238
+ prefix = _dotted_name(node.value)
239
+ return f"{prefix}.{node.attr}" if prefix else node.attr
240
+ return ""
241
+
242
+
243
+ def _python_sdk_requirements(content: str) -> list[tuple[str, str]]:
244
+ """Discover credentials consumed internally by constructors in submitted Python.
245
+
246
+ This is AST-based because model constructors routinely span multiple lines and contain
247
+ nested calls; a regex ending at the first ``)`` silently misses valid ``vertexai=True``
248
+ configurations. Parsing is read-only and never imports or executes customer code.
249
+ """
250
+ try:
251
+ tree = ast.parse(content)
252
+ except SyntaxError:
253
+ return []
254
+ aliases: dict[str, str] = {}
255
+ imports: dict[str, str] = {}
256
+ for node in ast.walk(tree):
257
+ if isinstance(node, ast.ImportFrom):
258
+ module = node.module or ""
259
+ for item in node.names:
260
+ local = item.asname or item.name
261
+ imports[local] = f"{module}.{item.name}" if module else item.name
262
+ if module == "livekit.plugins":
263
+ aliases[local] = item.name
264
+ elif isinstance(node, ast.Import):
265
+ for item in node.names:
266
+ local = item.asname or item.name.split(".")[0]
267
+ imports[local] = item.name
268
+ prefix = "livekit.plugins."
269
+ if item.name.startswith(prefix):
270
+ aliases[item.asname or item.name] = item.name[len(prefix) :].split(
271
+ "."
272
+ )[0]
273
+
274
+ found: set[tuple[str, str]] = set()
275
+ for node in ast.walk(tree):
276
+ if not isinstance(node, ast.Call):
277
+ continue
278
+ path = _dotted_name(node.func)
279
+ if not path:
280
+ continue
281
+ parts = path.split(".")
282
+ provider = aliases.get(parts[0], parts[-2] if len(parts) > 1 else "")
283
+ constructor = parts[-1]
284
+ imported_path = imports.get(parts[0], parts[0])
285
+ resolved_path = ".".join([imported_path, *parts[1:]])
286
+ if provider in _LIVEKIT_PROVIDER_CREDENTIALS and constructor in {
287
+ "LLM",
288
+ "STT",
289
+ "TTS",
290
+ "RealtimeModel",
291
+ }:
292
+ for name in _LIVEKIT_PROVIDER_CREDENTIALS[provider]:
293
+ found.add((name, f"sdk:livekit.plugins.{provider}"))
294
+ if provider == "google" and constructor in {"LLM", "STT", "TTS"}:
295
+ vertex = next(
296
+ (
297
+ keyword.value
298
+ for keyword in node.keywords
299
+ if keyword.arg == "vertexai"
300
+ ),
301
+ None,
302
+ )
303
+ if isinstance(vertex, ast.Constant) and vertex.value is True:
304
+ for name in (
305
+ "GOOGLE_APPLICATION_CREDENTIALS",
306
+ "GOOGLE_APPLICATION_CREDENTIALS_JSON",
307
+ "GOOGLE_CLOUD_PROJECT",
308
+ ):
309
+ found.add((name, "sdk:livekit.plugins.google.vertex"))
310
+ # LangChain and Google's first-party SDKs resolve Application Default Credentials
311
+ # internally, so customer code commonly reads only project/location and never calls
312
+ # getenv for the credential file itself. Recognize the constructor and explicit Vertex
313
+ # mode without importing or executing submitted code.
314
+ vertex = next(
315
+ (keyword.value for keyword in node.keywords if keyword.arg == "vertexai"),
316
+ None,
317
+ )
318
+ is_explicit_vertex_client = (
319
+ resolved_path == "langchain_google_genai.ChatGoogleGenerativeAI"
320
+ and isinstance(vertex, ast.Constant)
321
+ and vertex.value is True
322
+ ) or (
323
+ resolved_path in {"google.genai.Client", "genai.Client"}
324
+ and isinstance(vertex, ast.Constant)
325
+ and vertex.value is True
326
+ )
327
+ if is_explicit_vertex_client:
328
+ for name in (
329
+ "GOOGLE_APPLICATION_CREDENTIALS",
330
+ "GOOGLE_APPLICATION_CREDENTIALS_JSON",
331
+ "GOOGLE_CLOUD_PROJECT",
332
+ ):
333
+ found.add((name, f"sdk:{resolved_path}.vertex"))
334
+ return sorted(found)
335
+
336
+
337
+ def discover_credentials(
338
+ root: str | Path,
339
+ *,
340
+ secret_refs: dict[str, SecretRef] | None = None,
341
+ provided_environment: Iterable[str] = (),
342
+ scan_paths: Iterable[str | Path] | None = None,
343
+ ) -> CredentialManifest:
344
+ """Inspect declarations and environment reads without executing submitted code."""
345
+ root = Path(root).expanduser().resolve()
346
+ if not root.is_dir():
347
+ raise ValueError(f"credential_source_missing: {root}")
348
+ configured = {str(name).upper() for name in provided_environment}
349
+ for alias, ref in (secret_refs or {}).items():
350
+ configured.add(str(alias).upper())
351
+ configured.add(str(ref.key).upper())
352
+ configured.add(_identifier(ref.purpose).upper())
353
+ # Hosted process runtime materializes this vault value to a private file and exports the
354
+ # conventional Google variable to the child. Discovery deals in names only; no credential
355
+ # value is read or persisted here.
356
+ if "GOOGLE_APPLICATION_CREDENTIALS_JSON" in configured:
357
+ configured.add("GOOGLE_APPLICATION_CREDENTIALS")
358
+
359
+ findings: dict[str, dict[str, object]] = {}
360
+ connector_hits: set[str] = set()
361
+ scanned = 0
362
+ truncated = False
363
+ digest = hashlib.sha256()
364
+
365
+ for path in _candidate_files(root, scan_paths=scan_paths):
366
+ if scanned >= _MAX_FILES:
367
+ truncated = True
368
+ break
369
+ try:
370
+ size = path.stat().st_size
371
+ except OSError:
372
+ continue
373
+ if size > _MAX_FILE_BYTES:
374
+ continue
375
+ try:
376
+ content = path.read_text(encoding="utf-8", errors="replace")
377
+ except OSError:
378
+ continue
379
+ scanned += 1
380
+ relative = path.relative_to(root).as_posix()
381
+ digest.update(relative.encode())
382
+ digest.update(hashlib.sha256(content.encode()).digest())
383
+ lowered = content.lower()
384
+ for connector, signals in _CONNECTOR_SIGNALS.items():
385
+ if any(signal in lowered for signal in signals):
386
+ connector_hits.add(connector)
387
+
388
+ def record(
389
+ name: str, *, required: bool, declared_default: bool = False
390
+ ) -> None:
391
+ # ALK_* is a reserved control-plane namespace. Provider adapters and the
392
+ # hosted lifecycle legitimately read these values from their process
393
+ # environment, but they are injected at launch and must never be admitted as
394
+ # customer-supplied target configuration.
395
+ if name in _NON_USER_CONFIGURATION or name.startswith("ALK_"):
396
+ return
397
+ item = findings.setdefault(
398
+ name,
399
+ {
400
+ "required": False,
401
+ "declared_default": False,
402
+ "detected_from": set(),
403
+ },
404
+ )
405
+ item["required"] = bool(item["required"]) or required
406
+ item["declared_default"] = (
407
+ bool(item["declared_default"]) or declared_default
408
+ )
409
+ detected = item["detected_from"]
410
+ assert isinstance(detected, set)
411
+ detected.add(relative)
412
+
413
+ # An uppercase assignment in source is usually a constant, not an
414
+ # environment declaration. Only env templates use NAME=value syntax.
415
+ if path.name.lower() in _ENV_TEMPLATE_NAMES:
416
+ for match in _ENV_DECLARATION.finditer(content):
417
+ name = match.group(1)
418
+ value = match.group(2).strip().strip("\"'")
419
+ usable_default = bool(value) and not _PLACEHOLDER_VALUE.fullmatch(value)
420
+ record(
421
+ name,
422
+ # A blank non-secret setting in an example file documents a knob; it
423
+ # does not prove that the selected runtime path needs a value. Strict
424
+ # source reads and Compose's :? operator remain authoritative. Secret
425
+ # placeholders stay required because SDKs commonly consume them without
426
+ # an explicit getenv call in customer code.
427
+ required=(
428
+ not usable_default and _kind(name) is RequirementKind.SECRET
429
+ ),
430
+ declared_default=usable_default,
431
+ )
432
+ if path.name.lower() in _COMPOSE_NAMES:
433
+ for match in _COMPOSE_VARIABLE.finditer(content):
434
+ operator, fallback = match.group(2), (match.group(3) or "").strip()
435
+ # Compose substitutes an unset plain ${NAME} with an empty string. Only its
436
+ # explicit error operators are admission requirements; source-level strict
437
+ # reads can still make the same name required when a built runtime is scanned.
438
+ required = operator in {"?", ":?"}
439
+ record(
440
+ match.group(1),
441
+ required=required,
442
+ declared_default=bool(fallback) and operator not in {"?", ":?"},
443
+ )
444
+ for match in _PYTHON_REQUIRED.finditer(content):
445
+ # ``os.environ[NAME] if os.environ.get(NAME) else default`` is a common guarded
446
+ # access idiom. The indexed read alone looks mandatory, but the same-line guard
447
+ # proves the application has a fallback and should not block job submission.
448
+ line_start = content.rfind("\n", 0, match.start()) + 1
449
+ line_end = content.find("\n", match.end())
450
+ line = content[line_start : line_end if line_end >= 0 else len(content)]
451
+ name = match.group(1)
452
+ guarded = bool(
453
+ re.search(
454
+ rf"(?:os\.getenv|os\.environ\.get)\s*\(\s*['\"]{re.escape(name)}['\"]",
455
+ line,
456
+ )
457
+ )
458
+ record(name, required=not guarded, declared_default=guarded)
459
+ for match in _PYTHON_GETENV.finditer(content):
460
+ default = (match.group(2) or "").strip()
461
+ record(
462
+ match.group(1),
463
+ # getenv/get is explicitly nullable. Treat it as optional unless
464
+ # a blank env template or a strict indexed read says otherwise.
465
+ required=False,
466
+ declared_default=bool(default),
467
+ )
468
+ for match in _JS_ENV.finditer(content):
469
+ record(match.group(1), required=True)
470
+ if path.suffix.lower() == ".py":
471
+ for name, origin in _python_sdk_requirements(content):
472
+ record(name, required=True)
473
+ detected = findings[name]["detected_from"]
474
+ assert isinstance(detected, set)
475
+ detected.add(origin)
476
+
477
+ for connector in sorted(connector_hits):
478
+ for name in _CONNECTOR_REQUIREMENTS.get(connector, ()):
479
+ item = findings.setdefault(
480
+ name,
481
+ {
482
+ "required": True,
483
+ "declared_default": False,
484
+ "detected_from": set(),
485
+ },
486
+ )
487
+ item["required"] = True
488
+ detected = item["detected_from"]
489
+ assert isinstance(detected, set)
490
+ detected.add(f"connector:{connector}")
491
+
492
+ requirements = []
493
+ for name, finding in sorted(findings.items()):
494
+ kind = _kind(name)
495
+ required = bool(finding["required"])
496
+ if kind is RequirementKind.CONFIGURATION and finding["declared_default"]:
497
+ required = False
498
+ if kind is RequirementKind.HARNESS_INFRASTRUCTURE:
499
+ status = RequirementStatus.HARNESS_PROVIDED
500
+ elif name in configured:
501
+ status = RequirementStatus.CONFIGURED
502
+ elif not required:
503
+ status = RequirementStatus.OPTIONAL
504
+ else:
505
+ status = RequirementStatus.MISSING
506
+ provider, purpose = _provider(name)
507
+ requirements.append(
508
+ CredentialRequirement(
509
+ id=_identifier(name),
510
+ environment_name=name,
511
+ provider=provider,
512
+ purpose=purpose,
513
+ kind=kind,
514
+ required=required,
515
+ status=status,
516
+ detected_from=sorted(finding["detected_from"]),
517
+ accepted_secret_types=(
518
+ [_secret_type(name)] if kind is RequirementKind.SECRET else []
519
+ ),
520
+ )
521
+ )
522
+
523
+ choices = _credential_choices(requirements)
524
+ manifest_core = {
525
+ "files": digest.hexdigest(),
526
+ "connectors": sorted(connector_hits),
527
+ "requirements": [item.model_dump(mode="json") for item in requirements],
528
+ "credential_choices": [item.model_dump(mode="json") for item in choices],
529
+ }
530
+ return CredentialManifest(
531
+ source_digest=hashlib.sha256(
532
+ json.dumps(manifest_core, sort_keys=True, separators=(",", ":")).encode()
533
+ ).hexdigest(),
534
+ detected_connectors=sorted(connector_hits),
535
+ requirements=requirements,
536
+ credential_choices=choices,
537
+ scanned_files=scanned,
538
+ truncated=truncated,
539
+ )
540
+
541
+
542
+ def _candidate_files(
543
+ root: Path, *, scan_paths: Iterable[str | Path] | None = None
544
+ ) -> list[Path]:
545
+ scopes: list[Path] | None = None
546
+ if scan_paths is not None:
547
+ scopes = []
548
+ for value in scan_paths:
549
+ candidate = (root / value).resolve()
550
+ try:
551
+ candidate.relative_to(root)
552
+ except ValueError:
553
+ continue
554
+ if candidate.exists():
555
+ scopes.append(candidate)
556
+ result: list[Path] = []
557
+ for path in sorted(root.rglob("*")):
558
+ try:
559
+ relative = path.relative_to(root)
560
+ except ValueError:
561
+ continue
562
+ if any(part in _IGNORED_PARTS for part in relative.parts):
563
+ continue
564
+ if path.is_symlink() or not path.is_file():
565
+ continue
566
+ if scopes is not None and not any(
567
+ path == scope or (scope.is_dir() and scope in path.parents)
568
+ for scope in scopes
569
+ ):
570
+ continue
571
+ if path.name in _CONFIG_NAMES or path.suffix.lower() in _SOURCE_SUFFIXES:
572
+ result.append(path)
573
+ return result
574
+
575
+
576
+ def _kind(name: str) -> RequirementKind:
577
+ if name in _HARNESS_PROVIDED:
578
+ return RequirementKind.HARNESS_INFRASTRUCTURE
579
+ if _SECRET_NAME.search(name):
580
+ return RequirementKind.SECRET
581
+ return RequirementKind.CONFIGURATION
582
+
583
+
584
+ def _provider(name: str) -> tuple[str, str]:
585
+ for prefixes, provider, purpose in _PROVIDERS:
586
+ if name.startswith(prefixes):
587
+ return provider, purpose
588
+ if name in _HARNESS_PROVIDED:
589
+ return "harness", "generated test infrastructure"
590
+ return "agent", "agent runtime configuration"
591
+
592
+
593
+ def _secret_type(name: str) -> str:
594
+ if "PRIVATE_KEY" in name or name.endswith("CREDENTIALS"):
595
+ return "credential_file"
596
+ if "TOKEN" in name:
597
+ return "token"
598
+ if "SECRET" in name or "PASSWORD" in name:
599
+ return "secret"
600
+ return "api_key"
601
+
602
+
603
+ def _credential_choices(
604
+ requirements: list[CredentialRequirement],
605
+ ) -> list[CredentialChoice]:
606
+ """Describe common provider auth alternatives without reading secret values."""
607
+ by_name = {item.environment_name: item for item in requirements}
608
+ google_options = [
609
+ ["GEMINI_API_KEY"],
610
+ ["GOOGLE_API_KEY"],
611
+ ["GOOGLE_APPLICATION_CREDENTIALS", "GOOGLE_CLOUD_PROJECT"],
612
+ ["GOOGLE_APPLICATION_CREDENTIALS_JSON", "GOOGLE_CLOUD_PROJECT"],
613
+ ]
614
+ # Repositories often keep more than one model backend behind LLM_PROVIDER. A strict
615
+ # credential read inside one branch (or a standalone bakeoff utility in the same runtime
616
+ # tree) must not make every backend credential mandatory. The selected provider's one
617
+ # complete authentication route is the requirement. Without an explicit provider selector
618
+ # we remain conservative and do not merge independent integrations into one choice.
619
+ if "LLM_PROVIDER" in by_name:
620
+ google_options.extend(
621
+ [option for option in (["OPENAI_API_KEY"], ["AGENTCC_API_KEY"])]
622
+ )
623
+ present_options = [
624
+ option for option in google_options if all(name in by_name for name in option)
625
+ ]
626
+ google_names = {
627
+ "GEMINI_API_KEY",
628
+ "GOOGLE_API_KEY",
629
+ "GOOGLE_APPLICATION_CREDENTIALS",
630
+ "GOOGLE_APPLICATION_CREDENTIALS_JSON",
631
+ }
632
+ if len(present_options) < 2 or not any(
633
+ any(name in google_names for name in option) for option in present_options
634
+ ):
635
+ return []
636
+ configured = {
637
+ RequirementStatus.CONFIGURED,
638
+ RequirementStatus.HARNESS_PROVIDED,
639
+ }
640
+ satisfied = any(
641
+ all(by_name[name].status in configured for name in option)
642
+ for option in present_options
643
+ )
644
+ if not satisfied:
645
+ # These variables are optional individually but one complete route is
646
+ # mandatory. Mark the members input-capable; the choice prevents the UI
647
+ # from requiring every alternative.
648
+ for name in {name for option in present_options for name in option}:
649
+ if by_name[name].status not in configured:
650
+ by_name[name].required = True
651
+ by_name[name].status = RequirementStatus.MISSING
652
+ return [
653
+ CredentialChoice(
654
+ id="google_model_auth",
655
+ purpose="authenticate the Google/Gemini model runtime",
656
+ options=present_options,
657
+ satisfied=satisfied,
658
+ )
659
+ ]
660
+
661
+
662
+ def _identifier(value: str) -> str:
663
+ return re.sub(r"[^a-z0-9]+", "_", value.lower()).strip("_")
664
+
665
+
666
+ __all__ = [
667
+ "CREDENTIAL_MANIFEST_VERSION",
668
+ "CredentialManifest",
669
+ "CredentialChoice",
670
+ "CredentialRequirement",
671
+ "RequirementKind",
672
+ "RequirementStatus",
673
+ "discover_credentials",
674
+ ]