agent-learning-kit 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (642) hide show
  1. agent_learning_kit-0.1.0.dist-info/METADATA +381 -0
  2. agent_learning_kit-0.1.0.dist-info/RECORD +642 -0
  3. agent_learning_kit-0.1.0.dist-info/WHEEL +4 -0
  4. agent_learning_kit-0.1.0.dist-info/entry_points.txt +5 -0
  5. agent_learning_kit-0.1.0.dist-info/licenses/LICENSE +173 -0
  6. agent_learning_kit-0.1.0.dist-info/licenses/NOTICE +7 -0
  7. fi/__init__.py +5 -0
  8. fi/alk/__init__.py +57 -0
  9. fi/alk/_facade.py +31 -0
  10. fi/alk/_module_alias.py +68 -0
  11. fi/alk/_paths.py +14 -0
  12. fi/alk/_schema.py +522 -0
  13. fi/alk/actions.py +727 -0
  14. fi/alk/bench/__init__.py +517 -0
  15. fi/alk/bench/_codeexec.py +213 -0
  16. fi/alk/bench/_coding.py +215 -0
  17. fi/alk/bench/_docker.py +237 -0
  18. fi/alk/bench/_grader.py +286 -0
  19. fi/alk/bench/_pull.py +212 -0
  20. fi/alk/bench/_voice.py +147 -0
  21. fi/alk/capabilities.py +627 -0
  22. fi/alk/cli.py +6396 -0
  23. fi/alk/config.py +130 -0
  24. fi/alk/cua_loop.py +562 -0
  25. fi/alk/evals.py +2351 -0
  26. fi/alk/extensions.py +163 -0
  27. fi/alk/harness/ARCHITECTURE.md +231 -0
  28. fi/alk/harness/DESIGN.md +246 -0
  29. fi/alk/harness/ENVIRONMENT_CONFORMANCE.md +127 -0
  30. fi/alk/harness/HOW-IT-WORKS.md +297 -0
  31. fi/alk/harness/IMPLEMENTATION_AND_VALIDATION_STATUS.md +229 -0
  32. fi/alk/harness/README.md +417 -0
  33. fi/alk/harness/__init__.py +77 -0
  34. fi/alk/harness/__main__.py +3 -0
  35. fi/alk/harness/amend.py +312 -0
  36. fi/alk/harness/artifacts.py +319 -0
  37. fi/alk/harness/authoring_entrypoint.py +189 -0
  38. fi/alk/harness/authoring_runtime_validation.py +267 -0
  39. fi/alk/harness/backends/README.md +43 -0
  40. fi/alk/harness/backends/__init__.py +122 -0
  41. fi/alk/harness/backends/base.py +241 -0
  42. fi/alk/harness/backends/claude.py +211 -0
  43. fi/alk/harness/backends/files.py +182 -0
  44. fi/alk/harness/backends/vertex_gemini.py +457 -0
  45. fi/alk/harness/background_noise.py +95 -0
  46. fi/alk/harness/build.py +385 -0
  47. fi/alk/harness/bundle.py +593 -0
  48. fi/alk/harness/bundle_author_v2.py +1831 -0
  49. fi/alk/harness/bundle_v2.py +719 -0
  50. fi/alk/harness/call_runner.py +1440 -0
  51. fi/alk/harness/callback_http_adapter.py +111 -0
  52. fi/alk/harness/catalogue.py +287 -0
  53. fi/alk/harness/chat.py +428 -0
  54. fi/alk/harness/chat_call_runner.py +506 -0
  55. fi/alk/harness/checks.py +136 -0
  56. fi/alk/harness/cli.py +1354 -0
  57. fi/alk/harness/config.py +338 -0
  58. fi/alk/harness/contract.py +718 -0
  59. fi/alk/harness/credentials.py +674 -0
  60. fi/alk/harness/data/persona_vocabulary.json +111 -0
  61. fi/alk/harness/environment.py +99 -0
  62. fi/alk/harness/environment_plan.py +168 -0
  63. fi/alk/harness/events.py +125 -0
  64. fi/alk/harness/executor.py +304 -0
  65. fi/alk/harness/folder.py +234 -0
  66. fi/alk/harness/generated_runtime.py +815 -0
  67. fi/alk/harness/github.py +72 -0
  68. fi/alk/harness/hosted_authoring_entrypoint.py +183 -0
  69. fi/alk/harness/hosted_entrypoint.py +2402 -0
  70. fi/alk/harness/hosted_scheduler.py +2218 -0
  71. fi/alk/harness/job.py +426 -0
  72. fi/alk/harness/judge.py +184 -0
  73. fi/alk/harness/livekit_source.py +50 -0
  74. fi/alk/harness/livekit_tool_trace_bootstrap.py +71 -0
  75. fi/alk/harness/observability.py +208 -0
  76. fi/alk/harness/outbound.py +3252 -0
  77. fi/alk/harness/packaging.py +515 -0
  78. fi/alk/harness/persona_guides.py +157 -0
  79. fi/alk/harness/platform.py +692 -0
  80. fi/alk/harness/process_preflight.py +764 -0
  81. fi/alk/harness/process_runtime.py +5670 -0
  82. fi/alk/harness/prove.py +425 -0
  83. fi/alk/harness/provider_import.py +703 -0
  84. fi/alk/harness/provider_lifecycle.py +392 -0
  85. fi/alk/harness/provision.py +2896 -0
  86. fi/alk/harness/reception.py +147 -0
  87. fi/alk/harness/retell_chat_call_runner.py +373 -0
  88. fi/alk/harness/run/__init__.py +296 -0
  89. fi/alk/harness/run/alk.py +184 -0
  90. fi/alk/harness/run/call.py +162 -0
  91. fi/alk/harness/run/conversation.py +264 -0
  92. fi/alk/harness/run/data/voices_by_language_and_gender.json +693 -0
  93. fi/alk/harness/run/evidence.py +195 -0
  94. fi/alk/harness/run/grade.py +598 -0
  95. fi/alk/harness/run/live.py +297 -0
  96. fi/alk/harness/run/models.py +56 -0
  97. fi/alk/harness/run/platform_evals.py +227 -0
  98. fi/alk/harness/run/sdk_voice.py +130 -0
  99. fi/alk/harness/run/simulation.py +1209 -0
  100. fi/alk/harness/run/stage.py +91 -0
  101. fi/alk/harness/run/targets.py +508 -0
  102. fi/alk/harness/run/tools.py +601 -0
  103. fi/alk/harness/run/voice.py +340 -0
  104. fi/alk/harness/runtime.py +172 -0
  105. fi/alk/harness/sandbox_server.py +2011 -0
  106. fi/alk/harness/sandbox_worker.py +44 -0
  107. fi/alk/harness/scenario.py +1048 -0
  108. fi/alk/harness/scenario_source.py +879 -0
  109. fi/alk/harness/scenario_tools.py +1143 -0
  110. fi/alk/harness/scenarios.py +915 -0
  111. fi/alk/harness/secrets.py +168 -0
  112. fi/alk/harness/service_catalog.py +97 -0
  113. fi/alk/harness/session.py +391 -0
  114. fi/alk/harness/sessions.py +372 -0
  115. fi/alk/harness/simulator.py +76 -0
  116. fi/alk/harness/simulator_voice.py +928 -0
  117. fi/alk/harness/skills/build-environment/SKILL.md +538 -0
  118. fi/alk/harness/skills/harness.md +131 -0
  119. fi/alk/harness/skills/kinds/chat.md +48 -0
  120. fi/alk/harness/skills/kinds/voice-voicemail.md +63 -0
  121. fi/alk/harness/skills/kinds/voice.md +59 -0
  122. fi/alk/harness/skills/plan-suite/SKILL.md +103 -0
  123. fi/alk/harness/skills/provision-environment/SKILL.md +136 -0
  124. fi/alk/harness/skills/run-scenarios/SKILL.md +112 -0
  125. fi/alk/harness/skills/understand-agent/SKILL.md +251 -0
  126. fi/alk/harness/skills/write-scenarios/SKILL.md +606 -0
  127. fi/alk/harness/skills/write-scenarios/references/refusals.md +28 -0
  128. fi/alk/harness/skills/write-scenarios/references/world-api.md +92 -0
  129. fi/alk/harness/source_data_invariants.py +444 -0
  130. fi/alk/harness/source_tool_evidence.py +79 -0
  131. fi/alk/harness/sources.py +253 -0
  132. fi/alk/harness/spend.py +140 -0
  133. fi/alk/harness/tool_trace_proxy.py +104 -0
  134. fi/alk/harness/tools.py +1018 -0
  135. fi/alk/harness/understand.py +169 -0
  136. fi/alk/harness/voicemail_audio.py +74 -0
  137. fi/alk/harness/world/__init__.py +33 -0
  138. fi/alk/harness/world/errors.py +68 -0
  139. fi/alk/harness/world/expectations.py +91 -0
  140. fi/alk/harness/world/handle.py +538 -0
  141. fi/alk/harness/world/kinds.py +196 -0
  142. fi/alk/harness/world/mutate.py +186 -0
  143. fi/alk/harness/world/probe.py +413 -0
  144. fi/alk/harness/world/provision.py +511 -0
  145. fi/alk/harness/world/provisioned.py +191 -0
  146. fi/alk/harness/world/runtime.py +616 -0
  147. fi/alk/harness/world/snapshot.py +288 -0
  148. fi/alk/harness/world/stores/__init__.py +305 -0
  149. fi/alk/harness/world/stores/container.py +215 -0
  150. fi/alk/harness/world/stores/inprocess.py +346 -0
  151. fi/alk/harness/world/stores/postgres.py +481 -0
  152. fi/alk/harness/world/stores/prove.py +202 -0
  153. fi/alk/harness/world/stores/sqlite.py +245 -0
  154. fi/alk/harness/world/stores/written.py +182 -0
  155. fi/alk/harness/world/tools.py +1516 -0
  156. fi/alk/harness/world/workspace.py +144 -0
  157. fi/alk/image_loop.py +453 -0
  158. fi/alk/image_perturb.py +241 -0
  159. fi/alk/improve.py +274 -0
  160. fi/alk/live/__init__.py +154 -0
  161. fi/alk/live/_attribution.py +184 -0
  162. fi/alk/live/_capture.py +264 -0
  163. fi/alk/live/_codec.py +391 -0
  164. fi/alk/live/_contract.py +134 -0
  165. fi/alk/live/_loopback.py +316 -0
  166. fi/alk/live/_perturb.py +449 -0
  167. fi/alk/live/_runner.py +386 -0
  168. fi/alk/live/_stats.py +561 -0
  169. fi/alk/live/_transcript.py +240 -0
  170. fi/alk/live/_workers/__init__.py +9 -0
  171. fi/alk/live/_workers/a2a_worker.py +316 -0
  172. fi/alk/live/_workers/langgraph_worker.py +217 -0
  173. fi/alk/live/_workers/livekit_worker.py +207 -0
  174. fi/alk/live/_workers/mcp_loopback_server.py +46 -0
  175. fi/alk/live/_workers/mcp_worker.py +158 -0
  176. fi/alk/live/_workers/pipecat_worker.py +189 -0
  177. fi/alk/live/a2a_lane.py +138 -0
  178. fi/alk/live/langgraph_lane.py +339 -0
  179. fi/alk/live/livekit_lane.py +376 -0
  180. fi/alk/live/mcp_lane.py +172 -0
  181. fi/alk/live/pipecat_lane.py +341 -0
  182. fi/alk/live/voice_redteam.py +494 -0
  183. fi/alk/loss.py +306 -0
  184. fi/alk/optimize.py +36260 -0
  185. fi/alk/practice/__init__.py +51 -0
  186. fi/alk/practice/_assess.py +103 -0
  187. fi/alk/practice/_budget.py +81 -0
  188. fi/alk/practice/_calibrate.py +69 -0
  189. fi/alk/practice/_capstone.py +86 -0
  190. fi/alk/practice/_contract.py +91 -0
  191. fi/alk/practice/_diagnose.py +79 -0
  192. fi/alk/practice/_drill.py +196 -0
  193. fi/alk/practice/_experiment.py +720 -0
  194. fi/alk/practice/_schedule.py +102 -0
  195. fi/alk/practice/_store.py +194 -0
  196. fi/alk/practice/_trainer.py +245 -0
  197. fi/alk/practice/_update.py +125 -0
  198. fi/alk/redteam.py +2621 -0
  199. fi/alk/rewardhack.py +237 -0
  200. fi/alk/simulate.py +10351 -0
  201. fi/alk/studio/__init__.py +82 -0
  202. fi/alk/studio/_bias.py +314 -0
  203. fi/alk/studio/_calibration.py +522 -0
  204. fi/alk/studio/_coverage.py +262 -0
  205. fi/alk/studio/_download.py +665 -0
  206. fi/alk/studio/_fidelity_attack.py +114 -0
  207. fi/alk/studio/_generate.py +652 -0
  208. fi/alk/studio/_library.py +370 -0
  209. fi/alk/studio/_scan.py +134 -0
  210. fi/alk/studio/_upgrade.py +42 -0
  211. fi/alk/studio/_vendor.py +172 -0
  212. fi/alk/suite.py +4200 -0
  213. fi/alk/tasks.py +828 -0
  214. fi/alk/telemetry/__init__.py +149 -0
  215. fi/alk/telemetry/_contract.py +141 -0
  216. fi/alk/telemetry/_emit.py +182 -0
  217. fi/alk/telemetry/_ledger.py +296 -0
  218. fi/alk/telemetry/_queue.py +127 -0
  219. fi/alk/telemetry/_row.py +294 -0
  220. fi/alk/telemetry/_run.py +233 -0
  221. fi/alk/telemetry/_sync.py +193 -0
  222. fi/alk/telemetry/_url.py +119 -0
  223. fi/alk/trinity.py +49397 -0
  224. fi/alk/voice_loop.py +174 -0
  225. fi/api/__init__.py +1 -0
  226. fi/api/auth.py +137 -0
  227. fi/api/types.py +29 -0
  228. fi/cli/__init__.py +9 -0
  229. fi/cli/assertions/__init__.py +25 -0
  230. fi/cli/assertions/conditions.py +76 -0
  231. fi/cli/assertions/evaluator.py +286 -0
  232. fi/cli/assertions/exit_codes.py +20 -0
  233. fi/cli/assertions/parser.py +131 -0
  234. fi/cli/assertions/reporter.py +194 -0
  235. fi/cli/commands/__init__.py +9 -0
  236. fi/cli/commands/config.py +165 -0
  237. fi/cli/commands/export.py +208 -0
  238. fi/cli/commands/init.py +112 -0
  239. fi/cli/commands/list_cmd.py +213 -0
  240. fi/cli/commands/run.py +486 -0
  241. fi/cli/commands/validate.py +173 -0
  242. fi/cli/commands/view.py +424 -0
  243. fi/cli/config/__init__.py +6 -0
  244. fi/cli/config/defaults.py +206 -0
  245. fi/cli/config/loader.py +155 -0
  246. fi/cli/config/schema.py +174 -0
  247. fi/cli/main.py +78 -0
  248. fi/cli/output/__init__.py +6 -0
  249. fi/cli/output/formatters.py +106 -0
  250. fi/cli/output/reporters.py +46 -0
  251. fi/cli/storage/__init__.py +5 -0
  252. fi/cli/storage/run_history.py +249 -0
  253. fi/cli/utils/__init__.py +5 -0
  254. fi/cli/utils/console.py +44 -0
  255. fi/evals/__init__.py +131 -0
  256. fi/evals/autoeval/__init__.py +137 -0
  257. fi/evals/autoeval/analyzer.py +211 -0
  258. fi/evals/autoeval/config.py +244 -0
  259. fi/evals/autoeval/export.py +213 -0
  260. fi/evals/autoeval/interactive.py +283 -0
  261. fi/evals/autoeval/pipeline.py +625 -0
  262. fi/evals/autoeval/prompts.py +139 -0
  263. fi/evals/autoeval/recommender.py +242 -0
  264. fi/evals/autoeval/rules.py +589 -0
  265. fi/evals/autoeval/templates.py +299 -0
  266. fi/evals/autoeval/types.py +232 -0
  267. fi/evals/core/__init__.py +16 -0
  268. fi/evals/core/cloud_registry.py +184 -0
  269. fi/evals/core/engines.py +368 -0
  270. fi/evals/core/evaluate.py +319 -0
  271. fi/evals/core/judge_prompt.py +90 -0
  272. fi/evals/core/prompt_generator.py +83 -0
  273. fi/evals/core/registry.py +57 -0
  274. fi/evals/core/result.py +55 -0
  275. fi/evals/evaluator.py +721 -0
  276. fi/evals/execution.py +168 -0
  277. fi/evals/feedback/__init__.py +32 -0
  278. fi/evals/feedback/calibrator.py +160 -0
  279. fi/evals/feedback/collector.py +214 -0
  280. fi/evals/feedback/hooks.py +81 -0
  281. fi/evals/feedback/retriever.py +128 -0
  282. fi/evals/feedback/store.py +272 -0
  283. fi/evals/feedback/types.py +99 -0
  284. fi/evals/framework/README.md +79 -0
  285. fi/evals/framework/__init__.py +267 -0
  286. fi/evals/framework/backends/Dockerfile.eval-runner +33 -0
  287. fi/evals/framework/backends/__init__.py +99 -0
  288. fi/evals/framework/backends/_container.py +141 -0
  289. fi/evals/framework/backends/_utils.py +145 -0
  290. fi/evals/framework/backends/base.py +223 -0
  291. fi/evals/framework/backends/celery_backend.py +417 -0
  292. fi/evals/framework/backends/celery_worker.py +78 -0
  293. fi/evals/framework/backends/kubernetes_backend.py +665 -0
  294. fi/evals/framework/backends/ray_backend.py +521 -0
  295. fi/evals/framework/backends/temporal.py +350 -0
  296. fi/evals/framework/backends/temporal_worker.py +126 -0
  297. fi/evals/framework/backends/thread_pool.py +286 -0
  298. fi/evals/framework/context.py +258 -0
  299. fi/evals/framework/enrichment.py +306 -0
  300. fi/evals/framework/evals/__init__.py +68 -0
  301. fi/evals/framework/evals/agentic.py +399 -0
  302. fi/evals/framework/evals/builder.py +609 -0
  303. fi/evals/framework/evals/semantic.py +142 -0
  304. fi/evals/framework/evaluator.py +647 -0
  305. fi/evals/framework/evaluators/__init__.py +22 -0
  306. fi/evals/framework/evaluators/blocking.py +347 -0
  307. fi/evals/framework/evaluators/non_blocking.py +577 -0
  308. fi/evals/framework/propagation.py +421 -0
  309. fi/evals/framework/protocols.py +385 -0
  310. fi/evals/framework/registry.py +370 -0
  311. fi/evals/framework/resilience/__init__.py +150 -0
  312. fi/evals/framework/resilience/circuit_breaker.py +309 -0
  313. fi/evals/framework/resilience/degradation.py +355 -0
  314. fi/evals/framework/resilience/health.py +505 -0
  315. fi/evals/framework/resilience/rate_limiter.py +228 -0
  316. fi/evals/framework/resilience/retry.py +274 -0
  317. fi/evals/framework/resilience/types.py +288 -0
  318. fi/evals/framework/resilience/wrapper.py +433 -0
  319. fi/evals/framework/types.py +218 -0
  320. fi/evals/guardrails/README.md +915 -0
  321. fi/evals/guardrails/__init__.py +96 -0
  322. fi/evals/guardrails/backends/__init__.py +43 -0
  323. fi/evals/guardrails/backends/azure.py +361 -0
  324. fi/evals/guardrails/backends/base.py +88 -0
  325. fi/evals/guardrails/backends/generic_llm.py +163 -0
  326. fi/evals/guardrails/backends/granite.py +216 -0
  327. fi/evals/guardrails/backends/llamaguard.py +221 -0
  328. fi/evals/guardrails/backends/local_base.py +479 -0
  329. fi/evals/guardrails/backends/openai.py +365 -0
  330. fi/evals/guardrails/backends/qwen.py +170 -0
  331. fi/evals/guardrails/backends/shieldgemma.py +154 -0
  332. fi/evals/guardrails/backends/turing.py +235 -0
  333. fi/evals/guardrails/backends/vllm_client.py +321 -0
  334. fi/evals/guardrails/backends/wildguard.py +188 -0
  335. fi/evals/guardrails/base.py +888 -0
  336. fi/evals/guardrails/config.py +221 -0
  337. fi/evals/guardrails/discovery.py +243 -0
  338. fi/evals/guardrails/gateway.py +437 -0
  339. fi/evals/guardrails/registry.py +231 -0
  340. fi/evals/guardrails/scanners/__init__.py +127 -0
  341. fi/evals/guardrails/scanners/base.py +191 -0
  342. fi/evals/guardrails/scanners/code_injection.py +243 -0
  343. fi/evals/guardrails/scanners/eval_delegate.py +574 -0
  344. fi/evals/guardrails/scanners/invisible_chars.py +351 -0
  345. fi/evals/guardrails/scanners/jailbreak.py +412 -0
  346. fi/evals/guardrails/scanners/language.py +288 -0
  347. fi/evals/guardrails/scanners/pipeline.py +260 -0
  348. fi/evals/guardrails/scanners/regex.py +311 -0
  349. fi/evals/guardrails/scanners/secrets.py +274 -0
  350. fi/evals/guardrails/scanners/topics.py +649 -0
  351. fi/evals/guardrails/scanners/urls.py +341 -0
  352. fi/evals/guardrails/types.py +96 -0
  353. fi/evals/llm/__init__.py +3 -0
  354. fi/evals/llm/base_llm_provider.py +35 -0
  355. fi/evals/llm/providers/litellm.py +70 -0
  356. fi/evals/local/__init__.py +90 -0
  357. fi/evals/local/evaluator.py +690 -0
  358. fi/evals/local/execution_mode.py +121 -0
  359. fi/evals/local/llm.py +489 -0
  360. fi/evals/local/metrics/__init__.py +19 -0
  361. fi/evals/local/registry.py +360 -0
  362. fi/evals/manager.py +1018 -0
  363. fi/evals/manager_types.py +362 -0
  364. fi/evals/metrics/__init__.py +185 -0
  365. fi/evals/metrics/agents/__init__.py +74 -0
  366. fi/evals/metrics/agents/metrics.py +693 -0
  367. fi/evals/metrics/agents/report.py +36463 -0
  368. fi/evals/metrics/agents/types.py +160 -0
  369. fi/evals/metrics/base_llm_metric.py +111 -0
  370. fi/evals/metrics/base_metric.py +138 -0
  371. fi/evals/metrics/code_security/__init__.py +305 -0
  372. fi/evals/metrics/code_security/analyzer.py +985 -0
  373. fi/evals/metrics/code_security/benchmarks/__init__.py +73 -0
  374. fi/evals/metrics/code_security/benchmarks/builtin.py +750 -0
  375. fi/evals/metrics/code_security/benchmarks/loader.py +580 -0
  376. fi/evals/metrics/code_security/benchmarks/types.py +308 -0
  377. fi/evals/metrics/code_security/detectors/__init__.py +186 -0
  378. fi/evals/metrics/code_security/detectors/base.py +394 -0
  379. fi/evals/metrics/code_security/detectors/cryptography.py +345 -0
  380. fi/evals/metrics/code_security/detectors/injection.py +744 -0
  381. fi/evals/metrics/code_security/detectors/secrets.py +287 -0
  382. fi/evals/metrics/code_security/detectors/serialization.py +192 -0
  383. fi/evals/metrics/code_security/joint_metrics.py +588 -0
  384. fi/evals/metrics/code_security/judges/__init__.py +83 -0
  385. fi/evals/metrics/code_security/judges/base.py +238 -0
  386. fi/evals/metrics/code_security/judges/dual_judge.py +534 -0
  387. fi/evals/metrics/code_security/judges/llm_judge.py +301 -0
  388. fi/evals/metrics/code_security/judges/pattern_judge.py +515 -0
  389. fi/evals/metrics/code_security/metrics.py +388 -0
  390. fi/evals/metrics/code_security/modes/__init__.py +63 -0
  391. fi/evals/metrics/code_security/modes/adversarial.py +284 -0
  392. fi/evals/metrics/code_security/modes/autocomplete.py +198 -0
  393. fi/evals/metrics/code_security/modes/base.py +283 -0
  394. fi/evals/metrics/code_security/modes/instruct.py +253 -0
  395. fi/evals/metrics/code_security/modes/repair.py +230 -0
  396. fi/evals/metrics/code_security/reports/__init__.py +57 -0
  397. fi/evals/metrics/code_security/reports/generator.py +404 -0
  398. fi/evals/metrics/code_security/reports/leaderboard.py +509 -0
  399. fi/evals/metrics/code_security/types.py +534 -0
  400. fi/evals/metrics/function_calling/__init__.py +34 -0
  401. fi/evals/metrics/function_calling/metrics.py +573 -0
  402. fi/evals/metrics/function_calling/types.py +87 -0
  403. fi/evals/metrics/hallucination/__init__.py +54 -0
  404. fi/evals/metrics/hallucination/detector.py +149 -0
  405. fi/evals/metrics/hallucination/metrics.py +390 -0
  406. fi/evals/metrics/hallucination/nli.py +253 -0
  407. fi/evals/metrics/hallucination/sentinel.py +106 -0
  408. fi/evals/metrics/hallucination/types.py +132 -0
  409. fi/evals/metrics/heuristics/aggregation_metrics.py +85 -0
  410. fi/evals/metrics/heuristics/json_metrics.py +87 -0
  411. fi/evals/metrics/heuristics/similarity_metrics.py +375 -0
  412. fi/evals/metrics/heuristics/string_metrics.py +391 -0
  413. fi/evals/metrics/llm_as_judges/__init__.py +17 -0
  414. fi/evals/metrics/llm_as_judges/custom_judge/metric.py +112 -0
  415. fi/evals/metrics/llm_as_judges/custom_judge/prompts.py +26 -0
  416. fi/evals/metrics/llm_as_judges/types.py +48 -0
  417. fi/evals/metrics/rag/__init__.py +111 -0
  418. fi/evals/metrics/rag/advanced/__init__.py +14 -0
  419. fi/evals/metrics/rag/advanced/multi_hop.py +283 -0
  420. fi/evals/metrics/rag/advanced/source_attribution.py +344 -0
  421. fi/evals/metrics/rag/generation/__init__.py +17 -0
  422. fi/evals/metrics/rag/generation/answer_relevancy.py +176 -0
  423. fi/evals/metrics/rag/generation/context_utilization.py +245 -0
  424. fi/evals/metrics/rag/generation/faithfulness.py +241 -0
  425. fi/evals/metrics/rag/generation/groundedness.py +131 -0
  426. fi/evals/metrics/rag/rag_score.py +277 -0
  427. fi/evals/metrics/rag/retrieval/__init__.py +20 -0
  428. fi/evals/metrics/rag/retrieval/context_entity_recall.py +124 -0
  429. fi/evals/metrics/rag/retrieval/context_precision.py +158 -0
  430. fi/evals/metrics/rag/retrieval/context_recall.py +106 -0
  431. fi/evals/metrics/rag/retrieval/noise_sensitivity.py +163 -0
  432. fi/evals/metrics/rag/retrieval/ranking.py +261 -0
  433. fi/evals/metrics/rag/types.py +100 -0
  434. fi/evals/metrics/rag/utils/__init__.py +62 -0
  435. fi/evals/metrics/rag/utils/claims.py +189 -0
  436. fi/evals/metrics/rag/utils/entities.py +244 -0
  437. fi/evals/metrics/rag/utils/nli.py +92 -0
  438. fi/evals/metrics/rag/utils/similarity.py +345 -0
  439. fi/evals/metrics/structured/__init__.py +114 -0
  440. fi/evals/metrics/structured/field_completeness.py +313 -0
  441. fi/evals/metrics/structured/hierarchy_score.py +366 -0
  442. fi/evals/metrics/structured/json_validation.py +190 -0
  443. fi/evals/metrics/structured/schema_compliance.py +280 -0
  444. fi/evals/metrics/structured/structured_output_score.py +298 -0
  445. fi/evals/metrics/structured/types.py +108 -0
  446. fi/evals/metrics/structured/validators/__init__.py +30 -0
  447. fi/evals/metrics/structured/validators/base.py +189 -0
  448. fi/evals/metrics/structured/validators/json_validator.py +196 -0
  449. fi/evals/metrics/structured/validators/pydantic_validator.py +178 -0
  450. fi/evals/metrics/structured/validators/yaml_validator.py +248 -0
  451. fi/evals/otel/__init__.py +266 -0
  452. fi/evals/otel/config.py +400 -0
  453. fi/evals/otel/conventions.py +463 -0
  454. fi/evals/otel/enrichment.py +371 -0
  455. fi/evals/otel/instrumentors/__init__.py +140 -0
  456. fi/evals/otel/instrumentors/anthropic.py +517 -0
  457. fi/evals/otel/instrumentors/base.py +382 -0
  458. fi/evals/otel/instrumentors/openai.py +673 -0
  459. fi/evals/otel/processors/__init__.py +36 -0
  460. fi/evals/otel/processors/base.py +473 -0
  461. fi/evals/otel/processors/cost.py +445 -0
  462. fi/evals/otel/processors/evaluation.py +559 -0
  463. fi/evals/otel/processors/llm.py +462 -0
  464. fi/evals/otel/tracer.py +506 -0
  465. fi/evals/otel/types.py +232 -0
  466. fi/evals/otel_utils.py +23 -0
  467. fi/evals/protect.py +671 -0
  468. fi/evals/protect_input_adapter.py +154 -0
  469. fi/evals/streaming/__init__.py +88 -0
  470. fi/evals/streaming/buffer.py +213 -0
  471. fi/evals/streaming/evaluator.py +551 -0
  472. fi/evals/streaming/policy.py +307 -0
  473. fi/evals/streaming/scorers.py +368 -0
  474. fi/evals/streaming/types.py +238 -0
  475. fi/evals/templates.py +472 -0
  476. fi/evals/types.py +156 -0
  477. fi/opt/__init__.py +221 -0
  478. fi/opt/_objective_scoring.py +85 -0
  479. fi/opt/base/__init__.py +11 -0
  480. fi/opt/base/base_generator.py +33 -0
  481. fi/opt/base/base_mapper.py +26 -0
  482. fi/opt/base/base_optimizer.py +45 -0
  483. fi/opt/base/evaluator.py +211 -0
  484. fi/opt/components.py +3095 -0
  485. fi/opt/datamappers/__init__.py +3 -0
  486. fi/opt/datamappers/basic_mapper.py +40 -0
  487. fi/opt/deployment.py +1021 -0
  488. fi/opt/evidence.py +4332 -0
  489. fi/opt/generators/__init__.py +3 -0
  490. fi/opt/generators/litellm.py +66 -0
  491. fi/opt/integrations/__init__.py +23 -0
  492. fi/opt/integrations/generative_suite.py +410 -0
  493. fi/opt/integrations/simulate.py +1313 -0
  494. fi/opt/mutations.py +771 -0
  495. fi/opt/observability.py +4639 -0
  496. fi/opt/optimizer_trace.py +889 -0
  497. fi/opt/optimizers/__init__.py +80 -0
  498. fi/opt/optimizers/agent.py +331 -0
  499. fi/opt/optimizers/agent_bandit.py +392 -0
  500. fi/opt/optimizers/agent_curriculum.py +635 -0
  501. fi/opt/optimizers/agent_evolution.py +894 -0
  502. fi/opt/optimizers/agent_feedback.py +1863 -0
  503. fi/opt/optimizers/agent_pareto.py +547 -0
  504. fi/opt/optimizers/agent_social_memory.py +1113 -0
  505. fi/opt/optimizers/agent_tpe.py +321 -0
  506. fi/opt/optimizers/bayesian_search.py +449 -0
  507. fi/opt/optimizers/council.py +2075 -0
  508. fi/opt/optimizers/futureagi_replay.py +799 -0
  509. fi/opt/optimizers/gepa.py +322 -0
  510. fi/opt/optimizers/metaprompt.py +243 -0
  511. fi/opt/optimizers/promptwizard.py +417 -0
  512. fi/opt/optimizers/protegi.py +329 -0
  513. fi/opt/optimizers/random_search.py +224 -0
  514. fi/opt/research.py +518 -0
  515. fi/opt/simulation.py +260 -0
  516. fi/opt/targets.py +232 -0
  517. fi/opt/types.py +66 -0
  518. fi/opt/utils/__init__.py +4 -0
  519. fi/opt/utils/early_stopping.py +266 -0
  520. fi/opt/utils/setup_logging.py +82 -0
  521. fi/simulate/__init__.py +540 -0
  522. fi/simulate/_hashing.py +35 -0
  523. fi/simulate/_logging.py +10 -0
  524. fi/simulate/adapters.py +87 -0
  525. fi/simulate/agent/__init__.py +120 -0
  526. fi/simulate/agent/browser.py +658 -0
  527. fi/simulate/agent/definition.py +587 -0
  528. fi/simulate/agent/frameworks.py +3528 -0
  529. fi/simulate/agent/generic.py +8286 -0
  530. fi/simulate/agent/import_probe.py +227 -0
  531. fi/simulate/agent/memory.py +905 -0
  532. fi/simulate/agent/mocks.py +101 -0
  533. fi/simulate/agent/multi_agent.py +361 -0
  534. fi/simulate/agent/orchestration.py +903 -0
  535. fi/simulate/agent/realtime.py +665 -0
  536. fi/simulate/agent/wrapper.py +99 -0
  537. fi/simulate/agent/wrappers/__init__.py +18 -0
  538. fi/simulate/agent/wrappers/anthropic.py +62 -0
  539. fi/simulate/agent/wrappers/gemini.py +65 -0
  540. fi/simulate/agent/wrappers/http.py +404 -0
  541. fi/simulate/agent/wrappers/langchain.py +80 -0
  542. fi/simulate/agent/wrappers/openai.py +75 -0
  543. fi/simulate/agent/wrappers/websocket.py +326 -0
  544. fi/simulate/artifacts/__init__.py +11 -0
  545. fi/simulate/artifacts/manifest.py +62 -0
  546. fi/simulate/cli.py +20560 -0
  547. fi/simulate/endpoints/__init__.py +45 -0
  548. fi/simulate/endpoints/_http_actor.py +73 -0
  549. fi/simulate/endpoints/actor_sources.py +243 -0
  550. fi/simulate/endpoints/base.py +107 -0
  551. fi/simulate/endpoints/builtins.py +10 -0
  552. fi/simulate/endpoints/callable.py +95 -0
  553. fi/simulate/endpoints/http.py +76 -0
  554. fi/simulate/endpoints/livekit.py +138 -0
  555. fi/simulate/endpoints/originators.py +132 -0
  556. fi/simulate/endpoints/profiles.py +348 -0
  557. fi/simulate/endpoints/retell.py +633 -0
  558. fi/simulate/endpoints/vapi.py +205 -0
  559. fi/simulate/endpoints/websocket.py +76 -0
  560. fi/simulate/environment.py +33026 -0
  561. fi/simulate/environments/__init__.py +11 -0
  562. fi/simulate/environments/base.py +73 -0
  563. fi/simulate/environments/chat.py +697 -0
  564. fi/simulate/environments/voice.py +212 -0
  565. fi/simulate/evaluation/__init__.py +4 -0
  566. fi/simulate/evaluation/ai_eval.py +227 -0
  567. fi/simulate/evidence/__init__.py +35 -0
  568. fi/simulate/evidence/base.py +59 -0
  569. fi/simulate/evidence/caller_observed.py +50 -0
  570. fi/simulate/evidence/livekit_instrumentation.py +51 -0
  571. fi/simulate/evidence/livekit_room.py +50 -0
  572. fi/simulate/evidence/otel.py +49 -0
  573. fi/simulate/evidence/providers/__init__.py +24 -0
  574. fi/simulate/evidence/providers/base.py +61 -0
  575. fi/simulate/evidence/providers/retell.py +376 -0
  576. fi/simulate/evidence/providers/vapi.py +426 -0
  577. fi/simulate/hosted/__init__.py +32 -0
  578. fi/simulate/hosted/child_entrypoint.py +306 -0
  579. fi/simulate/hosted/job.py +150 -0
  580. fi/simulate/hosted/targets.py +53 -0
  581. fi/simulate/instrumentation/__init__.py +5 -0
  582. fi/simulate/instrumentation/livekit/__init__.py +122 -0
  583. fi/simulate/manifest.py +1033 -0
  584. fi/simulate/matrix_cli.py +165 -0
  585. fi/simulate/realtime/__init__.py +40 -0
  586. fi/simulate/realtime/events.py +107 -0
  587. fi/simulate/realtime/media.py +61 -0
  588. fi/simulate/realtime/session.py +91 -0
  589. fi/simulate/recording/__init__.py +5 -0
  590. fi/simulate/recording/room_recorder.py +326 -0
  591. fi/simulate/registry.py +185 -0
  592. fi/simulate/results/__init__.py +9 -0
  593. fi/simulate/results/base.py +18 -0
  594. fi/simulate/results/filesystem.py +71 -0
  595. fi/simulate/results/futureagi.py +1340 -0
  596. fi/simulate/runtime/__init__.py +85 -0
  597. fi/simulate/runtime/capabilities.py +40 -0
  598. fi/simulate/runtime/events.py +63 -0
  599. fi/simulate/runtime/failures.py +25 -0
  600. fi/simulate/runtime/ids.py +34 -0
  601. fi/simulate/runtime/plan.py +70 -0
  602. fi/simulate/runtime/planner.py +102 -0
  603. fi/simulate/runtime/report.py +174 -0
  604. fi/simulate/runtime/run.py +75 -0
  605. fi/simulate/runtime/runner.py +333 -0
  606. fi/simulate/runtime/spec.py +186 -0
  607. fi/simulate/simulation/__init__.py +30 -0
  608. fi/simulate/simulation/behavior_policy.py +425 -0
  609. fi/simulate/simulation/bridge/__init__.py +9 -0
  610. fi/simulate/simulation/bridge/audio.py +29 -0
  611. fi/simulate/simulation/bridge/connector.py +46 -0
  612. fi/simulate/simulation/bridge/livekit.py +252 -0
  613. fi/simulate/simulation/bridge/retell.py +188 -0
  614. fi/simulate/simulation/bridge/vapi.py +177 -0
  615. fi/simulate/simulation/contract.py +419 -0
  616. fi/simulate/simulation/engines/__init__.py +12 -0
  617. fi/simulate/simulation/engines/base.py +21 -0
  618. fi/simulate/simulation/engines/cloud.py +517 -0
  619. fi/simulate/simulation/engines/livekit.py +4167 -0
  620. fi/simulate/simulation/engines/local_text.py +89 -0
  621. fi/simulate/simulation/fidelity.py +374 -0
  622. fi/simulate/simulation/gemini_tts_stream.py +110 -0
  623. fi/simulate/simulation/generator.py +91 -0
  624. fi/simulate/simulation/goal_machine.py +185 -0
  625. fi/simulate/simulation/livekit_models.py +467 -0
  626. fi/simulate/simulation/matrix.py +170 -0
  627. fi/simulate/simulation/models.py +279 -0
  628. fi/simulate/simulation/runner.py +153 -0
  629. fi/simulate/simulation/synthetic.py +880 -0
  630. fi/simulate/simulation/voice_prompt.py +502 -0
  631. fi/simulate/simulator/__init__.py +55 -0
  632. fi/simulate/simulator/builtins.py +53 -0
  633. fi/simulate/suite.py +1288 -0
  634. fi/simulate/utils/routes.py +164 -0
  635. fi/simulate/voice.py +225 -0
  636. fi/simulate/voice_cli.py +182 -0
  637. fi/utils/__init__.py +1 -0
  638. fi/utils/constants.py +14 -0
  639. fi/utils/errors.py +200 -0
  640. fi/utils/executor.py +26 -0
  641. fi/utils/routes.py +119 -0
  642. fi/utils/utils.py +17 -0
@@ -0,0 +1,3528 @@
1
+ from __future__ import annotations
2
+
3
+ import asyncio
4
+ import copy
5
+ import inspect
6
+ from dataclasses import dataclass
7
+ from typing import Any, Callable, Dict, Mapping, Optional, Sequence
8
+ from urllib.parse import urlparse
9
+
10
+ from fi.simulate.agent.generic import GenericAgentWrapper, InputMode
11
+ from fi.simulate.agent.wrapper import AgentInput, AgentResponse, AgentWrapper
12
+
13
+
14
+ @dataclass(frozen=True)
15
+ class FrameworkAdapterSpec:
16
+ """Import-free adapter preset for a common agent/orchestration framework."""
17
+
18
+ name: str
19
+ method: Optional[str]
20
+ input_mode: InputMode
21
+ modality: str = "text"
22
+ transport: str = "in_process"
23
+ lifecycle_hooks: tuple[str, ...] = ()
24
+ capabilities: tuple[str, ...] = ()
25
+ notes: str = ""
26
+
27
+
28
+ FRAMEWORK_PRESETS: Dict[str, FrameworkAdapterSpec] = {
29
+ # Text/chat orchestration
30
+ "custom": FrameworkAdapterSpec("custom", None, "auto", notes="Bring-your-own framework adapter."),
31
+ "callable": FrameworkAdapterSpec("callable", None, "agent_input", notes="Plain Python callable."),
32
+ "a2a": FrameworkAdapterSpec("a2a", "send_message", "dict", notes="Agent2Agent protocol client/server session."),
33
+ "openenv": FrameworkAdapterSpec(
34
+ "openenv",
35
+ "run",
36
+ "dict",
37
+ capabilities=(
38
+ "environment_replay",
39
+ "reset_step_trace",
40
+ "runtime_trace",
41
+ "state",
42
+ "artifacts",
43
+ ),
44
+ notes="OpenEnv/Gymnasium-style environment replay adapter.",
45
+ ),
46
+ "gymnasium": FrameworkAdapterSpec(
47
+ "gymnasium",
48
+ "run",
49
+ "dict",
50
+ capabilities=(
51
+ "environment_replay",
52
+ "reset_step_trace",
53
+ "runtime_trace",
54
+ "state",
55
+ "artifacts",
56
+ ),
57
+ notes="Gymnasium Env reset/step replay adapter.",
58
+ ),
59
+ "langchain": FrameworkAdapterSpec("langchain", "ainvoke", "dict", notes="LangChain Runnable/Chain."),
60
+ "langgraph": FrameworkAdapterSpec("langgraph", "ainvoke", "dict", notes="LangGraph compiled graph."),
61
+ "llamaindex": FrameworkAdapterSpec("llamaindex", "achat", "text", notes="LlamaIndex chat/query engines."),
62
+ "crewai": FrameworkAdapterSpec("crewai", "kickoff", "dict", notes="CrewAI Crew kickoff."),
63
+ "autogen": FrameworkAdapterSpec("autogen", "run", "text", notes="AutoGen AgentChat style task run."),
64
+ "semantic_kernel": FrameworkAdapterSpec("semantic_kernel", "invoke", "dict", notes="Semantic Kernel function/agent."),
65
+ "openai_agents": FrameworkAdapterSpec("openai_agents", "run", "text", notes="OpenAI Agents SDK runner/agent."),
66
+ "pydantic_ai": FrameworkAdapterSpec("pydantic_ai", "run", "text", notes="PydanticAI agent."),
67
+ "haystack": FrameworkAdapterSpec("haystack", "run", "dict", notes="Haystack pipeline."),
68
+ "agno": FrameworkAdapterSpec("agno", "run", "dict", notes="Agno agent/team runner."),
69
+ "beeai": FrameworkAdapterSpec("beeai", "run", "dict", notes="BeeAI agent runner."),
70
+ "claude_agent_sdk": FrameworkAdapterSpec("claude_agent_sdk", "query", "text", notes="Claude Agent SDK query runner."),
71
+ "dspy": FrameworkAdapterSpec("dspy", "__call__", "dict", notes="DSPy module/program."),
72
+ "google_adk": FrameworkAdapterSpec("google_adk", "run", "dict", notes="Google ADK runner/agent."),
73
+ "guardrails": FrameworkAdapterSpec("guardrails", "__call__", "text", notes="Guardrails validation wrapper."),
74
+ "litellm": FrameworkAdapterSpec("litellm", "completion", "dict", notes="LiteLLM completion shim."),
75
+ "mcp": FrameworkAdapterSpec("mcp", "call_tool", "dict", notes="MCP client/server tool session."),
76
+ "portkey": FrameworkAdapterSpec("portkey", "chat", "dict", notes="Portkey gateway client."),
77
+ "smolagents": FrameworkAdapterSpec("smolagents", "run", "text", notes="SmolAgents runner."),
78
+ "strands": FrameworkAdapterSpec("strands", "__call__", "text", notes="Strands agent callable."),
79
+ # Voice and realtime
80
+ "livekit": FrameworkAdapterSpec("livekit", "respond", "text", modality="voice", notes="LiveKit agent/session shim."),
81
+ "pipecat": FrameworkAdapterSpec("pipecat", "process", "dict", modality="voice", notes="Pipecat pipeline/processor shim."),
82
+ "vapi": FrameworkAdapterSpec("vapi", "respond", "dict", modality="voice", notes="Webhook/local adapter shim."),
83
+ "retell": FrameworkAdapterSpec("retell", "respond", "dict", modality="voice", notes="Webhook/local adapter shim."),
84
+ "elevenlabs": FrameworkAdapterSpec("elevenlabs", "respond", "dict", modality="voice", notes="ElevenLabs conversational agent shim."),
85
+ "deepgram": FrameworkAdapterSpec("deepgram", "respond", "dict", modality="voice", notes="Deepgram voice agent shim."),
86
+ "agora": FrameworkAdapterSpec("agora", "respond", "dict", modality="voice", notes="Agora conversational AI shim."),
87
+ "twilio": FrameworkAdapterSpec("twilio", "respond", "dict", modality="voice", notes="Twilio voice/media stream webhook shim."),
88
+ # Model/provider clients commonly instrumented by TraceAI
89
+ "anthropic": FrameworkAdapterSpec("anthropic", "messages.create", "messages", notes="Anthropic messages client shim."),
90
+ "bedrock": FrameworkAdapterSpec("bedrock", "invoke_model", "dict", notes="AWS Bedrock client shim."),
91
+ "cerebras": FrameworkAdapterSpec("cerebras", "chat", "dict", notes="Cerebras client shim."),
92
+ "cohere": FrameworkAdapterSpec("cohere", "chat", "dict", notes="Cohere client shim."),
93
+ "deepseek": FrameworkAdapterSpec("deepseek", "chat", "dict", notes="DeepSeek OpenAI-compatible client shim."),
94
+ "fireworks": FrameworkAdapterSpec("fireworks", "chat", "dict", notes="Fireworks client shim."),
95
+ "google_genai": FrameworkAdapterSpec("google_genai", "generate_content", "dict", notes="Google GenAI client shim."),
96
+ "groq": FrameworkAdapterSpec("groq", "chat", "dict", notes="Groq client shim."),
97
+ "huggingface": FrameworkAdapterSpec("huggingface", "__call__", "dict", notes="Hugging Face pipeline/client shim."),
98
+ "instructor": FrameworkAdapterSpec("instructor", "chat", "dict", notes="Instructor structured output client shim."),
99
+ "mistralai": FrameworkAdapterSpec("mistralai", "chat", "dict", notes="Mistral AI client shim."),
100
+ "ollama": FrameworkAdapterSpec("ollama", "chat", "dict", notes="Ollama client shim."),
101
+ "openai": FrameworkAdapterSpec("openai", "chat.completions.create", "messages", notes="OpenAI chat client shim."),
102
+ "together": FrameworkAdapterSpec("together", "chat", "dict", notes="Together AI client shim."),
103
+ "vertexai": FrameworkAdapterSpec("vertexai", "generate_content", "dict", notes="Vertex AI client shim."),
104
+ "vllm": FrameworkAdapterSpec("vllm", "generate", "dict", notes="vLLM server/client shim."),
105
+ "xai": FrameworkAdapterSpec("xai", "chat", "dict", notes="xAI client shim."),
106
+ # Computer-use / browser / multimodal
107
+ "computer_use": FrameworkAdapterSpec("computer_use", "run", "dict", modality="cua", notes="Browser or desktop CUA runner."),
108
+ "browser_use": FrameworkAdapterSpec("browser_use", "run", "dict", modality="cua", notes="Browser automation agent."),
109
+ "playwright": FrameworkAdapterSpec("playwright", "run", "dict", modality="cua", notes="Playwright-backed agent harness."),
110
+ "vision_agent": FrameworkAdapterSpec("vision_agent", "run", "dict", modality="image", notes="Image or multimodal agent."),
111
+ }
112
+
113
+
114
+ _DISCOVERY_METHOD_ORDER = (
115
+ "ainvoke",
116
+ "invoke",
117
+ "astream",
118
+ "stream",
119
+ "stream_events",
120
+ "execute_task",
121
+ "process_frame",
122
+ "send_message",
123
+ "message_send",
124
+ "call",
125
+ "achat",
126
+ "chat",
127
+ "responses.create",
128
+ "chat.completions.create",
129
+ "messages.create",
130
+ "kickoff",
131
+ "query",
132
+ "process",
133
+ "respond",
134
+ "run",
135
+ "run_stream",
136
+ "arun",
137
+ "send",
138
+ "completion",
139
+ "call_tool",
140
+ "invoke_model",
141
+ "generate_content",
142
+ "generate",
143
+ "__call__",
144
+ )
145
+
146
+ _DISCOVERY_METHOD_INPUT_MODES: dict[str, InputMode] = {
147
+ "ainvoke": "dict",
148
+ "invoke": "dict",
149
+ "astream": "dict",
150
+ "stream": "dict",
151
+ "stream_events": "dict",
152
+ "execute_task": "dict",
153
+ "process_frame": "dict",
154
+ "send_message": "dict",
155
+ "message_send": "dict",
156
+ "responses.create": "text",
157
+ "chat.completions.create": "messages",
158
+ "messages.create": "messages",
159
+ "kickoff": "dict",
160
+ "process": "dict",
161
+ "completion": "dict",
162
+ "call_tool": "dict",
163
+ "invoke_model": "dict",
164
+ "generate_content": "dict",
165
+ "generate": "dict",
166
+ "call": "agent_input",
167
+ "achat": "text",
168
+ "chat": "text",
169
+ "query": "text",
170
+ "respond": "text",
171
+ "run": "text",
172
+ "run_stream": "text",
173
+ "arun": "text",
174
+ "send": "text",
175
+ }
176
+
177
+ _STREAMING_METHODS = {"astream", "stream", "stream_events", "run_stream"}
178
+
179
+ _KEYWORD_INPUT_NAMES = (
180
+ "inputs",
181
+ "input",
182
+ "payload",
183
+ "frame",
184
+ "request",
185
+ "contents",
186
+ "arguments",
187
+ "task",
188
+ "user_prompt",
189
+ "prompt",
190
+ "message",
191
+ "messages",
192
+ "query",
193
+ "data",
194
+ )
195
+
196
+ _METHOD_INPUT_KEY_PREFERENCES = {
197
+ "execute_task": ("task", "input", "payload"),
198
+ "kickoff": ("inputs", "input", "payload"),
199
+ "run": ("task", "user_prompt", "prompt", "input"),
200
+ "arun": ("task", "user_prompt", "prompt", "input"),
201
+ "run_stream": ("task", "user_prompt", "prompt", "input"),
202
+ "send_message": ("message", "payload", "input"),
203
+ "message_send": ("message", "payload", "input"),
204
+ "send": ("message", "messages", "input"),
205
+ "achat": ("message", "messages", "input"),
206
+ "chat": ("message", "messages", "input"),
207
+ "query": ("query", "input", "message"),
208
+ "respond": ("message", "input", "payload"),
209
+ "process": ("frame", "payload", "input", "data"),
210
+ "process_frame": ("frame", "payload", "input", "data"),
211
+ "responses.create": ("input", "messages", "payload"),
212
+ "chat.completions.create": ("messages", "input", "payload"),
213
+ "messages.create": ("messages", "input", "payload"),
214
+ "completion": ("request", "payload", "input"),
215
+ "call_tool": ("payload", "input", "arguments"),
216
+ "invoke_model": ("payload", "input", "request"),
217
+ "generate_content": ("contents", "input", "payload"),
218
+ "generate": ("prompt", "input", "payload"),
219
+ }
220
+
221
+ _DISCOVERY_INPUT_MODE_ORDER: tuple[InputMode, ...] = (
222
+ "dict",
223
+ "text",
224
+ "agent_input",
225
+ "messages",
226
+ "auto",
227
+ )
228
+
229
+
230
+ def supported_frameworks() -> list[str]:
231
+ """Return built-in framework preset names.
232
+
233
+ ``wrap_framework`` also accepts unknown framework names as custom adapters
234
+ when the caller supplies method/input-mode overrides or the generic wrapper
235
+ can infer a callable method.
236
+ """
237
+
238
+ return sorted(FRAMEWORK_PRESETS)
239
+
240
+
241
+ def framework_adapter_contract(
242
+ framework: str,
243
+ *,
244
+ target: str | None = None,
245
+ method: str | None = None,
246
+ input_mode: InputMode | None = None,
247
+ input_key: str | None = None,
248
+ input_kwargs: Mapping[str, Any] | None = None,
249
+ modality: str | None = None,
250
+ trace_runtime: bool = True,
251
+ metadata: Optional[Dict[str, Any]] = None,
252
+ ) -> dict[str, Any]:
253
+ """Return the native adapter contract used for framework simulation.
254
+
255
+ The contract is import-free and local: it describes the framework shim,
256
+ lifecycle, transport, capabilities, schemas, and replay requirements without
257
+ pulling in LangGraph, LiveKit, Pipecat, or any other framework package.
258
+ """
259
+
260
+ meta = dict(metadata or {})
261
+ key = _framework_key(framework)
262
+ spec = FRAMEWORK_PRESETS.get(key)
263
+ adapter_kind = "preset" if spec is not None else "custom"
264
+ resolved_method = str(method or (spec.method if spec else "") or "auto")
265
+ resolved_input_mode = str(input_mode or (spec.input_mode if spec else "") or "auto")
266
+ resolved_modality = str(modality or meta.get("modality") or (spec.modality if spec else "text"))
267
+ transport = str((spec.transport if spec else "") or _default_transport(resolved_modality))
268
+ lifecycle_hooks = list(
269
+ spec.lifecycle_hooks
270
+ if spec and spec.lifecycle_hooks
271
+ else _default_lifecycle_hooks(resolved_modality)
272
+ )
273
+ capabilities = list(
274
+ spec.capabilities
275
+ if spec and spec.capabilities
276
+ else _default_capabilities(resolved_modality, resolved_input_mode)
277
+ )
278
+ target_scheme = urlparse(str(target or "")).scheme.lower()
279
+ target_is_external = target_scheme in {"http", "https"}
280
+ local_fixture = not target_is_external
281
+
282
+ contract: dict[str, Any] = {
283
+ "kind": "agent-learning.framework-adapter-contract.v1",
284
+ "framework": key,
285
+ "adapter": adapter_kind,
286
+ "method": resolved_method,
287
+ "input_mode": resolved_input_mode,
288
+ "modality": resolved_modality,
289
+ "transport": transport,
290
+ "lifecycle_hooks": lifecycle_hooks,
291
+ "capabilities": capabilities,
292
+ "schemas": {
293
+ "input": _input_schema(resolved_input_mode),
294
+ "output": _output_schema(),
295
+ },
296
+ "trace_runtime": bool(trace_runtime),
297
+ "requires_external_service": False,
298
+ "local_executable_fixture": local_fixture,
299
+ "evidence_requirements": [
300
+ "framework_runtime",
301
+ "framework_trace",
302
+ "tool_calls",
303
+ "adapter_conformance",
304
+ "metric_evidence",
305
+ ],
306
+ }
307
+ if input_key:
308
+ contract["input_key"] = str(input_key)
309
+ input_kwargs_keys = sorted(str(key) for key in dict(input_kwargs or {}))
310
+ if input_kwargs_keys:
311
+ contract["input_kwargs_keys"] = input_kwargs_keys
312
+ if target:
313
+ contract["target"] = str(target)
314
+ contract["target_scheme"] = target_scheme
315
+ if spec and spec.notes:
316
+ contract["notes"] = spec.notes
317
+ if key in {"openenv", "gymnasium", "gymnasium_env", "environment_replay"}:
318
+ contract["evidence_requirements"] = [
319
+ *contract["evidence_requirements"],
320
+ "openenv",
321
+ ]
322
+ return contract
323
+
324
+
325
+ def framework_adapter_contract_matrix(
326
+ frameworks: Sequence[str] | str | None = None,
327
+ *,
328
+ targets: Mapping[str, str] | None = None,
329
+ methods: Mapping[str, str] | None = None,
330
+ input_modes: Mapping[str, InputMode] | None = None,
331
+ modalities: Mapping[str, str] | None = None,
332
+ trace_runtime: bool = True,
333
+ allow_external_targets: bool = False,
334
+ metadata: Optional[Dict[str, Any]] = None,
335
+ ) -> dict[str, Any]:
336
+ """Return a native, import-free adapter contract matrix.
337
+
338
+ The matrix is a first-party certification artifact: it proves what the
339
+ Agent Learning simulator can run through local adapter fixtures without
340
+ importing or calling LangGraph, LiveKit, Pipecat, or other frameworks.
341
+ HTTP/HTTPS targets are rejected by default so this path stays native unless
342
+ a caller explicitly opts into external target documentation.
343
+ """
344
+
345
+ framework_keys = _framework_matrix_keys(frameworks)
346
+ target_map = {
347
+ _framework_key(key): str(value)
348
+ for key, value in dict(targets or {}).items()
349
+ }
350
+ method_map = {
351
+ _framework_key(key): str(value)
352
+ for key, value in dict(methods or {}).items()
353
+ }
354
+ input_mode_map = {
355
+ _framework_key(key): value for key, value in dict(input_modes or {}).items()
356
+ }
357
+ modality_map = {
358
+ _framework_key(key): str(value)
359
+ for key, value in dict(modalities or {}).items()
360
+ }
361
+ external_targets = {
362
+ key: target
363
+ for key, target in target_map.items()
364
+ if _is_external_target(target)
365
+ }
366
+ if external_targets and not allow_external_targets:
367
+ blocked = ", ".join(
368
+ f"{key}={target}" for key, target in sorted(external_targets.items())
369
+ )
370
+ raise ValueError(
371
+ "external targets are disabled for native framework adapter matrices: "
372
+ f"{blocked}"
373
+ )
374
+
375
+ contracts = [
376
+ framework_adapter_contract(
377
+ key,
378
+ target=target_map.get(key) or _local_fixture_target(key),
379
+ method=method_map.get(key),
380
+ input_mode=input_mode_map.get(key),
381
+ modality=modality_map.get(key),
382
+ trace_runtime=trace_runtime,
383
+ metadata=copy_metadata,
384
+ )
385
+ for key in framework_keys
386
+ for copy_metadata in [dict(metadata or {})]
387
+ ]
388
+ profiles = [
389
+ _framework_adapter_capability_profile_from_contract(contract)
390
+ for contract in contracts
391
+ ]
392
+ findings = _framework_matrix_findings(contracts)
393
+ summary = _framework_matrix_summary(contracts)
394
+ return {
395
+ "kind": "agent-learning.framework-adapter-contract-matrix.v1",
396
+ "status": "passed" if not findings else "failed",
397
+ "requires_external_service": False,
398
+ "runtime": "in_process",
399
+ "allow_external_targets": bool(allow_external_targets),
400
+ "framework_count": len(framework_keys),
401
+ "frameworks": framework_keys,
402
+ "contracts": contracts,
403
+ "profiles": profiles,
404
+ "summary": summary,
405
+ "profile_summary": _framework_profile_collection_summary(profiles),
406
+ "findings": findings,
407
+ "contract_quality_gate": {
408
+ "kind": "agent-learning.framework-adapter-contract.v1",
409
+ "required_frameworks": framework_keys,
410
+ "require_trace_runtime": bool(trace_runtime),
411
+ "require_local_executable_fixture": not bool(allow_external_targets),
412
+ "require_no_external_service": True,
413
+ "require_target": True,
414
+ "forbidden_target_schemes": (
415
+ [] if allow_external_targets else ["http", "https"]
416
+ ),
417
+ "required_schema_sections": ["input", "output"],
418
+ "required_lifecycle_hooks": ["setup", "teardown"],
419
+ "required_capabilities": ["messages", "tool_calls", "runtime_trace"],
420
+ "required_evidence_requirements": [
421
+ "framework_runtime",
422
+ "framework_trace",
423
+ "tool_calls",
424
+ "adapter_conformance",
425
+ "metric_evidence",
426
+ ],
427
+ },
428
+ "evidence_requirements": [
429
+ "framework_runtime",
430
+ "framework_trace",
431
+ "adapter_conformance",
432
+ "metric_evidence",
433
+ "matrix_coverage",
434
+ ],
435
+ }
436
+
437
+
438
+ def framework_adapter_capability_profile(
439
+ framework: str,
440
+ *,
441
+ target: str | None = None,
442
+ method: str | None = None,
443
+ input_mode: InputMode | None = None,
444
+ input_key: str | None = None,
445
+ input_kwargs: Mapping[str, Any] | None = None,
446
+ modality: str | None = None,
447
+ trace_runtime: bool = True,
448
+ metadata: Optional[Dict[str, Any]] = None,
449
+ contract: Mapping[str, Any] | None = None,
450
+ ) -> dict[str, Any]:
451
+ """Return a portable trinity profile for one framework adapter.
452
+
453
+ Profiles are derived from the native adapter contract and intentionally do
454
+ not import LangChain, LiveKit, Pipecat, or any other framework. The payload
455
+ is the shared handshake between simulate-sdk, ai-evaluation, and agent-opt:
456
+ it tells each library which local contract, metric, and optimization layer
457
+ represent the framework surface.
458
+ """
459
+
460
+ selected_contract = (
461
+ copy.deepcopy(dict(contract))
462
+ if contract is not None
463
+ else framework_adapter_contract(
464
+ framework,
465
+ target=target,
466
+ method=method,
467
+ input_mode=input_mode,
468
+ input_key=input_key,
469
+ input_kwargs=input_kwargs,
470
+ modality=modality,
471
+ trace_runtime=trace_runtime,
472
+ metadata=metadata,
473
+ )
474
+ )
475
+ return _framework_adapter_capability_profile_from_contract(selected_contract)
476
+
477
+
478
+ def framework_adapter_capability_profiles(
479
+ frameworks: Sequence[str] | str | None = None,
480
+ *,
481
+ matrix: Mapping[str, Any] | None = None,
482
+ targets: Mapping[str, str] | None = None,
483
+ methods: Mapping[str, str] | None = None,
484
+ input_modes: Mapping[str, InputMode] | None = None,
485
+ modalities: Mapping[str, str] | None = None,
486
+ trace_runtime: bool = True,
487
+ allow_external_targets: bool = False,
488
+ metadata: Optional[Dict[str, Any]] = None,
489
+ ) -> dict[str, Any]:
490
+ """Return framework capability profiles for a matrix or framework list."""
491
+
492
+ matrix_payload = (
493
+ copy.deepcopy(dict(matrix))
494
+ if matrix is not None
495
+ else framework_adapter_contract_matrix(
496
+ frameworks,
497
+ targets=targets,
498
+ methods=methods,
499
+ input_modes=input_modes,
500
+ modalities=modalities,
501
+ trace_runtime=trace_runtime,
502
+ allow_external_targets=allow_external_targets,
503
+ metadata=metadata,
504
+ )
505
+ )
506
+ contracts = [
507
+ copy.deepcopy(dict(contract))
508
+ for contract in matrix_payload.get("contracts", []) or []
509
+ if isinstance(contract, Mapping)
510
+ ]
511
+ if not contracts:
512
+ framework_keys = _framework_matrix_keys(
513
+ matrix_payload.get("frameworks") or frameworks
514
+ )
515
+ contracts = [
516
+ framework_adapter_contract(
517
+ key,
518
+ target=(dict(targets or {}).get(key) if targets else None),
519
+ method=(dict(methods or {}).get(key) if methods else None),
520
+ input_mode=(dict(input_modes or {}).get(key) if input_modes else None),
521
+ modality=(dict(modalities or {}).get(key) if modalities else None),
522
+ trace_runtime=trace_runtime,
523
+ metadata=metadata,
524
+ )
525
+ for key in framework_keys
526
+ ]
527
+
528
+ profiles = [
529
+ _framework_adapter_capability_profile_from_contract(contract)
530
+ for contract in contracts
531
+ ]
532
+ required_frameworks = _framework_matrix_keys(
533
+ matrix_payload.get("frameworks") or frameworks
534
+ )
535
+ findings = _framework_profile_collection_findings(
536
+ profiles,
537
+ required_frameworks=required_frameworks,
538
+ )
539
+ return {
540
+ "kind": "agent-learning.framework-adapter-capability-profiles.v1",
541
+ "status": "passed" if not findings else "failed",
542
+ "passed": not findings,
543
+ "requires_external_service": False,
544
+ "framework_count": len(required_frameworks),
545
+ "profile_count": len(profiles),
546
+ "frameworks": required_frameworks,
547
+ "profiles": profiles,
548
+ "summary": _framework_profile_collection_summary(profiles),
549
+ "findings": findings,
550
+ "source_matrix_kind": matrix_payload.get("kind"),
551
+ "evidence_requirements": [
552
+ "framework_adapter_contract",
553
+ "framework_adapter_profile",
554
+ "framework_runtime",
555
+ "framework_trace",
556
+ "metric_evidence",
557
+ "optimization_lineage",
558
+ ],
559
+ }
560
+
561
+
562
+ def wrap_framework(
563
+ framework: str,
564
+ agent: Any,
565
+ *,
566
+ target: str | None = None,
567
+ method: str | None = None,
568
+ input_mode: InputMode | None = None,
569
+ input_key: str | None = None,
570
+ input_kwargs: Mapping[str, Any] | None = None,
571
+ system_prompt: str | None = None,
572
+ output_key: str | None = None,
573
+ metadata: Optional[Dict[str, Any]] = None,
574
+ trace_runtime: bool = False,
575
+ runtime_metadata: Optional[Dict[str, Any]] = None,
576
+ ) -> AgentWrapper:
577
+ """
578
+ Wrap a known or custom framework by name without importing that framework.
579
+
580
+ Presets are intentionally thin. They encode the most common method/payload
581
+ shape while leaving escape hatches for custom method, input_mode, and
582
+ output_key.
583
+ """
584
+
585
+ key = framework.lower().replace("-", "_")
586
+ spec = FRAMEWORK_PRESETS.get(key)
587
+ raw_metadata = dict(metadata or {})
588
+ contract = raw_metadata.get("framework_adapter_contract")
589
+ if not isinstance(contract, dict):
590
+ contract = framework_adapter_contract(
591
+ key,
592
+ target=target,
593
+ method=method,
594
+ input_mode=input_mode,
595
+ input_key=input_key,
596
+ input_kwargs=input_kwargs,
597
+ trace_runtime=trace_runtime,
598
+ metadata=raw_metadata,
599
+ )
600
+ runtime = dict(runtime_metadata or {})
601
+ runtime.setdefault("framework_adapter_contract", contract)
602
+ if spec is None:
603
+ return GenericAgentWrapper(
604
+ agent,
605
+ method=method,
606
+ input_mode=input_mode or "auto",
607
+ input_key=input_key,
608
+ input_kwargs=input_kwargs,
609
+ output_key=output_key,
610
+ system_prompt=system_prompt,
611
+ metadata={
612
+ "framework": key,
613
+ "modality": str(raw_metadata.get("modality") or "text"),
614
+ "adapter": "custom",
615
+ "framework_adapter_contract": contract,
616
+ **raw_metadata,
617
+ },
618
+ trace_runtime=trace_runtime,
619
+ runtime_metadata=runtime,
620
+ )
621
+
622
+ return GenericAgentWrapper(
623
+ agent,
624
+ method=method or spec.method,
625
+ input_mode=input_mode or spec.input_mode,
626
+ input_key=input_key,
627
+ input_kwargs=input_kwargs,
628
+ output_key=output_key,
629
+ system_prompt=system_prompt,
630
+ metadata={
631
+ "framework": spec.name,
632
+ "modality": spec.modality,
633
+ "framework_adapter_contract": contract,
634
+ **raw_metadata,
635
+ },
636
+ trace_runtime=trace_runtime,
637
+ runtime_metadata=runtime,
638
+ )
639
+
640
+
641
+ async def probe_framework_adapter(
642
+ framework: str,
643
+ agent: Any,
644
+ *,
645
+ cases: Sequence[Mapping[str, Any]] | None = None,
646
+ target: str | None = None,
647
+ method: str | Callable[..., Any] | None = None,
648
+ input_mode: InputMode | None = None,
649
+ input_key: str | None = None,
650
+ input_kwargs: Mapping[str, Any] | None = None,
651
+ system_prompt: str | None = None,
652
+ output_key: str | None = None,
653
+ metadata: Optional[Dict[str, Any]] = None,
654
+ trace_runtime: bool = True,
655
+ allow_external_target: bool = False,
656
+ ) -> dict[str, Any]:
657
+ """Run a local adapter conformance probe for any framework shim.
658
+
659
+ The probe is intentionally import-free: callers pass an already-created
660
+ LangChain/LangGraph/LiveKit/Pipecat/custom object or plain callable. The
661
+ function wraps it with the same generic adapter used by manifests, executes
662
+ representative cases, and returns runtime evidence that can feed evals,
663
+ reports, optimizer proofs, or Future AGI UI cards.
664
+ """
665
+
666
+ if target and _is_external_target(target) and not allow_external_target:
667
+ raise ValueError(
668
+ "external targets are disabled for framework adapter probes; "
669
+ "set allow_external_target=True only when the user explicitly "
670
+ "wants to test that live workload"
671
+ )
672
+
673
+ key = _framework_key(framework)
674
+ method_name = _probe_method_name(method)
675
+ selected_metadata = dict(metadata or {})
676
+ contract = framework_adapter_contract(
677
+ key,
678
+ target=target,
679
+ method=method_name,
680
+ input_mode=input_mode,
681
+ input_key=input_key,
682
+ input_kwargs=input_kwargs,
683
+ trace_runtime=trace_runtime,
684
+ metadata=selected_metadata,
685
+ )
686
+ callable_signature = _adapter_callable_signature(
687
+ agent,
688
+ method=method,
689
+ method_name=str(contract.get("method") or method_name or ""),
690
+ )
691
+ if callable_signature:
692
+ contract["callable_signature"] = callable_signature
693
+ selected_metadata["framework_adapter_contract"] = contract
694
+ wrapper = wrap_framework(
695
+ key,
696
+ agent,
697
+ target=target,
698
+ method=method,
699
+ input_mode=input_mode,
700
+ input_key=input_key,
701
+ input_kwargs=input_kwargs,
702
+ system_prompt=system_prompt,
703
+ output_key=output_key,
704
+ metadata=selected_metadata,
705
+ trace_runtime=trace_runtime,
706
+ runtime_metadata={"framework_adapter_contract": contract},
707
+ )
708
+
709
+ probe_cases = _probe_cases(cases)
710
+ case_results: list[dict[str, Any]] = []
711
+ findings: list[dict[str, Any]] = []
712
+
713
+ for index, case in enumerate(probe_cases, start=1):
714
+ case_result = await _run_probe_case(
715
+ wrapper,
716
+ key,
717
+ case,
718
+ index=index,
719
+ trace_runtime=trace_runtime,
720
+ contract=contract,
721
+ )
722
+ case_results.append(case_result)
723
+ findings.extend(case_result.get("findings", []))
724
+
725
+ summary = _probe_summary(case_results, contract)
726
+ status = "passed" if not findings else "failed"
727
+ return {
728
+ "kind": "agent-learning.framework-adapter-probe.v1",
729
+ "status": status,
730
+ "passed": status == "passed",
731
+ "framework": key,
732
+ "method": method_name or str(contract.get("method") or "auto"),
733
+ "input_mode": str(input_mode or contract.get("input_mode") or "auto"),
734
+ "input_key": str(input_key or contract.get("input_key") or "")
735
+ or None,
736
+ "input_kwargs_keys": sorted(str(key) for key in dict(input_kwargs or {})),
737
+ "requires_external_service": bool(contract.get("requires_external_service")),
738
+ "allow_external_target": bool(allow_external_target),
739
+ "contract": contract,
740
+ "summary": summary,
741
+ "cases": case_results,
742
+ "findings": findings,
743
+ }
744
+
745
+
746
+ def run_framework_adapter_probe(
747
+ framework: str,
748
+ agent: Any,
749
+ **kwargs: Any,
750
+ ) -> dict[str, Any]:
751
+ """Synchronous wrapper for :func:`probe_framework_adapter`.
752
+
753
+ Use ``await probe_framework_adapter(...)`` when already inside an event loop.
754
+ """
755
+
756
+ try:
757
+ asyncio.get_running_loop()
758
+ except RuntimeError:
759
+ return asyncio.run(probe_framework_adapter(framework, agent, **kwargs))
760
+ raise RuntimeError(
761
+ "run_framework_adapter_probe cannot run inside an active event loop; "
762
+ "await probe_framework_adapter(...) instead"
763
+ )
764
+
765
+
766
+ def discover_framework_adapter(
767
+ framework: str,
768
+ agent: Any = None,
769
+ *,
770
+ target: str | None = None,
771
+ method_candidates: Sequence[str | None] | None = None,
772
+ input_mode_candidates: Sequence[InputMode] | None = None,
773
+ modality: str | None = None,
774
+ trace_runtime: bool = True,
775
+ metadata: Optional[Dict[str, Any]] = None,
776
+ allow_external_target: bool = False,
777
+ max_candidates: int | None = 24,
778
+ ) -> dict[str, Any]:
779
+ """Discover local adapter candidates for an arbitrary framework object.
780
+
781
+ Discovery never imports optional framework packages or calls the supplied
782
+ agent. It inspects callable attributes, combines them with the built-in
783
+ framework presets, and returns ranked method/input-mode contracts that can
784
+ be passed directly to ``optimize_framework_adapter_probe``.
785
+ """
786
+
787
+ if target and _is_external_target(target) and not allow_external_target:
788
+ raise ValueError(
789
+ "external targets are disabled for framework adapter discovery; "
790
+ "set allow_external_target=True only when the user explicitly "
791
+ "wants to document that live workload"
792
+ )
793
+ if max_candidates is not None and int(max_candidates) <= 0:
794
+ raise ValueError("max_candidates must be greater than zero")
795
+
796
+ key = _framework_key(framework)
797
+ spec = FRAMEWORK_PRESETS.get(key)
798
+ selected_metadata = dict(metadata or {})
799
+ inventory = _adapter_discovery_inventory(agent)
800
+ methods = _adapter_discovery_methods(
801
+ agent,
802
+ spec=spec,
803
+ method_candidates=method_candidates,
804
+ )
805
+ input_modes = _adapter_discovery_input_modes(
806
+ spec=spec,
807
+ input_mode_candidates=input_mode_candidates,
808
+ )
809
+
810
+ candidates: list[dict[str, Any]] = []
811
+ seen_pairs: set[tuple[str, str]] = set()
812
+ explicit_methods = {
813
+ str(method)
814
+ for method in method_candidates or []
815
+ if method is not None and str(method)
816
+ }
817
+
818
+ for method_name in methods:
819
+ for input_mode in _adapter_discovery_modes_for_method(
820
+ method_name,
821
+ input_modes,
822
+ spec=spec,
823
+ ):
824
+ pair = (str(method_name or ""), str(input_mode))
825
+ if pair in seen_pairs:
826
+ continue
827
+ seen_pairs.add(pair)
828
+ input_key = _adapter_candidate_input_key(
829
+ agent,
830
+ method_name,
831
+ input_mode,
832
+ )
833
+ contract = framework_adapter_contract(
834
+ key,
835
+ target=target,
836
+ method=method_name,
837
+ input_mode=input_mode,
838
+ input_key=input_key,
839
+ modality=modality,
840
+ trace_runtime=trace_runtime,
841
+ metadata=selected_metadata,
842
+ )
843
+ scoring = _adapter_discovery_score(
844
+ method_name,
845
+ input_mode,
846
+ spec=spec,
847
+ inventory=inventory,
848
+ explicit_methods=explicit_methods,
849
+ )
850
+ adapter_candidate: dict[str, Any] = {
851
+ "input_mode": input_mode,
852
+ "trace_runtime": bool(trace_runtime),
853
+ }
854
+ if method_name:
855
+ adapter_candidate["method"] = method_name
856
+ if input_key:
857
+ adapter_candidate["input_key"] = input_key
858
+ if target:
859
+ adapter_candidate["target"] = str(target)
860
+ candidates.append(
861
+ {
862
+ "rank": 0,
863
+ "framework": key,
864
+ "method": method_name or "auto",
865
+ "input_mode": input_mode,
866
+ "score": scoring["score"],
867
+ "reasons": scoring["reasons"],
868
+ "agent_method_present": _adapter_method_present(
869
+ inventory,
870
+ method_name,
871
+ ),
872
+ "contract": contract,
873
+ "adapter_candidate": adapter_candidate,
874
+ }
875
+ )
876
+
877
+ candidates.sort(
878
+ key=lambda item: (
879
+ -float(item["score"]),
880
+ _adapter_method_rank(str(item.get("method") or "")),
881
+ _adapter_input_mode_rank(str(item.get("input_mode") or "")),
882
+ )
883
+ )
884
+ if max_candidates is not None:
885
+ candidates = candidates[: int(max_candidates)]
886
+ for rank, candidate in enumerate(candidates, start=1):
887
+ candidate["rank"] = rank
888
+
889
+ adapter_candidates = [
890
+ dict(candidate["adapter_candidate"]) for candidate in candidates
891
+ ]
892
+ findings = _adapter_discovery_findings(inventory, candidates)
893
+ status = "passed" if candidates else "failed"
894
+ top = candidates[0] if candidates else {}
895
+ return {
896
+ "kind": "agent-learning.framework-adapter-discovery.v1",
897
+ "status": status,
898
+ "passed": status == "passed",
899
+ "framework": key,
900
+ "target": str(target) if target else None,
901
+ "requires_external_service": False,
902
+ "allow_external_target": bool(allow_external_target),
903
+ "trace_runtime": bool(trace_runtime),
904
+ "agent": inventory,
905
+ "summary": {
906
+ "framework": key,
907
+ "candidate_count": len(candidates),
908
+ "adapter_candidate_count": len(adapter_candidates),
909
+ "max_candidates": max_candidates,
910
+ "top_method": top.get("method"),
911
+ "top_input_mode": top.get("input_mode"),
912
+ "top_score": top.get("score"),
913
+ "agent_provided": bool(inventory.get("provided")),
914
+ "agent_callable": bool(inventory.get("callable")),
915
+ "method_count": len(inventory.get("exposed_methods", [])),
916
+ "local_executable_fixture": not bool(target and _is_external_target(target)),
917
+ },
918
+ "candidates": candidates,
919
+ "adapter_candidates": adapter_candidates,
920
+ "findings": findings,
921
+ "evidence_requirements": [
922
+ "local_introspection",
923
+ "framework_adapter_contract",
924
+ "adapter_candidates",
925
+ "framework_adapter_probe",
926
+ "metric_evidence",
927
+ ],
928
+ }
929
+
930
+
931
+ def _framework_key(value: str) -> str:
932
+ return str(value or "custom").strip().lower().replace("-", "_") or "custom"
933
+
934
+
935
+ def _framework_matrix_keys(frameworks: Sequence[str] | str | None) -> list[str]:
936
+ default_frameworks = (
937
+ "langchain",
938
+ "langgraph",
939
+ "llamaindex",
940
+ "crewai",
941
+ "autogen",
942
+ "openai_agents",
943
+ "livekit",
944
+ "pipecat",
945
+ )
946
+ values: Sequence[str] | str = frameworks or default_frameworks
947
+ if isinstance(values, str):
948
+ values = [values]
949
+ keys: list[str] = []
950
+ seen: set[str] = set()
951
+ for value in values:
952
+ key = _framework_key(value)
953
+ if key in seen:
954
+ continue
955
+ seen.add(key)
956
+ keys.append(key)
957
+ if not keys:
958
+ raise ValueError("frameworks must contain at least one framework")
959
+ return keys
960
+
961
+
962
+ def _local_fixture_target(framework: str) -> str:
963
+ return f"agent-learning-fixture://framework/{_framework_key(framework)}"
964
+
965
+
966
+ def _is_external_target(target: str) -> bool:
967
+ return urlparse(str(target or "")).scheme.lower() in {"http", "https"}
968
+
969
+
970
+ def _adapter_discovery_inventory(agent: Any) -> dict[str, Any]:
971
+ if agent is None:
972
+ return {
973
+ "provided": False,
974
+ "callable": False,
975
+ "type": None,
976
+ "exposed_methods": [],
977
+ "wrapper": False,
978
+ }
979
+
980
+ exposed_methods = _adapter_public_callable_names(agent)
981
+ return {
982
+ "provided": True,
983
+ "callable": callable(agent),
984
+ "type": _adapter_agent_type(agent),
985
+ "exposed_methods": exposed_methods,
986
+ "wrapper": isinstance(agent, AgentWrapper),
987
+ }
988
+
989
+
990
+ def _adapter_agent_type(agent: Any) -> str:
991
+ if inspect.isfunction(agent) or inspect.ismethod(agent):
992
+ module = getattr(agent, "__module__", "")
993
+ qualname = getattr(agent, "__qualname__", getattr(agent, "__name__", "callable"))
994
+ return f"{module}.{qualname}".strip(".")
995
+ cls = type(agent)
996
+ module = getattr(cls, "__module__", "")
997
+ qualname = getattr(cls, "__qualname__", getattr(cls, "__name__", "object"))
998
+ return f"{module}.{qualname}".strip(".")
999
+
1000
+
1001
+ def _adapter_public_callable_names(agent: Any) -> list[str]:
1002
+ names: list[str] = []
1003
+ seen: set[str] = set()
1004
+
1005
+ for name in _DISCOVERY_METHOD_ORDER:
1006
+ if name == "__call__" and not callable(agent):
1007
+ continue
1008
+ if name != "__call__" and not _adapter_has_callable_method(agent, name):
1009
+ continue
1010
+ seen.add(name)
1011
+ names.append(name)
1012
+
1013
+ try:
1014
+ members = inspect.getmembers_static(agent)
1015
+ except Exception:
1016
+ members = []
1017
+
1018
+ public_names: list[str] = []
1019
+ for name, value in members:
1020
+ if name in seen:
1021
+ continue
1022
+ if name.startswith("_"):
1023
+ continue
1024
+ if not (
1025
+ inspect.isroutine(value)
1026
+ or isinstance(value, (classmethod, staticmethod))
1027
+ or _adapter_has_callable_method(agent, name)
1028
+ ):
1029
+ continue
1030
+ if not _adapter_has_callable_method(agent, name):
1031
+ continue
1032
+ public_names.append(name)
1033
+
1034
+ public_names.sort(key=lambda item: (_adapter_method_rank(item), item))
1035
+ names.extend(public_names[:24])
1036
+ return names
1037
+
1038
+
1039
+ def _adapter_resolve_callable_attr_path(agent: Any, method_name: str | None) -> Callable[..., Any] | None:
1040
+ if not method_name:
1041
+ return agent if callable(agent) else None
1042
+ value = agent
1043
+ for raw_part in str(method_name).split("."):
1044
+ part = raw_part.strip()
1045
+ if not part:
1046
+ return None
1047
+ try:
1048
+ value = getattr(value, part)
1049
+ except Exception:
1050
+ return None
1051
+ return value if callable(value) else None
1052
+
1053
+
1054
+ def _adapter_method_leaf(method_name: str | None) -> str:
1055
+ return str(method_name or "").rsplit(".", 1)[-1]
1056
+
1057
+
1058
+ def _adapter_has_callable_method(agent: Any, method_name: str) -> bool:
1059
+ if not method_name:
1060
+ return callable(agent)
1061
+ return _adapter_resolve_callable_attr_path(agent, method_name) is not None
1062
+
1063
+
1064
+ def _adapter_discovery_methods(
1065
+ agent: Any,
1066
+ *,
1067
+ spec: FrameworkAdapterSpec | None,
1068
+ method_candidates: Sequence[str | None] | None,
1069
+ ) -> list[str | None]:
1070
+ methods: list[str | None] = []
1071
+
1072
+ for method in method_candidates or []:
1073
+ methods.append(str(method) if method is not None and str(method) else None)
1074
+ if spec and spec.method:
1075
+ methods.append(spec.method)
1076
+ methods.extend(_adapter_public_callable_names(agent) if agent is not None else [])
1077
+ if agent is not None and callable(agent):
1078
+ methods.append(None)
1079
+ methods.extend(_DISCOVERY_METHOD_ORDER)
1080
+ if spec is None:
1081
+ methods.append(None)
1082
+
1083
+ unique: list[str | None] = []
1084
+ seen: set[str] = set()
1085
+ for method in methods:
1086
+ key = str(method or "")
1087
+ if key in seen:
1088
+ continue
1089
+ seen.add(key)
1090
+ unique.append(method)
1091
+ return unique or [None]
1092
+
1093
+
1094
+ def _adapter_discovery_input_modes(
1095
+ *,
1096
+ spec: FrameworkAdapterSpec | None,
1097
+ input_mode_candidates: Sequence[InputMode] | None,
1098
+ ) -> list[InputMode]:
1099
+ modes: list[InputMode] = []
1100
+ if input_mode_candidates is not None:
1101
+ modes.extend(input_mode_candidates)
1102
+ elif spec is not None and spec.input_mode != "auto":
1103
+ modes.append(spec.input_mode)
1104
+ modes.extend(_DISCOVERY_INPUT_MODE_ORDER)
1105
+
1106
+ unique: list[InputMode] = []
1107
+ seen: set[str] = set()
1108
+ for mode in modes:
1109
+ normalized = str(mode or "auto")
1110
+ if normalized not in _DISCOVERY_INPUT_MODE_ORDER:
1111
+ continue
1112
+ if normalized in seen:
1113
+ continue
1114
+ seen.add(normalized)
1115
+ unique.append(normalized) # type: ignore[arg-type]
1116
+ return unique or ["auto"]
1117
+
1118
+
1119
+ def _adapter_discovery_modes_for_method(
1120
+ method_name: str | None,
1121
+ input_modes: Sequence[InputMode],
1122
+ *,
1123
+ spec: FrameworkAdapterSpec | None,
1124
+ ) -> list[InputMode]:
1125
+ modes: list[InputMode] = []
1126
+ inferred = _adapter_inferred_input_mode(method_name)
1127
+ if inferred:
1128
+ modes.append(inferred)
1129
+ elif method_name is None and spec is not None:
1130
+ modes.append(spec.input_mode)
1131
+ modes.extend(input_modes)
1132
+
1133
+ unique: list[InputMode] = []
1134
+ seen: set[str] = set()
1135
+ for mode in modes:
1136
+ normalized = str(mode or "auto")
1137
+ if normalized not in _DISCOVERY_INPUT_MODE_ORDER:
1138
+ continue
1139
+ if normalized in seen:
1140
+ continue
1141
+ seen.add(normalized)
1142
+ unique.append(normalized) # type: ignore[arg-type]
1143
+ return unique or ["auto"]
1144
+
1145
+
1146
+ def _adapter_inferred_input_mode(method_name: str | None) -> InputMode | None:
1147
+ if method_name is None:
1148
+ return "agent_input"
1149
+ return (
1150
+ _DISCOVERY_METHOD_INPUT_MODES.get(method_name)
1151
+ or _DISCOVERY_METHOD_INPUT_MODES.get(_adapter_method_leaf(method_name))
1152
+ )
1153
+
1154
+
1155
+ def _adapter_candidate_input_key(
1156
+ agent: Any,
1157
+ method_name: str | None,
1158
+ input_mode: InputMode,
1159
+ ) -> str | None:
1160
+ if agent is None or not method_name:
1161
+ return None
1162
+ method = _adapter_resolve_callable_attr_path(agent, method_name)
1163
+ if method is None:
1164
+ return None
1165
+ try:
1166
+ signature = inspect.signature(method)
1167
+ except (TypeError, ValueError):
1168
+ return None
1169
+ params = list(signature.parameters.values())
1170
+ names = {param.name: param for param in params}
1171
+ preferred_names = (
1172
+ (
1173
+ _METHOD_INPUT_KEY_PREFERENCES.get(str(method_name or ""))
1174
+ or _METHOD_INPUT_KEY_PREFERENCES.get(_adapter_method_leaf(method_name), ())
1175
+ )
1176
+ + _KEYWORD_INPUT_NAMES
1177
+ )
1178
+ accepts_positional = any(
1179
+ param.kind
1180
+ in {
1181
+ inspect.Parameter.POSITIONAL_ONLY,
1182
+ inspect.Parameter.POSITIONAL_OR_KEYWORD,
1183
+ inspect.Parameter.VAR_POSITIONAL,
1184
+ }
1185
+ for param in params
1186
+ )
1187
+ for name in preferred_names:
1188
+ param = names.get(name)
1189
+ if param is None or param.kind == inspect.Parameter.POSITIONAL_ONLY:
1190
+ continue
1191
+ if name == "inputs" or not accepts_positional or param.kind == inspect.Parameter.KEYWORD_ONLY:
1192
+ return name
1193
+ if not accepts_positional:
1194
+ for param in params:
1195
+ if param.kind == inspect.Parameter.KEYWORD_ONLY:
1196
+ return param.name
1197
+ if any(param.kind == inspect.Parameter.VAR_KEYWORD for param in params):
1198
+ if input_mode == "dict":
1199
+ if method_name == "kickoff":
1200
+ return "inputs"
1201
+ if method_name in {"process", "process_frame"}:
1202
+ return "frame"
1203
+ return "payload"
1204
+ return "task" if method_name in {"run", "arun", "run_stream"} else "input"
1205
+ return None
1206
+
1207
+
1208
+ def _adapter_callable_signature(
1209
+ agent: Any,
1210
+ *,
1211
+ method: str | Callable[..., Any] | None,
1212
+ method_name: str | None,
1213
+ ) -> dict[str, Any]:
1214
+ callable_method = method if callable(method) else None
1215
+ resolved_method = str(method_name or "").strip()
1216
+ if callable_method is None:
1217
+ if resolved_method == "auto":
1218
+ resolved_method = ""
1219
+ callable_method = _adapter_resolve_callable_attr_path(agent, resolved_method)
1220
+ if callable_method is None and callable(agent):
1221
+ callable_method = agent
1222
+ if callable_method is None:
1223
+ return {}
1224
+
1225
+ try:
1226
+ signature = inspect.signature(callable_method)
1227
+ except (TypeError, ValueError):
1228
+ return {}
1229
+
1230
+ params = list(signature.parameters.values())
1231
+ parameter_rows = [
1232
+ {
1233
+ "name": param.name,
1234
+ "kind": str(param.kind).rsplit(".", 1)[-1].lower(),
1235
+ "required": param.default is inspect.Parameter.empty
1236
+ and param.kind
1237
+ not in {
1238
+ inspect.Parameter.VAR_POSITIONAL,
1239
+ inspect.Parameter.VAR_KEYWORD,
1240
+ },
1241
+ "has_default": param.default is not inspect.Parameter.empty,
1242
+ "annotation": _adapter_annotation_name(param.annotation),
1243
+ }
1244
+ for param in params
1245
+ ]
1246
+ selected_method = (
1247
+ str(method)
1248
+ if isinstance(method, str) and str(method)
1249
+ else resolved_method
1250
+ or getattr(callable_method, "__name__", None)
1251
+ or "callable"
1252
+ )
1253
+ method_leaf = _adapter_method_leaf(selected_method)
1254
+ preferred_input_key = _adapter_candidate_input_key(
1255
+ agent,
1256
+ selected_method if isinstance(method, str) else method_leaf,
1257
+ "auto",
1258
+ )
1259
+ return {
1260
+ "kind": "agent-learning.framework-adapter-callable-signature.v1",
1261
+ "inspectable": True,
1262
+ "method": selected_method,
1263
+ "method_leaf": method_leaf,
1264
+ "callable_type": _adapter_agent_type(callable_method),
1265
+ "parameters": parameter_rows,
1266
+ "parameter_names": [row["name"] for row in parameter_rows],
1267
+ "required_parameters": [
1268
+ row["name"] for row in parameter_rows if row["required"]
1269
+ ],
1270
+ "required_parameter_count": sum(1 for row in parameter_rows if row["required"]),
1271
+ "accepts_positional": _params_accept_positional(params),
1272
+ "accepts_var_positional": any(
1273
+ param.kind == inspect.Parameter.VAR_POSITIONAL for param in params
1274
+ ),
1275
+ "accepts_var_keyword": any(
1276
+ param.kind == inspect.Parameter.VAR_KEYWORD for param in params
1277
+ ),
1278
+ "keyword_only_parameters": [
1279
+ param.name
1280
+ for param in params
1281
+ if param.kind == inspect.Parameter.KEYWORD_ONLY
1282
+ ],
1283
+ "positional_parameters": [
1284
+ param.name
1285
+ for param in params
1286
+ if param.kind
1287
+ in {
1288
+ inspect.Parameter.POSITIONAL_ONLY,
1289
+ inspect.Parameter.POSITIONAL_OR_KEYWORD,
1290
+ }
1291
+ ],
1292
+ "preferred_input_key": preferred_input_key,
1293
+ "is_async": inspect.iscoroutinefunction(callable_method),
1294
+ "is_generator": inspect.isgeneratorfunction(callable_method),
1295
+ "is_async_generator": inspect.isasyncgenfunction(callable_method),
1296
+ "return_annotation": _adapter_annotation_name(signature.return_annotation),
1297
+ }
1298
+
1299
+
1300
+ def _adapter_annotation_name(annotation: Any) -> str | None:
1301
+ if annotation is inspect.Signature.empty or annotation is inspect.Parameter.empty:
1302
+ return None
1303
+ if isinstance(annotation, type):
1304
+ return annotation.__name__
1305
+ return str(annotation)
1306
+
1307
+
1308
+ def _params_accept_positional(params: Sequence[inspect.Parameter]) -> bool:
1309
+ return any(
1310
+ param.kind
1311
+ in {
1312
+ inspect.Parameter.POSITIONAL_ONLY,
1313
+ inspect.Parameter.POSITIONAL_OR_KEYWORD,
1314
+ inspect.Parameter.VAR_POSITIONAL,
1315
+ }
1316
+ for param in params
1317
+ )
1318
+
1319
+
1320
+ def _adapter_discovery_score(
1321
+ method_name: str | None,
1322
+ input_mode: InputMode,
1323
+ *,
1324
+ spec: FrameworkAdapterSpec | None,
1325
+ inventory: Mapping[str, Any],
1326
+ explicit_methods: set[str],
1327
+ ) -> dict[str, Any]:
1328
+ score = 0.15
1329
+ reasons: list[str] = ["local_contract_candidate"]
1330
+ method_present = _adapter_method_present(inventory, method_name)
1331
+ inferred_mode = _adapter_inferred_input_mode(method_name)
1332
+
1333
+ if method_name and method_name in explicit_methods:
1334
+ score += 0.15
1335
+ reasons.append("explicit_method_candidate")
1336
+ if method_present:
1337
+ score += 0.35
1338
+ reasons.append("agent_exposes_method")
1339
+ elif method_name is None and inventory.get("callable"):
1340
+ score += 0.35
1341
+ reasons.append("agent_is_direct_callable")
1342
+ elif inventory.get("provided") and method_name:
1343
+ score -= 0.15
1344
+ reasons.append("method_not_found_on_agent")
1345
+
1346
+ if spec and method_name and method_name == spec.method:
1347
+ score += 0.2
1348
+ reasons.append("matches_framework_preset_method")
1349
+ if spec and input_mode == spec.input_mode:
1350
+ score += 0.15
1351
+ reasons.append("matches_framework_preset_input_mode")
1352
+ if inferred_mode and input_mode == inferred_mode:
1353
+ score += 0.15
1354
+ reasons.append("matches_inferred_input_mode")
1355
+ if method_name == "execute_task" and input_mode == "dict":
1356
+ score += 0.1
1357
+ reasons.append("task_payload_adapter")
1358
+ if method_name == "process_frame" and input_mode == "dict":
1359
+ score += 0.1
1360
+ reasons.append("frame_payload_adapter")
1361
+ if method_name in _STREAMING_METHODS:
1362
+ score += 0.1
1363
+ reasons.append("streaming_adapter_surface")
1364
+ if input_mode == "auto":
1365
+ score -= 0.05
1366
+ reasons.append("auto_input_mode_requires_runtime_inference")
1367
+
1368
+ normalized = max(0.0, min(1.0, score))
1369
+ return {"score": round(normalized, 3), "reasons": reasons}
1370
+
1371
+
1372
+ def _adapter_method_present(
1373
+ inventory: Mapping[str, Any],
1374
+ method_name: str | None,
1375
+ ) -> bool:
1376
+ if method_name is None:
1377
+ return bool(inventory.get("callable"))
1378
+ return method_name in set(inventory.get("exposed_methods", []) or [])
1379
+
1380
+
1381
+ def _adapter_method_rank(method_name: str) -> int:
1382
+ normalized = str(method_name or "")
1383
+ if normalized == "auto":
1384
+ normalized = ""
1385
+ try:
1386
+ return _DISCOVERY_METHOD_ORDER.index(normalized)
1387
+ except ValueError:
1388
+ return len(_DISCOVERY_METHOD_ORDER)
1389
+
1390
+
1391
+ def _adapter_input_mode_rank(input_mode: str) -> int:
1392
+ try:
1393
+ return _DISCOVERY_INPUT_MODE_ORDER.index(input_mode) # type: ignore[arg-type]
1394
+ except ValueError:
1395
+ return len(_DISCOVERY_INPUT_MODE_ORDER)
1396
+
1397
+
1398
+ def _adapter_discovery_findings(
1399
+ inventory: Mapping[str, Any],
1400
+ candidates: Sequence[Mapping[str, Any]],
1401
+ ) -> list[dict[str, Any]]:
1402
+ findings: list[dict[str, Any]] = []
1403
+ if not inventory.get("provided"):
1404
+ findings.append(
1405
+ {
1406
+ "level": "info",
1407
+ "type": "agent_not_provided",
1408
+ "message": "Discovery used framework presets without inspecting an agent.",
1409
+ }
1410
+ )
1411
+ elif not inventory.get("callable") and not inventory.get("exposed_methods"):
1412
+ findings.append(
1413
+ {
1414
+ "level": "warning",
1415
+ "type": "no_callable_adapter_surface",
1416
+ "message": "Agent does not expose a discovered callable adapter method.",
1417
+ }
1418
+ )
1419
+ if not candidates:
1420
+ findings.append(
1421
+ {
1422
+ "level": "error",
1423
+ "type": "adapter_candidates_missing",
1424
+ "message": "No framework adapter candidates were discovered.",
1425
+ }
1426
+ )
1427
+ return findings
1428
+
1429
+
1430
+ def _framework_matrix_summary(contracts: Sequence[Mapping[str, Any]]) -> dict[str, Any]:
1431
+ frameworks: list[str] = []
1432
+ methods: dict[str, str] = {}
1433
+ input_modes: dict[str, str] = {}
1434
+ modalities: dict[str, str] = {}
1435
+ transports: dict[str, str] = {}
1436
+ capabilities: set[str] = set()
1437
+ evidence_requirements: set[str] = set()
1438
+ target_schemes: set[str] = set()
1439
+
1440
+ for contract in contracts:
1441
+ framework = _framework_key(str(contract.get("framework") or "custom"))
1442
+ frameworks.append(framework)
1443
+ methods[framework] = str(contract.get("method") or "")
1444
+ input_modes[framework] = str(contract.get("input_mode") or "")
1445
+ modalities[framework] = str(contract.get("modality") or "")
1446
+ transports[framework] = str(contract.get("transport") or "")
1447
+ capabilities.update(
1448
+ str(item) for item in contract.get("capabilities", []) or []
1449
+ )
1450
+ evidence_requirements.update(
1451
+ str(item) for item in contract.get("evidence_requirements", []) or []
1452
+ )
1453
+ target_scheme = str(contract.get("target_scheme") or "")
1454
+ if target_scheme:
1455
+ target_schemes.add(target_scheme)
1456
+
1457
+ return {
1458
+ "frameworks": frameworks,
1459
+ "modalities": sorted(set(modalities.values())),
1460
+ "transports": sorted(set(transports.values())),
1461
+ "methods": methods,
1462
+ "input_modes": input_modes,
1463
+ "framework_modalities": modalities,
1464
+ "framework_transports": transports,
1465
+ "capabilities": sorted(capabilities),
1466
+ "evidence_requirements": sorted(evidence_requirements),
1467
+ "target_schemes": sorted(target_schemes),
1468
+ "contract_count": len(contracts),
1469
+ "local_executable_fixture_count": sum(
1470
+ 1 for contract in contracts if bool(contract.get("local_executable_fixture"))
1471
+ ),
1472
+ "trace_runtime_count": sum(
1473
+ 1 for contract in contracts if bool(contract.get("trace_runtime"))
1474
+ ),
1475
+ "requires_external_service_count": sum(
1476
+ 1 for contract in contracts if bool(contract.get("requires_external_service"))
1477
+ ),
1478
+ "external_target_count": sum(
1479
+ 1
1480
+ for contract in contracts
1481
+ if _is_external_target(str(contract.get("target") or ""))
1482
+ ),
1483
+ }
1484
+
1485
+
1486
+ def _framework_matrix_findings(
1487
+ contracts: Sequence[Mapping[str, Any]],
1488
+ ) -> list[dict[str, Any]]:
1489
+ findings: list[dict[str, Any]] = []
1490
+ for contract in contracts:
1491
+ framework = _framework_key(str(contract.get("framework") or "custom"))
1492
+ if contract.get("kind") != "agent-learning.framework-adapter-contract.v1":
1493
+ findings.append({"framework": framework, "type": "contract_kind_mismatch"})
1494
+ if bool(contract.get("requires_external_service")):
1495
+ findings.append(
1496
+ {"framework": framework, "type": "external_service_required"}
1497
+ )
1498
+ if not bool(contract.get("local_executable_fixture")):
1499
+ findings.append({"framework": framework, "type": "local_fixture_missing"})
1500
+ if _is_external_target(str(contract.get("target") or "")):
1501
+ findings.append({"framework": framework, "type": "external_target_scheme"})
1502
+ return findings
1503
+
1504
+
1505
+ def _framework_adapter_capability_profile_from_contract(
1506
+ contract: Mapping[str, Any],
1507
+ ) -> dict[str, Any]:
1508
+ payload = copy.deepcopy(dict(contract))
1509
+ framework = _framework_key(str(payload.get("framework") or "custom"))
1510
+ capabilities = _framework_profile_capabilities(payload)
1511
+ task_surfaces = _framework_profile_task_surfaces(payload)
1512
+ bindings = _framework_profile_bindings(payload)
1513
+ findings = _framework_profile_findings(payload, bindings)
1514
+ summary = _framework_profile_summary(
1515
+ payload,
1516
+ capabilities=capabilities,
1517
+ task_surfaces=task_surfaces,
1518
+ bindings=bindings,
1519
+ )
1520
+ status = "passed" if not findings else "failed"
1521
+ return {
1522
+ "kind": "agent-learning.framework-adapter-capability-profile.v1",
1523
+ "status": status,
1524
+ "passed": status == "passed",
1525
+ "framework": framework,
1526
+ "method": str(payload.get("method") or "auto"),
1527
+ "input_mode": str(payload.get("input_mode") or "auto"),
1528
+ "modality": str(payload.get("modality") or "text"),
1529
+ "transport": str(payload.get("transport") or "in_process"),
1530
+ "requires_external_service": bool(payload.get("requires_external_service")),
1531
+ "local_executable_fixture": bool(payload.get("local_executable_fixture")),
1532
+ "trace_runtime": bool(payload.get("trace_runtime")),
1533
+ "contract": payload,
1534
+ "capabilities": capabilities,
1535
+ "task_surfaces": task_surfaces,
1536
+ "bindings": bindings,
1537
+ "summary": summary,
1538
+ "findings": findings,
1539
+ "evidence_requirements": sorted(
1540
+ {
1541
+ "framework_adapter_profile",
1542
+ "framework_adapter_contract",
1543
+ *[
1544
+ str(item)
1545
+ for item in payload.get("evidence_requirements", []) or []
1546
+ if str(item)
1547
+ ],
1548
+ }
1549
+ ),
1550
+ }
1551
+
1552
+
1553
+ def _framework_profile_capabilities(
1554
+ contract: Mapping[str, Any],
1555
+ ) -> list[dict[str, Any]]:
1556
+ names = [
1557
+ *[str(item) for item in contract.get("capabilities", []) or []],
1558
+ "adapter_contract",
1559
+ "local_fixture",
1560
+ "metric_evidence",
1561
+ "optimization_search",
1562
+ ]
1563
+ if bool(contract.get("trace_runtime")):
1564
+ names.append("trace_runtime")
1565
+ if str(contract.get("modality") or "") == "voice":
1566
+ names.extend(["voice", "realtime"])
1567
+ if str(contract.get("modality") or "") == "cua":
1568
+ names.extend(["browser", "computer_use"])
1569
+
1570
+ capabilities: list[dict[str, Any]] = []
1571
+ seen: set[str] = set()
1572
+ for raw_name in names:
1573
+ name = _framework_key(raw_name)
1574
+ if not name or name in seen:
1575
+ continue
1576
+ seen.add(name)
1577
+ capabilities.append(
1578
+ {
1579
+ "name": name,
1580
+ "category": _framework_profile_capability_category(name),
1581
+ "status": "supported",
1582
+ "source": "framework_adapter_contract",
1583
+ }
1584
+ )
1585
+ return capabilities
1586
+
1587
+
1588
+ def _framework_profile_capability_category(name: str) -> str:
1589
+ normalized = _framework_key(name)
1590
+ if normalized in {"tool_calls", "tools", "call_tool"}:
1591
+ return "tools"
1592
+ if normalized in {"runtime_trace", "trace_runtime", "metric_evidence"}:
1593
+ return "observability"
1594
+ if normalized in {"structured_input", "messages", "artifacts", "state"}:
1595
+ return "io"
1596
+ if normalized in {"voice", "realtime", "streaming"}:
1597
+ return "realtime"
1598
+ if normalized in {"browser", "computer_use"}:
1599
+ return "computer_use"
1600
+ if normalized in {"environment_replay", "reset_step_trace"}:
1601
+ return "world"
1602
+ if normalized in {"optimization_search"}:
1603
+ return "optimization"
1604
+ if normalized in {"adapter_contract", "local_fixture"}:
1605
+ return "adapter"
1606
+ return "framework"
1607
+
1608
+
1609
+ def _framework_profile_task_surfaces(
1610
+ contract: Mapping[str, Any],
1611
+ ) -> list[dict[str, Any]]:
1612
+ framework = _framework_key(str(contract.get("framework") or "custom"))
1613
+ method = str(contract.get("method") or "auto")
1614
+ input_mode = str(contract.get("input_mode") or "auto")
1615
+ return [
1616
+ {
1617
+ "name": "framework_adapter_simulation",
1618
+ "library": "simulate-sdk",
1619
+ "framework": framework,
1620
+ "method": method,
1621
+ "input_mode": input_mode,
1622
+ "evidence": [
1623
+ "framework_adapter_contract",
1624
+ "framework_runtime",
1625
+ "framework_trace",
1626
+ ],
1627
+ },
1628
+ {
1629
+ "name": "framework_adapter_evaluation",
1630
+ "library": "ai-evaluation",
1631
+ "metric": "framework_adapter_contract_quality",
1632
+ "evidence": [
1633
+ "adapter_conformance",
1634
+ "metric_evidence",
1635
+ "tool_calls",
1636
+ ],
1637
+ },
1638
+ {
1639
+ "name": "framework_adapter_optimization",
1640
+ "library": "agent-opt",
1641
+ "layers": ["framework", "integration", "harness", "evaluator"],
1642
+ "search_paths": [
1643
+ "agent.method",
1644
+ "agent.input_mode",
1645
+ "simulation.environments",
1646
+ ],
1647
+ },
1648
+ ]
1649
+
1650
+
1651
+ def _framework_profile_bindings(
1652
+ contract: Mapping[str, Any],
1653
+ ) -> dict[str, Any]:
1654
+ method = str(contract.get("method") or "auto")
1655
+ input_mode = str(contract.get("input_mode") or "auto")
1656
+ framework = _framework_key(str(contract.get("framework") or "custom"))
1657
+ return {
1658
+ "simulate-sdk": {
1659
+ "adapter": "wrap_framework",
1660
+ "contract": "framework_adapter_contract",
1661
+ "matrix": "framework_adapter_contract_matrix",
1662
+ "probe": "probe_framework_adapter",
1663
+ "framework": framework,
1664
+ "method": method,
1665
+ "input_mode": input_mode,
1666
+ "local_executable_fixture": bool(
1667
+ contract.get("local_executable_fixture")
1668
+ ),
1669
+ },
1670
+ "ai-evaluation": {
1671
+ "metric": "framework_adapter_contract_quality",
1672
+ "contract_kind": "agent-learning.framework-adapter-contract.v1",
1673
+ "required_frameworks": [framework],
1674
+ "required_methods": [method] if method != "auto" else [],
1675
+ "required_input_modes": [input_mode] if input_mode != "auto" else [],
1676
+ "required_capabilities": [
1677
+ str(item)
1678
+ for item in contract.get("capabilities", []) or []
1679
+ if str(item)
1680
+ ],
1681
+ },
1682
+ "agent-opt": {
1683
+ "target": "OptimizationTarget",
1684
+ "candidate": "AgentCandidate",
1685
+ "optimizer": "AgentOptimizer",
1686
+ "layers": ["framework", "integration", "harness", "evaluator"],
1687
+ "search_paths": [
1688
+ "agent.method",
1689
+ "agent.input_mode",
1690
+ "simulation.environments",
1691
+ ],
1692
+ },
1693
+ }
1694
+
1695
+
1696
+ def _framework_profile_findings(
1697
+ contract: Mapping[str, Any],
1698
+ bindings: Mapping[str, Any],
1699
+ ) -> list[dict[str, Any]]:
1700
+ findings: list[dict[str, Any]] = []
1701
+ framework = _framework_key(str(contract.get("framework") or "custom"))
1702
+ if contract.get("kind") != "agent-learning.framework-adapter-contract.v1":
1703
+ findings.append(
1704
+ {
1705
+ "level": "error",
1706
+ "framework": framework,
1707
+ "type": "contract_kind_mismatch",
1708
+ }
1709
+ )
1710
+ if bool(contract.get("requires_external_service")):
1711
+ findings.append(
1712
+ {
1713
+ "level": "error",
1714
+ "framework": framework,
1715
+ "type": "external_service_required",
1716
+ }
1717
+ )
1718
+ if not bool(contract.get("local_executable_fixture")):
1719
+ findings.append(
1720
+ {
1721
+ "level": "error",
1722
+ "framework": framework,
1723
+ "type": "local_fixture_missing",
1724
+ }
1725
+ )
1726
+ if _is_external_target(str(contract.get("target") or "")):
1727
+ findings.append(
1728
+ {
1729
+ "level": "error",
1730
+ "framework": framework,
1731
+ "type": "external_target_scheme",
1732
+ }
1733
+ )
1734
+ for library in ("simulate-sdk", "ai-evaluation", "agent-opt"):
1735
+ if not isinstance(bindings.get(library), Mapping):
1736
+ findings.append(
1737
+ {
1738
+ "level": "error",
1739
+ "framework": framework,
1740
+ "type": "trinity_binding_missing",
1741
+ "library": library,
1742
+ }
1743
+ )
1744
+ return findings
1745
+
1746
+
1747
+ def _framework_profile_summary(
1748
+ contract: Mapping[str, Any],
1749
+ *,
1750
+ capabilities: Sequence[Mapping[str, Any]],
1751
+ task_surfaces: Sequence[Mapping[str, Any]],
1752
+ bindings: Mapping[str, Any],
1753
+ ) -> dict[str, Any]:
1754
+ return {
1755
+ "framework": _framework_key(str(contract.get("framework") or "custom")),
1756
+ "method": str(contract.get("method") or "auto"),
1757
+ "input_mode": str(contract.get("input_mode") or "auto"),
1758
+ "modality": str(contract.get("modality") or "text"),
1759
+ "transport": str(contract.get("transport") or "in_process"),
1760
+ "capability_count": len(capabilities),
1761
+ "task_surface_count": len(task_surfaces),
1762
+ "binding_count": len(bindings),
1763
+ "libraries": sorted(str(key) for key in bindings),
1764
+ "local_executable_fixture": bool(contract.get("local_executable_fixture")),
1765
+ "trace_runtime": bool(contract.get("trace_runtime")),
1766
+ "requires_external_service": bool(contract.get("requires_external_service")),
1767
+ }
1768
+
1769
+
1770
+ def _framework_profile_collection_summary(
1771
+ profiles: Sequence[Mapping[str, Any]],
1772
+ ) -> dict[str, Any]:
1773
+ frameworks = [
1774
+ _framework_key(str(profile.get("framework") or "custom"))
1775
+ for profile in profiles
1776
+ ]
1777
+ libraries = sorted(
1778
+ {
1779
+ str(library)
1780
+ for profile in profiles
1781
+ for library in (profile.get("bindings") or {})
1782
+ }
1783
+ )
1784
+ capabilities = sorted(
1785
+ {
1786
+ _framework_key(str(capability.get("name") or ""))
1787
+ for profile in profiles
1788
+ for capability in profile.get("capabilities", []) or []
1789
+ if isinstance(capability, Mapping)
1790
+ }
1791
+ )
1792
+ return {
1793
+ "frameworks": frameworks,
1794
+ "profile_count": len(profiles),
1795
+ "passed_profile_count": sum(
1796
+ 1 for profile in profiles if str(profile.get("status")) == "passed"
1797
+ ),
1798
+ "failed_profile_count": sum(
1799
+ 1 for profile in profiles if str(profile.get("status")) != "passed"
1800
+ ),
1801
+ "libraries": libraries,
1802
+ "capabilities": capabilities,
1803
+ "local_executable_fixture_count": sum(
1804
+ 1 for profile in profiles if bool(profile.get("local_executable_fixture"))
1805
+ ),
1806
+ "requires_external_service_count": sum(
1807
+ 1 for profile in profiles if bool(profile.get("requires_external_service"))
1808
+ ),
1809
+ }
1810
+
1811
+
1812
+ def _framework_profile_collection_findings(
1813
+ profiles: Sequence[Mapping[str, Any]],
1814
+ *,
1815
+ required_frameworks: Sequence[str],
1816
+ ) -> list[dict[str, Any]]:
1817
+ findings: list[dict[str, Any]] = []
1818
+ observed = {
1819
+ _framework_key(str(profile.get("framework") or "custom"))
1820
+ for profile in profiles
1821
+ }
1822
+ for framework in required_frameworks:
1823
+ key = _framework_key(framework)
1824
+ if key not in observed:
1825
+ findings.append(
1826
+ {
1827
+ "level": "error",
1828
+ "framework": key,
1829
+ "type": "framework_profile_missing",
1830
+ }
1831
+ )
1832
+ for profile in profiles:
1833
+ for finding in profile.get("findings", []) or []:
1834
+ if isinstance(finding, Mapping):
1835
+ findings.append(copy.deepcopy(dict(finding)))
1836
+ return findings
1837
+
1838
+
1839
+ def _default_transport(modality: str) -> str:
1840
+ if modality == "voice":
1841
+ return "realtime_adapter"
1842
+ if modality == "cua":
1843
+ return "browser_adapter"
1844
+ if modality == "image":
1845
+ return "multimodal_adapter"
1846
+ return "in_process"
1847
+
1848
+
1849
+ def _default_lifecycle_hooks(modality: str) -> tuple[str, ...]:
1850
+ if modality == "voice":
1851
+ return ("setup", "connect", "stream", "respond", "teardown")
1852
+ if modality == "cua":
1853
+ return ("setup", "observe", "act", "verify", "teardown")
1854
+ return ("setup", "invoke", "observe", "teardown")
1855
+
1856
+
1857
+ def _default_capabilities(modality: str, input_mode: str) -> tuple[str, ...]:
1858
+ capabilities = [
1859
+ "messages",
1860
+ "tool_calls",
1861
+ "runtime_trace",
1862
+ "state",
1863
+ "artifacts",
1864
+ ]
1865
+ if input_mode == "dict":
1866
+ capabilities.append("structured_input")
1867
+ if input_mode in {"dict", "messages"}:
1868
+ capabilities.append("streaming_trace")
1869
+ if modality == "voice":
1870
+ capabilities.extend(["voice_frames", "realtime_events"])
1871
+ elif modality == "cua":
1872
+ capabilities.extend(["browser_actions", "visual_grounding"])
1873
+ elif modality == "image":
1874
+ capabilities.extend(["image_context", "multimodal_grounding"])
1875
+ return tuple(capabilities)
1876
+
1877
+
1878
+ def _input_schema(input_mode: str) -> dict[str, Any]:
1879
+ if input_mode == "dict":
1880
+ return {
1881
+ "type": "object",
1882
+ "required": ["messages", "scenario"],
1883
+ "additionalProperties": True,
1884
+ }
1885
+ if input_mode == "messages":
1886
+ return {
1887
+ "type": "array",
1888
+ "items": {"type": "object", "required": ["role", "content"]},
1889
+ }
1890
+ if input_mode == "agent_input":
1891
+ return {"type": "object", "class": "AgentInput"}
1892
+ if input_mode == "text":
1893
+ return {"type": "string"}
1894
+ return {"type": "any"}
1895
+
1896
+
1897
+ def _output_schema() -> dict[str, Any]:
1898
+ return {
1899
+ "oneOf": [
1900
+ {"type": "string"},
1901
+ {"type": "object", "class": "AgentResponse"},
1902
+ ],
1903
+ "required_trace_state": ["framework_runtime"],
1904
+ }
1905
+
1906
+
1907
+ def _probe_method_name(method: str | Callable[..., Any] | None) -> str | None:
1908
+ if method is None:
1909
+ return None
1910
+ if isinstance(method, str):
1911
+ return method
1912
+ return getattr(method, "__name__", None) or "callable"
1913
+
1914
+
1915
+ def _probe_cases(cases: Sequence[Mapping[str, Any]] | None) -> list[dict[str, Any]]:
1916
+ if not cases:
1917
+ return [
1918
+ {
1919
+ "id": "default",
1920
+ "input": "Return a short adapter probe result.",
1921
+ }
1922
+ ]
1923
+ return [dict(case) for case in cases]
1924
+
1925
+
1926
+ async def _run_probe_case(
1927
+ wrapper: AgentWrapper,
1928
+ framework: str,
1929
+ case: Mapping[str, Any],
1930
+ *,
1931
+ index: int,
1932
+ trace_runtime: bool,
1933
+ contract: Mapping[str, Any],
1934
+ ) -> dict[str, Any]:
1935
+ case_id = str(case.get("id") or f"case_{index}")
1936
+ agent_input = _probe_agent_input(framework, case, index=index)
1937
+ raw_response = await wrapper.call(agent_input)
1938
+ response = (
1939
+ raw_response
1940
+ if isinstance(raw_response, AgentResponse)
1941
+ else AgentResponse(content=str(raw_response))
1942
+ )
1943
+ response_payload = _probe_response_payload(response)
1944
+ runtime_trace = dict((response.state or {}).get("framework_runtime") or {})
1945
+ observed_io_contract = _probe_observed_io_contract(
1946
+ framework,
1947
+ case_id=case_id,
1948
+ runtime_trace=runtime_trace,
1949
+ response_payload=response_payload,
1950
+ contract=contract,
1951
+ )
1952
+ checks = _probe_case_checks(
1953
+ case,
1954
+ response_payload,
1955
+ runtime_trace=runtime_trace,
1956
+ observed_io_contract=observed_io_contract,
1957
+ trace_runtime=trace_runtime,
1958
+ contract=contract,
1959
+ )
1960
+ findings = [
1961
+ {
1962
+ "case_id": case_id,
1963
+ "check": check["id"],
1964
+ "level": "error",
1965
+ "message": check["message"],
1966
+ "expected": check.get("expected"),
1967
+ "observed": check.get("observed"),
1968
+ }
1969
+ for check in checks
1970
+ if not check["passed"]
1971
+ ]
1972
+ return {
1973
+ "id": case_id,
1974
+ "status": "passed" if not findings else "failed",
1975
+ "passed": not findings,
1976
+ "input": {
1977
+ "message_count": len(agent_input.messages),
1978
+ "tool_count": len(agent_input.tools),
1979
+ "artifact_count": len(agent_input.artifacts),
1980
+ "event_count": len(agent_input.events),
1981
+ "modality": agent_input.modality,
1982
+ },
1983
+ "response": response_payload,
1984
+ "runtime_trace": runtime_trace,
1985
+ "observed_io_contract": observed_io_contract,
1986
+ "checks": checks,
1987
+ "findings": findings,
1988
+ }
1989
+
1990
+
1991
+ def _probe_agent_input(
1992
+ framework: str,
1993
+ case: Mapping[str, Any],
1994
+ *,
1995
+ index: int,
1996
+ ) -> AgentInput:
1997
+ messages = _probe_messages(case)
1998
+ new_message = dict(case.get("new_message") or messages[-1])
1999
+ metadata = {
2000
+ "framework": framework,
2001
+ "probe_case_id": str(case.get("id") or f"case_{index}"),
2002
+ **dict(case.get("metadata") or {}),
2003
+ }
2004
+ return AgentInput(
2005
+ thread_id=str(case.get("thread_id") or f"{framework}-probe-{index}"),
2006
+ messages=messages,
2007
+ new_message=new_message,
2008
+ execution_id=str(case.get("execution_id") or f"{framework}-probe"),
2009
+ turn_index=int(case.get("turn_index") or index - 1),
2010
+ scenario_name=str(case.get("scenario_name") or "framework-adapter-probe"),
2011
+ persona=dict(case.get("persona") or {}),
2012
+ situation=str(case.get("situation") or ""),
2013
+ expected_outcome=str(case.get("expected_outcome") or ""),
2014
+ modality=str(case.get("modality") or ""),
2015
+ artifacts=list(case.get("artifacts") or []),
2016
+ events=list(case.get("events") or []),
2017
+ memory=dict(case.get("memory") or {}),
2018
+ tools=[dict(tool) for tool in case.get("tools", []) if isinstance(tool, Mapping)],
2019
+ metadata=metadata,
2020
+ )
2021
+
2022
+
2023
+ def _probe_messages(case: Mapping[str, Any]) -> list[dict[str, Any]]:
2024
+ raw_messages = case.get("messages")
2025
+ if isinstance(raw_messages, Sequence) and not isinstance(raw_messages, (str, bytes)):
2026
+ messages = [
2027
+ dict(message)
2028
+ for message in raw_messages
2029
+ if isinstance(message, Mapping)
2030
+ ]
2031
+ if messages:
2032
+ return messages
2033
+ message = case.get("input", case.get("message", "Run the adapter probe."))
2034
+ return [{"role": "user", "content": str(message)}]
2035
+
2036
+
2037
+ def _probe_case_checks(
2038
+ case: Mapping[str, Any],
2039
+ response: Mapping[str, Any],
2040
+ *,
2041
+ runtime_trace: Mapping[str, Any],
2042
+ observed_io_contract: Mapping[str, Any],
2043
+ trace_runtime: bool,
2044
+ contract: Mapping[str, Any],
2045
+ ) -> list[dict[str, Any]]:
2046
+ checks: list[dict[str, Any]] = []
2047
+ content = str(response.get("content") or "")
2048
+ tool_names = set(response.get("tool_names") or [])
2049
+ event_types = set(response.get("event_types") or [])
2050
+ state_keys = set(response.get("state_keys") or [])
2051
+
2052
+ for term in _probe_strings(case.get("expected_contains")):
2053
+ checks.append(
2054
+ _probe_check(
2055
+ f"content_contains_{_framework_key(term)}",
2056
+ term.lower() in content.lower(),
2057
+ f"response content should contain {term!r}",
2058
+ expected=term,
2059
+ observed=content,
2060
+ )
2061
+ )
2062
+ for tool in _probe_strings(case.get("required_tools")):
2063
+ checks.append(
2064
+ _probe_check(
2065
+ f"required_tool_{_framework_key(tool)}",
2066
+ tool in tool_names,
2067
+ f"response should emit required tool {tool!r}",
2068
+ expected=tool,
2069
+ observed=sorted(tool_names),
2070
+ )
2071
+ )
2072
+ for event_type in _probe_strings(case.get("required_events")):
2073
+ checks.append(
2074
+ _probe_check(
2075
+ f"required_event_{_framework_key(event_type)}",
2076
+ event_type in event_types,
2077
+ f"response should emit required event {event_type!r}",
2078
+ expected=event_type,
2079
+ observed=sorted(event_types),
2080
+ )
2081
+ )
2082
+ for state_key in _probe_strings(case.get("required_state_keys")):
2083
+ checks.append(
2084
+ _probe_check(
2085
+ f"required_state_{_framework_key(state_key)}",
2086
+ state_key in state_keys,
2087
+ f"response should include required state key {state_key!r}",
2088
+ expected=state_key,
2089
+ observed=sorted(state_keys),
2090
+ )
2091
+ )
2092
+ if trace_runtime:
2093
+ runtime_summary = dict(runtime_trace.get("summary") or {})
2094
+ runtime_contract = dict(
2095
+ dict(runtime_trace.get("metadata") or {}).get("framework_adapter_contract")
2096
+ or {}
2097
+ )
2098
+ signature = dict(contract.get("callable_signature") or {})
2099
+ io_summary = dict(observed_io_contract.get("summary") or {})
2100
+ checks.extend(
2101
+ [
2102
+ _probe_check(
2103
+ "framework_runtime_trace_present",
2104
+ bool(runtime_trace),
2105
+ "trace_runtime=True should attach framework_runtime state",
2106
+ expected=True,
2107
+ observed=bool(runtime_trace),
2108
+ ),
2109
+ _probe_check(
2110
+ "framework_runtime_contract_present",
2111
+ runtime_contract.get("kind")
2112
+ == "agent-learning.framework-adapter-contract.v1",
2113
+ "runtime trace should carry the adapter contract",
2114
+ expected="agent-learning.framework-adapter-contract.v1",
2115
+ observed=runtime_contract.get("kind"),
2116
+ ),
2117
+ _probe_check(
2118
+ "framework_runtime_invocation_present",
2119
+ int(runtime_summary.get("invocation_count") or 0) >= 1,
2120
+ "runtime trace should record at least one invocation",
2121
+ expected=">=1",
2122
+ observed=runtime_summary.get("invocation_count"),
2123
+ ),
2124
+ _probe_check(
2125
+ "framework_adapter_callable_signature_present",
2126
+ signature.get("kind")
2127
+ == "agent-learning.framework-adapter-callable-signature.v1",
2128
+ "probe contract should carry deterministic callable signature evidence",
2129
+ expected="agent-learning.framework-adapter-callable-signature.v1",
2130
+ observed=signature.get("kind"),
2131
+ ),
2132
+ _probe_check(
2133
+ "framework_adapter_observed_io_contract_present",
2134
+ observed_io_contract.get("kind")
2135
+ == "agent-learning.framework-adapter-observed-io-contract.v1"
2136
+ and int(io_summary.get("invocation_count") or 0) >= 1,
2137
+ "probe case should carry observed input/output contract evidence",
2138
+ expected="agent-learning.framework-adapter-observed-io-contract.v1",
2139
+ observed={
2140
+ "kind": observed_io_contract.get("kind"),
2141
+ "invocation_count": io_summary.get("invocation_count"),
2142
+ },
2143
+ ),
2144
+ _probe_check(
2145
+ "framework_adapter_observed_io_matches_signature",
2146
+ io_summary.get("signature_bound") is True,
2147
+ "observed adapter invocation should bind to the callable signature",
2148
+ expected=True,
2149
+ observed=io_summary.get("signature_bound"),
2150
+ ),
2151
+ ]
2152
+ )
2153
+ checks.append(
2154
+ _probe_check(
2155
+ "adapter_contract_local_first",
2156
+ contract.get("requires_external_service") is False,
2157
+ "adapter contract should not require a hosted service",
2158
+ expected=False,
2159
+ observed=contract.get("requires_external_service"),
2160
+ )
2161
+ )
2162
+ return checks
2163
+
2164
+
2165
+ def _probe_check(
2166
+ check_id: str,
2167
+ passed: bool,
2168
+ message: str,
2169
+ *,
2170
+ expected: Any = None,
2171
+ observed: Any = None,
2172
+ ) -> dict[str, Any]:
2173
+ return {
2174
+ "id": check_id,
2175
+ "passed": bool(passed),
2176
+ "message": message,
2177
+ "expected": expected,
2178
+ "observed": observed,
2179
+ }
2180
+
2181
+
2182
+ def _probe_strings(value: Any) -> list[str]:
2183
+ if value is None:
2184
+ return []
2185
+ if isinstance(value, str):
2186
+ return [value]
2187
+ if isinstance(value, Sequence):
2188
+ return [str(item) for item in value if str(item)]
2189
+ return [str(value)]
2190
+
2191
+
2192
+ def _unique_probe_strings(values: Sequence[Any]) -> list[str]:
2193
+ seen: set[str] = set()
2194
+ unique: list[str] = []
2195
+ for value in values:
2196
+ text = str(value or "")
2197
+ if not text or text in seen:
2198
+ continue
2199
+ seen.add(text)
2200
+ unique.append(text)
2201
+ return unique
2202
+
2203
+
2204
+ def _probe_observed_io_contract(
2205
+ framework: str,
2206
+ *,
2207
+ case_id: str,
2208
+ runtime_trace: Mapping[str, Any],
2209
+ response_payload: Mapping[str, Any],
2210
+ contract: Mapping[str, Any],
2211
+ ) -> dict[str, Any]:
2212
+ invocations = [
2213
+ dict(invocation)
2214
+ for invocation in list(runtime_trace.get("invocations") or [])
2215
+ if isinstance(invocation, Mapping)
2216
+ ]
2217
+ signature = (
2218
+ dict(contract.get("callable_signature") or {})
2219
+ if isinstance(contract.get("callable_signature"), Mapping)
2220
+ else {}
2221
+ )
2222
+ observed_invocations: list[dict[str, Any]] = []
2223
+ for index, invocation in enumerate(invocations, start=1):
2224
+ input_shape = dict(invocation.get("input") or {})
2225
+ output_shape = dict(invocation.get("output") or {})
2226
+ observed_invocations.append(
2227
+ {
2228
+ "id": str(invocation.get("id") or f"{case_id}_invocation_{index}"),
2229
+ "framework": str(invocation.get("framework") or framework),
2230
+ "method": str(invocation.get("method") or ""),
2231
+ "input_mode": str(invocation.get("input_mode") or ""),
2232
+ "call_style": str(invocation.get("call_style") or ""),
2233
+ "input_key": invocation.get("input_key"),
2234
+ "input_kwargs_keys": _unique_probe_strings(
2235
+ list(invocation.get("input_kwargs_keys") or [])
2236
+ ),
2237
+ "input_shape": input_shape,
2238
+ "output_shape": output_shape,
2239
+ "duration_ms": int(invocation.get("duration_ms") or 0),
2240
+ "signals": _unique_probe_strings(list(invocation.get("signals") or [])),
2241
+ }
2242
+ )
2243
+ signature_bound = bool(signature) and bool(observed_invocations) and all(
2244
+ _probe_invocation_matches_signature(invocation, signature)
2245
+ for invocation in observed_invocations
2246
+ )
2247
+ input_keys = _unique_probe_strings(
2248
+ invocation.get("input_key")
2249
+ for invocation in observed_invocations
2250
+ if invocation.get("input_key") not in (None, "", [], {})
2251
+ )
2252
+ output_shapes = [dict(item.get("output_shape") or {}) for item in observed_invocations]
2253
+ return {
2254
+ "kind": "agent-learning.framework-adapter-observed-io-contract.v1",
2255
+ "framework": framework,
2256
+ "case_id": case_id,
2257
+ "method": contract.get("method"),
2258
+ "input_mode": contract.get("input_mode"),
2259
+ "signature_method": signature.get("method"),
2260
+ "signature_bound": signature_bound,
2261
+ "invocations": observed_invocations,
2262
+ "summary": {
2263
+ "invocation_count": len(observed_invocations),
2264
+ "methods": _unique_probe_strings(
2265
+ invocation.get("method") for invocation in observed_invocations
2266
+ ),
2267
+ "input_modes": _unique_probe_strings(
2268
+ invocation.get("input_mode") for invocation in observed_invocations
2269
+ ),
2270
+ "call_styles": _unique_probe_strings(
2271
+ invocation.get("call_style") for invocation in observed_invocations
2272
+ ),
2273
+ "input_keys": input_keys,
2274
+ "input_kwargs_keys": _unique_probe_strings(
2275
+ key
2276
+ for invocation in observed_invocations
2277
+ for key in list(invocation.get("input_kwargs_keys") or [])
2278
+ ),
2279
+ "input_types": _unique_probe_strings(
2280
+ dict(invocation.get("input_shape") or {}).get("type")
2281
+ for invocation in observed_invocations
2282
+ ),
2283
+ "output_types": _unique_probe_strings(
2284
+ output.get("type") for output in output_shapes
2285
+ ),
2286
+ "output_state_keys": _unique_probe_strings(
2287
+ key for output in output_shapes for key in list(output.get("state_keys") or [])
2288
+ ),
2289
+ "output_metadata_keys": _unique_probe_strings(
2290
+ key
2291
+ for output in output_shapes
2292
+ for key in list(output.get("metadata_keys") or [])
2293
+ ),
2294
+ "output_tool_names": _unique_probe_strings(
2295
+ key for output in output_shapes for key in list(output.get("tool_names") or [])
2296
+ ),
2297
+ "output_event_types": _unique_probe_strings(
2298
+ key for output in output_shapes for key in list(output.get("event_types") or [])
2299
+ ),
2300
+ "output_artifact_types": _unique_probe_strings(
2301
+ key
2302
+ for output in output_shapes
2303
+ for key in list(output.get("artifact_types") or [])
2304
+ ),
2305
+ "content_observed": bool(response_payload.get("content")),
2306
+ "signature_bound": signature_bound,
2307
+ },
2308
+ }
2309
+
2310
+
2311
+ def _probe_invocation_matches_signature(
2312
+ invocation: Mapping[str, Any],
2313
+ signature: Mapping[str, Any],
2314
+ ) -> bool:
2315
+ if not signature:
2316
+ return False
2317
+ method = str(invocation.get("method") or "")
2318
+ signature_method = str(signature.get("method") or "")
2319
+ signature_leaf = str(signature.get("method_leaf") or "")
2320
+ method_matches = method in {signature_method, signature_leaf, "callable"} or (
2321
+ signature_method in {"", "auto"} and bool(method)
2322
+ )
2323
+ if not method_matches:
2324
+ return False
2325
+
2326
+ call_style = str(invocation.get("call_style") or "")
2327
+ input_key = invocation.get("input_key")
2328
+ parameter_names = set(str(name) for name in list(signature.get("parameter_names") or []))
2329
+ input_kwargs_keys = set(
2330
+ str(key) for key in list(invocation.get("input_kwargs_keys") or []) if str(key)
2331
+ )
2332
+ required_parameters = set(
2333
+ str(name) for name in list(signature.get("required_parameters") or []) if str(name)
2334
+ )
2335
+ accepts_var_keyword = bool(signature.get("accepts_var_keyword"))
2336
+
2337
+ if call_style in {"keyword", "positional_with_kwargs"} and input_key:
2338
+ return (
2339
+ str(input_key) in parameter_names
2340
+ or accepts_var_keyword
2341
+ or str(input_key) == str(signature.get("preferred_input_key") or "")
2342
+ )
2343
+ if call_style == "expanded_kwargs":
2344
+ return accepts_var_keyword
2345
+ if call_style in {"positional", "positional_with_kwargs"}:
2346
+ return bool(signature.get("accepts_positional"))
2347
+ if call_style == "none":
2348
+ return required_parameters <= input_kwargs_keys
2349
+ return False
2350
+
2351
+
2352
+ def _probe_response_payload(response: AgentResponse) -> dict[str, Any]:
2353
+ tool_calls = [dict(call) for call in response.tool_calls or []]
2354
+ events = [event.model_dump() for event in response.events]
2355
+ artifacts = [artifact.model_dump() for artifact in response.artifacts]
2356
+ state = dict(response.state or {})
2357
+ metadata = dict(response.metadata or {})
2358
+ streaming_trace = (
2359
+ state.get("streaming_trace")
2360
+ if isinstance(state.get("streaming_trace"), Mapping)
2361
+ else {}
2362
+ )
2363
+ return {
2364
+ "content": response.content,
2365
+ "tool_call_count": len(tool_calls),
2366
+ "tool_names": sorted(
2367
+ {
2368
+ str(
2369
+ call.get("name")
2370
+ or call.get("tool")
2371
+ or dict(call.get("function") or {}).get("name")
2372
+ or ""
2373
+ )
2374
+ for call in tool_calls
2375
+ if isinstance(call, Mapping)
2376
+ }
2377
+ ),
2378
+ "event_count": len(events),
2379
+ "event_types": sorted({str(event.get("type") or "") for event in events}),
2380
+ "artifact_count": len(artifacts),
2381
+ "artifact_types": sorted({str(artifact.get("type") or "") for artifact in artifacts}),
2382
+ "artifact_evidence": [
2383
+ _probe_artifact_evidence(artifact) for artifact in artifacts
2384
+ ],
2385
+ "state_keys": sorted(str(key) for key in state),
2386
+ "framework_lifecycle_summary": _probe_framework_lifecycle_summary(
2387
+ state.get("framework_lifecycle_trace")
2388
+ ),
2389
+ "framework_trace_summary": _probe_framework_trace_summary(
2390
+ state.get("framework_trace")
2391
+ ),
2392
+ "message_history_summary": _probe_message_history_summary(
2393
+ state.get("message_history")
2394
+ ),
2395
+ "framework_handoff_summary": _probe_framework_handoff_summary(
2396
+ state.get("framework_handoffs")
2397
+ ),
2398
+ "orchestration_trace_summary": _probe_orchestration_trace_summary(
2399
+ state.get("orchestration_trace")
2400
+ ),
2401
+ "workflow_trace_summary": _probe_workflow_trace_summary(
2402
+ state.get("workflow_trace")
2403
+ ),
2404
+ "realtime_trace_summary": _probe_realtime_trace_summary(
2405
+ state.get("realtime_trace")
2406
+ ),
2407
+ "framework_memory_summary": _probe_framework_memory_summary(
2408
+ state.get("framework_memory")
2409
+ ),
2410
+ "retrieval_memory_summary": _probe_retrieval_memory_summary(
2411
+ state.get("retrieval_memory")
2412
+ ),
2413
+ "agent_memory_lineage_summary": _probe_agent_memory_lineage_summary(
2414
+ state.get("agent_memory_lineage")
2415
+ ),
2416
+ "browser_cua_summary": _probe_browser_cua_summary(
2417
+ state.get("browser_cua")
2418
+ ),
2419
+ "mcp_tool_session_summary": _probe_mcp_tool_session_summary(
2420
+ state.get("mcp_tool_session")
2421
+ ),
2422
+ "a2a_protocol_summary": _probe_a2a_protocol_summary(
2423
+ state.get("a2a_protocol_trace")
2424
+ ),
2425
+ "openenv_summary": _probe_openenv_summary(state.get("openenv")),
2426
+ "agent_trust_boundary_summary": _probe_agent_trust_boundary_summary(
2427
+ state.get("agent_trust_boundary_model")
2428
+ ),
2429
+ "agent_control_plane_summary": _probe_agent_control_plane_summary(
2430
+ state.get("agent_control_plane")
2431
+ ),
2432
+ "metadata_keys": sorted(str(key) for key in metadata),
2433
+ "streaming": bool(streaming_trace or metadata.get("streaming")),
2434
+ "streaming_trace_signals": sorted(
2435
+ str(signal)
2436
+ for signal in (dict(streaming_trace).get("signals") or [])
2437
+ if str(signal)
2438
+ ),
2439
+ "streaming_trace_summary": dict(
2440
+ dict(streaming_trace).get("summary") or {}
2441
+ ),
2442
+ }
2443
+
2444
+
2445
+ def _probe_framework_lifecycle_summary(value: Any) -> dict[str, Any]:
2446
+ trace = dict(value or {}) if isinstance(value, Mapping) else {}
2447
+ summary = trace.get("summary")
2448
+ return dict(summary) if isinstance(summary, Mapping) else {}
2449
+
2450
+
2451
+ def _probe_agent_trust_boundary_summary(value: Any) -> dict[str, Any]:
2452
+ payload = dict(value or {}) if isinstance(value, Mapping) else {}
2453
+ summary = dict(payload.get("summary") or {}) if payload else {}
2454
+ if not payload and not summary:
2455
+ return {}
2456
+ return {
2457
+ "framework": payload.get("framework"),
2458
+ "control_count": summary.get("control_count"),
2459
+ "required_control_rate": summary.get("required_control_rate"),
2460
+ "high_risk_unmitigated_count": summary.get(
2461
+ "high_risk_unmitigated_count"
2462
+ ),
2463
+ "evidence_count": summary.get("evidence_count"),
2464
+ "gaps": list(summary.get("gaps") or []),
2465
+ "present_controls": list(summary.get("present_controls") or []),
2466
+ "present_categories": list(summary.get("present_categories") or []),
2467
+ "assets": [
2468
+ str(item.get("id") or item.get("name") or "")
2469
+ for item in _probe_mappings(payload.get("assets"))
2470
+ if item.get("id") or item.get("name")
2471
+ ],
2472
+ "tools": [
2473
+ str(item.get("id") or item.get("name") or "")
2474
+ for item in _probe_mappings(payload.get("tools"))
2475
+ if item.get("id") or item.get("name")
2476
+ ],
2477
+ "surfaces": [
2478
+ str(item.get("id") or item.get("name") or "")
2479
+ for item in _probe_mappings(payload.get("surfaces"))
2480
+ if item.get("id") or item.get("name")
2481
+ ],
2482
+ "threats": list(summary.get("threats") or []),
2483
+ "mitigated_threats": list(summary.get("mitigated_threats") or []),
2484
+ "signals": list(payload.get("signals") or []),
2485
+ "has_identity": bool(summary.get("has_identity")),
2486
+ "has_permissions": bool(summary.get("has_permissions")),
2487
+ "has_sandbox": bool(summary.get("has_sandbox")),
2488
+ "has_audit": bool(summary.get("has_audit")),
2489
+ "has_canaries": bool(summary.get("has_canaries")),
2490
+ "has_human_approval": bool(summary.get("has_human_approval")),
2491
+ "has_memory_isolation": bool(summary.get("has_memory_isolation")),
2492
+ "has_network_egress_controls": bool(
2493
+ summary.get("has_network_egress_controls")
2494
+ ),
2495
+ "has_tool_allowlist": bool(summary.get("has_tool_allowlist")),
2496
+ "has_data_boundary": bool(summary.get("has_data_boundary")),
2497
+ "has_secret_handling": bool(summary.get("has_secret_handling")),
2498
+ }
2499
+
2500
+
2501
+ def _probe_agent_control_plane_summary(value: Any) -> dict[str, Any]:
2502
+ payload = dict(value or {}) if isinstance(value, Mapping) else {}
2503
+ summary = dict(payload.get("summary") or {}) if payload else {}
2504
+ if not payload and not summary:
2505
+ return {}
2506
+ return {
2507
+ "framework": payload.get("framework"),
2508
+ "control_count": summary.get("control_count"),
2509
+ "required_control_rate": summary.get("required_control_rate"),
2510
+ "exceeded_budget_count": summary.get("exceeded_budget_count"),
2511
+ "high_risk_uncontained_count": summary.get(
2512
+ "high_risk_uncontained_count"
2513
+ ),
2514
+ "approval_required_action_count": summary.get(
2515
+ "approval_required_action_count"
2516
+ ),
2517
+ "approved_action_count": summary.get("approved_action_count"),
2518
+ "blocked_action_count": summary.get("blocked_action_count"),
2519
+ "rolled_back_action_count": summary.get("rolled_back_action_count"),
2520
+ "contained_incident_count": summary.get("contained_incident_count"),
2521
+ "within_budget_count": summary.get("within_budget_count"),
2522
+ "evidence_count": summary.get("evidence_count"),
2523
+ "gaps": list(summary.get("gaps") or []),
2524
+ "present_controls": list(summary.get("present_controls") or []),
2525
+ "present_categories": list(summary.get("present_categories") or []),
2526
+ "actions": list(summary.get("actions") or []),
2527
+ "budgets": list(summary.get("budgets") or []),
2528
+ "incidents": list(summary.get("incidents") or []),
2529
+ "signals": list(payload.get("signals") or []),
2530
+ "has_risk_scoring": bool(summary.get("has_risk_scoring")),
2531
+ "has_action_policy": bool(summary.get("has_action_policy")),
2532
+ "has_approval_gates": bool(summary.get("has_approval_gates")),
2533
+ "has_rollback": bool(summary.get("has_rollback")),
2534
+ "has_kill_switch": bool(summary.get("has_kill_switch")),
2535
+ "has_circuit_breakers": bool(summary.get("has_circuit_breakers")),
2536
+ "has_rate_limits": bool(summary.get("has_rate_limits")),
2537
+ "has_budgets": bool(summary.get("has_budgets")),
2538
+ "has_audit": bool(summary.get("has_audit")),
2539
+ "has_containment": bool(summary.get("has_containment")),
2540
+ "has_drift_detection": bool(summary.get("has_drift_detection")),
2541
+ }
2542
+
2543
+
2544
+ def _probe_framework_trace_summary(value: Any) -> dict[str, Any]:
2545
+ trace = dict(value or {}) if isinstance(value, Mapping) else {}
2546
+ if not trace:
2547
+ return {}
2548
+ summary = (
2549
+ dict(trace.get("summary") or {})
2550
+ if isinstance(trace.get("summary"), Mapping)
2551
+ else {}
2552
+ )
2553
+ for count_key, trace_key in (
2554
+ ("span_count", "spans"),
2555
+ ("event_count", "events"),
2556
+ ("checkpoint_count", "checkpoints"),
2557
+ ("session_count", "sessions"),
2558
+ ):
2559
+ if count_key not in summary and isinstance(trace.get(trace_key), list):
2560
+ summary[count_key] = len(trace.get(trace_key, []))
2561
+ elif count_key not in summary and trace.get(count_key) is not None:
2562
+ summary[count_key] = trace.get(count_key)
2563
+ for key in ("signals", "tool_names"):
2564
+ if trace.get(key):
2565
+ summary[key] = sorted(str(item) for item in trace.get(key, []) if str(item))
2566
+ adapter_conformance = (
2567
+ dict(trace.get("adapter_conformance"))
2568
+ if isinstance(trace.get("adapter_conformance"), Mapping)
2569
+ else {}
2570
+ )
2571
+ if adapter_conformance:
2572
+ summary["adapter_conformance_passed"] = bool(
2573
+ adapter_conformance.get("passed")
2574
+ )
2575
+ findings = _probe_mappings(adapter_conformance.get("findings"))
2576
+ summary["adapter_conformance_finding_count"] = len(findings)
2577
+ required_signals = adapter_conformance.get("required_signals")
2578
+ if required_signals:
2579
+ summary["adapter_required_signals"] = sorted(
2580
+ str(item) for item in _probe_list(required_signals) if str(item)
2581
+ )
2582
+ spans = _probe_mappings(trace.get("spans"))
2583
+ events = _probe_mappings(trace.get("events"))
2584
+ if spans and "span_names" not in summary:
2585
+ summary["span_names"] = sorted(
2586
+ {
2587
+ str(span.get("name") or span.get("id") or "")
2588
+ for span in spans
2589
+ if span.get("name") or span.get("id")
2590
+ }
2591
+ )
2592
+ if events and "event_names" not in summary:
2593
+ summary["event_names"] = sorted(
2594
+ {
2595
+ str(event.get("name") or event.get("id") or event.get("type") or "")
2596
+ for event in events
2597
+ if event.get("name") or event.get("id") or event.get("type")
2598
+ }
2599
+ )
2600
+ return summary
2601
+
2602
+
2603
+ def _probe_message_history_summary(value: Any) -> dict[str, Any]:
2604
+ history = dict(value or {}) if isinstance(value, Mapping) else {}
2605
+ if not history:
2606
+ return {}
2607
+ messages = _probe_mappings(history.get("messages"))
2608
+ summary: dict[str, Any] = {}
2609
+ for key in (
2610
+ "message_count",
2611
+ "tool_call_count",
2612
+ "tool_response_count",
2613
+ "handoff_count",
2614
+ ):
2615
+ if history.get(key) not in (None, "", [], {}):
2616
+ summary[key] = history.get(key)
2617
+ for key in ("roles", "sources", "types", "tool_names"):
2618
+ values = _probe_list(history.get(key))
2619
+ if values:
2620
+ summary[key] = sorted(str(item) for item in values if str(item))
2621
+ speaker_sequence = [
2622
+ str(
2623
+ message.get("source")
2624
+ or message.get("speaker")
2625
+ or message.get("role")
2626
+ or ""
2627
+ )
2628
+ for message in messages
2629
+ if message.get("source") or message.get("speaker") or message.get("role")
2630
+ ]
2631
+ if speaker_sequence:
2632
+ summary["speaker_sequence"] = speaker_sequence
2633
+ message_types = [
2634
+ str(message.get("type") or message.get("message_type") or "")
2635
+ for message in messages
2636
+ if message.get("type") or message.get("message_type")
2637
+ ]
2638
+ if message_types:
2639
+ summary["message_types"] = sorted(set(message_types))
2640
+ stop_reason = str(history.get("stop_reason") or "")
2641
+ if stop_reason:
2642
+ summary["stop_reason"] = stop_reason
2643
+ last_content = str(history.get("last_content") or "")
2644
+ if last_content:
2645
+ summary["last_content"] = last_content
2646
+ handoffs = _probe_mappings(history.get("handoffs"))
2647
+ if handoffs:
2648
+ summary["handoffs"] = [
2649
+ {
2650
+ key: str(handoff.get(key) or "")
2651
+ for key in ("from", "to", "task")
2652
+ if handoff.get(key) not in (None, "", [], {})
2653
+ }
2654
+ for handoff in handoffs
2655
+ ]
2656
+ return summary
2657
+
2658
+
2659
+ def _probe_framework_handoff_summary(value: Any) -> dict[str, Any]:
2660
+ coordination = dict(value or {}) if isinstance(value, Mapping) else {}
2661
+ if not coordination:
2662
+ return {}
2663
+ summary: dict[str, Any] = {}
2664
+ for key in ("handoff_count", "review_count", "reconciliation_count"):
2665
+ if coordination.get(key) not in (None, "", [], {}):
2666
+ summary[key] = coordination.get(key)
2667
+ participants = _probe_list(coordination.get("participants"))
2668
+ if participants:
2669
+ summary["participants"] = sorted(str(item) for item in participants if str(item))
2670
+ handoffs = _probe_mappings(coordination.get("handoffs"))
2671
+ if handoffs:
2672
+ summary["handoffs"] = [
2673
+ {
2674
+ key: str(handoff.get(key) or "")
2675
+ for key in ("from", "to", "task", "reason", "message_type")
2676
+ if handoff.get(key) not in (None, "", [], {})
2677
+ }
2678
+ for handoff in handoffs
2679
+ ]
2680
+ reviews = _probe_mappings(coordination.get("reviews"))
2681
+ if reviews:
2682
+ summary["reviews"] = [
2683
+ {
2684
+ key: str(review.get(key) or "")
2685
+ for key in ("reviewer", "target", "status", "message_type")
2686
+ if review.get(key) not in (None, "", [], {})
2687
+ }
2688
+ for review in reviews
2689
+ ]
2690
+ reconciliations = _probe_mappings(coordination.get("reconciliations"))
2691
+ if reconciliations:
2692
+ summary["reconciliations"] = [
2693
+ {
2694
+ key: str(reconciliation.get(key) or "")
2695
+ for key in (
2696
+ "source",
2697
+ "accepted_source",
2698
+ "status",
2699
+ "message_type",
2700
+ )
2701
+ if reconciliation.get(key) not in (None, "", [], {})
2702
+ }
2703
+ for reconciliation in reconciliations
2704
+ ]
2705
+ summary["has_handoffs"] = bool(handoffs)
2706
+ summary["has_reviews"] = bool(reviews)
2707
+ summary["has_reconciliation"] = bool(reconciliations)
2708
+ return summary
2709
+
2710
+
2711
+ def _probe_orchestration_trace_summary(value: Any) -> dict[str, Any]:
2712
+ trace = dict(value or {}) if isinstance(value, Mapping) else {}
2713
+ summary = dict(trace.get("summary") or {}) if isinstance(trace.get("summary"), Mapping) else {}
2714
+ if trace.get("signals"):
2715
+ summary["signals"] = sorted(str(signal) for signal in trace.get("signals", []) if str(signal))
2716
+ if trace.get("nodes"):
2717
+ summary["node_names"] = sorted(
2718
+ {
2719
+ str(dict(node).get("name") or dict(node).get("id") or "")
2720
+ for node in trace.get("nodes", [])
2721
+ if isinstance(node, Mapping)
2722
+ and (dict(node).get("name") or dict(node).get("id"))
2723
+ }
2724
+ )
2725
+ for key, trace_key in (
2726
+ ("node_count", "nodes"),
2727
+ ("edge_count", "edges"),
2728
+ ("step_count", "steps"),
2729
+ ):
2730
+ if key not in summary and isinstance(trace.get(trace_key), list):
2731
+ summary[key] = len(trace.get(trace_key, []))
2732
+ return summary
2733
+
2734
+
2735
+ def _probe_workflow_trace_summary(value: Any) -> dict[str, Any]:
2736
+ trace = dict(value or {}) if isinstance(value, Mapping) else {}
2737
+ if not trace:
2738
+ return {}
2739
+ summary = (
2740
+ dict(trace.get("summary") or {})
2741
+ if isinstance(trace.get("summary"), Mapping)
2742
+ else {}
2743
+ )
2744
+ for count_key, trace_key in (
2745
+ ("node_count", "nodes"),
2746
+ ("edge_count", "edges"),
2747
+ ("step_count", "steps"),
2748
+ ("checkpoint_count", "checkpoints"),
2749
+ ("route_decision_count", "route_decisions"),
2750
+ ("interrupt_count", "interrupts"),
2751
+ ("replay_count", "replay"),
2752
+ ("write_count", "writes"),
2753
+ ("state_snapshot_count", "state_snapshots"),
2754
+ ):
2755
+ if count_key not in summary and isinstance(trace.get(trace_key), list):
2756
+ summary[count_key] = len(trace.get(trace_key, []))
2757
+ elif count_key not in summary and trace.get(count_key) is not None:
2758
+ summary[count_key] = trace.get(count_key)
2759
+ if "tool_call_count" not in summary and trace.get("tool_call_count") is not None:
2760
+ summary["tool_call_count"] = trace.get("tool_call_count")
2761
+
2762
+ nodes = _probe_mappings(trace.get("nodes"))
2763
+ steps = _probe_mappings(trace.get("steps"))
2764
+ checkpoints = _probe_mappings(trace.get("checkpoints"))
2765
+ routes = _probe_mappings(trace.get("route_decisions"))
2766
+ interrupts = _probe_mappings(trace.get("interrupts"))
2767
+ replay = _probe_mappings(trace.get("replay"))
2768
+ topology = dict(trace.get("topology") or {}) if isinstance(trace.get("topology"), Mapping) else {}
2769
+ final_state = (
2770
+ dict(trace.get("final_state"))
2771
+ if isinstance(trace.get("final_state"), Mapping)
2772
+ else {}
2773
+ )
2774
+
2775
+ node_names = [
2776
+ node.get("name") or node.get("id")
2777
+ for node in nodes
2778
+ if node.get("name") or node.get("id")
2779
+ ]
2780
+ step_names = [
2781
+ step.get("name") or step.get("node") or step.get("id")
2782
+ for step in steps
2783
+ if step.get("name") or step.get("node") or step.get("id")
2784
+ ]
2785
+ tool_names = [
2786
+ *[str(item) for item in _probe_list(trace.get("tool_names")) if str(item)],
2787
+ *[
2788
+ str(call.get("name") or call.get("tool") or "")
2789
+ for step in steps
2790
+ for call in _probe_mappings(step.get("tool_calls"))
2791
+ if call.get("name") or call.get("tool")
2792
+ ],
2793
+ ]
2794
+ final_state_keys = trace.get("final_state_keys") or list(final_state.keys())
2795
+
2796
+ for key, values in (
2797
+ ("node_names", node_names),
2798
+ ("step_names", step_names),
2799
+ ("checkpoint_ids", [item.get("id") or item.get("checkpoint_id") for item in checkpoints]),
2800
+ ("route_targets", [item.get("target") or item.get("selected") for item in routes]),
2801
+ ("interrupt_nodes", [item.get("node") or item.get("id") for item in interrupts]),
2802
+ ("replay_ids", [item.get("id") or item.get("replay_id") for item in replay]),
2803
+ ("tool_names", tool_names),
2804
+ ("step_statuses", trace.get("step_statuses")),
2805
+ ("final_state_keys", final_state_keys),
2806
+ ):
2807
+ cleaned = sorted({str(item) for item in _probe_list(values) if str(item)})
2808
+ if cleaned:
2809
+ summary[key] = cleaned
2810
+
2811
+ for key in ("entry_nodes", "terminal_nodes"):
2812
+ values = sorted(str(item) for item in _probe_list(topology.get(key)) if str(item))
2813
+ if values:
2814
+ summary[key] = values
2815
+ for key in ("has_replay", "has_interrupts", "has_routes"):
2816
+ if trace.get(key) is not None:
2817
+ summary[key] = bool(trace.get(key))
2818
+ if topology:
2819
+ summary["has_topology"] = True
2820
+ if trace.get("framework"):
2821
+ summary["framework"] = str(trace.get("framework"))
2822
+ return summary
2823
+
2824
+
2825
+ def _probe_realtime_trace_summary(value: Any) -> dict[str, Any]:
2826
+ trace = dict(value or {}) if isinstance(value, Mapping) else {}
2827
+ summary = (
2828
+ dict(trace.get("summary") or {})
2829
+ if isinstance(trace.get("summary"), Mapping)
2830
+ else {}
2831
+ )
2832
+ for key in (
2833
+ "signals",
2834
+ "tool_names",
2835
+ "frame_types",
2836
+ "event_types",
2837
+ "categories",
2838
+ "directions",
2839
+ "modalities",
2840
+ ):
2841
+ if trace.get(key):
2842
+ summary[key] = sorted(str(item) for item in trace.get(key, []) if str(item))
2843
+ for count_key, value_key in (
2844
+ ("frame_count", "frames"),
2845
+ ("event_count", "events"),
2846
+ ):
2847
+ if count_key not in summary and isinstance(trace.get(value_key), list):
2848
+ summary[count_key] = len(trace.get(value_key, []))
2849
+ return summary
2850
+
2851
+
2852
+ def _probe_framework_memory_summary(value: Any) -> dict[str, Any]:
2853
+ trace = dict(value or {}) if isinstance(value, Mapping) else {}
2854
+ if not trace:
2855
+ return {}
2856
+ summary = (
2857
+ dict(trace.get("summary") or {})
2858
+ if isinstance(trace.get("summary"), Mapping)
2859
+ else {}
2860
+ )
2861
+ for count_key, value_key in (
2862
+ ("operation_count", "operations"),
2863
+ ("checkpoint_count", "checkpoints"),
2864
+ ("memory_count", "memories"),
2865
+ ("retrieval_count", "retrievals"),
2866
+ ("store_count", "stores"),
2867
+ ):
2868
+ if count_key not in summary and isinstance(trace.get(value_key), list):
2869
+ summary[count_key] = len(trace.get(value_key, []))
2870
+ elif count_key not in summary and trace.get(count_key) is not None:
2871
+ summary[count_key] = trace.get(count_key)
2872
+ policies = dict(trace.get("policies") or {}) if isinstance(trace.get("policies"), Mapping) else {}
2873
+ if "policy_count" not in summary:
2874
+ summary["policy_count"] = len(policies) or trace.get("policy_count", 0)
2875
+ for key in ("operation_types", "source_ids", "namespaces", "retrieval_doc_ids"):
2876
+ if trace.get(key):
2877
+ summary[key] = sorted(str(item) for item in trace.get(key, []) if str(item))
2878
+ policy_keys = trace.get("policy_keys") or list(policies.keys())
2879
+ if policy_keys:
2880
+ summary["policy_keys"] = sorted(str(item) for item in policy_keys if str(item))
2881
+ return summary
2882
+
2883
+
2884
+ def _probe_retrieval_memory_summary(value: Any) -> dict[str, Any]:
2885
+ trace = dict(value or {}) if isinstance(value, Mapping) else {}
2886
+ if not trace:
2887
+ return {}
2888
+ summary = (
2889
+ dict(trace.get("summary") or {})
2890
+ if isinstance(trace.get("summary"), Mapping)
2891
+ else {}
2892
+ )
2893
+ documents = [
2894
+ dict(item)
2895
+ for item in (trace.get("documents") or [])
2896
+ if isinstance(item, Mapping)
2897
+ ]
2898
+ queries = [
2899
+ dict(item)
2900
+ for item in (trace.get("queries") or [])
2901
+ if isinstance(item, Mapping)
2902
+ ]
2903
+ citations = [
2904
+ dict(item)
2905
+ for item in (trace.get("citations") or [])
2906
+ if isinstance(item, Mapping)
2907
+ ]
2908
+ memory_writes = [
2909
+ dict(item)
2910
+ for item in (trace.get("memory_writes") or [])
2911
+ if isinstance(item, Mapping)
2912
+ ]
2913
+ summary.setdefault("document_count", len(documents))
2914
+ summary.setdefault("query_count", len(queries))
2915
+ summary.setdefault("citation_count", len(citations))
2916
+ summary.setdefault("memory_write_count", len(memory_writes))
2917
+ summary.setdefault(
2918
+ "current_document_count",
2919
+ len([document for document in documents if document.get("current") is not False]),
2920
+ )
2921
+ doc_ids = sorted(
2922
+ {
2923
+ str(
2924
+ document.get("id")
2925
+ or document.get("doc_id")
2926
+ or document.get("source")
2927
+ or ""
2928
+ )
2929
+ for document in documents
2930
+ if document.get("id") or document.get("doc_id") or document.get("source")
2931
+ }
2932
+ )
2933
+ cited_doc_ids = sorted(
2934
+ {
2935
+ str(doc_id)
2936
+ for citation in citations
2937
+ for doc_id in _probe_list(citation.get("doc_ids"))
2938
+ if str(doc_id)
2939
+ }
2940
+ )
2941
+ if doc_ids:
2942
+ summary["document_ids"] = doc_ids
2943
+ if cited_doc_ids:
2944
+ summary["citation_doc_ids"] = cited_doc_ids
2945
+ if trace.get("require_current") is not None:
2946
+ summary["require_current"] = bool(trace.get("require_current"))
2947
+ return summary
2948
+
2949
+
2950
+ def _probe_agent_memory_lineage_summary(value: Any) -> dict[str, Any]:
2951
+ trace = dict(value or {}) if isinstance(value, Mapping) else {}
2952
+ if not trace:
2953
+ return {}
2954
+ summary = (
2955
+ dict(trace.get("summary") or {})
2956
+ if isinstance(trace.get("summary"), Mapping)
2957
+ else {}
2958
+ )
2959
+ stores = _probe_mappings(trace.get("stores"))
2960
+ memories = _probe_mappings(trace.get("memories"))
2961
+ operations = _probe_mappings(trace.get("operations"))
2962
+ lineage = _probe_mappings(trace.get("lineage"))
2963
+ policies = dict(trace.get("policies") or {}) if isinstance(trace.get("policies"), Mapping) else {}
2964
+ poison_tests = _probe_mappings(trace.get("poison_tests") or trace.get("poisoning_tests"))
2965
+ isolation_tests = _probe_mappings(trace.get("isolation_tests"))
2966
+ retention_tests = _probe_mappings(trace.get("retention_tests") or trace.get("deletion_tests"))
2967
+ observability = (
2968
+ dict(trace.get("observability"))
2969
+ if isinstance(trace.get("observability"), Mapping)
2970
+ else {}
2971
+ )
2972
+ artifacts = _probe_mappings(trace.get("artifacts"))
2973
+ operation_type_values = [
2974
+ operation_type
2975
+ for operation_type in (
2976
+ _probe_memory_key(
2977
+ operation.get("operation")
2978
+ or operation.get("type")
2979
+ or operation.get("op")
2980
+ )
2981
+ for operation in operations
2982
+ )
2983
+ if operation_type
2984
+ ]
2985
+ operation_types = sorted(set(operation_type_values))
2986
+ policy_keys = sorted(_probe_memory_key(key) for key in policies if _probe_memory_key(key))
2987
+ attributed_memories = [
2988
+ memory
2989
+ for memory in memories
2990
+ if _probe_list(
2991
+ memory.get("source_ids")
2992
+ or memory.get("sources")
2993
+ or memory.get("doc_ids")
2994
+ )
2995
+ ]
2996
+ unattributed_memories = [
2997
+ str(memory.get("id") or memory.get("key") or index)
2998
+ for index, memory in enumerate(memories, start=1)
2999
+ if not _probe_list(
3000
+ memory.get("source_ids")
3001
+ or memory.get("sources")
3002
+ or memory.get("doc_ids")
3003
+ )
3004
+ and str(memory.get("status") or "active").lower() not in {"deleted", "expired", "blocked"}
3005
+ ]
3006
+ poison_good = {"passed", "blocked", "mitigated", "contained", "accepted"}
3007
+ isolation_good = {"passed", "blocked", "mitigated", "contained"}
3008
+ retention_good = {"passed", "deleted", "expired", "purged", "mitigated"}
3009
+ poisoning_failures = [
3010
+ test
3011
+ for test in poison_tests
3012
+ if _probe_memory_key(test.get("status")) not in poison_good
3013
+ ]
3014
+ isolation_violations = [
3015
+ test
3016
+ for test in isolation_tests
3017
+ if _probe_memory_key(test.get("status")) not in isolation_good
3018
+ ]
3019
+ retention_violations = [
3020
+ test
3021
+ for test in retention_tests
3022
+ if _probe_memory_key(test.get("status")) not in retention_good
3023
+ ]
3024
+ policy_violations = [
3025
+ operation
3026
+ for operation in operations
3027
+ if _probe_memory_key(operation.get("status")) in {"policy_violation", "violation", "failed_policy"}
3028
+ or _probe_memory_key(operation.get("policy_decision")) in {"violation", "failed", "bypassed"}
3029
+ ]
3030
+ summary.setdefault("store_count", len(stores))
3031
+ summary.setdefault("memory_count", len(memories))
3032
+ summary.setdefault("operation_count", len(operations))
3033
+ summary.setdefault("lineage_count", len(lineage))
3034
+ summary.setdefault("policy_count", len(policies))
3035
+ summary.setdefault("artifact_count", len(artifacts))
3036
+ summary.setdefault("observability_hook_count", _probe_observability_count(observability))
3037
+ summary.setdefault("attributed_memory_count", len(attributed_memories))
3038
+ summary.setdefault("unattributed_memory_count", len(unattributed_memories))
3039
+ summary.setdefault("poisoned_memory_count", 0)
3040
+ summary.setdefault("open_poisoning_count", len(poisoning_failures))
3041
+ summary.setdefault("isolation_violation_count", len(isolation_violations))
3042
+ summary.setdefault("retention_violation_count", len(retention_violations))
3043
+ summary.setdefault("policy_violation_count", len(policy_violations))
3044
+ for operation_type in ("read", "write", "update", "delete", "recall"):
3045
+ summary.setdefault(
3046
+ f"{operation_type}_operation_count",
3047
+ operation_type_values.count(operation_type),
3048
+ )
3049
+ summary.setdefault("has_target", bool(trace.get("target")))
3050
+ summary.setdefault("has_stores", bool(stores))
3051
+ summary.setdefault("has_memory_records", bool(memories))
3052
+ summary.setdefault("has_operations", bool(operations))
3053
+ summary.setdefault("has_lineage", bool(lineage))
3054
+ summary.setdefault("has_source_attribution", bool(attributed_memories) and not unattributed_memories)
3055
+ summary.setdefault(
3056
+ "has_tenant_isolation",
3057
+ "tenant_isolation" in policy_keys or bool(isolation_tests),
3058
+ )
3059
+ summary.setdefault("has_audit", "audit" in policy_keys)
3060
+ summary.setdefault(
3061
+ "has_retention_policy",
3062
+ any(key in policy_keys for key in ("retention", "retention_policy")),
3063
+ )
3064
+ summary.setdefault(
3065
+ "has_deletion_policy",
3066
+ any(key in policy_keys for key in ("deletion", "deletion_policy")),
3067
+ )
3068
+ summary.setdefault("has_redaction", "redaction" in policy_keys)
3069
+ summary.setdefault("has_canaries", "canary" in policy_keys or bool(poison_tests))
3070
+ summary.setdefault("has_observability", bool(observability))
3071
+ summary.setdefault("has_artifacts", bool(artifacts))
3072
+ if operation_types:
3073
+ summary["operation_types"] = operation_types
3074
+ if policy_keys:
3075
+ summary["policy_keys"] = policy_keys
3076
+ observed_evidence = {
3077
+ signal
3078
+ for flag, signal in (
3079
+ ("has_target", "target"),
3080
+ ("has_stores", "store"),
3081
+ ("has_memory_records", "memory_record"),
3082
+ ("has_operations", "operation"),
3083
+ ("has_lineage", "lineage"),
3084
+ ("has_source_attribution", "source_attribution"),
3085
+ ("has_tenant_isolation", "tenant_isolation"),
3086
+ ("has_audit", "audit"),
3087
+ ("has_retention_policy", "retention_policy"),
3088
+ ("has_deletion_policy", "deletion_policy"),
3089
+ ("has_redaction", "redaction"),
3090
+ ("has_canaries", "canary"),
3091
+ ("has_observability", "observability"),
3092
+ ("has_artifacts", "artifact"),
3093
+ )
3094
+ if summary.get(flag)
3095
+ }
3096
+ observed_evidence.update(f"{operation_type}_operation" for operation_type in operation_types)
3097
+ observed_signals = {
3098
+ *observed_evidence,
3099
+ *operation_types,
3100
+ *policy_keys,
3101
+ "agent_memory_lineage",
3102
+ "memory_lineage",
3103
+ "memory_provenance",
3104
+ "memory",
3105
+ "provenance",
3106
+ }
3107
+ summary.setdefault("observed_evidence", sorted(observed_evidence))
3108
+ summary.setdefault("observed_signals", sorted(observed_signals))
3109
+ summary.setdefault("blocking_gap_count", 0)
3110
+ return summary
3111
+
3112
+
3113
+ def _probe_browser_cua_summary(value: Any) -> dict[str, Any]:
3114
+ trace = dict(value or {}) if isinstance(value, Mapping) else {}
3115
+ if not trace:
3116
+ return {}
3117
+ summary = (
3118
+ dict(trace.get("summary") or {})
3119
+ if isinstance(trace.get("summary"), Mapping)
3120
+ else {}
3121
+ )
3122
+ snapshots = _probe_mappings(trace.get("snapshots"))
3123
+ actions = _probe_mappings(trace.get("action_replay") or trace.get("actions"))
3124
+ screenshots = _probe_mappings(trace.get("screenshots"))
3125
+ regions = dict(trace.get("regions") or {}) if isinstance(trace.get("regions"), Mapping) else {}
3126
+ network_log = _probe_mappings(trace.get("network_log"))
3127
+ runtime_events = _probe_mappings(trace.get("runtime_events"))
3128
+ performance_entries = _probe_mappings(trace.get("performance_entries"))
3129
+ prompt_injections = _probe_mappings(trace.get("prompt_injections"))
3130
+ mutation_pack = (
3131
+ dict(trace.get("mutation_pack"))
3132
+ if isinstance(trace.get("mutation_pack"), Mapping)
3133
+ else {}
3134
+ )
3135
+ mutations = _probe_mappings(
3136
+ mutation_pack.get("mutations")
3137
+ or trace.get("browser_mutations")
3138
+ or trace.get("mutations")
3139
+ )
3140
+ for count_key, count in (
3141
+ ("snapshot_count", len(snapshots)),
3142
+ ("action_count", len(actions)),
3143
+ ("screenshot_count", len(screenshots)),
3144
+ ("region_count", len(regions)),
3145
+ ("network_request_count", len(network_log)),
3146
+ ("runtime_event_count", len(runtime_events)),
3147
+ ("performance_entry_count", len(performance_entries)),
3148
+ ("prompt_injection_surface_count", len(prompt_injections)),
3149
+ ("mutation_count", len(mutations)),
3150
+ ):
3151
+ summary.setdefault(count_key, trace.get(count_key, count))
3152
+ for count_key in (
3153
+ "successful_action_count",
3154
+ "blocked_action_count",
3155
+ "matched_action_count",
3156
+ "prompt_injection_touched_count",
3157
+ "screenshot_diff_count",
3158
+ ):
3159
+ if trace.get(count_key) is not None:
3160
+ summary[count_key] = trace.get(count_key)
3161
+ summary.setdefault(
3162
+ "stale_action_count",
3163
+ len([action for action in actions if action.get("stale_screenshot")]),
3164
+ )
3165
+ summary.setdefault(
3166
+ "dom_snapshot_count",
3167
+ len([snapshot for snapshot in snapshots if snapshot.get("dom")]),
3168
+ )
3169
+ summary.setdefault(
3170
+ "screenshot_snapshot_count",
3171
+ len(
3172
+ [
3173
+ snapshot
3174
+ for snapshot in snapshots
3175
+ if snapshot.get("screenshot_uri") or snapshot.get("screenshot_path")
3176
+ ]
3177
+ ),
3178
+ )
3179
+ summary["layout_shift_present"] = bool(
3180
+ trace.get("layout_shift_present")
3181
+ or trace.get("layout_shift_distribution")
3182
+ )
3183
+ summary["storage_present"] = bool(trace.get("storage_present"))
3184
+ action_types = trace.get("action_types") or [
3185
+ action.get("action") or action.get("type")
3186
+ for action in actions
3187
+ if action.get("action") or action.get("type")
3188
+ ]
3189
+ tool_names = trace.get("tool_names") or [
3190
+ action.get("tool") or action.get("tool_name")
3191
+ for action in actions
3192
+ if action.get("tool") or action.get("tool_name")
3193
+ ]
3194
+ mutation_ids = [
3195
+ mutation.get("id") or mutation.get("name")
3196
+ for mutation in mutations
3197
+ if mutation.get("id") or mutation.get("name")
3198
+ ]
3199
+ mutation_types = [
3200
+ mutation.get("type") or mutation.get("kind")
3201
+ for mutation in mutations
3202
+ if mutation.get("type") or mutation.get("kind")
3203
+ ]
3204
+ region_ids = [
3205
+ *list(regions.keys()),
3206
+ *[
3207
+ region.get("id") or region.get("name")
3208
+ for region in regions.values()
3209
+ if isinstance(region, Mapping) and (region.get("id") or region.get("name"))
3210
+ ],
3211
+ ]
3212
+ prompt_injection_ids = [
3213
+ injection.get("id") or injection.get("selector")
3214
+ for injection in prompt_injections
3215
+ if injection.get("id") or injection.get("selector")
3216
+ ]
3217
+ summary["action_types"] = sorted(str(item) for item in action_types if str(item))
3218
+ summary["tool_names"] = sorted(str(item) for item in tool_names if str(item))
3219
+ summary["mutation_ids"] = sorted(str(item) for item in mutation_ids if str(item))
3220
+ summary["mutation_types"] = sorted(str(item) for item in mutation_types if str(item))
3221
+ summary["region_ids"] = sorted(str(item) for item in region_ids if str(item))
3222
+ summary["prompt_injection_ids"] = sorted(
3223
+ str(item) for item in prompt_injection_ids if str(item)
3224
+ )
3225
+ layout_distribution = (
3226
+ dict(trace.get("layout_shift_distribution"))
3227
+ if isinstance(trace.get("layout_shift_distribution"), Mapping)
3228
+ else {}
3229
+ )
3230
+ layout_values = [
3231
+ _probe_float(layout_distribution.get(key))
3232
+ for key in ("max", "p95", "score", "value")
3233
+ ]
3234
+ layout_values = [value for value in layout_values if value is not None]
3235
+ if layout_values:
3236
+ summary["max_layout_shift_score"] = max(layout_values)
3237
+ performance_durations = [
3238
+ _probe_float(entry.get("duration_ms") or entry.get("duration"))
3239
+ for entry in performance_entries
3240
+ ]
3241
+ performance_durations = [value for value in performance_durations if value is not None]
3242
+ if performance_durations:
3243
+ summary["max_performance_duration_ms"] = max(performance_durations)
3244
+ compact_actions = []
3245
+ for action in actions[:5]:
3246
+ compact_action = {
3247
+ key: action.get(key)
3248
+ for key in (
3249
+ "id",
3250
+ "tool",
3251
+ "tool_name",
3252
+ "action",
3253
+ "selector",
3254
+ "success",
3255
+ "matched",
3256
+ "blocked",
3257
+ "mutation_id",
3258
+ "mutation_type",
3259
+ )
3260
+ if action.get(key) not in (None, "", [], {})
3261
+ }
3262
+ region = action.get("region")
3263
+ if isinstance(region, Mapping):
3264
+ compact_action["region"] = {
3265
+ key: region.get(key)
3266
+ for key in ("id", "name", "selector", "x", "y", "width", "height")
3267
+ if region.get(key) not in (None, "", [], {})
3268
+ }
3269
+ if compact_action:
3270
+ compact_actions.append(compact_action)
3271
+ if compact_actions:
3272
+ summary["actions"] = compact_actions
3273
+ if runtime_events:
3274
+ summary["runtime_events"] = [
3275
+ {
3276
+ key: event.get(key)
3277
+ for key in ("id", "name", "type", "level", "message")
3278
+ if event.get(key) not in (None, "", [], {})
3279
+ }
3280
+ for event in runtime_events[:5]
3281
+ ]
3282
+ return summary
3283
+
3284
+
3285
+ def _probe_mcp_tool_session_summary(value: Any) -> dict[str, Any]:
3286
+ trace = dict(value or {}) if isinstance(value, Mapping) else {}
3287
+ summary = trace.get("summary")
3288
+ return dict(summary) if isinstance(summary, Mapping) else {}
3289
+
3290
+
3291
+ def _probe_a2a_protocol_summary(value: Any) -> dict[str, Any]:
3292
+ trace = dict(value or {}) if isinstance(value, Mapping) else {}
3293
+ summary = trace.get("summary")
3294
+ return dict(summary) if isinstance(summary, Mapping) else {}
3295
+
3296
+
3297
+ def _probe_openenv_summary(value: Any) -> dict[str, Any]:
3298
+ trace = dict(value or {}) if isinstance(value, Mapping) else {}
3299
+ if not trace:
3300
+ return {}
3301
+ summary = (
3302
+ dict(trace.get("summary") or {})
3303
+ if isinstance(trace.get("summary"), Mapping)
3304
+ else {}
3305
+ )
3306
+ trajectory = _probe_mappings(trace.get("trajectory") or trace.get("steps"))
3307
+ action_log = _probe_mappings(trace.get("action_log") or trace.get("actions"))
3308
+ error_log = _probe_mappings(trace.get("error_log") or trace.get("errors"))
3309
+ sandbox = (
3310
+ dict(trace.get("sandbox"))
3311
+ if isinstance(trace.get("sandbox"), Mapping)
3312
+ else {}
3313
+ )
3314
+ for count_key, fallback in (
3315
+ ("reset_count", 1 if trace.get("initial_observation") is not None else 0),
3316
+ ("step_count", len(trajectory)),
3317
+ ("action_route_count", len(action_log) or len(trajectory)),
3318
+ (
3319
+ "failure_count",
3320
+ max(
3321
+ len(_probe_mappings(trace.get("failure_injections") or trace.get("faults"))),
3322
+ sum(
3323
+ 1
3324
+ for step in trajectory
3325
+ if step.get("failure_injected") or step.get("failure")
3326
+ ),
3327
+ ),
3328
+ ),
3329
+ (
3330
+ "metadata_capture_count",
3331
+ sum(
3332
+ 1
3333
+ for step in trajectory
3334
+ if step.get("metadata") not in (None, "", [], {})
3335
+ or step.get("info") not in (None, "", [], {})
3336
+ )
3337
+ + (1 if trace.get("reset_info") else 0),
3338
+ ),
3339
+ ("error_count", len(error_log)),
3340
+ ):
3341
+ if summary.get(count_key) in (None, "", [], {}):
3342
+ summary[count_key] = fallback
3343
+ if summary.get("reward_total") in (None, "", [], {}):
3344
+ summary["reward_total"] = round(
3345
+ sum(_probe_float(step.get("reward")) or 0.0 for step in trajectory),
3346
+ 4,
3347
+ )
3348
+ for key in ("done", "terminated", "truncated"):
3349
+ if summary.get(key) in (None, "", [], {}):
3350
+ summary[key] = any(bool(step.get(key)) for step in trajectory)
3351
+ if summary.get("sandbox_enabled") in (None, "", [], {}):
3352
+ summary["sandbox_enabled"] = bool(sandbox.get("enabled", bool(sandbox)))
3353
+ if summary.get("isolation") in (None, "", [], {}):
3354
+ summary["isolation"] = str(sandbox.get("isolation") or "process")
3355
+ for key in (
3356
+ "runtime",
3357
+ "transport",
3358
+ "requires_external_service",
3359
+ "deterministic_reset",
3360
+ ):
3361
+ if summary.get(key) in (None, "", [], {}) and trace.get(key) not in (
3362
+ None,
3363
+ "",
3364
+ [],
3365
+ {},
3366
+ ):
3367
+ summary[key] = trace.get(key)
3368
+ signals = trace.get("signals")
3369
+ if signals and summary.get("signals") in (None, "", [], {}):
3370
+ summary["signals"] = sorted(str(signal) for signal in signals if str(signal))
3371
+ return summary
3372
+
3373
+
3374
+ def _probe_artifact_evidence(artifact: Mapping[str, Any]) -> dict[str, Any]:
3375
+ metadata = artifact.get("metadata")
3376
+ return {
3377
+ "type": str(artifact.get("type") or ""),
3378
+ "uri": str(artifact.get("uri") or ""),
3379
+ "path": str(artifact.get("path") or ""),
3380
+ "mime_type": str(artifact.get("mime_type") or ""),
3381
+ "role": str(artifact.get("role") or ""),
3382
+ "metadata": dict(metadata) if isinstance(metadata, Mapping) else {},
3383
+ }
3384
+
3385
+
3386
+ def _probe_list(value: Any) -> list[Any]:
3387
+ if value is None:
3388
+ return []
3389
+ if isinstance(value, list):
3390
+ return value
3391
+ if isinstance(value, tuple):
3392
+ return list(value)
3393
+ return [value]
3394
+
3395
+
3396
+ def _probe_mappings(value: Any) -> list[dict[str, Any]]:
3397
+ return [dict(item) for item in _probe_list(value) if isinstance(item, Mapping)]
3398
+
3399
+
3400
+ def _probe_memory_key(value: Any) -> str:
3401
+ return str(value or "").strip().lower().replace("-", "_").replace(" ", "_")
3402
+
3403
+
3404
+ def _probe_observability_count(observability: Mapping[str, Any]) -> int:
3405
+ count = 0
3406
+ for value in observability.values():
3407
+ if isinstance(value, Mapping):
3408
+ count += len(value)
3409
+ elif isinstance(value, (list, tuple, set)):
3410
+ count += len([item for item in value if item])
3411
+ elif value:
3412
+ count += 1
3413
+ return count
3414
+
3415
+
3416
+ def _probe_float(value: Any) -> float | None:
3417
+ if isinstance(value, bool):
3418
+ return None
3419
+ if isinstance(value, (int, float)):
3420
+ return float(value)
3421
+ if isinstance(value, str):
3422
+ try:
3423
+ return float(value.strip())
3424
+ except ValueError:
3425
+ return None
3426
+ return None
3427
+
3428
+
3429
+ def _probe_summary(
3430
+ cases: Sequence[Mapping[str, Any]],
3431
+ contract: Mapping[str, Any],
3432
+ ) -> dict[str, Any]:
3433
+ passed = sum(1 for case in cases if case.get("passed"))
3434
+ failed = len(cases) - passed
3435
+ response_tool_count = sum(
3436
+ int(dict(case.get("response") or {}).get("tool_call_count") or 0)
3437
+ for case in cases
3438
+ )
3439
+ runtime_trace_count = sum(1 for case in cases if case.get("runtime_trace"))
3440
+ streaming_trace_count = sum(
3441
+ 1 for case in cases if dict(case.get("response") or {}).get("streaming")
3442
+ )
3443
+ observed_io_contracts = [
3444
+ dict(case.get("observed_io_contract") or {})
3445
+ for case in cases
3446
+ if isinstance(case.get("observed_io_contract"), Mapping)
3447
+ ]
3448
+ observed_io_contract_count = sum(
3449
+ 1
3450
+ for item in observed_io_contracts
3451
+ if item.get("kind")
3452
+ == "agent-learning.framework-adapter-observed-io-contract.v1"
3453
+ and int(dict(item.get("summary") or {}).get("invocation_count") or 0) >= 1
3454
+ )
3455
+ signature_bound_count = sum(
3456
+ 1
3457
+ for item in observed_io_contracts
3458
+ if dict(item.get("summary") or {}).get("signature_bound") is True
3459
+ )
3460
+ call_contract_count = sum(
3461
+ 1
3462
+ for case in cases
3463
+ for invocation in dict(case.get("runtime_trace") or {}).get("invocations", [])
3464
+ if isinstance(invocation, Mapping)
3465
+ and dict(invocation.get("call_contract") or {}).get("kind")
3466
+ == "agent-learning.framework-adapter-call-contract.v1"
3467
+ )
3468
+ input_keys = sorted(
3469
+ {
3470
+ str(invocation.get("input_key"))
3471
+ for case in cases
3472
+ for invocation in dict(case.get("runtime_trace") or {}).get("invocations", [])
3473
+ if isinstance(invocation, Mapping)
3474
+ and invocation.get("input_key") not in (None, "", [], {})
3475
+ }
3476
+ )
3477
+ call_styles = sorted(
3478
+ {
3479
+ str(invocation.get("call_style"))
3480
+ for case in cases
3481
+ for invocation in dict(case.get("runtime_trace") or {}).get("invocations", [])
3482
+ if isinstance(invocation, Mapping)
3483
+ and invocation.get("call_style") not in (None, "", [], {})
3484
+ }
3485
+ )
3486
+ input_kwargs_keys = sorted(
3487
+ {
3488
+ str(key)
3489
+ for case in cases
3490
+ for invocation in dict(case.get("runtime_trace") or {}).get("invocations", [])
3491
+ if isinstance(invocation, Mapping)
3492
+ for key in invocation.get("input_kwargs_keys", [])
3493
+ if key not in (None, "", [], {})
3494
+ }
3495
+ )
3496
+ input_types = _unique_probe_strings(
3497
+ input_type
3498
+ for item in observed_io_contracts
3499
+ for input_type in list(dict(item.get("summary") or {}).get("input_types") or [])
3500
+ )
3501
+ output_types = _unique_probe_strings(
3502
+ output_type
3503
+ for item in observed_io_contracts
3504
+ for output_type in list(dict(item.get("summary") or {}).get("output_types") or [])
3505
+ )
3506
+ return {
3507
+ "case_count": len(cases),
3508
+ "passed_case_count": passed,
3509
+ "failed_case_count": failed,
3510
+ "runtime_trace_count": runtime_trace_count,
3511
+ "streaming_trace_count": streaming_trace_count,
3512
+ "call_contract_count": call_contract_count,
3513
+ "callable_signature_present": bool(contract.get("callable_signature")),
3514
+ "observed_io_contract_count": observed_io_contract_count,
3515
+ "signature_bound_count": signature_bound_count,
3516
+ "tool_call_count": response_tool_count,
3517
+ "framework": contract.get("framework"),
3518
+ "method": contract.get("method"),
3519
+ "input_mode": contract.get("input_mode"),
3520
+ "input_keys": input_keys,
3521
+ "input_types": input_types,
3522
+ "output_types": output_types,
3523
+ "call_styles": call_styles,
3524
+ "input_kwargs_keys": input_kwargs_keys,
3525
+ "local_executable_fixture": bool(contract.get("local_executable_fixture")),
3526
+ "requires_external_service": bool(contract.get("requires_external_service")),
3527
+ "trace_runtime": bool(contract.get("trace_runtime")),
3528
+ }