agent-learning-kit 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (642) hide show
  1. agent_learning_kit-0.1.0.dist-info/METADATA +381 -0
  2. agent_learning_kit-0.1.0.dist-info/RECORD +642 -0
  3. agent_learning_kit-0.1.0.dist-info/WHEEL +4 -0
  4. agent_learning_kit-0.1.0.dist-info/entry_points.txt +5 -0
  5. agent_learning_kit-0.1.0.dist-info/licenses/LICENSE +173 -0
  6. agent_learning_kit-0.1.0.dist-info/licenses/NOTICE +7 -0
  7. fi/__init__.py +5 -0
  8. fi/alk/__init__.py +57 -0
  9. fi/alk/_facade.py +31 -0
  10. fi/alk/_module_alias.py +68 -0
  11. fi/alk/_paths.py +14 -0
  12. fi/alk/_schema.py +522 -0
  13. fi/alk/actions.py +727 -0
  14. fi/alk/bench/__init__.py +517 -0
  15. fi/alk/bench/_codeexec.py +213 -0
  16. fi/alk/bench/_coding.py +215 -0
  17. fi/alk/bench/_docker.py +237 -0
  18. fi/alk/bench/_grader.py +286 -0
  19. fi/alk/bench/_pull.py +212 -0
  20. fi/alk/bench/_voice.py +147 -0
  21. fi/alk/capabilities.py +627 -0
  22. fi/alk/cli.py +6396 -0
  23. fi/alk/config.py +130 -0
  24. fi/alk/cua_loop.py +562 -0
  25. fi/alk/evals.py +2351 -0
  26. fi/alk/extensions.py +163 -0
  27. fi/alk/harness/ARCHITECTURE.md +231 -0
  28. fi/alk/harness/DESIGN.md +246 -0
  29. fi/alk/harness/ENVIRONMENT_CONFORMANCE.md +127 -0
  30. fi/alk/harness/HOW-IT-WORKS.md +297 -0
  31. fi/alk/harness/IMPLEMENTATION_AND_VALIDATION_STATUS.md +229 -0
  32. fi/alk/harness/README.md +417 -0
  33. fi/alk/harness/__init__.py +77 -0
  34. fi/alk/harness/__main__.py +3 -0
  35. fi/alk/harness/amend.py +312 -0
  36. fi/alk/harness/artifacts.py +319 -0
  37. fi/alk/harness/authoring_entrypoint.py +189 -0
  38. fi/alk/harness/authoring_runtime_validation.py +267 -0
  39. fi/alk/harness/backends/README.md +43 -0
  40. fi/alk/harness/backends/__init__.py +122 -0
  41. fi/alk/harness/backends/base.py +241 -0
  42. fi/alk/harness/backends/claude.py +211 -0
  43. fi/alk/harness/backends/files.py +182 -0
  44. fi/alk/harness/backends/vertex_gemini.py +457 -0
  45. fi/alk/harness/background_noise.py +95 -0
  46. fi/alk/harness/build.py +385 -0
  47. fi/alk/harness/bundle.py +593 -0
  48. fi/alk/harness/bundle_author_v2.py +1831 -0
  49. fi/alk/harness/bundle_v2.py +719 -0
  50. fi/alk/harness/call_runner.py +1440 -0
  51. fi/alk/harness/callback_http_adapter.py +111 -0
  52. fi/alk/harness/catalogue.py +287 -0
  53. fi/alk/harness/chat.py +428 -0
  54. fi/alk/harness/chat_call_runner.py +506 -0
  55. fi/alk/harness/checks.py +136 -0
  56. fi/alk/harness/cli.py +1354 -0
  57. fi/alk/harness/config.py +338 -0
  58. fi/alk/harness/contract.py +718 -0
  59. fi/alk/harness/credentials.py +674 -0
  60. fi/alk/harness/data/persona_vocabulary.json +111 -0
  61. fi/alk/harness/environment.py +99 -0
  62. fi/alk/harness/environment_plan.py +168 -0
  63. fi/alk/harness/events.py +125 -0
  64. fi/alk/harness/executor.py +304 -0
  65. fi/alk/harness/folder.py +234 -0
  66. fi/alk/harness/generated_runtime.py +815 -0
  67. fi/alk/harness/github.py +72 -0
  68. fi/alk/harness/hosted_authoring_entrypoint.py +183 -0
  69. fi/alk/harness/hosted_entrypoint.py +2402 -0
  70. fi/alk/harness/hosted_scheduler.py +2218 -0
  71. fi/alk/harness/job.py +426 -0
  72. fi/alk/harness/judge.py +184 -0
  73. fi/alk/harness/livekit_source.py +50 -0
  74. fi/alk/harness/livekit_tool_trace_bootstrap.py +71 -0
  75. fi/alk/harness/observability.py +208 -0
  76. fi/alk/harness/outbound.py +3252 -0
  77. fi/alk/harness/packaging.py +515 -0
  78. fi/alk/harness/persona_guides.py +157 -0
  79. fi/alk/harness/platform.py +692 -0
  80. fi/alk/harness/process_preflight.py +764 -0
  81. fi/alk/harness/process_runtime.py +5670 -0
  82. fi/alk/harness/prove.py +425 -0
  83. fi/alk/harness/provider_import.py +703 -0
  84. fi/alk/harness/provider_lifecycle.py +392 -0
  85. fi/alk/harness/provision.py +2896 -0
  86. fi/alk/harness/reception.py +147 -0
  87. fi/alk/harness/retell_chat_call_runner.py +373 -0
  88. fi/alk/harness/run/__init__.py +296 -0
  89. fi/alk/harness/run/alk.py +184 -0
  90. fi/alk/harness/run/call.py +162 -0
  91. fi/alk/harness/run/conversation.py +264 -0
  92. fi/alk/harness/run/data/voices_by_language_and_gender.json +693 -0
  93. fi/alk/harness/run/evidence.py +195 -0
  94. fi/alk/harness/run/grade.py +598 -0
  95. fi/alk/harness/run/live.py +297 -0
  96. fi/alk/harness/run/models.py +56 -0
  97. fi/alk/harness/run/platform_evals.py +227 -0
  98. fi/alk/harness/run/sdk_voice.py +130 -0
  99. fi/alk/harness/run/simulation.py +1209 -0
  100. fi/alk/harness/run/stage.py +91 -0
  101. fi/alk/harness/run/targets.py +508 -0
  102. fi/alk/harness/run/tools.py +601 -0
  103. fi/alk/harness/run/voice.py +340 -0
  104. fi/alk/harness/runtime.py +172 -0
  105. fi/alk/harness/sandbox_server.py +2011 -0
  106. fi/alk/harness/sandbox_worker.py +44 -0
  107. fi/alk/harness/scenario.py +1048 -0
  108. fi/alk/harness/scenario_source.py +879 -0
  109. fi/alk/harness/scenario_tools.py +1143 -0
  110. fi/alk/harness/scenarios.py +915 -0
  111. fi/alk/harness/secrets.py +168 -0
  112. fi/alk/harness/service_catalog.py +97 -0
  113. fi/alk/harness/session.py +391 -0
  114. fi/alk/harness/sessions.py +372 -0
  115. fi/alk/harness/simulator.py +76 -0
  116. fi/alk/harness/simulator_voice.py +928 -0
  117. fi/alk/harness/skills/build-environment/SKILL.md +538 -0
  118. fi/alk/harness/skills/harness.md +131 -0
  119. fi/alk/harness/skills/kinds/chat.md +48 -0
  120. fi/alk/harness/skills/kinds/voice-voicemail.md +63 -0
  121. fi/alk/harness/skills/kinds/voice.md +59 -0
  122. fi/alk/harness/skills/plan-suite/SKILL.md +103 -0
  123. fi/alk/harness/skills/provision-environment/SKILL.md +136 -0
  124. fi/alk/harness/skills/run-scenarios/SKILL.md +112 -0
  125. fi/alk/harness/skills/understand-agent/SKILL.md +251 -0
  126. fi/alk/harness/skills/write-scenarios/SKILL.md +606 -0
  127. fi/alk/harness/skills/write-scenarios/references/refusals.md +28 -0
  128. fi/alk/harness/skills/write-scenarios/references/world-api.md +92 -0
  129. fi/alk/harness/source_data_invariants.py +444 -0
  130. fi/alk/harness/source_tool_evidence.py +79 -0
  131. fi/alk/harness/sources.py +253 -0
  132. fi/alk/harness/spend.py +140 -0
  133. fi/alk/harness/tool_trace_proxy.py +104 -0
  134. fi/alk/harness/tools.py +1018 -0
  135. fi/alk/harness/understand.py +169 -0
  136. fi/alk/harness/voicemail_audio.py +74 -0
  137. fi/alk/harness/world/__init__.py +33 -0
  138. fi/alk/harness/world/errors.py +68 -0
  139. fi/alk/harness/world/expectations.py +91 -0
  140. fi/alk/harness/world/handle.py +538 -0
  141. fi/alk/harness/world/kinds.py +196 -0
  142. fi/alk/harness/world/mutate.py +186 -0
  143. fi/alk/harness/world/probe.py +413 -0
  144. fi/alk/harness/world/provision.py +511 -0
  145. fi/alk/harness/world/provisioned.py +191 -0
  146. fi/alk/harness/world/runtime.py +616 -0
  147. fi/alk/harness/world/snapshot.py +288 -0
  148. fi/alk/harness/world/stores/__init__.py +305 -0
  149. fi/alk/harness/world/stores/container.py +215 -0
  150. fi/alk/harness/world/stores/inprocess.py +346 -0
  151. fi/alk/harness/world/stores/postgres.py +481 -0
  152. fi/alk/harness/world/stores/prove.py +202 -0
  153. fi/alk/harness/world/stores/sqlite.py +245 -0
  154. fi/alk/harness/world/stores/written.py +182 -0
  155. fi/alk/harness/world/tools.py +1516 -0
  156. fi/alk/harness/world/workspace.py +144 -0
  157. fi/alk/image_loop.py +453 -0
  158. fi/alk/image_perturb.py +241 -0
  159. fi/alk/improve.py +274 -0
  160. fi/alk/live/__init__.py +154 -0
  161. fi/alk/live/_attribution.py +184 -0
  162. fi/alk/live/_capture.py +264 -0
  163. fi/alk/live/_codec.py +391 -0
  164. fi/alk/live/_contract.py +134 -0
  165. fi/alk/live/_loopback.py +316 -0
  166. fi/alk/live/_perturb.py +449 -0
  167. fi/alk/live/_runner.py +386 -0
  168. fi/alk/live/_stats.py +561 -0
  169. fi/alk/live/_transcript.py +240 -0
  170. fi/alk/live/_workers/__init__.py +9 -0
  171. fi/alk/live/_workers/a2a_worker.py +316 -0
  172. fi/alk/live/_workers/langgraph_worker.py +217 -0
  173. fi/alk/live/_workers/livekit_worker.py +207 -0
  174. fi/alk/live/_workers/mcp_loopback_server.py +46 -0
  175. fi/alk/live/_workers/mcp_worker.py +158 -0
  176. fi/alk/live/_workers/pipecat_worker.py +189 -0
  177. fi/alk/live/a2a_lane.py +138 -0
  178. fi/alk/live/langgraph_lane.py +339 -0
  179. fi/alk/live/livekit_lane.py +376 -0
  180. fi/alk/live/mcp_lane.py +172 -0
  181. fi/alk/live/pipecat_lane.py +341 -0
  182. fi/alk/live/voice_redteam.py +494 -0
  183. fi/alk/loss.py +306 -0
  184. fi/alk/optimize.py +36260 -0
  185. fi/alk/practice/__init__.py +51 -0
  186. fi/alk/practice/_assess.py +103 -0
  187. fi/alk/practice/_budget.py +81 -0
  188. fi/alk/practice/_calibrate.py +69 -0
  189. fi/alk/practice/_capstone.py +86 -0
  190. fi/alk/practice/_contract.py +91 -0
  191. fi/alk/practice/_diagnose.py +79 -0
  192. fi/alk/practice/_drill.py +196 -0
  193. fi/alk/practice/_experiment.py +720 -0
  194. fi/alk/practice/_schedule.py +102 -0
  195. fi/alk/practice/_store.py +194 -0
  196. fi/alk/practice/_trainer.py +245 -0
  197. fi/alk/practice/_update.py +125 -0
  198. fi/alk/redteam.py +2621 -0
  199. fi/alk/rewardhack.py +237 -0
  200. fi/alk/simulate.py +10351 -0
  201. fi/alk/studio/__init__.py +82 -0
  202. fi/alk/studio/_bias.py +314 -0
  203. fi/alk/studio/_calibration.py +522 -0
  204. fi/alk/studio/_coverage.py +262 -0
  205. fi/alk/studio/_download.py +665 -0
  206. fi/alk/studio/_fidelity_attack.py +114 -0
  207. fi/alk/studio/_generate.py +652 -0
  208. fi/alk/studio/_library.py +370 -0
  209. fi/alk/studio/_scan.py +134 -0
  210. fi/alk/studio/_upgrade.py +42 -0
  211. fi/alk/studio/_vendor.py +172 -0
  212. fi/alk/suite.py +4200 -0
  213. fi/alk/tasks.py +828 -0
  214. fi/alk/telemetry/__init__.py +149 -0
  215. fi/alk/telemetry/_contract.py +141 -0
  216. fi/alk/telemetry/_emit.py +182 -0
  217. fi/alk/telemetry/_ledger.py +296 -0
  218. fi/alk/telemetry/_queue.py +127 -0
  219. fi/alk/telemetry/_row.py +294 -0
  220. fi/alk/telemetry/_run.py +233 -0
  221. fi/alk/telemetry/_sync.py +193 -0
  222. fi/alk/telemetry/_url.py +119 -0
  223. fi/alk/trinity.py +49397 -0
  224. fi/alk/voice_loop.py +174 -0
  225. fi/api/__init__.py +1 -0
  226. fi/api/auth.py +137 -0
  227. fi/api/types.py +29 -0
  228. fi/cli/__init__.py +9 -0
  229. fi/cli/assertions/__init__.py +25 -0
  230. fi/cli/assertions/conditions.py +76 -0
  231. fi/cli/assertions/evaluator.py +286 -0
  232. fi/cli/assertions/exit_codes.py +20 -0
  233. fi/cli/assertions/parser.py +131 -0
  234. fi/cli/assertions/reporter.py +194 -0
  235. fi/cli/commands/__init__.py +9 -0
  236. fi/cli/commands/config.py +165 -0
  237. fi/cli/commands/export.py +208 -0
  238. fi/cli/commands/init.py +112 -0
  239. fi/cli/commands/list_cmd.py +213 -0
  240. fi/cli/commands/run.py +486 -0
  241. fi/cli/commands/validate.py +173 -0
  242. fi/cli/commands/view.py +424 -0
  243. fi/cli/config/__init__.py +6 -0
  244. fi/cli/config/defaults.py +206 -0
  245. fi/cli/config/loader.py +155 -0
  246. fi/cli/config/schema.py +174 -0
  247. fi/cli/main.py +78 -0
  248. fi/cli/output/__init__.py +6 -0
  249. fi/cli/output/formatters.py +106 -0
  250. fi/cli/output/reporters.py +46 -0
  251. fi/cli/storage/__init__.py +5 -0
  252. fi/cli/storage/run_history.py +249 -0
  253. fi/cli/utils/__init__.py +5 -0
  254. fi/cli/utils/console.py +44 -0
  255. fi/evals/__init__.py +131 -0
  256. fi/evals/autoeval/__init__.py +137 -0
  257. fi/evals/autoeval/analyzer.py +211 -0
  258. fi/evals/autoeval/config.py +244 -0
  259. fi/evals/autoeval/export.py +213 -0
  260. fi/evals/autoeval/interactive.py +283 -0
  261. fi/evals/autoeval/pipeline.py +625 -0
  262. fi/evals/autoeval/prompts.py +139 -0
  263. fi/evals/autoeval/recommender.py +242 -0
  264. fi/evals/autoeval/rules.py +589 -0
  265. fi/evals/autoeval/templates.py +299 -0
  266. fi/evals/autoeval/types.py +232 -0
  267. fi/evals/core/__init__.py +16 -0
  268. fi/evals/core/cloud_registry.py +184 -0
  269. fi/evals/core/engines.py +368 -0
  270. fi/evals/core/evaluate.py +319 -0
  271. fi/evals/core/judge_prompt.py +90 -0
  272. fi/evals/core/prompt_generator.py +83 -0
  273. fi/evals/core/registry.py +57 -0
  274. fi/evals/core/result.py +55 -0
  275. fi/evals/evaluator.py +721 -0
  276. fi/evals/execution.py +168 -0
  277. fi/evals/feedback/__init__.py +32 -0
  278. fi/evals/feedback/calibrator.py +160 -0
  279. fi/evals/feedback/collector.py +214 -0
  280. fi/evals/feedback/hooks.py +81 -0
  281. fi/evals/feedback/retriever.py +128 -0
  282. fi/evals/feedback/store.py +272 -0
  283. fi/evals/feedback/types.py +99 -0
  284. fi/evals/framework/README.md +79 -0
  285. fi/evals/framework/__init__.py +267 -0
  286. fi/evals/framework/backends/Dockerfile.eval-runner +33 -0
  287. fi/evals/framework/backends/__init__.py +99 -0
  288. fi/evals/framework/backends/_container.py +141 -0
  289. fi/evals/framework/backends/_utils.py +145 -0
  290. fi/evals/framework/backends/base.py +223 -0
  291. fi/evals/framework/backends/celery_backend.py +417 -0
  292. fi/evals/framework/backends/celery_worker.py +78 -0
  293. fi/evals/framework/backends/kubernetes_backend.py +665 -0
  294. fi/evals/framework/backends/ray_backend.py +521 -0
  295. fi/evals/framework/backends/temporal.py +350 -0
  296. fi/evals/framework/backends/temporal_worker.py +126 -0
  297. fi/evals/framework/backends/thread_pool.py +286 -0
  298. fi/evals/framework/context.py +258 -0
  299. fi/evals/framework/enrichment.py +306 -0
  300. fi/evals/framework/evals/__init__.py +68 -0
  301. fi/evals/framework/evals/agentic.py +399 -0
  302. fi/evals/framework/evals/builder.py +609 -0
  303. fi/evals/framework/evals/semantic.py +142 -0
  304. fi/evals/framework/evaluator.py +647 -0
  305. fi/evals/framework/evaluators/__init__.py +22 -0
  306. fi/evals/framework/evaluators/blocking.py +347 -0
  307. fi/evals/framework/evaluators/non_blocking.py +577 -0
  308. fi/evals/framework/propagation.py +421 -0
  309. fi/evals/framework/protocols.py +385 -0
  310. fi/evals/framework/registry.py +370 -0
  311. fi/evals/framework/resilience/__init__.py +150 -0
  312. fi/evals/framework/resilience/circuit_breaker.py +309 -0
  313. fi/evals/framework/resilience/degradation.py +355 -0
  314. fi/evals/framework/resilience/health.py +505 -0
  315. fi/evals/framework/resilience/rate_limiter.py +228 -0
  316. fi/evals/framework/resilience/retry.py +274 -0
  317. fi/evals/framework/resilience/types.py +288 -0
  318. fi/evals/framework/resilience/wrapper.py +433 -0
  319. fi/evals/framework/types.py +218 -0
  320. fi/evals/guardrails/README.md +915 -0
  321. fi/evals/guardrails/__init__.py +96 -0
  322. fi/evals/guardrails/backends/__init__.py +43 -0
  323. fi/evals/guardrails/backends/azure.py +361 -0
  324. fi/evals/guardrails/backends/base.py +88 -0
  325. fi/evals/guardrails/backends/generic_llm.py +163 -0
  326. fi/evals/guardrails/backends/granite.py +216 -0
  327. fi/evals/guardrails/backends/llamaguard.py +221 -0
  328. fi/evals/guardrails/backends/local_base.py +479 -0
  329. fi/evals/guardrails/backends/openai.py +365 -0
  330. fi/evals/guardrails/backends/qwen.py +170 -0
  331. fi/evals/guardrails/backends/shieldgemma.py +154 -0
  332. fi/evals/guardrails/backends/turing.py +235 -0
  333. fi/evals/guardrails/backends/vllm_client.py +321 -0
  334. fi/evals/guardrails/backends/wildguard.py +188 -0
  335. fi/evals/guardrails/base.py +888 -0
  336. fi/evals/guardrails/config.py +221 -0
  337. fi/evals/guardrails/discovery.py +243 -0
  338. fi/evals/guardrails/gateway.py +437 -0
  339. fi/evals/guardrails/registry.py +231 -0
  340. fi/evals/guardrails/scanners/__init__.py +127 -0
  341. fi/evals/guardrails/scanners/base.py +191 -0
  342. fi/evals/guardrails/scanners/code_injection.py +243 -0
  343. fi/evals/guardrails/scanners/eval_delegate.py +574 -0
  344. fi/evals/guardrails/scanners/invisible_chars.py +351 -0
  345. fi/evals/guardrails/scanners/jailbreak.py +412 -0
  346. fi/evals/guardrails/scanners/language.py +288 -0
  347. fi/evals/guardrails/scanners/pipeline.py +260 -0
  348. fi/evals/guardrails/scanners/regex.py +311 -0
  349. fi/evals/guardrails/scanners/secrets.py +274 -0
  350. fi/evals/guardrails/scanners/topics.py +649 -0
  351. fi/evals/guardrails/scanners/urls.py +341 -0
  352. fi/evals/guardrails/types.py +96 -0
  353. fi/evals/llm/__init__.py +3 -0
  354. fi/evals/llm/base_llm_provider.py +35 -0
  355. fi/evals/llm/providers/litellm.py +70 -0
  356. fi/evals/local/__init__.py +90 -0
  357. fi/evals/local/evaluator.py +690 -0
  358. fi/evals/local/execution_mode.py +121 -0
  359. fi/evals/local/llm.py +489 -0
  360. fi/evals/local/metrics/__init__.py +19 -0
  361. fi/evals/local/registry.py +360 -0
  362. fi/evals/manager.py +1018 -0
  363. fi/evals/manager_types.py +362 -0
  364. fi/evals/metrics/__init__.py +185 -0
  365. fi/evals/metrics/agents/__init__.py +74 -0
  366. fi/evals/metrics/agents/metrics.py +693 -0
  367. fi/evals/metrics/agents/report.py +36463 -0
  368. fi/evals/metrics/agents/types.py +160 -0
  369. fi/evals/metrics/base_llm_metric.py +111 -0
  370. fi/evals/metrics/base_metric.py +138 -0
  371. fi/evals/metrics/code_security/__init__.py +305 -0
  372. fi/evals/metrics/code_security/analyzer.py +985 -0
  373. fi/evals/metrics/code_security/benchmarks/__init__.py +73 -0
  374. fi/evals/metrics/code_security/benchmarks/builtin.py +750 -0
  375. fi/evals/metrics/code_security/benchmarks/loader.py +580 -0
  376. fi/evals/metrics/code_security/benchmarks/types.py +308 -0
  377. fi/evals/metrics/code_security/detectors/__init__.py +186 -0
  378. fi/evals/metrics/code_security/detectors/base.py +394 -0
  379. fi/evals/metrics/code_security/detectors/cryptography.py +345 -0
  380. fi/evals/metrics/code_security/detectors/injection.py +744 -0
  381. fi/evals/metrics/code_security/detectors/secrets.py +287 -0
  382. fi/evals/metrics/code_security/detectors/serialization.py +192 -0
  383. fi/evals/metrics/code_security/joint_metrics.py +588 -0
  384. fi/evals/metrics/code_security/judges/__init__.py +83 -0
  385. fi/evals/metrics/code_security/judges/base.py +238 -0
  386. fi/evals/metrics/code_security/judges/dual_judge.py +534 -0
  387. fi/evals/metrics/code_security/judges/llm_judge.py +301 -0
  388. fi/evals/metrics/code_security/judges/pattern_judge.py +515 -0
  389. fi/evals/metrics/code_security/metrics.py +388 -0
  390. fi/evals/metrics/code_security/modes/__init__.py +63 -0
  391. fi/evals/metrics/code_security/modes/adversarial.py +284 -0
  392. fi/evals/metrics/code_security/modes/autocomplete.py +198 -0
  393. fi/evals/metrics/code_security/modes/base.py +283 -0
  394. fi/evals/metrics/code_security/modes/instruct.py +253 -0
  395. fi/evals/metrics/code_security/modes/repair.py +230 -0
  396. fi/evals/metrics/code_security/reports/__init__.py +57 -0
  397. fi/evals/metrics/code_security/reports/generator.py +404 -0
  398. fi/evals/metrics/code_security/reports/leaderboard.py +509 -0
  399. fi/evals/metrics/code_security/types.py +534 -0
  400. fi/evals/metrics/function_calling/__init__.py +34 -0
  401. fi/evals/metrics/function_calling/metrics.py +573 -0
  402. fi/evals/metrics/function_calling/types.py +87 -0
  403. fi/evals/metrics/hallucination/__init__.py +54 -0
  404. fi/evals/metrics/hallucination/detector.py +149 -0
  405. fi/evals/metrics/hallucination/metrics.py +390 -0
  406. fi/evals/metrics/hallucination/nli.py +253 -0
  407. fi/evals/metrics/hallucination/sentinel.py +106 -0
  408. fi/evals/metrics/hallucination/types.py +132 -0
  409. fi/evals/metrics/heuristics/aggregation_metrics.py +85 -0
  410. fi/evals/metrics/heuristics/json_metrics.py +87 -0
  411. fi/evals/metrics/heuristics/similarity_metrics.py +375 -0
  412. fi/evals/metrics/heuristics/string_metrics.py +391 -0
  413. fi/evals/metrics/llm_as_judges/__init__.py +17 -0
  414. fi/evals/metrics/llm_as_judges/custom_judge/metric.py +112 -0
  415. fi/evals/metrics/llm_as_judges/custom_judge/prompts.py +26 -0
  416. fi/evals/metrics/llm_as_judges/types.py +48 -0
  417. fi/evals/metrics/rag/__init__.py +111 -0
  418. fi/evals/metrics/rag/advanced/__init__.py +14 -0
  419. fi/evals/metrics/rag/advanced/multi_hop.py +283 -0
  420. fi/evals/metrics/rag/advanced/source_attribution.py +344 -0
  421. fi/evals/metrics/rag/generation/__init__.py +17 -0
  422. fi/evals/metrics/rag/generation/answer_relevancy.py +176 -0
  423. fi/evals/metrics/rag/generation/context_utilization.py +245 -0
  424. fi/evals/metrics/rag/generation/faithfulness.py +241 -0
  425. fi/evals/metrics/rag/generation/groundedness.py +131 -0
  426. fi/evals/metrics/rag/rag_score.py +277 -0
  427. fi/evals/metrics/rag/retrieval/__init__.py +20 -0
  428. fi/evals/metrics/rag/retrieval/context_entity_recall.py +124 -0
  429. fi/evals/metrics/rag/retrieval/context_precision.py +158 -0
  430. fi/evals/metrics/rag/retrieval/context_recall.py +106 -0
  431. fi/evals/metrics/rag/retrieval/noise_sensitivity.py +163 -0
  432. fi/evals/metrics/rag/retrieval/ranking.py +261 -0
  433. fi/evals/metrics/rag/types.py +100 -0
  434. fi/evals/metrics/rag/utils/__init__.py +62 -0
  435. fi/evals/metrics/rag/utils/claims.py +189 -0
  436. fi/evals/metrics/rag/utils/entities.py +244 -0
  437. fi/evals/metrics/rag/utils/nli.py +92 -0
  438. fi/evals/metrics/rag/utils/similarity.py +345 -0
  439. fi/evals/metrics/structured/__init__.py +114 -0
  440. fi/evals/metrics/structured/field_completeness.py +313 -0
  441. fi/evals/metrics/structured/hierarchy_score.py +366 -0
  442. fi/evals/metrics/structured/json_validation.py +190 -0
  443. fi/evals/metrics/structured/schema_compliance.py +280 -0
  444. fi/evals/metrics/structured/structured_output_score.py +298 -0
  445. fi/evals/metrics/structured/types.py +108 -0
  446. fi/evals/metrics/structured/validators/__init__.py +30 -0
  447. fi/evals/metrics/structured/validators/base.py +189 -0
  448. fi/evals/metrics/structured/validators/json_validator.py +196 -0
  449. fi/evals/metrics/structured/validators/pydantic_validator.py +178 -0
  450. fi/evals/metrics/structured/validators/yaml_validator.py +248 -0
  451. fi/evals/otel/__init__.py +266 -0
  452. fi/evals/otel/config.py +400 -0
  453. fi/evals/otel/conventions.py +463 -0
  454. fi/evals/otel/enrichment.py +371 -0
  455. fi/evals/otel/instrumentors/__init__.py +140 -0
  456. fi/evals/otel/instrumentors/anthropic.py +517 -0
  457. fi/evals/otel/instrumentors/base.py +382 -0
  458. fi/evals/otel/instrumentors/openai.py +673 -0
  459. fi/evals/otel/processors/__init__.py +36 -0
  460. fi/evals/otel/processors/base.py +473 -0
  461. fi/evals/otel/processors/cost.py +445 -0
  462. fi/evals/otel/processors/evaluation.py +559 -0
  463. fi/evals/otel/processors/llm.py +462 -0
  464. fi/evals/otel/tracer.py +506 -0
  465. fi/evals/otel/types.py +232 -0
  466. fi/evals/otel_utils.py +23 -0
  467. fi/evals/protect.py +671 -0
  468. fi/evals/protect_input_adapter.py +154 -0
  469. fi/evals/streaming/__init__.py +88 -0
  470. fi/evals/streaming/buffer.py +213 -0
  471. fi/evals/streaming/evaluator.py +551 -0
  472. fi/evals/streaming/policy.py +307 -0
  473. fi/evals/streaming/scorers.py +368 -0
  474. fi/evals/streaming/types.py +238 -0
  475. fi/evals/templates.py +472 -0
  476. fi/evals/types.py +156 -0
  477. fi/opt/__init__.py +221 -0
  478. fi/opt/_objective_scoring.py +85 -0
  479. fi/opt/base/__init__.py +11 -0
  480. fi/opt/base/base_generator.py +33 -0
  481. fi/opt/base/base_mapper.py +26 -0
  482. fi/opt/base/base_optimizer.py +45 -0
  483. fi/opt/base/evaluator.py +211 -0
  484. fi/opt/components.py +3095 -0
  485. fi/opt/datamappers/__init__.py +3 -0
  486. fi/opt/datamappers/basic_mapper.py +40 -0
  487. fi/opt/deployment.py +1021 -0
  488. fi/opt/evidence.py +4332 -0
  489. fi/opt/generators/__init__.py +3 -0
  490. fi/opt/generators/litellm.py +66 -0
  491. fi/opt/integrations/__init__.py +23 -0
  492. fi/opt/integrations/generative_suite.py +410 -0
  493. fi/opt/integrations/simulate.py +1313 -0
  494. fi/opt/mutations.py +771 -0
  495. fi/opt/observability.py +4639 -0
  496. fi/opt/optimizer_trace.py +889 -0
  497. fi/opt/optimizers/__init__.py +80 -0
  498. fi/opt/optimizers/agent.py +331 -0
  499. fi/opt/optimizers/agent_bandit.py +392 -0
  500. fi/opt/optimizers/agent_curriculum.py +635 -0
  501. fi/opt/optimizers/agent_evolution.py +894 -0
  502. fi/opt/optimizers/agent_feedback.py +1863 -0
  503. fi/opt/optimizers/agent_pareto.py +547 -0
  504. fi/opt/optimizers/agent_social_memory.py +1113 -0
  505. fi/opt/optimizers/agent_tpe.py +321 -0
  506. fi/opt/optimizers/bayesian_search.py +449 -0
  507. fi/opt/optimizers/council.py +2075 -0
  508. fi/opt/optimizers/futureagi_replay.py +799 -0
  509. fi/opt/optimizers/gepa.py +322 -0
  510. fi/opt/optimizers/metaprompt.py +243 -0
  511. fi/opt/optimizers/promptwizard.py +417 -0
  512. fi/opt/optimizers/protegi.py +329 -0
  513. fi/opt/optimizers/random_search.py +224 -0
  514. fi/opt/research.py +518 -0
  515. fi/opt/simulation.py +260 -0
  516. fi/opt/targets.py +232 -0
  517. fi/opt/types.py +66 -0
  518. fi/opt/utils/__init__.py +4 -0
  519. fi/opt/utils/early_stopping.py +266 -0
  520. fi/opt/utils/setup_logging.py +82 -0
  521. fi/simulate/__init__.py +540 -0
  522. fi/simulate/_hashing.py +35 -0
  523. fi/simulate/_logging.py +10 -0
  524. fi/simulate/adapters.py +87 -0
  525. fi/simulate/agent/__init__.py +120 -0
  526. fi/simulate/agent/browser.py +658 -0
  527. fi/simulate/agent/definition.py +587 -0
  528. fi/simulate/agent/frameworks.py +3528 -0
  529. fi/simulate/agent/generic.py +8286 -0
  530. fi/simulate/agent/import_probe.py +227 -0
  531. fi/simulate/agent/memory.py +905 -0
  532. fi/simulate/agent/mocks.py +101 -0
  533. fi/simulate/agent/multi_agent.py +361 -0
  534. fi/simulate/agent/orchestration.py +903 -0
  535. fi/simulate/agent/realtime.py +665 -0
  536. fi/simulate/agent/wrapper.py +99 -0
  537. fi/simulate/agent/wrappers/__init__.py +18 -0
  538. fi/simulate/agent/wrappers/anthropic.py +62 -0
  539. fi/simulate/agent/wrappers/gemini.py +65 -0
  540. fi/simulate/agent/wrappers/http.py +404 -0
  541. fi/simulate/agent/wrappers/langchain.py +80 -0
  542. fi/simulate/agent/wrappers/openai.py +75 -0
  543. fi/simulate/agent/wrappers/websocket.py +326 -0
  544. fi/simulate/artifacts/__init__.py +11 -0
  545. fi/simulate/artifacts/manifest.py +62 -0
  546. fi/simulate/cli.py +20560 -0
  547. fi/simulate/endpoints/__init__.py +45 -0
  548. fi/simulate/endpoints/_http_actor.py +73 -0
  549. fi/simulate/endpoints/actor_sources.py +243 -0
  550. fi/simulate/endpoints/base.py +107 -0
  551. fi/simulate/endpoints/builtins.py +10 -0
  552. fi/simulate/endpoints/callable.py +95 -0
  553. fi/simulate/endpoints/http.py +76 -0
  554. fi/simulate/endpoints/livekit.py +138 -0
  555. fi/simulate/endpoints/originators.py +132 -0
  556. fi/simulate/endpoints/profiles.py +348 -0
  557. fi/simulate/endpoints/retell.py +633 -0
  558. fi/simulate/endpoints/vapi.py +205 -0
  559. fi/simulate/endpoints/websocket.py +76 -0
  560. fi/simulate/environment.py +33026 -0
  561. fi/simulate/environments/__init__.py +11 -0
  562. fi/simulate/environments/base.py +73 -0
  563. fi/simulate/environments/chat.py +697 -0
  564. fi/simulate/environments/voice.py +212 -0
  565. fi/simulate/evaluation/__init__.py +4 -0
  566. fi/simulate/evaluation/ai_eval.py +227 -0
  567. fi/simulate/evidence/__init__.py +35 -0
  568. fi/simulate/evidence/base.py +59 -0
  569. fi/simulate/evidence/caller_observed.py +50 -0
  570. fi/simulate/evidence/livekit_instrumentation.py +51 -0
  571. fi/simulate/evidence/livekit_room.py +50 -0
  572. fi/simulate/evidence/otel.py +49 -0
  573. fi/simulate/evidence/providers/__init__.py +24 -0
  574. fi/simulate/evidence/providers/base.py +61 -0
  575. fi/simulate/evidence/providers/retell.py +376 -0
  576. fi/simulate/evidence/providers/vapi.py +426 -0
  577. fi/simulate/hosted/__init__.py +32 -0
  578. fi/simulate/hosted/child_entrypoint.py +306 -0
  579. fi/simulate/hosted/job.py +150 -0
  580. fi/simulate/hosted/targets.py +53 -0
  581. fi/simulate/instrumentation/__init__.py +5 -0
  582. fi/simulate/instrumentation/livekit/__init__.py +122 -0
  583. fi/simulate/manifest.py +1033 -0
  584. fi/simulate/matrix_cli.py +165 -0
  585. fi/simulate/realtime/__init__.py +40 -0
  586. fi/simulate/realtime/events.py +107 -0
  587. fi/simulate/realtime/media.py +61 -0
  588. fi/simulate/realtime/session.py +91 -0
  589. fi/simulate/recording/__init__.py +5 -0
  590. fi/simulate/recording/room_recorder.py +326 -0
  591. fi/simulate/registry.py +185 -0
  592. fi/simulate/results/__init__.py +9 -0
  593. fi/simulate/results/base.py +18 -0
  594. fi/simulate/results/filesystem.py +71 -0
  595. fi/simulate/results/futureagi.py +1340 -0
  596. fi/simulate/runtime/__init__.py +85 -0
  597. fi/simulate/runtime/capabilities.py +40 -0
  598. fi/simulate/runtime/events.py +63 -0
  599. fi/simulate/runtime/failures.py +25 -0
  600. fi/simulate/runtime/ids.py +34 -0
  601. fi/simulate/runtime/plan.py +70 -0
  602. fi/simulate/runtime/planner.py +102 -0
  603. fi/simulate/runtime/report.py +174 -0
  604. fi/simulate/runtime/run.py +75 -0
  605. fi/simulate/runtime/runner.py +333 -0
  606. fi/simulate/runtime/spec.py +186 -0
  607. fi/simulate/simulation/__init__.py +30 -0
  608. fi/simulate/simulation/behavior_policy.py +425 -0
  609. fi/simulate/simulation/bridge/__init__.py +9 -0
  610. fi/simulate/simulation/bridge/audio.py +29 -0
  611. fi/simulate/simulation/bridge/connector.py +46 -0
  612. fi/simulate/simulation/bridge/livekit.py +252 -0
  613. fi/simulate/simulation/bridge/retell.py +188 -0
  614. fi/simulate/simulation/bridge/vapi.py +177 -0
  615. fi/simulate/simulation/contract.py +419 -0
  616. fi/simulate/simulation/engines/__init__.py +12 -0
  617. fi/simulate/simulation/engines/base.py +21 -0
  618. fi/simulate/simulation/engines/cloud.py +517 -0
  619. fi/simulate/simulation/engines/livekit.py +4167 -0
  620. fi/simulate/simulation/engines/local_text.py +89 -0
  621. fi/simulate/simulation/fidelity.py +374 -0
  622. fi/simulate/simulation/gemini_tts_stream.py +110 -0
  623. fi/simulate/simulation/generator.py +91 -0
  624. fi/simulate/simulation/goal_machine.py +185 -0
  625. fi/simulate/simulation/livekit_models.py +467 -0
  626. fi/simulate/simulation/matrix.py +170 -0
  627. fi/simulate/simulation/models.py +279 -0
  628. fi/simulate/simulation/runner.py +153 -0
  629. fi/simulate/simulation/synthetic.py +880 -0
  630. fi/simulate/simulation/voice_prompt.py +502 -0
  631. fi/simulate/simulator/__init__.py +55 -0
  632. fi/simulate/simulator/builtins.py +53 -0
  633. fi/simulate/suite.py +1288 -0
  634. fi/simulate/utils/routes.py +164 -0
  635. fi/simulate/voice.py +225 -0
  636. fi/simulate/voice_cli.py +182 -0
  637. fi/utils/__init__.py +1 -0
  638. fi/utils/constants.py +14 -0
  639. fi/utils/errors.py +200 -0
  640. fi/utils/executor.py +26 -0
  641. fi/utils/routes.py +119 -0
  642. fi/utils/utils.py +17 -0
@@ -0,0 +1,744 @@
1
+ """
2
+ Injection Vulnerability Detectors.
3
+
4
+ Detects various injection vulnerabilities:
5
+ - CWE-89: SQL Injection
6
+ - CWE-78: OS Command Injection
7
+ - CWE-79: Cross-Site Scripting (XSS)
8
+ - CWE-94: Code Injection
9
+ - CWE-611: XXE Injection
10
+ - CWE-918: SSRF
11
+ """
12
+
13
+ import re
14
+ from typing import List, Optional
15
+
16
+ from .base import BaseDetector, register_detector
17
+ from ..types import (
18
+ SecurityFinding,
19
+ Severity,
20
+ VulnerabilityCategory,
21
+ )
22
+ from ..analyzer import AnalysisResult
23
+
24
+
25
+ @register_detector("sql_injection")
26
+ class SQLInjectionDetector(BaseDetector):
27
+ """
28
+ Detects SQL injection vulnerabilities (CWE-89).
29
+
30
+ Identifies:
31
+ - String concatenation in SQL queries
32
+ - f-string/format string in SQL
33
+ - Raw string interpolation
34
+ - Missing parameterized queries
35
+
36
+ Examples of vulnerable code:
37
+ query = "SELECT * FROM users WHERE id = " + user_id
38
+ query = f"SELECT * FROM users WHERE name = '{name}'"
39
+ cursor.execute("SELECT * FROM users WHERE id = %s" % user_id)
40
+ """
41
+
42
+ name = "sql_injection"
43
+ cwe_ids = ["CWE-89"]
44
+ category = VulnerabilityCategory.INJECTION
45
+ description = "SQL Injection via string concatenation or interpolation"
46
+ default_severity = Severity.HIGH
47
+
48
+ # SQL keywords to look for
49
+ SQL_KEYWORDS = [
50
+ "select", "insert", "update", "delete", "drop", "create",
51
+ "alter", "exec", "execute", "union", "truncate", "grant",
52
+ ]
53
+
54
+ # Patterns indicating unsafe SQL construction
55
+ UNSAFE_PATTERNS = {
56
+ "python": [
57
+ # f-string with SQL
58
+ (r'f["\'].*(?:' + "|".join(SQL_KEYWORDS) + r').*\{', "f-string SQL interpolation"),
59
+ # String concatenation
60
+ (r'["\'].*(?:' + "|".join(SQL_KEYWORDS) + r').*["\']\s*\+', "String concatenation in SQL"),
61
+ (r'\+\s*["\'].*(?:' + "|".join(SQL_KEYWORDS) + r')', "String concatenation in SQL"),
62
+ # % formatting
63
+ (r'["\'].*(?:' + "|".join(SQL_KEYWORDS) + r').*%s.*["\']\s*%', "%-format SQL interpolation"),
64
+ # .format()
65
+ (r'["\'].*(?:' + "|".join(SQL_KEYWORDS) + r').*\{\}.*["\']\.format\(', ".format() SQL interpolation"),
66
+ ],
67
+ "javascript": [
68
+ # Template literal
69
+ (r'`.*(?:' + "|".join(SQL_KEYWORDS) + r').*\$\{', "Template literal SQL interpolation"),
70
+ # String concatenation
71
+ (r'["\'].*(?:' + "|".join(SQL_KEYWORDS) + r').*["\']\s*\+', "String concatenation in SQL"),
72
+ ],
73
+ "java": [
74
+ # String concatenation
75
+ (r'".*(?:' + "|".join(SQL_KEYWORDS) + r').*"\s*\+', "String concatenation in SQL"),
76
+ # String.format
77
+ (r'String\.format\s*\(\s*".*(?:' + "|".join(SQL_KEYWORDS) + r')', "String.format SQL"),
78
+ ],
79
+ "php": [
80
+ # Variable interpolation
81
+ (r'".*(?:' + "|".join(SQL_KEYWORDS) + r').*\$', "Variable interpolation in SQL"),
82
+ # String concatenation
83
+ (r'["\'].*(?:' + "|".join(SQL_KEYWORDS) + r').*["\']\s*\.', "String concatenation in SQL"),
84
+ ],
85
+ }
86
+
87
+ # Safe patterns (parameterized queries)
88
+ SAFE_PATTERNS = {
89
+ "python": [
90
+ r"execute\s*\([^,]+,\s*[\[\(]", # execute(query, (params,))
91
+ r"executemany\s*\([^,]+,", # executemany(query, params)
92
+ r"\.filter\s*\(", # ORM filter
93
+ r"\.where\s*\(", # ORM where
94
+ ],
95
+ "javascript": [
96
+ r"\?\s*,", # Prepared statement placeholder
97
+ r"query\s*\([^,]+,\s*\[", # query(sql, [params])
98
+ ],
99
+ "java": [
100
+ r"prepareStatement", # Prepared statement
101
+ r"setString\s*\(", # Parameter binding
102
+ r"setInt\s*\(",
103
+ ],
104
+ }
105
+
106
+ def detect(
107
+ self,
108
+ code: str,
109
+ language: str,
110
+ analysis: Optional[AnalysisResult] = None,
111
+ ) -> List[SecurityFinding]:
112
+ """Detect SQL injection vulnerabilities."""
113
+ if not self.enabled:
114
+ return []
115
+
116
+ findings = []
117
+ lines = code.split("\n")
118
+ lang_lower = language.lower()
119
+
120
+ # Get patterns for this language
121
+ unsafe_patterns = self.UNSAFE_PATTERNS.get(lang_lower, self.UNSAFE_PATTERNS["python"])
122
+ safe_patterns = self.SAFE_PATTERNS.get(lang_lower, [])
123
+
124
+ for i, line in enumerate(lines, 1):
125
+ line_lower = line.lower()
126
+
127
+ # Skip if no SQL keywords
128
+ if not any(kw in line_lower for kw in self.SQL_KEYWORDS):
129
+ continue
130
+
131
+ # Skip if matches safe pattern
132
+ if any(re.search(p, line, re.IGNORECASE) for p in safe_patterns):
133
+ continue
134
+
135
+ # Check unsafe patterns
136
+ for pattern, description in unsafe_patterns:
137
+ if re.search(pattern, line, re.IGNORECASE):
138
+ # Determine confidence based on context
139
+ confidence = 0.8
140
+ if analysis and analysis.functions:
141
+ # Higher confidence if in a route/endpoint
142
+ func = self._find_function(analysis, i)
143
+ if func and any(d in ["route", "get", "post", "api"] for d in func.decorators):
144
+ confidence = 0.95
145
+
146
+ findings.append(self.create_finding(
147
+ vulnerability_type="SQL Injection",
148
+ description=f"{description}: {line.strip()[:60]}...",
149
+ line=i,
150
+ snippet=line.strip()[:100],
151
+ confidence=confidence,
152
+ suggested_fix="Use parameterized queries: cursor.execute(query, (param,))",
153
+ ))
154
+ break # One finding per line
155
+
156
+ return findings
157
+
158
+ def _find_function(self, analysis: AnalysisResult, line: int):
159
+ """Find the function containing the given line."""
160
+ for func in analysis.functions:
161
+ if func.line <= line and (func.end_line is None or func.end_line >= line):
162
+ return func
163
+ return None
164
+
165
+
166
+ @register_detector("command_injection")
167
+ class CommandInjectionDetector(BaseDetector):
168
+ """
169
+ Detects OS command injection vulnerabilities (CWE-78).
170
+
171
+ Identifies:
172
+ - os.system() with variable input
173
+ - subprocess with shell=True
174
+ - eval/exec with user input
175
+ - Backtick execution
176
+
177
+ Examples of vulnerable code:
178
+ os.system("ping " + host)
179
+ subprocess.call(cmd, shell=True)
180
+ eval(user_input)
181
+ """
182
+
183
+ name = "command_injection"
184
+ cwe_ids = ["CWE-78"]
185
+ category = VulnerabilityCategory.INJECTION
186
+ description = "OS Command Injection via shell execution"
187
+ default_severity = Severity.CRITICAL
188
+
189
+ DANGEROUS_FUNCTIONS = {
190
+ "python": {
191
+ "os.system": "Shell command execution",
192
+ "os.popen": "Shell command execution",
193
+ "subprocess.call": "Subprocess execution",
194
+ "subprocess.run": "Subprocess execution",
195
+ "subprocess.Popen": "Subprocess execution",
196
+ "subprocess.check_output": "Subprocess execution",
197
+ "subprocess.check_call": "Subprocess execution",
198
+ "commands.getoutput": "Shell command execution",
199
+ "commands.getstatusoutput": "Shell command execution",
200
+ "eval": "Code evaluation",
201
+ "exec": "Code execution",
202
+ },
203
+ "javascript": {
204
+ "eval": "Code evaluation",
205
+ "exec": "Shell command execution",
206
+ "execSync": "Shell command execution",
207
+ "spawn": "Process spawn",
208
+ "spawnSync": "Process spawn",
209
+ "execFile": "File execution",
210
+ "child_process.exec": "Shell command execution",
211
+ "child_process.execSync": "Shell command execution",
212
+ },
213
+ "java": {
214
+ "Runtime.exec": "Shell command execution",
215
+ "Runtime.getRuntime().exec": "Shell command execution",
216
+ "ProcessBuilder": "Process execution",
217
+ },
218
+ "php": {
219
+ "exec": "Shell command execution",
220
+ "shell_exec": "Shell command execution",
221
+ "system": "Shell command execution",
222
+ "passthru": "Shell command execution",
223
+ "popen": "Shell command execution",
224
+ "proc_open": "Process execution",
225
+ },
226
+ }
227
+
228
+ # Patterns indicating variable/user input
229
+ VARIABLE_INPUT_PATTERNS = [
230
+ r"\([^)]*\+", # function(... +
231
+ r"\(f['\"]", # function(f"...
232
+ r"\([^)]*\{", # function(...{
233
+ r"\([^)]*\$", # function(...$
234
+ r"\([^)]*%", # function(... %
235
+ r"shell\s*=\s*True", # shell=True
236
+ r"`[^`]*\$\{", # Backtick with ${...}
237
+ ]
238
+
239
+ # Backtick execution patterns (shell command in template literal)
240
+ BACKTICK_PATTERNS = [
241
+ (r"`[^`]*\$\{[^}]+\}[^`]*`", "backtick template with variable"),
242
+ ]
243
+
244
+ def detect(
245
+ self,
246
+ code: str,
247
+ language: str,
248
+ analysis: Optional[AnalysisResult] = None,
249
+ ) -> List[SecurityFinding]:
250
+ """Detect command injection vulnerabilities."""
251
+ if not self.enabled:
252
+ return []
253
+
254
+ findings = []
255
+ lines = code.split("\n")
256
+ lang_lower = language.lower()
257
+
258
+ dangerous_funcs = self.DANGEROUS_FUNCTIONS.get(lang_lower, {})
259
+
260
+ for i, line in enumerate(lines, 1):
261
+ # Check for backtick command execution (JavaScript)
262
+ if lang_lower in ["javascript", "js", "typescript", "ts"]:
263
+ for pattern, description in self.BACKTICK_PATTERNS:
264
+ if re.search(pattern, line):
265
+ findings.append(self.create_finding(
266
+ vulnerability_type="Command Injection",
267
+ description=f"Shell command via {description}",
268
+ line=i,
269
+ snippet=line.strip()[:100],
270
+ severity=Severity.CRITICAL,
271
+ confidence=0.85,
272
+ suggested_fix="Avoid template literal shell commands. Use proper command builders.",
273
+ ))
274
+ break
275
+
276
+ for func, description in dangerous_funcs.items():
277
+ # Check if function is called
278
+ func_pattern = rf"\b{re.escape(func.split('.')[-1])}\s*\("
279
+ if not re.search(func_pattern, line):
280
+ continue
281
+
282
+ # Check for variable input or shell=True
283
+ has_variable_input = any(
284
+ re.search(p, line) for p in self.VARIABLE_INPUT_PATTERNS
285
+ )
286
+
287
+ if has_variable_input or "shell=True" in line.lower():
288
+ severity = Severity.CRITICAL
289
+ confidence = 0.9 if has_variable_input else 0.85
290
+
291
+ findings.append(self.create_finding(
292
+ vulnerability_type="Command Injection",
293
+ description=f"{description} with variable input: {func}",
294
+ line=i,
295
+ snippet=line.strip()[:100],
296
+ severity=severity,
297
+ confidence=confidence,
298
+ suggested_fix="Use subprocess with shell=False and list arguments. Validate/sanitize input.",
299
+ ))
300
+
301
+ return findings
302
+
303
+
304
+ @register_detector("xss")
305
+ class XSSDetector(BaseDetector):
306
+ """
307
+ Detects Cross-Site Scripting vulnerabilities (CWE-79).
308
+
309
+ Identifies:
310
+ - innerHTML assignments with variable
311
+ - document.write with variable
312
+ - Unescaped template rendering
313
+ - Response write without encoding
314
+
315
+ Examples of vulnerable code:
316
+ element.innerHTML = userInput
317
+ document.write(data)
318
+ res.send("<div>" + name + "</div>")
319
+ """
320
+
321
+ name = "xss"
322
+ cwe_ids = ["CWE-79"]
323
+ category = VulnerabilityCategory.INJECTION
324
+ description = "Cross-Site Scripting via unescaped output"
325
+ default_severity = Severity.MEDIUM
326
+ supported_languages = {"javascript", "js", "typescript", "ts", "python", "php"}
327
+
328
+ XSS_SINKS = {
329
+ "javascript": [
330
+ (r"\.innerHTML\s*=", "innerHTML assignment"),
331
+ (r"\.outerHTML\s*=", "outerHTML assignment"),
332
+ (r"document\.write\s*\(", "document.write"),
333
+ (r"document\.writeln\s*\(", "document.writeln"),
334
+ (r"\.insertAdjacentHTML\s*\(", "insertAdjacentHTML"),
335
+ (r"eval\s*\(", "eval"),
336
+ ],
337
+ "python": [
338
+ (r"render_template_string\s*\(", "render_template_string"),
339
+ (r"Markup\s*\(", "Markup without escape"),
340
+ (r"\|safe\b", "Jinja2 safe filter"),
341
+ # Flask/Django returning HTML with f-strings
342
+ (r'return\s+f["\'].*<[a-zA-Z]', "returning HTML with f-string"),
343
+ (r'return\s+["\']<.*\+', "returning HTML with concatenation"),
344
+ ],
345
+ "php": [
346
+ (r"echo\s+\$", "echo with variable"),
347
+ (r"print\s+\$", "print with variable"),
348
+ ],
349
+ }
350
+
351
+ def detect(
352
+ self,
353
+ code: str,
354
+ language: str,
355
+ analysis: Optional[AnalysisResult] = None,
356
+ ) -> List[SecurityFinding]:
357
+ """Detect XSS vulnerabilities."""
358
+ if not self.enabled or not self.supports_language(language):
359
+ return []
360
+
361
+ findings = []
362
+ lines = code.split("\n")
363
+ lang_lower = language.lower()
364
+
365
+ # Normalize language
366
+ if lang_lower in ["ts", "typescript"]:
367
+ lang_lower = "javascript"
368
+ if lang_lower == "js":
369
+ lang_lower = "javascript"
370
+
371
+ sinks = self.XSS_SINKS.get(lang_lower, [])
372
+
373
+ for i, line in enumerate(lines, 1):
374
+ for pattern, description in sinks:
375
+ if re.search(pattern, line, re.IGNORECASE):
376
+ # Check if there's variable involvement
377
+ has_variable = bool(re.search(r"[\+\$\{]|f['\"]|\bvar\b|\blet\b|\bconst\b", line))
378
+
379
+ if has_variable or "innerHTML" in line:
380
+ findings.append(self.create_finding(
381
+ vulnerability_type="Cross-Site Scripting (XSS)",
382
+ description=f"Potential XSS via {description}",
383
+ line=i,
384
+ snippet=line.strip()[:100],
385
+ confidence=0.75 if has_variable else 0.6,
386
+ suggested_fix="Escape user input before rendering. Use textContent instead of innerHTML.",
387
+ ))
388
+ break
389
+
390
+ return findings
391
+
392
+
393
+ @register_detector("code_injection")
394
+ class CodeInjectionDetector(BaseDetector):
395
+ """
396
+ Detects code injection vulnerabilities (CWE-94).
397
+
398
+ Identifies:
399
+ - eval() with user input
400
+ - exec() with user input
401
+ - Dynamic code compilation
402
+ - Template injection
403
+
404
+ Examples of vulnerable code:
405
+ eval(user_input)
406
+ exec(f"print({value})")
407
+ compile(code, '<string>', 'exec')
408
+ """
409
+
410
+ name = "code_injection"
411
+ cwe_ids = ["CWE-94"]
412
+ category = VulnerabilityCategory.INJECTION
413
+ description = "Code Injection via dynamic execution"
414
+ default_severity = Severity.CRITICAL
415
+
416
+ CODE_EXECUTION_PATTERNS = {
417
+ "python": [
418
+ # Any eval/exec call is dangerous if passed variable
419
+ (r"\beval\s*\(\s*[a-zA-Z_]", "eval with variable", Severity.CRITICAL),
420
+ (r"\bexec\s*\(\s*[a-zA-Z_]", "exec with variable", Severity.CRITICAL),
421
+ (r"\beval\s*\([^)]*[\+\$\{]", "eval with concatenation", Severity.CRITICAL),
422
+ (r"\bexec\s*\([^)]*[\+\$\{]", "exec with concatenation", Severity.CRITICAL),
423
+ (r"\beval\s*\(f['\"]", "eval with f-string", Severity.CRITICAL),
424
+ (r"\bexec\s*\(f['\"]", "exec with f-string", Severity.CRITICAL),
425
+ (r"\bcompile\s*\([^)]+,\s*['\"]<string>['\"]", "compile", Severity.HIGH),
426
+ ],
427
+ "javascript": [
428
+ # Any eval call is dangerous
429
+ (r"\beval\s*\(\s*[a-zA-Z_]", "eval with variable", Severity.CRITICAL),
430
+ (r"\beval\s*\([^)]*[\+\$\{]", "eval with concatenation", Severity.CRITICAL),
431
+ (r"\bnew\s+Function\s*\([^)]*[\+\$\{]", "Function constructor", Severity.CRITICAL),
432
+ (r"setTimeout\s*\([^)]*[\+\$\{]", "setTimeout with string", Severity.MEDIUM),
433
+ (r"setInterval\s*\([^)]*[\+\$\{]", "setInterval with string", Severity.MEDIUM),
434
+ ],
435
+ }
436
+
437
+ def detect(
438
+ self,
439
+ code: str,
440
+ language: str,
441
+ analysis: Optional[AnalysisResult] = None,
442
+ ) -> List[SecurityFinding]:
443
+ """Detect code injection vulnerabilities."""
444
+ if not self.enabled:
445
+ return []
446
+
447
+ findings = []
448
+ lines = code.split("\n")
449
+ lang_lower = language.lower()
450
+
451
+ patterns = self.CODE_EXECUTION_PATTERNS.get(lang_lower, [])
452
+
453
+ for i, line in enumerate(lines, 1):
454
+ for pattern, description, severity in patterns:
455
+ if re.search(pattern, line, re.IGNORECASE):
456
+ findings.append(self.create_finding(
457
+ vulnerability_type="Code Injection",
458
+ description=f"Dynamic code execution via {description}",
459
+ line=i,
460
+ snippet=line.strip()[:100],
461
+ severity=severity,
462
+ confidence=0.9,
463
+ suggested_fix="Avoid eval/exec. Use safer alternatives like ast.literal_eval for Python.",
464
+ ))
465
+ break # One finding per line
466
+
467
+ return findings
468
+
469
+
470
+ @register_detector("xxe")
471
+ class XXEDetector(BaseDetector):
472
+ """
473
+ Detects XML External Entity vulnerabilities (CWE-611).
474
+
475
+ Identifies:
476
+ - XML parsing without disabling external entities
477
+ - Unsafe XML parser configurations
478
+
479
+ Examples of vulnerable code:
480
+ etree.parse(xml_file)
481
+ xml.etree.ElementTree.fromstring(xml)
482
+ DocumentBuilderFactory.newInstance() # without setFeature
483
+ """
484
+
485
+ name = "xxe"
486
+ cwe_ids = ["CWE-611"]
487
+ category = VulnerabilityCategory.INJECTION
488
+ description = "XML External Entity (XXE) Injection"
489
+ default_severity = Severity.HIGH
490
+ supported_languages = {"python", "java"}
491
+
492
+ XXE_PATTERNS = {
493
+ "python": [
494
+ (r"etree\.parse\s*\(", "etree.parse without defuse"),
495
+ (r"etree\.fromstring\s*\(", "etree.fromstring without defuse"),
496
+ (r"ET\.parse\s*\(", "ET.parse without defuse"),
497
+ (r"ET\.fromstring\s*\(", "ET.fromstring without defuse"),
498
+ (r"ElementTree\.parse\s*\(", "ElementTree.parse"),
499
+ (r"ElementTree\.fromstring\s*\(", "ElementTree.fromstring"),
500
+ (r"xml\.dom\.minidom\.parse\s*\(", "minidom.parse"),
501
+ (r"xml\.sax\.parse\s*\(", "sax.parse"),
502
+ (r"lxml\.etree\.parse\s*\(", "lxml.etree.parse"),
503
+ (r"lxml\.etree\.fromstring\s*\(", "lxml.etree.fromstring"),
504
+ (r"XMLParser\s*\(", "XMLParser without defuse"),
505
+ ],
506
+ "java": [
507
+ (r"DocumentBuilderFactory\.newInstance\s*\(", "DocumentBuilderFactory without secure config"),
508
+ (r"SAXParserFactory\.newInstance\s*\(", "SAXParserFactory without secure config"),
509
+ (r"XMLInputFactory\.newInstance\s*\(", "XMLInputFactory without secure config"),
510
+ ],
511
+ }
512
+
513
+ SAFE_PATTERNS = {
514
+ "python": [
515
+ r"defusedxml",
516
+ r"resolve_entities\s*=\s*False",
517
+ ],
518
+ "java": [
519
+ r"setFeature.*XMLConstants\.FEATURE_SECURE_PROCESSING",
520
+ r"setFeature.*disallow-doctype-decl",
521
+ ],
522
+ }
523
+
524
+ def detect(
525
+ self,
526
+ code: str,
527
+ language: str,
528
+ analysis: Optional[AnalysisResult] = None,
529
+ ) -> List[SecurityFinding]:
530
+ """Detect XXE vulnerabilities."""
531
+ if not self.enabled or not self.supports_language(language):
532
+ return []
533
+
534
+ findings = []
535
+ lang_lower = language.lower()
536
+
537
+ # Check if safe patterns are present in entire code
538
+ safe_patterns = self.SAFE_PATTERNS.get(lang_lower, [])
539
+ has_safe_config = any(re.search(p, code, re.IGNORECASE) for p in safe_patterns)
540
+
541
+ if has_safe_config:
542
+ return []
543
+
544
+ lines = code.split("\n")
545
+ patterns = self.XXE_PATTERNS.get(lang_lower, [])
546
+
547
+ for i, line in enumerate(lines, 1):
548
+ for pattern, description in patterns:
549
+ if re.search(pattern, line, re.IGNORECASE):
550
+ findings.append(self.create_finding(
551
+ vulnerability_type="XML External Entity (XXE)",
552
+ description=f"Potentially unsafe XML parsing: {description}",
553
+ line=i,
554
+ snippet=line.strip()[:100],
555
+ confidence=0.75,
556
+ suggested_fix="Use defusedxml (Python) or disable external entities (Java).",
557
+ ))
558
+
559
+ return findings
560
+
561
+
562
+ @register_detector("ssrf")
563
+ class SSRFDetector(BaseDetector):
564
+ """
565
+ Detects Server-Side Request Forgery vulnerabilities (CWE-918).
566
+
567
+ Identifies:
568
+ - HTTP requests with user-controlled URLs
569
+ - URL construction with user input
570
+
571
+ Examples of vulnerable code:
572
+ requests.get(user_url)
573
+ urllib.request.urlopen(url)
574
+ fetch(userInput)
575
+ """
576
+
577
+ name = "ssrf"
578
+ cwe_ids = ["CWE-918"]
579
+ category = VulnerabilityCategory.INJECTION
580
+ description = "Server-Side Request Forgery"
581
+ default_severity = Severity.HIGH
582
+
583
+ HTTP_FUNCTIONS = {
584
+ "python": [
585
+ "requests.get", "requests.post", "requests.put", "requests.delete",
586
+ "requests.head", "requests.patch", "requests.request",
587
+ "urllib.request.urlopen", "urllib2.urlopen",
588
+ "httplib.HTTPConnection", "http.client.HTTPConnection",
589
+ "aiohttp.ClientSession", "httpx.get", "httpx.post",
590
+ ],
591
+ "javascript": [
592
+ "fetch", "axios.get", "axios.post", "axios.put",
593
+ "http.get", "https.get", "request",
594
+ ],
595
+ "java": [
596
+ "URL.openConnection", "HttpURLConnection",
597
+ "HttpClient.execute", "RestTemplate",
598
+ ],
599
+ }
600
+
601
+ def detect(
602
+ self,
603
+ code: str,
604
+ language: str,
605
+ analysis: Optional[AnalysisResult] = None,
606
+ ) -> List[SecurityFinding]:
607
+ """Detect SSRF vulnerabilities."""
608
+ if not self.enabled:
609
+ return []
610
+
611
+ findings = []
612
+ lines = code.split("\n")
613
+ lang_lower = language.lower()
614
+
615
+ functions = self.HTTP_FUNCTIONS.get(lang_lower, [])
616
+
617
+ for i, line in enumerate(lines, 1):
618
+ for func in functions:
619
+ func_name = func.split(".")[-1]
620
+ pattern = rf"\b{re.escape(func_name)}\s*\("
621
+
622
+ if re.search(pattern, line, re.IGNORECASE):
623
+ # Check if URL appears to be user-controlled (variable passed)
624
+ # Match: function(variable) or function(url) - any non-string first arg
625
+ has_variable = bool(re.search(
626
+ rf"{re.escape(func_name)}\s*\(\s*[a-zA-Z_][a-zA-Z0-9_]*\s*[,\)]",
627
+ line
628
+ ))
629
+ # Also check for concatenation or f-strings
630
+ has_concat = bool(re.search(r"[\+\$\{]|f['\"]", line))
631
+
632
+ if has_variable or has_concat:
633
+ findings.append(self.create_finding(
634
+ vulnerability_type="Server-Side Request Forgery (SSRF)",
635
+ description=f"HTTP request with potentially user-controlled URL: {func}",
636
+ line=i,
637
+ snippet=line.strip()[:100],
638
+ confidence=0.7 if has_variable else 0.6,
639
+ suggested_fix="Validate and whitelist allowed URLs. Don't allow user control of full URL.",
640
+ ))
641
+ break
642
+
643
+ return findings
644
+
645
+
646
+ @register_detector("path_traversal")
647
+ class PathTraversalDetector(BaseDetector):
648
+ """
649
+ Detects Path Traversal vulnerabilities (CWE-22).
650
+
651
+ Identifies:
652
+ - File operations with user-controlled paths
653
+ - Lack of path sanitization
654
+
655
+ Examples of vulnerable code:
656
+ open(user_path)
657
+ os.path.join(base, user_input)
658
+ fs.readFile(userPath)
659
+ """
660
+
661
+ name = "path_traversal"
662
+ cwe_ids = ["CWE-22"]
663
+ category = VulnerabilityCategory.INPUT_VALIDATION
664
+ description = "Path Traversal via unsanitized file paths"
665
+ default_severity = Severity.HIGH
666
+
667
+ FILE_OPERATIONS = {
668
+ "python": [
669
+ "open", "file", "os.path.join", "pathlib.Path",
670
+ "shutil.copy", "shutil.move", "os.remove", "os.unlink",
671
+ "os.rename", "os.listdir", "os.makedirs", "send_file",
672
+ "send_from_directory",
673
+ ],
674
+ "javascript": [
675
+ "fs.readFile", "fs.writeFile", "fs.unlink", "fs.mkdir",
676
+ "fs.readFileSync", "fs.writeFileSync", "path.join",
677
+ "fs.createReadStream", "fs.createWriteStream",
678
+ ],
679
+ "java": [
680
+ "FileInputStream", "FileOutputStream", "FileReader",
681
+ "FileWriter", "File", "Files.readAllBytes",
682
+ ],
683
+ }
684
+
685
+ def detect(
686
+ self,
687
+ code: str,
688
+ language: str,
689
+ analysis: Optional[AnalysisResult] = None,
690
+ ) -> List[SecurityFinding]:
691
+ """Detect path traversal vulnerabilities."""
692
+ if not self.enabled:
693
+ return []
694
+
695
+ findings = []
696
+ lines = code.split("\n")
697
+ lang_lower = language.lower()
698
+
699
+ operations = self.FILE_OPERATIONS.get(lang_lower, [])
700
+
701
+ for i, line in enumerate(lines, 1):
702
+ for op in operations:
703
+ op_name = op.split(".")[-1]
704
+ pattern = rf"\b{re.escape(op_name)}\s*\("
705
+
706
+ if re.search(pattern, line, re.IGNORECASE):
707
+ # Check for variable input - any non-string literal arg
708
+ has_variable = bool(re.search(
709
+ rf"{re.escape(op_name)}\s*\(\s*[a-zA-Z_][a-zA-Z0-9_]*\s*[,\)]",
710
+ line
711
+ ))
712
+
713
+ # Also check for concatenation or f-strings
714
+ has_concat = bool(re.search(r"[\+\$\{]|f['\"]", line))
715
+
716
+ # Check for path traversal indicators
717
+ has_traversal = bool(re.search(r"\.\./|\.\.\\", line))
718
+
719
+ if has_variable or has_concat or has_traversal:
720
+ confidence = 0.85 if has_traversal else (0.7 if has_variable else 0.6)
721
+
722
+ findings.append(self.create_finding(
723
+ vulnerability_type="Path Traversal",
724
+ description=f"File operation with potentially unsafe path: {op}",
725
+ line=i,
726
+ snippet=line.strip()[:100],
727
+ confidence=confidence,
728
+ suggested_fix="Validate and sanitize file paths. Use os.path.realpath() and check against allowed directories.",
729
+ ))
730
+ break
731
+
732
+ return findings
733
+
734
+
735
+ # Export all detectors
736
+ __all__ = [
737
+ "SQLInjectionDetector",
738
+ "CommandInjectionDetector",
739
+ "XSSDetector",
740
+ "CodeInjectionDetector",
741
+ "XXEDetector",
742
+ "SSRFDetector",
743
+ "PathTraversalDetector",
744
+ ]