agno 0.1.2__py3-none-any.whl → 2.3.13__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (723) hide show
  1. agno/__init__.py +8 -0
  2. agno/agent/__init__.py +44 -5
  3. agno/agent/agent.py +10531 -2975
  4. agno/api/agent.py +14 -53
  5. agno/api/api.py +7 -46
  6. agno/api/evals.py +22 -0
  7. agno/api/os.py +17 -0
  8. agno/api/routes.py +6 -25
  9. agno/api/schemas/__init__.py +9 -0
  10. agno/api/schemas/agent.py +6 -9
  11. agno/api/schemas/evals.py +16 -0
  12. agno/api/schemas/os.py +14 -0
  13. agno/api/schemas/team.py +10 -10
  14. agno/api/schemas/utils.py +21 -0
  15. agno/api/schemas/workflows.py +16 -0
  16. agno/api/settings.py +53 -0
  17. agno/api/team.py +22 -26
  18. agno/api/workflow.py +28 -0
  19. agno/cloud/aws/base.py +214 -0
  20. agno/cloud/aws/s3/__init__.py +2 -0
  21. agno/cloud/aws/s3/api_client.py +43 -0
  22. agno/cloud/aws/s3/bucket.py +195 -0
  23. agno/cloud/aws/s3/object.py +57 -0
  24. agno/compression/__init__.py +3 -0
  25. agno/compression/manager.py +247 -0
  26. agno/culture/__init__.py +3 -0
  27. agno/culture/manager.py +956 -0
  28. agno/db/__init__.py +24 -0
  29. agno/db/async_postgres/__init__.py +3 -0
  30. agno/db/base.py +946 -0
  31. agno/db/dynamo/__init__.py +3 -0
  32. agno/db/dynamo/dynamo.py +2781 -0
  33. agno/db/dynamo/schemas.py +442 -0
  34. agno/db/dynamo/utils.py +743 -0
  35. agno/db/firestore/__init__.py +3 -0
  36. agno/db/firestore/firestore.py +2379 -0
  37. agno/db/firestore/schemas.py +181 -0
  38. agno/db/firestore/utils.py +376 -0
  39. agno/db/gcs_json/__init__.py +3 -0
  40. agno/db/gcs_json/gcs_json_db.py +1791 -0
  41. agno/db/gcs_json/utils.py +228 -0
  42. agno/db/in_memory/__init__.py +3 -0
  43. agno/db/in_memory/in_memory_db.py +1312 -0
  44. agno/db/in_memory/utils.py +230 -0
  45. agno/db/json/__init__.py +3 -0
  46. agno/db/json/json_db.py +1777 -0
  47. agno/db/json/utils.py +230 -0
  48. agno/db/migrations/manager.py +199 -0
  49. agno/db/migrations/v1_to_v2.py +635 -0
  50. agno/db/migrations/versions/v2_3_0.py +938 -0
  51. agno/db/mongo/__init__.py +17 -0
  52. agno/db/mongo/async_mongo.py +2760 -0
  53. agno/db/mongo/mongo.py +2597 -0
  54. agno/db/mongo/schemas.py +119 -0
  55. agno/db/mongo/utils.py +276 -0
  56. agno/db/mysql/__init__.py +4 -0
  57. agno/db/mysql/async_mysql.py +2912 -0
  58. agno/db/mysql/mysql.py +2923 -0
  59. agno/db/mysql/schemas.py +186 -0
  60. agno/db/mysql/utils.py +488 -0
  61. agno/db/postgres/__init__.py +4 -0
  62. agno/db/postgres/async_postgres.py +2579 -0
  63. agno/db/postgres/postgres.py +2870 -0
  64. agno/db/postgres/schemas.py +187 -0
  65. agno/db/postgres/utils.py +442 -0
  66. agno/db/redis/__init__.py +3 -0
  67. agno/db/redis/redis.py +2141 -0
  68. agno/db/redis/schemas.py +159 -0
  69. agno/db/redis/utils.py +346 -0
  70. agno/db/schemas/__init__.py +4 -0
  71. agno/db/schemas/culture.py +120 -0
  72. agno/db/schemas/evals.py +34 -0
  73. agno/db/schemas/knowledge.py +40 -0
  74. agno/db/schemas/memory.py +61 -0
  75. agno/db/singlestore/__init__.py +3 -0
  76. agno/db/singlestore/schemas.py +179 -0
  77. agno/db/singlestore/singlestore.py +2877 -0
  78. agno/db/singlestore/utils.py +384 -0
  79. agno/db/sqlite/__init__.py +4 -0
  80. agno/db/sqlite/async_sqlite.py +2911 -0
  81. agno/db/sqlite/schemas.py +181 -0
  82. agno/db/sqlite/sqlite.py +2908 -0
  83. agno/db/sqlite/utils.py +429 -0
  84. agno/db/surrealdb/__init__.py +3 -0
  85. agno/db/surrealdb/metrics.py +292 -0
  86. agno/db/surrealdb/models.py +334 -0
  87. agno/db/surrealdb/queries.py +71 -0
  88. agno/db/surrealdb/surrealdb.py +1908 -0
  89. agno/db/surrealdb/utils.py +147 -0
  90. agno/db/utils.py +118 -0
  91. agno/eval/__init__.py +24 -0
  92. agno/eval/accuracy.py +666 -276
  93. agno/eval/agent_as_judge.py +861 -0
  94. agno/eval/base.py +29 -0
  95. agno/eval/performance.py +779 -0
  96. agno/eval/reliability.py +241 -62
  97. agno/eval/utils.py +120 -0
  98. agno/exceptions.py +143 -1
  99. agno/filters.py +354 -0
  100. agno/guardrails/__init__.py +6 -0
  101. agno/guardrails/base.py +19 -0
  102. agno/guardrails/openai.py +144 -0
  103. agno/guardrails/pii.py +94 -0
  104. agno/guardrails/prompt_injection.py +52 -0
  105. agno/hooks/__init__.py +3 -0
  106. agno/hooks/decorator.py +164 -0
  107. agno/integrations/discord/__init__.py +3 -0
  108. agno/integrations/discord/client.py +203 -0
  109. agno/knowledge/__init__.py +5 -1
  110. agno/{document → knowledge}/chunking/agentic.py +22 -14
  111. agno/{document → knowledge}/chunking/document.py +2 -2
  112. agno/{document → knowledge}/chunking/fixed.py +7 -6
  113. agno/knowledge/chunking/markdown.py +151 -0
  114. agno/{document → knowledge}/chunking/recursive.py +15 -3
  115. agno/knowledge/chunking/row.py +39 -0
  116. agno/knowledge/chunking/semantic.py +91 -0
  117. agno/knowledge/chunking/strategy.py +165 -0
  118. agno/knowledge/content.py +74 -0
  119. agno/knowledge/document/__init__.py +5 -0
  120. agno/{document → knowledge/document}/base.py +12 -2
  121. agno/knowledge/embedder/__init__.py +5 -0
  122. agno/knowledge/embedder/aws_bedrock.py +343 -0
  123. agno/knowledge/embedder/azure_openai.py +210 -0
  124. agno/{embedder → knowledge/embedder}/base.py +8 -0
  125. agno/knowledge/embedder/cohere.py +323 -0
  126. agno/knowledge/embedder/fastembed.py +62 -0
  127. agno/{embedder → knowledge/embedder}/fireworks.py +1 -1
  128. agno/knowledge/embedder/google.py +258 -0
  129. agno/knowledge/embedder/huggingface.py +94 -0
  130. agno/knowledge/embedder/jina.py +182 -0
  131. agno/knowledge/embedder/langdb.py +22 -0
  132. agno/knowledge/embedder/mistral.py +206 -0
  133. agno/knowledge/embedder/nebius.py +13 -0
  134. agno/knowledge/embedder/ollama.py +154 -0
  135. agno/knowledge/embedder/openai.py +195 -0
  136. agno/knowledge/embedder/sentence_transformer.py +63 -0
  137. agno/{embedder → knowledge/embedder}/together.py +1 -1
  138. agno/knowledge/embedder/vllm.py +262 -0
  139. agno/knowledge/embedder/voyageai.py +165 -0
  140. agno/knowledge/knowledge.py +3006 -0
  141. agno/knowledge/reader/__init__.py +7 -0
  142. agno/knowledge/reader/arxiv_reader.py +81 -0
  143. agno/knowledge/reader/base.py +95 -0
  144. agno/knowledge/reader/csv_reader.py +164 -0
  145. agno/knowledge/reader/docx_reader.py +82 -0
  146. agno/knowledge/reader/field_labeled_csv_reader.py +290 -0
  147. agno/knowledge/reader/firecrawl_reader.py +201 -0
  148. agno/knowledge/reader/json_reader.py +88 -0
  149. agno/knowledge/reader/markdown_reader.py +137 -0
  150. agno/knowledge/reader/pdf_reader.py +431 -0
  151. agno/knowledge/reader/pptx_reader.py +101 -0
  152. agno/knowledge/reader/reader_factory.py +313 -0
  153. agno/knowledge/reader/s3_reader.py +89 -0
  154. agno/knowledge/reader/tavily_reader.py +193 -0
  155. agno/knowledge/reader/text_reader.py +127 -0
  156. agno/knowledge/reader/web_search_reader.py +325 -0
  157. agno/knowledge/reader/website_reader.py +455 -0
  158. agno/knowledge/reader/wikipedia_reader.py +91 -0
  159. agno/knowledge/reader/youtube_reader.py +78 -0
  160. agno/knowledge/remote_content/remote_content.py +88 -0
  161. agno/knowledge/reranker/__init__.py +3 -0
  162. agno/{reranker → knowledge/reranker}/base.py +1 -1
  163. agno/{reranker → knowledge/reranker}/cohere.py +2 -2
  164. agno/knowledge/reranker/infinity.py +195 -0
  165. agno/knowledge/reranker/sentence_transformer.py +54 -0
  166. agno/knowledge/types.py +39 -0
  167. agno/knowledge/utils.py +234 -0
  168. agno/media.py +439 -95
  169. agno/memory/__init__.py +16 -3
  170. agno/memory/manager.py +1474 -123
  171. agno/memory/strategies/__init__.py +15 -0
  172. agno/memory/strategies/base.py +66 -0
  173. agno/memory/strategies/summarize.py +196 -0
  174. agno/memory/strategies/types.py +37 -0
  175. agno/models/aimlapi/__init__.py +5 -0
  176. agno/models/aimlapi/aimlapi.py +62 -0
  177. agno/models/anthropic/__init__.py +4 -0
  178. agno/models/anthropic/claude.py +960 -496
  179. agno/models/aws/__init__.py +15 -0
  180. agno/models/aws/bedrock.py +686 -451
  181. agno/models/aws/claude.py +190 -183
  182. agno/models/azure/__init__.py +18 -1
  183. agno/models/azure/ai_foundry.py +489 -0
  184. agno/models/azure/openai_chat.py +89 -40
  185. agno/models/base.py +2477 -550
  186. agno/models/cerebras/__init__.py +12 -0
  187. agno/models/cerebras/cerebras.py +565 -0
  188. agno/models/cerebras/cerebras_openai.py +131 -0
  189. agno/models/cohere/__init__.py +4 -0
  190. agno/models/cohere/chat.py +306 -492
  191. agno/models/cometapi/__init__.py +5 -0
  192. agno/models/cometapi/cometapi.py +74 -0
  193. agno/models/dashscope/__init__.py +5 -0
  194. agno/models/dashscope/dashscope.py +90 -0
  195. agno/models/deepinfra/__init__.py +5 -0
  196. agno/models/deepinfra/deepinfra.py +45 -0
  197. agno/models/deepseek/__init__.py +4 -0
  198. agno/models/deepseek/deepseek.py +110 -9
  199. agno/models/fireworks/__init__.py +4 -0
  200. agno/models/fireworks/fireworks.py +19 -22
  201. agno/models/google/__init__.py +3 -7
  202. agno/models/google/gemini.py +1717 -662
  203. agno/models/google/utils.py +22 -0
  204. agno/models/groq/__init__.py +4 -0
  205. agno/models/groq/groq.py +391 -666
  206. agno/models/huggingface/__init__.py +4 -0
  207. agno/models/huggingface/huggingface.py +266 -538
  208. agno/models/ibm/__init__.py +5 -0
  209. agno/models/ibm/watsonx.py +432 -0
  210. agno/models/internlm/__init__.py +3 -0
  211. agno/models/internlm/internlm.py +20 -3
  212. agno/models/langdb/__init__.py +1 -0
  213. agno/models/langdb/langdb.py +60 -0
  214. agno/models/litellm/__init__.py +14 -0
  215. agno/models/litellm/chat.py +503 -0
  216. agno/models/litellm/litellm_openai.py +42 -0
  217. agno/models/llama_cpp/__init__.py +5 -0
  218. agno/models/llama_cpp/llama_cpp.py +22 -0
  219. agno/models/lmstudio/__init__.py +5 -0
  220. agno/models/lmstudio/lmstudio.py +25 -0
  221. agno/models/message.py +361 -39
  222. agno/models/meta/__init__.py +12 -0
  223. agno/models/meta/llama.py +502 -0
  224. agno/models/meta/llama_openai.py +79 -0
  225. agno/models/metrics.py +120 -0
  226. agno/models/mistral/__init__.py +4 -0
  227. agno/models/mistral/mistral.py +293 -393
  228. agno/models/nebius/__init__.py +3 -0
  229. agno/models/nebius/nebius.py +53 -0
  230. agno/models/nexus/__init__.py +3 -0
  231. agno/models/nexus/nexus.py +22 -0
  232. agno/models/nvidia/__init__.py +4 -0
  233. agno/models/nvidia/nvidia.py +22 -3
  234. agno/models/ollama/__init__.py +4 -2
  235. agno/models/ollama/chat.py +257 -492
  236. agno/models/openai/__init__.py +7 -0
  237. agno/models/openai/chat.py +725 -770
  238. agno/models/openai/like.py +16 -2
  239. agno/models/openai/responses.py +1121 -0
  240. agno/models/openrouter/__init__.py +4 -0
  241. agno/models/openrouter/openrouter.py +62 -5
  242. agno/models/perplexity/__init__.py +5 -0
  243. agno/models/perplexity/perplexity.py +203 -0
  244. agno/models/portkey/__init__.py +3 -0
  245. agno/models/portkey/portkey.py +82 -0
  246. agno/models/requesty/__init__.py +5 -0
  247. agno/models/requesty/requesty.py +69 -0
  248. agno/models/response.py +177 -7
  249. agno/models/sambanova/__init__.py +4 -0
  250. agno/models/sambanova/sambanova.py +23 -4
  251. agno/models/siliconflow/__init__.py +5 -0
  252. agno/models/siliconflow/siliconflow.py +42 -0
  253. agno/models/together/__init__.py +4 -0
  254. agno/models/together/together.py +21 -164
  255. agno/models/utils.py +266 -0
  256. agno/models/vercel/__init__.py +3 -0
  257. agno/models/vercel/v0.py +43 -0
  258. agno/models/vertexai/__init__.py +0 -1
  259. agno/models/vertexai/claude.py +190 -0
  260. agno/models/vllm/__init__.py +3 -0
  261. agno/models/vllm/vllm.py +83 -0
  262. agno/models/xai/__init__.py +2 -0
  263. agno/models/xai/xai.py +111 -7
  264. agno/os/__init__.py +3 -0
  265. agno/os/app.py +1027 -0
  266. agno/os/auth.py +244 -0
  267. agno/os/config.py +126 -0
  268. agno/os/interfaces/__init__.py +1 -0
  269. agno/os/interfaces/a2a/__init__.py +3 -0
  270. agno/os/interfaces/a2a/a2a.py +42 -0
  271. agno/os/interfaces/a2a/router.py +249 -0
  272. agno/os/interfaces/a2a/utils.py +924 -0
  273. agno/os/interfaces/agui/__init__.py +3 -0
  274. agno/os/interfaces/agui/agui.py +47 -0
  275. agno/os/interfaces/agui/router.py +147 -0
  276. agno/os/interfaces/agui/utils.py +574 -0
  277. agno/os/interfaces/base.py +25 -0
  278. agno/os/interfaces/slack/__init__.py +3 -0
  279. agno/os/interfaces/slack/router.py +148 -0
  280. agno/os/interfaces/slack/security.py +30 -0
  281. agno/os/interfaces/slack/slack.py +47 -0
  282. agno/os/interfaces/whatsapp/__init__.py +3 -0
  283. agno/os/interfaces/whatsapp/router.py +210 -0
  284. agno/os/interfaces/whatsapp/security.py +55 -0
  285. agno/os/interfaces/whatsapp/whatsapp.py +36 -0
  286. agno/os/mcp.py +293 -0
  287. agno/os/middleware/__init__.py +9 -0
  288. agno/os/middleware/jwt.py +797 -0
  289. agno/os/router.py +258 -0
  290. agno/os/routers/__init__.py +3 -0
  291. agno/os/routers/agents/__init__.py +3 -0
  292. agno/os/routers/agents/router.py +599 -0
  293. agno/os/routers/agents/schema.py +261 -0
  294. agno/os/routers/evals/__init__.py +3 -0
  295. agno/os/routers/evals/evals.py +450 -0
  296. agno/os/routers/evals/schemas.py +174 -0
  297. agno/os/routers/evals/utils.py +231 -0
  298. agno/os/routers/health.py +31 -0
  299. agno/os/routers/home.py +52 -0
  300. agno/os/routers/knowledge/__init__.py +3 -0
  301. agno/os/routers/knowledge/knowledge.py +1008 -0
  302. agno/os/routers/knowledge/schemas.py +178 -0
  303. agno/os/routers/memory/__init__.py +3 -0
  304. agno/os/routers/memory/memory.py +661 -0
  305. agno/os/routers/memory/schemas.py +88 -0
  306. agno/os/routers/metrics/__init__.py +3 -0
  307. agno/os/routers/metrics/metrics.py +190 -0
  308. agno/os/routers/metrics/schemas.py +47 -0
  309. agno/os/routers/session/__init__.py +3 -0
  310. agno/os/routers/session/session.py +997 -0
  311. agno/os/routers/teams/__init__.py +3 -0
  312. agno/os/routers/teams/router.py +512 -0
  313. agno/os/routers/teams/schema.py +257 -0
  314. agno/os/routers/traces/__init__.py +3 -0
  315. agno/os/routers/traces/schemas.py +414 -0
  316. agno/os/routers/traces/traces.py +499 -0
  317. agno/os/routers/workflows/__init__.py +3 -0
  318. agno/os/routers/workflows/router.py +624 -0
  319. agno/os/routers/workflows/schema.py +75 -0
  320. agno/os/schema.py +534 -0
  321. agno/os/scopes.py +469 -0
  322. agno/{playground → os}/settings.py +7 -15
  323. agno/os/utils.py +973 -0
  324. agno/reasoning/anthropic.py +80 -0
  325. agno/reasoning/azure_ai_foundry.py +67 -0
  326. agno/reasoning/deepseek.py +63 -0
  327. agno/reasoning/default.py +97 -0
  328. agno/reasoning/gemini.py +73 -0
  329. agno/reasoning/groq.py +71 -0
  330. agno/reasoning/helpers.py +24 -1
  331. agno/reasoning/ollama.py +67 -0
  332. agno/reasoning/openai.py +86 -0
  333. agno/reasoning/step.py +2 -1
  334. agno/reasoning/vertexai.py +76 -0
  335. agno/run/__init__.py +6 -0
  336. agno/run/agent.py +822 -0
  337. agno/run/base.py +247 -0
  338. agno/run/cancel.py +81 -0
  339. agno/run/requirement.py +181 -0
  340. agno/run/team.py +767 -0
  341. agno/run/workflow.py +708 -0
  342. agno/session/__init__.py +10 -0
  343. agno/session/agent.py +260 -0
  344. agno/session/summary.py +265 -0
  345. agno/session/team.py +342 -0
  346. agno/session/workflow.py +501 -0
  347. agno/table.py +10 -0
  348. agno/team/__init__.py +37 -0
  349. agno/team/team.py +9536 -0
  350. agno/tools/__init__.py +7 -0
  351. agno/tools/agentql.py +120 -0
  352. agno/tools/airflow.py +22 -12
  353. agno/tools/api.py +122 -0
  354. agno/tools/apify.py +276 -83
  355. agno/tools/{arxiv_toolkit.py → arxiv.py} +20 -12
  356. agno/tools/aws_lambda.py +28 -7
  357. agno/tools/aws_ses.py +66 -0
  358. agno/tools/baidusearch.py +11 -4
  359. agno/tools/bitbucket.py +292 -0
  360. agno/tools/brandfetch.py +213 -0
  361. agno/tools/bravesearch.py +106 -0
  362. agno/tools/brightdata.py +367 -0
  363. agno/tools/browserbase.py +209 -0
  364. agno/tools/calcom.py +32 -23
  365. agno/tools/calculator.py +24 -37
  366. agno/tools/cartesia.py +187 -0
  367. agno/tools/{clickup_tool.py → clickup.py} +17 -28
  368. agno/tools/confluence.py +91 -26
  369. agno/tools/crawl4ai.py +139 -43
  370. agno/tools/csv_toolkit.py +28 -22
  371. agno/tools/dalle.py +36 -22
  372. agno/tools/daytona.py +475 -0
  373. agno/tools/decorator.py +169 -14
  374. agno/tools/desi_vocal.py +23 -11
  375. agno/tools/discord.py +32 -29
  376. agno/tools/docker.py +716 -0
  377. agno/tools/duckdb.py +76 -81
  378. agno/tools/duckduckgo.py +43 -40
  379. agno/tools/e2b.py +703 -0
  380. agno/tools/eleven_labs.py +65 -54
  381. agno/tools/email.py +13 -5
  382. agno/tools/evm.py +129 -0
  383. agno/tools/exa.py +324 -42
  384. agno/tools/fal.py +39 -35
  385. agno/tools/file.py +196 -30
  386. agno/tools/file_generation.py +356 -0
  387. agno/tools/financial_datasets.py +288 -0
  388. agno/tools/firecrawl.py +108 -33
  389. agno/tools/function.py +960 -122
  390. agno/tools/giphy.py +34 -12
  391. agno/tools/github.py +1294 -97
  392. agno/tools/gmail.py +922 -0
  393. agno/tools/google_bigquery.py +117 -0
  394. agno/tools/google_drive.py +271 -0
  395. agno/tools/google_maps.py +253 -0
  396. agno/tools/googlecalendar.py +607 -107
  397. agno/tools/googlesheets.py +377 -0
  398. agno/tools/hackernews.py +20 -12
  399. agno/tools/jina.py +24 -14
  400. agno/tools/jira.py +48 -19
  401. agno/tools/knowledge.py +218 -0
  402. agno/tools/linear.py +82 -43
  403. agno/tools/linkup.py +58 -0
  404. agno/tools/local_file_system.py +15 -7
  405. agno/tools/lumalab.py +41 -26
  406. agno/tools/mcp/__init__.py +10 -0
  407. agno/tools/mcp/mcp.py +331 -0
  408. agno/tools/mcp/multi_mcp.py +347 -0
  409. agno/tools/mcp/params.py +24 -0
  410. agno/tools/mcp_toolbox.py +284 -0
  411. agno/tools/mem0.py +193 -0
  412. agno/tools/memory.py +419 -0
  413. agno/tools/mlx_transcribe.py +11 -9
  414. agno/tools/models/azure_openai.py +190 -0
  415. agno/tools/models/gemini.py +203 -0
  416. agno/tools/models/groq.py +158 -0
  417. agno/tools/models/morph.py +186 -0
  418. agno/tools/models/nebius.py +124 -0
  419. agno/tools/models_labs.py +163 -82
  420. agno/tools/moviepy_video.py +18 -13
  421. agno/tools/nano_banana.py +151 -0
  422. agno/tools/neo4j.py +134 -0
  423. agno/tools/newspaper.py +15 -4
  424. agno/tools/newspaper4k.py +19 -6
  425. agno/tools/notion.py +204 -0
  426. agno/tools/openai.py +181 -17
  427. agno/tools/openbb.py +27 -20
  428. agno/tools/opencv.py +321 -0
  429. agno/tools/openweather.py +233 -0
  430. agno/tools/oxylabs.py +385 -0
  431. agno/tools/pandas.py +25 -15
  432. agno/tools/parallel.py +314 -0
  433. agno/tools/postgres.py +238 -185
  434. agno/tools/pubmed.py +125 -13
  435. agno/tools/python.py +48 -35
  436. agno/tools/reasoning.py +283 -0
  437. agno/tools/reddit.py +207 -29
  438. agno/tools/redshift.py +406 -0
  439. agno/tools/replicate.py +69 -26
  440. agno/tools/resend.py +11 -6
  441. agno/tools/scrapegraph.py +179 -19
  442. agno/tools/searxng.py +23 -31
  443. agno/tools/serpapi.py +15 -10
  444. agno/tools/serper.py +255 -0
  445. agno/tools/shell.py +23 -12
  446. agno/tools/shopify.py +1519 -0
  447. agno/tools/slack.py +56 -14
  448. agno/tools/sleep.py +8 -6
  449. agno/tools/spider.py +35 -11
  450. agno/tools/spotify.py +919 -0
  451. agno/tools/sql.py +34 -19
  452. agno/tools/tavily.py +158 -8
  453. agno/tools/telegram.py +18 -8
  454. agno/tools/todoist.py +218 -0
  455. agno/tools/toolkit.py +134 -9
  456. agno/tools/trafilatura.py +388 -0
  457. agno/tools/trello.py +25 -28
  458. agno/tools/twilio.py +18 -9
  459. agno/tools/user_control_flow.py +78 -0
  460. agno/tools/valyu.py +228 -0
  461. agno/tools/visualization.py +467 -0
  462. agno/tools/webbrowser.py +28 -0
  463. agno/tools/webex.py +76 -0
  464. agno/tools/website.py +23 -19
  465. agno/tools/webtools.py +45 -0
  466. agno/tools/whatsapp.py +286 -0
  467. agno/tools/wikipedia.py +28 -19
  468. agno/tools/workflow.py +285 -0
  469. agno/tools/{twitter.py → x.py} +142 -46
  470. agno/tools/yfinance.py +41 -39
  471. agno/tools/youtube.py +34 -17
  472. agno/tools/zendesk.py +15 -5
  473. agno/tools/zep.py +454 -0
  474. agno/tools/zoom.py +86 -37
  475. agno/tracing/__init__.py +12 -0
  476. agno/tracing/exporter.py +157 -0
  477. agno/tracing/schemas.py +276 -0
  478. agno/tracing/setup.py +111 -0
  479. agno/utils/agent.py +938 -0
  480. agno/utils/audio.py +37 -1
  481. agno/utils/certs.py +27 -0
  482. agno/utils/code_execution.py +11 -0
  483. agno/utils/common.py +103 -20
  484. agno/utils/cryptography.py +22 -0
  485. agno/utils/dttm.py +33 -0
  486. agno/utils/events.py +700 -0
  487. agno/utils/functions.py +107 -37
  488. agno/utils/gemini.py +426 -0
  489. agno/utils/hooks.py +171 -0
  490. agno/utils/http.py +185 -0
  491. agno/utils/json_schema.py +159 -37
  492. agno/utils/knowledge.py +36 -0
  493. agno/utils/location.py +19 -0
  494. agno/utils/log.py +221 -8
  495. agno/utils/mcp.py +214 -0
  496. agno/utils/media.py +335 -14
  497. agno/utils/merge_dict.py +22 -1
  498. agno/utils/message.py +77 -2
  499. agno/utils/models/ai_foundry.py +50 -0
  500. agno/utils/models/claude.py +373 -0
  501. agno/utils/models/cohere.py +94 -0
  502. agno/utils/models/llama.py +85 -0
  503. agno/utils/models/mistral.py +100 -0
  504. agno/utils/models/openai_responses.py +140 -0
  505. agno/utils/models/schema_utils.py +153 -0
  506. agno/utils/models/watsonx.py +41 -0
  507. agno/utils/openai.py +257 -0
  508. agno/utils/pickle.py +1 -1
  509. agno/utils/pprint.py +124 -8
  510. agno/utils/print_response/agent.py +930 -0
  511. agno/utils/print_response/team.py +1914 -0
  512. agno/utils/print_response/workflow.py +1668 -0
  513. agno/utils/prompts.py +111 -0
  514. agno/utils/reasoning.py +108 -0
  515. agno/utils/response.py +163 -0
  516. agno/utils/serialize.py +32 -0
  517. agno/utils/shell.py +4 -4
  518. agno/utils/streamlit.py +487 -0
  519. agno/utils/string.py +204 -51
  520. agno/utils/team.py +139 -0
  521. agno/utils/timer.py +9 -2
  522. agno/utils/tokens.py +657 -0
  523. agno/utils/tools.py +19 -1
  524. agno/utils/whatsapp.py +305 -0
  525. agno/utils/yaml_io.py +3 -3
  526. agno/vectordb/__init__.py +2 -0
  527. agno/vectordb/base.py +87 -9
  528. agno/vectordb/cassandra/__init__.py +5 -1
  529. agno/vectordb/cassandra/cassandra.py +383 -27
  530. agno/vectordb/chroma/__init__.py +4 -0
  531. agno/vectordb/chroma/chromadb.py +748 -83
  532. agno/vectordb/clickhouse/__init__.py +7 -1
  533. agno/vectordb/clickhouse/clickhousedb.py +554 -53
  534. agno/vectordb/couchbase/__init__.py +3 -0
  535. agno/vectordb/couchbase/couchbase.py +1446 -0
  536. agno/vectordb/lancedb/__init__.py +5 -0
  537. agno/vectordb/lancedb/lance_db.py +730 -98
  538. agno/vectordb/langchaindb/__init__.py +5 -0
  539. agno/vectordb/langchaindb/langchaindb.py +163 -0
  540. agno/vectordb/lightrag/__init__.py +5 -0
  541. agno/vectordb/lightrag/lightrag.py +388 -0
  542. agno/vectordb/llamaindex/__init__.py +3 -0
  543. agno/vectordb/llamaindex/llamaindexdb.py +166 -0
  544. agno/vectordb/milvus/__init__.py +3 -0
  545. agno/vectordb/milvus/milvus.py +966 -78
  546. agno/vectordb/mongodb/__init__.py +9 -1
  547. agno/vectordb/mongodb/mongodb.py +1175 -172
  548. agno/vectordb/pgvector/__init__.py +8 -0
  549. agno/vectordb/pgvector/pgvector.py +599 -115
  550. agno/vectordb/pineconedb/__init__.py +5 -1
  551. agno/vectordb/pineconedb/pineconedb.py +406 -43
  552. agno/vectordb/qdrant/__init__.py +4 -0
  553. agno/vectordb/qdrant/qdrant.py +914 -61
  554. agno/vectordb/redis/__init__.py +9 -0
  555. agno/vectordb/redis/redisdb.py +682 -0
  556. agno/vectordb/singlestore/__init__.py +8 -1
  557. agno/vectordb/singlestore/singlestore.py +771 -0
  558. agno/vectordb/surrealdb/__init__.py +3 -0
  559. agno/vectordb/surrealdb/surrealdb.py +663 -0
  560. agno/vectordb/upstashdb/__init__.py +5 -0
  561. agno/vectordb/upstashdb/upstashdb.py +718 -0
  562. agno/vectordb/weaviate/__init__.py +8 -0
  563. agno/vectordb/weaviate/index.py +15 -0
  564. agno/vectordb/weaviate/weaviate.py +1009 -0
  565. agno/workflow/__init__.py +23 -1
  566. agno/workflow/agent.py +299 -0
  567. agno/workflow/condition.py +759 -0
  568. agno/workflow/loop.py +756 -0
  569. agno/workflow/parallel.py +853 -0
  570. agno/workflow/router.py +723 -0
  571. agno/workflow/step.py +1564 -0
  572. agno/workflow/steps.py +613 -0
  573. agno/workflow/types.py +556 -0
  574. agno/workflow/workflow.py +4327 -514
  575. agno-2.3.13.dist-info/METADATA +639 -0
  576. agno-2.3.13.dist-info/RECORD +613 -0
  577. {agno-0.1.2.dist-info → agno-2.3.13.dist-info}/WHEEL +1 -1
  578. agno-2.3.13.dist-info/licenses/LICENSE +201 -0
  579. agno/api/playground.py +0 -91
  580. agno/api/schemas/playground.py +0 -22
  581. agno/api/schemas/user.py +0 -22
  582. agno/api/schemas/workspace.py +0 -46
  583. agno/api/user.py +0 -160
  584. agno/api/workspace.py +0 -151
  585. agno/cli/auth_server.py +0 -118
  586. agno/cli/config.py +0 -275
  587. agno/cli/console.py +0 -88
  588. agno/cli/credentials.py +0 -23
  589. agno/cli/entrypoint.py +0 -571
  590. agno/cli/operator.py +0 -355
  591. agno/cli/settings.py +0 -85
  592. agno/cli/ws/ws_cli.py +0 -817
  593. agno/constants.py +0 -13
  594. agno/document/__init__.py +0 -1
  595. agno/document/chunking/semantic.py +0 -47
  596. agno/document/chunking/strategy.py +0 -31
  597. agno/document/reader/__init__.py +0 -1
  598. agno/document/reader/arxiv_reader.py +0 -41
  599. agno/document/reader/base.py +0 -22
  600. agno/document/reader/csv_reader.py +0 -84
  601. agno/document/reader/docx_reader.py +0 -46
  602. agno/document/reader/firecrawl_reader.py +0 -99
  603. agno/document/reader/json_reader.py +0 -43
  604. agno/document/reader/pdf_reader.py +0 -219
  605. agno/document/reader/s3/pdf_reader.py +0 -46
  606. agno/document/reader/s3/text_reader.py +0 -51
  607. agno/document/reader/text_reader.py +0 -41
  608. agno/document/reader/website_reader.py +0 -175
  609. agno/document/reader/youtube_reader.py +0 -50
  610. agno/embedder/__init__.py +0 -1
  611. agno/embedder/azure_openai.py +0 -86
  612. agno/embedder/cohere.py +0 -72
  613. agno/embedder/fastembed.py +0 -37
  614. agno/embedder/google.py +0 -73
  615. agno/embedder/huggingface.py +0 -54
  616. agno/embedder/mistral.py +0 -80
  617. agno/embedder/ollama.py +0 -57
  618. agno/embedder/openai.py +0 -74
  619. agno/embedder/sentence_transformer.py +0 -38
  620. agno/embedder/voyageai.py +0 -64
  621. agno/eval/perf.py +0 -201
  622. agno/file/__init__.py +0 -1
  623. agno/file/file.py +0 -16
  624. agno/file/local/csv.py +0 -32
  625. agno/file/local/txt.py +0 -19
  626. agno/infra/app.py +0 -240
  627. agno/infra/base.py +0 -144
  628. agno/infra/context.py +0 -20
  629. agno/infra/db_app.py +0 -52
  630. agno/infra/resource.py +0 -205
  631. agno/infra/resources.py +0 -55
  632. agno/knowledge/agent.py +0 -230
  633. agno/knowledge/arxiv.py +0 -22
  634. agno/knowledge/combined.py +0 -22
  635. agno/knowledge/csv.py +0 -28
  636. agno/knowledge/csv_url.py +0 -19
  637. agno/knowledge/document.py +0 -20
  638. agno/knowledge/docx.py +0 -30
  639. agno/knowledge/json.py +0 -28
  640. agno/knowledge/langchain.py +0 -71
  641. agno/knowledge/llamaindex.py +0 -66
  642. agno/knowledge/pdf.py +0 -28
  643. agno/knowledge/pdf_url.py +0 -26
  644. agno/knowledge/s3/base.py +0 -60
  645. agno/knowledge/s3/pdf.py +0 -21
  646. agno/knowledge/s3/text.py +0 -23
  647. agno/knowledge/text.py +0 -30
  648. agno/knowledge/website.py +0 -88
  649. agno/knowledge/wikipedia.py +0 -31
  650. agno/knowledge/youtube.py +0 -22
  651. agno/memory/agent.py +0 -392
  652. agno/memory/classifier.py +0 -104
  653. agno/memory/db/__init__.py +0 -1
  654. agno/memory/db/base.py +0 -42
  655. agno/memory/db/mongodb.py +0 -189
  656. agno/memory/db/postgres.py +0 -203
  657. agno/memory/db/sqlite.py +0 -193
  658. agno/memory/memory.py +0 -15
  659. agno/memory/row.py +0 -36
  660. agno/memory/summarizer.py +0 -192
  661. agno/memory/summary.py +0 -19
  662. agno/memory/workflow.py +0 -38
  663. agno/models/google/gemini_openai.py +0 -26
  664. agno/models/ollama/hermes.py +0 -221
  665. agno/models/ollama/tools.py +0 -362
  666. agno/models/vertexai/gemini.py +0 -595
  667. agno/playground/__init__.py +0 -3
  668. agno/playground/async_router.py +0 -421
  669. agno/playground/deploy.py +0 -249
  670. agno/playground/operator.py +0 -92
  671. agno/playground/playground.py +0 -91
  672. agno/playground/schemas.py +0 -76
  673. agno/playground/serve.py +0 -55
  674. agno/playground/sync_router.py +0 -405
  675. agno/reasoning/agent.py +0 -68
  676. agno/run/response.py +0 -112
  677. agno/storage/agent/__init__.py +0 -0
  678. agno/storage/agent/base.py +0 -38
  679. agno/storage/agent/dynamodb.py +0 -350
  680. agno/storage/agent/json.py +0 -92
  681. agno/storage/agent/mongodb.py +0 -228
  682. agno/storage/agent/postgres.py +0 -367
  683. agno/storage/agent/session.py +0 -79
  684. agno/storage/agent/singlestore.py +0 -303
  685. agno/storage/agent/sqlite.py +0 -357
  686. agno/storage/agent/yaml.py +0 -93
  687. agno/storage/workflow/__init__.py +0 -0
  688. agno/storage/workflow/base.py +0 -40
  689. agno/storage/workflow/mongodb.py +0 -233
  690. agno/storage/workflow/postgres.py +0 -366
  691. agno/storage/workflow/session.py +0 -60
  692. agno/storage/workflow/sqlite.py +0 -359
  693. agno/tools/googlesearch.py +0 -88
  694. agno/utils/defaults.py +0 -57
  695. agno/utils/filesystem.py +0 -39
  696. agno/utils/git.py +0 -52
  697. agno/utils/json_io.py +0 -30
  698. agno/utils/load_env.py +0 -19
  699. agno/utils/py_io.py +0 -19
  700. agno/utils/pyproject.py +0 -18
  701. agno/utils/resource_filter.py +0 -31
  702. agno/vectordb/singlestore/s2vectordb.py +0 -390
  703. agno/vectordb/singlestore/s2vectordb2.py +0 -355
  704. agno/workspace/__init__.py +0 -0
  705. agno/workspace/config.py +0 -325
  706. agno/workspace/enums.py +0 -6
  707. agno/workspace/helpers.py +0 -48
  708. agno/workspace/operator.py +0 -758
  709. agno/workspace/settings.py +0 -63
  710. agno-0.1.2.dist-info/LICENSE +0 -375
  711. agno-0.1.2.dist-info/METADATA +0 -502
  712. agno-0.1.2.dist-info/RECORD +0 -352
  713. agno-0.1.2.dist-info/entry_points.txt +0 -3
  714. /agno/{cli → db/migrations}/__init__.py +0 -0
  715. /agno/{cli/ws → db/migrations/versions}/__init__.py +0 -0
  716. /agno/{document/chunking/__init__.py → db/schemas/metrics.py} +0 -0
  717. /agno/{document/reader/s3 → integrations}/__init__.py +0 -0
  718. /agno/{file/local → knowledge/chunking}/__init__.py +0 -0
  719. /agno/{infra → knowledge/remote_content}/__init__.py +0 -0
  720. /agno/{knowledge/s3 → tools/models}/__init__.py +0 -0
  721. /agno/{reranker → utils/models}/__init__.py +0 -0
  722. /agno/{storage → utils/print_response}/__init__.py +0 -0
  723. {agno-0.1.2.dist-info → agno-2.3.13.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,771 @@
1
+ import asyncio
2
+ import json
3
+ from hashlib import md5
4
+ from typing import Any, Dict, List, Optional, Union
5
+
6
+ try:
7
+ from sqlalchemy.dialects import mysql
8
+ from sqlalchemy.engine import Engine, create_engine
9
+ from sqlalchemy.inspection import inspect
10
+ from sqlalchemy.orm import Session, sessionmaker
11
+ from sqlalchemy.schema import Column, MetaData, Table
12
+ from sqlalchemy.sql.expression import func, select, text, update
13
+ from sqlalchemy.types import DateTime
14
+ except ImportError:
15
+ raise ImportError("`sqlalchemy` not installed")
16
+
17
+ from agno.filters import FilterExpr
18
+ from agno.knowledge.document import Document
19
+ from agno.knowledge.embedder import Embedder
20
+ from agno.knowledge.reranker.base import Reranker
21
+ from agno.utils.log import log_debug, log_error, log_info, log_warning
22
+ from agno.vectordb.base import VectorDb
23
+ from agno.vectordb.distance import Distance
24
+
25
+
26
+ class SingleStore(VectorDb):
27
+ def __init__(
28
+ self,
29
+ collection: str,
30
+ schema: Optional[str] = "ai",
31
+ db_url: Optional[str] = None,
32
+ db_engine: Optional[Engine] = None,
33
+ embedder: Optional[Embedder] = None,
34
+ distance: Distance = Distance.cosine,
35
+ reranker: Optional[Reranker] = None,
36
+ name: Optional[str] = None,
37
+ description: Optional[str] = None,
38
+ # index: Optional[Union[Ivfflat, HNSW]] = HNSW(),
39
+ ):
40
+ _engine: Optional[Engine] = db_engine
41
+ if _engine is None and db_url is not None:
42
+ _engine = create_engine(db_url)
43
+
44
+ if _engine is None:
45
+ raise ValueError("Must provide either db_url or db_engine")
46
+
47
+ self.collection: str = collection
48
+ self.schema: Optional[str] = schema
49
+ self.db_url: Optional[str] = db_url
50
+ # Initialize base class with name and description
51
+ super().__init__(name=name, description=description)
52
+
53
+ self.db_engine: Engine = _engine
54
+ self.metadata: MetaData = MetaData(schema=self.schema)
55
+ if embedder is None:
56
+ from agno.knowledge.embedder.openai import OpenAIEmbedder
57
+
58
+ embedder = OpenAIEmbedder()
59
+ log_info("Embedder not provided, using OpenAIEmbedder as default.")
60
+ self.embedder: Embedder = embedder
61
+ self.dimensions: Optional[int] = self.embedder.dimensions
62
+
63
+ self.distance: Distance = distance
64
+ # self.index: Optional[Union[Ivfflat, HNSW]] = index
65
+ self.Session: sessionmaker[Session] = sessionmaker(bind=self.db_engine)
66
+ self.reranker: Optional[Reranker] = reranker
67
+ self.table: Table = self.get_table()
68
+
69
+ def get_table(self) -> Table:
70
+ """
71
+ Define the table structure.
72
+
73
+ Returns:
74
+ Table: SQLAlchemy Table object.
75
+ """
76
+ return Table(
77
+ self.collection,
78
+ self.metadata,
79
+ Column("id", mysql.TEXT),
80
+ Column("name", mysql.TEXT),
81
+ Column("meta_data", mysql.TEXT),
82
+ Column("content", mysql.TEXT),
83
+ Column("embedding", mysql.TEXT), # Placeholder for the vector column
84
+ Column("usage", mysql.TEXT),
85
+ Column("created_at", DateTime(timezone=True), server_default=text("now()")),
86
+ Column("updated_at", DateTime(timezone=True), onupdate=text("now()")),
87
+ Column("content_hash", mysql.TEXT),
88
+ Column("content_id", mysql.TEXT),
89
+ extend_existing=True,
90
+ )
91
+
92
+ def create(self) -> None:
93
+ """
94
+ Create the table if it does not exist.
95
+ """
96
+ if not self.table_exists():
97
+ log_info(f"Creating table: {self.collection}")
98
+ with self.db_engine.connect() as connection:
99
+ connection.execute(
100
+ text(f"""
101
+ CREATE TABLE IF NOT EXISTS {self.schema}.{self.collection} (
102
+ id TEXT,
103
+ name TEXT,
104
+ meta_data TEXT,
105
+ content TEXT,
106
+ embedding VECTOR({self.dimensions}) NOT NULL,
107
+ `usage` TEXT,
108
+ created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
109
+ updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
110
+ content_hash TEXT,
111
+ content_id TEXT
112
+ );
113
+ """)
114
+ )
115
+ # Call optimize to create indexes
116
+ self.optimize()
117
+
118
+ def table_exists(self) -> bool:
119
+ """
120
+ Check if the table exists.
121
+
122
+ Returns:
123
+ bool: True if the table exists, False otherwise.
124
+ """
125
+ log_debug(f"Checking if table exists: {self.table.name}")
126
+ try:
127
+ return inspect(self.db_engine).has_table(self.table.name, schema=self.schema)
128
+ except Exception as e:
129
+ log_error(e)
130
+ return False
131
+
132
+ def content_hash_exists(self, content_hash: str) -> bool:
133
+ """
134
+ Validating if the document exists or not
135
+
136
+ Args:
137
+ document (Document): Document to validate
138
+ """
139
+ with self.Session.begin() as sess:
140
+ stmt = select(self.table.c.name).where(self.table.c.content_hash == content_hash)
141
+ result = sess.execute(stmt).first()
142
+ return result is not None
143
+
144
+ def name_exists(self, name: str) -> bool:
145
+ """
146
+ Validate if a row with this name exists or not
147
+
148
+ Args:
149
+ name (str): Name to check
150
+ """
151
+ with self.Session.begin() as sess:
152
+ stmt = select(self.table.c.name).where(self.table.c.name == name)
153
+ result = sess.execute(stmt).first()
154
+ return result is not None
155
+
156
+ def id_exists(self, id: str) -> bool:
157
+ """
158
+ Validate if a row with this id exists or not
159
+
160
+ Args:
161
+ id (str): Id to check
162
+ """
163
+ with self.Session.begin() as sess:
164
+ stmt = select(self.table.c.id).where(self.table.c.id == id)
165
+ result = sess.execute(stmt).first()
166
+ return result is not None
167
+
168
+ def insert(
169
+ self,
170
+ content_hash: str,
171
+ documents: List[Document],
172
+ filters: Optional[Dict[str, Any]] = None,
173
+ batch_size: int = 10,
174
+ ) -> None:
175
+ """
176
+ Insert documents into the table.
177
+
178
+ Args:
179
+ documents (List[Document]): List of documents to insert.
180
+ filters (Optional[Dict[str, Any]]): Optional filters for the insert.
181
+ batch_size (int): Number of documents to insert in each batch.
182
+ """
183
+ with self.Session.begin() as sess:
184
+ counter = 0
185
+ for document in documents:
186
+ document.embed(embedder=self.embedder)
187
+ cleaned_content = document.content.replace("\x00", "\ufffd")
188
+ # Include content_hash in ID to ensure uniqueness across different content hashes
189
+ base_id = document.id or md5(cleaned_content.encode()).hexdigest()
190
+ record_id = md5(f"{base_id}_{content_hash}".encode()).hexdigest()
191
+ _id = record_id
192
+
193
+ meta_data_json = json.dumps(document.meta_data)
194
+ usage_json = json.dumps(document.usage)
195
+
196
+ # Convert embedding list to SingleStore VECTOR format
197
+ embeddings = f"[{','.join(map(str, document.embedding))}]" if document.embedding else None
198
+
199
+ stmt = mysql.insert(self.table).values(
200
+ id=_id,
201
+ name=document.name,
202
+ meta_data=meta_data_json,
203
+ content=cleaned_content,
204
+ embedding=embeddings,
205
+ usage=usage_json,
206
+ content_hash=content_hash,
207
+ content_id=document.content_id,
208
+ )
209
+ sess.execute(stmt)
210
+ counter += 1
211
+ log_debug(f"Inserted document: {document.name} ({document.meta_data})")
212
+
213
+ sess.commit()
214
+ log_debug(f"Committed {counter} documents")
215
+
216
+ def upsert_available(self) -> bool:
217
+ """Indicate that upsert functionality is available."""
218
+ return True
219
+
220
+ def upsert(
221
+ self,
222
+ content_hash: str,
223
+ documents: List[Document],
224
+ filters: Optional[Dict[str, Any]] = None,
225
+ batch_size: int = 20,
226
+ ) -> None:
227
+ if self.content_hash_exists(content_hash):
228
+ self._delete_by_content_hash(content_hash)
229
+ self._upsert(content_hash=content_hash, documents=documents, filters=filters, batch_size=batch_size)
230
+
231
+ def _upsert(
232
+ self,
233
+ content_hash: str,
234
+ documents: List[Document],
235
+ filters: Optional[Dict[str, Any]] = None,
236
+ batch_size: int = 20,
237
+ ) -> None:
238
+ """
239
+ Upsert (insert or update) documents in the table.
240
+
241
+ Args:
242
+ documents (List[Document]): List of documents to upsert.
243
+ filters (Optional[Dict[str, Any]]): Optional filters for the upsert.
244
+ batch_size (int): Number of documents to upsert in each batch.
245
+ """
246
+ with self.Session.begin() as sess:
247
+ counter = 0
248
+ for document in documents:
249
+ document.embed(embedder=self.embedder)
250
+ cleaned_content = document.content.replace("\x00", "\ufffd")
251
+ # Include content_hash in ID to ensure uniqueness across different content hashes
252
+ base_id = document.id or md5(cleaned_content.encode()).hexdigest()
253
+ record_id = md5(f"{base_id}_{content_hash}".encode()).hexdigest()
254
+ _id = record_id
255
+
256
+ meta_data_json = json.dumps(document.meta_data)
257
+ usage_json = json.dumps(document.usage)
258
+
259
+ # Convert embedding list to SingleStore VECTOR format
260
+ embeddings = f"[{','.join(map(str, document.embedding))}]" if document.embedding else None
261
+ stmt = (
262
+ mysql.insert(self.table)
263
+ .values(
264
+ id=_id,
265
+ name=document.name,
266
+ meta_data=meta_data_json,
267
+ content=cleaned_content,
268
+ embedding=embeddings,
269
+ usage=usage_json,
270
+ content_hash=content_hash,
271
+ content_id=document.content_id,
272
+ )
273
+ .on_duplicate_key_update(
274
+ name=document.name,
275
+ meta_data=meta_data_json,
276
+ content=cleaned_content,
277
+ embedding=embeddings,
278
+ usage=usage_json,
279
+ content_hash=content_hash,
280
+ content_id=document.content_id,
281
+ )
282
+ )
283
+ sess.execute(stmt)
284
+ counter += 1
285
+ log_debug(f"Upserted document: {document.name} ({document.meta_data})")
286
+
287
+ sess.commit()
288
+ log_debug(f"Committed {counter} documents")
289
+
290
+ def search(
291
+ self, query: str, limit: int = 5, filters: Optional[Union[Dict[str, Any], List[FilterExpr]]] = None
292
+ ) -> List[Document]:
293
+ """
294
+ Search for documents based on a query and optional filters.
295
+
296
+ Args:
297
+ query (str): The search query.
298
+ limit (int): The maximum number of results to return.
299
+ filters (Optional[Dict[str, Any]]): Optional filters for the search.
300
+
301
+ Returns:
302
+ List[Document]: List of documents that match the query.
303
+ """
304
+ if filters is not None:
305
+ log_warning("Filters are not supported in SingleStore. No filters will be applied.")
306
+ query_embedding = self.embedder.get_embedding(query)
307
+ if query_embedding is None:
308
+ log_error(f"Error getting embedding for Query: {query}")
309
+ return []
310
+
311
+ columns = [
312
+ self.table.c.name,
313
+ self.table.c.meta_data,
314
+ self.table.c.content,
315
+ self.table.c.embedding,
316
+ self.table.c.usage,
317
+ self.table.c.content_id,
318
+ ]
319
+
320
+ stmt = select(*columns)
321
+
322
+ # if filters is not None:
323
+ # for key, value in filters.items():
324
+ # if hasattr(self.table.c, key):
325
+ # stmt = stmt.where(getattr(self.table.c, key) == value)
326
+
327
+ if self.distance == Distance.l2:
328
+ stmt = stmt.order_by(self.table.c.embedding.max_inner_product(query_embedding))
329
+ if self.distance == Distance.cosine:
330
+ embeddings = json.dumps(query_embedding)
331
+ dot_product_expr = func.dot_product(self.table.c.embedding, text(":embedding"))
332
+ stmt = stmt.order_by(dot_product_expr.desc())
333
+ stmt = stmt.params(embedding=embeddings)
334
+ # stmt = stmt.order_by(self.table.c.embedding.cosine_distance(query_embedding))
335
+ if self.distance == Distance.max_inner_product:
336
+ stmt = stmt.order_by(self.table.c.embedding.max_inner_product(query_embedding))
337
+
338
+ stmt = stmt.limit(limit=limit)
339
+ log_debug(f"Query: {stmt}")
340
+
341
+ # Get neighbors
342
+ # This will only work if embedding column is created with `vector` data type.
343
+ with self.Session.begin() as sess:
344
+ sess.execute(text("SET vector_type_project_format = JSON"))
345
+ neighbors = sess.execute(stmt).fetchall() or []
346
+ # if self.index is not None:
347
+ # if isinstance(self.index, Ivfflat):
348
+ # # Assuming 'nprobe' is a relevant parameter to be set for the session
349
+ # # Update the session settings based on the Ivfflat index configuration
350
+ # sess.execute(text(f"SET SESSION nprobe = {self.index.nprobe}"))
351
+ # elif isinstance(self.index, HNSWFlat):
352
+ # # Assuming 'ef_search' is a relevant parameter to be set for the session
353
+ # # Update the session settings based on the HNSW index configuration
354
+ # sess.execute(text(f"SET SESSION ef_search = {self.index.ef_search}"))
355
+
356
+ # Build search results
357
+ search_results: List[Document] = []
358
+ for neighbor in neighbors:
359
+ meta_data_dict = json.loads(neighbor.meta_data) if neighbor.meta_data else {}
360
+ usage_dict = json.loads(neighbor.usage) if neighbor.usage else {}
361
+
362
+ # Convert SingleStore VECTOR type to list
363
+ embedding_list = []
364
+ if neighbor.embedding:
365
+ try:
366
+ embedding_list = json.loads(neighbor.embedding)
367
+ except Exception as e:
368
+ log_error(f"Error extracting vector: {e}")
369
+ embedding_list = []
370
+
371
+ search_results.append(
372
+ Document(
373
+ name=neighbor.name,
374
+ meta_data=meta_data_dict,
375
+ content=neighbor.content,
376
+ embedder=self.embedder,
377
+ embedding=embedding_list,
378
+ usage=usage_dict,
379
+ )
380
+ )
381
+
382
+ if self.reranker:
383
+ search_results = self.reranker.rerank(query=query, documents=search_results)
384
+
385
+ return search_results
386
+
387
+ def drop(self) -> None:
388
+ """
389
+ Delete the table.
390
+ """
391
+ if self.table_exists():
392
+ log_debug(f"Deleting table: {self.collection}")
393
+ self.table.drop(self.db_engine)
394
+
395
+ def exists(self) -> bool:
396
+ """
397
+ Check if the table exists.
398
+
399
+ Returns:
400
+ bool: True if the table exists, False otherwise.
401
+ """
402
+ return self.table_exists()
403
+
404
+ def get_count(self) -> int:
405
+ """
406
+ Get the count of rows in the table.
407
+
408
+ Returns:
409
+ int: The count of rows.
410
+ """
411
+ with self.Session.begin() as sess:
412
+ stmt = select(func.count(self.table.c.name)).select_from(self.table)
413
+ result = sess.execute(stmt).scalar()
414
+ if result is not None:
415
+ return int(result)
416
+ return 0
417
+
418
+ def optimize(self) -> None:
419
+ pass
420
+
421
+ def delete(self) -> bool:
422
+ """
423
+ Clear all rows from the table.
424
+
425
+ Returns:
426
+ bool: True if the table was cleared, False otherwise.
427
+ """
428
+ from sqlalchemy import delete
429
+
430
+ with self.Session.begin() as sess:
431
+ stmt = delete(self.table)
432
+ sess.execute(stmt)
433
+ return True
434
+
435
+ def delete_by_id(self, id: str) -> bool:
436
+ """
437
+ Delete a document by its ID.
438
+ """
439
+ from sqlalchemy import delete
440
+
441
+ try:
442
+ with self.Session.begin() as sess:
443
+ stmt = delete(self.table).where(self.table.c.id == id)
444
+ result = sess.execute(stmt) # type: ignore
445
+ log_info(f"Deleted {result.rowcount} records with ID {id} from table '{self.table.name}'.") # type: ignore
446
+ return result.rowcount > 0 # type: ignore
447
+ except Exception as e:
448
+ log_error(f"Error deleting document with ID {id}: {e}")
449
+ return False
450
+
451
+ def delete_by_content_id(self, content_id: str) -> bool:
452
+ """
453
+ Delete a document by its content ID.
454
+ """
455
+ from sqlalchemy import delete
456
+
457
+ try:
458
+ with self.Session.begin() as sess:
459
+ stmt = delete(self.table).where(self.table.c.content_id == content_id)
460
+ result = sess.execute(stmt) # type: ignore
461
+ log_info(
462
+ f"Deleted {result.rowcount} records with content_id {content_id} from table '{self.table.name}'." # type: ignore
463
+ )
464
+ return result.rowcount > 0 # type: ignore
465
+ except Exception as e:
466
+ log_error(f"Error deleting document with content_id {content_id}: {e}")
467
+ return False
468
+
469
+ def delete_by_name(self, name: str) -> bool:
470
+ """
471
+ Delete a document by its name.
472
+ """
473
+ from sqlalchemy import delete
474
+
475
+ try:
476
+ with self.Session.begin() as sess:
477
+ stmt = delete(self.table).where(self.table.c.name == name)
478
+ result = sess.execute(stmt) # type: ignore
479
+ log_info(f"Deleted {result.rowcount} records with name '{name}' from table '{self.table.name}'.") # type: ignore
480
+ return result.rowcount > 0 # type: ignore
481
+ except Exception as e:
482
+ log_error(f"Error deleting document with name {name}: {e}")
483
+ return False
484
+
485
+ def delete_by_metadata(self, metadata: Dict[str, Any]) -> bool:
486
+ """
487
+ Delete documents by metadata.
488
+ """
489
+ from sqlalchemy import delete
490
+
491
+ try:
492
+ with self.Session.begin() as sess:
493
+ # Convert metadata to JSON string for comparison
494
+ metadata_json = json.dumps(metadata, sort_keys=True)
495
+ stmt = delete(self.table).where(self.table.c.meta_data == metadata_json)
496
+ result = sess.execute(stmt) # type: ignore
497
+ log_info(f"Deleted {result.rowcount} records with metadata {metadata} from table '{self.table.name}'.") # type: ignore
498
+ return result.rowcount > 0 # type: ignore
499
+ except Exception as e:
500
+ log_error(f"Error deleting documents with metadata {metadata}: {e}")
501
+ return False
502
+
503
+ async def async_create(self) -> None:
504
+ raise NotImplementedError(f"Async not supported on {self.__class__.__name__}.")
505
+
506
+ async def async_insert(
507
+ self,
508
+ content_hash: str,
509
+ documents: List[Document],
510
+ filters: Optional[Dict[str, Any]] = None,
511
+ ) -> None:
512
+ if self.embedder.enable_batch and hasattr(self.embedder, "async_get_embeddings_batch_and_usage"):
513
+ # Use batch embedding when enabled and supported
514
+ try:
515
+ # Extract content from all documents
516
+ doc_contents = [doc.content for doc in documents]
517
+
518
+ # Get batch embeddings and usage
519
+ embeddings, usages = await self.embedder.async_get_embeddings_batch_and_usage(doc_contents)
520
+
521
+ # Process documents with pre-computed embeddings
522
+ for j, doc in enumerate(documents):
523
+ try:
524
+ if j < len(embeddings):
525
+ doc.embedding = embeddings[j]
526
+ doc.usage = usages[j] if j < len(usages) else None
527
+ except Exception as e:
528
+ log_error(f"Error assigning batch embedding to document '{doc.name}': {e}")
529
+
530
+ except Exception as e:
531
+ # Check if this is a rate limit error - don't fall back as it would make things worse
532
+ error_str = str(e).lower()
533
+ is_rate_limit = any(
534
+ phrase in error_str
535
+ for phrase in ["rate limit", "too many requests", "429", "trial key", "api calls / minute"]
536
+ )
537
+
538
+ if is_rate_limit:
539
+ log_error(f"Rate limit detected during batch embedding. {e}")
540
+ raise e
541
+ else:
542
+ log_error(f"Async batch embedding failed, falling back to individual embeddings: {e}")
543
+ # Fall back to individual embedding
544
+ embed_tasks = [doc.async_embed(embedder=self.embedder) for doc in documents]
545
+ await asyncio.gather(*embed_tasks, return_exceptions=True)
546
+ else:
547
+ # Use individual embedding
548
+ embed_tasks = [document.async_embed(embedder=self.embedder) for document in documents]
549
+ await asyncio.gather(*embed_tasks, return_exceptions=True)
550
+
551
+ with self.Session.begin() as sess:
552
+ counter = 0
553
+ for document in documents:
554
+ cleaned_content = document.content.replace("\x00", "\ufffd")
555
+ # Include content_hash in ID to ensure uniqueness across different content hashes
556
+ base_id = document.id or md5(cleaned_content.encode()).hexdigest()
557
+ record_id = md5(f"{base_id}_{content_hash}".encode()).hexdigest()
558
+ _id = record_id
559
+
560
+ meta_data_json = json.dumps(document.meta_data)
561
+ usage_json = json.dumps(document.usage)
562
+
563
+ # Convert embedding list to SingleStore VECTOR format
564
+ embeddings = f"[{','.join(map(str, document.embedding))}]" if document.embedding else None
565
+
566
+ stmt = mysql.insert(self.table).values(
567
+ id=_id,
568
+ name=document.name,
569
+ meta_data=meta_data_json,
570
+ content=cleaned_content,
571
+ embedding=embeddings,
572
+ usage=usage_json,
573
+ content_hash=content_hash,
574
+ content_id=document.content_id,
575
+ )
576
+ sess.execute(stmt)
577
+ counter += 1
578
+ log_debug(f"Inserted document: {document.name} ({document.meta_data})")
579
+
580
+ sess.commit()
581
+ log_debug(f"Committed {counter} documents")
582
+
583
+ async def async_upsert(
584
+ self,
585
+ content_hash: str,
586
+ documents: List[Document],
587
+ filters: Optional[Dict[str, Any]] = None,
588
+ ) -> None:
589
+ """
590
+ Upsert (insert or update) documents in the table.
591
+
592
+ Args:
593
+ documents (List[Document]): List of documents to upsert.
594
+ filters (Optional[Dict[str, Any]]): Optional filters for the upsert.
595
+ batch_size (int): Number of documents to upsert in each batch.
596
+ """
597
+
598
+ if self.embedder.enable_batch and hasattr(self.embedder, "async_get_embeddings_batch_and_usage"):
599
+ # Use batch embedding when enabled and supported
600
+ try:
601
+ # Extract content from all documents
602
+ doc_contents = [doc.content for doc in documents]
603
+
604
+ # Get batch embeddings and usage
605
+ embeddings, usages = await self.embedder.async_get_embeddings_batch_and_usage(doc_contents)
606
+
607
+ # Process documents with pre-computed embeddings
608
+ for j, doc in enumerate(documents):
609
+ try:
610
+ if j < len(embeddings):
611
+ doc.embedding = embeddings[j]
612
+ doc.usage = usages[j] if j < len(usages) else None
613
+ except Exception as e:
614
+ log_error(f"Error assigning batch embedding to document '{doc.name}': {e}")
615
+
616
+ except Exception as e:
617
+ # Check if this is a rate limit error - don't fall back as it would make things worse
618
+ error_str = str(e).lower()
619
+ is_rate_limit = any(
620
+ phrase in error_str
621
+ for phrase in ["rate limit", "too many requests", "429", "trial key", "api calls / minute"]
622
+ )
623
+
624
+ if is_rate_limit:
625
+ log_error(f"Rate limit detected during batch embedding. {e}")
626
+ raise e
627
+ else:
628
+ log_error(f"Async batch embedding failed, falling back to individual embeddings: {e}")
629
+ # Fall back to individual embedding
630
+ embed_tasks = [doc.async_embed(embedder=self.embedder) for doc in documents]
631
+ await asyncio.gather(*embed_tasks, return_exceptions=True)
632
+ else:
633
+ # Use individual embedding
634
+ embed_tasks = [document.async_embed(embedder=self.embedder) for document in documents]
635
+ await asyncio.gather(*embed_tasks, return_exceptions=True)
636
+
637
+ with self.Session.begin() as sess:
638
+ counter = 0
639
+ for document in documents:
640
+ cleaned_content = document.content.replace("\x00", "\ufffd")
641
+ # Include content_hash in ID to ensure uniqueness across different content hashes
642
+ base_id = document.id or md5(cleaned_content.encode()).hexdigest()
643
+ record_id = md5(f"{base_id}_{content_hash}".encode()).hexdigest()
644
+ _id = record_id
645
+
646
+ meta_data_json = json.dumps(document.meta_data)
647
+ usage_json = json.dumps(document.usage)
648
+
649
+ # Convert embedding list to SingleStore VECTOR format
650
+ embeddings = f"[{','.join(map(str, document.embedding))}]" if document.embedding else None
651
+ stmt = (
652
+ mysql.insert(self.table)
653
+ .values(
654
+ id=_id,
655
+ name=document.name,
656
+ meta_data=meta_data_json,
657
+ content=cleaned_content,
658
+ embedding=embeddings,
659
+ usage=usage_json,
660
+ content_hash=content_hash,
661
+ content_id=document.content_id,
662
+ )
663
+ .on_duplicate_key_update(
664
+ name=document.name,
665
+ meta_data=meta_data_json,
666
+ content=cleaned_content,
667
+ embedding=embeddings,
668
+ usage=usage_json,
669
+ content_hash=content_hash,
670
+ content_id=document.content_id,
671
+ )
672
+ )
673
+ sess.execute(stmt)
674
+ counter += 1
675
+ log_debug(f"Upserted document: {document.name} ({document.meta_data})")
676
+
677
+ sess.commit()
678
+ log_debug(f"Committed {counter} documents")
679
+
680
+ async def async_search(
681
+ self, query: str, limit: int = 5, filters: Optional[Union[Dict[str, Any], List[FilterExpr]]] = None
682
+ ) -> List[Document]:
683
+ return self.search(query=query, limit=limit, filters=filters)
684
+
685
+ async def async_drop(self) -> None:
686
+ raise NotImplementedError(f"Async not supported on {self.__class__.__name__}.")
687
+
688
+ async def async_exists(self) -> bool:
689
+ raise NotImplementedError(f"Async not supported on {self.__class__.__name__}.")
690
+
691
+ async def async_name_exists(self, name: str) -> bool:
692
+ raise NotImplementedError(f"Async not supported on {self.__class__.__name__}.")
693
+
694
+ def _delete_by_content_hash(self, content_hash: str) -> bool:
695
+ """
696
+ Delete documents by their content hash.
697
+
698
+ Args:
699
+ content_hash (str): The content hash to delete.
700
+
701
+ Returns:
702
+ bool: True if documents were deleted, False otherwise.
703
+ """
704
+ from sqlalchemy import delete
705
+
706
+ try:
707
+ with self.Session.begin() as sess:
708
+ stmt = delete(self.table).where(self.table.c.content_hash == content_hash)
709
+ result = sess.execute(stmt) # type: ignore
710
+ log_info(
711
+ f"Deleted {result.rowcount} records with content_hash '{content_hash}' from table '{self.table.name}'." # type: ignore
712
+ )
713
+ return result.rowcount > 0 # type: ignore
714
+ except Exception as e:
715
+ log_error(f"Error deleting documents with content_hash {content_hash}: {e}")
716
+ return False
717
+
718
+ def update_metadata(self, content_id: str, metadata: Dict[str, Any]) -> None:
719
+ """
720
+ Update the metadata for documents with the given content_id.
721
+
722
+ Args:
723
+ content_id (str): The content ID to update
724
+ metadata (Dict[str, Any]): The metadata to update
725
+ """
726
+ import json
727
+
728
+ try:
729
+ with self.Session.begin() as sess:
730
+ # Find documents with the given content_id
731
+ stmt = select(self.table).where(self.table.c.content_id == content_id)
732
+ result = sess.execute(stmt) # type: ignore
733
+
734
+ updated_count = 0
735
+ for row in result:
736
+ # Parse existing metadata
737
+ current_metadata = json.loads(row.meta_data) if row.meta_data else {}
738
+
739
+ # Merge existing metadata with new metadata
740
+ updated_metadata = current_metadata.copy()
741
+ updated_metadata.update(metadata)
742
+
743
+ # Also update filters field within the metadata JSON
744
+ if "filters" not in updated_metadata:
745
+ updated_metadata["filters"] = {}
746
+ if isinstance(updated_metadata["filters"], dict):
747
+ updated_metadata["filters"].update(metadata)
748
+ else:
749
+ updated_metadata["filters"] = metadata
750
+
751
+ # Update the document (only meta_data column exists)
752
+ update_stmt = (
753
+ update(self.table)
754
+ .where(self.table.c.id == row.id)
755
+ .values(meta_data=json.dumps(updated_metadata))
756
+ )
757
+ sess.execute(update_stmt)
758
+ updated_count += 1
759
+
760
+ if updated_count == 0:
761
+ log_debug(f"No documents found with content_id: {content_id}")
762
+ else:
763
+ log_debug(f"Updated metadata for {updated_count} documents with content_id: {content_id}")
764
+
765
+ except Exception as e:
766
+ log_error(f"Error updating metadata for content_id '{content_id}': {e}")
767
+ raise
768
+
769
+ def get_supported_search_types(self) -> List[str]:
770
+ """Get the supported search types for this vector database."""
771
+ return [] # SingleStore doesn't use SearchType enum