camel-ai 0.2.59__py3-none-any.whl → 0.2.82__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of camel-ai might be problematic. Click here for more details.

Files changed (506) hide show
  1. camel/__init__.py +3 -3
  2. camel/agents/__init__.py +2 -2
  3. camel/agents/_types.py +9 -4
  4. camel/agents/_utils.py +40 -2
  5. camel/agents/base.py +2 -2
  6. camel/agents/chat_agent.py +5012 -902
  7. camel/agents/critic_agent.py +2 -2
  8. camel/agents/deductive_reasoner_agent.py +56 -56
  9. camel/agents/embodied_agent.py +2 -2
  10. camel/agents/knowledge_graph_agent.py +20 -20
  11. camel/agents/mcp_agent.py +39 -36
  12. camel/agents/multi_hop_generator_agent.py +3 -3
  13. camel/agents/programmed_agent_instruction.py +2 -2
  14. camel/agents/repo_agent.py +4 -3
  15. camel/agents/role_assignment_agent.py +2 -2
  16. camel/agents/search_agent.py +2 -2
  17. camel/agents/task_agent.py +2 -2
  18. camel/agents/tool_agents/__init__.py +2 -2
  19. camel/agents/tool_agents/base.py +2 -2
  20. camel/agents/tool_agents/hugging_face_tool_agent.py +3 -3
  21. camel/benchmarks/__init__.py +2 -2
  22. camel/benchmarks/apibank.py +5 -5
  23. camel/benchmarks/apibench.py +2 -2
  24. camel/benchmarks/base.py +2 -2
  25. camel/benchmarks/browsecomp.py +44 -33
  26. camel/benchmarks/gaia.py +17 -13
  27. camel/benchmarks/mock_website/README.md +94 -0
  28. camel/benchmarks/mock_website/mock_web.py +299 -0
  29. camel/benchmarks/mock_website/requirements.txt +3 -0
  30. camel/benchmarks/mock_website/shopping_mall/app.py +465 -0
  31. camel/benchmarks/mock_website/task.json +104 -0
  32. camel/benchmarks/nexus.py +3 -3
  33. camel/benchmarks/ragbench.py +2 -2
  34. camel/bots/__init__.py +2 -2
  35. camel/bots/discord/__init__.py +2 -2
  36. camel/bots/discord/discord_app.py +2 -2
  37. camel/bots/discord/discord_installation.py +2 -2
  38. camel/bots/discord/discord_store.py +3 -3
  39. camel/bots/slack/__init__.py +2 -2
  40. camel/bots/slack/models.py +4 -4
  41. camel/bots/slack/slack_app.py +2 -2
  42. camel/bots/telegram_bot.py +2 -2
  43. camel/configs/__init__.py +26 -2
  44. camel/configs/aihubmix_config.py +90 -0
  45. camel/configs/aiml_config.py +2 -2
  46. camel/configs/amd_config.py +70 -0
  47. camel/configs/anthropic_config.py +8 -7
  48. camel/configs/base_config.py +2 -2
  49. camel/configs/bedrock_config.py +5 -3
  50. camel/configs/cerebras_config.py +98 -0
  51. camel/configs/cohere_config.py +3 -3
  52. camel/configs/cometapi_config.py +106 -0
  53. camel/configs/crynux_config.py +94 -0
  54. camel/configs/deepseek_config.py +9 -8
  55. camel/configs/gemini_config.py +6 -4
  56. camel/configs/groq_config.py +6 -4
  57. camel/configs/internlm_config.py +6 -4
  58. camel/configs/litellm_config.py +2 -2
  59. camel/configs/lmstudio_config.py +6 -4
  60. camel/configs/minimax_config.py +95 -0
  61. camel/configs/mistral_config.py +3 -3
  62. camel/configs/modelscope_config.py +5 -3
  63. camel/configs/moonshot_config.py +2 -2
  64. camel/configs/nebius_config.py +105 -0
  65. camel/configs/netmind_config.py +2 -2
  66. camel/configs/novita_config.py +2 -2
  67. camel/configs/nvidia_config.py +2 -2
  68. camel/configs/ollama_config.py +2 -2
  69. camel/configs/openai_config.py +8 -3
  70. camel/configs/openrouter_config.py +6 -4
  71. camel/configs/ppio_config.py +2 -2
  72. camel/configs/qianfan_config.py +85 -0
  73. camel/configs/qwen_config.py +2 -2
  74. camel/configs/reka_config.py +3 -3
  75. camel/configs/samba_config.py +8 -6
  76. camel/configs/sglang_config.py +2 -2
  77. camel/configs/siliconflow_config.py +2 -2
  78. camel/configs/togetherai_config.py +2 -2
  79. camel/configs/vllm_config.py +4 -2
  80. camel/configs/watsonx_config.py +2 -2
  81. camel/configs/yi_config.py +6 -4
  82. camel/configs/zhipuai_config.py +6 -4
  83. camel/{data_collector → data_collectors}/__init__.py +2 -2
  84. camel/{data_collector → data_collectors}/alpaca_collector.py +19 -10
  85. camel/{data_collector → data_collectors}/base.py +2 -2
  86. camel/{data_collector → data_collectors}/sharegpt_collector.py +3 -3
  87. camel/datagen/__init__.py +2 -2
  88. camel/datagen/cot_datagen.py +32 -37
  89. camel/datagen/evol_instruct/__init__.py +2 -2
  90. camel/datagen/evol_instruct/evol_instruct.py +2 -2
  91. camel/datagen/evol_instruct/scorer.py +24 -25
  92. camel/datagen/evol_instruct/templates.py +48 -48
  93. camel/datagen/self_improving_cot.py +5 -5
  94. camel/datagen/self_instruct/__init__.py +2 -2
  95. camel/datagen/self_instruct/filter/__init__.py +2 -2
  96. camel/datagen/self_instruct/filter/filter_function.py +2 -2
  97. camel/datagen/self_instruct/filter/filter_registry.py +2 -2
  98. camel/datagen/self_instruct/filter/instruction_filter.py +2 -2
  99. camel/datagen/self_instruct/self_instruct.py +2 -2
  100. camel/datagen/self_instruct/templates.py +47 -47
  101. camel/datagen/source2synth/__init__.py +2 -2
  102. camel/datagen/source2synth/data_processor.py +2 -2
  103. camel/datagen/source2synth/models.py +2 -2
  104. camel/datagen/source2synth/user_data_processor_config.py +2 -2
  105. camel/datahubs/__init__.py +2 -2
  106. camel/datahubs/base.py +2 -2
  107. camel/datahubs/huggingface.py +2 -2
  108. camel/datahubs/models.py +2 -2
  109. camel/datasets/__init__.py +2 -2
  110. camel/datasets/base_generator.py +41 -12
  111. camel/datasets/few_shot_generator.py +18 -18
  112. camel/datasets/models.py +3 -3
  113. camel/datasets/self_instruct_generator.py +2 -2
  114. camel/datasets/static_dataset.py +152 -2
  115. camel/embeddings/__init__.py +2 -2
  116. camel/embeddings/azure_embedding.py +2 -2
  117. camel/embeddings/base.py +2 -2
  118. camel/embeddings/gemini_embedding.py +2 -2
  119. camel/embeddings/jina_embedding.py +10 -3
  120. camel/embeddings/mistral_embedding.py +2 -2
  121. camel/embeddings/openai_compatible_embedding.py +2 -2
  122. camel/embeddings/openai_embedding.py +2 -2
  123. camel/embeddings/sentence_transformers_embeddings.py +4 -4
  124. camel/embeddings/together_embedding.py +2 -2
  125. camel/embeddings/vlm_embedding.py +11 -4
  126. camel/environments/__init__.py +14 -2
  127. camel/environments/models.py +2 -2
  128. camel/environments/multi_step.py +2 -2
  129. camel/environments/rlcards_env.py +860 -0
  130. camel/environments/single_step.py +30 -5
  131. camel/environments/tic_tac_toe.py +3 -3
  132. camel/extractors/__init__.py +2 -2
  133. camel/extractors/base.py +2 -2
  134. camel/extractors/python_strategies.py +2 -2
  135. camel/generators.py +2 -2
  136. camel/human.py +2 -2
  137. camel/interpreters/__init__.py +4 -2
  138. camel/interpreters/base.py +16 -3
  139. camel/interpreters/docker/Dockerfile +53 -7
  140. camel/interpreters/docker_interpreter.py +70 -11
  141. camel/interpreters/e2b_interpreter.py +59 -11
  142. camel/interpreters/internal_python_interpreter.py +81 -4
  143. camel/interpreters/interpreter_error.py +2 -2
  144. camel/interpreters/ipython_interpreter.py +23 -5
  145. camel/interpreters/microsandbox_interpreter.py +395 -0
  146. camel/interpreters/subprocess_interpreter.py +36 -4
  147. camel/loaders/__init__.py +17 -5
  148. camel/loaders/apify_reader.py +2 -2
  149. camel/loaders/base_io.py +2 -2
  150. camel/loaders/base_loader.py +85 -0
  151. camel/loaders/chunkr_reader.py +128 -93
  152. camel/loaders/crawl4ai_reader.py +2 -2
  153. camel/loaders/firecrawl_reader.py +6 -6
  154. camel/loaders/jina_url_reader.py +2 -2
  155. camel/loaders/markitdown.py +2 -2
  156. camel/loaders/mineru_extractor.py +2 -2
  157. camel/loaders/mistral_reader.py +148 -0
  158. camel/loaders/scrapegraph_reader.py +2 -2
  159. camel/loaders/unstructured_io.py +2 -2
  160. camel/logger.py +5 -5
  161. camel/memories/__init__.py +2 -2
  162. camel/memories/agent_memories.py +86 -3
  163. camel/memories/base.py +36 -2
  164. camel/memories/blocks/__init__.py +2 -2
  165. camel/memories/blocks/chat_history_block.py +126 -9
  166. camel/memories/blocks/vectordb_block.py +10 -3
  167. camel/memories/context_creators/__init__.py +2 -2
  168. camel/memories/context_creators/score_based.py +31 -239
  169. camel/memories/records.py +98 -13
  170. camel/messages/__init__.py +2 -2
  171. camel/messages/base.py +193 -46
  172. camel/messages/conversion/__init__.py +2 -2
  173. camel/messages/conversion/alpaca.py +2 -2
  174. camel/messages/conversion/conversation_models.py +2 -2
  175. camel/messages/conversion/sharegpt/__init__.py +2 -2
  176. camel/messages/conversion/sharegpt/function_call_formatter.py +2 -2
  177. camel/messages/conversion/sharegpt/hermes/__init__.py +2 -2
  178. camel/messages/conversion/sharegpt/hermes/hermes_function_formatter.py +2 -2
  179. camel/messages/func_message.py +54 -17
  180. camel/models/__init__.py +18 -2
  181. camel/models/_utils.py +3 -3
  182. camel/models/aihubmix_model.py +83 -0
  183. camel/models/aiml_model.py +11 -18
  184. camel/models/amd_model.py +101 -0
  185. camel/models/anthropic_model.py +127 -20
  186. camel/models/aws_bedrock_model.py +12 -35
  187. camel/models/azure_openai_model.py +263 -63
  188. camel/models/base_audio_model.py +5 -3
  189. camel/models/base_model.py +195 -26
  190. camel/models/cerebras_model.py +83 -0
  191. camel/models/cohere_model.py +81 -21
  192. camel/models/cometapi_model.py +83 -0
  193. camel/models/crynux_model.py +87 -0
  194. camel/models/deepseek_model.py +61 -59
  195. camel/models/fish_audio_model.py +8 -2
  196. camel/models/gemini_model.py +439 -30
  197. camel/models/groq_model.py +11 -19
  198. camel/models/internlm_model.py +11 -18
  199. camel/models/litellm_model.py +94 -34
  200. camel/models/lmstudio_model.py +17 -20
  201. camel/models/minimax_model.py +83 -0
  202. camel/models/mistral_model.py +84 -19
  203. camel/models/model_factory.py +49 -6
  204. camel/models/model_manager.py +33 -11
  205. camel/models/modelscope_model.py +13 -193
  206. camel/models/moonshot_model.py +195 -21
  207. camel/models/nebius_model.py +83 -0
  208. camel/models/nemotron_model.py +19 -9
  209. camel/models/netmind_model.py +11 -18
  210. camel/models/novita_model.py +11 -18
  211. camel/models/nvidia_model.py +11 -18
  212. camel/models/ollama_model.py +14 -21
  213. camel/models/openai_audio_models.py +2 -2
  214. camel/models/openai_compatible_model.py +234 -27
  215. camel/models/openai_model.py +255 -39
  216. camel/models/openrouter_model.py +11 -19
  217. camel/models/ppio_model.py +11 -18
  218. camel/models/qianfan_model.py +89 -0
  219. camel/models/qwen_model.py +13 -193
  220. camel/models/reka_model.py +90 -21
  221. camel/models/reward/__init__.py +2 -2
  222. camel/models/reward/base_reward_model.py +2 -2
  223. camel/models/reward/evaluator.py +2 -2
  224. camel/models/reward/nemotron_model.py +2 -2
  225. camel/models/reward/skywork_model.py +2 -2
  226. camel/models/samba_model.py +117 -49
  227. camel/models/sglang_model.py +162 -42
  228. camel/models/siliconflow_model.py +12 -35
  229. camel/models/stub_model.py +10 -7
  230. camel/models/togetherai_model.py +11 -18
  231. camel/models/vllm_model.py +10 -18
  232. camel/models/volcano_model.py +16 -20
  233. camel/models/watsonx_model.py +69 -19
  234. camel/models/yi_model.py +11 -18
  235. camel/models/zhipuai_model.py +70 -18
  236. camel/parsers/__init__.py +18 -0
  237. camel/parsers/mcp_tool_call_parser.py +176 -0
  238. camel/personas/__init__.py +2 -2
  239. camel/personas/persona.py +2 -2
  240. camel/personas/persona_hub.py +2 -2
  241. camel/prompts/__init__.py +2 -2
  242. camel/prompts/ai_society.py +2 -2
  243. camel/prompts/base.py +2 -2
  244. camel/prompts/code.py +2 -2
  245. camel/prompts/evaluation.py +2 -2
  246. camel/prompts/generate_text_embedding_data.py +2 -2
  247. camel/prompts/image_craft.py +2 -2
  248. camel/prompts/misalignment.py +2 -2
  249. camel/prompts/multi_condition_image_craft.py +2 -2
  250. camel/prompts/object_recognition.py +2 -2
  251. camel/prompts/persona_hub.py +3 -3
  252. camel/prompts/prompt_templates.py +2 -2
  253. camel/prompts/role_description_prompt_template.py +2 -2
  254. camel/prompts/solution_extraction.py +8 -8
  255. camel/prompts/task_prompt_template.py +2 -2
  256. camel/prompts/translation.py +2 -2
  257. camel/prompts/video_description_prompt.py +3 -3
  258. camel/responses/__init__.py +2 -2
  259. camel/responses/agent_responses.py +2 -2
  260. camel/retrievers/__init__.py +2 -2
  261. camel/retrievers/auto_retriever.py +23 -3
  262. camel/retrievers/base.py +2 -2
  263. camel/retrievers/bm25_retriever.py +3 -4
  264. camel/retrievers/cohere_rerank_retriever.py +2 -2
  265. camel/retrievers/hybrid_retrival.py +4 -4
  266. camel/retrievers/vector_retriever.py +2 -2
  267. camel/runtimes/Dockerfile.multi-toolkit +90 -0
  268. camel/{runtime → runtimes}/__init__.py +2 -2
  269. camel/runtimes/api.py +153 -0
  270. camel/{runtime → runtimes}/base.py +2 -2
  271. camel/{runtime → runtimes}/configs.py +13 -13
  272. camel/{runtime → runtimes}/daytona_runtime.py +18 -19
  273. camel/{runtime → runtimes}/docker_runtime.py +13 -13
  274. camel/{runtime → runtimes}/llm_guard_runtime.py +28 -28
  275. camel/{runtime → runtimes}/remote_http_runtime.py +12 -12
  276. camel/{runtime → runtimes}/ubuntu_docker_runtime.py +3 -3
  277. camel/{runtime → runtimes}/utils/__init__.py +2 -2
  278. camel/{runtime → runtimes}/utils/function_risk_toolkit.py +2 -2
  279. camel/{runtime → runtimes}/utils/ignore_risk_toolkit.py +2 -2
  280. camel/schemas/__init__.py +2 -2
  281. camel/schemas/base.py +2 -2
  282. camel/schemas/openai_converter.py +3 -3
  283. camel/schemas/outlines_converter.py +2 -2
  284. camel/services/agent_openapi_server.py +380 -0
  285. camel/societies/__init__.py +4 -2
  286. camel/societies/babyagi_playing.py +2 -2
  287. camel/societies/role_playing.py +201 -80
  288. camel/societies/workforce/__init__.py +10 -3
  289. camel/societies/workforce/base.py +9 -5
  290. camel/societies/workforce/events.py +143 -0
  291. camel/societies/workforce/prompts.py +258 -33
  292. camel/societies/workforce/role_playing_worker.py +95 -30
  293. camel/societies/workforce/single_agent_worker.py +659 -30
  294. camel/societies/workforce/structured_output_handler.py +512 -0
  295. camel/societies/workforce/task_channel.py +182 -38
  296. camel/societies/workforce/utils.py +784 -18
  297. camel/societies/workforce/worker.py +96 -28
  298. camel/societies/workforce/workflow_memory_manager.py +1746 -0
  299. camel/societies/workforce/workforce.py +5730 -366
  300. camel/societies/workforce/workforce_callback.py +103 -0
  301. camel/societies/workforce/workforce_logger.py +647 -0
  302. camel/societies/workforce/workforce_metrics.py +33 -0
  303. camel/storages/__init__.py +10 -2
  304. camel/storages/graph_storages/__init__.py +2 -2
  305. camel/storages/graph_storages/base.py +2 -2
  306. camel/storages/graph_storages/graph_element.py +2 -2
  307. camel/storages/graph_storages/nebula_graph.py +4 -4
  308. camel/storages/graph_storages/neo4j_graph.py +7 -7
  309. camel/storages/key_value_storages/__init__.py +2 -2
  310. camel/storages/key_value_storages/base.py +2 -2
  311. camel/storages/key_value_storages/in_memory.py +2 -2
  312. camel/storages/key_value_storages/json.py +17 -4
  313. camel/storages/key_value_storages/mem0_cloud.py +50 -49
  314. camel/storages/key_value_storages/redis.py +2 -2
  315. camel/storages/object_storages/__init__.py +2 -2
  316. camel/storages/object_storages/amazon_s3.py +2 -2
  317. camel/storages/object_storages/azure_blob.py +2 -2
  318. camel/storages/object_storages/base.py +2 -2
  319. camel/storages/object_storages/google_cloud.py +3 -3
  320. camel/storages/vectordb_storages/__init__.py +12 -2
  321. camel/storages/vectordb_storages/base.py +2 -2
  322. camel/storages/vectordb_storages/chroma.py +731 -0
  323. camel/storages/vectordb_storages/faiss.py +712 -0
  324. camel/storages/vectordb_storages/milvus.py +2 -2
  325. camel/storages/vectordb_storages/oceanbase.py +16 -17
  326. camel/storages/vectordb_storages/pgvector.py +349 -0
  327. camel/storages/vectordb_storages/qdrant.py +6 -6
  328. camel/storages/vectordb_storages/surreal.py +372 -0
  329. camel/storages/vectordb_storages/tidb.py +11 -8
  330. camel/storages/vectordb_storages/weaviate.py +714 -0
  331. camel/tasks/__init__.py +2 -2
  332. camel/tasks/task.py +366 -27
  333. camel/tasks/task_prompt.py +3 -3
  334. camel/terminators/__init__.py +2 -2
  335. camel/terminators/base.py +2 -2
  336. camel/terminators/response_terminator.py +2 -2
  337. camel/terminators/token_limit_terminator.py +2 -2
  338. camel/toolkits/__init__.py +58 -10
  339. camel/toolkits/aci_toolkit.py +66 -21
  340. camel/toolkits/arxiv_toolkit.py +8 -8
  341. camel/toolkits/ask_news_toolkit.py +2 -2
  342. camel/toolkits/async_browser_toolkit.py +174 -575
  343. camel/toolkits/audio_analysis_toolkit.py +3 -3
  344. camel/toolkits/base.py +65 -7
  345. camel/toolkits/bohrium_toolkit.py +318 -0
  346. camel/toolkits/browser_toolkit.py +306 -566
  347. camel/toolkits/browser_toolkit_commons.py +568 -0
  348. camel/toolkits/code_execution.py +67 -11
  349. camel/toolkits/context_summarizer_toolkit.py +684 -0
  350. camel/toolkits/craw4ai_toolkit.py +93 -0
  351. camel/toolkits/dappier_toolkit.py +12 -8
  352. camel/toolkits/data_commons_toolkit.py +2 -2
  353. camel/toolkits/dingtalk.py +1135 -0
  354. camel/toolkits/earth_science_toolkit.py +5367 -0
  355. camel/toolkits/edgeone_pages_mcp_toolkit.py +49 -0
  356. camel/toolkits/excel_toolkit.py +910 -70
  357. camel/toolkits/file_toolkit.py +1402 -0
  358. camel/toolkits/function_tool.py +128 -20
  359. camel/toolkits/github_toolkit.py +148 -43
  360. camel/toolkits/gmail_toolkit.py +1839 -0
  361. camel/toolkits/google_calendar_toolkit.py +40 -6
  362. camel/toolkits/google_drive_mcp_toolkit.py +54 -0
  363. camel/toolkits/google_maps_toolkit.py +2 -2
  364. camel/toolkits/google_scholar_toolkit.py +2 -2
  365. camel/toolkits/human_toolkit.py +36 -12
  366. camel/toolkits/hybrid_browser_toolkit/__init__.py +18 -0
  367. camel/toolkits/hybrid_browser_toolkit/config_loader.py +185 -0
  368. camel/toolkits/hybrid_browser_toolkit/hybrid_browser_toolkit.py +246 -0
  369. camel/toolkits/hybrid_browser_toolkit/hybrid_browser_toolkit_ts.py +1973 -0
  370. camel/toolkits/hybrid_browser_toolkit/installer.py +203 -0
  371. camel/toolkits/hybrid_browser_toolkit/ts/package-lock.json +4589 -0
  372. camel/toolkits/hybrid_browser_toolkit/ts/package.json +33 -0
  373. camel/toolkits/hybrid_browser_toolkit/ts/src/browser-scripts.js +125 -0
  374. camel/toolkits/hybrid_browser_toolkit/ts/src/browser-session.ts +1929 -0
  375. camel/toolkits/hybrid_browser_toolkit/ts/src/config-loader.ts +233 -0
  376. camel/toolkits/hybrid_browser_toolkit/ts/src/hybrid-browser-toolkit.ts +589 -0
  377. camel/toolkits/hybrid_browser_toolkit/ts/src/index.ts +7 -0
  378. camel/toolkits/hybrid_browser_toolkit/ts/src/parent-child-filter.ts +226 -0
  379. camel/toolkits/hybrid_browser_toolkit/ts/src/snapshot-parser.ts +219 -0
  380. camel/toolkits/hybrid_browser_toolkit/ts/src/som-screenshot-injected.ts +543 -0
  381. camel/toolkits/hybrid_browser_toolkit/ts/src/types.ts +129 -0
  382. camel/toolkits/hybrid_browser_toolkit/ts/tsconfig.json +27 -0
  383. camel/toolkits/hybrid_browser_toolkit/ts/websocket-server.js +319 -0
  384. camel/toolkits/hybrid_browser_toolkit/ws_wrapper.py +1037 -0
  385. camel/toolkits/hybrid_browser_toolkit_py/__init__.py +17 -0
  386. camel/toolkits/hybrid_browser_toolkit_py/actions.py +575 -0
  387. camel/toolkits/hybrid_browser_toolkit_py/agent.py +311 -0
  388. camel/toolkits/hybrid_browser_toolkit_py/browser_session.py +787 -0
  389. camel/toolkits/hybrid_browser_toolkit_py/config_loader.py +490 -0
  390. camel/toolkits/hybrid_browser_toolkit_py/hybrid_browser_toolkit.py +2390 -0
  391. camel/toolkits/hybrid_browser_toolkit_py/snapshot.py +233 -0
  392. camel/toolkits/hybrid_browser_toolkit_py/stealth_script.js +0 -0
  393. camel/toolkits/hybrid_browser_toolkit_py/unified_analyzer.js +1043 -0
  394. camel/toolkits/image_analysis_toolkit.py +3 -3
  395. camel/toolkits/image_generation_toolkit.py +390 -0
  396. camel/toolkits/jina_reranker_toolkit.py +195 -79
  397. camel/toolkits/klavis_toolkit.py +7 -3
  398. camel/toolkits/linkedin_toolkit.py +2 -2
  399. camel/toolkits/markitdown_toolkit.py +104 -0
  400. camel/toolkits/math_toolkit.py +66 -12
  401. camel/toolkits/mcp_toolkit.py +841 -600
  402. camel/toolkits/memory_toolkit.py +7 -3
  403. camel/toolkits/meshy_toolkit.py +2 -2
  404. camel/toolkits/message_agent_toolkit.py +608 -0
  405. camel/toolkits/message_integration.py +724 -0
  406. camel/toolkits/mineru_toolkit.py +2 -2
  407. camel/toolkits/minimax_mcp_toolkit.py +195 -0
  408. camel/toolkits/networkx_toolkit.py +2 -2
  409. camel/toolkits/note_taking_toolkit.py +277 -0
  410. camel/toolkits/notion_mcp_toolkit.py +224 -0
  411. camel/toolkits/notion_toolkit.py +2 -2
  412. camel/toolkits/open_api_specs/biztoc/__init__.py +2 -2
  413. camel/toolkits/open_api_specs/biztoc/ai-plugin.json +1 -1
  414. camel/toolkits/open_api_specs/coursera/__init__.py +2 -2
  415. camel/toolkits/open_api_specs/create_qr_code/__init__.py +2 -2
  416. camel/toolkits/open_api_specs/klarna/__init__.py +2 -2
  417. camel/toolkits/open_api_specs/nasa_apod/__init__.py +2 -2
  418. camel/toolkits/open_api_specs/outschool/__init__.py +2 -2
  419. camel/toolkits/open_api_specs/outschool/ai-plugin.json +1 -1
  420. camel/toolkits/open_api_specs/outschool/openapi.yaml +1 -1
  421. camel/toolkits/open_api_specs/outschool/paths/__init__.py +2 -2
  422. camel/toolkits/open_api_specs/outschool/paths/get_classes.py +2 -2
  423. camel/toolkits/open_api_specs/outschool/paths/search_teachers.py +2 -2
  424. camel/toolkits/open_api_specs/security_config.py +2 -2
  425. camel/toolkits/open_api_specs/speak/__init__.py +2 -2
  426. camel/toolkits/open_api_specs/web_scraper/__init__.py +2 -2
  427. camel/toolkits/open_api_specs/web_scraper/ai-plugin.json +1 -1
  428. camel/toolkits/open_api_specs/web_scraper/paths/__init__.py +2 -2
  429. camel/toolkits/open_api_specs/web_scraper/paths/scraper.py +2 -2
  430. camel/toolkits/open_api_toolkit.py +2 -2
  431. camel/toolkits/openbb_toolkit.py +7 -3
  432. camel/toolkits/origene_mcp_toolkit.py +56 -0
  433. camel/toolkits/page_script.js +86 -74
  434. camel/toolkits/playwright_mcp_toolkit.py +27 -32
  435. camel/toolkits/pptx_toolkit.py +790 -0
  436. camel/toolkits/pubmed_toolkit.py +2 -2
  437. camel/toolkits/pulse_mcp_search_toolkit.py +2 -2
  438. camel/toolkits/pyautogui_toolkit.py +2 -2
  439. camel/toolkits/reddit_toolkit.py +2 -2
  440. camel/toolkits/resend_toolkit.py +168 -0
  441. camel/toolkits/retrieval_toolkit.py +2 -2
  442. camel/toolkits/screenshot_toolkit.py +213 -0
  443. camel/toolkits/search_toolkit.py +539 -146
  444. camel/toolkits/searxng_toolkit.py +2 -2
  445. camel/toolkits/semantic_scholar_toolkit.py +2 -2
  446. camel/toolkits/slack_toolkit.py +108 -58
  447. camel/toolkits/sql_toolkit.py +712 -0
  448. camel/toolkits/stripe_toolkit.py +2 -2
  449. camel/toolkits/sympy_toolkit.py +3 -3
  450. camel/toolkits/task_planning_toolkit.py +134 -0
  451. camel/toolkits/terminal_toolkit/__init__.py +18 -0
  452. camel/toolkits/terminal_toolkit/terminal_toolkit.py +1070 -0
  453. camel/toolkits/terminal_toolkit/utils.py +532 -0
  454. camel/toolkits/thinking_toolkit.py +3 -3
  455. camel/toolkits/twitter_toolkit.py +8 -3
  456. camel/toolkits/vertex_ai_veo_toolkit.py +590 -0
  457. camel/toolkits/video_analysis_toolkit.py +112 -29
  458. camel/toolkits/video_download_toolkit.py +22 -16
  459. camel/toolkits/weather_toolkit.py +2 -2
  460. camel/toolkits/web_deploy_toolkit.py +1219 -0
  461. camel/toolkits/wechat_official_toolkit.py +483 -0
  462. camel/toolkits/whatsapp_toolkit.py +2 -2
  463. camel/toolkits/wolfram_alpha_toolkit.py +53 -25
  464. camel/toolkits/zapier_toolkit.py +7 -3
  465. camel/types/__init__.py +4 -4
  466. camel/types/agents/__init__.py +2 -2
  467. camel/types/agents/tool_calling_record.py +6 -3
  468. camel/types/enums.py +454 -35
  469. camel/types/mcp_registries.py +2 -2
  470. camel/types/openai_types.py +4 -4
  471. camel/types/unified_model_type.py +43 -6
  472. camel/utils/__init__.py +20 -2
  473. camel/utils/async_func.py +2 -2
  474. camel/utils/chunker/__init__.py +2 -2
  475. camel/utils/chunker/base.py +2 -2
  476. camel/utils/chunker/code_chunker.py +2 -2
  477. camel/utils/chunker/uio_chunker.py +2 -2
  478. camel/utils/commons.py +65 -7
  479. camel/utils/constants.py +5 -2
  480. camel/utils/context_utils.py +1134 -0
  481. camel/utils/deduplication.py +2 -2
  482. camel/utils/filename.py +2 -2
  483. camel/utils/langfuse.py +258 -0
  484. camel/utils/mcp.py +140 -6
  485. camel/utils/mcp_client.py +1056 -0
  486. camel/utils/message_summarizer.py +148 -0
  487. camel/utils/response_format.py +2 -2
  488. camel/utils/token_counting.py +45 -22
  489. camel/utils/tool_result.py +44 -0
  490. camel/verifiers/__init__.py +2 -2
  491. camel/verifiers/base.py +2 -2
  492. camel/verifiers/math_verifier.py +2 -2
  493. camel/verifiers/models.py +2 -2
  494. camel/verifiers/physics_verifier.py +2 -2
  495. camel/verifiers/python_verifier.py +2 -2
  496. {camel_ai-0.2.59.dist-info → camel_ai-0.2.82.dist-info}/METADATA +349 -108
  497. camel_ai-0.2.82.dist-info/RECORD +507 -0
  498. {camel_ai-0.2.59.dist-info → camel_ai-0.2.82.dist-info}/WHEEL +1 -1
  499. {camel_ai-0.2.59.dist-info → camel_ai-0.2.82.dist-info}/licenses/LICENSE +1 -1
  500. camel/loaders/pandas_reader.py +0 -368
  501. camel/runtime/api.py +0 -97
  502. camel/toolkits/dalle_toolkit.py +0 -171
  503. camel/toolkits/file_write_toolkit.py +0 -395
  504. camel/toolkits/openai_agent_toolkit.py +0 -135
  505. camel/toolkits/terminal_toolkit.py +0 -1037
  506. camel_ai-0.2.59.dist-info/RECORD +0 -410
@@ -0,0 +1,714 @@
1
+ # ========= Copyright 2023-2025 @ CAMEL-AI.org. All Rights Reserved. =========
2
+ # Licensed under the Apache License, Version 2.0 (the "License");
3
+ # you may not use this file except in compliance with the License.
4
+ # You may obtain a copy of the License at
5
+ #
6
+ # http://www.apache.org/licenses/LICENSE-2.0
7
+ #
8
+ # Unless required by applicable law or agreed to in writing, software
9
+ # distributed under the License is distributed on an "AS IS" BASIS,
10
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11
+ # See the License for the specific language governing permissions and
12
+ # limitations under the License.
13
+ # ========= Copyright 2023-2025 @ CAMEL-AI.org. All Rights Reserved. =========
14
+ import json
15
+ import math
16
+ import re
17
+ from datetime import datetime
18
+ from typing import (
19
+ TYPE_CHECKING,
20
+ Any,
21
+ Dict,
22
+ List,
23
+ Literal,
24
+ Optional,
25
+ Union,
26
+ cast,
27
+ )
28
+
29
+ if TYPE_CHECKING:
30
+ from weaviate import WeaviateClient
31
+
32
+ from camel.logger import get_logger
33
+ from camel.storages.vectordb_storages import (
34
+ BaseVectorStorage,
35
+ VectorDBQuery,
36
+ VectorDBQueryResult,
37
+ VectorDBStatus,
38
+ VectorRecord,
39
+ )
40
+ from camel.utils import dependencies_required
41
+
42
+ logger = get_logger(__name__)
43
+
44
+
45
+ # Type definitions for configuration options
46
+ ConnectionType = Literal["local", "cloud", "embedded", "custom"]
47
+ VectorIndexType = Literal["hnsw", "flat"]
48
+ DistanceMetric = Literal["cosine", "dot", "l2-squared", "hamming", "manhattan"]
49
+
50
+
51
+ class WeaviateStorage(BaseVectorStorage):
52
+ r"""An implementation of the `BaseVectorStorage` for interacting with
53
+ Weaviate, a cloud-native vector search engine.
54
+
55
+ This class provides multiple ways to connect to Weaviate instances:
56
+ - Weaviate Cloud (WCD)
57
+ - Local Docker/Kubernetes instances
58
+ - Embedded Weaviate
59
+ - Custom connection parameters
60
+
61
+ Args:
62
+ vector_dim (int): The dimension of storing vectors.
63
+ collection_name (Optional[str], optional): Name for the collection in
64
+ Weaviate. If not provided, generates a unique name based on current
65
+ timestamp. (default: :obj:`None`)
66
+ connection_type (ConnectionType, optional): Type of connection to use.
67
+ Supported types: 'local', 'cloud', 'embedded', 'custom'.
68
+ (default: :obj:`"local"`)
69
+
70
+ # Weaviate Cloud parameters
71
+ wcd_cluster_url (Optional[str], optional): Weaviate Cloud cluster URL.
72
+ Required when connection_type='cloud'.
73
+ wcd_api_key (Optional[str], optional): Weaviate Cloud API key.
74
+ Required when connection_type='cloud'.
75
+
76
+ # Local instance parameters
77
+ local_host (str, optional): Local Weaviate host.
78
+ (default: :obj:`"localhost"`)
79
+ local_port (int, optional): Local Weaviate HTTP port.
80
+ (default: :obj:`8080`)
81
+ local_grpc_port (int, optional): Local Weaviate gRPC port.
82
+ (default: :obj:`50051`)
83
+ local_auth_credentials (Optional[Union[str, Any]], optional):
84
+ Authentication credentials for local instance. Can be an API key
85
+ string or Auth object. (default: :obj:`None`)
86
+
87
+ # Embedded Weaviate parameters
88
+ embedded_hostname (str, optional): Embedded instance hostname.
89
+ (default: :obj:`"127.0.0.1"`)
90
+ embedded_port (int, optional): Embedded instance HTTP port.
91
+ (default: :obj:`8079`)
92
+ embedded_grpc_port (int, optional): Embedded instance gRPC port.
93
+ (default: :obj:`50050`)
94
+ embedded_version (Optional[str], optional): Weaviate version for
95
+ embedded instance. If None, uses the default version.
96
+ (default: :obj:`None`)
97
+ embedded_persistence_data_path (Optional[str], optional): Directory
98
+ for embedded database files. (default: :obj:`None`)
99
+ embedded_binary_path (Optional[str], optional): Directory for
100
+ Weaviate binary. (default: :obj:`None`)
101
+ embedded_environment_variables (Optional[Dict[str, str]], optional):
102
+ Environment variables for embedded instance. (default: :obj:`None`)
103
+
104
+ # Custom connection parameters
105
+ custom_http_host (Optional[str], optional): Custom HTTP host.
106
+ custom_http_port (Optional[int], optional): Custom HTTP port.
107
+ custom_http_secure (Optional[bool], optional): Use HTTPS.
108
+ custom_grpc_host (Optional[str], optional): Custom gRPC host.
109
+ custom_grpc_port (Optional[int], optional): Custom gRPC port.
110
+ custom_grpc_secure (Optional[bool], optional): Use secure gRPC.
111
+ custom_auth_credentials (Optional[Any], optional): Custom auth.
112
+
113
+ # Vector index configuration parameters
114
+ vector_index_type (VectorIndexType, optional): Vector index type.
115
+ Supported types: 'hnsw', 'flat'. (default: :obj:`"hnsw"`)
116
+ distance_metric (DistanceMetric, optional): Distance metric for vector
117
+ similarity. Supported metrics: 'cosine', 'dot', 'l2-squared',
118
+ 'hamming', 'manhattan'. (default: :obj:`"cosine"`)
119
+
120
+ # Common parameters for all connection types
121
+ headers (Optional[Dict[str, str]], optional): Additional headers for
122
+ third-party API keys (e.g., OpenAI, Cohere). (default: :obj:`None`)
123
+ additional_config (Optional[Any], optional): Advanced configuration
124
+ options like timeouts. (default: :obj:`None`)
125
+ skip_init_checks (bool, optional): Skip initialization checks.
126
+ (default: :obj:`False`)
127
+
128
+ Raises:
129
+ ImportError: If `weaviate` package is not installed.
130
+ ValueError: If connection parameters are invalid or missing.
131
+ RuntimeError: If there's an error setting up the collection.
132
+
133
+ Note:
134
+ This implementation supports synchronous operations only.
135
+ The client connection is automatically handled and closed when the
136
+ storage instance is destroyed.
137
+ """
138
+
139
+ @dependencies_required('weaviate')
140
+ def __init__(
141
+ self,
142
+ vector_dim: int,
143
+ collection_name: Optional[str] = None,
144
+ connection_type: ConnectionType = "local",
145
+ # Weaviate Cloud parameters
146
+ wcd_cluster_url: Optional[str] = None,
147
+ wcd_api_key: Optional[str] = None,
148
+ # Local instance parameters
149
+ local_host: str = "localhost",
150
+ local_port: int = 8080,
151
+ local_grpc_port: int = 50051,
152
+ local_auth_credentials: Optional[Union[str, Any]] = None,
153
+ # Embedded Weaviate parameters
154
+ embedded_hostname: str = "127.0.0.1",
155
+ embedded_port: int = 8079,
156
+ embedded_grpc_port: int = 50050,
157
+ embedded_version: Optional[str] = None,
158
+ embedded_persistence_data_path: Optional[str] = None,
159
+ embedded_binary_path: Optional[str] = None,
160
+ embedded_environment_variables: Optional[Dict[str, str]] = None,
161
+ # Custom connection parameters
162
+ custom_http_host: Optional[str] = None,
163
+ custom_http_port: Optional[int] = None,
164
+ custom_http_secure: Optional[bool] = None,
165
+ custom_grpc_host: Optional[str] = None,
166
+ custom_grpc_port: Optional[int] = None,
167
+ custom_grpc_secure: Optional[bool] = None,
168
+ custom_auth_credentials: Optional[Any] = None,
169
+ # Vector index configuration parameters
170
+ vector_index_type: VectorIndexType = "hnsw",
171
+ distance_metric: DistanceMetric = "cosine",
172
+ # Common parameters
173
+ headers: Optional[Dict[str, str]] = None,
174
+ additional_config: Optional[Any] = None,
175
+ skip_init_checks: bool = False,
176
+ **kwargs: Any,
177
+ ) -> None:
178
+ self.vector_dim = vector_dim
179
+ self.collection_name = (
180
+ collection_name or self._generate_collection_name()
181
+ )
182
+
183
+ # Store vector index configuration
184
+ self.vector_index_type: VectorIndexType = vector_index_type
185
+ self.distance_metric: DistanceMetric = distance_metric
186
+
187
+ # Store connection type configuration
188
+ self.connection_type: ConnectionType = connection_type
189
+
190
+ # Store connection parameters for later use
191
+ self._connection_params = {
192
+ 'wcd_cluster_url': wcd_cluster_url,
193
+ 'wcd_api_key': wcd_api_key,
194
+ 'local_host': local_host,
195
+ 'local_port': local_port,
196
+ 'local_grpc_port': local_grpc_port,
197
+ 'local_auth_credentials': local_auth_credentials,
198
+ 'embedded_hostname': embedded_hostname,
199
+ 'embedded_port': embedded_port,
200
+ 'embedded_grpc_port': embedded_grpc_port,
201
+ 'embedded_version': embedded_version,
202
+ 'embedded_persistence_data_path': embedded_persistence_data_path,
203
+ 'embedded_binary_path': embedded_binary_path,
204
+ 'embedded_environment_variables': embedded_environment_variables,
205
+ 'custom_http_host': custom_http_host,
206
+ 'custom_http_port': custom_http_port,
207
+ 'custom_http_secure': custom_http_secure,
208
+ 'custom_grpc_host': custom_grpc_host,
209
+ 'custom_grpc_port': custom_grpc_port,
210
+ 'custom_grpc_secure': custom_grpc_secure,
211
+ 'custom_auth_credentials': custom_auth_credentials,
212
+ 'headers': headers,
213
+ 'additional_config': additional_config,
214
+ 'skip_init_checks': skip_init_checks,
215
+ **kwargs,
216
+ }
217
+
218
+ # Store original collection creation kwargs for clear() method
219
+ self._original_collection_kwargs = kwargs.copy()
220
+
221
+ # Create client using the new unified approach
222
+ self._client = self._get_connection_client()
223
+ logger.info(
224
+ f"Created Weaviate client with connection type: "
225
+ f"{self.connection_type}"
226
+ )
227
+
228
+ self._check_and_create_collection(**kwargs)
229
+
230
+ def _get_connection_client(self) -> "WeaviateClient":
231
+ r"""Get Weaviate client based on connection type and user settings."""
232
+ import weaviate
233
+
234
+ # Map connection types to handler methods
235
+ connection_handlers: Dict[ConnectionType, Any] = {
236
+ 'cloud': self._create_cloud_client,
237
+ 'local': self._create_local_client,
238
+ 'embedded': self._create_embedded_client,
239
+ 'custom': self._create_custom_client,
240
+ }
241
+
242
+ # Get connection handler
243
+ handler = connection_handlers[self.connection_type]
244
+
245
+ try:
246
+ return handler(weaviate)
247
+ except Exception as e:
248
+ logger.error(
249
+ f"Failed to create {self.connection_type} client: {e}"
250
+ )
251
+ raise
252
+
253
+ def _create_cloud_client(self, weaviate_module: Any) -> "WeaviateClient":
254
+ r"""Create a Weaviate Cloud client."""
255
+ cluster_url = self._connection_params.get('wcd_cluster_url')
256
+ api_key = self._connection_params.get('wcd_api_key')
257
+
258
+ if not cluster_url:
259
+ raise ValueError(
260
+ "wcd_cluster_url is required for cloud connection"
261
+ )
262
+ if not api_key:
263
+ raise ValueError("wcd_api_key is required for cloud connection")
264
+
265
+ return weaviate_module.connect_to_weaviate_cloud(
266
+ cluster_url=cluster_url,
267
+ auth_credentials=api_key,
268
+ headers=self._connection_params.get('headers'),
269
+ additional_config=self._connection_params.get('additional_config'),
270
+ skip_init_checks=self._connection_params.get(
271
+ 'skip_init_checks', False
272
+ ),
273
+ )
274
+
275
+ def _create_local_client(self, weaviate_module: Any) -> "WeaviateClient":
276
+ r"""Create a local Weaviate client."""
277
+ return weaviate_module.connect_to_local(
278
+ host=self._connection_params.get('local_host', 'localhost'),
279
+ port=self._connection_params.get('local_port', 8080),
280
+ grpc_port=self._connection_params.get('local_grpc_port', 50051),
281
+ headers=self._connection_params.get('headers'),
282
+ additional_config=self._connection_params.get('additional_config'),
283
+ skip_init_checks=self._connection_params.get(
284
+ 'skip_init_checks', False
285
+ ),
286
+ auth_credentials=self._connection_params.get(
287
+ 'local_auth_credentials'
288
+ ),
289
+ )
290
+
291
+ def _create_embedded_client(
292
+ self, weaviate_module: Any
293
+ ) -> "WeaviateClient":
294
+ r"""Create an embedded Weaviate client."""
295
+ embedded_kwargs = {
296
+ 'hostname': self._connection_params.get(
297
+ 'embedded_hostname', '127.0.0.1'
298
+ ),
299
+ 'port': self._connection_params.get('embedded_port', 8079),
300
+ 'grpc_port': self._connection_params.get(
301
+ 'embedded_grpc_port', 50050
302
+ ),
303
+ 'headers': self._connection_params.get('headers'),
304
+ 'additional_config': self._connection_params.get(
305
+ 'additional_config'
306
+ ),
307
+ }
308
+
309
+ # Add optional embedded parameters
310
+ if self._connection_params.get('embedded_version') is not None:
311
+ embedded_kwargs['version'] = self._connection_params[
312
+ 'embedded_version'
313
+ ]
314
+ if (
315
+ self._connection_params.get('embedded_persistence_data_path')
316
+ is not None
317
+ ):
318
+ embedded_kwargs['persistence_data_path'] = self._connection_params[
319
+ 'embedded_persistence_data_path'
320
+ ]
321
+ if self._connection_params.get('embedded_binary_path') is not None:
322
+ embedded_kwargs['binary_path'] = self._connection_params[
323
+ 'embedded_binary_path'
324
+ ]
325
+ if (
326
+ self._connection_params.get('embedded_environment_variables')
327
+ is not None
328
+ ):
329
+ embedded_kwargs['environment_variables'] = self._connection_params[
330
+ 'embedded_environment_variables'
331
+ ]
332
+
333
+ return weaviate_module.connect_to_embedded(**embedded_kwargs)
334
+
335
+ def _create_custom_client(self, weaviate_module: Any) -> "WeaviateClient":
336
+ r"""Create a custom Weaviate client."""
337
+ # Validate required custom parameters
338
+ required_params = [
339
+ 'custom_http_host',
340
+ 'custom_http_port',
341
+ 'custom_http_secure',
342
+ 'custom_grpc_host',
343
+ 'custom_grpc_port',
344
+ 'custom_grpc_secure',
345
+ ]
346
+
347
+ for param in required_params:
348
+ if self._connection_params.get(param) is None:
349
+ raise ValueError(f"{param} is required for custom connection")
350
+
351
+ return weaviate_module.connect_to_custom(
352
+ http_host=self._connection_params['custom_http_host'],
353
+ http_port=self._connection_params['custom_http_port'],
354
+ http_secure=self._connection_params['custom_http_secure'],
355
+ grpc_host=self._connection_params['custom_grpc_host'],
356
+ grpc_port=self._connection_params['custom_grpc_port'],
357
+ grpc_secure=self._connection_params['custom_grpc_secure'],
358
+ headers=self._connection_params.get('headers'),
359
+ additional_config=self._connection_params.get('additional_config'),
360
+ auth_credentials=self._connection_params.get(
361
+ 'custom_auth_credentials'
362
+ ),
363
+ skip_init_checks=self._connection_params.get(
364
+ 'skip_init_checks', False
365
+ ),
366
+ )
367
+
368
+ def __del__(self):
369
+ r"""Clean up client connection."""
370
+ try:
371
+ if hasattr(self, '_client') and self._client is not None:
372
+ self._client.close()
373
+ logger.debug("Closed Weaviate client connection")
374
+ except Exception as e:
375
+ logger.warning(f"Error closing Weaviate client: {e}")
376
+
377
+ def close(self) -> None:
378
+ r"""Explicitly close the client connection."""
379
+ if self._client is not None:
380
+ self._client.close()
381
+ logger.info("Explicitly closed Weaviate client connection")
382
+
383
+ def _generate_collection_name(self) -> str:
384
+ r"""Generate a collection name if user doesn't provide one."""
385
+ timestamp = datetime.now().isoformat()
386
+ # Weaviate collection names must start with uppercase and be valid
387
+ # GraphQL names
388
+ valid_name = "Collection_" + re.sub(r'[^a-zA-Z0-9_]', '_', timestamp)
389
+ return valid_name
390
+
391
+ def _check_and_create_collection(self, **kwargs: Any) -> None:
392
+ r"""Check if collection exists and create if it doesn't."""
393
+ if not self._collection_exists(self.collection_name):
394
+ self._create_collection(**kwargs)
395
+
396
+ def _collection_exists(self, collection_name: str) -> bool:
397
+ r"""Check if the collection exists."""
398
+ try:
399
+ collection = self._client.collections.get(collection_name)
400
+ collection.config.get()
401
+ return True
402
+ except Exception:
403
+ return False
404
+
405
+ def _get_vector_index_config(self, **kwargs: Any) -> Any:
406
+ r"""Get vector index configuration based on user settings."""
407
+ import weaviate.classes.config as wvc
408
+
409
+ # Map distance metrics - type safety guaranteed by Literal
410
+ distance_metric_mapping: Dict[DistanceMetric, Any] = {
411
+ 'cosine': wvc.VectorDistances.COSINE,
412
+ 'dot': wvc.VectorDistances.DOT,
413
+ 'l2-squared': wvc.VectorDistances.L2_SQUARED,
414
+ 'hamming': wvc.VectorDistances.HAMMING,
415
+ 'manhattan': wvc.VectorDistances.MANHATTAN,
416
+ }
417
+
418
+ # Get distance metric - no need for None check due to Literal typing
419
+ distance_metric = distance_metric_mapping[self.distance_metric]
420
+
421
+ # Configure vector index based on type
422
+ if self.vector_index_type == 'hnsw':
423
+ return wvc.Configure.VectorIndex.hnsw(
424
+ distance_metric=distance_metric,
425
+ **kwargs,
426
+ )
427
+ else: # must be 'flat' due to Literal typing
428
+ return wvc.Configure.VectorIndex.flat(
429
+ distance_metric=distance_metric,
430
+ **kwargs,
431
+ )
432
+
433
+ def _create_collection(self, **kwargs: Any) -> None:
434
+ r"""Create a new collection in Weaviate."""
435
+ import weaviate.classes.config as wvc
436
+
437
+ # Separate vector index kwargs from general kwargs to avoid conflicts
438
+ vector_index_kwargs = {}
439
+ collection_kwargs = {}
440
+
441
+ # Known vector index parameters
442
+ vector_index_params = {
443
+ 'ef_construction',
444
+ 'max_connections',
445
+ 'ef',
446
+ 'dynamic_ef_min',
447
+ 'dynamic_ef_max',
448
+ 'dynamic_ef_factor',
449
+ 'vector_cache_max_objects',
450
+ 'flat_search_cutoff',
451
+ 'cleanup_interval_seconds',
452
+ 'pq_enabled',
453
+ 'pq_segments',
454
+ 'pq_centroids',
455
+ 'pq_training_limit',
456
+ 'pq_encoder',
457
+ }
458
+
459
+ for key, value in kwargs.items():
460
+ if key in vector_index_params:
461
+ vector_index_kwargs[key] = value
462
+ else:
463
+ collection_kwargs[key] = value
464
+
465
+ self._client.collections.create(
466
+ name=self.collection_name,
467
+ vectorizer_config=wvc.Configure.Vectorizer.none(),
468
+ properties=[
469
+ wvc.Property(name="payload", data_type=wvc.DataType.TEXT),
470
+ ],
471
+ vector_index_config=self._get_vector_index_config(
472
+ **vector_index_kwargs
473
+ ),
474
+ **collection_kwargs,
475
+ )
476
+
477
+ def add(
478
+ self,
479
+ records: List[VectorRecord],
480
+ **kwargs: Any,
481
+ ) -> None:
482
+ r"""Saves a list of vector records to the storage.
483
+
484
+ Args:
485
+ records (List[VectorRecord]): List of vector records to be saved.
486
+ **kwargs (Any): Additional keyword arguments.
487
+
488
+ Raises:
489
+ RuntimeError: If there is an error during the saving process.
490
+ """
491
+ if not records:
492
+ return
493
+
494
+ try:
495
+ collection = self._client.collections.get(self.collection_name)
496
+
497
+ with collection.batch.dynamic() as batch:
498
+ for record in records:
499
+ payload_str = (
500
+ json.dumps(record.payload) if record.payload else ""
501
+ )
502
+ batch.add_object(
503
+ properties={"payload": payload_str},
504
+ vector=record.vector,
505
+ uuid=record.id,
506
+ **kwargs,
507
+ )
508
+ logger.debug(
509
+ f"Successfully added vectors to Weaviate collection: "
510
+ f"{self.collection_name}"
511
+ )
512
+
513
+ except Exception as e:
514
+ raise RuntimeError(f"Failed to add vectors to Weaviate: {e}")
515
+
516
+ def delete(
517
+ self,
518
+ ids: List[str],
519
+ **kwargs: Any,
520
+ ) -> None:
521
+ r"""Deletes a list of vectors identified by their IDs from the storage.
522
+
523
+ Args:
524
+ ids (List[str]): List of unique identifiers for the vectors to be
525
+ deleted.
526
+ **kwargs (Any): Additional keyword arguments.
527
+
528
+ Raises:
529
+ RuntimeError: If there is an error during the deletion process.
530
+ """
531
+ if not ids:
532
+ return
533
+
534
+ try:
535
+ collection = self._client.collections.get(self.collection_name)
536
+ from weaviate.classes.query import Filter
537
+
538
+ collection.data.delete_many(
539
+ where=Filter.by_id().contains_any(ids), **kwargs
540
+ )
541
+ logger.debug(
542
+ f"Successfully deleted vectors in Weaviate: "
543
+ f"{self.collection_name}"
544
+ )
545
+
546
+ except Exception as e:
547
+ raise RuntimeError(f"Failed to delete vectors from Weaviate: {e}")
548
+
549
+ def _calculate_similarity_from_distance(
550
+ self, distance: Optional[float]
551
+ ) -> float:
552
+ r"""Calculate similarity score based on distance metric.
553
+
554
+ Args:
555
+ distance (Optional[float]): The distance value from Weaviate.
556
+
557
+ Returns:
558
+ float: Normalized similarity score between 0 and 1.
559
+ """
560
+ # Return 0.0 if distance is not available
561
+ if distance is None:
562
+ return 0.0
563
+
564
+ # Calculate similarity based on distance metric
565
+ if self.distance_metric == "cosine":
566
+ # Cosine: 0 (identical) to 2 (opposite)
567
+ # Convert to similarity: 1 - (distance / 2)
568
+ return max(0.0, 1.0 - (distance / 2.0))
569
+
570
+ elif self.distance_metric == "dot":
571
+ # Dot product: Weaviate returns -dot,
572
+ # smaller (more negative) is more similar
573
+ # Use sigmoid function to normalize to [0,1] range
574
+ # For dot product, we use: 1 / (1 + exp(distance))
575
+ return 1.0 / (1.0 + math.exp(distance))
576
+
577
+ elif self.distance_metric in ["l2-squared", "manhattan"]:
578
+ # L2-squared and Manhattan: 0 (identical) to ∞
579
+ # Convert to similarity using: 1 / (1 + distance)
580
+ # This provides a smoother decay than exponential
581
+ return 1.0 / (1.0 + distance)
582
+
583
+ elif self.distance_metric == "hamming":
584
+ # Hamming: 0 (identical) to vector_dim (completely different)
585
+ # Convert to similarity: 1 - (distance / vector_dim)
586
+ return max(0.0, 1.0 - (distance / self.vector_dim))
587
+
588
+ else:
589
+ # Unknown metric, return 0.0
590
+ return 0.0
591
+
592
+ def status(self) -> VectorDBStatus:
593
+ r"""Returns status of the vector database.
594
+
595
+ Returns:
596
+ VectorDBStatus: The vector database status.
597
+ """
598
+ try:
599
+ collection = self._client.collections.get(self.collection_name)
600
+ objects = collection.aggregate.over_all(total_count=True)
601
+
602
+ vector_count = (
603
+ objects.total_count if objects.total_count is not None else 0
604
+ )
605
+
606
+ return VectorDBStatus(
607
+ vector_dim=self.vector_dim, vector_count=vector_count
608
+ )
609
+
610
+ except Exception as e:
611
+ logger.warning(f"Failed to get status from Weaviate: {e}")
612
+ return VectorDBStatus(vector_dim=self.vector_dim, vector_count=0)
613
+
614
+ def query(
615
+ self,
616
+ query: VectorDBQuery,
617
+ **kwargs: Any,
618
+ ) -> List[VectorDBQueryResult]:
619
+ r"""Searches for similar vectors in the storage based on the provided
620
+ query.
621
+
622
+ Args:
623
+ query (VectorDBQuery): The query object containing the search
624
+ vector and the number of top similar vectors to retrieve.
625
+ **kwargs (Any): Additional keyword arguments.
626
+
627
+ Returns:
628
+ List[VectorDBQueryResult]: A list of vectors retrieved from the
629
+ storage based on similarity to the query vector.
630
+ """
631
+ from weaviate.classes.query import MetadataQuery
632
+
633
+ try:
634
+ collection = self._client.collections.get(self.collection_name)
635
+
636
+ response = collection.query.near_vector(
637
+ near_vector=query.query_vector,
638
+ limit=query.top_k,
639
+ include_vector=True,
640
+ return_metadata=MetadataQuery(distance=True),
641
+ **kwargs,
642
+ )
643
+
644
+ results = []
645
+ for obj in response.objects:
646
+ # Calculate similarity score based on distance metric
647
+ similarity = self._calculate_similarity_from_distance(
648
+ obj.metadata.distance
649
+ )
650
+
651
+ # Handle payload
652
+ payload = None
653
+ if obj.properties.get('payload'):
654
+ payload_value = obj.properties['payload']
655
+ if isinstance(payload_value, str):
656
+ try:
657
+ payload = json.loads(payload_value)
658
+ except (json.JSONDecodeError, TypeError):
659
+ payload = {"raw": payload_value}
660
+ else:
661
+ payload = {"raw": str(payload_value)}
662
+
663
+ # Handle vector data
664
+ # In newer versions of Weaviate, obj.vector returns a dict
665
+ # with 'default' key
666
+
667
+ vector: List[float] = []
668
+ if hasattr(obj, 'vector') and obj.vector is not None:
669
+ # New format: {'default': [vector_values]}
670
+ vector_data = obj.vector.get('default', [])
671
+ if (
672
+ vector_data
673
+ and isinstance(vector_data, list)
674
+ and len(vector_data) > 0
675
+ and isinstance(vector_data[0], float)
676
+ ):
677
+ vector = cast(List[float], vector_data)
678
+ else:
679
+ # unsupported vector data format
680
+ vector = []
681
+ else:
682
+ vector = []
683
+
684
+ result = VectorDBQueryResult.create(
685
+ similarity=similarity,
686
+ vector=vector,
687
+ id=str(obj.uuid),
688
+ payload=payload,
689
+ )
690
+ results.append(result)
691
+
692
+ return results
693
+
694
+ except Exception as e:
695
+ raise RuntimeError(f"Failed to query vectors from Weaviate: {e}")
696
+
697
+ def clear(self) -> None:
698
+ r"""Remove all vectors from the storage."""
699
+ try:
700
+ self._client.collections.delete(self.collection_name)
701
+ self._create_collection(**self._original_collection_kwargs)
702
+ except Exception as e:
703
+ raise RuntimeError(f"Failed to clear Weaviate collection: {e}")
704
+
705
+ def load(self) -> None:
706
+ r"""Load the collection hosted on cloud service."""
707
+ # For Weaviate, collections are automatically available when client
708
+ # connects
709
+ pass
710
+
711
+ @property
712
+ def client(self) -> "WeaviateClient":
713
+ r"""Provides access to the underlying vector database client."""
714
+ return self._client