camel-ai 0.2.59__py3-none-any.whl → 0.2.82__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of camel-ai might be problematic. Click here for more details.
- camel/__init__.py +3 -3
- camel/agents/__init__.py +2 -2
- camel/agents/_types.py +9 -4
- camel/agents/_utils.py +40 -2
- camel/agents/base.py +2 -2
- camel/agents/chat_agent.py +5012 -902
- camel/agents/critic_agent.py +2 -2
- camel/agents/deductive_reasoner_agent.py +56 -56
- camel/agents/embodied_agent.py +2 -2
- camel/agents/knowledge_graph_agent.py +20 -20
- camel/agents/mcp_agent.py +39 -36
- camel/agents/multi_hop_generator_agent.py +3 -3
- camel/agents/programmed_agent_instruction.py +2 -2
- camel/agents/repo_agent.py +4 -3
- camel/agents/role_assignment_agent.py +2 -2
- camel/agents/search_agent.py +2 -2
- camel/agents/task_agent.py +2 -2
- camel/agents/tool_agents/__init__.py +2 -2
- camel/agents/tool_agents/base.py +2 -2
- camel/agents/tool_agents/hugging_face_tool_agent.py +3 -3
- camel/benchmarks/__init__.py +2 -2
- camel/benchmarks/apibank.py +5 -5
- camel/benchmarks/apibench.py +2 -2
- camel/benchmarks/base.py +2 -2
- camel/benchmarks/browsecomp.py +44 -33
- camel/benchmarks/gaia.py +17 -13
- camel/benchmarks/mock_website/README.md +94 -0
- camel/benchmarks/mock_website/mock_web.py +299 -0
- camel/benchmarks/mock_website/requirements.txt +3 -0
- camel/benchmarks/mock_website/shopping_mall/app.py +465 -0
- camel/benchmarks/mock_website/task.json +104 -0
- camel/benchmarks/nexus.py +3 -3
- camel/benchmarks/ragbench.py +2 -2
- camel/bots/__init__.py +2 -2
- camel/bots/discord/__init__.py +2 -2
- camel/bots/discord/discord_app.py +2 -2
- camel/bots/discord/discord_installation.py +2 -2
- camel/bots/discord/discord_store.py +3 -3
- camel/bots/slack/__init__.py +2 -2
- camel/bots/slack/models.py +4 -4
- camel/bots/slack/slack_app.py +2 -2
- camel/bots/telegram_bot.py +2 -2
- camel/configs/__init__.py +26 -2
- camel/configs/aihubmix_config.py +90 -0
- camel/configs/aiml_config.py +2 -2
- camel/configs/amd_config.py +70 -0
- camel/configs/anthropic_config.py +8 -7
- camel/configs/base_config.py +2 -2
- camel/configs/bedrock_config.py +5 -3
- camel/configs/cerebras_config.py +98 -0
- camel/configs/cohere_config.py +3 -3
- camel/configs/cometapi_config.py +106 -0
- camel/configs/crynux_config.py +94 -0
- camel/configs/deepseek_config.py +9 -8
- camel/configs/gemini_config.py +6 -4
- camel/configs/groq_config.py +6 -4
- camel/configs/internlm_config.py +6 -4
- camel/configs/litellm_config.py +2 -2
- camel/configs/lmstudio_config.py +6 -4
- camel/configs/minimax_config.py +95 -0
- camel/configs/mistral_config.py +3 -3
- camel/configs/modelscope_config.py +5 -3
- camel/configs/moonshot_config.py +2 -2
- camel/configs/nebius_config.py +105 -0
- camel/configs/netmind_config.py +2 -2
- camel/configs/novita_config.py +2 -2
- camel/configs/nvidia_config.py +2 -2
- camel/configs/ollama_config.py +2 -2
- camel/configs/openai_config.py +8 -3
- camel/configs/openrouter_config.py +6 -4
- camel/configs/ppio_config.py +2 -2
- camel/configs/qianfan_config.py +85 -0
- camel/configs/qwen_config.py +2 -2
- camel/configs/reka_config.py +3 -3
- camel/configs/samba_config.py +8 -6
- camel/configs/sglang_config.py +2 -2
- camel/configs/siliconflow_config.py +2 -2
- camel/configs/togetherai_config.py +2 -2
- camel/configs/vllm_config.py +4 -2
- camel/configs/watsonx_config.py +2 -2
- camel/configs/yi_config.py +6 -4
- camel/configs/zhipuai_config.py +6 -4
- camel/{data_collector → data_collectors}/__init__.py +2 -2
- camel/{data_collector → data_collectors}/alpaca_collector.py +19 -10
- camel/{data_collector → data_collectors}/base.py +2 -2
- camel/{data_collector → data_collectors}/sharegpt_collector.py +3 -3
- camel/datagen/__init__.py +2 -2
- camel/datagen/cot_datagen.py +32 -37
- camel/datagen/evol_instruct/__init__.py +2 -2
- camel/datagen/evol_instruct/evol_instruct.py +2 -2
- camel/datagen/evol_instruct/scorer.py +24 -25
- camel/datagen/evol_instruct/templates.py +48 -48
- camel/datagen/self_improving_cot.py +5 -5
- camel/datagen/self_instruct/__init__.py +2 -2
- camel/datagen/self_instruct/filter/__init__.py +2 -2
- camel/datagen/self_instruct/filter/filter_function.py +2 -2
- camel/datagen/self_instruct/filter/filter_registry.py +2 -2
- camel/datagen/self_instruct/filter/instruction_filter.py +2 -2
- camel/datagen/self_instruct/self_instruct.py +2 -2
- camel/datagen/self_instruct/templates.py +47 -47
- camel/datagen/source2synth/__init__.py +2 -2
- camel/datagen/source2synth/data_processor.py +2 -2
- camel/datagen/source2synth/models.py +2 -2
- camel/datagen/source2synth/user_data_processor_config.py +2 -2
- camel/datahubs/__init__.py +2 -2
- camel/datahubs/base.py +2 -2
- camel/datahubs/huggingface.py +2 -2
- camel/datahubs/models.py +2 -2
- camel/datasets/__init__.py +2 -2
- camel/datasets/base_generator.py +41 -12
- camel/datasets/few_shot_generator.py +18 -18
- camel/datasets/models.py +3 -3
- camel/datasets/self_instruct_generator.py +2 -2
- camel/datasets/static_dataset.py +152 -2
- camel/embeddings/__init__.py +2 -2
- camel/embeddings/azure_embedding.py +2 -2
- camel/embeddings/base.py +2 -2
- camel/embeddings/gemini_embedding.py +2 -2
- camel/embeddings/jina_embedding.py +10 -3
- camel/embeddings/mistral_embedding.py +2 -2
- camel/embeddings/openai_compatible_embedding.py +2 -2
- camel/embeddings/openai_embedding.py +2 -2
- camel/embeddings/sentence_transformers_embeddings.py +4 -4
- camel/embeddings/together_embedding.py +2 -2
- camel/embeddings/vlm_embedding.py +11 -4
- camel/environments/__init__.py +14 -2
- camel/environments/models.py +2 -2
- camel/environments/multi_step.py +2 -2
- camel/environments/rlcards_env.py +860 -0
- camel/environments/single_step.py +30 -5
- camel/environments/tic_tac_toe.py +3 -3
- camel/extractors/__init__.py +2 -2
- camel/extractors/base.py +2 -2
- camel/extractors/python_strategies.py +2 -2
- camel/generators.py +2 -2
- camel/human.py +2 -2
- camel/interpreters/__init__.py +4 -2
- camel/interpreters/base.py +16 -3
- camel/interpreters/docker/Dockerfile +53 -7
- camel/interpreters/docker_interpreter.py +70 -11
- camel/interpreters/e2b_interpreter.py +59 -11
- camel/interpreters/internal_python_interpreter.py +81 -4
- camel/interpreters/interpreter_error.py +2 -2
- camel/interpreters/ipython_interpreter.py +23 -5
- camel/interpreters/microsandbox_interpreter.py +395 -0
- camel/interpreters/subprocess_interpreter.py +36 -4
- camel/loaders/__init__.py +17 -5
- camel/loaders/apify_reader.py +2 -2
- camel/loaders/base_io.py +2 -2
- camel/loaders/base_loader.py +85 -0
- camel/loaders/chunkr_reader.py +128 -93
- camel/loaders/crawl4ai_reader.py +2 -2
- camel/loaders/firecrawl_reader.py +6 -6
- camel/loaders/jina_url_reader.py +2 -2
- camel/loaders/markitdown.py +2 -2
- camel/loaders/mineru_extractor.py +2 -2
- camel/loaders/mistral_reader.py +148 -0
- camel/loaders/scrapegraph_reader.py +2 -2
- camel/loaders/unstructured_io.py +2 -2
- camel/logger.py +5 -5
- camel/memories/__init__.py +2 -2
- camel/memories/agent_memories.py +86 -3
- camel/memories/base.py +36 -2
- camel/memories/blocks/__init__.py +2 -2
- camel/memories/blocks/chat_history_block.py +126 -9
- camel/memories/blocks/vectordb_block.py +10 -3
- camel/memories/context_creators/__init__.py +2 -2
- camel/memories/context_creators/score_based.py +31 -239
- camel/memories/records.py +98 -13
- camel/messages/__init__.py +2 -2
- camel/messages/base.py +193 -46
- camel/messages/conversion/__init__.py +2 -2
- camel/messages/conversion/alpaca.py +2 -2
- camel/messages/conversion/conversation_models.py +2 -2
- camel/messages/conversion/sharegpt/__init__.py +2 -2
- camel/messages/conversion/sharegpt/function_call_formatter.py +2 -2
- camel/messages/conversion/sharegpt/hermes/__init__.py +2 -2
- camel/messages/conversion/sharegpt/hermes/hermes_function_formatter.py +2 -2
- camel/messages/func_message.py +54 -17
- camel/models/__init__.py +18 -2
- camel/models/_utils.py +3 -3
- camel/models/aihubmix_model.py +83 -0
- camel/models/aiml_model.py +11 -18
- camel/models/amd_model.py +101 -0
- camel/models/anthropic_model.py +127 -20
- camel/models/aws_bedrock_model.py +12 -35
- camel/models/azure_openai_model.py +263 -63
- camel/models/base_audio_model.py +5 -3
- camel/models/base_model.py +195 -26
- camel/models/cerebras_model.py +83 -0
- camel/models/cohere_model.py +81 -21
- camel/models/cometapi_model.py +83 -0
- camel/models/crynux_model.py +87 -0
- camel/models/deepseek_model.py +61 -59
- camel/models/fish_audio_model.py +8 -2
- camel/models/gemini_model.py +439 -30
- camel/models/groq_model.py +11 -19
- camel/models/internlm_model.py +11 -18
- camel/models/litellm_model.py +94 -34
- camel/models/lmstudio_model.py +17 -20
- camel/models/minimax_model.py +83 -0
- camel/models/mistral_model.py +84 -19
- camel/models/model_factory.py +49 -6
- camel/models/model_manager.py +33 -11
- camel/models/modelscope_model.py +13 -193
- camel/models/moonshot_model.py +195 -21
- camel/models/nebius_model.py +83 -0
- camel/models/nemotron_model.py +19 -9
- camel/models/netmind_model.py +11 -18
- camel/models/novita_model.py +11 -18
- camel/models/nvidia_model.py +11 -18
- camel/models/ollama_model.py +14 -21
- camel/models/openai_audio_models.py +2 -2
- camel/models/openai_compatible_model.py +234 -27
- camel/models/openai_model.py +255 -39
- camel/models/openrouter_model.py +11 -19
- camel/models/ppio_model.py +11 -18
- camel/models/qianfan_model.py +89 -0
- camel/models/qwen_model.py +13 -193
- camel/models/reka_model.py +90 -21
- camel/models/reward/__init__.py +2 -2
- camel/models/reward/base_reward_model.py +2 -2
- camel/models/reward/evaluator.py +2 -2
- camel/models/reward/nemotron_model.py +2 -2
- camel/models/reward/skywork_model.py +2 -2
- camel/models/samba_model.py +117 -49
- camel/models/sglang_model.py +162 -42
- camel/models/siliconflow_model.py +12 -35
- camel/models/stub_model.py +10 -7
- camel/models/togetherai_model.py +11 -18
- camel/models/vllm_model.py +10 -18
- camel/models/volcano_model.py +16 -20
- camel/models/watsonx_model.py +69 -19
- camel/models/yi_model.py +11 -18
- camel/models/zhipuai_model.py +70 -18
- camel/parsers/__init__.py +18 -0
- camel/parsers/mcp_tool_call_parser.py +176 -0
- camel/personas/__init__.py +2 -2
- camel/personas/persona.py +2 -2
- camel/personas/persona_hub.py +2 -2
- camel/prompts/__init__.py +2 -2
- camel/prompts/ai_society.py +2 -2
- camel/prompts/base.py +2 -2
- camel/prompts/code.py +2 -2
- camel/prompts/evaluation.py +2 -2
- camel/prompts/generate_text_embedding_data.py +2 -2
- camel/prompts/image_craft.py +2 -2
- camel/prompts/misalignment.py +2 -2
- camel/prompts/multi_condition_image_craft.py +2 -2
- camel/prompts/object_recognition.py +2 -2
- camel/prompts/persona_hub.py +3 -3
- camel/prompts/prompt_templates.py +2 -2
- camel/prompts/role_description_prompt_template.py +2 -2
- camel/prompts/solution_extraction.py +8 -8
- camel/prompts/task_prompt_template.py +2 -2
- camel/prompts/translation.py +2 -2
- camel/prompts/video_description_prompt.py +3 -3
- camel/responses/__init__.py +2 -2
- camel/responses/agent_responses.py +2 -2
- camel/retrievers/__init__.py +2 -2
- camel/retrievers/auto_retriever.py +23 -3
- camel/retrievers/base.py +2 -2
- camel/retrievers/bm25_retriever.py +3 -4
- camel/retrievers/cohere_rerank_retriever.py +2 -2
- camel/retrievers/hybrid_retrival.py +4 -4
- camel/retrievers/vector_retriever.py +2 -2
- camel/runtimes/Dockerfile.multi-toolkit +90 -0
- camel/{runtime → runtimes}/__init__.py +2 -2
- camel/runtimes/api.py +153 -0
- camel/{runtime → runtimes}/base.py +2 -2
- camel/{runtime → runtimes}/configs.py +13 -13
- camel/{runtime → runtimes}/daytona_runtime.py +18 -19
- camel/{runtime → runtimes}/docker_runtime.py +13 -13
- camel/{runtime → runtimes}/llm_guard_runtime.py +28 -28
- camel/{runtime → runtimes}/remote_http_runtime.py +12 -12
- camel/{runtime → runtimes}/ubuntu_docker_runtime.py +3 -3
- camel/{runtime → runtimes}/utils/__init__.py +2 -2
- camel/{runtime → runtimes}/utils/function_risk_toolkit.py +2 -2
- camel/{runtime → runtimes}/utils/ignore_risk_toolkit.py +2 -2
- camel/schemas/__init__.py +2 -2
- camel/schemas/base.py +2 -2
- camel/schemas/openai_converter.py +3 -3
- camel/schemas/outlines_converter.py +2 -2
- camel/services/agent_openapi_server.py +380 -0
- camel/societies/__init__.py +4 -2
- camel/societies/babyagi_playing.py +2 -2
- camel/societies/role_playing.py +201 -80
- camel/societies/workforce/__init__.py +10 -3
- camel/societies/workforce/base.py +9 -5
- camel/societies/workforce/events.py +143 -0
- camel/societies/workforce/prompts.py +258 -33
- camel/societies/workforce/role_playing_worker.py +95 -30
- camel/societies/workforce/single_agent_worker.py +659 -30
- camel/societies/workforce/structured_output_handler.py +512 -0
- camel/societies/workforce/task_channel.py +182 -38
- camel/societies/workforce/utils.py +784 -18
- camel/societies/workforce/worker.py +96 -28
- camel/societies/workforce/workflow_memory_manager.py +1746 -0
- camel/societies/workforce/workforce.py +5730 -366
- camel/societies/workforce/workforce_callback.py +103 -0
- camel/societies/workforce/workforce_logger.py +647 -0
- camel/societies/workforce/workforce_metrics.py +33 -0
- camel/storages/__init__.py +10 -2
- camel/storages/graph_storages/__init__.py +2 -2
- camel/storages/graph_storages/base.py +2 -2
- camel/storages/graph_storages/graph_element.py +2 -2
- camel/storages/graph_storages/nebula_graph.py +4 -4
- camel/storages/graph_storages/neo4j_graph.py +7 -7
- camel/storages/key_value_storages/__init__.py +2 -2
- camel/storages/key_value_storages/base.py +2 -2
- camel/storages/key_value_storages/in_memory.py +2 -2
- camel/storages/key_value_storages/json.py +17 -4
- camel/storages/key_value_storages/mem0_cloud.py +50 -49
- camel/storages/key_value_storages/redis.py +2 -2
- camel/storages/object_storages/__init__.py +2 -2
- camel/storages/object_storages/amazon_s3.py +2 -2
- camel/storages/object_storages/azure_blob.py +2 -2
- camel/storages/object_storages/base.py +2 -2
- camel/storages/object_storages/google_cloud.py +3 -3
- camel/storages/vectordb_storages/__init__.py +12 -2
- camel/storages/vectordb_storages/base.py +2 -2
- camel/storages/vectordb_storages/chroma.py +731 -0
- camel/storages/vectordb_storages/faiss.py +712 -0
- camel/storages/vectordb_storages/milvus.py +2 -2
- camel/storages/vectordb_storages/oceanbase.py +16 -17
- camel/storages/vectordb_storages/pgvector.py +349 -0
- camel/storages/vectordb_storages/qdrant.py +6 -6
- camel/storages/vectordb_storages/surreal.py +372 -0
- camel/storages/vectordb_storages/tidb.py +11 -8
- camel/storages/vectordb_storages/weaviate.py +714 -0
- camel/tasks/__init__.py +2 -2
- camel/tasks/task.py +366 -27
- camel/tasks/task_prompt.py +3 -3
- camel/terminators/__init__.py +2 -2
- camel/terminators/base.py +2 -2
- camel/terminators/response_terminator.py +2 -2
- camel/terminators/token_limit_terminator.py +2 -2
- camel/toolkits/__init__.py +58 -10
- camel/toolkits/aci_toolkit.py +66 -21
- camel/toolkits/arxiv_toolkit.py +8 -8
- camel/toolkits/ask_news_toolkit.py +2 -2
- camel/toolkits/async_browser_toolkit.py +174 -575
- camel/toolkits/audio_analysis_toolkit.py +3 -3
- camel/toolkits/base.py +65 -7
- camel/toolkits/bohrium_toolkit.py +318 -0
- camel/toolkits/browser_toolkit.py +306 -566
- camel/toolkits/browser_toolkit_commons.py +568 -0
- camel/toolkits/code_execution.py +67 -11
- camel/toolkits/context_summarizer_toolkit.py +684 -0
- camel/toolkits/craw4ai_toolkit.py +93 -0
- camel/toolkits/dappier_toolkit.py +12 -8
- camel/toolkits/data_commons_toolkit.py +2 -2
- camel/toolkits/dingtalk.py +1135 -0
- camel/toolkits/earth_science_toolkit.py +5367 -0
- camel/toolkits/edgeone_pages_mcp_toolkit.py +49 -0
- camel/toolkits/excel_toolkit.py +910 -70
- camel/toolkits/file_toolkit.py +1402 -0
- camel/toolkits/function_tool.py +128 -20
- camel/toolkits/github_toolkit.py +148 -43
- camel/toolkits/gmail_toolkit.py +1839 -0
- camel/toolkits/google_calendar_toolkit.py +40 -6
- camel/toolkits/google_drive_mcp_toolkit.py +54 -0
- camel/toolkits/google_maps_toolkit.py +2 -2
- camel/toolkits/google_scholar_toolkit.py +2 -2
- camel/toolkits/human_toolkit.py +36 -12
- camel/toolkits/hybrid_browser_toolkit/__init__.py +18 -0
- camel/toolkits/hybrid_browser_toolkit/config_loader.py +185 -0
- camel/toolkits/hybrid_browser_toolkit/hybrid_browser_toolkit.py +246 -0
- camel/toolkits/hybrid_browser_toolkit/hybrid_browser_toolkit_ts.py +1973 -0
- camel/toolkits/hybrid_browser_toolkit/installer.py +203 -0
- camel/toolkits/hybrid_browser_toolkit/ts/package-lock.json +4589 -0
- camel/toolkits/hybrid_browser_toolkit/ts/package.json +33 -0
- camel/toolkits/hybrid_browser_toolkit/ts/src/browser-scripts.js +125 -0
- camel/toolkits/hybrid_browser_toolkit/ts/src/browser-session.ts +1929 -0
- camel/toolkits/hybrid_browser_toolkit/ts/src/config-loader.ts +233 -0
- camel/toolkits/hybrid_browser_toolkit/ts/src/hybrid-browser-toolkit.ts +589 -0
- camel/toolkits/hybrid_browser_toolkit/ts/src/index.ts +7 -0
- camel/toolkits/hybrid_browser_toolkit/ts/src/parent-child-filter.ts +226 -0
- camel/toolkits/hybrid_browser_toolkit/ts/src/snapshot-parser.ts +219 -0
- camel/toolkits/hybrid_browser_toolkit/ts/src/som-screenshot-injected.ts +543 -0
- camel/toolkits/hybrid_browser_toolkit/ts/src/types.ts +129 -0
- camel/toolkits/hybrid_browser_toolkit/ts/tsconfig.json +27 -0
- camel/toolkits/hybrid_browser_toolkit/ts/websocket-server.js +319 -0
- camel/toolkits/hybrid_browser_toolkit/ws_wrapper.py +1037 -0
- camel/toolkits/hybrid_browser_toolkit_py/__init__.py +17 -0
- camel/toolkits/hybrid_browser_toolkit_py/actions.py +575 -0
- camel/toolkits/hybrid_browser_toolkit_py/agent.py +311 -0
- camel/toolkits/hybrid_browser_toolkit_py/browser_session.py +787 -0
- camel/toolkits/hybrid_browser_toolkit_py/config_loader.py +490 -0
- camel/toolkits/hybrid_browser_toolkit_py/hybrid_browser_toolkit.py +2390 -0
- camel/toolkits/hybrid_browser_toolkit_py/snapshot.py +233 -0
- camel/toolkits/hybrid_browser_toolkit_py/stealth_script.js +0 -0
- camel/toolkits/hybrid_browser_toolkit_py/unified_analyzer.js +1043 -0
- camel/toolkits/image_analysis_toolkit.py +3 -3
- camel/toolkits/image_generation_toolkit.py +390 -0
- camel/toolkits/jina_reranker_toolkit.py +195 -79
- camel/toolkits/klavis_toolkit.py +7 -3
- camel/toolkits/linkedin_toolkit.py +2 -2
- camel/toolkits/markitdown_toolkit.py +104 -0
- camel/toolkits/math_toolkit.py +66 -12
- camel/toolkits/mcp_toolkit.py +841 -600
- camel/toolkits/memory_toolkit.py +7 -3
- camel/toolkits/meshy_toolkit.py +2 -2
- camel/toolkits/message_agent_toolkit.py +608 -0
- camel/toolkits/message_integration.py +724 -0
- camel/toolkits/mineru_toolkit.py +2 -2
- camel/toolkits/minimax_mcp_toolkit.py +195 -0
- camel/toolkits/networkx_toolkit.py +2 -2
- camel/toolkits/note_taking_toolkit.py +277 -0
- camel/toolkits/notion_mcp_toolkit.py +224 -0
- camel/toolkits/notion_toolkit.py +2 -2
- camel/toolkits/open_api_specs/biztoc/__init__.py +2 -2
- camel/toolkits/open_api_specs/biztoc/ai-plugin.json +1 -1
- camel/toolkits/open_api_specs/coursera/__init__.py +2 -2
- camel/toolkits/open_api_specs/create_qr_code/__init__.py +2 -2
- camel/toolkits/open_api_specs/klarna/__init__.py +2 -2
- camel/toolkits/open_api_specs/nasa_apod/__init__.py +2 -2
- camel/toolkits/open_api_specs/outschool/__init__.py +2 -2
- camel/toolkits/open_api_specs/outschool/ai-plugin.json +1 -1
- camel/toolkits/open_api_specs/outschool/openapi.yaml +1 -1
- camel/toolkits/open_api_specs/outschool/paths/__init__.py +2 -2
- camel/toolkits/open_api_specs/outschool/paths/get_classes.py +2 -2
- camel/toolkits/open_api_specs/outschool/paths/search_teachers.py +2 -2
- camel/toolkits/open_api_specs/security_config.py +2 -2
- camel/toolkits/open_api_specs/speak/__init__.py +2 -2
- camel/toolkits/open_api_specs/web_scraper/__init__.py +2 -2
- camel/toolkits/open_api_specs/web_scraper/ai-plugin.json +1 -1
- camel/toolkits/open_api_specs/web_scraper/paths/__init__.py +2 -2
- camel/toolkits/open_api_specs/web_scraper/paths/scraper.py +2 -2
- camel/toolkits/open_api_toolkit.py +2 -2
- camel/toolkits/openbb_toolkit.py +7 -3
- camel/toolkits/origene_mcp_toolkit.py +56 -0
- camel/toolkits/page_script.js +86 -74
- camel/toolkits/playwright_mcp_toolkit.py +27 -32
- camel/toolkits/pptx_toolkit.py +790 -0
- camel/toolkits/pubmed_toolkit.py +2 -2
- camel/toolkits/pulse_mcp_search_toolkit.py +2 -2
- camel/toolkits/pyautogui_toolkit.py +2 -2
- camel/toolkits/reddit_toolkit.py +2 -2
- camel/toolkits/resend_toolkit.py +168 -0
- camel/toolkits/retrieval_toolkit.py +2 -2
- camel/toolkits/screenshot_toolkit.py +213 -0
- camel/toolkits/search_toolkit.py +539 -146
- camel/toolkits/searxng_toolkit.py +2 -2
- camel/toolkits/semantic_scholar_toolkit.py +2 -2
- camel/toolkits/slack_toolkit.py +108 -58
- camel/toolkits/sql_toolkit.py +712 -0
- camel/toolkits/stripe_toolkit.py +2 -2
- camel/toolkits/sympy_toolkit.py +3 -3
- camel/toolkits/task_planning_toolkit.py +134 -0
- camel/toolkits/terminal_toolkit/__init__.py +18 -0
- camel/toolkits/terminal_toolkit/terminal_toolkit.py +1070 -0
- camel/toolkits/terminal_toolkit/utils.py +532 -0
- camel/toolkits/thinking_toolkit.py +3 -3
- camel/toolkits/twitter_toolkit.py +8 -3
- camel/toolkits/vertex_ai_veo_toolkit.py +590 -0
- camel/toolkits/video_analysis_toolkit.py +112 -29
- camel/toolkits/video_download_toolkit.py +22 -16
- camel/toolkits/weather_toolkit.py +2 -2
- camel/toolkits/web_deploy_toolkit.py +1219 -0
- camel/toolkits/wechat_official_toolkit.py +483 -0
- camel/toolkits/whatsapp_toolkit.py +2 -2
- camel/toolkits/wolfram_alpha_toolkit.py +53 -25
- camel/toolkits/zapier_toolkit.py +7 -3
- camel/types/__init__.py +4 -4
- camel/types/agents/__init__.py +2 -2
- camel/types/agents/tool_calling_record.py +6 -3
- camel/types/enums.py +454 -35
- camel/types/mcp_registries.py +2 -2
- camel/types/openai_types.py +4 -4
- camel/types/unified_model_type.py +43 -6
- camel/utils/__init__.py +20 -2
- camel/utils/async_func.py +2 -2
- camel/utils/chunker/__init__.py +2 -2
- camel/utils/chunker/base.py +2 -2
- camel/utils/chunker/code_chunker.py +2 -2
- camel/utils/chunker/uio_chunker.py +2 -2
- camel/utils/commons.py +65 -7
- camel/utils/constants.py +5 -2
- camel/utils/context_utils.py +1134 -0
- camel/utils/deduplication.py +2 -2
- camel/utils/filename.py +2 -2
- camel/utils/langfuse.py +258 -0
- camel/utils/mcp.py +140 -6
- camel/utils/mcp_client.py +1056 -0
- camel/utils/message_summarizer.py +148 -0
- camel/utils/response_format.py +2 -2
- camel/utils/token_counting.py +45 -22
- camel/utils/tool_result.py +44 -0
- camel/verifiers/__init__.py +2 -2
- camel/verifiers/base.py +2 -2
- camel/verifiers/math_verifier.py +2 -2
- camel/verifiers/models.py +2 -2
- camel/verifiers/physics_verifier.py +2 -2
- camel/verifiers/python_verifier.py +2 -2
- {camel_ai-0.2.59.dist-info → camel_ai-0.2.82.dist-info}/METADATA +349 -108
- camel_ai-0.2.82.dist-info/RECORD +507 -0
- {camel_ai-0.2.59.dist-info → camel_ai-0.2.82.dist-info}/WHEEL +1 -1
- {camel_ai-0.2.59.dist-info → camel_ai-0.2.82.dist-info}/licenses/LICENSE +1 -1
- camel/loaders/pandas_reader.py +0 -368
- camel/runtime/api.py +0 -97
- camel/toolkits/dalle_toolkit.py +0 -171
- camel/toolkits/file_write_toolkit.py +0 -395
- camel/toolkits/openai_agent_toolkit.py +0 -135
- camel/toolkits/terminal_toolkit.py +0 -1037
- camel_ai-0.2.59.dist-info/RECORD +0 -410
|
@@ -0,0 +1,714 @@
|
|
|
1
|
+
# ========= Copyright 2023-2025 @ CAMEL-AI.org. All Rights Reserved. =========
|
|
2
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
3
|
+
# you may not use this file except in compliance with the License.
|
|
4
|
+
# You may obtain a copy of the License at
|
|
5
|
+
#
|
|
6
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
7
|
+
#
|
|
8
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
9
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
10
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
11
|
+
# See the License for the specific language governing permissions and
|
|
12
|
+
# limitations under the License.
|
|
13
|
+
# ========= Copyright 2023-2025 @ CAMEL-AI.org. All Rights Reserved. =========
|
|
14
|
+
import json
|
|
15
|
+
import math
|
|
16
|
+
import re
|
|
17
|
+
from datetime import datetime
|
|
18
|
+
from typing import (
|
|
19
|
+
TYPE_CHECKING,
|
|
20
|
+
Any,
|
|
21
|
+
Dict,
|
|
22
|
+
List,
|
|
23
|
+
Literal,
|
|
24
|
+
Optional,
|
|
25
|
+
Union,
|
|
26
|
+
cast,
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
if TYPE_CHECKING:
|
|
30
|
+
from weaviate import WeaviateClient
|
|
31
|
+
|
|
32
|
+
from camel.logger import get_logger
|
|
33
|
+
from camel.storages.vectordb_storages import (
|
|
34
|
+
BaseVectorStorage,
|
|
35
|
+
VectorDBQuery,
|
|
36
|
+
VectorDBQueryResult,
|
|
37
|
+
VectorDBStatus,
|
|
38
|
+
VectorRecord,
|
|
39
|
+
)
|
|
40
|
+
from camel.utils import dependencies_required
|
|
41
|
+
|
|
42
|
+
logger = get_logger(__name__)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
# Type definitions for configuration options
|
|
46
|
+
ConnectionType = Literal["local", "cloud", "embedded", "custom"]
|
|
47
|
+
VectorIndexType = Literal["hnsw", "flat"]
|
|
48
|
+
DistanceMetric = Literal["cosine", "dot", "l2-squared", "hamming", "manhattan"]
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class WeaviateStorage(BaseVectorStorage):
|
|
52
|
+
r"""An implementation of the `BaseVectorStorage` for interacting with
|
|
53
|
+
Weaviate, a cloud-native vector search engine.
|
|
54
|
+
|
|
55
|
+
This class provides multiple ways to connect to Weaviate instances:
|
|
56
|
+
- Weaviate Cloud (WCD)
|
|
57
|
+
- Local Docker/Kubernetes instances
|
|
58
|
+
- Embedded Weaviate
|
|
59
|
+
- Custom connection parameters
|
|
60
|
+
|
|
61
|
+
Args:
|
|
62
|
+
vector_dim (int): The dimension of storing vectors.
|
|
63
|
+
collection_name (Optional[str], optional): Name for the collection in
|
|
64
|
+
Weaviate. If not provided, generates a unique name based on current
|
|
65
|
+
timestamp. (default: :obj:`None`)
|
|
66
|
+
connection_type (ConnectionType, optional): Type of connection to use.
|
|
67
|
+
Supported types: 'local', 'cloud', 'embedded', 'custom'.
|
|
68
|
+
(default: :obj:`"local"`)
|
|
69
|
+
|
|
70
|
+
# Weaviate Cloud parameters
|
|
71
|
+
wcd_cluster_url (Optional[str], optional): Weaviate Cloud cluster URL.
|
|
72
|
+
Required when connection_type='cloud'.
|
|
73
|
+
wcd_api_key (Optional[str], optional): Weaviate Cloud API key.
|
|
74
|
+
Required when connection_type='cloud'.
|
|
75
|
+
|
|
76
|
+
# Local instance parameters
|
|
77
|
+
local_host (str, optional): Local Weaviate host.
|
|
78
|
+
(default: :obj:`"localhost"`)
|
|
79
|
+
local_port (int, optional): Local Weaviate HTTP port.
|
|
80
|
+
(default: :obj:`8080`)
|
|
81
|
+
local_grpc_port (int, optional): Local Weaviate gRPC port.
|
|
82
|
+
(default: :obj:`50051`)
|
|
83
|
+
local_auth_credentials (Optional[Union[str, Any]], optional):
|
|
84
|
+
Authentication credentials for local instance. Can be an API key
|
|
85
|
+
string or Auth object. (default: :obj:`None`)
|
|
86
|
+
|
|
87
|
+
# Embedded Weaviate parameters
|
|
88
|
+
embedded_hostname (str, optional): Embedded instance hostname.
|
|
89
|
+
(default: :obj:`"127.0.0.1"`)
|
|
90
|
+
embedded_port (int, optional): Embedded instance HTTP port.
|
|
91
|
+
(default: :obj:`8079`)
|
|
92
|
+
embedded_grpc_port (int, optional): Embedded instance gRPC port.
|
|
93
|
+
(default: :obj:`50050`)
|
|
94
|
+
embedded_version (Optional[str], optional): Weaviate version for
|
|
95
|
+
embedded instance. If None, uses the default version.
|
|
96
|
+
(default: :obj:`None`)
|
|
97
|
+
embedded_persistence_data_path (Optional[str], optional): Directory
|
|
98
|
+
for embedded database files. (default: :obj:`None`)
|
|
99
|
+
embedded_binary_path (Optional[str], optional): Directory for
|
|
100
|
+
Weaviate binary. (default: :obj:`None`)
|
|
101
|
+
embedded_environment_variables (Optional[Dict[str, str]], optional):
|
|
102
|
+
Environment variables for embedded instance. (default: :obj:`None`)
|
|
103
|
+
|
|
104
|
+
# Custom connection parameters
|
|
105
|
+
custom_http_host (Optional[str], optional): Custom HTTP host.
|
|
106
|
+
custom_http_port (Optional[int], optional): Custom HTTP port.
|
|
107
|
+
custom_http_secure (Optional[bool], optional): Use HTTPS.
|
|
108
|
+
custom_grpc_host (Optional[str], optional): Custom gRPC host.
|
|
109
|
+
custom_grpc_port (Optional[int], optional): Custom gRPC port.
|
|
110
|
+
custom_grpc_secure (Optional[bool], optional): Use secure gRPC.
|
|
111
|
+
custom_auth_credentials (Optional[Any], optional): Custom auth.
|
|
112
|
+
|
|
113
|
+
# Vector index configuration parameters
|
|
114
|
+
vector_index_type (VectorIndexType, optional): Vector index type.
|
|
115
|
+
Supported types: 'hnsw', 'flat'. (default: :obj:`"hnsw"`)
|
|
116
|
+
distance_metric (DistanceMetric, optional): Distance metric for vector
|
|
117
|
+
similarity. Supported metrics: 'cosine', 'dot', 'l2-squared',
|
|
118
|
+
'hamming', 'manhattan'. (default: :obj:`"cosine"`)
|
|
119
|
+
|
|
120
|
+
# Common parameters for all connection types
|
|
121
|
+
headers (Optional[Dict[str, str]], optional): Additional headers for
|
|
122
|
+
third-party API keys (e.g., OpenAI, Cohere). (default: :obj:`None`)
|
|
123
|
+
additional_config (Optional[Any], optional): Advanced configuration
|
|
124
|
+
options like timeouts. (default: :obj:`None`)
|
|
125
|
+
skip_init_checks (bool, optional): Skip initialization checks.
|
|
126
|
+
(default: :obj:`False`)
|
|
127
|
+
|
|
128
|
+
Raises:
|
|
129
|
+
ImportError: If `weaviate` package is not installed.
|
|
130
|
+
ValueError: If connection parameters are invalid or missing.
|
|
131
|
+
RuntimeError: If there's an error setting up the collection.
|
|
132
|
+
|
|
133
|
+
Note:
|
|
134
|
+
This implementation supports synchronous operations only.
|
|
135
|
+
The client connection is automatically handled and closed when the
|
|
136
|
+
storage instance is destroyed.
|
|
137
|
+
"""
|
|
138
|
+
|
|
139
|
+
@dependencies_required('weaviate')
|
|
140
|
+
def __init__(
|
|
141
|
+
self,
|
|
142
|
+
vector_dim: int,
|
|
143
|
+
collection_name: Optional[str] = None,
|
|
144
|
+
connection_type: ConnectionType = "local",
|
|
145
|
+
# Weaviate Cloud parameters
|
|
146
|
+
wcd_cluster_url: Optional[str] = None,
|
|
147
|
+
wcd_api_key: Optional[str] = None,
|
|
148
|
+
# Local instance parameters
|
|
149
|
+
local_host: str = "localhost",
|
|
150
|
+
local_port: int = 8080,
|
|
151
|
+
local_grpc_port: int = 50051,
|
|
152
|
+
local_auth_credentials: Optional[Union[str, Any]] = None,
|
|
153
|
+
# Embedded Weaviate parameters
|
|
154
|
+
embedded_hostname: str = "127.0.0.1",
|
|
155
|
+
embedded_port: int = 8079,
|
|
156
|
+
embedded_grpc_port: int = 50050,
|
|
157
|
+
embedded_version: Optional[str] = None,
|
|
158
|
+
embedded_persistence_data_path: Optional[str] = None,
|
|
159
|
+
embedded_binary_path: Optional[str] = None,
|
|
160
|
+
embedded_environment_variables: Optional[Dict[str, str]] = None,
|
|
161
|
+
# Custom connection parameters
|
|
162
|
+
custom_http_host: Optional[str] = None,
|
|
163
|
+
custom_http_port: Optional[int] = None,
|
|
164
|
+
custom_http_secure: Optional[bool] = None,
|
|
165
|
+
custom_grpc_host: Optional[str] = None,
|
|
166
|
+
custom_grpc_port: Optional[int] = None,
|
|
167
|
+
custom_grpc_secure: Optional[bool] = None,
|
|
168
|
+
custom_auth_credentials: Optional[Any] = None,
|
|
169
|
+
# Vector index configuration parameters
|
|
170
|
+
vector_index_type: VectorIndexType = "hnsw",
|
|
171
|
+
distance_metric: DistanceMetric = "cosine",
|
|
172
|
+
# Common parameters
|
|
173
|
+
headers: Optional[Dict[str, str]] = None,
|
|
174
|
+
additional_config: Optional[Any] = None,
|
|
175
|
+
skip_init_checks: bool = False,
|
|
176
|
+
**kwargs: Any,
|
|
177
|
+
) -> None:
|
|
178
|
+
self.vector_dim = vector_dim
|
|
179
|
+
self.collection_name = (
|
|
180
|
+
collection_name or self._generate_collection_name()
|
|
181
|
+
)
|
|
182
|
+
|
|
183
|
+
# Store vector index configuration
|
|
184
|
+
self.vector_index_type: VectorIndexType = vector_index_type
|
|
185
|
+
self.distance_metric: DistanceMetric = distance_metric
|
|
186
|
+
|
|
187
|
+
# Store connection type configuration
|
|
188
|
+
self.connection_type: ConnectionType = connection_type
|
|
189
|
+
|
|
190
|
+
# Store connection parameters for later use
|
|
191
|
+
self._connection_params = {
|
|
192
|
+
'wcd_cluster_url': wcd_cluster_url,
|
|
193
|
+
'wcd_api_key': wcd_api_key,
|
|
194
|
+
'local_host': local_host,
|
|
195
|
+
'local_port': local_port,
|
|
196
|
+
'local_grpc_port': local_grpc_port,
|
|
197
|
+
'local_auth_credentials': local_auth_credentials,
|
|
198
|
+
'embedded_hostname': embedded_hostname,
|
|
199
|
+
'embedded_port': embedded_port,
|
|
200
|
+
'embedded_grpc_port': embedded_grpc_port,
|
|
201
|
+
'embedded_version': embedded_version,
|
|
202
|
+
'embedded_persistence_data_path': embedded_persistence_data_path,
|
|
203
|
+
'embedded_binary_path': embedded_binary_path,
|
|
204
|
+
'embedded_environment_variables': embedded_environment_variables,
|
|
205
|
+
'custom_http_host': custom_http_host,
|
|
206
|
+
'custom_http_port': custom_http_port,
|
|
207
|
+
'custom_http_secure': custom_http_secure,
|
|
208
|
+
'custom_grpc_host': custom_grpc_host,
|
|
209
|
+
'custom_grpc_port': custom_grpc_port,
|
|
210
|
+
'custom_grpc_secure': custom_grpc_secure,
|
|
211
|
+
'custom_auth_credentials': custom_auth_credentials,
|
|
212
|
+
'headers': headers,
|
|
213
|
+
'additional_config': additional_config,
|
|
214
|
+
'skip_init_checks': skip_init_checks,
|
|
215
|
+
**kwargs,
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
# Store original collection creation kwargs for clear() method
|
|
219
|
+
self._original_collection_kwargs = kwargs.copy()
|
|
220
|
+
|
|
221
|
+
# Create client using the new unified approach
|
|
222
|
+
self._client = self._get_connection_client()
|
|
223
|
+
logger.info(
|
|
224
|
+
f"Created Weaviate client with connection type: "
|
|
225
|
+
f"{self.connection_type}"
|
|
226
|
+
)
|
|
227
|
+
|
|
228
|
+
self._check_and_create_collection(**kwargs)
|
|
229
|
+
|
|
230
|
+
def _get_connection_client(self) -> "WeaviateClient":
|
|
231
|
+
r"""Get Weaviate client based on connection type and user settings."""
|
|
232
|
+
import weaviate
|
|
233
|
+
|
|
234
|
+
# Map connection types to handler methods
|
|
235
|
+
connection_handlers: Dict[ConnectionType, Any] = {
|
|
236
|
+
'cloud': self._create_cloud_client,
|
|
237
|
+
'local': self._create_local_client,
|
|
238
|
+
'embedded': self._create_embedded_client,
|
|
239
|
+
'custom': self._create_custom_client,
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
# Get connection handler
|
|
243
|
+
handler = connection_handlers[self.connection_type]
|
|
244
|
+
|
|
245
|
+
try:
|
|
246
|
+
return handler(weaviate)
|
|
247
|
+
except Exception as e:
|
|
248
|
+
logger.error(
|
|
249
|
+
f"Failed to create {self.connection_type} client: {e}"
|
|
250
|
+
)
|
|
251
|
+
raise
|
|
252
|
+
|
|
253
|
+
def _create_cloud_client(self, weaviate_module: Any) -> "WeaviateClient":
|
|
254
|
+
r"""Create a Weaviate Cloud client."""
|
|
255
|
+
cluster_url = self._connection_params.get('wcd_cluster_url')
|
|
256
|
+
api_key = self._connection_params.get('wcd_api_key')
|
|
257
|
+
|
|
258
|
+
if not cluster_url:
|
|
259
|
+
raise ValueError(
|
|
260
|
+
"wcd_cluster_url is required for cloud connection"
|
|
261
|
+
)
|
|
262
|
+
if not api_key:
|
|
263
|
+
raise ValueError("wcd_api_key is required for cloud connection")
|
|
264
|
+
|
|
265
|
+
return weaviate_module.connect_to_weaviate_cloud(
|
|
266
|
+
cluster_url=cluster_url,
|
|
267
|
+
auth_credentials=api_key,
|
|
268
|
+
headers=self._connection_params.get('headers'),
|
|
269
|
+
additional_config=self._connection_params.get('additional_config'),
|
|
270
|
+
skip_init_checks=self._connection_params.get(
|
|
271
|
+
'skip_init_checks', False
|
|
272
|
+
),
|
|
273
|
+
)
|
|
274
|
+
|
|
275
|
+
def _create_local_client(self, weaviate_module: Any) -> "WeaviateClient":
|
|
276
|
+
r"""Create a local Weaviate client."""
|
|
277
|
+
return weaviate_module.connect_to_local(
|
|
278
|
+
host=self._connection_params.get('local_host', 'localhost'),
|
|
279
|
+
port=self._connection_params.get('local_port', 8080),
|
|
280
|
+
grpc_port=self._connection_params.get('local_grpc_port', 50051),
|
|
281
|
+
headers=self._connection_params.get('headers'),
|
|
282
|
+
additional_config=self._connection_params.get('additional_config'),
|
|
283
|
+
skip_init_checks=self._connection_params.get(
|
|
284
|
+
'skip_init_checks', False
|
|
285
|
+
),
|
|
286
|
+
auth_credentials=self._connection_params.get(
|
|
287
|
+
'local_auth_credentials'
|
|
288
|
+
),
|
|
289
|
+
)
|
|
290
|
+
|
|
291
|
+
def _create_embedded_client(
|
|
292
|
+
self, weaviate_module: Any
|
|
293
|
+
) -> "WeaviateClient":
|
|
294
|
+
r"""Create an embedded Weaviate client."""
|
|
295
|
+
embedded_kwargs = {
|
|
296
|
+
'hostname': self._connection_params.get(
|
|
297
|
+
'embedded_hostname', '127.0.0.1'
|
|
298
|
+
),
|
|
299
|
+
'port': self._connection_params.get('embedded_port', 8079),
|
|
300
|
+
'grpc_port': self._connection_params.get(
|
|
301
|
+
'embedded_grpc_port', 50050
|
|
302
|
+
),
|
|
303
|
+
'headers': self._connection_params.get('headers'),
|
|
304
|
+
'additional_config': self._connection_params.get(
|
|
305
|
+
'additional_config'
|
|
306
|
+
),
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
# Add optional embedded parameters
|
|
310
|
+
if self._connection_params.get('embedded_version') is not None:
|
|
311
|
+
embedded_kwargs['version'] = self._connection_params[
|
|
312
|
+
'embedded_version'
|
|
313
|
+
]
|
|
314
|
+
if (
|
|
315
|
+
self._connection_params.get('embedded_persistence_data_path')
|
|
316
|
+
is not None
|
|
317
|
+
):
|
|
318
|
+
embedded_kwargs['persistence_data_path'] = self._connection_params[
|
|
319
|
+
'embedded_persistence_data_path'
|
|
320
|
+
]
|
|
321
|
+
if self._connection_params.get('embedded_binary_path') is not None:
|
|
322
|
+
embedded_kwargs['binary_path'] = self._connection_params[
|
|
323
|
+
'embedded_binary_path'
|
|
324
|
+
]
|
|
325
|
+
if (
|
|
326
|
+
self._connection_params.get('embedded_environment_variables')
|
|
327
|
+
is not None
|
|
328
|
+
):
|
|
329
|
+
embedded_kwargs['environment_variables'] = self._connection_params[
|
|
330
|
+
'embedded_environment_variables'
|
|
331
|
+
]
|
|
332
|
+
|
|
333
|
+
return weaviate_module.connect_to_embedded(**embedded_kwargs)
|
|
334
|
+
|
|
335
|
+
def _create_custom_client(self, weaviate_module: Any) -> "WeaviateClient":
|
|
336
|
+
r"""Create a custom Weaviate client."""
|
|
337
|
+
# Validate required custom parameters
|
|
338
|
+
required_params = [
|
|
339
|
+
'custom_http_host',
|
|
340
|
+
'custom_http_port',
|
|
341
|
+
'custom_http_secure',
|
|
342
|
+
'custom_grpc_host',
|
|
343
|
+
'custom_grpc_port',
|
|
344
|
+
'custom_grpc_secure',
|
|
345
|
+
]
|
|
346
|
+
|
|
347
|
+
for param in required_params:
|
|
348
|
+
if self._connection_params.get(param) is None:
|
|
349
|
+
raise ValueError(f"{param} is required for custom connection")
|
|
350
|
+
|
|
351
|
+
return weaviate_module.connect_to_custom(
|
|
352
|
+
http_host=self._connection_params['custom_http_host'],
|
|
353
|
+
http_port=self._connection_params['custom_http_port'],
|
|
354
|
+
http_secure=self._connection_params['custom_http_secure'],
|
|
355
|
+
grpc_host=self._connection_params['custom_grpc_host'],
|
|
356
|
+
grpc_port=self._connection_params['custom_grpc_port'],
|
|
357
|
+
grpc_secure=self._connection_params['custom_grpc_secure'],
|
|
358
|
+
headers=self._connection_params.get('headers'),
|
|
359
|
+
additional_config=self._connection_params.get('additional_config'),
|
|
360
|
+
auth_credentials=self._connection_params.get(
|
|
361
|
+
'custom_auth_credentials'
|
|
362
|
+
),
|
|
363
|
+
skip_init_checks=self._connection_params.get(
|
|
364
|
+
'skip_init_checks', False
|
|
365
|
+
),
|
|
366
|
+
)
|
|
367
|
+
|
|
368
|
+
def __del__(self):
|
|
369
|
+
r"""Clean up client connection."""
|
|
370
|
+
try:
|
|
371
|
+
if hasattr(self, '_client') and self._client is not None:
|
|
372
|
+
self._client.close()
|
|
373
|
+
logger.debug("Closed Weaviate client connection")
|
|
374
|
+
except Exception as e:
|
|
375
|
+
logger.warning(f"Error closing Weaviate client: {e}")
|
|
376
|
+
|
|
377
|
+
def close(self) -> None:
|
|
378
|
+
r"""Explicitly close the client connection."""
|
|
379
|
+
if self._client is not None:
|
|
380
|
+
self._client.close()
|
|
381
|
+
logger.info("Explicitly closed Weaviate client connection")
|
|
382
|
+
|
|
383
|
+
def _generate_collection_name(self) -> str:
|
|
384
|
+
r"""Generate a collection name if user doesn't provide one."""
|
|
385
|
+
timestamp = datetime.now().isoformat()
|
|
386
|
+
# Weaviate collection names must start with uppercase and be valid
|
|
387
|
+
# GraphQL names
|
|
388
|
+
valid_name = "Collection_" + re.sub(r'[^a-zA-Z0-9_]', '_', timestamp)
|
|
389
|
+
return valid_name
|
|
390
|
+
|
|
391
|
+
def _check_and_create_collection(self, **kwargs: Any) -> None:
|
|
392
|
+
r"""Check if collection exists and create if it doesn't."""
|
|
393
|
+
if not self._collection_exists(self.collection_name):
|
|
394
|
+
self._create_collection(**kwargs)
|
|
395
|
+
|
|
396
|
+
def _collection_exists(self, collection_name: str) -> bool:
|
|
397
|
+
r"""Check if the collection exists."""
|
|
398
|
+
try:
|
|
399
|
+
collection = self._client.collections.get(collection_name)
|
|
400
|
+
collection.config.get()
|
|
401
|
+
return True
|
|
402
|
+
except Exception:
|
|
403
|
+
return False
|
|
404
|
+
|
|
405
|
+
def _get_vector_index_config(self, **kwargs: Any) -> Any:
|
|
406
|
+
r"""Get vector index configuration based on user settings."""
|
|
407
|
+
import weaviate.classes.config as wvc
|
|
408
|
+
|
|
409
|
+
# Map distance metrics - type safety guaranteed by Literal
|
|
410
|
+
distance_metric_mapping: Dict[DistanceMetric, Any] = {
|
|
411
|
+
'cosine': wvc.VectorDistances.COSINE,
|
|
412
|
+
'dot': wvc.VectorDistances.DOT,
|
|
413
|
+
'l2-squared': wvc.VectorDistances.L2_SQUARED,
|
|
414
|
+
'hamming': wvc.VectorDistances.HAMMING,
|
|
415
|
+
'manhattan': wvc.VectorDistances.MANHATTAN,
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
# Get distance metric - no need for None check due to Literal typing
|
|
419
|
+
distance_metric = distance_metric_mapping[self.distance_metric]
|
|
420
|
+
|
|
421
|
+
# Configure vector index based on type
|
|
422
|
+
if self.vector_index_type == 'hnsw':
|
|
423
|
+
return wvc.Configure.VectorIndex.hnsw(
|
|
424
|
+
distance_metric=distance_metric,
|
|
425
|
+
**kwargs,
|
|
426
|
+
)
|
|
427
|
+
else: # must be 'flat' due to Literal typing
|
|
428
|
+
return wvc.Configure.VectorIndex.flat(
|
|
429
|
+
distance_metric=distance_metric,
|
|
430
|
+
**kwargs,
|
|
431
|
+
)
|
|
432
|
+
|
|
433
|
+
def _create_collection(self, **kwargs: Any) -> None:
|
|
434
|
+
r"""Create a new collection in Weaviate."""
|
|
435
|
+
import weaviate.classes.config as wvc
|
|
436
|
+
|
|
437
|
+
# Separate vector index kwargs from general kwargs to avoid conflicts
|
|
438
|
+
vector_index_kwargs = {}
|
|
439
|
+
collection_kwargs = {}
|
|
440
|
+
|
|
441
|
+
# Known vector index parameters
|
|
442
|
+
vector_index_params = {
|
|
443
|
+
'ef_construction',
|
|
444
|
+
'max_connections',
|
|
445
|
+
'ef',
|
|
446
|
+
'dynamic_ef_min',
|
|
447
|
+
'dynamic_ef_max',
|
|
448
|
+
'dynamic_ef_factor',
|
|
449
|
+
'vector_cache_max_objects',
|
|
450
|
+
'flat_search_cutoff',
|
|
451
|
+
'cleanup_interval_seconds',
|
|
452
|
+
'pq_enabled',
|
|
453
|
+
'pq_segments',
|
|
454
|
+
'pq_centroids',
|
|
455
|
+
'pq_training_limit',
|
|
456
|
+
'pq_encoder',
|
|
457
|
+
}
|
|
458
|
+
|
|
459
|
+
for key, value in kwargs.items():
|
|
460
|
+
if key in vector_index_params:
|
|
461
|
+
vector_index_kwargs[key] = value
|
|
462
|
+
else:
|
|
463
|
+
collection_kwargs[key] = value
|
|
464
|
+
|
|
465
|
+
self._client.collections.create(
|
|
466
|
+
name=self.collection_name,
|
|
467
|
+
vectorizer_config=wvc.Configure.Vectorizer.none(),
|
|
468
|
+
properties=[
|
|
469
|
+
wvc.Property(name="payload", data_type=wvc.DataType.TEXT),
|
|
470
|
+
],
|
|
471
|
+
vector_index_config=self._get_vector_index_config(
|
|
472
|
+
**vector_index_kwargs
|
|
473
|
+
),
|
|
474
|
+
**collection_kwargs,
|
|
475
|
+
)
|
|
476
|
+
|
|
477
|
+
def add(
|
|
478
|
+
self,
|
|
479
|
+
records: List[VectorRecord],
|
|
480
|
+
**kwargs: Any,
|
|
481
|
+
) -> None:
|
|
482
|
+
r"""Saves a list of vector records to the storage.
|
|
483
|
+
|
|
484
|
+
Args:
|
|
485
|
+
records (List[VectorRecord]): List of vector records to be saved.
|
|
486
|
+
**kwargs (Any): Additional keyword arguments.
|
|
487
|
+
|
|
488
|
+
Raises:
|
|
489
|
+
RuntimeError: If there is an error during the saving process.
|
|
490
|
+
"""
|
|
491
|
+
if not records:
|
|
492
|
+
return
|
|
493
|
+
|
|
494
|
+
try:
|
|
495
|
+
collection = self._client.collections.get(self.collection_name)
|
|
496
|
+
|
|
497
|
+
with collection.batch.dynamic() as batch:
|
|
498
|
+
for record in records:
|
|
499
|
+
payload_str = (
|
|
500
|
+
json.dumps(record.payload) if record.payload else ""
|
|
501
|
+
)
|
|
502
|
+
batch.add_object(
|
|
503
|
+
properties={"payload": payload_str},
|
|
504
|
+
vector=record.vector,
|
|
505
|
+
uuid=record.id,
|
|
506
|
+
**kwargs,
|
|
507
|
+
)
|
|
508
|
+
logger.debug(
|
|
509
|
+
f"Successfully added vectors to Weaviate collection: "
|
|
510
|
+
f"{self.collection_name}"
|
|
511
|
+
)
|
|
512
|
+
|
|
513
|
+
except Exception as e:
|
|
514
|
+
raise RuntimeError(f"Failed to add vectors to Weaviate: {e}")
|
|
515
|
+
|
|
516
|
+
def delete(
|
|
517
|
+
self,
|
|
518
|
+
ids: List[str],
|
|
519
|
+
**kwargs: Any,
|
|
520
|
+
) -> None:
|
|
521
|
+
r"""Deletes a list of vectors identified by their IDs from the storage.
|
|
522
|
+
|
|
523
|
+
Args:
|
|
524
|
+
ids (List[str]): List of unique identifiers for the vectors to be
|
|
525
|
+
deleted.
|
|
526
|
+
**kwargs (Any): Additional keyword arguments.
|
|
527
|
+
|
|
528
|
+
Raises:
|
|
529
|
+
RuntimeError: If there is an error during the deletion process.
|
|
530
|
+
"""
|
|
531
|
+
if not ids:
|
|
532
|
+
return
|
|
533
|
+
|
|
534
|
+
try:
|
|
535
|
+
collection = self._client.collections.get(self.collection_name)
|
|
536
|
+
from weaviate.classes.query import Filter
|
|
537
|
+
|
|
538
|
+
collection.data.delete_many(
|
|
539
|
+
where=Filter.by_id().contains_any(ids), **kwargs
|
|
540
|
+
)
|
|
541
|
+
logger.debug(
|
|
542
|
+
f"Successfully deleted vectors in Weaviate: "
|
|
543
|
+
f"{self.collection_name}"
|
|
544
|
+
)
|
|
545
|
+
|
|
546
|
+
except Exception as e:
|
|
547
|
+
raise RuntimeError(f"Failed to delete vectors from Weaviate: {e}")
|
|
548
|
+
|
|
549
|
+
def _calculate_similarity_from_distance(
|
|
550
|
+
self, distance: Optional[float]
|
|
551
|
+
) -> float:
|
|
552
|
+
r"""Calculate similarity score based on distance metric.
|
|
553
|
+
|
|
554
|
+
Args:
|
|
555
|
+
distance (Optional[float]): The distance value from Weaviate.
|
|
556
|
+
|
|
557
|
+
Returns:
|
|
558
|
+
float: Normalized similarity score between 0 and 1.
|
|
559
|
+
"""
|
|
560
|
+
# Return 0.0 if distance is not available
|
|
561
|
+
if distance is None:
|
|
562
|
+
return 0.0
|
|
563
|
+
|
|
564
|
+
# Calculate similarity based on distance metric
|
|
565
|
+
if self.distance_metric == "cosine":
|
|
566
|
+
# Cosine: 0 (identical) to 2 (opposite)
|
|
567
|
+
# Convert to similarity: 1 - (distance / 2)
|
|
568
|
+
return max(0.0, 1.0 - (distance / 2.0))
|
|
569
|
+
|
|
570
|
+
elif self.distance_metric == "dot":
|
|
571
|
+
# Dot product: Weaviate returns -dot,
|
|
572
|
+
# smaller (more negative) is more similar
|
|
573
|
+
# Use sigmoid function to normalize to [0,1] range
|
|
574
|
+
# For dot product, we use: 1 / (1 + exp(distance))
|
|
575
|
+
return 1.0 / (1.0 + math.exp(distance))
|
|
576
|
+
|
|
577
|
+
elif self.distance_metric in ["l2-squared", "manhattan"]:
|
|
578
|
+
# L2-squared and Manhattan: 0 (identical) to ∞
|
|
579
|
+
# Convert to similarity using: 1 / (1 + distance)
|
|
580
|
+
# This provides a smoother decay than exponential
|
|
581
|
+
return 1.0 / (1.0 + distance)
|
|
582
|
+
|
|
583
|
+
elif self.distance_metric == "hamming":
|
|
584
|
+
# Hamming: 0 (identical) to vector_dim (completely different)
|
|
585
|
+
# Convert to similarity: 1 - (distance / vector_dim)
|
|
586
|
+
return max(0.0, 1.0 - (distance / self.vector_dim))
|
|
587
|
+
|
|
588
|
+
else:
|
|
589
|
+
# Unknown metric, return 0.0
|
|
590
|
+
return 0.0
|
|
591
|
+
|
|
592
|
+
def status(self) -> VectorDBStatus:
|
|
593
|
+
r"""Returns status of the vector database.
|
|
594
|
+
|
|
595
|
+
Returns:
|
|
596
|
+
VectorDBStatus: The vector database status.
|
|
597
|
+
"""
|
|
598
|
+
try:
|
|
599
|
+
collection = self._client.collections.get(self.collection_name)
|
|
600
|
+
objects = collection.aggregate.over_all(total_count=True)
|
|
601
|
+
|
|
602
|
+
vector_count = (
|
|
603
|
+
objects.total_count if objects.total_count is not None else 0
|
|
604
|
+
)
|
|
605
|
+
|
|
606
|
+
return VectorDBStatus(
|
|
607
|
+
vector_dim=self.vector_dim, vector_count=vector_count
|
|
608
|
+
)
|
|
609
|
+
|
|
610
|
+
except Exception as e:
|
|
611
|
+
logger.warning(f"Failed to get status from Weaviate: {e}")
|
|
612
|
+
return VectorDBStatus(vector_dim=self.vector_dim, vector_count=0)
|
|
613
|
+
|
|
614
|
+
def query(
|
|
615
|
+
self,
|
|
616
|
+
query: VectorDBQuery,
|
|
617
|
+
**kwargs: Any,
|
|
618
|
+
) -> List[VectorDBQueryResult]:
|
|
619
|
+
r"""Searches for similar vectors in the storage based on the provided
|
|
620
|
+
query.
|
|
621
|
+
|
|
622
|
+
Args:
|
|
623
|
+
query (VectorDBQuery): The query object containing the search
|
|
624
|
+
vector and the number of top similar vectors to retrieve.
|
|
625
|
+
**kwargs (Any): Additional keyword arguments.
|
|
626
|
+
|
|
627
|
+
Returns:
|
|
628
|
+
List[VectorDBQueryResult]: A list of vectors retrieved from the
|
|
629
|
+
storage based on similarity to the query vector.
|
|
630
|
+
"""
|
|
631
|
+
from weaviate.classes.query import MetadataQuery
|
|
632
|
+
|
|
633
|
+
try:
|
|
634
|
+
collection = self._client.collections.get(self.collection_name)
|
|
635
|
+
|
|
636
|
+
response = collection.query.near_vector(
|
|
637
|
+
near_vector=query.query_vector,
|
|
638
|
+
limit=query.top_k,
|
|
639
|
+
include_vector=True,
|
|
640
|
+
return_metadata=MetadataQuery(distance=True),
|
|
641
|
+
**kwargs,
|
|
642
|
+
)
|
|
643
|
+
|
|
644
|
+
results = []
|
|
645
|
+
for obj in response.objects:
|
|
646
|
+
# Calculate similarity score based on distance metric
|
|
647
|
+
similarity = self._calculate_similarity_from_distance(
|
|
648
|
+
obj.metadata.distance
|
|
649
|
+
)
|
|
650
|
+
|
|
651
|
+
# Handle payload
|
|
652
|
+
payload = None
|
|
653
|
+
if obj.properties.get('payload'):
|
|
654
|
+
payload_value = obj.properties['payload']
|
|
655
|
+
if isinstance(payload_value, str):
|
|
656
|
+
try:
|
|
657
|
+
payload = json.loads(payload_value)
|
|
658
|
+
except (json.JSONDecodeError, TypeError):
|
|
659
|
+
payload = {"raw": payload_value}
|
|
660
|
+
else:
|
|
661
|
+
payload = {"raw": str(payload_value)}
|
|
662
|
+
|
|
663
|
+
# Handle vector data
|
|
664
|
+
# In newer versions of Weaviate, obj.vector returns a dict
|
|
665
|
+
# with 'default' key
|
|
666
|
+
|
|
667
|
+
vector: List[float] = []
|
|
668
|
+
if hasattr(obj, 'vector') and obj.vector is not None:
|
|
669
|
+
# New format: {'default': [vector_values]}
|
|
670
|
+
vector_data = obj.vector.get('default', [])
|
|
671
|
+
if (
|
|
672
|
+
vector_data
|
|
673
|
+
and isinstance(vector_data, list)
|
|
674
|
+
and len(vector_data) > 0
|
|
675
|
+
and isinstance(vector_data[0], float)
|
|
676
|
+
):
|
|
677
|
+
vector = cast(List[float], vector_data)
|
|
678
|
+
else:
|
|
679
|
+
# unsupported vector data format
|
|
680
|
+
vector = []
|
|
681
|
+
else:
|
|
682
|
+
vector = []
|
|
683
|
+
|
|
684
|
+
result = VectorDBQueryResult.create(
|
|
685
|
+
similarity=similarity,
|
|
686
|
+
vector=vector,
|
|
687
|
+
id=str(obj.uuid),
|
|
688
|
+
payload=payload,
|
|
689
|
+
)
|
|
690
|
+
results.append(result)
|
|
691
|
+
|
|
692
|
+
return results
|
|
693
|
+
|
|
694
|
+
except Exception as e:
|
|
695
|
+
raise RuntimeError(f"Failed to query vectors from Weaviate: {e}")
|
|
696
|
+
|
|
697
|
+
def clear(self) -> None:
|
|
698
|
+
r"""Remove all vectors from the storage."""
|
|
699
|
+
try:
|
|
700
|
+
self._client.collections.delete(self.collection_name)
|
|
701
|
+
self._create_collection(**self._original_collection_kwargs)
|
|
702
|
+
except Exception as e:
|
|
703
|
+
raise RuntimeError(f"Failed to clear Weaviate collection: {e}")
|
|
704
|
+
|
|
705
|
+
def load(self) -> None:
|
|
706
|
+
r"""Load the collection hosted on cloud service."""
|
|
707
|
+
# For Weaviate, collections are automatically available when client
|
|
708
|
+
# connects
|
|
709
|
+
pass
|
|
710
|
+
|
|
711
|
+
@property
|
|
712
|
+
def client(self) -> "WeaviateClient":
|
|
713
|
+
r"""Provides access to the underlying vector database client."""
|
|
714
|
+
return self._client
|