camel-ai 0.2.37__py3-none-any.whl → 0.2.38__py3-none-any.whl

camel/models/sglang_model.py

@@ -12,6 +12,7 @@
 # limitations under the License.
 # ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
 import logging
+import os
 import subprocess
 import threading
 import time
@@ -51,6 +52,10 @@ class SGLangModel(BaseModelBackend):
             use for the model. If not provided, :obj:`OpenAITokenCounter(
             ModelType.GPT_4O_MINI)` will be used.
             (default: :obj:`None`)
+        timeout (Optional[float], optional): The timeout value in seconds for
+            API calls. If not provided, will fall back to the MODEL_TIMEOUT
+            environment variable or default to 180 seconds.
+            (default: :obj:`None`)
 
     Reference: https://sgl-project.github.io/backend/openai_api_completions.html
     """
@@ -62,6 +67,7 @@ class SGLangModel(BaseModelBackend):
         api_key: Optional[str] = None,
         url: Optional[str] = None,
         token_counter: Optional[BaseTokenCounter] = None,
+        timeout: Optional[float] = None,
     ) -> None:
         if model_config_dict is None:
             model_config_dict = SGLangConfig().as_dict()
@@ -73,8 +79,9 @@ class SGLangModel(BaseModelBackend):
         self._lock = threading.Lock()
         self._inactivity_thread: Optional[threading.Thread] = None
 
+        timeout = timeout or float(os.environ.get("MODEL_TIMEOUT", 180))
         super().__init__(
-            model_type, model_config_dict, api_key, url, token_counter
+            model_type, model_config_dict, api_key, url, token_counter, timeout
         )
 
         self._client = None
@@ -82,13 +89,13 @@ class SGLangModel(BaseModelBackend):
         if self._url:
             # Initialize the client if an existing URL is provided
             self._client = OpenAI(
-                timeout=180,
+                timeout=self._timeout,
                 max_retries=3,
                 api_key="Set-but-ignored",  # required but ignored
                 base_url=self._url,
             )
             self._async_client = AsyncOpenAI(
-                timeout=180,
+                timeout=self._timeout,
                 max_retries=3,
                 api_key="Set-but-ignored",  # required but ignored
                 base_url=self._url,
@@ -123,7 +130,7 @@ class SGLangModel(BaseModelBackend):
             self.last_run_time = time.time()
             # Initialize the client after the server starts
             self._client = OpenAI(
-                timeout=180,
+                timeout=self._timeout,
                 max_retries=3,
                 api_key="Set-but-ignored",  # required but ignored
                 base_url=self._url,

camel/models/siliconflow_model.py

@@ -51,6 +51,10 @@ class SiliconFlowModel(BaseModelBackend):
             use for the model. If not provided, :obj:`OpenAITokenCounter(
             ModelType.GPT_4O_MINI)` will be used.
             (default: :obj:`None`)
+        timeout (Optional[float], optional): The timeout value in seconds for
+            API calls. If not provided, will fall back to the MODEL_TIMEOUT
+            environment variable or default to 180 seconds.
+            (default: :obj:`None`)
     """
 
     @api_keys_required(
@@ -65,6 +69,7 @@ class SiliconFlowModel(BaseModelBackend):
         api_key: Optional[str] = None,
         url: Optional[str] = None,
         token_counter: Optional[BaseTokenCounter] = None,
+        timeout: Optional[float] = None,
     ) -> None:
         if model_config_dict is None:
             model_config_dict = SiliconFlowConfig().as_dict()
@@ -73,11 +78,12 @@ class SiliconFlowModel(BaseModelBackend):
             "SILICONFLOW_API_BASE_URL",
             "https://api.siliconflow.cn/v1/",
         )
+        timeout = timeout or float(os.environ.get("MODEL_TIMEOUT", 180))
         super().__init__(
-            model_type, model_config_dict, api_key, url, token_counter
+            model_type, model_config_dict, api_key, url, token_counter, timeout
         )
         self._client = OpenAI(
-            timeout=180,
+            timeout=self._timeout,
             max_retries=3,
             api_key=self._api_key,
             base_url=self._url,

camel/models/stub_model.py

@@ -82,10 +82,11 @@ class StubModel(BaseModelBackend):
         api_key: Optional[str] = None,
         url: Optional[str] = None,
         token_counter: Optional[BaseTokenCounter] = None,
+        timeout: Optional[float] = None,
     ) -> None:
         r"""All arguments are unused for the dummy model."""
         super().__init__(
-            model_type, model_config_dict, api_key, url, token_counter
+            model_type, model_config_dict, api_key, url, token_counter, timeout
         )
 
     @property

camel/models/togetherai_model.py

@@ -52,6 +52,10 @@ class TogetherAIModel(BaseModelBackend):
         token_counter (Optional[BaseTokenCounter], optional): Token counter to
             use for the model. If not provided, :obj:`OpenAITokenCounter(
             ModelType.GPT_4O_MINI)` will be used.
+        timeout (Optional[float], optional): The timeout value in seconds for
+            API calls. If not provided, will fall back to the MODEL_TIMEOUT
+            environment variable or default to 180 seconds.
+            (default: :obj:`None`)
     """
 
     @api_keys_required(
@@ -66,6 +70,7 @@ class TogetherAIModel(BaseModelBackend):
         api_key: Optional[str] = None,
         url: Optional[str] = None,
         token_counter: Optional[BaseTokenCounter] = None,
+        timeout: Optional[float] = None,
     ) -> None:
         if model_config_dict is None:
             model_config_dict = TogetherAIConfig().as_dict()
@@ -73,18 +78,19 @@ class TogetherAIModel(BaseModelBackend):
         url = url or os.environ.get(
             "TOGETHER_API_BASE_URL", "https://api.together.xyz/v1"
         )
+        timeout = timeout or float(os.environ.get("MODEL_TIMEOUT", 180))
         super().__init__(
-            model_type, model_config_dict, api_key, url, token_counter
+            model_type, model_config_dict, api_key, url, token_counter, timeout
         )
 
         self._client = OpenAI(
-            timeout=180,
+            timeout=self._timeout,
             max_retries=3,
             api_key=self._api_key,
             base_url=self._url,
         )
         self._async_client = AsyncOpenAI(
-            timeout=180,
+            timeout=self._timeout,
             max_retries=3,
             api_key=self._api_key,
             base_url=self._url,

camel/models/vllm_model.py

@@ -50,6 +50,10 @@ class VLLMModel(BaseModelBackend):
             use for the model. If not provided, :obj:`OpenAITokenCounter(
             ModelType.GPT_4O_MINI)` will be used.
             (default: :obj:`None`)
+        timeout (Optional[float], optional): The timeout value in seconds for
+            API calls. If not provided, will fall back to the MODEL_TIMEOUT
+            environment variable or default to 180 seconds.
+            (default: :obj:`None`)
 
     References:
         https://docs.vllm.ai/en/latest/serving/openai_compatible_server.html
@@ -62,24 +66,26 @@ class VLLMModel(BaseModelBackend):
         api_key: Optional[str] = None,
         url: Optional[str] = None,
         token_counter: Optional[BaseTokenCounter] = None,
+        timeout: Optional[float] = None,
     ) -> None:
         if model_config_dict is None:
             model_config_dict = VLLMConfig().as_dict()
         url = url or os.environ.get("VLLM_BASE_URL")
+        timeout = timeout or float(os.environ.get("MODEL_TIMEOUT", 180))
         super().__init__(
-            model_type, model_config_dict, api_key, url, token_counter
+            model_type, model_config_dict, api_key, url, token_counter, timeout
         )
         if not self._url:
             self._start_server()
         # Use OpenAI cilent as interface call vLLM
         self._client = OpenAI(
-            timeout=180,
+            timeout=self._timeout,
             max_retries=3,
             api_key="EMPTY",  # required but ignored
             base_url=self._url,
         )
         self._async_client = AsyncOpenAI(
-            timeout=180,
+            timeout=self._timeout,
             max_retries=3,
             api_key="EMPTY",  # required but ignored
             base_url=self._url,

camel/models/yi_model.py

@@ -51,6 +51,10 @@ class YiModel(BaseModelBackend):
             use for the model. If not provided, :obj:`OpenAITokenCounter(
             ModelType.GPT_4O_MINI)` will be used.
             (default: :obj:`None`)
+        timeout (Optional[float], optional): The timeout value in seconds for
+            API calls. If not provided, will fall back to the MODEL_TIMEOUT
+            environment variable or default to 180 seconds.
+            (default: :obj:`None`)
     """
 
     @api_keys_required(
@@ -65,6 +69,7 @@ class YiModel(BaseModelBackend):
         api_key: Optional[str] = None,
         url: Optional[str] = None,
         token_counter: Optional[BaseTokenCounter] = None,
+        timeout: Optional[float] = None,
     ) -> None:
         if model_config_dict is None:
             model_config_dict = YiConfig().as_dict()
@@ -72,17 +77,18 @@ class YiModel(BaseModelBackend):
         url = url or os.environ.get(
             "YI_API_BASE_URL", "https://api.lingyiwanwu.com/v1"
         )
+        timeout = timeout or float(os.environ.get("MODEL_TIMEOUT", 180))
         super().__init__(
-            model_type, model_config_dict, api_key, url, token_counter
+            model_type, model_config_dict, api_key, url, token_counter, timeout
         )
         self._client = OpenAI(
-            timeout=180,
+            timeout=self._timeout,
             max_retries=3,
             api_key=self._api_key,
             base_url=self._url,
         )
         self._async_client = AsyncOpenAI(
-            timeout=180,
+            timeout=self._timeout,
             max_retries=3,
             api_key=self._api_key,
             base_url=self._url,

camel/models/zhipuai_model.py

@@ -51,6 +51,10 @@ class ZhipuAIModel(BaseModelBackend):
             use for the model. If not provided, :obj:`OpenAITokenCounter(
             ModelType.GPT_4O_MINI)` will be used.
             (default: :obj:`None`)
+        timeout (Optional[float], optional): The timeout value in seconds for
+            API calls. If not provided, will fall back to the MODEL_TIMEOUT
+            environment variable or default to 180 seconds.
+            (default: :obj:`None`)
     """
 
     @api_keys_required(
@@ -65,6 +69,7 @@ class ZhipuAIModel(BaseModelBackend):
         api_key: Optional[str] = None,
         url: Optional[str] = None,
         token_counter: Optional[BaseTokenCounter] = None,
+        timeout: Optional[float] = None,
     ) -> None:
         if model_config_dict is None:
             model_config_dict = ZhipuAIConfig().as_dict()
@@ -72,17 +77,18 @@ class ZhipuAIModel(BaseModelBackend):
         url = url or os.environ.get(
             "ZHIPUAI_API_BASE_URL", "https://open.bigmodel.cn/api/paas/v4/"
         )
+        timeout = timeout or float(os.environ.get("MODEL_TIMEOUT", 180))
         super().__init__(
-            model_type, model_config_dict, api_key, url, token_counter
+            model_type, model_config_dict, api_key, url, token_counter, timeout
         )
         self._client = OpenAI(
-            timeout=180,
+            timeout=self._timeout,
             max_retries=3,
             api_key=self._api_key,
             base_url=self._url,
         )
         self._async_client = AsyncOpenAI(
-            timeout=180,
+            timeout=self._timeout,
             max_retries=3,
             api_key=self._api_key,
             base_url=self._url,

camel/retrievers/auto_retriever.py

@@ -29,6 +29,7 @@ from camel.storages import (
     BaseVectorStorage,
     MilvusStorage,
     QdrantStorage,
+    TiDBStorage,
 )
 from camel.types import StorageType
 from camel.utils import Constants
@@ -90,6 +91,19 @@ class AutoRetriever:
                 url_and_api_key=self.url_and_api_key,
             )
 
+        if self.storage_type == StorageType.TIDB:
+            if self.url_and_api_key is None:
+                raise ValueError(
+                    "URL (database url) and API key required for TiDB storage "
+                    "are not provided. Format: "
+                    "mysql+pymysql://<username>:<password>@<host>:4000/test"
+                )
+            return TiDBStorage(
+                vector_dim=self.embedding_model.get_output_dim(),
+                collection_name=collection_name,
+                url_and_api_key=self.url_and_api_key,
+            )
+
         if self.storage_type == StorageType.QDRANT:
             return QdrantStorage(
                 vector_dim=self.embedding_model.get_output_dim(),

camel/storages/__init__.py

@@ -28,6 +28,7 @@ from .vectordb_storages.base import (
 )
 from .vectordb_storages.milvus import MilvusStorage
 from .vectordb_storages.qdrant import QdrantStorage
+from .vectordb_storages.tidb import TiDBStorage
 
 __all__ = [
     'BaseKeyValueStorage',
@@ -40,6 +41,7 @@ __all__ = [
     'VectorDBQueryResult',
     'QdrantStorage',
     'MilvusStorage',
+    "TiDBStorage",
     'BaseGraphStorage',
     'Neo4jGraph',
     'NebulaGraph',

camel/storages/vectordb_storages/__init__.py

@@ -21,6 +21,7 @@ from .base import (
 )
 from .milvus import MilvusStorage
 from .qdrant import QdrantStorage
+from .tidb import TiDBStorage
 
 __all__ = [
     'BaseVectorStorage',
@@ -28,6 +29,7 @@ __all__ = [
     'VectorDBQueryResult',
     'QdrantStorage',
     'MilvusStorage',
+    "TiDBStorage",
     'VectorRecord',
     'VectorDBStatus',
 ]

camel/storages/vectordb_storages/tidb.py

@@ -0,0 +1,332 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+import json
+import logging
+import re
+from datetime import datetime
+from enum import Enum
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
+
+from camel.storages.vectordb_storages import (
+    BaseVectorStorage,
+    VectorDBQuery,
+    VectorDBQueryResult,
+    VectorDBStatus,
+    VectorRecord,
+)
+from camel.utils import dependencies_required
+
+if TYPE_CHECKING:
+    from pytidb import Table, TiDBClient
+
+logger = logging.getLogger(__name__)
+
+
+class EnumEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, Enum):
+            return obj.value
+        return super().default(obj)
+
+
+class TiDBStorage(BaseVectorStorage):
+    r"""An implementation of the `BaseVectorStorage` for interacting with TiDB.
+
+    The detailed information about TiDB is available at:
+    `TiDB Vector Search <https://ai.pingcap.com/>`_
+
+    Args:
+        vector_dim (int): The dimension of storing vectors.
+        url_and_api_key (Optional[Union[Tuple[str, str], str]]): A tuple
+            containing the database url and API key for connecting to a TiDB
+            cluster. The URL should be in the format:
+           "mysql+pymysql://<username>:<password>@<host>:<port>/<db_name>".
+           TiDB will not use the API Key, but retains the definition for
+           interface compatible.
+        collection_name (Optional[str]): Name of the collection.
+            The collection name will be used as the table name in TiDB. If not
+            provided, set it to the current time with iso format.
+        **kwargs (Any): Additional keyword arguments for initializing
+            TiDB connection.
+
+    Raises:
+        ImportError: If `pytidb` package is not installed.
+    """
+
+    @dependencies_required('pytidb')
+    def __init__(
+        self,
+        vector_dim: int,
+        collection_name: Optional[str] = None,
+        url_and_api_key: Optional[Union[Tuple[str, str], str]] = None,
+        **kwargs: Any,
+    ) -> None:
+        from pytidb import TiDBClient
+
+        self._client: TiDBClient
+        database_url = None
+        if isinstance(url_and_api_key, str):
+            database_url = url_and_api_key
+        elif isinstance(url_and_api_key, tuple):
+            database_url = url_and_api_key[0]
+        self._create_client(database_url, **kwargs)
+        self.vector_dim = vector_dim
+        self.collection_name = collection_name or self._generate_table_name()
+        self._table = self._open_and_create_table()
+        self._table_model = self._table.table_model
+        self._check_table()
+
+    def _create_client(
+        self,
+        database_url: Optional[str] = None,
+        **kwargs: Any,
+    ) -> None:
+        r"""Initializes the TiDB client with the provided connection details.
+
+        Args:
+            database_url (Optional[str]): The database connection string for
+                the TiDB server.
+            **kwargs: Additional keyword arguments passed to the TiDB client.
+        """
+        from pytidb import TiDBClient
+
+        self._client = TiDBClient.connect(
+            database_url,
+            **kwargs,
+        )
+
+    def _get_table_model(self, collection_name: str) -> Any:
+        from pytidb.schema import Field, TableModel, VectorField
+        from sqlalchemy import JSON
+
+        class VectorDBRecord(TableModel):
+            id: Optional[str] = Field(None, primary_key=True)
+            vector: list[float] = VectorField(self.vector_dim)
+            payload: Optional[dict[str, Any]] = Field(None, sa_type=JSON)
+
+        # Notice: Avoid repeated definition warnings by dynamically generating
+        # class names.
+        return type(
+            f"VectorDBRecord_{collection_name}",
+            (VectorDBRecord,),
+            {"__tablename__": collection_name},
+            table=True,
+        )
+
+    def _open_and_create_table(self) -> "Table[Any]":
+        r"""Opens an existing table or creates a new table in TiDB."""
+        table = self._client.open_table(self.collection_name)
+        if table is None:
+            table = self._client.create_table(
+                schema=self._get_table_model(self.collection_name)
+            )
+        return table
+
+    def _check_table(self):
+        r"""Ensuring the specified table matches the specified vector
+        dimensionality.
+        """
+        in_dim = self._get_table_info()["vector_dim"]
+        if in_dim != self.vector_dim:
+            raise ValueError(
+                "Vector dimension of the existing table "
+                f'"{self.collection_name}" ({in_dim}) is different from '
+                f"the given embedding dim ({self.vector_dim})."
+            )
+
+    def _generate_table_name(self) -> str:
+        r"""Generates a unique name for a new table based on the current
+        timestamp. TiDB table names can only contain alphanumeric
+        characters and underscores.
+
+        Returns:
+            str: A unique, valid table name.
+        """
+        timestamp = datetime.now().isoformat()
+        transformed_name = re.sub(r'[^a-zA-Z0-9_]', '_', timestamp)
+        valid_name = "vectors_" + transformed_name
+        return valid_name
+
+    def _get_table_info(self) -> Dict[str, Any]:
+        r"""Retrieves details of an existing table.
+
+        Returns:
+            Dict[str, Any]: A dictionary containing details about the
+                table.
+        """
+        vector_count = self._table.rows()
+        # Get vector dimension from table schema
+        columns = self._table.columns()
+        dim_value = None
+        for col in columns:
+            match = re.search(r'vector\((\d+)\)', col.column_type)
+            if match:
+                dim_value = int(match.group(1))
+                break
+
+        # If no vector column found, log a warning
+        if dim_value is None:
+            logger.warning(
+                f"No vector column found in table {self.collection_name}. "
+                "This may indicate an incompatible table schema."
+            )
+
+        return {
+            "vector_count": vector_count,
+            "vector_dim": dim_value,
+        }
+
+    def _validate_and_convert_vectors(
+        self, records: List[VectorRecord]
+    ) -> List[Any]:
+        r"""Validates and converts VectorRecord instances to VectorDBRecord
+        instances.
+
+        Args:
+            records (List[VectorRecord]): List of vector records to validate
+                and convert.
+
+        Returns:
+            List[VectorDBRecord]: A list of VectorDBRecord instances.
+        """
+        db_records = []
+        for record in records:
+            payload = record.payload
+            if isinstance(payload, str):
+                payload = json.loads(payload)
+            elif isinstance(payload, dict):
+                payload = json.loads(json.dumps(payload, cls=EnumEncoder))
+            else:
+                payload = None
+
+            db_records.append(
+                self._table_model(
+                    id=record.id,
+                    vector=record.vector,
+                    payload=payload,
+                )
+            )
+        return db_records
+
+    def add(
+        self,
+        records: List[VectorRecord],
+        **kwargs,
+    ) -> None:
+        r"""Adds a list of vectors to the specified table.
+
+        Args:
+            records (List[VectorRecord]): List of vectors to be added.
+            **kwargs (Any): Additional keyword arguments pass to insert.
+
+        Raises:
+            RuntimeError: If there was an error in the addition process.
+        """
+
+        db_records = self._validate_and_convert_vectors(records)
+        if len(db_records) == 0:
+            return
+        self._table.bulk_insert(db_records)
+
+        logger.debug(
+            f"Successfully added vectors to TiDB table: {self.collection_name}"
+        )
+
+    def delete(
+        self,
+        ids: List[str],
+        **kwargs: Any,
+    ) -> None:
+        r"""Deletes a list of vectors identified by their IDs from the
+        storage.
+
+        Args:
+            ids (List[str]): List of unique identifiers for the vectors to be
+                deleted.
+            **kwargs (Any): Additional keyword arguments passed to delete.
+
+        Raises:
+            RuntimeError: If there is an error during the deletion process.
+        """
+        self._table.delete({"id": {"$in": ids}})
+        logger.debug(
+            f"Successfully deleted vectors from TiDB table "
+            f"<{self.collection_name}>"
+        )
+
+    def status(self) -> VectorDBStatus:
+        r"""Retrieves the current status of the TiDB table.
+
+        Returns:
+            VectorDBStatus: An object containing information about the
+                table's status.
+        """
+        status = self._get_table_info()
+        return VectorDBStatus(
+            vector_dim=status["vector_dim"],
+            vector_count=status["vector_count"],
+        )
+
+    def query(
+        self,
+        query: VectorDBQuery,
+        **kwargs: Any,
+    ) -> List[VectorDBQueryResult]:
+        r"""Searches for similar vectors in the storage based on the provided
+        query.
+
+        Args:
+            query (VectorDBQuery): The query object containing the search
+                vector and the number of top similar vectors to retrieve.
+            **kwargs (Any): Additional keyword arguments passed to search.
+
+        Returns:
+            List[VectorDBQueryResult]: A list of vectors retrieved from the
+                storage based on similarity to the query vector.
+        """
+        rows = (
+            self._table.search(query.query_vector).limit(query.top_k).to_list()
+        )
+
+        query_results = []
+        for row in rows:
+            query_results.append(
+                VectorDBQueryResult.create(
+                    similarity=float(row['similarity_score']),
+                    id=str(row['id']),
+                    payload=row['payload'],
+                    vector=row['vector'],
+                )
+            )
+        return query_results
+
+    def clear(self) -> None:
+        r"""Removes all vectors from the TiDB table. This method
+        deletes the existing table and then recreates it with the same
+        schema to effectively remove all stored vectors.
+        """
+        self._table.truncate()
+
+    def load(self) -> None:
+        r"""Load the collection hosted on cloud service."""
+        pass
+
+    @property
+    def client(self) -> "TiDBClient":
+        r"""Provides direct access to the TiDB client.
+
+        Returns:
+            Any: The TiDB client instance.
+        """
+        return self._client

camel/toolkits/__init__.py

@@ -61,6 +61,9 @@ from .file_write_toolkit import FileWriteToolkit
 from .terminal_toolkit import TerminalToolkit
 from .pubmed_toolkit import PubMedToolkit
 from .thinking_toolkit import ThinkingToolkit
+from .openai_agent_toolkit import OpenAIAgentToolkit
+from .searxng_toolkit import SearxNGToolkit
+
 
 __all__ = [
     'BaseToolkit',
@@ -110,4 +113,6 @@ __all__ = [
     'TerminalToolkit',
     'PubMedToolkit',
     'ThinkingToolkit',
+    'OpenAIAgentToolkit',
+    'SearxNGToolkit',
 ]