camel-ai 0.1.5.4__py3-none-any.whl → 0.1.5.5__py3-none-any.whl

camel/loaders/__init__.py

@@ -13,10 +13,12 @@
 # =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
 
 from .base_io import File, read_file
+from .jina_url_reader import JinaURLReader
 from .unstructured_io import UnstructuredIO
 
 __all__ = [
     'File',
     'read_file',
     'UnstructuredIO',
+    'JinaURLReader',
 ]

camel/loaders/base_io.py

@@ -19,6 +19,8 @@ from hashlib import md5
 from io import BytesIO
 from typing import Any, Dict, List, Optional
 
+from camel.utils import dependencies_required
+
 
 class File(ABC):
     r"""Represents an uploaded file comprised of Documents"""
@@ -94,6 +96,7 @@ def strip_consecutive_newlines(text: str) -> str:
 
 class DocxFile(File):
     @classmethod
+    @dependencies_required('docx2txt')
     def from_bytes(cls, file: BytesIO) -> "DocxFile":
         r"""Creates a DocxFile object from a BytesIO object.
 
@@ -104,15 +107,8 @@ class DocxFile(File):
         Returns:
             DocxFile: A DocxFile object.
         """
-        # Use docx2txt to extract text from docx files
-        try:
-            import docx2txt
-        except ImportError:
-            raise ImportError(
-                "Please install `docx2txt` first. "
-                "You can install it by running "
-                "`pip install docx2txt`."
-            )
+        import docx2txt
+
         text = docx2txt.process(file)
         text = strip_consecutive_newlines(text)
         # Create a dictionary with the extracted text

camel/loaders/jina_url_reader.py

@@ -0,0 +1,99 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+
+import os
+from typing import Any, Optional
+
+from camel.types.enums import JinaReturnFormat
+
+JINA_ENDPOINT = "https://r.jina.ai/"
+
+
+class JinaURLReader:
+    r"""URL Reader provided by Jina AI. The output is cleaner and more
+    LLM-friendly than the URL Reader of UnstructuredIO. Can be configured to
+    replace the UnstructuredIO URL Reader in the pipeline.
+
+    Args:
+        api_key (Optional[str], optional): The API key for Jina AI. If not
+            provided, the reader will have a lower rate limit. Defaults to
+            None.
+        return_format (ReturnFormat, optional): The level of detail
+            of the returned content, which is optimized for LLMs. For
+            now screenshots are not supported. Defaults to
+            ReturnFormat.DEFAULT.
+        json_response (bool, optional): Whether to return the response
+            in JSON format. Defaults to False.
+        timeout (int, optional): The maximum time in seconds to wait for
+            the page to be rendered. Defaults to 30.
+        **kwargs (Any): Additional keyword arguments, including proxies,
+            cookies, etc. It should align with the HTTP Header field and
+            value pairs listed in the reference.
+
+    References:
+        https://jina.ai/reader
+    """
+
+    def __init__(
+        self,
+        api_key: Optional[str] = None,
+        return_format: JinaReturnFormat = JinaReturnFormat.DEFAULT,
+        json_response: bool = False,
+        timeout: int = 30,
+        **kwargs: Any,
+    ) -> None:
+        api_key = api_key or os.getenv('JINA_API_KEY')
+        if api_key is None:
+            print(
+                "[JinaURLReader] JINA_API_KEY not set. This will result in a "
+                "low rate limit of Jina URL Reader. Get API key here: "
+                "https://jina.ai/reader."
+            )
+
+        # if the following field not provided, it will be None
+        api_field = f"Bearer {api_key}" if api_key else None
+        json_field = "application/json" if json_response else None
+
+        raw_headers = {
+            "Authorization": api_field,
+            "X-Return-Format": return_format.value,
+            "Accept": json_field,
+            "X-Timeout": str(timeout),
+            **kwargs,
+        }
+
+        # eliminate None values
+        self._headers = {k: v for k, v in raw_headers.items() if v}
+
+    def read_content(self, url: str) -> str:
+        r"""Reads the content of a URL and returns it as a string with
+        given form.
+
+        Args:
+            url (str): The URL to read.
+
+        Returns:
+            str: The content of the URL.
+        """
+
+        import requests
+
+        full_url = f"{JINA_ENDPOINT}{url}"
+        try:
+            resp = requests.get(full_url, headers=self._headers)
+            resp.raise_for_status()
+        except Exception as e:
+            raise Exception(f"Failed to read content from {url}: {e}") from e
+
+        return resp.text

camel/loaders/unstructured_io.py

@@ -16,6 +16,8 @@ from typing import Any, Dict, List, Literal, Optional, Tuple, Union
 
 from unstructured.documents.elements import Element
 
+from camel.utils import dependencies_required
+
 
 class UnstructuredIO:
     r"""A class to handle various functionalities provided by the
@@ -37,6 +39,7 @@ class UnstructuredIO:
         """
         self._ensure_unstructured_version(self.UNSTRUCTURED_MIN_VERSION)
 
+    @dependencies_required('unstructured')
     def _ensure_unstructured_version(self, min_version: str) -> None:
         r"""Validates that the installed 'Unstructured' library version
         satisfies the specified minimum version requirement. This function is
@@ -58,12 +61,7 @@ class UnstructuredIO:
                 strings.
         """
         from packaging import version
-
-        try:
-            from unstructured.__version__ import __version__
-
-        except ImportError as e:
-            raise ImportError("Package `unstructured` not installed.") from e
+        from unstructured.__version__ import __version__
 
         # Use packaging.version to compare versions
         min_ver = version.parse(min_version)

camel/models/anthropic_model.py

@@ -23,7 +23,7 @@ from camel.types import ChatCompletion, ModelType
 from camel.utils import (
     AnthropicTokenCounter,
     BaseTokenCounter,
-    model_api_key_required,
+    api_keys_required,
 )
 
 
@@ -46,11 +46,13 @@ class AnthropicModel(BaseModelBackend):
                 be fed into Anthropic.messages.create().
             api_key (Optional[str]): The API key for authenticating with the
                 Anthropic service. (default: :obj:`None`)
-            url (Optional[str]): The url to the model service.
+            url (Optional[str]): The url to the Anthropic service. (default:
+                :obj:`None`)
         """
         super().__init__(model_type, model_config_dict, api_key, url)
         self._api_key = api_key or os.environ.get("ANTHROPIC_API_KEY")
-        self.client = Anthropic(api_key=self._api_key, base_url=url)
+        self._url = url or os.environ.get("ANTHROPIC_API_BASE_URL")
+        self.client = Anthropic(api_key=self._api_key, base_url=self._url)
         self._token_counter: Optional[BaseTokenCounter] = None
 
     def _convert_response_from_anthropic_to_openai(self, response):
@@ -96,7 +98,7 @@ class AnthropicModel(BaseModelBackend):
         """
         return self.client.count_tokens(prompt)
 
-    @model_api_key_required
+    @api_keys_required("ANTHROPIC_API_KEY")
     def run(
         self,
         messages: List[OpenAIMessage],

camel/models/litellm_model.py

@@ -11,24 +11,25 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
+from typing import Any, Dict, List, Optional
 
 from camel.configs import LITELLM_API_PARAMS
 from camel.messages import OpenAIMessage
+from camel.types import ChatCompletion
 from camel.utils import LiteLLMTokenCounter
 
-if TYPE_CHECKING:
-    from litellm.utils import CustomStreamWrapper, ModelResponse
-
 
 class LiteLLMModel:
     r"""Constructor for LiteLLM backend with OpenAI compatibility."""
 
-    # NOTE: Currently "stream": True is not supported with LiteLLM due to the
-    # limitation of the current camel design.
+    # NOTE: Currently stream mode is not supported.
 
     def __init__(
-        self, model_type: str, model_config_dict: Dict[str, Any]
+        self,
+        model_type: str,
+        model_config_dict: Dict[str, Any],
+        api_key: Optional[str] = None,
+        url: Optional[str] = None,
     ) -> None:
         r"""Constructor for LiteLLM backend.
 
@@ -37,12 +38,48 @@ class LiteLLMModel:
                 such as GPT-3.5-turbo, Claude-2, etc.
             model_config_dict (Dict[str, Any]): A dictionary of parameters for
                 the model configuration.
+            api_key (Optional[str]): The API key for authenticating with the
+                model service. (default: :obj:`None`)
+            url (Optional[str]): The url to the model service. (default:
+                :obj:`None`)
         """
         self.model_type = model_type
         self.model_config_dict = model_config_dict
         self._client = None
         self._token_counter: Optional[LiteLLMTokenCounter] = None
         self.check_model_config()
+        self._url = url
+        self._api_key = api_key
+
+    def _convert_response_from_litellm_to_openai(
+        self, response
+    ) -> ChatCompletion:
+        r"""Converts a response from the LiteLLM format to the OpenAI format.
+
+        Parameters:
+            response (LiteLLMResponse): The response object from LiteLLM.
+
+        Returns:
+            ChatCompletion: The response object in OpenAI's format.
+        """
+        return ChatCompletion.construct(
+            id=response.id,
+            choices=[
+                {
+                    "index": response.choices[0].index,
+                    "message": {
+                        "role": response.choices[0].message.role,
+                        "content": response.choices[0].message.content,
+                    },
+                    "finish_reason": response.choices[0].finish_reason,
+                }
+            ],
+            created=response.created,
+            model=response.model,
+            object=response.object,
+            system_fingerprint=response.system_fingerprint,
+            usage=response.usage,
+        )
 
     @property
     def client(self):
@@ -67,7 +104,7 @@ class LiteLLMModel:
     def run(
         self,
         messages: List[OpenAIMessage],
-    ) -> Union['ModelResponse', 'CustomStreamWrapper']:
+    ) -> ChatCompletion:
         r"""Runs inference of LiteLLM chat completion.
 
         Args:
@@ -75,15 +112,16 @@ class LiteLLMModel:
                 in OpenAI format.
 
         Returns:
-            Union[ModelResponse, CustomStreamWrapper]:
-                `ModelResponse` in the non-stream mode, or
-                `CustomStreamWrapper` in the stream mode.
+            ChatCompletion
         """
         response = self.client(
+            api_key=self._api_key,
+            base_url=self._url,
             model=self.model_type,
             messages=messages,
             **self.model_config_dict,
         )
+        response = self._convert_response_from_litellm_to_openai(response)
         return response
 
     def check_model_config(self):
@@ -100,13 +138,3 @@ class LiteLLMModel:
                     f"Unexpected argument `{param}` is "
                     "input into LiteLLM model backend."
                 )
-
-    @property
-    def stream(self) -> bool:
-        r"""Returns whether the model is in stream mode, which sends partial
-        results each time.
-
-        Returns:
-            bool: Whether the model is in stream mode.
-        """
-        return self.model_config_dict.get('stream', False)

camel/models/model_factory.py

@@ -80,6 +80,7 @@ class ModelFactory:
         elif isinstance(model_type, str):
             if model_platform.is_ollama:
                 model_class = OllamaModel
+                return model_class(model_type, model_config_dict, url)
             elif model_platform.is_litellm:
                 model_class = LiteLLMModel
             else:

camel/models/nemotron_model.py

@@ -20,7 +20,7 @@ from camel.messages import OpenAIMessage
 from camel.types import ChatCompletion, ModelType
 from camel.utils import (
     BaseTokenCounter,
-    model_api_key_required,
+    api_keys_required,
 )
 
 
@@ -33,6 +33,7 @@ class NemotronModel:
         self,
         model_type: ModelType,
         api_key: Optional[str] = None,
+        url: Optional[str] = None,
     ) -> None:
         r"""Constructor for Nvidia backend.
 
@@ -40,18 +41,25 @@ class NemotronModel:
             model_type (ModelType): Model for which a backend is created.
             api_key (Optional[str]): The API key for authenticating with the
                 Nvidia service. (default: :obj:`None`)
+            url (Optional[str]): The url to the Nvidia service. (default:
+                :obj:`None`)
         """
         self.model_type = model_type
-        url = os.environ.get('NVIDIA_API_BASE_URL', None)
+        self._url = url or os.environ.get("NVIDIA_API_BASE_URL")
         self._api_key = api_key or os.environ.get("NVIDIA_API_KEY")
-        if not url or not self._api_key:
-            raise ValueError("The NVIDIA API base url and key should be set.")
+        if not self._url or not self._api_key:
+            raise ValueError(
+                "NVIDIA_API_BASE_URL and NVIDIA_API_KEY should be set."
+            )
         self._client = OpenAI(
-            timeout=60, max_retries=3, base_url=url, api_key=self._api_key
+            timeout=60,
+            max_retries=3,
+            base_url=self._url,
+            api_key=self._api_key,
         )
         self._token_counter: Optional[BaseTokenCounter] = None
 
-    @model_api_key_required
+    @api_keys_required("NVIDIA_API_KEY")
     def run(
         self,
         messages: List[OpenAIMessage],

camel/models/ollama_model.py

@@ -11,12 +11,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
-import os
 from typing import Any, Dict, List, Optional, Union
 
 from openai import OpenAI, Stream
 
-from camel.configs import OPENAI_API_PARAMS
+from camel.configs import OLLAMA_API_PARAMS
 from camel.messages import OpenAIMessage
 from camel.types import ChatCompletion, ChatCompletionChunk, ModelType
 from camel.utils import BaseTokenCounter, OpenAITokenCounter
@@ -25,39 +24,34 @@ from camel.utils import BaseTokenCounter, OpenAITokenCounter
 class OllamaModel:
     r"""Ollama service interface."""
 
-    # NOTE: Current `ModelType and `TokenCounter` desigen is not suitable,
-    # stream mode is not supported
-
     def __init__(
         self,
         model_type: str,
         model_config_dict: Dict[str, Any],
-        api_key: Optional[str] = None,
         url: Optional[str] = None,
     ) -> None:
         r"""Constructor for Ollama backend with OpenAI compatibility.
 
+        # Reference: https://github.com/ollama/ollama/blob/main/docs/openai.md
+
         Args:
             model_type (str): Model for which a backend is created.
             model_config_dict (Dict[str, Any]): A dictionary that will
                 be fed into openai.ChatCompletion.create().
-            api_key (Optional[str]): The API key for authenticating with the
-                model service. (default: :obj:`None`)
-            url (Optional[str]): The url to the model service.
+            url (Optional[str]): The url to the model service. (default:
+                :obj:`None`)
         """
         self.model_type = model_type
         self.model_config_dict = model_config_dict
-        self._url = url or os.environ.get('OPENAI_API_BASE_URL')
-        self._api_key = api_key or os.environ.get("OPENAI_API_KEY")
         # Use OpenAI cilent as interface call Ollama
-        # Reference: https://github.com/ollama/ollama/blob/main/docs/openai.md
         self._client = OpenAI(
             timeout=60,
             max_retries=3,
-            base_url=self._url,
-            api_key=self._api_key,
+            base_url=url,
+            api_key="ollama",  # required but ignored
         )
         self._token_counter: Optional[BaseTokenCounter] = None
+        self.check_model_config()
 
     @property
     def token_counter(self) -> BaseTokenCounter:
@@ -74,17 +68,17 @@ class OllamaModel:
 
     def check_model_config(self):
         r"""Check whether the model configuration contains any
-        unexpected arguments to OpenAI API.
+        unexpected arguments to Ollama API.
 
         Raises:
             ValueError: If the model configuration dictionary contains any
                 unexpected arguments to OpenAI API.
         """
         for param in self.model_config_dict:
-            if param not in OPENAI_API_PARAMS:
+            if param not in OLLAMA_API_PARAMS:
                 raise ValueError(
                     f"Unexpected argument `{param}` is "
-                    "input into OpenAI model backend."
+                    "input into Ollama model backend."
                 )
 
     def run(

camel/models/openai_audio_models.py

@@ -25,10 +25,18 @@ class OpenAIAudioModels:
 
     def __init__(
         self,
+        api_key: Optional[str] = None,
+        url: Optional[str] = None,
     ) -> None:
         r"""Initialize an instance of OpenAI."""
-        url = os.environ.get('OPENAI_API_BASE_URL')
-        self._client = OpenAI(timeout=120, max_retries=3, base_url=url)
+        self._url = url or os.environ.get("OPENAI_API_BASE_URL")
+        self._api_key = api_key or os.environ.get("OPENAI_API_KEY")
+        self._client = OpenAI(
+            timeout=120,
+            max_retries=3,
+            base_url=self._url,
+            api_key=self._api_key,
+        )
 
     def text_to_speech(
         self,

camel/models/openai_model.py

@@ -23,7 +23,7 @@ from camel.types import ChatCompletion, ChatCompletionChunk, ModelType
 from camel.utils import (
     BaseTokenCounter,
     OpenAITokenCounter,
-    model_api_key_required,
+    api_keys_required,
 )
 
 
@@ -46,7 +46,8 @@ class OpenAIModel(BaseModelBackend):
                 be fed into openai.ChatCompletion.create().
             api_key (Optional[str]): The API key for authenticating with the
                 OpenAI service. (default: :obj:`None`)
-            url (Optional[str]): The url to the OpenAI service.
+            url (Optional[str]): The url to the OpenAI service. (default:
+                :obj:`None`)
         """
         super().__init__(model_type, model_config_dict, api_key, url)
         self._url = url or os.environ.get("OPENAI_API_BASE_URL")
@@ -71,7 +72,7 @@ class OpenAIModel(BaseModelBackend):
             self._token_counter = OpenAITokenCounter(self.model_type)
         return self._token_counter
 
-    @model_api_key_required
+    @api_keys_required("OPENAI_API_KEY")
     def run(
         self,
         messages: List[OpenAIMessage],

camel/models/zhipuai_model.py

@@ -17,14 +17,14 @@ from typing import Any, Dict, List, Optional, Union
 
 from openai import OpenAI, Stream
 
-from camel.configs import OPENAI_API_PARAMS
+from camel.configs import ZHIPUAI_API_PARAMS
 from camel.messages import OpenAIMessage
 from camel.models import BaseModelBackend
 from camel.types import ChatCompletion, ChatCompletionChunk, ModelType
 from camel.utils import (
     BaseTokenCounter,
     OpenAITokenCounter,
-    model_api_key_required,
+    api_keys_required,
 )
 
 
@@ -47,10 +47,16 @@ class ZhipuAIModel(BaseModelBackend):
                 be fed into openai.ChatCompletion.create().
             api_key (Optional[str]): The API key for authenticating with the
                 ZhipuAI service. (default: :obj:`None`)
+            url (Optional[str]): The url to the ZhipuAI service. (default:
+                :obj:`None`)
         """
         super().__init__(model_type, model_config_dict)
         self._url = url or os.environ.get("ZHIPUAI_API_BASE_URL")
         self._api_key = api_key or os.environ.get("ZHIPUAI_API_KEY")
+        if not self._url or not self._api_key:
+            raise ValueError(
+                "ZHIPUAI_API_BASE_URL and ZHIPUAI_API_KEY should be set."
+            )
         self._client = OpenAI(
             timeout=60,
             max_retries=3,
@@ -59,7 +65,7 @@ class ZhipuAIModel(BaseModelBackend):
         )
         self._token_counter: Optional[BaseTokenCounter] = None
 
-    @model_api_key_required
+    @api_keys_required("ZHIPUAI_API_KEY")
     def run(
         self,
         messages: List[OpenAIMessage],
@@ -104,13 +110,13 @@ class ZhipuAIModel(BaseModelBackend):
 
         Raises:
             ValueError: If the model configuration dictionary contains any
-                unexpected arguments to OpenAI API.
+                unexpected arguments to ZhipuAI API.
         """
         for param in self.model_config_dict:
-            if param not in OPENAI_API_PARAMS:
+            if param not in ZHIPUAI_API_PARAMS:
                 raise ValueError(
                     f"Unexpected argument `{param}` is "
-                    "input into OpenAI model backend."
+                    "input into ZhipuAI model backend."
                 )
         pass
 

camel/retrievers/bm25_retriever.py

@@ -17,6 +17,7 @@ import numpy as np
 
 from camel.loaders import UnstructuredIO
 from camel.retrievers import BaseRetriever
+from camel.utils import dependencies_required
 
 DEFAULT_TOP_K_RESULTS = 1
 
@@ -40,16 +41,10 @@ class BM25Retriever(BaseRetriever):
         https://github.com/dorianbrown/rank_bm25
     """
 
+    @dependencies_required('rank_bm25')
     def __init__(self) -> None:
         r"""Initializes the BM25Retriever."""
-
-        try:
-            from rank_bm25 import BM25Okapi
-        except ImportError as e:
-            raise ImportError(
-                "Package `rank_bm25` not installed, install by running 'pip "
-                "install rank_bm25'"
-            ) from e
+        from rank_bm25 import BM25Okapi
 
         self.bm25: BM25Okapi = None
         self.content_input_path: str = ""

camel/retrievers/cohere_rerank_retriever.py

@@ -15,6 +15,7 @@ import os
 from typing import Any, Dict, List, Optional
 
 from camel.retrievers import BaseRetriever
+from camel.utils import dependencies_required
 
 DEFAULT_TOP_K_RESULTS = 1
 
@@ -32,6 +33,7 @@ class CohereRerankRetriever(BaseRetriever):
         https://txt.cohere.com/rerank/
     """
 
+    @dependencies_required('cohere')
     def __init__(
         self,
         model_name: str = "rerank-multilingual-v2.0",
@@ -56,11 +58,7 @@ class CohereRerankRetriever(BaseRetriever):
             ValueError: If the API key is neither passed as an argument nor
                 set in the environment variable.
         """
-
-        try:
-            import cohere
-        except ImportError as e:
-            raise ImportError("Package 'cohere' is not installed") from e
+        import cohere
 
         try:
             self.api_key = api_key or os.environ["COHERE_API_KEY"]

camel/storages/__init__.py

@@ -17,6 +17,7 @@ from .graph_storages.neo4j_graph import Neo4jGraph
 from .key_value_storages.base import BaseKeyValueStorage
 from .key_value_storages.in_memory import InMemoryKeyValueStorage
 from .key_value_storages.json import JsonStorage
+from .key_value_storages.redis import RedisStorage
 from .vectordb_storages.base import (
     BaseVectorStorage,
     VectorDBQuery,
@@ -30,6 +31,7 @@ __all__ = [
     'BaseKeyValueStorage',
     'InMemoryKeyValueStorage',
     'JsonStorage',
+    'RedisStorage',
     'VectorRecord',
     'BaseVectorStorage',
     'VectorDBQuery',