camel-ai 0.1.5.1__py3-none-any.whl → 0.1.5.3__py3-none-any.whl

camel/loaders/unstructured_io.py

@@ -131,30 +131,22 @@ class UnstructuredIO:
         self,
         input_path: str,
         **kwargs: Any,
-    ) -> Union[Any, List[Any]]:
-        r"""Loads a file or a URL and parses its contents as unstructured data.
+    ) -> List[Element]:
+        r"""Loads a file or a URL and parses its contents into elements.
 
         Args:
             input_path (str): Path to the file or URL to be parsed.
             **kwargs: Extra kwargs passed to the partition function.
 
         Returns:
-            List[Any]: The elements after parsing the file or URL, could be a
-                dict, list, etc., depending on the content. If return_str is
-                True, returns a tuple with a string representation of the
-                elements and the elements themselves.
+            List[Element]: List of elements after parsing the file or URL.
 
         Raises:
-            FileNotFoundError: If the file does not exist
-                at the path specified.
+            FileNotFoundError: If the file does not exist at the path
+                specified.
             Exception: For any other issues during file or URL parsing.
 
         Notes:
-            By default we use the basic "unstructured" library,
-            if you are processing document types beyond the basics,
-            you can install the necessary extras like:
-                `pip install "unstructured[docx,pptx]"` or
-                `pip install "unstructured[all-docs]"`.
             Available document types:
                 "csv", "doc", "docx", "epub", "image", "md", "msg", "odt",
                 "org", "pdf", "ppt", "pptx", "rtf", "rst", "tsv", "xlsx".
@@ -185,7 +177,9 @@ class UnstructuredIO:
 
             # Check if the file exists
             if not os.path.exists(input_path):
-                raise FileNotFoundError(f"The file {input_path} was not found.")
+                raise FileNotFoundError(
+                    f"The file {input_path} was not found."
+                )
 
             # Read the file
             try:
@@ -193,7 +187,9 @@ class UnstructuredIO:
                     elements = partition(file=f, **kwargs)
                     return elements
             except Exception as e:
-                raise Exception("Failed to parse the unstructured file.") from e
+                raise Exception(
+                    "Failed to parse the unstructured file."
+                ) from e
 
     def clean_text_data(
         self,
@@ -433,9 +429,8 @@ class UnstructuredIO:
                 els, kw.get('metadata', [])
             ),
             "stage_for_baseplate": baseplate.stage_for_baseplate,
-            "stage_for_datasaur": lambda els, **kw: datasaur.stage_for_datasaur(
-                els, kw.get('entities', [])
-            ),
+            "stage_for_datasaur": lambda els,
+            **kw: datasaur.stage_for_datasaur(els, kw.get('entities', [])),
             "stage_for_label_box": lambda els,
             **kw: label_box.stage_for_label_box(els, **kw),
             "stage_for_label_studio": lambda els,
@@ -450,11 +445,11 @@ class UnstructuredIO:
 
     def chunk_elements(
         self, elements: List[Any], chunk_type: str, **kwargs
-    ) -> List[Any]:
+    ) -> List[Element]:
         r"""Chunks elements by titles.
 
         Args:
-            elements (List[Any]): List of Element objects to be chunked.
+            elements (List[Element]): List of Element objects to be chunked.
             chunk_type (str): Type chunk going to apply. Supported types:
                 'chunk_by_title'.
             **kwargs: Additional keyword arguments for chunking.
@@ -531,8 +526,7 @@ class UnstructuredIO:
         account_name: str,
         num_processes: int = 2,
     ) -> None:
-        """
-        Processes documents from an Azure storage container and stores
+        r"""Processes documents from an Azure storage container and stores
         structured outputs locally.
 
         Args:

camel/messages/base.py

@@ -16,6 +16,7 @@ import io
 from dataclasses import dataclass
 from typing import Any, Dict, List, Literal, Optional, Tuple, Union
 
+import numpy as np
 from PIL import Image
 
 from camel.messages import (
@@ -27,10 +28,11 @@ from camel.messages import (
 from camel.prompts import CodePrompt, TextPrompt
 from camel.types import (
     OpenAIBackendRole,
-    OpenAIImageDetailType,
     OpenAIImageType,
+    OpenAIVisionDetailType,
     RoleType,
 )
+from camel.utils import Constants
 
 
 @dataclass
@@ -39,19 +41,29 @@ class BaseMessage:
 
     Args:
         role_name (str): The name of the user or assistant role.
-        role_type (RoleType): The type of role, either
-            :obj:`RoleType.ASSISTANT` or :obj:`RoleType.USER`.
+        role_type (RoleType): The type of role, either :obj:`RoleType.
+            ASSISTANT` or :obj:`RoleType.USER`.
         meta_dict (Optional[Dict[str, str]]): Additional metadata dictionary
             for the message.
         content (str): The content of the message.
+        video_bytes (Optional[bytes]): Optional bytes of a video associated
+            with the message. Default is None.
+        image_list (Optional[List[Image.Image]]): Optional list of PIL Image
+            objects associated with the message. Default is None.
+        image_detail (Literal["auto", "low", "high"]): Detail level of the
+            images associated with the message. Default is "auto".
+        video_detail (Literal["auto", "low", "high"]): Detail level of the
+            videos associated with the message. Default is "low".
     """
 
     role_name: str
     role_type: RoleType
     meta_dict: Optional[Dict[str, str]]
     content: str
-    image: Optional[Image.Image] = None
+    video_bytes: Optional[bytes] = None
+    image_list: Optional[List[Image.Image]] = None
     image_detail: Literal["auto", "low", "high"] = "auto"
+    video_detail: Literal["auto", "low", "high"] = "low"
 
     @classmethod
     def make_user_message(
@@ -59,16 +71,24 @@ class BaseMessage:
         role_name: str,
         content: str,
         meta_dict: Optional[Dict[str, str]] = None,
-        image: Optional[Image.Image] = None,
-        image_detail: Union[OpenAIImageDetailType, str] = "auto",
-    ) -> 'BaseMessage':
+        video_bytes: Optional[bytes] = None,
+        image_list: Optional[List[Image.Image]] = None,
+        image_detail: Union[
+            OpenAIVisionDetailType, str
+        ] = OpenAIVisionDetailType.AUTO,
+        video_detail: Union[
+            OpenAIVisionDetailType, str
+        ] = OpenAIVisionDetailType.LOW,
+    ) -> "BaseMessage":
         return cls(
             role_name,
             RoleType.USER,
             meta_dict,
             content,
-            image,
-            OpenAIImageDetailType(image_detail).value,
+            video_bytes,
+            image_list,
+            OpenAIVisionDetailType(image_detail).value,
+            OpenAIVisionDetailType(video_detail).value,
         )
 
     @classmethod
@@ -77,16 +97,24 @@ class BaseMessage:
         role_name: str,
         content: str,
         meta_dict: Optional[Dict[str, str]] = None,
-        image: Optional[Image.Image] = None,
-        image_detail: Union[OpenAIImageDetailType, str] = "auto",
-    ) -> 'BaseMessage':
+        video_bytes: Optional[bytes] = None,
+        image_list: Optional[List[Image.Image]] = None,
+        image_detail: Union[
+            OpenAIVisionDetailType, str
+        ] = OpenAIVisionDetailType.AUTO,
+        video_detail: Union[
+            OpenAIVisionDetailType, str
+        ] = OpenAIVisionDetailType.LOW,
+    ) -> "BaseMessage":
         return cls(
             role_name,
             RoleType.ASSISTANT,
             meta_dict,
             content,
-            image,
-            OpenAIImageDetailType(image_detail).value,
+            video_bytes,
+            image_list,
+            OpenAIVisionDetailType(image_detail).value,
+            OpenAIVisionDetailType(video_detail).value,
         )
 
     def create_new_instance(self, content: str) -> "BaseMessage":
@@ -241,46 +269,107 @@ class BaseMessage:
         Returns:
             OpenAIUserMessage: The converted :obj:`OpenAIUserMessage` object.
         """
-        if self.image is None:
-            return {"role": "user", "content": self.content}
-        else:
-            #
-            if self.image.format is None:
-                raise ValueError(
-                    f"Image's `format` is `None`, please "
-                    f"transform the `PIL.Image.Image` to  one of "
-                    f"following supported formats, such as "
-                    f"{list(OpenAIImageType)}"
-                )
-
-            image_type: str = self.image.format.lower()
-            if image_type not in OpenAIImageType:
-                raise ValueError(
-                    f"Image type {self.image.format} "
-                    f"is not supported by OpenAI vision model"
-                )
-            with io.BytesIO() as buffer:
-                self.image.save(fp=buffer, format=self.image.format)
-                encoded_image = base64.b64encode(buffer.getvalue()).decode(
-                    "utf-8"
-                )
-            image_prefix = f"data:image/{image_type};base64,"
+        hybird_content: List[Any] = []
+        hybird_content.append(
+            {
+                "type": "text",
+                "text": self.content,
+            }
+        )
 
-            return {
-                "role": "user",
-                "content": [
-                    {
-                        "type": "text",
-                        "text": self.content,
-                    },
+        if self.image_list and len(self.image_list) > 0:
+            for image in self.image_list:
+                if image.format is None:
+                    raise ValueError(
+                        f"Image's `format` is `None`, please "
+                        f"transform the `PIL.Image.Image` to  one of "
+                        f"following supported formats, such as "
+                        f"{list(OpenAIImageType)}"
+                    )
+
+                image_type: str = image.format.lower()
+                if image_type not in OpenAIImageType:
+                    raise ValueError(
+                        f"Image type {image.format} "
+                        f"is not supported by OpenAI vision model"
+                    )
+                with io.BytesIO() as buffer:
+                    image.save(fp=buffer, format=image.format)
+                    encoded_image = base64.b64encode(buffer.getvalue()).decode(
+                        "utf-8"
+                    )
+                image_prefix = f"data:image/{image_type};base64,"
+                hybird_content.append(
                     {
                         "type": "image_url",
                         "image_url": {
                             "url": f"{image_prefix}{encoded_image}",
                             "detail": self.image_detail,
                         },
+                    }
+                )
+
+        if self.video_bytes:
+            import imageio.v3 as iio
+
+            base64Frames: List[str] = []
+            frame_count = 0
+            # read video bytes
+            video = iio.imiter(
+                self.video_bytes, plugin=Constants.VIDEO_DEFAULT_PLUG_PYAV
+            )
+
+            for frame in video:
+                frame_count += 1
+                if (
+                    frame_count % Constants.VIDEO_IMAGE_EXTRACTION_INTERVAL
+                    == 0
+                ):
+                    # convert frame to numpy array
+                    frame_array = np.asarray(frame)
+                    frame_image = Image.fromarray(frame_array)
+
+                    # Get the dimensions of the frame
+                    width, height = frame_image.size
+
+                    # resize the frame to the default image size
+                    new_width = Constants.VIDEO_DEFAULT_IMAGE_SIZE
+                    aspect_ratio = width / height
+                    new_height = int(new_width / aspect_ratio)
+                    resized_img = frame_image.resize((new_width, new_height))
+
+                    # encode the image to base64
+                    with io.BytesIO() as buffer:
+                        image_format = OpenAIImageType.JPEG.value
+                        image_format = image_format.upper()
+                        resized_img.save(fp=buffer, format=image_format)
+                        encoded_image = base64.b64encode(
+                            buffer.getvalue()
+                        ).decode("utf-8")
+
+                    base64Frames.append(encoded_image)
+
+            for encoded_image in base64Frames:
+                item = {
+                    "type": "image_url",
+                    "image_url": {
+                        "url": f"data:image/jpeg;base64,{encoded_image}",
+                        "detail": self.video_detail,
                     },
-                ],
+                }
+
+                hybird_content.append(item)
+
+        if len(hybird_content) > 1:
+            return {
+                "role": "user",
+                "content": hybird_content,
+            }
+        # This return just for str message
+        else:
+            return {
+                "role": "user",
+                "content": self.content,
             }
 
     def to_openai_assistant_message(self) -> OpenAIAssistantMessage:

camel/models/__init__.py

@@ -13,18 +13,26 @@
 # =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
 from .anthropic_model import AnthropicModel
 from .base_model import BaseModelBackend
+from .litellm_model import LiteLLMModel
 from .model_factory import ModelFactory
+from .nemotron_model import NemotronModel
+from .ollama_model import OllamaModel
 from .open_source_model import OpenSourceModel
 from .openai_audio_models import OpenAIAudioModels
 from .openai_model import OpenAIModel
 from .stub_model import StubModel
+from .zhipuai_model import ZhipuAIModel
 
 __all__ = [
     'BaseModelBackend',
     'OpenAIModel',
     'AnthropicModel',
     'StubModel',
+    'ZhipuAIModel',
     'OpenSourceModel',
     'ModelFactory',
+    'LiteLLMModel',
     'OpenAIAudioModels',
+    'NemotronModel',
+    'OllamaModel',
 ]

camel/models/anthropic_model.py

@@ -12,15 +12,19 @@
 # limitations under the License.
 # =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
 import os
-from typing import Any, Dict, Optional
+from typing import Any, Dict, List, Optional
 
-from anthropic import Anthropic
-from anthropic._types import NOT_GIVEN
+from anthropic import NOT_GIVEN, Anthropic
 
 from camel.configs import ANTHROPIC_API_PARAMS
+from camel.messages import OpenAIMessage
 from camel.models.base_model import BaseModelBackend
 from camel.types import ChatCompletion, ModelType
-from camel.utils import AnthropicTokenCounter, BaseTokenCounter
+from camel.utils import (
+    AnthropicTokenCounter,
+    BaseTokenCounter,
+    model_api_key_required,
+)
 
 
 class AnthropicModel(BaseModelBackend):
@@ -31,20 +35,22 @@ class AnthropicModel(BaseModelBackend):
         model_type: ModelType,
         model_config_dict: Dict[str, Any],
         api_key: Optional[str] = None,
+        url: Optional[str] = None,
     ) -> None:
         r"""Constructor for Anthropic backend.
 
         Args:
             model_type (ModelType): Model for which a backend is created,
-                one of GPT_* series.
+                one of CLAUDE_* series.
             model_config_dict (Dict[str, Any]): A dictionary that will
-                be fed into openai.ChatCompletion.create().
+                be fed into Anthropic.messages.create().
             api_key (Optional[str]): The API key for authenticating with the
                 Anthropic service. (default: :obj:`None`)
+            url (Optional[str]): The url to the model service.
         """
-        super().__init__(model_type, model_config_dict)
+        super().__init__(model_type, model_config_dict, api_key, url)
         self._api_key = api_key or os.environ.get("ANTHROPIC_API_KEY")
-        self.client = Anthropic(api_key=self._api_key)
+        self.client = Anthropic(api_key=self._api_key, base_url=url)
         self._token_counter: Optional[BaseTokenCounter] = None
 
     def _convert_response_from_anthropic_to_openai(self, response):
@@ -90,28 +96,29 @@ class AnthropicModel(BaseModelBackend):
         """
         return self.client.count_tokens(prompt)
 
+    @model_api_key_required
     def run(
         self,
-        messages,
+        messages: List[OpenAIMessage],
     ):
         r"""Run inference of Anthropic chat completion.
 
         Args:
-            messages (List[Dict]): Message list with the chat history
+            messages (List[OpenAIMessage]): Message list with the chat history
                 in OpenAI API format.
 
         Returns:
-            Dict[str, Any]: Response in the OpenAI API format.
+            ChatCompletion: Response in the OpenAI API format.
         """
 
         if messages[0]["role"] == "system":
-            sys_msg = messages.pop(0)["content"]
+            sys_msg = str(messages.pop(0)["content"])
         else:
-            sys_msg = NOT_GIVEN
+            sys_msg = NOT_GIVEN  # type: ignore[assignment]
         response = self.client.messages.create(
             model=self.model_type.value,
             system=sys_msg,
-            messages=messages,
+            messages=messages,  # type: ignore[arg-type]
             **self.model_config_dict,
         )
 
@@ -138,8 +145,9 @@ class AnthropicModel(BaseModelBackend):
 
     @property
     def stream(self) -> bool:
-        r"""Returns whether the model is in stream mode,
-            which sends partial results each time.
+        r"""Returns whether the model is in stream mode, which sends partial
+        results each time.
+
         Returns:
             bool: Whether the model is in stream mode.
         """

camel/models/base_model.py

@@ -31,6 +31,7 @@ class BaseModelBackend(ABC):
         model_type: ModelType,
         model_config_dict: Dict[str, Any],
         api_key: Optional[str] = None,
+        url: Optional[str] = None,
     ) -> None:
         r"""Constructor for the model backend.
 
@@ -38,12 +39,14 @@ class BaseModelBackend(ABC):
             model_type (ModelType): Model for which a backend is created.
             model_config_dict (Dict[str, Any]): A config dictionary.
             api_key (Optional[str]): The API key for authenticating with the
-                LLM service.
+                model service.
+            url (Optional[str]): The url to the model service.
         """
         self.model_type = model_type
 
         self.model_config_dict = model_config_dict
         self._api_key = api_key
+        self._url = url
         self.check_model_config()
 
     @property
@@ -102,6 +105,7 @@ class BaseModelBackend(ABC):
     @property
     def token_limit(self) -> int:
         r"""Returns the maximum token limit for a given model.
+
         Returns:
             int: The maximum token limit for the given model.
         """
@@ -111,6 +115,7 @@ class BaseModelBackend(ABC):
     def stream(self) -> bool:
         r"""Returns whether the model is in stream mode,
             which sends partial results each time.
+
         Returns:
             bool: Whether the model is in stream mode.
         """

camel/models/litellm_model.py

@@ -0,0 +1,112 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
+
+from camel.configs import LITELLM_API_PARAMS
+from camel.messages import OpenAIMessage
+from camel.utils import LiteLLMTokenCounter
+
+if TYPE_CHECKING:
+    from litellm.utils import CustomStreamWrapper, ModelResponse
+
+
+class LiteLLMModel:
+    r"""Constructor for LiteLLM backend with OpenAI compatibility."""
+
+    # NOTE: Currently "stream": True is not supported with LiteLLM due to the
+    # limitation of the current camel design.
+
+    def __init__(
+        self, model_type: str, model_config_dict: Dict[str, Any]
+    ) -> None:
+        r"""Constructor for LiteLLM backend.
+
+        Args:
+            model_type (str): Model for which a backend is created,
+                such as GPT-3.5-turbo, Claude-2, etc.
+            model_config_dict (Dict[str, Any]): A dictionary of parameters for
+                the model configuration.
+        """
+        self.model_type = model_type
+        self.model_config_dict = model_config_dict
+        self._client = None
+        self._token_counter: Optional[LiteLLMTokenCounter] = None
+        self.check_model_config()
+
+    @property
+    def client(self):
+        if self._client is None:
+            from litellm import completion
+
+            self._client = completion
+        return self._client
+
+    @property
+    def token_counter(self) -> LiteLLMTokenCounter:
+        r"""Initialize the token counter for the model backend.
+
+        Returns:
+            LiteLLMTokenCounter: The token counter following the model's
+                tokenization style.
+        """
+        if not self._token_counter:
+            self._token_counter = LiteLLMTokenCounter(self.model_type)
+        return self._token_counter
+
+    def run(
+        self,
+        messages: List[OpenAIMessage],
+    ) -> Union['ModelResponse', 'CustomStreamWrapper']:
+        r"""Runs inference of LiteLLM chat completion.
+
+        Args:
+            messages (List[OpenAIMessage]): Message list with the chat history
+                in OpenAI format.
+
+        Returns:
+            Union[ModelResponse, CustomStreamWrapper]:
+                `ModelResponse` in the non-stream mode, or
+                `CustomStreamWrapper` in the stream mode.
+        """
+        response = self.client(
+            model=self.model_type,
+            messages=messages,
+            **self.model_config_dict,
+        )
+        return response
+
+    def check_model_config(self):
+        r"""Check whether the model configuration contains any unexpected
+        arguments to LiteLLM API.
+
+        Raises:
+            ValueError: If the model configuration dictionary contains any
+                unexpected arguments.
+        """
+        for param in self.model_config_dict:
+            if param not in LITELLM_API_PARAMS:
+                raise ValueError(
+                    f"Unexpected argument `{param}` is "
+                    "input into LiteLLM model backend."
+                )
+
+    @property
+    def stream(self) -> bool:
+        r"""Returns whether the model is in stream mode, which sends partial
+        results each time.
+
+        Returns:
+            bool: Whether the model is in stream mode.
+        """
+        return self.model_config_dict.get('stream', False)