camel-ai 0.1.5__py3-none-any.whl → 0.1.5.2__py3-none-any.whl

camel/messages/base.py

@@ -16,6 +16,7 @@ import io
 from dataclasses import dataclass
 from typing import Any, Dict, List, Literal, Optional, Tuple, Union
 
+import numpy as np
 from PIL import Image
 
 from camel.messages import (
@@ -27,10 +28,11 @@ from camel.messages import (
 from camel.prompts import CodePrompt, TextPrompt
 from camel.types import (
     OpenAIBackendRole,
-    OpenAIImageDetailType,
     OpenAIImageType,
+    OpenAIVisionDetailType,
     RoleType,
 )
+from camel.utils import Constants
 
 
 @dataclass
@@ -39,19 +41,29 @@ class BaseMessage:
 
     Args:
         role_name (str): The name of the user or assistant role.
-        role_type (RoleType): The type of role, either
-            :obj:`RoleType.ASSISTANT` or :obj:`RoleType.USER`.
+        role_type (RoleType): The type of role, either :obj:`RoleType.
+            ASSISTANT` or :obj:`RoleType.USER`.
         meta_dict (Optional[Dict[str, str]]): Additional metadata dictionary
             for the message.
         content (str): The content of the message.
+        video_bytes (Optional[bytes]): Optional bytes of a video associated
+            with the message. Default is None.
+        image_list (Optional[List[Image.Image]]): Optional list of PIL Image
+            objects associated with the message. Default is None.
+        image_detail (Literal["auto", "low", "high"]): Detail level of the
+            images associated with the message. Default is "auto".
+        video_detail (Literal["auto", "low", "high"]): Detail level of the
+            videos associated with the message. Default is "low".
     """
 
     role_name: str
     role_type: RoleType
     meta_dict: Optional[Dict[str, str]]
     content: str
-    image: Optional[Image.Image] = None
+    video_bytes: Optional[bytes] = None
+    image_list: Optional[List[Image.Image]] = None
     image_detail: Literal["auto", "low", "high"] = "auto"
+    video_detail: Literal["auto", "low", "high"] = "low"
 
     @classmethod
     def make_user_message(
@@ -59,16 +71,24 @@ class BaseMessage:
         role_name: str,
         content: str,
         meta_dict: Optional[Dict[str, str]] = None,
-        image: Optional[Image.Image] = None,
-        image_detail: Union[OpenAIImageDetailType, str] = "auto",
-    ) -> 'BaseMessage':
+        video_bytes: Optional[bytes] = None,
+        image_list: Optional[List[Image.Image]] = None,
+        image_detail: Union[
+            OpenAIVisionDetailType, str
+        ] = OpenAIVisionDetailType.AUTO,
+        video_detail: Union[
+            OpenAIVisionDetailType, str
+        ] = OpenAIVisionDetailType.LOW,
+    ) -> "BaseMessage":
         return cls(
             role_name,
             RoleType.USER,
             meta_dict,
             content,
-            image,
-            OpenAIImageDetailType(image_detail).value,
+            video_bytes,
+            image_list,
+            OpenAIVisionDetailType(image_detail).value,
+            OpenAIVisionDetailType(video_detail).value,
         )
 
     @classmethod
@@ -77,16 +97,24 @@ class BaseMessage:
         role_name: str,
         content: str,
         meta_dict: Optional[Dict[str, str]] = None,
-        image: Optional[Image.Image] = None,
-        image_detail: Union[OpenAIImageDetailType, str] = "auto",
-    ) -> 'BaseMessage':
+        video_bytes: Optional[bytes] = None,
+        image_list: Optional[List[Image.Image]] = None,
+        image_detail: Union[
+            OpenAIVisionDetailType, str
+        ] = OpenAIVisionDetailType.AUTO,
+        video_detail: Union[
+            OpenAIVisionDetailType, str
+        ] = OpenAIVisionDetailType.LOW,
+    ) -> "BaseMessage":
         return cls(
             role_name,
             RoleType.ASSISTANT,
             meta_dict,
             content,
-            image,
-            OpenAIImageDetailType(image_detail).value,
+            video_bytes,
+            image_list,
+            OpenAIVisionDetailType(image_detail).value,
+            OpenAIVisionDetailType(video_detail).value,
         )
 
     def create_new_instance(self, content: str) -> "BaseMessage":
@@ -241,46 +269,107 @@ class BaseMessage:
         Returns:
             OpenAIUserMessage: The converted :obj:`OpenAIUserMessage` object.
         """
-        if self.image is None:
-            return {"role": "user", "content": self.content}
-        else:
-            #
-            if self.image.format is None:
-                raise ValueError(
-                    f"Image's `format` is `None`, please "
-                    f"transform the `PIL.Image.Image` to  one of "
-                    f"following supported formats, such as "
-                    f"{list(OpenAIImageType)}"
-                )
-
-            image_type: str = self.image.format.lower()
-            if image_type not in OpenAIImageType:
-                raise ValueError(
-                    f"Image type {self.image.format} "
-                    f"is not supported by OpenAI vision model"
-                )
-            with io.BytesIO() as buffer:
-                self.image.save(fp=buffer, format=self.image.format)
-                encoded_image = base64.b64encode(buffer.getvalue()).decode(
-                    "utf-8"
-                )
-            image_prefix = f"data:image/{image_type};base64,"
+        hybird_content: List[Any] = []
+        hybird_content.append(
+            {
+                "type": "text",
+                "text": self.content,
+            }
+        )
 
-            return {
-                "role": "user",
-                "content": [
-                    {
-                        "type": "text",
-                        "text": self.content,
-                    },
+        if self.image_list and len(self.image_list) > 0:
+            for image in self.image_list:
+                if image.format is None:
+                    raise ValueError(
+                        f"Image's `format` is `None`, please "
+                        f"transform the `PIL.Image.Image` to  one of "
+                        f"following supported formats, such as "
+                        f"{list(OpenAIImageType)}"
+                    )
+
+                image_type: str = image.format.lower()
+                if image_type not in OpenAIImageType:
+                    raise ValueError(
+                        f"Image type {image.format} "
+                        f"is not supported by OpenAI vision model"
+                    )
+                with io.BytesIO() as buffer:
+                    image.save(fp=buffer, format=image.format)
+                    encoded_image = base64.b64encode(buffer.getvalue()).decode(
+                        "utf-8"
+                    )
+                image_prefix = f"data:image/{image_type};base64,"
+                hybird_content.append(
                     {
                         "type": "image_url",
                         "image_url": {
                             "url": f"{image_prefix}{encoded_image}",
                             "detail": self.image_detail,
                         },
+                    }
+                )
+
+        if self.video_bytes:
+            import imageio.v3 as iio
+
+            base64Frames: List[str] = []
+            frame_count = 0
+            # read video bytes
+            video = iio.imiter(
+                self.video_bytes, plugin=Constants.VIDEO_DEFAULT_PLUG_PYAV
+            )
+
+            for frame in video:
+                frame_count += 1
+                if (
+                    frame_count % Constants.VIDEO_IMAGE_EXTRACTION_INTERVAL
+                    == 0
+                ):
+                    # convert frame to numpy array
+                    frame_array = np.asarray(frame)
+                    frame_image = Image.fromarray(frame_array)
+
+                    # Get the dimensions of the frame
+                    width, height = frame_image.size
+
+                    # resize the frame to the default image size
+                    new_width = Constants.VIDEO_DEFAULT_IMAGE_SIZE
+                    aspect_ratio = width / height
+                    new_height = int(new_width / aspect_ratio)
+                    resized_img = frame_image.resize((new_width, new_height))
+
+                    # encode the image to base64
+                    with io.BytesIO() as buffer:
+                        image_format = OpenAIImageType.JPEG.value
+                        image_format = image_format.upper()
+                        resized_img.save(fp=buffer, format=image_format)
+                        encoded_image = base64.b64encode(
+                            buffer.getvalue()
+                        ).decode("utf-8")
+
+                    base64Frames.append(encoded_image)
+
+            for encoded_image in base64Frames:
+                item = {
+                    "type": "image_url",
+                    "image_url": {
+                        "url": f"data:image/jpeg;base64,{encoded_image}",
+                        "detail": self.video_detail,
                     },
-                ],
+                }
+
+                hybird_content.append(item)
+
+        if len(hybird_content) > 1:
+            return {
+                "role": "user",
+                "content": hybird_content,
+            }
+        # This return just for str message
+        else:
+            return {
+                "role": "user",
+                "content": self.content,
             }
 
     def to_openai_assistant_message(self) -> OpenAIAssistantMessage:

camel/models/__init__.py

@@ -13,18 +13,22 @@
 # =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
 from .anthropic_model import AnthropicModel
 from .base_model import BaseModelBackend
+from .litellm_model import LiteLLMModel
 from .model_factory import ModelFactory
 from .open_source_model import OpenSourceModel
 from .openai_audio_models import OpenAIAudioModels
 from .openai_model import OpenAIModel
 from .stub_model import StubModel
+from .zhipuai_model import ZhipuAIModel
 
 __all__ = [
     'BaseModelBackend',
     'OpenAIModel',
     'AnthropicModel',
     'StubModel',
+    'ZhipuAIModel',
     'OpenSourceModel',
     'ModelFactory',
+    'LiteLLMModel',
     'OpenAIAudioModels',
 ]

camel/models/anthropic_model.py

@@ -12,15 +12,19 @@
 # limitations under the License.
 # =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
 import os
-from typing import Any, Dict, Optional
+from typing import Any, Dict, List, Optional
 
-from anthropic import Anthropic
-from anthropic._types import NOT_GIVEN
+from anthropic import NOT_GIVEN, Anthropic
 
 from camel.configs import ANTHROPIC_API_PARAMS
+from camel.messages import OpenAIMessage
 from camel.models.base_model import BaseModelBackend
 from camel.types import ChatCompletion, ModelType
-from camel.utils import AnthropicTokenCounter, BaseTokenCounter
+from camel.utils import (
+    AnthropicTokenCounter,
+    BaseTokenCounter,
+    model_api_key_required,
+)
 
 
 class AnthropicModel(BaseModelBackend):
@@ -36,9 +40,9 @@ class AnthropicModel(BaseModelBackend):
 
         Args:
             model_type (ModelType): Model for which a backend is created,
-                one of GPT_* series.
+                one of CLAUDE_* series.
             model_config_dict (Dict[str, Any]): A dictionary that will
-                be fed into openai.ChatCompletion.create().
+                be fed into Anthropic.messages.create().
             api_key (Optional[str]): The API key for authenticating with the
                 Anthropic service. (default: :obj:`None`)
         """
@@ -90,28 +94,29 @@ class AnthropicModel(BaseModelBackend):
         """
         return self.client.count_tokens(prompt)
 
+    @model_api_key_required
     def run(
         self,
-        messages,
+        messages: List[OpenAIMessage],
     ):
         r"""Run inference of Anthropic chat completion.
 
         Args:
-            messages (List[Dict]): Message list with the chat history
+            messages (List[OpenAIMessage]): Message list with the chat history
                 in OpenAI API format.
 
         Returns:
-            Dict[str, Any]: Response in the OpenAI API format.
+            ChatCompletion: Response in the OpenAI API format.
         """
 
         if messages[0]["role"] == "system":
-            sys_msg = messages.pop(0)["content"]
+            sys_msg = str(messages.pop(0)["content"])
         else:
-            sys_msg = NOT_GIVEN
+            sys_msg = NOT_GIVEN  # type: ignore[assignment]
         response = self.client.messages.create(
             model=self.model_type.value,
             system=sys_msg,
-            messages=messages,
+            messages=messages,  # type: ignore[arg-type]
             **self.model_config_dict,
         )
 
@@ -138,8 +143,9 @@ class AnthropicModel(BaseModelBackend):
 
     @property
     def stream(self) -> bool:
-        r"""Returns whether the model is in stream mode,
-            which sends partial results each time.
+        r"""Returns whether the model is in stream mode, which sends partial
+        results each time.
+
         Returns:
             bool: Whether the model is in stream mode.
         """

camel/models/base_model.py

@@ -102,6 +102,7 @@ class BaseModelBackend(ABC):
     @property
     def token_limit(self) -> int:
         r"""Returns the maximum token limit for a given model.
+
         Returns:
             int: The maximum token limit for the given model.
         """
@@ -111,6 +112,7 @@ class BaseModelBackend(ABC):
     def stream(self) -> bool:
         r"""Returns whether the model is in stream mode,
             which sends partial results each time.
+
         Returns:
             bool: Whether the model is in stream mode.
         """

camel/models/litellm_model.py

@@ -0,0 +1,112 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
+
+from camel.configs import LITELLM_API_PARAMS
+from camel.messages import OpenAIMessage
+from camel.utils import LiteLLMTokenCounter
+
+if TYPE_CHECKING:
+    from litellm.utils import CustomStreamWrapper, ModelResponse
+
+
+class LiteLLMModel:
+    r"""Constructor for LiteLLM backend with OpenAI compatibility."""
+
+    # NOTE: Currently "stream": True is not supported with LiteLLM due to the
+    # limitation of the current camel design.
+
+    def __init__(
+        self, model_type: str, model_config_dict: Dict[str, Any]
+    ) -> None:
+        r"""Constructor for LiteLLM backend.
+
+        Args:
+            model_type (str): Model for which a backend is created,
+                such as GPT-3.5-turbo, Claude-2, etc.
+            model_config_dict (Dict[str, Any]): A dictionary of parameters for
+                the model configuration.
+        """
+        self.model_type = model_type
+        self.model_config_dict = model_config_dict
+        self._client = None
+        self._token_counter: Optional[LiteLLMTokenCounter] = None
+        self.check_model_config()
+
+    @property
+    def client(self):
+        if self._client is None:
+            from litellm import completion
+
+            self._client = completion
+        return self._client
+
+    @property
+    def token_counter(self) -> LiteLLMTokenCounter:
+        r"""Initialize the token counter for the model backend.
+
+        Returns:
+            LiteLLMTokenCounter: The token counter following the model's
+                tokenization style.
+        """
+        if not self._token_counter:
+            self._token_counter = LiteLLMTokenCounter(self.model_type)
+        return self._token_counter
+
+    def run(
+        self,
+        messages: List[OpenAIMessage],
+    ) -> Union['ModelResponse', 'CustomStreamWrapper']:
+        r"""Runs inference of LiteLLM chat completion.
+
+        Args:
+            messages (List[OpenAIMessage]): Message list with the chat history
+                in OpenAI format.
+
+        Returns:
+            Union[ModelResponse, CustomStreamWrapper]:
+                `ModelResponse` in the non-stream mode, or
+                `CustomStreamWrapper` in the stream mode.
+        """
+        response = self.client(
+            model=self.model_type,
+            messages=messages,
+            **self.model_config_dict,
+        )
+        return response
+
+    def check_model_config(self):
+        r"""Check whether the model configuration contains any unexpected
+        arguments to LiteLLM API.
+
+        Raises:
+            ValueError: If the model configuration dictionary contains any
+                unexpected arguments.
+        """
+        for param in self.model_config_dict:
+            if param not in LITELLM_API_PARAMS:
+                raise ValueError(
+                    f"Unexpected argument `{param}` is "
+                    "input into LiteLLM model backend."
+                )
+
+    @property
+    def stream(self) -> bool:
+        r"""Returns whether the model is in stream mode, which sends partial
+        results each time.
+
+        Returns:
+            bool: Whether the model is in stream mode.
+        """
+        return self.model_config_dict.get('stream', False)

camel/models/model_factory.py

@@ -18,6 +18,7 @@ from camel.models.base_model import BaseModelBackend
 from camel.models.open_source_model import OpenSourceModel
 from camel.models.openai_model import OpenAIModel
 from camel.models.stub_model import StubModel
+from camel.models.zhipuai_model import ZhipuAIModel
 from camel.types import ModelType
 
 
@@ -58,8 +59,14 @@ class ModelFactory:
             model_class = OpenSourceModel
         elif model_type.is_anthropic:
             model_class = AnthropicModel
+        elif model_type.is_zhipuai:
+            model_class = ZhipuAIModel
         else:
             raise ValueError(f"Unknown model type `{model_type}` is input")
 
-        inst = model_class(model_type, model_config_dict, api_key)
+        if model_type.is_open_source:
+            inst = model_class(model_type, model_config_dict)
+        else:
+            inst = model_class(model_type, model_config_dict, api_key)
+
         return inst

camel/models/open_source_model.py

@@ -150,6 +150,7 @@ class OpenSourceModel(BaseModelBackend):
     def stream(self) -> bool:
         r"""Returns whether the model is in stream mode,
             which sends partial results each time.
+
         Returns:
             bool: Whether the model is in stream mode.
         """

camel/models/openai_model.py

@@ -20,7 +20,11 @@ from camel.configs import OPENAI_API_PARAMS
 from camel.messages import OpenAIMessage
 from camel.models import BaseModelBackend
 from camel.types import ChatCompletion, ChatCompletionChunk, ModelType
-from camel.utils import BaseTokenCounter, OpenAITokenCounter, api_key_required
+from camel.utils import (
+    BaseTokenCounter,
+    OpenAITokenCounter,
+    model_api_key_required,
+)
 
 
 class OpenAIModel(BaseModelBackend):
@@ -62,7 +66,7 @@ class OpenAIModel(BaseModelBackend):
             self._token_counter = OpenAITokenCounter(self.model_type)
         return self._token_counter
 
-    @api_key_required
+    @model_api_key_required
     def run(
         self,
         messages: List[OpenAIMessage],

camel/models/zhipuai_model.py

@@ -0,0 +1,125 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+
+import os
+from typing import Any, Dict, List, Optional, Union
+
+from openai import OpenAI, Stream
+
+from camel.configs import OPENAI_API_PARAMS
+from camel.messages import OpenAIMessage
+from camel.models import BaseModelBackend
+from camel.types import ChatCompletion, ChatCompletionChunk, ModelType
+from camel.utils import (
+    BaseTokenCounter,
+    OpenAITokenCounter,
+    model_api_key_required,
+)
+
+
+class ZhipuAIModel(BaseModelBackend):
+    r"""ZhipuAI API in a unified BaseModelBackend interface."""
+
+    def __init__(
+        self,
+        model_type: ModelType,
+        model_config_dict: Dict[str, Any],
+        api_key: Optional[str] = None,
+        url: Optional[str] = None,
+    ) -> None:
+        r"""Constructor for ZhipuAI backend.
+
+        Args:
+            model_type (ModelType): Model for which a backend is created,
+                such as GLM_* series.
+            model_config_dict (Dict[str, Any]): A dictionary that will
+                be fed into openai.ChatCompletion.create().
+            api_key (Optional[str]): The API key for authenticating with the
+                ZhipuAI service. (default: :obj:`None`)
+        """
+        super().__init__(model_type, model_config_dict)
+        self._url = url or os.environ.get("ZHIPUAI_API_BASE_URL")
+        self._api_key = api_key or os.environ.get("ZHIPUAI_API_KEY")
+        self._client = OpenAI(
+            timeout=60,
+            max_retries=3,
+            api_key=self._api_key,
+            base_url=self._url,
+        )
+        self._token_counter: Optional[BaseTokenCounter] = None
+
+    @model_api_key_required
+    def run(
+        self,
+        messages: List[OpenAIMessage],
+    ) -> Union[ChatCompletion, Stream[ChatCompletionChunk]]:
+        r"""Runs inference of OpenAI chat completion.
+
+        Args:
+            messages (List[OpenAIMessage]): Message list with the chat history
+                in OpenAI API format.
+
+        Returns:
+            Union[ChatCompletion, Stream[ChatCompletionChunk]]:
+                `ChatCompletion` in the non-stream mode, or
+                `Stream[ChatCompletionChunk]` in the stream mode.
+        """
+        # Use OpenAI cilent as interface call ZhipuAI
+        # Reference: https://open.bigmodel.cn/dev/api#openai_sdk
+        response = self._client.chat.completions.create(
+            messages=messages,
+            model=self.model_type.value,
+            **self.model_config_dict,
+        )
+        return response
+
+    @property
+    def token_counter(self) -> BaseTokenCounter:
+        r"""Initialize the token counter for the model backend.
+
+        Returns:
+            OpenAITokenCounter: The token counter following the model's
+                tokenization style.
+        """
+
+        if not self._token_counter:
+            # It's a temporary setting for token counter.
+            self._token_counter = OpenAITokenCounter(ModelType.GPT_3_5_TURBO)
+        return self._token_counter
+
+    def check_model_config(self):
+        r"""Check whether the model configuration contains any
+        unexpected arguments to OpenAI API.
+
+        Raises:
+            ValueError: If the model configuration dictionary contains any
+                unexpected arguments to OpenAI API.
+        """
+        for param in self.model_config_dict:
+            if param not in OPENAI_API_PARAMS:
+                raise ValueError(
+                    f"Unexpected argument `{param}` is "
+                    "input into OpenAI model backend."
+                )
+        pass
+
+    @property
+    def stream(self) -> bool:
+        r"""Returns whether the model is in stream mode, which sends partial
+        results each time.
+
+        Returns:
+            bool: Whether the model is in stream mode.
+        """
+        return self.model_config_dict.get('stream', False)

camel/prompts/__init__.py

@@ -14,6 +14,7 @@
 from .ai_society import AISocietyPromptTemplateDict
 from .base import CodePrompt, TextPrompt, TextPromptDict
 from .code import CodePromptTemplateDict
+from .descripte_video_prompt import DescriptionVideoPromptTemplateDict
 from .evaluation import EvaluationPromptTemplateDict
 from .misalignment import MisalignmentPromptTemplateDict
 from .object_recognition import ObjectRecognitionPromptTemplateDict
@@ -37,4 +38,5 @@ __all__ = [
     'PromptTemplateGenerator',
     'SolutionExtractionPromptTemplateDict',
     'ObjectRecognitionPromptTemplateDict',
+    'DescriptionVideoPromptTemplateDict',
 ]