camel-ai 0.1.1__py3-none-any.whl → 0.1.3__py3-none-any.whl

camel/messages/base.py

@@ -11,8 +11,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+import base64
+import io
 from dataclasses import dataclass
-from typing import Any, Dict, List, Optional, Tuple, Union
+from typing import Any, Dict, List, Literal, Optional, Tuple, Union
+
+from PIL import Image
 
 from camel.messages import (
     OpenAIAssistantMessage,
@@ -21,7 +25,12 @@ from camel.messages import (
     OpenAIUserMessage,
 )
 from camel.prompts import CodePrompt, TextPrompt
-from camel.types import OpenAIBackendRole, RoleType
+from camel.types import (
+    OpenAIBackendRole,
+    OpenAIImageDetailType,
+    OpenAIImageType,
+    RoleType,
+)
 
 
 @dataclass
@@ -36,22 +45,49 @@ class BaseMessage:
             for the message.
         content (str): The content of the message.
     """
+
     role_name: str
     role_type: RoleType
     meta_dict: Optional[Dict[str, str]]
     content: str
+    image: Optional[Image.Image] = None
+    image_detail: Literal["auto", "low", "high"] = "auto"
 
     @classmethod
     def make_user_message(
-            cls, role_name: str, content: str,
-            meta_dict: Optional[Dict[str, str]] = None) -> 'BaseMessage':
-        return cls(role_name, RoleType.USER, meta_dict, content)
+        cls,
+        role_name: str,
+        content: str,
+        meta_dict: Optional[Dict[str, str]] = None,
+        image: Optional[Image.Image] = None,
+        image_detail: Union[OpenAIImageDetailType, str] = "auto",
+    ) -> 'BaseMessage':
+        return cls(
+            role_name,
+            RoleType.USER,
+            meta_dict,
+            content,
+            image,
+            OpenAIImageDetailType(image_detail).value,
+        )
 
     @classmethod
     def make_assistant_message(
-            cls, role_name: str, content: str,
-            meta_dict: Optional[Dict[str, str]] = None) -> 'BaseMessage':
-        return cls(role_name, RoleType.ASSISTANT, meta_dict, content)
+        cls,
+        role_name: str,
+        content: str,
+        meta_dict: Optional[Dict[str, str]] = None,
+        image: Optional[Image.Image] = None,
+        image_detail: Union[OpenAIImageDetailType, str] = "auto",
+    ) -> 'BaseMessage':
+        return cls(
+            role_name,
+            RoleType.ASSISTANT,
+            meta_dict,
+            content,
+            image,
+            OpenAIImageDetailType(image_detail).value,
+        )
 
     def create_new_instance(self, content: str) -> "BaseMessage":
         r"""Create a new instance of the :obj:`BaseMessage` with updated
@@ -63,9 +99,12 @@ class BaseMessage:
         Returns:
             BaseMessage: The new instance of :obj:`BaseMessage`.
         """
-        return self.__class__(role_name=self.role_name,
-                              role_type=self.role_type,
-                              meta_dict=self.meta_dict, content=content)
+        return self.__class__(
+            role_name=self.role_name,
+            role_type=self.role_type,
+            meta_dict=self.meta_dict,
+            content=content,
+        )
 
     def __add__(self, other: Any) -> Union["BaseMessage", Any]:
         r"""Addition operator override for :obj:`BaseMessage`.
@@ -83,7 +122,8 @@ class BaseMessage:
         else:
             raise TypeError(
                 f"Unsupported operand type(s) for +: '{type(self)}' and "
-                f"'{type(other)}'")
+                f"'{type(other)}'"
+            )
         return self.create_new_instance(combined_content)
 
     def __mul__(self, other: Any) -> Union["BaseMessage", Any]:
@@ -101,7 +141,8 @@ class BaseMessage:
         else:
             raise TypeError(
                 f"Unsupported operand type(s) for *: '{type(self)}' and "
-                f"'{type(other)}'")
+                f"'{type(other)}'"
+            )
 
     def __len__(self) -> int:
         r"""Length operator override for :obj:`BaseMessage`.
@@ -124,7 +165,8 @@ class BaseMessage:
         return item in self.content
 
     def extract_text_and_code_prompts(
-            self) -> Tuple[List[TextPrompt], List[CodePrompt]]:
+        self,
+    ) -> Tuple[List[TextPrompt], List[CodePrompt]]:
         r"""Extract text and code prompts from the message content.
 
         Returns:
@@ -140,7 +182,8 @@ class BaseMessage:
         start_idx = 0
         while idx < len(lines):
             while idx < len(lines) and (
-                    not lines[idx].lstrip().startswith("```")):
+                not lines[idx].lstrip().startswith("```")
+            ):
                 idx += 1
             text = "\n".join(lines[start_idx:idx]).strip()
             text_prompts.append(TextPrompt(text))
@@ -198,7 +241,47 @@ class BaseMessage:
         Returns:
             OpenAIUserMessage: The converted :obj:`OpenAIUserMessage` object.
         """
-        return {"role": "user", "content": self.content}
+        if self.image is None:
+            return {"role": "user", "content": self.content}
+        else:
+            #
+            if self.image.format is None:
+                raise ValueError(
+                    f"Image's `format` is `None`, please "
+                    f"transform the `PIL.Image.Image` to  one of "
+                    f"following supported formats, such as "
+                    f"{list(OpenAIImageType)}"
+                )
+
+            image_type: str = self.image.format.lower()
+            if image_type not in OpenAIImageType:
+                raise ValueError(
+                    f"Image type {self.image.format} "
+                    f"is not supported by OpenAI vision model"
+                )
+            with io.BytesIO() as buffer:
+                self.image.save(fp=buffer, format=self.image.format)
+                encoded_image = base64.b64encode(buffer.getvalue()).decode(
+                    "utf-8"
+                )
+            image_prefix = f"data:image/{image_type};base64,"
+
+            return {
+                "role": "user",
+                "content": [
+                    {
+                        "type": "text",
+                        "text": self.content,
+                    },
+                    {
+                        "type": "image_url",
+                        "image_url": {
+                            "url": f"{image_prefix}{encoded_image}",
+                            "detail": self.image_detail,
+                        },
+                    },
+                ],
+            }
 
     def to_openai_assistant_message(self) -> OpenAIAssistantMessage:
         r"""Converts the message to an :obj:`OpenAIAssistantMessage` object.

camel/messages/func_message.py

@@ -36,6 +36,7 @@ class FunctionCallingMessage(BaseMessage):
         result (Optional[Any]): The result of function execution.
             (default: :obj:`None`)
     """
+
     func_name: Optional[str] = None
     args: Optional[Dict] = None
     result: Optional[Any] = None
@@ -67,10 +68,11 @@ class FunctionCallingMessage(BaseMessage):
             OpenAIAssistantMessage: The converted :obj:`OpenAIAssistantMessage`
                 object.
         """
-        if (not self.func_name) or (not self.args):
+        if (not self.func_name) or (self.args is None):
             raise ValueError(
                 "Invalid request for converting into assistant message"
-                " due to missing function name or arguments.")
+                " due to missing function name or arguments."
+            )
 
         msg_dict: OpenAIAssistantMessage = {
             "role": "assistant",
@@ -78,7 +80,7 @@ class FunctionCallingMessage(BaseMessage):
             "function_call": {
                 "name": self.func_name,
                 "arguments": str(self.args),
-            }
+            },
         }
 
         return msg_dict
@@ -94,7 +96,8 @@ class FunctionCallingMessage(BaseMessage):
         if (not self.func_name) or (not self.result):
             raise ValueError(
                 "Invalid request for converting into function message"
-                " due to missing function name or results.")
+                " due to missing function name or results."
+            )
 
         result_content = {"result": {str(self.result)}}
         msg_dict: OpenAIFunctionMessage = {

camel/models/__init__.py

@@ -11,15 +11,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+from .anthropic_model import AnthropicModel
 from .base_model import BaseModelBackend
+from .model_factory import ModelFactory
+from .open_source_model import OpenSourceModel
 from .openai_model import OpenAIModel
 from .stub_model import StubModel
-from .open_source_model import OpenSourceModel
-from .model_factory import ModelFactory
 
 __all__ = [
     'BaseModelBackend',
     'OpenAIModel',
+    'AnthropicModel',
     'StubModel',
     'OpenSourceModel',
     'ModelFactory',

camel/models/anthropic_model.py

@@ -0,0 +1,132 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+import os
+from typing import Any, Dict, Optional
+
+from anthropic import Anthropic
+from anthropic._types import NOT_GIVEN
+
+from camel.configs import ANTHROPIC_API_PARAMS
+from camel.models.base_model import BaseModelBackend
+from camel.types import ChatCompletion, ModelType
+from camel.utils import AnthropicTokenCounter, BaseTokenCounter
+
+
+class AnthropicModel(BaseModelBackend):
+    r"""Anthropic API in a unified BaseModelBackend interface."""
+
+    def __init__(
+        self, model_type: ModelType, model_config_dict: Dict[str, Any]
+    ) -> None:
+        super().__init__(model_type, model_config_dict)
+        self.client = Anthropic(api_key=os.environ["ANTHROPIC_API_KEY"])
+        self._token_counter: Optional[BaseTokenCounter] = None
+
+    def _convert_response_from_anthropic_to_openai(self, response):
+        # openai ^1.0.0 format, reference openai/types/chat/chat_completion.py
+        obj = ChatCompletion.construct(
+            id=None,
+            choices=[
+                dict(
+                    index=0,
+                    message={
+                        "role": "assistant",
+                        "content": response.content[0].text,
+                    },
+                    finish_reason=response.stop_reason,
+                )
+            ],
+            created=None,
+            model=response.model,
+            object="chat.completion",
+        )
+        return obj
+
+    @property
+    def token_counter(self) -> BaseTokenCounter:
+        r"""Initialize the token counter for the model backend.
+
+        Returns:
+            BaseTokenCounter: The token counter following the model's
+                tokenization style.
+        """
+        if not self._token_counter:
+            self._token_counter = AnthropicTokenCounter(self.model_type)
+        return self._token_counter
+
+    def count_tokens_from_prompt(self, prompt: str) -> int:
+        r"""Count the number of tokens from a prompt.
+
+        Args:
+            prompt (str): The prompt string.
+
+        Returns:
+            int: The number of tokens in the prompt.
+        """
+        return self.client.count_tokens(prompt)
+
+    def run(
+        self,
+        messages,
+    ):
+        r"""Run inference of Anthropic chat completion.
+
+        Args:
+            messages (List[Dict]): Message list with the chat history
+                in OpenAI API format.
+
+        Returns:
+            Dict[str, Any]: Response in the OpenAI API format.
+        """
+
+        if messages[0]["role"] == "system":
+            sys_msg = messages.pop(0)["content"]
+        else:
+            sys_msg = NOT_GIVEN
+        response = self.client.messages.create(
+            model=self.model_type.value,
+            system=sys_msg,
+            messages=messages,
+            **self.model_config_dict,
+        )
+
+        # format response to openai format
+        response = self._convert_response_from_anthropic_to_openai(response)
+
+        return response
+
+    def check_model_config(self):
+        r"""Check whether the model configuration is valid for anthropic
+        model backends.
+
+        Raises:
+            ValueError: If the model configuration dictionary contains any
+                unexpected arguments to OpenAI API, or it does not contain
+                :obj:`model_path` or :obj:`server_url`.
+        """
+        for param in self.model_config_dict:
+            if param not in ANTHROPIC_API_PARAMS:
+                raise ValueError(
+                    f"Unexpected argument `{param}` is "
+                    "input into Anthropic model backend."
+                )
+
+    @property
+    def stream(self) -> bool:
+        r"""Returns whether the model is in stream mode,
+            which sends partial results each time.
+        Returns:
+            bool: Whether the model is in stream mode.
+        """
+        return self.model_config_dict.get("stream", False)

camel/models/base_model.py

@@ -26,8 +26,9 @@ class BaseModelBackend(ABC):
     May be OpenAI API, a local LLM, a stub for unit tests, etc.
     """
 
-    def __init__(self, model_type: ModelType,
-                 model_config_dict: Dict[str, Any]) -> None:
+    def __init__(
+        self, model_type: ModelType, model_config_dict: Dict[str, Any]
+    ) -> None:
         r"""Constructor for the model backend.
 
         Args:

camel/models/model_factory.py

@@ -13,12 +13,11 @@
 # =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
 from typing import Any, Dict
 
-from camel.models import (
-    BaseModelBackend,
-    OpenAIModel,
-    OpenSourceModel,
-    StubModel,
-)
+from camel.models.anthropic_model import AnthropicModel
+from camel.models.base_model import BaseModelBackend
+from camel.models.open_source_model import OpenSourceModel
+from camel.models.openai_model import OpenAIModel
+from camel.models.stub_model import StubModel
 from camel.types import ModelType
 
 
@@ -30,8 +29,9 @@ class ModelFactory:
     """
 
     @staticmethod
-    def create(model_type: ModelType,
-               model_config_dict: Dict) -> BaseModelBackend:
+    def create(
+        model_type: ModelType, model_config_dict: Dict
+    ) -> BaseModelBackend:
         r"""Creates an instance of `BaseModelBackend` of the specified type.
 
         Args:
@@ -52,6 +52,8 @@ class ModelFactory:
             model_class = StubModel
         elif model_type.is_open_source:
             model_class = OpenSourceModel
+        elif model_type.is_anthropic:
+            model_class = AnthropicModel
         else:
             raise ValueError(f"Unknown model type `{model_type}` is input")
 

camel/models/open_source_model.py

@@ -45,11 +45,13 @@ class OpenSourceModel(BaseModelBackend):
         # Check whether the input model type is open-source
         if not model_type.is_open_source:
             raise ValueError(
-                f"Model `{model_type}` is not a supported open-source model.")
+                f"Model `{model_type}` is not a supported open-source model."
+            )
 
         # Check whether input model path is empty
-        model_path: Optional[str] = (self.model_config_dict.get(
-            "model_path", None))
+        model_path: Optional[str] = self.model_config_dict.get(
+            "model_path", None
+        )
         if not model_path:
             raise ValueError("Path to open-source model is not provided.")
         self.model_path: str = model_path
@@ -59,19 +61,23 @@ class OpenSourceModel(BaseModelBackend):
         if not self.model_type.validate_model_name(self.model_name):
             raise ValueError(
                 f"Model name `{self.model_name}` does not match model type "
-                f"`{self.model_type.value}`.")
+                f"`{self.model_type.value}`."
+            )
 
         # Load the server URL and check whether it is None
-        server_url: Optional[str] = (self.model_config_dict.get(
-            "server_url", None))
+        server_url: Optional[str] = self.model_config_dict.get(
+            "server_url", None
+        )
         if not server_url:
             raise ValueError(
-                "URL to server running open-source LLM is not provided.")
+                "URL to server running open-source LLM is not provided."
+            )
         self.server_url: str = server_url
         self._client = OpenAI(
             base_url=self.server_url,
             timeout=60,
             max_retries=3,
+            api_key="fake_key",
         )
 
         # Replace `model_config_dict` with only the params to be
@@ -88,7 +94,8 @@ class OpenSourceModel(BaseModelBackend):
         """
         if not self._token_counter:
             self._token_counter = OpenSourceTokenCounter(
-                self.model_type, self.model_path)
+                self.model_type, self.model_path
+            )
         return self._token_counter
 
     def run(
@@ -123,16 +130,21 @@ class OpenSourceModel(BaseModelBackend):
                 unexpected arguments to OpenAI API, or it does not contain
                 :obj:`model_path` or :obj:`server_url`.
         """
-        if ("model_path" not in self.model_config_dict
-                or "server_url" not in self.model_config_dict):
+        if (
+            "model_path" not in self.model_config_dict
+            or "server_url" not in self.model_config_dict
+        ):
             raise ValueError(
                 "Invalid configuration for open-source model backend with "
-                ":obj:`model_path` or :obj:`server_url` missing.")
+                ":obj:`model_path` or :obj:`server_url` missing."
+            )
 
         for param in self.model_config_dict["api_params"].__dict__:
             if param not in OPENAI_API_PARAMS:
-                raise ValueError(f"Unexpected argument `{param}` is "
-                                 "input into open-source model backend.")
+                raise ValueError(
+                    f"Unexpected argument `{param}` is "
+                    "input into open-source model backend."
+                )
 
     @property
     def stream(self) -> bool:

camel/models/openai_model.py

@@ -20,18 +20,15 @@ from camel.configs import OPENAI_API_PARAMS_WITH_FUNCTIONS
 from camel.messages import OpenAIMessage
 from camel.models import BaseModelBackend
 from camel.types import ChatCompletion, ChatCompletionChunk, ModelType
-from camel.utils import (
-    BaseTokenCounter,
-    OpenAITokenCounter,
-    openai_api_key_required,
-)
+from camel.utils import BaseTokenCounter, OpenAITokenCounter, api_key_required
 
 
 class OpenAIModel(BaseModelBackend):
     r"""OpenAI API in a unified BaseModelBackend interface."""
 
-    def __init__(self, model_type: ModelType,
-                 model_config_dict: Dict[str, Any]) -> None:
+    def __init__(
+        self, model_type: ModelType, model_config_dict: Dict[str, Any]
+    ) -> None:
         r"""Constructor for OpenAI backend.
 
         Args:
@@ -57,7 +54,7 @@ class OpenAIModel(BaseModelBackend):
             self._token_counter = OpenAITokenCounter(self.model_type)
         return self._token_counter
 
-    @openai_api_key_required
+    @api_key_required
     def run(
         self,
         messages: List[OpenAIMessage],
@@ -90,8 +87,10 @@ class OpenAIModel(BaseModelBackend):
         """
         for param in self.model_config_dict:
             if param not in OPENAI_API_PARAMS_WITH_FUNCTIONS:
-                raise ValueError(f"Unexpected argument `{param}` is "
-                                 "input into OpenAI model backend.")
+                raise ValueError(
+                    f"Unexpected argument `{param}` is "
+                    "input into OpenAI model backend."
+                )
 
     @property
     def stream(self) -> bool:

camel/models/stub_model.py

@@ -30,7 +30,6 @@ from camel.utils import BaseTokenCounter
 
 
 class StubTokenCounter(BaseTokenCounter):
-
     def count_tokens_from_messages(self, messages: List[OpenAIMessage]) -> int:
         r"""Token counting for STUB models, directly returning a constant.
 
@@ -47,10 +46,12 @@ class StubTokenCounter(BaseTokenCounter):
 
 class StubModel(BaseModelBackend):
     r"""A dummy model used for unit tests."""
+
     model_type = ModelType.STUB
 
-    def __init__(self, model_type: ModelType,
-                 model_config_dict: Dict[str, Any]) -> None:
+    def __init__(
+        self, model_type: ModelType, model_config_dict: Dict[str, Any]
+    ) -> None:
         r"""All arguments are unused for the dummy model."""
         super().__init__(model_type, model_config_dict)
         self._token_counter: Optional[BaseTokenCounter] = None
@@ -90,6 +91,7 @@ class StubModel(BaseModelBackend):
                         content=ARBITRARY_STRING,
                         role="assistant",
                     ),
+                    logprobs=None,
                 )
             ],
             usage=CompletionUsage(
@@ -101,6 +103,5 @@ class StubModel(BaseModelBackend):
         return response
 
     def check_model_config(self):
-        r"""Directly pass the check on arguments to STUB model.
-        """
+        r"""Directly pass the check on arguments to STUB model."""
         pass

camel/prompts/__init__.py

@@ -11,16 +11,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
-from .base import TextPrompt, CodePrompt, TextPromptDict
 from .ai_society import AISocietyPromptTemplateDict
+from .base import CodePrompt, TextPrompt, TextPromptDict
 from .code import CodePromptTemplateDict
-from .misalignment import MisalignmentPromptTemplateDict
-from .translation import TranslationPromptTemplateDict
-from .solution_extraction import SolutionExtractionPromptTemplateDict
 from .evaluation import EvaluationPromptTemplateDict
+from .misalignment import MisalignmentPromptTemplateDict
+from .object_recognition import ObjectRecognitionPromptTemplateDict
+from .prompt_templates import PromptTemplateGenerator
 from .role_description_prompt_template import RoleDescriptionPromptTemplateDict
+from .solution_extraction import SolutionExtractionPromptTemplateDict
 from .task_prompt_template import TaskPromptTemplateDict
-from .prompt_templates import PromptTemplateGenerator
+from .translation import TranslationPromptTemplateDict
 
 __all__ = [
     'TextPrompt',
@@ -35,4 +36,5 @@ __all__ = [
     'TaskPromptTemplateDict',
     'PromptTemplateGenerator',
     'SolutionExtractionPromptTemplateDict',
+    'ObjectRecognitionPromptTemplateDict',
 ]