camel-ai 0.2.21__py3-none-any.whl → 0.2.23a0__py3-none-any.whl

camel/models/qwen_model.py

@@ -13,13 +13,15 @@
 # ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
 
 import os
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, List, Optional, Type, Union
 
-from openai import OpenAI, Stream
+from openai import AsyncOpenAI, AsyncStream, OpenAI, Stream
+from pydantic import BaseModel
 
 from camel.configs import QWEN_API_PARAMS, QwenConfig
 from camel.messages import OpenAIMessage
 from camel.models import BaseModelBackend
+from camel.models._utils import try_modify_message_with_format
 from camel.types import (
     ChatCompletion,
     ChatCompletionChunk,
@@ -81,10 +83,46 @@ class QwenModel(BaseModelBackend):
             api_key=self._api_key,
             base_url=self._url,
         )
+        self._async_client = AsyncOpenAI(
+            timeout=180,
+            max_retries=3,
+            api_key=self._api_key,
+            base_url=self._url,
+        )
+
+    async def _arun(
+        self,
+        messages: List[OpenAIMessage],
+        response_format: Optional[Type[BaseModel]] = None,
+        tools: Optional[List[Dict[str, Any]]] = None,
+    ) -> Union[ChatCompletion, AsyncStream[ChatCompletionChunk]]:
+        r"""Runs inference of Qwen chat completion.
+
+        Args:
+            messages (List[OpenAIMessage]): Message list with the chat history
+                in OpenAI API format.
+
+        Returns:
+            Union[ChatCompletion, AsyncStream[ChatCompletionChunk]]:
+                `ChatCompletion` in the non-stream mode, or
+                `AsyncStream[ChatCompletionChunk]` in the stream mode.
+        """
+        request_config = self._prepare_request(
+            messages, response_format, tools
+        )
+
+        response = await self._async_client.chat.completions.create(
+            messages=messages,
+            model=self.model_type,
+            **request_config,
+        )
+        return response
 
-    def run(
+    def _run(
         self,
         messages: List[OpenAIMessage],
+        response_format: Optional[Type[BaseModel]] = None,
+        tools: Optional[List[Dict[str, Any]]] = None,
     ) -> Union[ChatCompletion, Stream[ChatCompletionChunk]]:
         r"""Runs inference of Qwen chat completion.
 
@@ -97,13 +135,32 @@ class QwenModel(BaseModelBackend):
                 `ChatCompletion` in the non-stream mode, or
                 `Stream[ChatCompletionChunk]` in the stream mode.
         """
+        request_config = self._prepare_request(
+            messages, response_format, tools
+        )
+
         response = self._client.chat.completions.create(
             messages=messages,
             model=self.model_type,
-            **self.model_config_dict,
+            **request_config,
         )
         return response
 
+    def _prepare_request(
+        self,
+        messages: List[OpenAIMessage],
+        response_format: Optional[Type[BaseModel]] = None,
+        tools: Optional[List[Dict[str, Any]]] = None,
+    ) -> Dict[str, Any]:
+        request_config = self.model_config_dict.copy()
+        if tools:
+            request_config["tools"] = tools
+        elif response_format:
+            try_modify_message_with_format(messages[-1], response_format)
+            request_config["response_format"] = {"type": "json_object"}
+
+        return request_config
+
     @property
     def token_counter(self) -> BaseTokenCounter:
         r"""Initialize the token counter for the model backend.

camel/models/reka_model.py

@@ -11,7 +11,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Type, Union
+
+from pydantic import BaseModel
 
 from camel.configs import REKA_API_PARAMS, RekaConfig
 from camel.messages import OpenAIMessage
@@ -70,7 +72,7 @@ class RekaModel(BaseModelBackend):
         url: Optional[str] = None,
         token_counter: Optional[BaseTokenCounter] = None,
     ) -> None:
-        from reka.client import Reka
+        from reka.client import AsyncReka, Reka
 
         if model_config_dict is None:
             model_config_dict = RekaConfig().as_dict()
@@ -80,6 +82,9 @@ class RekaModel(BaseModelBackend):
             model_type, model_config_dict, api_key, url, token_counter
         )
         self._client = Reka(api_key=self._api_key, base_url=self._url)
+        self._async_client = AsyncReka(
+            api_key=self._api_key, base_url=self._url
+        )
 
     def _convert_reka_to_openai_response(
         self, response: 'ChatResponse'
@@ -117,6 +122,8 @@ class RekaModel(BaseModelBackend):
     def _convert_openai_to_reka_messages(
         self,
         messages: List[OpenAIMessage],
+        response_format: Optional[Type[BaseModel]] = None,
+        tools: Optional[List[str]] = None,
     ) -> List["ChatMessage"]:
         r"""Converts OpenAI API messages to Reka API messages.
 
@@ -173,9 +180,52 @@ class RekaModel(BaseModelBackend):
             )
         return self._token_counter
 
-    def run(
+    async def _arun(
+        self,
+        messages: List[OpenAIMessage],
+        response_format: Optional[Type[BaseModel]] = None,
+        tools: Optional[List[Dict[str, Any]]] = None,
+    ) -> ChatCompletion:
+        r"""Runs inference of Mistral chat completion.
+
+        Args:
+            messages (List[OpenAIMessage]): Message list with the chat history
+                in OpenAI API format.
+
+        Returns:
+            ChatCompletion.
+        """
+        reka_messages = self._convert_openai_to_reka_messages(messages)
+
+        response = await self._async_client.chat.create(
+            messages=reka_messages,
+            model=self.model_type,
+            **self.model_config_dict,
+        )
+
+        openai_response = self._convert_reka_to_openai_response(response)
+
+        # Add AgentOps LLM Event tracking
+        if LLMEvent:
+            llm_event = LLMEvent(
+                thread_id=openai_response.id,
+                prompt=" ".join(
+                    [message.get("content") for message in messages]  # type: ignore[misc]
+                ),
+                prompt_tokens=openai_response.usage.input_tokens,  # type: ignore[union-attr]
+                completion=openai_response.choices[0].message.content,
+                completion_tokens=openai_response.usage.output_tokens,  # type: ignore[union-attr]
+                model=self.model_type,
+            )
+            record(llm_event)
+
+        return openai_response
+
+    def _run(
         self,
         messages: List[OpenAIMessage],
+        response_format: Optional[Type[BaseModel]] = None,
+        tools: Optional[List[Dict[str, Any]]] = None,
     ) -> ChatCompletion:
         r"""Runs inference of Mistral chat completion.
 

camel/models/samba_model.py

@@ -15,10 +15,11 @@ import json
 import os
 import time
 import uuid
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, List, Optional, Type, Union
 
 import httpx
-from openai import OpenAI, Stream
+from openai import AsyncOpenAI, AsyncStream, OpenAI, Stream
+from pydantic import BaseModel
 
 from camel.configs import (
     SAMBA_CLOUD_API_PARAMS,
@@ -105,6 +106,12 @@ class SambaModel(BaseModelBackend):
                 base_url=self._url,
                 api_key=self._api_key,
             )
+            self._async_client = AsyncOpenAI(
+                timeout=180,
+                max_retries=3,
+                base_url=self._url,
+                api_key=self._api_key,
+            )
 
     @property
     def token_counter(self) -> BaseTokenCounter:
@@ -148,8 +155,35 @@ class SambaModel(BaseModelBackend):
                 " SambaNova service"
             )
 
-    def run(  # type: ignore[misc]
-        self, messages: List[OpenAIMessage]
+    async def _arun(  # type: ignore[misc]
+        self,
+        messages: List[OpenAIMessage],
+        response_format: Optional[Type[BaseModel]] = None,
+        tools: Optional[List[Dict[str, Any]]] = None,
+    ) -> Union[ChatCompletion, AsyncStream[ChatCompletionChunk]]:
+        r"""Runs SambaNova's service.
+
+        Args:
+            messages (List[OpenAIMessage]): Message list with the chat history
+                in OpenAI API format.
+
+        Returns:
+            Union[ChatCompletion, AsyncStream[ChatCompletionChunk]]:
+                `ChatCompletion` in the non-stream mode, or
+                `AsyncStream[ChatCompletionChunk]` in the stream mode.
+        """
+        if "tools" in self.model_config_dict:
+            del self.model_config_dict["tools"]
+        if self.model_config_dict.get("stream") is True:
+            return await self._arun_streaming(messages)
+        else:
+            return await self._arun_non_streaming(messages)
+
+    def _run(  # type: ignore[misc]
+        self,
+        messages: List[OpenAIMessage],
+        response_format: Optional[Type[BaseModel]] = None,
+        tools: Optional[List[Dict[str, Any]]] = None,
     ) -> Union[ChatCompletion, Stream[ChatCompletionChunk]]:
         r"""Runs SambaNova's service.
 
@@ -398,3 +432,174 @@ class SambaModel(BaseModelBackend):
             bool: Whether the model is in stream mode.
         """
         return self.model_config_dict.get('stream', False)
+
+    async def _arun_streaming(
+        self, messages: List[OpenAIMessage]
+    ) -> AsyncStream[ChatCompletionChunk]:
+        r"""Handles streaming inference with SambaNova's API.
+
+        Args:
+            messages (List[OpenAIMessage]): A list of messages representing the
+                chat history in OpenAI API format.
+
+        Returns:
+            AsyncStream[ChatCompletionChunk]: A generator yielding
+                `ChatCompletionChunk` objects as they are received from the
+                API.
+
+        Raises:
+            RuntimeError: If the HTTP request fails.
+            ValueError: If the API doesn't support stream mode.
+        """
+        # Handle SambaNova's Cloud API
+        if self._url == "https://api.sambanova.ai/v1":
+            response = await self._async_client.chat.completions.create(
+                messages=messages,
+                model=self.model_type,
+                **self.model_config_dict,
+            )
+
+            # Add AgentOps LLM Event tracking
+            if LLMEvent:
+                llm_event = LLMEvent(
+                    thread_id=response.id,
+                    prompt=" ".join(
+                        [message.get("content") for message in messages]  # type: ignore[misc]
+                    ),
+                    prompt_tokens=response.usage.prompt_tokens,  # type: ignore[union-attr]
+                    completion=response.choices[0].message.content,
+                    completion_tokens=response.usage.completion_tokens,  # type: ignore[union-attr]
+                    model=self.model_type,
+                )
+                record(llm_event)
+
+            return response
+
+        elif self._url == "https://sambaverse.sambanova.ai/api/predict":
+            raise ValueError(
+                "https://sambaverse.sambanova.ai/api/predict doesn't support"
+                " stream mode"
+            )
+        raise RuntimeError(f"Unknown URL: {self._url}")
+
+    async def _arun_non_streaming(
+        self, messages: List[OpenAIMessage]
+    ) -> ChatCompletion:
+        r"""Handles non-streaming inference with SambaNova's API.
+
+        Args:
+            messages (List[OpenAIMessage]): A list of messages representing the
+                message in OpenAI API format.
+
+        Returns:
+            ChatCompletion: A `ChatCompletion` object containing the complete
+                response from the API.
+
+        Raises:
+            RuntimeError: If the HTTP request fails.
+            ValueError: If the JSON response cannot be decoded or is missing
+                expected data.
+        """
+        # Handle SambaNova's Cloud API
+        if self._url == "https://api.sambanova.ai/v1":
+            response = await self._async_client.chat.completions.create(
+                messages=messages,
+                model=self.model_type,
+                **self.model_config_dict,
+            )
+
+            # Add AgentOps LLM Event tracking
+            if LLMEvent:
+                llm_event = LLMEvent(
+                    thread_id=response.id,
+                    prompt=" ".join(
+                        [message.get("content") for message in messages]  # type: ignore[misc]
+                    ),
+                    prompt_tokens=response.usage.prompt_tokens,  # type: ignore[union-attr]
+                    completion=response.choices[0].message.content,
+                    completion_tokens=response.usage.completion_tokens,  # type: ignore[union-attr]
+                    model=self.model_type,
+                )
+                record(llm_event)
+
+            return response
+
+        # Handle SambaNova's Sambaverse API
+        else:
+            headers = {
+                "Content-Type": "application/json",
+                "key": str(self._api_key),
+                "modelName": self.model_type,
+            }
+
+            data = {
+                "instance": json.dumps(
+                    {
+                        "conversation_id": str(uuid.uuid4()),
+                        "messages": messages,
+                    }
+                ),
+                "params": {
+                    "do_sample": {"type": "bool", "value": "true"},
+                    "max_tokens_to_generate": {
+                        "type": "int",
+                        "value": str(self.model_config_dict.get("max_tokens")),
+                    },
+                    "process_prompt": {"type": "bool", "value": "true"},
+                    "repetition_penalty": {
+                        "type": "float",
+                        "value": str(
+                            self.model_config_dict.get("repetition_penalty")
+                        ),
+                    },
+                    "return_token_count_only": {
+                        "type": "bool",
+                        "value": "false",
+                    },
+                    "select_expert": {
+                        "type": "str",
+                        "value": self.model_type.split("/")[1],
+                    },
+                    "stop_sequences": {
+                        "type": "str",
+                        "value": self.model_config_dict.get("stop_sequences"),
+                    },
+                    "temperature": {
+                        "type": "float",
+                        "value": str(
+                            self.model_config_dict.get("temperature")
+                        ),
+                    },
+                    "top_k": {
+                        "type": "int",
+                        "value": str(self.model_config_dict.get("top_k")),
+                    },
+                    "top_p": {
+                        "type": "float",
+                        "value": str(self.model_config_dict.get("top_p")),
+                    },
+                },
+            }
+
+            try:
+                # Send the request and handle the response
+                with httpx.Client() as client:
+                    response = client.post(
+                        self._url,  # type: ignore[arg-type]
+                        headers=headers,
+                        json=data,
+                    )
+
+                raw_text = response.text
+                # Split the string into two dictionaries
+                dicts = raw_text.split("}\n{")
+
+                # Keep only the last dictionary
+                last_dict = "{" + dicts[-1]
+
+                # Parse the dictionary
+                last_dict = json.loads(last_dict)
+                return self._sambaverse_to_openai_response(last_dict)  # type: ignore[arg-type]
+
+            except httpx.HTTPStatusError:
+                raise RuntimeError(f"HTTP request failed: {raw_text}")

camel/models/sglang_model.py

@@ -12,11 +12,13 @@
 # limitations under the License.
 # ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
 import logging
+import subprocess
 import threading
 import time
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, List, Optional, Type, Union
 
-from openai import OpenAI, Stream
+from openai import AsyncOpenAI, AsyncStream, OpenAI, Stream
+from pydantic import BaseModel
 
 from camel.configs import SGLANG_API_PARAMS, SGLangConfig
 from camel.messages import OpenAIMessage
@@ -85,13 +87,14 @@ class SGLangModel(BaseModelBackend):
                 api_key="Set-but-ignored",  # required but ignored
                 base_url=self._url,
             )
+            self._async_client = AsyncOpenAI(
+                timeout=180,
+                max_retries=3,
+                api_key="Set-but-ignored",  # required but ignored
+                base_url=self._url,
+            )
 
     def _start_server(self) -> None:
-        from sglang.utils import (  # type: ignore[import-untyped]
-            execute_shell_command,
-            wait_for_server,
-        )
-
         try:
             if not self._url:
                 cmd = (
@@ -101,10 +104,10 @@ class SGLangModel(BaseModelBackend):
                     f"--host 0.0.0.0"
                 )
 
-                server_process = execute_shell_command(cmd)
-                wait_for_server("http://localhost:30000")
+                server_process = _execute_shell_command(cmd)
+                _wait_for_server("http://localhost:30000")
                 self._url = "http://127.0.0.1:30000/v1"
-                self.server_process = server_process
+                self.server_process = server_process  # type: ignore[assignment]
                 # Start the inactivity monitor in a background thread
                 self._inactivity_thread = threading.Thread(
                     target=self._monitor_inactivity, daemon=True
@@ -131,8 +134,6 @@ class SGLangModel(BaseModelBackend):
         r"""Monitor whether the server process has been inactive for over 10
         minutes.
         """
-        from sglang.utils import terminate_process
-
         while True:
             # Check every 10 seconds
             time.sleep(10)
@@ -143,7 +144,7 @@ class SGLangModel(BaseModelBackend):
                     time.time() - self.last_run_time > 600
                 ):
                     if self.server_process:
-                        terminate_process(self.server_process)
+                        _terminate_process(self.server_process)
                         self.server_process = None
                         self._client = None  # Invalidate the client
                         logging.info(
@@ -178,9 +179,49 @@ class SGLangModel(BaseModelBackend):
                     "input into SGLang model backend."
                 )
 
-    def run(
+    async def _arun(
         self,
         messages: List[OpenAIMessage],
+        response_format: Optional[Type[BaseModel]] = None,
+        tools: Optional[List[Dict[str, Any]]] = None,
+    ) -> Union[ChatCompletion, AsyncStream[ChatCompletionChunk]]:
+        r"""Runs inference of OpenAI chat completion.
+
+        Args:
+            messages (List[OpenAIMessage]): Message list with the chat history
+                in OpenAI API format.
+
+        Returns:
+            Union[ChatCompletion, AsyncStream[ChatCompletionChunk]]:
+                `ChatCompletion` in the non-stream mode, or
+                `AsyncStream[ChatCompletionChunk]` in the stream mode.
+        """
+
+        # Ensure server is running
+        self._ensure_server_running()
+
+        with self._lock:
+            # Update last run time
+            self.last_run_time = time.time()
+
+        if self._client is None:
+            raise RuntimeError(
+                "Client is not initialized. Ensure the server is running."
+            )
+
+        response = await self._async_client.chat.completions.create(
+            messages=messages,
+            model=self.model_type,
+            **self.model_config_dict,
+        )
+
+        return response
+
+    def _run(
+        self,
+        messages: List[OpenAIMessage],
+        response_format: Optional[Type[BaseModel]] = None,
+        tools: Optional[List[Dict[str, Any]]] = None,
     ) -> Union[ChatCompletion, Stream[ChatCompletionChunk]]:
         r"""Runs inference of OpenAI chat completion.
 
@@ -223,3 +264,101 @@ class SGLangModel(BaseModelBackend):
             bool: Whether the model is in stream mode.
         """
         return self.model_config_dict.get('stream', False)
+
+
+# Below are helper functions from sglang.utils
+def _terminate_process(process):
+    _kill_process_tree(process.pid)
+
+
+def _kill_process_tree(
+    parent_pid, include_parent: bool = True, skip_pid: Optional[int] = None
+):
+    r"""Kill the process and all its child processes."""
+    import os
+    import signal
+
+    import psutil
+
+    if parent_pid is None:
+        parent_pid = os.getpid()
+        include_parent = False
+
+    try:
+        itself = psutil.Process(parent_pid)
+    except psutil.NoSuchProcess:
+        return
+
+    children = itself.children(recursive=True)
+    for child in children:
+        if child.pid == skip_pid:
+            continue
+        try:
+            child.kill()
+        except psutil.NoSuchProcess:
+            pass
+
+    if include_parent:
+        try:
+            itself.kill()
+
+            # Sometime processes cannot be killed with SIGKILL
+            # so we send an additional signal to kill them.
+            itself.send_signal(signal.SIGQUIT)
+        except psutil.NoSuchProcess:
+            pass
+
+
+def _execute_shell_command(command: str) -> subprocess.Popen:
+    r"""Execute a shell command and return the process handle
+
+    Args:
+        command: Shell command as a string (can include \\ line continuations)
+    Returns:
+        subprocess.Popen: Process handle
+    """
+    import subprocess
+
+    # Replace \ newline with space and split
+    command = command.replace("\\\n", " ").replace("\\", " ")
+    parts = command.split()
+
+    return subprocess.Popen(parts, text=True, stderr=subprocess.STDOUT)
+
+
+def _wait_for_server(base_url: str, timeout: Optional[int] = None) -> None:
+    r"""Wait for the server to be ready by polling the /v1/models endpoint.
+
+    Args:
+        base_url: The base URL of the server
+        timeout: Maximum time to wait in seconds. None means wait forever.
+    """
+    import requests
+
+    start_time = time.time()
+    while True:
+        try:
+            response = requests.get(
+                f"{base_url}/v1/models",
+                headers={"Authorization": "Bearer None"},
+            )
+            if response.status_code == 200:
+                time.sleep(5)
+                print(
+                    """\n
+                    NOTE: Typically, the server runs in a separate terminal.
+                    In this notebook, we run the server and notebook code 
+                    together, so their outputs are combined.
+                    To improve clarity, the server logs are displayed in the 
+                    original black color, while the notebook outputs are 
+                    highlighted in blue.
+                    """
+                )
+                break
+
+            if timeout and time.time() - start_time > timeout:
+                raise TimeoutError(
+                    "Server did not become ready within timeout period"
+                )
+        except requests.exceptions.RequestException:
+            time.sleep(1)

camel/models/siliconflow_model.py

@@ -12,9 +12,10 @@
 # limitations under the License.
 # ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
 import os
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, List, Optional, Type, Union
 
-from openai import OpenAI, Stream
+from openai import AsyncStream, OpenAI, Stream
+from pydantic import BaseModel
 
 from camel.configs import SILICONFLOW_API_PARAMS, SiliconFlowConfig
 from camel.messages import OpenAIMessage
@@ -82,9 +83,11 @@ class SiliconFlowModel(BaseModelBackend):
             base_url=self._url,
         )
 
-    def run(
+    def _run(
         self,
         messages: List[OpenAIMessage],
+        response_format: Optional[Type[BaseModel]] = None,
+        tools: Optional[List[Dict[str, Any]]] = None,
     ) -> Union[ChatCompletion, Stream[ChatCompletionChunk]]:
         r"""Runs inference of SiliconFlow chat completion.
 
@@ -104,6 +107,16 @@ class SiliconFlowModel(BaseModelBackend):
         )
         return response
 
+    async def _arun(
+        self,
+        messages: List[OpenAIMessage],
+        response_format: Optional[Type[BaseModel]] = None,
+        tools: Optional[List[Dict[str, Any]]] = None,
+    ) -> Union[ChatCompletion, AsyncStream[ChatCompletionChunk]]:
+        raise NotImplementedError(
+            "SiliconFlow does not support async inference."
+        )
+
     @property
     def token_counter(self) -> BaseTokenCounter:
         r"""Initialize the token counter for the model backend.