camel-ai 0.2.23a0__py3-none-any.whl → 0.2.24__py3-none-any.whl

camel/societies/role_playing.py

@@ -468,6 +468,42 @@ class RolePlaying:
 
         return init_msg
 
+    async def ainit_chat(
+        self, init_msg_content: Optional[str] = None
+    ) -> BaseMessage:
+        r"""Asynchronously initializes the chat by resetting both of the
+        assistant and user agents. Returns an initial message for the
+        role-playing session.
+
+        Args:
+            init_msg_content (str, optional): A user-specified initial message.
+                Will be sent to the role-playing session as the initial
+                message. (default: :obj:`None`)
+
+        Returns:
+            BaseMessage: A single `BaseMessage` representing the initial
+                message.
+        """
+        # Currently, reset() is synchronous, but if it becomes async in the
+        # future, we can await it here
+        self.assistant_agent.reset()
+        self.user_agent.reset()
+        default_init_msg_content = (
+            "Now start to give me instructions one by one. "
+            "Only reply with Instruction and Input."
+        )
+        if init_msg_content is None:
+            init_msg_content = default_init_msg_content
+
+        # Initialize a message sent by the assistant
+        init_msg = BaseMessage.make_assistant_message(
+            role_name=getattr(self.assistant_sys_msg, 'role_name', None)
+            or "assistant",
+            content=init_msg_content,
+        )
+
+        return init_msg
+
     def step(
         self,
         assistant_msg: BaseMessage,
@@ -549,3 +585,86 @@ class RolePlaying:
                 info=user_response.info,
             ),
         )
+
+    async def astep(
+        self,
+        assistant_msg: BaseMessage,
+    ) -> Tuple[ChatAgentResponse, ChatAgentResponse]:
+        r"""Asynchronously advances the conversation by taking a message from
+        the assistant, processing it using the user agent, and then processing
+        the resulting message using the assistant agent. Returns a tuple
+        containing the resulting assistant message, whether the assistant
+        agent terminated the conversation, and any additional assistant
+        information, as well as a tuple containing the resulting user message,
+        whether the user agent terminated the conversation, and any additional
+        user information.
+
+        Args:
+            assistant_msg: A `BaseMessage` representing the message from the
+                assistant.
+
+        Returns:
+            Tuple[ChatAgentResponse, ChatAgentResponse]: A tuple containing two
+                ChatAgentResponse: the first struct contains the resulting
+                assistant message, whether the assistant agent terminated the
+                conversation, and any additional assistant information; the
+                second struct contains the resulting user message, whether the
+                user agent terminated the conversation, and any additional user
+                information.
+        """
+        user_response = await self.user_agent.astep(assistant_msg)
+        if user_response.terminated or user_response.msgs is None:
+            return (
+                ChatAgentResponse(msgs=[], terminated=False, info={}),
+                ChatAgentResponse(
+                    msgs=[],
+                    terminated=user_response.terminated,
+                    info=user_response.info,
+                ),
+            )
+        user_msg = self._reduce_message_options(user_response.msgs)
+
+        # To prevent recording the same memory more than once (once in chat
+        # step and once in role play), and the model generates only one
+        # response when multi-response support is enabled.
+        if (
+            'n' in self.user_agent.model_backend.model_config_dict.keys()
+            and self.user_agent.model_backend.model_config_dict['n'] > 1
+        ):
+            self.user_agent.record_message(user_msg)
+
+        assistant_response = await self.assistant_agent.astep(user_msg)
+        if assistant_response.terminated or assistant_response.msgs is None:
+            return (
+                ChatAgentResponse(
+                    msgs=[],
+                    terminated=assistant_response.terminated,
+                    info=assistant_response.info,
+                ),
+                ChatAgentResponse(
+                    msgs=[user_msg], terminated=False, info=user_response.info
+                ),
+            )
+        assistant_msg = self._reduce_message_options(assistant_response.msgs)
+
+        # To prevent recording the same memory more than once (once in chat
+        # step and once in role play), and the model generates only one
+        # response when multi-response support is enabled.
+        if (
+            'n' in self.assistant_agent.model_backend.model_config_dict.keys()
+            and self.assistant_agent.model_backend.model_config_dict['n'] > 1
+        ):
+            self.assistant_agent.record_message(assistant_msg)
+
+        return (
+            ChatAgentResponse(
+                msgs=[assistant_msg],
+                terminated=assistant_response.terminated,
+                info=assistant_response.info,
+            ),
+            ChatAgentResponse(
+                msgs=[user_msg],
+                terminated=user_response.terminated,
+                info=user_response.info,
+            ),
+        )

camel/toolkits/__init__.py

@@ -43,13 +43,21 @@ from .retrieval_toolkit import RetrievalToolkit
 from .notion_toolkit import NotionToolkit
 from .human_toolkit import HumanToolkit
 from .stripe_toolkit import StripeToolkit
-from .video_toolkit import VideoDownloaderToolkit
+from .video_download_toolkit import VideoDownloaderToolkit
 from .dappier_toolkit import DappierToolkit
 from .networkx_toolkit import NetworkXToolkit
 from .semantic_scholar_toolkit import SemanticScholarToolkit
 from .zapier_toolkit import ZapierToolkit
 from .sympy_toolkit import SymPyToolkit
 from .mineru_toolkit import MinerUToolkit
+from .audio_analysis_toolkit import AudioAnalysisToolkit
+from .excel_toolkit import ExcelToolkit
+from .video_analysis_toolkit import VideoAnalysisToolkit
+from .image_analysis_toolkit import ImageAnalysisToolkit
+from .mcp_toolkit import MCPToolkit
+from .web_toolkit import WebToolkit
+from .file_write_toolkit import FileWriteToolkit
+from .terminal_toolkit import TerminalToolkit
 
 
 __all__ = [
@@ -88,4 +96,12 @@ __all__ = [
     'ZapierToolkit',
     'SymPyToolkit',
     'MinerUToolkit',
+    'MCPToolkit',
+    'AudioAnalysisToolkit',
+    'ExcelToolkit',
+    'VideoAnalysisToolkit',
+    'ImageAnalysisToolkit',
+    'WebToolkit',
+    'FileWriteToolkit',
+    'TerminalToolkit',
 ]

camel/toolkits/audio_analysis_toolkit.py

@@ -0,0 +1,238 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+import os
+import uuid
+from typing import List, Optional
+from urllib.parse import urlparse
+
+import requests
+
+from camel.logger import get_logger
+from camel.messages import BaseMessage
+from camel.models import BaseAudioModel, BaseModelBackend, OpenAIAudioModels
+from camel.toolkits.base import BaseToolkit
+from camel.toolkits.function_tool import FunctionTool
+
+logger = get_logger(__name__)
+
+
+def download_file(url: str, cache_dir: str) -> str:
+    r"""Download a file from a URL to a local cache directory.
+
+    Args:
+        url (str): The URL of the file to download.
+        cache_dir (str): The directory to save the downloaded file.
+
+    Returns:
+        str: The path to the downloaded file.
+
+    Raises:
+        Exception: If the download fails.
+    """
+    # Create cache directory if it doesn't exist
+    os.makedirs(cache_dir, exist_ok=True)
+
+    # Extract filename from URL or generate a unique one
+    parsed_url = urlparse(url)
+    filename = os.path.basename(parsed_url.path)
+    if not filename:
+        # Generate a unique filename if none is provided in the URL
+        file_ext = ".mp3"  # Default extension
+        content_type = None
+
+        # Try to get the file extension from the content type
+        try:
+            response = requests.head(url)
+            content_type = response.headers.get('Content-Type', '')
+            if 'audio/wav' in content_type:
+                file_ext = '.wav'
+            elif 'audio/mpeg' in content_type:
+                file_ext = '.mp3'
+            elif 'audio/ogg' in content_type:
+                file_ext = '.ogg'
+        except Exception:
+            pass
+
+        filename = f"{uuid.uuid4()}{file_ext}"
+
+    local_path = os.path.join(cache_dir, filename)
+
+    # Download the file
+    response = requests.get(url, stream=True)
+    response.raise_for_status()
+
+    with open(local_path, 'wb') as f:
+        for chunk in response.iter_content(chunk_size=8192):
+            f.write(chunk)
+
+    logger.debug(f"Downloaded file from {url} to {local_path}")
+    return local_path
+
+
+class AudioAnalysisToolkit(BaseToolkit):
+    r"""A toolkit for audio processing and analysis.
+
+    This class provides methods for processing, transcribing, and extracting
+    information from audio data, including direct question answering about
+    audio content.
+
+    Args:
+        cache_dir (Optional[str]): Directory path for caching downloaded audio
+            files. If not provided, 'tmp/' will be used. (default: :obj:`None`)
+        transcribe_model (Optional[BaseAudioModel]): Model used for audio
+            transcription. If not provided, OpenAIAudioModels will be used.
+            (default: :obj:`None`)
+        audio_reasoning_model (Optional[BaseModelBackend]): Model used for
+            audio reasoning and question answering. If not provided, uses the
+            default model from ChatAgent. (default: :obj:`None`)
+    """
+
+    def __init__(
+        self,
+        cache_dir: Optional[str] = None,
+        transcribe_model: Optional[BaseAudioModel] = None,
+        audio_reasoning_model: Optional[BaseModelBackend] = None,
+    ):
+        self.cache_dir = 'tmp/'
+        if cache_dir:
+            self.cache_dir = cache_dir
+
+        if transcribe_model:
+            self.transcribe_model = transcribe_model
+        else:
+            self.transcribe_model = OpenAIAudioModels()
+            logger.warning(
+                "No audio transcription model provided. "
+                "Using OpenAIAudioModels."
+            )
+
+        from camel.agents import ChatAgent
+
+        if audio_reasoning_model:
+            self.audio_agent = ChatAgent(model=audio_reasoning_model)
+        else:
+            self.audio_agent = ChatAgent()
+            logger.warning(
+                "No audio reasoning model provided. Using default model in"
+                " ChatAgent."
+            )
+
+    def audio2text(self, audio_path: str) -> str:
+        r"""Transcribe audio to text.
+
+        Args:
+            audio_path (str): The path to the audio file or URL.
+
+        Returns:
+            str: The transcribed text.
+        """
+        logger.debug(
+            f"Calling transcribe_audio method for audio file `{audio_path}`."
+        )
+
+        try:
+            audio_transcript = self.transcribe_model.speech_to_text(audio_path)
+            if not audio_transcript:
+                logger.warning("Audio transcription returned empty result")
+                return "No audio transcription available."
+            return audio_transcript
+        except Exception as e:
+            logger.error(f"Audio transcription failed: {e}")
+            return "Audio transcription failed."
+
+    def ask_question_about_audio(self, audio_path: str, question: str) -> str:
+        r"""Ask any question about the audio and get the answer using
+        multimodal model.
+
+        Args:
+            audio_path (str): The path to the audio file.
+            question (str): The question to ask about the audio.
+
+        Returns:
+            str: The answer to the question.
+        """
+
+        logger.debug(
+            f"Calling ask_question_about_audio method for audio file \
+            `{audio_path}` and question `{question}`."
+        )
+
+        parsed_url = urlparse(audio_path)
+        is_url = all([parsed_url.scheme, parsed_url.netloc])
+        local_audio_path = audio_path
+
+        # If the audio is a URL, download it first
+        if is_url:
+            try:
+                local_audio_path = download_file(audio_path, self.cache_dir)
+            except Exception as e:
+                logger.error(f"Failed to download audio file: {e}")
+                return f"Failed to download audio file: {e!s}"
+
+        # Try direct audio question answering first
+        try:
+            # Check if the transcribe_model supports audio_question_answering
+            if hasattr(self.transcribe_model, 'audio_question_answering'):
+                logger.debug("Using direct audio question answering")
+                response = self.transcribe_model.audio_question_answering(
+                    local_audio_path, question
+                )
+                return response
+        except Exception as e:
+            logger.warning(
+                f"Direct audio question answering failed: {e}. "
+                "Falling back to transcription-based approach."
+            )
+
+        # Fallback to transcription-based approach
+        try:
+            transcript = self.audio2text(local_audio_path)
+            reasoning_prompt = f"""
+            <speech_transcription_result>{transcript}</
+            speech_transcription_result>
+
+            Please answer the following question based on the speech 
+            transcription result above:
+            <question>{question}</question>
+            """
+            msg = BaseMessage.make_user_message(
+                role_name="User", content=reasoning_prompt
+            )
+            response = self.audio_agent.step(msg)
+
+            if not response or not response.msgs:
+                logger.error("Model returned empty response")
+                return (
+                    "Failed to generate an answer. "
+                    "The model returned an empty response."
+                )
+
+            answer = response.msgs[0].content
+            return answer
+        except Exception as e:
+            logger.error(f"Audio question answering failed: {e}")
+            return f"Failed to answer question about audio: {e!s}"
+
+    def get_tools(self) -> List[FunctionTool]:
+        r"""Returns a list of FunctionTool objects representing the functions
+            in the toolkit.
+
+        Returns:
+            List[FunctionTool]: A list of FunctionTool objects representing the
+                functions in the toolkit.
+        """
+        return [
+            FunctionTool(self.ask_question_about_audio),
+            FunctionTool(self.audio2text),
+        ]

camel/toolkits/excel_toolkit.py

@@ -0,0 +1,172 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+from typing import List
+
+import pandas as pd
+
+from camel.logger import get_logger
+from camel.toolkits.base import BaseToolkit
+from camel.toolkits.function_tool import FunctionTool
+
+logger = get_logger(__name__)
+
+
+class ExcelToolkit(BaseToolkit):
+    r"""A class representing a toolkit for extract detailed cell information
+    from an Excel file.
+
+    This class provides method for processing docx, pdf, pptx, etc. It cannot
+    process excel files.
+    """
+
+    def _convert_to_markdown(self, df: pd.DataFrame) -> str:
+        r"""Convert DataFrame to Markdown format table.
+
+        Args:
+            df (pd.DataFrame): DataFrame containing the Excel data.
+
+        Returns:
+            str: Markdown formatted table.
+        """
+        from tabulate import tabulate
+
+        md_table = tabulate(df, headers='keys', tablefmt='pipe')
+        return str(md_table)
+
+    def extract_excel_content(self, document_path: str) -> str:
+        r"""Extract detailed cell information from an Excel file, including
+        multiple sheets.
+
+        Args:
+            document_path (str): The path of the Excel file.
+
+        Returns:
+            str: Extracted excel information, including details of each sheet.
+        """
+        from openpyxl import load_workbook
+        from xls2xlsx import XLS2XLSX
+
+        logger.debug(
+            f"Calling extract_excel_content with document_path"
+            f": {document_path}"
+        )
+
+        if not (
+            document_path.endswith("xls")
+            or document_path.endswith("xlsx")
+            or document_path.endswith("csv")
+        ):
+            logger.error("Only xls, xlsx, csv files are supported.")
+            return (
+                f"Failed to process file {document_path}: "
+                f"It is not excel format. Please try other ways."
+            )
+
+        if document_path.endswith("csv"):
+            try:
+                df = pd.read_csv(document_path)
+                md_table = self._convert_to_markdown(df)
+                return f"CSV File Processed:\n{md_table}"
+            except Exception as e:
+                logger.error(f"Failed to process file {document_path}: {e}")
+                return f"Failed to process file {document_path}: {e}"
+
+        if document_path.endswith("xls"):
+            output_path = document_path.replace(".xls", ".xlsx")
+            x2x = XLS2XLSX(document_path)
+            x2x.to_xlsx(output_path)
+            document_path = output_path
+
+        # Load the Excel workbook
+        wb = load_workbook(document_path, data_only=True)
+        sheet_info_list = []
+
+        # Iterate through all sheets
+        for sheet in wb.sheetnames:
+            ws = wb[sheet]
+            cell_info_list = []
+
+            for row in ws.iter_rows():
+                for cell in row:
+                    row_num = cell.row
+                    col_letter = cell.column_letter
+
+                    cell_value = cell.value
+
+                    font_color = None
+                    if (
+                        cell.font
+                        and cell.font.color
+                        and "rgb=None" not in str(cell.font.color)
+                    ):  # Handle font color
+                        font_color = cell.font.color.rgb
+
+                    fill_color = None
+                    if (
+                        cell.fill
+                        and cell.fill.fgColor
+                        and "rgb=None" not in str(cell.fill.fgColor)
+                    ):  # Handle fill color
+                        fill_color = cell.fill.fgColor.rgb
+
+                    cell_info_list.append(
+                        {
+                            "index": f"{row_num}{col_letter}",
+                            "value": cell_value,
+                            "font_color": font_color,
+                            "fill_color": fill_color,
+                        }
+                    )
+
+            # Convert the sheet to a DataFrame and then to markdown
+            sheet_df = pd.read_excel(
+                document_path, sheet_name=sheet, engine='openpyxl'
+            )
+            markdown_content = self._convert_to_markdown(sheet_df)
+
+            # Collect all information for the sheet
+            sheet_info = {
+                "sheet_name": sheet,
+                "cell_info_list": cell_info_list,
+                "markdown_content": markdown_content,
+            }
+            sheet_info_list.append(sheet_info)
+
+        result_str = ""
+        for sheet_info in sheet_info_list:
+            result_str += f"""
+            Sheet Name: {sheet_info['sheet_name']}
+            Cell information list:
+            {sheet_info['cell_info_list']}
+            
+            Markdown View of the content:
+            {sheet_info['markdown_content']}
+            
+            {'-'*40}
+            """
+
+        return result_str
+
+    def get_tools(self) -> List[FunctionTool]:
+        r"""Returns a list of FunctionTool objects representing the functions
+        in the toolkit.
+
+        Returns:
+            List[FunctionTool]: A list of FunctionTool objects representing
+                the functions in the toolkit.
+        """
+        return [
+            FunctionTool(self.extract_excel_content),
+        ]