camel-ai 0.2.21__py3-none-any.whl → 0.2.23__py3-none-any.whl

camel/retrievers/hybrid_retrival.py

@@ -0,0 +1,237 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+from typing import Any, Collection, Dict, List, Optional, Sequence, Union
+
+import numpy as np
+
+from camel.embeddings import BaseEmbedding
+from camel.retrievers import BaseRetriever, BM25Retriever, VectorRetriever
+from camel.storages import BaseVectorStorage
+
+
+class HybridRetriever(BaseRetriever):
+    def __init__(
+        self,
+        embedding_model: Optional[BaseEmbedding] = None,
+        vector_storage: Optional[BaseVectorStorage] = None,
+    ) -> None:
+        r"""Initializes the HybridRetriever with optional embedding model and
+        vector storage.
+
+        Args:
+            embedding_model (Optional[BaseEmbedding]): An optional embedding
+                model used by the VectorRetriever. Defaults to None.
+            vector_storage (Optional[BaseVectorStorage]): An optional vector
+                storage used by the VectorRetriever. Defaults to None.
+        """
+        self.vr = VectorRetriever(embedding_model, vector_storage)
+        self.bm25 = BM25Retriever()
+
+    def process(self, content_input_path: str) -> None:
+        r"""Processes the content input path for both vector and BM25
+        retrievers.
+
+        Args:
+            content_input_path (str): File path or URL of the content to be
+                processed.
+
+        Raises:
+            ValueError: If the content_input_path is empty.
+        """
+        if not content_input_path:
+            raise ValueError("content_input_path cannot be empty.")
+
+        self.content_input_path = content_input_path
+        self.vr.process(content=self.content_input_path)
+        self.bm25.process(content_input_path=self.content_input_path)
+
+    def _sort_rrf_scores(
+        self,
+        vector_retriever_results: List[Dict[str, Any]],
+        bm25_retriever_results: List[Dict[str, Any]],
+        top_k: int,
+        vector_weight: float,
+        bm25_weight: float,
+        rank_smoothing_factor: float,
+    ) -> List[Dict[str, Union[str, float]]]:
+        r"""Sorts and combines results from vector and BM25 retrievers using
+        Reciprocal Rank Fusion (RRF).
+
+        Args:
+            vector_retriever_results: A list of dictionaries containing the
+                results from the vector retriever, where each dictionary
+                contains a 'text' entry.
+            bm25_retriever_results: A list of dictionaries containing the
+                results from the BM25 retriever, where each dictionary
+                contains a 'text' entry.
+            top_k: The number of top results to return after sorting by RRF
+                score.
+            vector_weight: The weight to assign to the vector retriever
+                results in the RRF calculation.
+            bm25_weight: The weight to assign to the BM25 retriever results in
+                the RRF calculation.
+            rank_smoothing_factor: A hyperparameter for the RRF calculation
+                that helps smooth the rank positions.
+
+        Returns:
+            List[Dict[str, Union[str, float]]]: A list of dictionaries
+            representing the sorted results. Each dictionary contains the
+            'text'from the retrieved items and their corresponding 'rrf_score'.
+
+        Raises:
+            ValueError: If any of the input weights are negative.
+
+        References:
+            https://medium.com/@devalshah1619/mathematical-intuition-behind-reciprocal-rank-fusion-rrf-explained-in-2-mins-002df0cc5e2a
+            https://colab.research.google.com/drive/1iwVJrN96fiyycxN1pBqWlEr_4EPiGdGy#scrollTo=0qh83qGV2dY8
+        """
+        text_to_id = {}
+        id_to_info = {}
+        current_id = 1
+
+        # Iterate over vector_retriever_results
+        for rank, result in enumerate(vector_retriever_results, start=1):
+            text = result.get('text', None)  # type: ignore[attr-defined]
+            if text is None:
+                raise KeyError("Each result must contain a 'text' key")
+
+            if text not in text_to_id:
+                text_to_id[text] = current_id
+                id_to_info[current_id] = {'text': text, 'vector_rank': rank}
+                current_id += 1
+            else:
+                id_to_info[text_to_id[text]]['vector_rank'] = rank
+
+        # Iterate over bm25_retriever_results
+        for rank, result in enumerate(bm25_retriever_results, start=1):
+            text = result['text']
+            if text not in text_to_id:
+                text_to_id[text] = current_id
+                id_to_info[current_id] = {'text': text, 'bm25_rank': rank}
+                current_id += 1
+            else:
+                id_to_info[text_to_id[text]].setdefault('bm25_rank', rank)
+
+        vector_ranks = np.array(
+            [
+                info.get('vector_rank', float('inf'))
+                for info in id_to_info.values()
+            ]
+        )
+        bm25_ranks = np.array(
+            [
+                info.get('bm25_rank', float('inf'))
+                for info in id_to_info.values()
+            ]
+        )
+
+        # Calculate RRF scores
+        vector_rrf_scores = vector_weight / (
+            rank_smoothing_factor + vector_ranks
+        )
+        bm25_rrf_scores = bm25_weight / (rank_smoothing_factor + bm25_ranks)
+        rrf_scores = vector_rrf_scores + bm25_rrf_scores
+
+        for idx, (_, info) in enumerate(id_to_info.items()):
+            info['rrf_score'] = rrf_scores[idx]
+        sorted_results = sorted(
+            id_to_info.values(), key=lambda x: x['rrf_score'], reverse=True
+        )
+        return sorted_results[:top_k]
+
+    def query(
+        self,
+        query: str,
+        top_k: int = 20,
+        vector_weight: float = 0.8,
+        bm25_weight: float = 0.2,
+        rank_smoothing_factor: int = 60,
+        vector_retriever_top_k: int = 50,
+        vector_retriever_similarity_threshold: float = 0.5,
+        bm25_retriever_top_k: int = 50,
+        return_detailed_info: bool = False,
+    ) -> Union[
+        dict[str, Sequence[Collection[str]]],
+        dict[str, Sequence[Union[str, float]]],
+    ]:
+        r"""Executes a hybrid retrieval query using both vector and BM25
+        retrievers.
+
+        Args:
+            query (str): The search query.
+            top_k (int): Number of top results to return (default 20).
+            vector_weight (float): Weight for vector retriever results in RRF.
+            bm25_weight (float): Weight for BM25 retriever results in RRF.
+            rank_smoothing_factor (int): RRF hyperparameter for rank smoothing.
+            vector_retriever_top_k (int): Top results from vector retriever.
+            vector_retriever_similarity_threshold (float): Similarity
+                threshold for vector retriever.
+            bm25_retriever_top_k (int): Top results from BM25 retriever.
+            return_detailed_info (bool): Return detailed info if True.
+
+        Returns:
+            Union[
+                dict[str, Sequence[Collection[str]]],
+                dict[str, Sequence[Union[str, float]]]
+            ]: By default, returns only the text information. If
+                `return_detailed_info` is `True`, return detailed information
+                including rrf scores.
+        """
+        if top_k > max(vector_retriever_top_k, bm25_retriever_top_k):
+            raise ValueError(
+                "top_k needs to be less than or equal to the "
+                "maximum value among vector_retriever_top_k and "
+                "bm25_retriever_top_k."
+            )
+        if vector_weight < 0 or bm25_weight < 0:
+            raise ValueError(
+                "Neither `vector_weight` nor `bm25_weight` can be negative."
+            )
+
+        vr_raw_results: List[Dict[str, Any]] = self.vr.query(
+            query=query,
+            top_k=vector_retriever_top_k,
+            similarity_threshold=vector_retriever_similarity_threshold,
+        )
+        # if the number of results is less than top_k, return all results
+        with_score = [
+            info for info in vr_raw_results if 'similarity score' in info
+        ]
+        vector_retriever_results = sorted(
+            with_score, key=lambda x: x['similarity score'], reverse=True
+        )
+
+        bm25_retriever_results = self.bm25.query(
+            query=query,
+            top_k=bm25_retriever_top_k,
+        )
+
+        all_retrieved_info = self._sort_rrf_scores(
+            vector_retriever_results,
+            bm25_retriever_results,
+            top_k,
+            vector_weight,
+            bm25_weight,
+            rank_smoothing_factor,
+        )
+
+        retrieved_info = {
+            "Original Query": query,
+            "Retrieved Context": (
+                all_retrieved_info
+                if return_detailed_info
+                else [item['text'] for item in all_retrieved_info]  # type: ignore[misc]
+            ),
+        }
+        return retrieved_info

camel/toolkits/__init__.py

@@ -43,10 +43,19 @@ from .retrieval_toolkit import RetrievalToolkit
 from .notion_toolkit import NotionToolkit
 from .human_toolkit import HumanToolkit
 from .stripe_toolkit import StripeToolkit
-from .video_toolkit import VideoDownloaderToolkit
+from .video_download_toolkit import VideoDownloaderToolkit
 from .dappier_toolkit import DappierToolkit
-from .sympy_toolkit import SymPyToolkit
+from .networkx_toolkit import NetworkXToolkit
 from .semantic_scholar_toolkit import SemanticScholarToolkit
+from .zapier_toolkit import ZapierToolkit
+from .sympy_toolkit import SymPyToolkit
+from .mineru_toolkit import MinerUToolkit
+from .audio_analysis_toolkit import AudioAnalysisToolkit
+from .excel_toolkit import ExcelToolkit
+from .video_analysis_toolkit import VideoAnalysisToolkit
+from .image_analysis_toolkit import ImageAnalysisToolkit
+from .web_toolkit import WebToolkit
+
 
 __all__ = [
     'BaseToolkit',
@@ -79,6 +88,14 @@ __all__ = [
     'MeshyToolkit',
     'OpenBBToolkit',
     'DappierToolkit',
-    'SymPyToolkit',
+    'NetworkXToolkit',
     'SemanticScholarToolkit',
+    'ZapierToolkit',
+    'SymPyToolkit',
+    'MinerUToolkit',
+    'AudioAnalysisToolkit',
+    'ExcelToolkit',
+    'VideoAnalysisToolkit',
+    'ImageAnalysisToolkit',
+    'WebToolkit',
 ]

camel/toolkits/arxiv_toolkit.py

@@ -28,8 +28,9 @@ class ArxivToolkit(BaseToolkit):
     """
 
     @dependencies_required('arxiv')
-    def __init__(self) -> None:
+    def __init__(self, timeout: Optional[float] = None) -> None:
         r"""Initializes the ArxivToolkit and sets up the arXiv client."""
+        super().__init__(timeout=timeout)
         import arxiv
 
         self.client = arxiv.Client()

camel/toolkits/ask_news_toolkit.py

@@ -62,11 +62,13 @@ class AskNewsToolkit(BaseToolkit):
     based on user queries using the AskNews API.
     """
 
-    def __init__(self):
+    def __init__(self, timeout: Optional[float] = None):
         r"""Initialize the AskNewsToolkit with API clients.The API keys and
         credentials are retrieved from environment variables.
         """
-        from asknews_sdk import AskNewsSDK
+        super().__init__(timeout=timeout)
+
+        from asknews_sdk import AskNewsSDK  # type: ignore[import-not-found]
 
         client_id = os.environ.get("ASKNEWS_CLIENT_ID")
         client_secret = os.environ.get("ASKNEWS_CLIENT_SECRET")

camel/toolkits/audio_analysis_toolkit.py

@@ -0,0 +1,238 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+import os
+import uuid
+from typing import List, Optional
+from urllib.parse import urlparse
+
+import requests
+
+from camel.logger import get_logger
+from camel.messages import BaseMessage
+from camel.models import BaseAudioModel, BaseModelBackend, OpenAIAudioModels
+from camel.toolkits.base import BaseToolkit
+from camel.toolkits.function_tool import FunctionTool
+
+logger = get_logger(__name__)
+
+
+def download_file(url: str, cache_dir: str) -> str:
+    r"""Download a file from a URL to a local cache directory.
+
+    Args:
+        url (str): The URL of the file to download.
+        cache_dir (str): The directory to save the downloaded file.
+
+    Returns:
+        str: The path to the downloaded file.
+
+    Raises:
+        Exception: If the download fails.
+    """
+    # Create cache directory if it doesn't exist
+    os.makedirs(cache_dir, exist_ok=True)
+
+    # Extract filename from URL or generate a unique one
+    parsed_url = urlparse(url)
+    filename = os.path.basename(parsed_url.path)
+    if not filename:
+        # Generate a unique filename if none is provided in the URL
+        file_ext = ".mp3"  # Default extension
+        content_type = None
+
+        # Try to get the file extension from the content type
+        try:
+            response = requests.head(url)
+            content_type = response.headers.get('Content-Type', '')
+            if 'audio/wav' in content_type:
+                file_ext = '.wav'
+            elif 'audio/mpeg' in content_type:
+                file_ext = '.mp3'
+            elif 'audio/ogg' in content_type:
+                file_ext = '.ogg'
+        except Exception:
+            pass
+
+        filename = f"{uuid.uuid4()}{file_ext}"
+
+    local_path = os.path.join(cache_dir, filename)
+
+    # Download the file
+    response = requests.get(url, stream=True)
+    response.raise_for_status()
+
+    with open(local_path, 'wb') as f:
+        for chunk in response.iter_content(chunk_size=8192):
+            f.write(chunk)
+
+    logger.debug(f"Downloaded file from {url} to {local_path}")
+    return local_path
+
+
+class AudioAnalysisToolkit(BaseToolkit):
+    r"""A toolkit for audio processing and analysis.
+
+    This class provides methods for processing, transcribing, and extracting
+    information from audio data, including direct question answering about
+    audio content.
+
+    Args:
+        cache_dir (Optional[str]): Directory path for caching downloaded audio
+            files. If not provided, 'tmp/' will be used. (default: :obj:`None`)
+        transcribe_model (Optional[BaseAudioModel]): Model used for audio
+            transcription. If not provided, OpenAIAudioModels will be used.
+            (default: :obj:`None`)
+        audio_reasoning_model (Optional[BaseModelBackend]): Model used for
+            audio reasoning and question answering. If not provided, uses the
+            default model from ChatAgent. (default: :obj:`None`)
+    """
+
+    def __init__(
+        self,
+        cache_dir: Optional[str] = None,
+        transcribe_model: Optional[BaseAudioModel] = None,
+        audio_reasoning_model: Optional[BaseModelBackend] = None,
+    ):
+        self.cache_dir = 'tmp/'
+        if cache_dir:
+            self.cache_dir = cache_dir
+
+        if transcribe_model:
+            self.transcribe_model = transcribe_model
+        else:
+            self.transcribe_model = OpenAIAudioModels()
+            logger.warning(
+                "No audio transcription model provided. "
+                "Using OpenAIAudioModels."
+            )
+
+        from camel.agents import ChatAgent
+
+        if audio_reasoning_model:
+            self.audio_agent = ChatAgent(model=audio_reasoning_model)
+        else:
+            self.audio_agent = ChatAgent()
+            logger.warning(
+                "No audio reasoning model provided. Using default model in"
+                " ChatAgent."
+            )
+
+    def audio2text(self, audio_path: str) -> str:
+        r"""Transcribe audio to text.
+
+        Args:
+            audio_path (str): The path to the audio file or URL.
+
+        Returns:
+            str: The transcribed text.
+        """
+        logger.debug(
+            f"Calling transcribe_audio method for audio file `{audio_path}`."
+        )
+
+        try:
+            audio_transcript = self.transcribe_model.speech_to_text(audio_path)
+            if not audio_transcript:
+                logger.warning("Audio transcription returned empty result")
+                return "No audio transcription available."
+            return audio_transcript
+        except Exception as e:
+            logger.error(f"Audio transcription failed: {e}")
+            return "Audio transcription failed."
+
+    def ask_question_about_audio(self, audio_path: str, question: str) -> str:
+        r"""Ask any question about the audio and get the answer using
+        multimodal model.
+
+        Args:
+            audio_path (str): The path to the audio file.
+            question (str): The question to ask about the audio.
+
+        Returns:
+            str: The answer to the question.
+        """
+
+        logger.debug(
+            f"Calling ask_question_about_audio method for audio file \
+            `{audio_path}` and question `{question}`."
+        )
+
+        parsed_url = urlparse(audio_path)
+        is_url = all([parsed_url.scheme, parsed_url.netloc])
+        local_audio_path = audio_path
+
+        # If the audio is a URL, download it first
+        if is_url:
+            try:
+                local_audio_path = download_file(audio_path, self.cache_dir)
+            except Exception as e:
+                logger.error(f"Failed to download audio file: {e}")
+                return f"Failed to download audio file: {e!s}"
+
+        # Try direct audio question answering first
+        try:
+            # Check if the transcribe_model supports audio_question_answering
+            if hasattr(self.transcribe_model, 'audio_question_answering'):
+                logger.debug("Using direct audio question answering")
+                response = self.transcribe_model.audio_question_answering(
+                    local_audio_path, question
+                )
+                return response
+        except Exception as e:
+            logger.warning(
+                f"Direct audio question answering failed: {e}. "
+                "Falling back to transcription-based approach."
+            )
+
+        # Fallback to transcription-based approach
+        try:
+            transcript = self.audio2text(local_audio_path)
+            reasoning_prompt = f"""
+            <speech_transcription_result>{transcript}</
+            speech_transcription_result>
+
+            Please answer the following question based on the speech 
+            transcription result above:
+            <question>{question}</question>
+            """
+            msg = BaseMessage.make_user_message(
+                role_name="User", content=reasoning_prompt
+            )
+            response = self.audio_agent.step(msg)
+
+            if not response or not response.msgs:
+                logger.error("Model returned empty response")
+                return (
+                    "Failed to generate an answer. "
+                    "The model returned an empty response."
+                )
+
+            answer = response.msgs[0].content
+            return answer
+        except Exception as e:
+            logger.error(f"Audio question answering failed: {e}")
+            return f"Failed to answer question about audio: {e!s}"
+
+    def get_tools(self) -> List[FunctionTool]:
+        r"""Returns a list of FunctionTool objects representing the functions
+            in the toolkit.
+
+        Returns:
+            List[FunctionTool]: A list of FunctionTool objects representing the
+                functions in the toolkit.
+        """
+        return [
+            FunctionTool(self.ask_question_about_audio),
+            FunctionTool(self.audio2text),
+        ]

camel/toolkits/base.py

@@ -12,14 +12,33 @@
 # limitations under the License.
 # ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
 
-from typing import List
+from typing import List, Optional
 
 from camel.toolkits import FunctionTool
-from camel.utils import AgentOpsMeta
+from camel.utils import AgentOpsMeta, with_timeout
 
 
 class BaseToolkit(metaclass=AgentOpsMeta):
-    r"""Base class for toolkits."""
+    r"""Base class for toolkits.
+
+    Args:
+        timeout (Optional[float]): The timeout for the toolkit.
+    """
+
+    timeout: Optional[float] = None
+
+    def __init__(self, timeout: Optional[float] = None):
+        # check if timeout is a positive number
+        if timeout is not None and timeout <= 0:
+            raise ValueError("Timeout must be a positive number.")
+        self.timeout = timeout
+
+    # Add timeout to all callable methods in the toolkit
+    def __init_subclass__(cls, **kwargs):
+        super().__init_subclass__(**kwargs)
+        for attr_name, attr_value in cls.__dict__.items():
+            if callable(attr_value) and not attr_name.startswith("__"):
+                setattr(cls, attr_name, with_timeout(attr_value))
 
     def get_tools(self) -> List[FunctionTool]:
         r"""Returns a list of FunctionTool objects representing the

camel/toolkits/code_execution.py

@@ -48,7 +48,9 @@ class CodeExecutionToolkit(BaseToolkit):
         unsafe_mode: bool = False,
         import_white_list: Optional[List[str]] = None,
         require_confirm: bool = False,
+        timeout: Optional[float] = None,
     ) -> None:
+        super().__init__(timeout=timeout)
         self.verbose = verbose
         self.unsafe_mode = unsafe_mode
         self.import_white_list = import_white_list or list()

camel/toolkits/dappier_toolkit.py

@@ -33,10 +33,11 @@ class DappierToolkit(BaseToolkit):
             (None, "DAPPIER_API_KEY"),
         ]
     )
-    def __init__(self):
+    def __init__(self, timeout: Optional[float] = None):
         r"""Initialize the DappierTookit with API clients.The API keys and
         credentials are retrieved from environment variables.
         """
+        super().__init__(timeout=timeout)
         from dappier import Dappier
 
         dappier_api_key = os.environ.get("DAPPIER_API_KEY")