camel-ai 0.2.21__py3-none-any.whl → 0.2.23__py3-none-any.whl

camel/toolkits/mineru_toolkit.py

@@ -0,0 +1,178 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+from typing import Dict, List, Optional
+
+from camel.loaders.mineru_extractor import MinerU
+from camel.toolkits.base import BaseToolkit
+from camel.toolkits.function_tool import FunctionTool
+from camel.utils import api_keys_required
+
+
+class MinerUToolkit(BaseToolkit):
+    r"""Toolkit for extracting and processing document content
+        using MinerU API.
+
+    Provides comprehensive document processing capabilities including content
+    extraction from URLs and files, with support for OCR, formula recognition,
+    and table detection through the MinerU API service.
+
+    Note:
+        - Maximum file size: 200MB per file
+        - Maximum pages: 600 pages per file
+        - Daily quota: 2000 pages for high-priority parsing
+        - Network restrictions may affect certain URLs (e.g., GitHub, AWS)
+    """
+
+    @api_keys_required(
+        [
+            (None, "MINERU_API_KEY"),
+        ]
+    )
+    def __init__(
+        self,
+        api_key: Optional[str] = None,
+        api_url: Optional[str] = "https://mineru.net/api/v4",
+        is_ocr: bool = False,
+        enable_formula: bool = False,
+        enable_table: bool = True,
+        layout_model: str = "doclayout_yolo",
+        language: str = "en",
+        wait: bool = True,
+        timeout: float = 300,
+    ) -> None:
+        r"""Initialize the MinerU document processing toolkit.
+
+        Args:
+            api_key (Optional[str]): Authentication key for MinerU API access.
+                If not provided, uses MINERU_API_KEY environment variable.
+                (default: :obj:`None`)
+            api_url (Optional[str]): Base endpoint URL for MinerU API service.
+                (default: :obj:`"https://mineru.net/api/v4"`)
+            is_ocr (bool): Enable Optical Character Recognition for image-based
+                text extraction. (default: :obj:`False`)
+            enable_formula (bool): Enable mathematical formula detection and
+                recognition. (default: :obj:`False`)
+            enable_table (bool): Enable table structure detection and
+                extraction. (default: :obj:`True`)
+            layout_model (str): Document layout analysis model selection.
+                Available options: 'doclayout_yolo', 'layoutlmv3'.
+                (default: :obj:`"doclayout_yolo"`)
+            language (str): Primary language of the document for processing.
+                (default: :obj:`"en"`)
+            wait (bool): Block execution until processing completion.
+                (default: :obj:`True`)
+            timeout (float): Maximum duration in seconds to wait for task
+                completion. (default: :obj:`300`)
+        """
+        self.client = MinerU(
+            api_key=api_key,
+            api_url=api_url,
+            is_ocr=is_ocr,
+            enable_formula=enable_formula,
+            enable_table=enable_table,
+            layout_model=layout_model,
+            language=language,
+        )
+        self.wait = wait
+        self.timeout = timeout
+
+    def extract_from_urls(
+        self,
+        urls: str | List[str],
+    ) -> Dict:
+        r"""Process and extract content from one or multiple URLs.
+
+        Args:
+            urls (str | List[str]): Target URL or list of URLs for content
+                extraction. Supports both single URL string and multiple URLs
+                in a list.
+
+        Returns:
+            Dict: Response containing either completed task results when wait
+                is True, or task/batch identifiers for status tracking when
+                wait is False.
+        """
+        if isinstance(urls, str):
+            # Single URL case
+            response = self.client.extract_url(url=urls)
+
+            if self.wait:
+                return self.client.wait_for_completion(
+                    response['task_id'],
+                    timeout=self.timeout,  # type: ignore[arg-type]
+                )
+            return response
+        else:
+            # Multiple URLs case
+            files: List[Dict[str, str | bool]] = [
+                {"url": str(url)} for url in urls
+            ]
+            batch_id = self.client.batch_extract_urls(files=files)
+
+            if self.wait:
+                return self.client.wait_for_completion(
+                    batch_id,
+                    is_batch=True,
+                    timeout=self.timeout if self.timeout > 300 else 600,  # type: ignore[arg-type,operator]
+                )
+            return {"batch_id": batch_id}
+
+    def get_task_status(self, task_id: str) -> Dict:
+        r"""Retrieve current status of an individual extraction task.
+
+        Args:
+            task_id (str): Unique identifier for the extraction task to check.
+
+        Returns:
+            Dict: Status information and results (if task is completed) for
+                the specified task.
+
+        Note:
+            This is a low-level status checking method. For most use cases,
+            prefer using extract_from_url with wait=True for automatic
+            completion handling.
+        """
+        return self.client.get_task_status(task_id)
+
+    def get_batch_status(self, batch_id: str) -> Dict:
+        r"""Retrieve current status of a batch extraction task.
+
+        Args:
+            batch_id (str): Unique identifier for the batch extraction task
+                to check.
+
+        Returns:
+            Dict: Comprehensive status information and results for all files
+                in the batch task.
+
+        Note:
+            This is a low-level status checking method. For most use cases,
+            prefer using batch_extract_from_urls with wait=True for automatic
+            completion handling.
+        """
+        return self.client.get_batch_status(batch_id)
+
+    def get_tools(self) -> List[FunctionTool]:
+        r"""Retrieve available toolkit functions as FunctionTool objects.
+
+        Returns:
+            List[FunctionTool]: Collection of FunctionTool objects representing
+                the available document processing functions in this toolkit.
+        """
+        return [
+            FunctionTool(self.extract_from_urls),
+            FunctionTool(self.get_task_status),
+            FunctionTool(self.get_batch_status),
+        ]

camel/toolkits/networkx_toolkit.py

@@ -0,0 +1,240 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+import json
+from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Union
+
+from camel.logger import get_logger
+from camel.toolkits import FunctionTool
+from camel.toolkits.base import BaseToolkit
+
+logger = get_logger(__name__)
+
+
+class NetworkXToolkit(BaseToolkit):
+    _nx = None  # Class variable to store the networkx module
+
+    @classmethod
+    def _get_nx(cls):
+        r"""Lazily import networkx module when needed."""
+        if cls._nx is None:
+            import networkx
+
+            cls._nx = networkx
+        return cls._nx
+
+    def __init__(
+        self,
+        graph_type: Literal[
+            'graph', 'digraph', 'multigraph', 'multidigraph'
+        ] = 'graph',
+    ):
+        r"""Initializes the NetworkX graph client.
+
+        Args:
+            graph_type (Literal['graph', 'digraph', 'multigraph',
+            'multidigraph']):
+                Type of graph to create. Options are:
+                - 'graph': Undirected graph
+                - 'digraph': Directed graph
+                - 'multigraph': Undirected graph with parallel edges
+                - 'multidigraph': Directed graph with parallel edges
+                (default: :obj:`'graph'`)
+        """
+        nx = self._get_nx()
+        graph_types = {
+            'graph': nx.Graph,
+            'digraph': nx.DiGraph,
+            'multigraph': nx.MultiGraph,
+            'multidigraph': nx.MultiDiGraph,
+        }
+        graph_class = graph_types.get(graph_type.lower())
+        if graph_class is None:
+            raise ValueError(
+                f"Invalid graph type: {graph_type}. Must be one "
+                f"of: {list(graph_types.keys())}"
+            )
+
+        self.graph = graph_class()
+        logger.info(f"Initialized NetworkX {graph_type} instance.")
+
+    def add_node(self, node_id: str, **attributes: Any) -> None:
+        r"""Adds a node to the graph.
+
+        Args:
+            node_id (str): The ID of the node.
+            attributes (dict): Additional node attributes.
+        """
+        logger.info(f"Adding node: {node_id}, attributes: {attributes}")
+        self.graph.add_node(node_id, **attributes)
+
+    def add_edge(self, source: str, target: str, **attributes: Any) -> None:
+        r"""Adds an edge to the graph.
+
+        Args:
+            source (str): Source node ID.
+            target (str): Target node ID.
+            attributes (dict): Additional edge attributes.
+        """
+        logger.info(
+            f"Adding edge: {source} -> {target}, attributes: {attributes}"
+        )
+        self.graph.add_edge(source, target, **attributes)
+
+    def get_nodes(self) -> List[str]:
+        r"""Returns all nodes in the graph.
+
+        Returns:
+            List[str]: A list of node IDs.
+        """
+        logger.info("Fetching all nodes.")
+        return list(self.graph.nodes)
+
+    def get_edges(self) -> List[Tuple[str, str]]:
+        r"""Returns all edges in the graph.
+
+        Returns:
+            List[Tuple[str, str]]: A list of edges as (source, target).
+        """
+        logger.info("Fetching all edges.")
+        return list(self.graph.edges)
+
+    def get_shortest_path(
+        self,
+        source: str,
+        target: str,
+        weight: Optional[Union[str, Callable]] = None,
+        method: Literal['dijkstra', 'bellman-ford'] = 'dijkstra',
+    ) -> List[str]:
+        r"""Finds the shortest path between two nodes.
+
+        Args:
+            source (str): The source node ID.
+            target (str): The target node ID.
+            weight (None, str or function, optional): Edge weights/distances.
+                If None, every edge has weight/distance/cost 1.
+                If string, use this edge attribute as the edge weight.
+                If function, the weight of an edge is the value returned by
+                the function. The function must accept three positional
+                arguments: the two endpoints and the edge attribute
+                dictionary. (default: :obj:`None`)
+            method (Literal['dijkstra', 'bellman-ford'], optional): Algorithm
+                to compute the path. Ignored if weight is None. (default:
+                :obj:`'dijkstra'`)
+
+        Returns:
+            List[str]: A list of nodes in the shortest path.
+        """
+        logger.info(
+            f"Finding shortest path from '{source}' to '{target}' "
+            f"using {method} algorithm"
+        )
+        try:
+            nx = self._get_nx()
+            path = nx.shortest_path(
+                self.graph,
+                source=source,
+                target=target,
+                weight=weight,
+                method=method,
+            )
+            logger.debug(f"Found path: {' -> '.join(path)}")
+            return path
+        except nx.NetworkXNoPath:
+            error_msg = f"No path exists between '{source}' and '{target}'"
+            logger.error(error_msg)
+            return [error_msg]
+        except nx.NodeNotFound as e:
+            error_msg = f"Node not found in graph: {e!s}"
+            logger.error(error_msg)
+            return [error_msg]
+
+    def compute_centrality(self) -> Dict[str, float]:
+        r"""Computes centrality measures for the graph.
+
+        Returns:
+            Dict[str, float]: Centrality values for each node.
+        """
+        logger.info("Computing centrality measures.")
+        nx = self._get_nx()
+        return nx.degree_centrality(self.graph)
+
+    def serialize_graph(self) -> str:
+        r"""Serializes the graph to a JSON string.
+
+        Returns:
+            str: The serialized graph in JSON format.
+        """
+        logger.info("Serializing the graph.")
+        nx = self._get_nx()
+        return json.dumps(nx.node_link_data(self.graph))
+
+    def deserialize_graph(self, data: str) -> None:
+        r"""Loads a graph from a serialized JSON string.
+
+        Args:
+            data (str): The JSON string representing the graph.
+        """
+        logger.info("Deserializing graph from JSON data.")
+        nx = self._get_nx()
+        self.graph = nx.node_link_graph(json.loads(data))
+
+    def export_to_file(self, file_path: str) -> None:
+        r"""Exports the graph to a file in JSON format.
+
+        Args:
+            file_path (str): The file path to save the graph.
+        """
+        logger.info(f"Exporting graph to file: {file_path}")
+        nx = self._get_nx()
+        with open(file_path, "w") as file:
+            json.dump(nx.node_link_data(self.graph), file)
+
+    def import_from_file(self, file_path: str) -> None:
+        r"""Imports a graph from a JSON file.
+
+        Args:
+            file_path (str): The file path to load the graph from.
+        """
+        logger.info(f"Importing graph from file: {file_path}")
+        nx = self._get_nx()
+        with open(file_path, "r") as file:
+            self.graph = nx.node_link_graph(json.load(file))
+
+    def clear_graph(self) -> None:
+        r"""Clears the current graph."""
+        logger.info("Clearing the graph.")
+        self.graph.clear()
+
+    def get_tools(self) -> List[FunctionTool]:
+        r"""Returns a list of FunctionTool objects representing the
+        functions in the toolkit.
+
+        Returns:
+            List[FunctionTool]: A list of FunctionTool objects for the
+                toolkit methods.
+        """
+        return [
+            FunctionTool(self.add_edge),
+            FunctionTool(self.add_node),
+            FunctionTool(self.clear_graph),
+            FunctionTool(self.compute_centrality),
+            FunctionTool(self.deserialize_graph),
+            FunctionTool(self.export_to_file),
+            FunctionTool(self.get_edges),
+            FunctionTool(self.get_nodes),
+            FunctionTool(self.import_from_file),
+            FunctionTool(self.serialize_graph),
+            FunctionTool(self.get_shortest_path),
+        ]

camel/toolkits/notion_toolkit.py

@@ -79,6 +79,7 @@ class NotionToolkit(BaseToolkit):
     def __init__(
         self,
         notion_token: Optional[str] = None,
+        timeout: Optional[float] = None,
     ) -> None:
         r"""Initializes the NotionToolkit.
 
@@ -86,6 +87,7 @@ class NotionToolkit(BaseToolkit):
             notion_token (Optional[str], optional): The optional notion_token
                 used to interact with notion APIs.(default: :obj:`None`)
         """
+        super().__init__(timeout=timeout)
         from notion_client import Client
 
         self.notion_token = notion_token or os.environ.get("NOTION_TOKEN")

camel/toolkits/openbb_toolkit.py

@@ -37,15 +37,16 @@ class OpenBBToolkit(BaseToolkit):
             (None, "OPENBB_TOKEN"),
         ]
     )
-    def __init__(self) -> None:
+    def __init__(self, timeout: Optional[float] = None) -> None:
         r"""Initialize the OpenBBToolkit.
 
         This method sets up the OpenBB client and initializes the OpenBB
         Hub account system.
         """
+        super().__init__(timeout=timeout)
         import os
 
-        from openbb import obb
+        from openbb import obb  # type: ignore[import-not-found]
 
         self.client = obb
         # Initialize OpenBB Hub account with access token