aip-agents-binary 0.6.4__py3-none-any.whl

aip_agents/tools/document_loader/excel_reader_tool.py

@@ -0,0 +1,171 @@
+"""Tool to read and extract content from Excel files using gllm_docproc.
+
+This tool uses the gllm_docproc loader pipeline to extract content from Excel files,
+providing a consistent interface with other document reader tools (PDF, DOCX).
+
+Authors:
+    Douglas Raevan Faisal (douglas.raevan.faisal@gdplabs.id)
+"""
+
+import zipfile
+from pathlib import Path
+
+try:
+    from gllm_docproc.loader.xlsx import OpenpyxlLoader
+except ImportError as exc:
+    OpenpyxlLoader = None  # type: ignore[assignment]
+    _DOCPROC_IMPORT_ERROR: Exception | None = exc
+else:
+    _DOCPROC_IMPORT_ERROR = None
+
+from aip_agents.tools.document_loader.base_reader import DOCPROC_MISSING_MESSAGE, BaseDocumentReaderTool
+from aip_agents.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class ExcelReaderTool(BaseDocumentReaderTool):
+    """Tool to read and extract content from Excel files.
+
+    This tool reads Excel files (.xlsx, .xlsm) and extracts their content using
+    the gllm_docproc loader pipeline. The content is formatted as Markdown tables
+    for easy readability.
+
+    Features:
+    - Supports .xlsx and .xlsm formats
+    - Extracts all sheets or specific sheets
+    - Formats output as Markdown tables
+    - Configurable row limits and file size limits
+
+    Examples:
+        >>> tool = ExcelReaderTool()
+        >>> result = tool._run("/tmp/data.xlsx")
+        >>> print(result)
+    """
+
+    name: str = "excel_reader_tool"
+    description: str = (
+        "Read an Excel file and extract its content as Markdown tables. "
+        "Input should be the path to the Excel file (.xlsx or .xlsm format). "
+        "Each sheet will be formatted as a Markdown table with the sheet name as a header."
+    )
+
+    def _setup_loader(self) -> None:
+        """Set up the XLSX loader for the pipeline.
+
+        This method initializes the OpenpyxlLoader which handles extraction
+        of content from Excel files and formatting as Markdown tables.
+        """
+        if OpenpyxlLoader is None:
+            logger.warning(DOCPROC_MISSING_MESSAGE)
+            if _DOCPROC_IMPORT_ERROR is not None:
+                logger.debug("gllm_docproc import failed: %s", _DOCPROC_IMPORT_ERROR)
+            return
+        self.loader.add_loader(OpenpyxlLoader())
+
+    def _run_with_batching(self, file_path: str, batch_size: int) -> str:
+        """Run without batching until Excel splitting is implemented.
+
+        Args:
+            file_path: Path to the Excel file to be processed
+            batch_size: Number of sheets to include in each batch (not used currently)
+
+        Returns:
+            Extracted text content from the document
+        """
+        # TODO: implement Excel batching with real splits before enabling batching flow.
+        return self._run_standard(file_path)
+
+    def _split_file(self, file_path: str, batch_size: int) -> list[str]:
+        """Split Excel file for batch processing.
+
+        Note: This is a placeholder implementation. Excel batching by sheets
+        could be implemented in the future, but for now we process the entire
+        file at once as Excel files are typically smaller than PDFs.
+
+        Args:
+            file_path: Path to the Excel file to be split
+            batch_size: Number of sheets to include in each split file (not used currently)
+
+        Returns:
+            List containing the original file path (no splitting performed)
+        """
+        # Placeholder implementation - Excel batching not implemented
+        # Excel files are typically smaller and batching by sheets would require
+        # more complex logic to split workbooks
+        logger.info(f"Excel batching not implemented, processing entire file: {file_path}")
+        return [file_path]
+
+    def _validate_excel_file(self, file_path: str) -> str | None:
+        """Validate Excel file and check for potential issues.
+
+        Args:
+            file_path: Path to the Excel file
+
+        Returns:
+            Error message if validation fails, None if successful
+        """
+        path = Path(file_path)
+
+        # Check file extension
+        if path.suffix.lower() not in [".xlsx", ".xlsm"]:
+            return f"Invalid file extension: {path.suffix}. Only .xlsx and .xlsm files are supported."
+
+        # Check for macro-enabled files and warn
+        if path.suffix.lower() == ".xlsm":
+            logger.warning(f"Macro-enabled file detected: {file_path}")
+            # Check if file actually contains macros
+            try:
+                with zipfile.ZipFile(file_path, "r") as zip_file:
+                    if "xl/vbaProject.bin" in zip_file.namelist():
+                        logger.warning(
+                            f"VBA macros found in {file_path}. "
+                            f"Macros will not be executed; only data will be extracted."
+                        )
+            except Exception as e:
+                logger.debug(f"Could not check for macros: {e}")
+
+        return None
+
+    def _run_standard(self, file_path: str) -> str:
+        """Override standard processing to add Excel-specific error handling.
+
+        Args:
+            file_path: Path to the Excel file to be read
+
+        Returns:
+            Extracted text content from the document
+        """
+        # Validate Excel file
+        validation_error = self._validate_excel_file(file_path)
+        if validation_error:
+            return f"Validation Error: {validation_error}"
+
+        try:
+            return self._process_single_file(file_path)
+
+        except PermissionError:
+            return f"Error: Permission denied accessing file: {file_path}"
+
+        except zipfile.BadZipFile:
+            return (
+                "Error: File appears to be corrupted or is not a valid Excel format. "
+                "Please check the file and try again."
+            )
+
+        except Exception as e:
+            error_msg = str(e).lower()
+
+            # Check for specific Excel errors
+            if "password" in error_msg or "encrypted" in error_msg:
+                return "Error: File is password-protected or encrypted. Please provide an unprotected file."
+
+            elif "invalid" in error_msg or "corrupt" in error_msg:
+                return "Error: File appears to be corrupted or invalid Excel format. Please verify the file integrity."
+
+            elif "not supported" in error_msg or "unsupported" in error_msg:
+                return f"Error: File contains unsupported features: {str(e)}"
+
+            else:
+                logger.exception(f"Unexpected error processing Excel file: {file_path}")
+                return f"Error processing Excel file: {str(e)}"

aip_agents/tools/document_loader/excel_reader_tool.pyi

@@ -0,0 +1,26 @@
+from _typeshed import Incomplete
+from aip_agents.tools.document_loader.base_reader import BaseDocumentReaderTool as BaseDocumentReaderTool, DOCPROC_MISSING_MESSAGE as DOCPROC_MISSING_MESSAGE
+from aip_agents.utils.logger import get_logger as get_logger
+
+logger: Incomplete
+
+class ExcelReaderTool(BaseDocumentReaderTool):
+    '''Tool to read and extract content from Excel files.
+
+    This tool reads Excel files (.xlsx, .xlsm) and extracts their content using
+    the gllm_docproc loader pipeline. The content is formatted as Markdown tables
+    for easy readability.
+
+    Features:
+    - Supports .xlsx and .xlsm formats
+    - Extracts all sheets or specific sheets
+    - Formats output as Markdown tables
+    - Configurable row limits and file size limits
+
+    Examples:
+        >>> tool = ExcelReaderTool()
+        >>> result = tool._run("/tmp/data.xlsx")
+        >>> print(result)
+    '''
+    name: str
+    description: str

aip_agents/tools/document_loader/pdf_reader_tool.py

@@ -0,0 +1,79 @@
+"""Tool to read and extract text from PDF files.
+
+Authors:
+    Christian Trisno Sen Long Chen (christian.t.s.l.chen@gdplabs.id)
+
+References:
+    https://github.com/GDP-ADMIN/gdplabs-exploration/blob/ai-agent-app/backend/aip_agents/tools/
+      reader/pdf_reader_tool.py
+"""
+
+try:
+    from gllm_docproc.loader.pdf import PDFPlumberLoader, PyMuPDFLoader
+except ImportError as exc:
+    PDFPlumberLoader = None  # type: ignore[assignment]
+    PyMuPDFLoader = None  # type: ignore[assignment]
+    _DOCPROC_IMPORT_ERROR: Exception | None = exc
+else:
+    _DOCPROC_IMPORT_ERROR = None
+
+from aip_agents.tools.document_loader.base_reader import DOCPROC_MISSING_MESSAGE, BaseDocumentReaderTool
+from aip_agents.tools.document_loader.pdf_splitter import PDFSplitter
+from aip_agents.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class PDFReaderTool(BaseDocumentReaderTool):
+    """Tool to read and extract text from PDF files."""
+
+    name: str = "pdf_reader_tool"
+    description: str = "Read a PDF file and extract its text content. Input should be the path to the PDF file."
+
+    def _setup_loader(self):
+        if PDFPlumberLoader is None or PyMuPDFLoader is None:
+            logger.warning(DOCPROC_MISSING_MESSAGE)
+            if _DOCPROC_IMPORT_ERROR is not None:
+                logger.debug("gllm_docproc import failed: %s", _DOCPROC_IMPORT_ERROR)
+            return
+        self.loader.add_loader(PyMuPDFLoader())
+        self.loader.add_loader(PDFPlumberLoader())
+
+    def _split_file(self, file_path: str, batch_size: int) -> list[str]:
+        """Split PDF file into temporary files for batch processing.
+
+        This method uses PDFSplitter.split_by_pages to split the PDF into
+        temporary files containing the specified number of pages per batch.
+
+        Args:
+            file_path: Path to the PDF file to be split
+            batch_size: Number of pages to include in each split file (1-10)
+
+        Returns:
+            List of temporary file paths containing the split content
+
+        Raises:
+            FileNotFoundError: If the input PDF file doesn't exist
+            ValueError: If batch_size is invalid or PDF processing fails
+            RuntimeError: For other unexpected errors during PDF splitting
+        """
+        logger.info(f"Splitting PDF file '{file_path}' with batch_size={batch_size}")
+
+        try:
+            # Use PDFSplitter utility to split the PDF by pages
+            temp_files = PDFSplitter.split_by_pages(file_path, batch_size)
+
+            logger.info(f"Successfully split PDF into {len(temp_files)} batch files")
+            logger.debug(f"Created temporary files: {temp_files}")
+
+            return temp_files
+
+        except FileNotFoundError:
+            logger.error(f"PDF file not found: {file_path}")
+            raise
+        except ValueError as e:
+            logger.error(f"Invalid parameters or corrupted PDF: {str(e)}")
+            raise
+        except Exception as e:
+            logger.error(f"Unexpected error splitting PDF '{file_path}': {str(e)}")
+            raise RuntimeError(f"Failed to split PDF file: {str(e)}") from e

aip_agents/tools/document_loader/pdf_reader_tool.pyi

@@ -0,0 +1,11 @@
+from _typeshed import Incomplete
+from aip_agents.tools.document_loader.base_reader import BaseDocumentReaderTool as BaseDocumentReaderTool, DOCPROC_MISSING_MESSAGE as DOCPROC_MISSING_MESSAGE
+from aip_agents.tools.document_loader.pdf_splitter import PDFSplitter as PDFSplitter
+from aip_agents.utils.logger import get_logger as get_logger
+
+logger: Incomplete
+
+class PDFReaderTool(BaseDocumentReaderTool):
+    """Tool to read and extract text from PDF files."""
+    name: str
+    description: str

aip_agents/tools/document_loader/pdf_splitter.py

@@ -0,0 +1,169 @@
+"""PDF splitting utility for batching optimization.
+
+Authors
+    Christian Trisno Sen Long Chen (christian.t.s.l.chen@gdplabs.id)
+"""
+
+import os
+import tempfile
+
+from PyPDF2 import PdfReader, PdfWriter
+
+
+class PDFSplitter:
+    """Utility class for splitting PDF files into page-based temporary files."""
+
+    @staticmethod
+    def split_by_pages(file_path: str, batch_size: int) -> list[str]:
+        """Split PDF into temporary files containing batch_size pages each.
+
+        Args:
+            file_path: Path to the PDF file to split
+            batch_size: Number of pages to include in each batch (1-10)
+
+        Returns:
+            List of temporary file paths containing the split PDF batches
+
+        Raises:
+            FileNotFoundError: If the input PDF file doesn't exist
+            ValueError: If batch_size is invalid or PDF is corrupted
+            Exception: For other PDF processing errors
+        """
+        PDFSplitter._validate_inputs(file_path, batch_size)
+
+        try:
+            reader = PdfReader(file_path)
+            PDFSplitter._validate_pdf_content(reader, file_path)
+
+            temp_files = []
+
+            # Split PDF into batches
+            for i in range(0, len(reader.pages), batch_size):
+                batch_number = i // batch_size + 1
+                temp_path = PDFSplitter._create_batch_file(reader, i, batch_size, batch_number, temp_files)
+                temp_files.append(temp_path)
+
+            return temp_files
+
+        except Exception as e:
+            # Handle corrupted or invalid PDF files
+            if "temp_files" in locals():
+                PDFSplitter._cleanup_temp_files(temp_files)
+
+            if isinstance(e, FileNotFoundError | ValueError):
+                raise
+            else:
+                raise ValueError(f"Error processing PDF file {file_path}: {str(e)}") from e
+
+    @staticmethod
+    def _validate_inputs(file_path: str, batch_size: int) -> None:
+        """Validate input parameters.
+
+        Args:
+            file_path: Path to the PDF file to validate
+            batch_size: Batch size to validate
+
+        Raises:
+            FileNotFoundError: If the input PDF file doesn't exist
+            ValueError: If batch_size is invalid
+        """
+        if not os.path.exists(file_path):
+            raise FileNotFoundError(f"PDF file not found: {file_path}")
+
+    @staticmethod
+    def _validate_pdf_content(reader: PdfReader, file_path: str) -> None:
+        """Validate PDF content has pages.
+
+        Args:
+            reader: PdfReader instance to validate
+            file_path: Path to the PDF file for error messages
+
+        Raises:
+            ValueError: If PDF has no pages
+        """
+        if len(reader.pages) == 0:
+            raise ValueError(f"PDF file has no pages: {file_path}")
+
+    @staticmethod
+    def _create_batch_file(
+        reader: PdfReader,
+        start_page: int,
+        batch_size: int,
+        batch_number: int,
+        temp_files: list[str],
+    ) -> str:
+        """Create a temporary file containing a batch of PDF pages.
+
+        Args:
+            reader: PdfReader instance containing the source pages
+            start_page: Starting page index for this batch
+            batch_size: Number of pages to include in the batch
+            batch_number: Batch number for file naming
+            temp_files: List of existing temp files for cleanup on error
+
+        Returns:
+            Path to the created temporary file
+
+        Raises:
+            ValueError: If page processing or file writing fails
+        """
+        writer = PdfWriter()
+
+        # Add pages to this batch
+        batch_end = min(start_page + batch_size, len(reader.pages))
+        for j in range(start_page, batch_end):
+            try:
+                writer.add_page(reader.pages[j])
+            except Exception as e:
+                # Clean up any created temp files before re-raising
+                PDFSplitter._cleanup_temp_files(temp_files)
+                raise ValueError(f"Error processing page {j + 1}: {str(e)}") from e
+
+        # Create and write temporary file
+        return PDFSplitter._write_batch_to_temp_file(writer, batch_number, temp_files)
+
+    @staticmethod
+    def _write_batch_to_temp_file(writer: PdfWriter, batch_number: int, temp_files: list[str]) -> str:
+        """Write a PDF batch to a temporary file.
+
+        Args:
+            writer: PdfWriter instance containing the batch pages
+            batch_number: Batch number for file naming
+            temp_files: List of existing temp files for cleanup on error
+
+        Returns:
+            Path to the created temporary file
+
+        Raises:
+            ValueError: If file writing fails
+        """
+        # Create temporary file with proper naming
+        temp_fd, temp_path = tempfile.mkstemp(suffix=f"_batch_{batch_number}.pdf", prefix="pdf_split_")
+
+        try:
+            with os.fdopen(temp_fd, "wb") as temp_file:
+                writer.write(temp_file)
+            return temp_path
+        except Exception as e:
+            # Close the file descriptor if write fails
+            try:
+                os.close(temp_fd)
+            except OSError:
+                pass
+            # Clean up any created temp files
+            PDFSplitter._cleanup_temp_files(temp_files)
+            raise ValueError(f"Error writing batch {batch_number}: {str(e)}") from e
+
+    @staticmethod
+    def _cleanup_temp_files(temp_files: list[str]) -> None:
+        """Clean up temporary files.
+
+        Args:
+            temp_files: List of temporary file paths to clean up
+        """
+        for temp_file in temp_files:
+            try:
+                if os.path.exists(temp_file):
+                    os.unlink(temp_file)
+            except OSError:
+                pass

aip_agents/tools/document_loader/pdf_splitter.pyi

@@ -0,0 +1,18 @@
+class PDFSplitter:
+    """Utility class for splitting PDF files into page-based temporary files."""
+    @staticmethod
+    def split_by_pages(file_path: str, batch_size: int) -> list[str]:
+        """Split PDF into temporary files containing batch_size pages each.
+
+        Args:
+            file_path: Path to the PDF file to split
+            batch_size: Number of pages to include in each batch (1-10)
+
+        Returns:
+            List of temporary file paths containing the split PDF batches
+
+        Raises:
+            FileNotFoundError: If the input PDF file doesn't exist
+            ValueError: If batch_size is invalid or PDF is corrupted
+            Exception: For other PDF processing errors
+        """