aip-agents-binary 0.0.0b2__py3-none-any.whl → 0.5.12__py3-none-any.whl

aip_agents/tools/constants.py

@@ -0,0 +1,165 @@
+"""Constants for tools using BOSA Connector.
+
+Authors:
+    Saul Sayers (saul.sayers@gdplabs.id)
+"""
+
+import os
+from enum import Enum, StrEnum
+
+BOSA_API_BASE_URL = os.getenv("BOSA_API_BASE_URL")
+BOSA_API_KEY = os.getenv("BOSA_API_KEY")
+BOSA_FETCH_MAX_RETRIES = 3
+
+
+class ToolType(StrEnum):
+    """Tool types for BOSA Connector."""
+
+    GLLM = "gllm"
+    LANGCHAIN = "langchain"
+
+
+class Action(Enum):
+    """Actions for BOSA Connector."""
+
+    GITHUB = "github"
+    GOOGLE = "google"
+    GOOGLE_DRIVE = "google_drive"
+    GOOGLE_MAIL = "google_mail"
+    TWITTER = "twitter"
+
+
+class GitHubEndpoint(Enum):
+    """GitHub endpoints for BOSA Connector."""
+
+    INTEGRATIONS = "integrations"
+    USER_HAS_INTEGRATION = "integration-exists"
+    SUCCESS_AUTHORIZE_CALLBACK = "success-authorize-callback"
+    CREATE_ISSUE_HANDLER = "create_issue"
+    GET_ISSUE_HANDLER = "get_issue"
+    LIST_ISSUES = "list_issues"
+    LIST_ISSUES_COMMENTS = "list_issues_comments"
+    SEARCH_ALL_ISSUES = "search_issues"
+    GET_COMMITS = "list_commits"
+    SEARCH_COMMITS = "search_commits"
+    GET_COLLABORATORS = "list_collaborators"
+    GET_RELEASES = "list_releases"
+    GET_CONTRIBUTORS = "list_contributors"
+    GET_LANGUAGES = "list_languages"
+    SEARCH_CONTRIBUTIONS = "search_contributions"
+    LIST_PULL_REQUESTS = "list_pull_requests"
+    SEARCH_PR = "search_pull_requests"
+    GET_PULL = "get_pull"
+    GET_ALL_CONTRIBUTOR_COMMIT_ACTIVITIES = "get_all_contributor_commit_activities"
+    GET_COMMIT_ACTIVITY = "get_the_last_year_of_commit_activity"
+    GET_WEEKLY_COMMIT_COUNTS = "get_weekly_commit_count "
+    GET_USER_CONTRIBUTION_STATISTICS = "get_user_contribution_statistics"
+    LIST_PROJECT_ITEMS = "list_project_items"
+    LIST_PROJECTS = "list_projects"
+
+
+class GoogleDriveEndpoint(Enum):
+    """Google Drive endpoints for BOSA Connector."""
+
+    INTEGRATIONS = "integrations"
+    USER_HAS_INTEGRATION = "integration-exists"
+    SUCCESS_AUTHORIZE_CALLBACK = "success-authorize-callback"
+    SEARCH_FILES = "search_files"
+    GET_FILE = "get_file"
+    CREATE_FILE = "create_file"
+    CREATE_FOLDER = "create_folder"
+    UPDATE_FILE = "update_file"
+    UPDATE_FOLDER = "update_folder"
+    COPY_FILE = "copy_file"
+    DELETE_FILE = "delete_file"
+    SUMMARIZE_FOLDER_FILES_BY_TYPE = "summarize_folder_files_by_type"
+    SUMMARIZE_TOTAL_FILES_BY_TYPE = "summarize_total_files_by_type"
+    RECENT_FILES = "list_recent_files_from_yesterday"
+    CREATE_PERMISSION = "create_permission"
+    LIST_PERMISSIONS = "list_permissions"
+    GET_PERMISSION = "get_permission"
+    UPDATE_PERMISSION = "update_permission"
+    DELETE_PERMISSION = "delete_permission"
+    DOWNLOAD_FILE = "download_file"
+
+
+class GoogleDocsEndpoint(Enum):
+    """Google Docs endpoints for BOSA Connector."""
+
+    INTEGRATIONS = "integrations"
+    USER_HAS_INTEGRATION = "integration-exists"
+    SUCCESS_AUTHORIZE_CALLBACK = "success-authorize-callback"
+    GET_DOCUMENT = "get_document"
+    LIST_DOCUMENTS = "list_documents"
+    CREATE_DOCUMENT = "create_document"
+    UPDATE_DOCUMENT = "update_document"
+    COPY_CONTENT = "copy_content"
+    UPDATE_DOCUMENT_MARKDOWN = "update_document_markdown"
+    LIST_COMMENTS = "list_comments"
+    SUMMARIZE_COMMENTS = "summarize_comments"
+
+
+class GoogleEndpoint(Enum):
+    """Google endpoints for BOSA Connector."""
+
+    INTEGRATIONS = "integrations"
+    USER_HAS_INTEGRATION = "integration-exists"
+    SUCCESS_AUTHORIZE_CALLBACK = "success-authorize-callback"
+    USERINFO = "userinfo"
+
+
+class TwitterEndpoint(Enum):
+    """Twitter endpoints for BOSA Connector."""
+
+    INTEGRATIONS = "integrations"
+    USER_HAS_INTEGRATION = "integration-exists"
+    SUCCESS_AUTHORIZE_CALLBACK = "success-authorize-callback"
+    SEARCH = "tweet_search"
+    GET_TWEETS = "get_tweets"
+    GET_THREAD = "get_thread"
+    GET_USERS = "get_users"
+
+
+class GoogleMailEndpoint(Enum):
+    """Google Mail endpoints for BOSA Connector."""
+
+    INTEGRATIONS = "integrations"
+    USER_HAS_INTEGRATION = "integration-exists"
+    SUCCESS_AUTHORIZE_CALLBACK = "success-authorize-callback"
+    CREATE_DRAFT = "create_draft"
+    LIST_DRAFTS = "list_drafts"
+    SEND_DRAFT = "send_draft"
+    GET_DRAFT = "get_draft"
+    MODIFY_DRAFT = "modify_draft"
+    LIST_LABELS = "list_labels"
+    LABEL_STATS = "label_stats"
+    GET_LABEL_DETAILS = "get_label_details"
+    CREATE_LABELS = "create_labels"
+    MODIFY_LABELS = "modify_labels"
+    DELETE_LABELS = "delete_labels"
+    SEND_EMAIL = "send_email"
+    LIST_EMAILS = "list_emails"
+    GET_EMAIL_DETAILS = "get_email_details"
+    MODIFY_EMAIL = "modify_email"
+    DELETE_EMAIL = "delete_email"
+    TRASH_EMAIL = "trash_email"
+    UNTRASH_EMAIL = "untrash_email"
+    LIST_THREADS = "list_threads"
+    THREAD_DETAILS = "thread_details"
+    MODIFY_THREAD = "modify_thread"
+    GET_AUTO_REPLY = "get_auto_reply"
+    SET_AUTO_REPLY = "set_auto_reply"
+    GET_ATTACHMENT = "get_attachment"
+    USERINFO = "userinfo"
+
+
+class ActionEndpointMap:
+    """Maps Action enums to their corresponding Endpoint enums."""
+
+    MAP: dict[Action, type[Enum]] = {
+        Action.GITHUB: GitHubEndpoint,
+        Action.GOOGLE: GoogleEndpoint,
+        Action.GOOGLE_DRIVE: GoogleDriveEndpoint,
+        Action.GOOGLE_MAIL: GoogleMailEndpoint,
+        Action.TWITTER: TwitterEndpoint,
+    }

aip_agents/tools/document_loader/__init__.py

@@ -0,0 +1,37 @@
+"""Document loader tools package.
+
+This package provides tools for reading and extracting content from various document formats.
+
+Authors:
+    Christian Trisno Sen Long Chen (christian.t.s.l.chen@gdplabs.id)
+    Douglas Raevan Faisal (douglas.raevan.faisal@gdplabs.id)
+
+References:
+    NONE
+"""
+
+try:
+    from aip_agents.tools.document_loader.base_reader import (  # noqa: F401
+        BaseDocumentReaderTool,
+        DocumentReaderInput,
+    )
+    from aip_agents.tools.document_loader.docx_reader_tool import DocxReaderTool  # noqa: F401
+    from aip_agents.tools.document_loader.excel_reader_tool import ExcelReaderTool  # noqa: F401
+    from aip_agents.tools.document_loader.pdf_reader_tool import PDFReaderTool  # noqa: F401
+    from aip_agents.tools.document_loader.pdf_splitter import PDFSplitter  # noqa: F401
+
+    __all__ = [
+        "BaseDocumentReaderTool",
+        "DocumentReaderInput",
+        "PDFReaderTool",
+        "DocxReaderTool",
+        "ExcelReaderTool",
+        "PDFSplitter",
+    ]
+except ImportError:
+    import warnings
+
+    warnings.warn(
+        "Document loader tools not available. Install with: pip install aip-agents[document-loader]", ImportWarning
+    )
+    __all__ = []

aip_agents/tools/document_loader/base_reader.py

@@ -0,0 +1,262 @@
+"""Base document reader tool.
+
+Authors:
+    Christian Trisno Sen Long Chen (christian.t.s.l.chen@gdplabs.id)
+
+References:
+    https://github.com/GDP-ADMIN/gdplabs-exploration/blob/ai-agent-app/backend/aip_agents/tools/reader/base_reader.py
+"""
+
+import gc
+import os
+from abc import ABC, abstractmethod
+
+from gllm_docproc.loader.pipeline_loader import PipelineLoader
+from langchain_core.runnables import RunnableConfig
+from langchain_core.tools import BaseTool
+from pydantic import BaseModel, Field
+from unidecode import unidecode
+
+from aip_agents.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class BaseDocumentConfig(BaseModel):
+    """Base tool configuration schema for document processing with batching functionality.
+
+    This configuration enables page-by-page batching to optimize memory usage when
+    processing large document files. When batching is enabled, documents are processed
+    sequentially by pages rather than loading the entire document into memory at once.
+
+    Attributes:
+        batching (bool): Enable page-by-page batching to reduce memory usage.
+            When True, documents are processed page by page sequentially.
+            When False, maintains current behavior of loading entire document.
+            Defaults to False for backward compatibility.
+        batch_size (int): Number of pages to process in each batch.
+            Must be between 1 and 100 pages inclusive.
+            Larger batch sizes may use more memory but could be more efficient.
+            Smaller batch sizes use less memory but may have more overhead.
+            Defaults to 10 for balanced memory usage and efficiency.
+
+    Examples:
+        >>> # Default configuration (no batching)
+        >>> config = BaseDocumentConfig()
+        >>> print(config.batching)  # False
+        >>> print(config.batch_size)  # 10
+
+        >>> # Enable batching with single page processing
+        >>> config = BaseDocumentConfig(batching=True, batch_size=1)
+
+        >>> # Enable batching with multi-page batches
+        >>> config = BaseDocumentConfig(batching=True, batch_size=3)
+    """
+
+    batching: bool = Field(
+        default=False,
+        description="Enable page-by-page batching to reduce memory usage when processing large documents",
+    )
+    batch_size: int = Field(
+        default=10,
+        ge=1,
+        le=100,
+        description="Number of pages to process per batch (1-100 pages)",
+    )
+
+
+class DocumentReaderInput(BaseModel):
+    """Input schema for the DocumentReader tool."""
+
+    file_path: str = Field(..., description="Path to the document file to be read")
+
+
+class BaseDocumentReaderTool(BaseTool, ABC):
+    """Base tool to read and extract text from document files."""
+
+    name: str = "base_document_reader_tool"
+    description: str = "Read a document file and extract its text content."
+    args_schema: type[BaseModel] = DocumentReaderInput
+    tool_config_schema: type[BaseModel] = BaseDocumentConfig
+    loader: PipelineLoader = Field(default_factory=PipelineLoader)
+
+    def __init__(self):
+        """Initialize the base document reader tool."""
+        super().__init__()
+        self._setup_loader()
+
+    @abstractmethod
+    def _setup_loader(self):
+        """Set up the specific loaders for each document type."""
+        pass  # pragma: no cover
+
+    def _run(self, file_path: str, config: RunnableConfig | None = None) -> str:
+        """Run with optional batching based on configuration.
+
+        Args:
+            file_path: Path to the document file to be read
+            config: Optional RunnableConfig containing tool configuration
+
+        Returns:
+            Extracted text content from the document
+        """
+        tool_config = None
+        if hasattr(self, "get_tool_config"):
+            tool_config = self.get_tool_config(config)
+
+        tool_config = tool_config or BaseDocumentConfig()
+
+        logger.info(f"Batching: {tool_config.batching}, Batch size: {tool_config.batch_size}")
+        if tool_config.batching:
+            return self._run_with_batching(file_path, tool_config.batch_size)
+        else:
+            return self._run_standard(file_path)
+
+    def _run_standard(self, file_path: str) -> str:
+        """Standard processing (existing behavior).
+
+        Args:
+            file_path: Path to the document file to be read
+
+        Returns:
+            Extracted text content from the document
+        """
+        try:
+            return self._process_single_file(file_path)
+        except Exception as e:
+            return f"Error reading file: {str(e)}"
+
+    def _run_with_batching(self, file_path: str, batch_size: int) -> str:
+        """Process file using batching with existing loader.
+
+        Args:
+            file_path: Path to the document file to be read
+            batch_size: Number of pages to process per batch
+
+        Returns:
+            Extracted text content from the document
+        """
+        try:
+            logger.info(f"Splitting file: {file_path} into {batch_size} pages per batch")
+            split_files = self._split_file(file_path, batch_size)
+            return self._process_file_batch(split_files)
+        except Exception as e:
+            return f"Error reading file: {str(e)}"
+
+    def _process_single_file(self, file_path: str) -> str:
+        """Process a single file and return extracted text.
+
+        Args:
+            file_path: Path to the document file to be read
+
+        Returns:
+            Extracted text content from the document (ASCII-normalized)
+        """
+        try:
+            # Load and process the file
+            loaded_elements = self.loader.load(file_path)
+            full_text = "\n".join(element["text"] for element in loaded_elements)
+
+            # Apply unidecode to convert non-ASCII characters to ASCII equivalents
+            # This prevents encoding errors by transliterating characters like:
+            # "Café" -> "Cafe", "François" -> "Francois", "北京" -> "Bei Jing"
+            result = unidecode(full_text).strip()
+
+            # Explicit memory cleanup
+            del loaded_elements
+            del full_text
+
+            # Force garbage collection to free memory immediately
+            gc.collect()
+
+            return result
+
+        except Exception:
+            # Ensure cleanup even on error
+            gc.collect()
+            raise
+
+    def _process_file_batch(self, split_files: list[str]) -> str:
+        """Process a batch of split files and return combined text.
+
+        Args:
+            split_files: List of temporary file paths to process
+
+        Returns:
+            Combined extracted text content from all files
+        """
+        results = []
+        errors = []
+
+        for split_file in split_files:
+            try:
+                text = self._process_single_file(split_file)
+                results.append(text)
+
+                # Clear the text variable to free memory immediately
+                del text
+
+            except Exception as e:
+                error_msg = f"Error processing batch: {str(e)}"
+                errors.append(error_msg)
+            finally:
+                self._cleanup_temp_file(split_file)
+                # Force garbage collection after each file to minimize memory usage
+                gc.collect()
+
+        # Combine results
+        full_text = "\n".join(results).strip()
+
+        # Clear intermediate results to free memory
+        del results
+        gc.collect()
+
+        if errors:
+            error_summary = f"\n\nProcessing completed with {len(errors)} errors:\n" + "\n".join(errors)
+            full_text += error_summary
+
+        return full_text
+
+    def _cleanup_temp_file(self, file_path: str) -> None:
+        """Clean up temporary file.
+
+        Args:
+            file_path: Path to the temporary file to clean up
+        """
+        try:
+            os.unlink(file_path)
+        except OSError:
+            pass
+
+    def cleanup_memory(self) -> None:
+        """Explicitly clean up memory and force garbage collection.
+
+        This method can be called after processing to minimize memory usage.
+        While it won't reset memory to exactly 0, it will free up as much
+        memory as possible by clearing internal caches and forcing garbage collection.
+        """
+        # Clear any cached data in the loader if it has a cleanup method
+        if hasattr(self.loader, "clear_cache"):
+            self.loader.clear_cache()
+
+        # Force garbage collection multiple times to ensure cleanup
+        for _ in range(3):
+            gc.collect()
+
+        # Optional: Clear loader entirely and reinitialize (more aggressive)
+        # This is commented out as it may affect performance for subsequent calls
+        # self.loader = PipelineLoader()
+        # self._setup_loader()
+
+    @abstractmethod
+    def _split_file(self, file_path: str, batch_size: int) -> list[str]:
+        """Split file into temporary files for batch processing.
+
+        Args:
+            file_path: Path to the document file to be split
+            batch_size: Number of pages to include in each split file
+
+        Returns:
+            List of temporary file paths containing the split content
+        """
+        pass  # pragma: no cover

aip_agents/tools/document_loader/docx_reader_tool.py

@@ -0,0 +1,53 @@
+"""Base tool to read and extract text from document files.
+
+Authors:
+    Christian Trisno Sen Long Chen (christian.t.s.l.chen@gdplabs.id)
+
+References:
+    https://github.com/GDP-ADMIN/gdplabs-exploration/blob/ai-agent-app/backend/aip_agents/tools/
+      reader/docx_reader_tool.py
+"""
+
+from gllm_docproc.loader.docx import DOCX2PythonLoader, PythonDOCXTableLoader
+
+from aip_agents.tools.document_loader.base_reader import BaseDocumentReaderTool
+
+
+class DocxReaderTool(BaseDocumentReaderTool):
+    """Tool to read and extract text from Word documents."""
+
+    name: str = "docx_reader_tool"
+    description: str = "Read a Word document and extract its text content. Input should be the path to the Word file."
+
+    def _setup_loader(self):
+        self.loader.add_loader(DOCX2PythonLoader())
+        self.loader.add_loader(PythonDOCXTableLoader())
+
+    def _run_with_batching(self, file_path: str, batch_size: int) -> str:
+        """Run without batching until DOCX splitting is implemented.
+
+        Args:
+            file_path: Path to the DOCX file to be processed
+            batch_size: Number of pages to include in each batch (not used currently)
+
+        Returns:
+            Extracted text content from the document
+        """
+        # TODO: implement DOCX batching with real splits before enabling batching flow.
+        return self._run_standard(file_path)
+
+    def _split_file(self, file_path: str, batch_size: int) -> list[str]:
+        """Split DOCX file into temporary files for batch processing.
+
+        Note: This is a placeholder implementation. DOCX batching is not
+        implemented in this feature but the method is required by the abstract base class.
+
+        Args:
+            file_path: Path to the DOCX file to be split
+            batch_size: Number of pages to include in each split file
+
+        Returns:
+            List of temporary file paths containing the split content
+        """
+        # Placeholder implementation - DOCX batching not implemented in this feature
+        return [file_path]

aip_agents/tools/document_loader/excel_reader_tool.py

@@ -0,0 +1,160 @@
+"""Tool to read and extract content from Excel files using gllm_docproc.
+
+This tool uses the gllm_docproc loader pipeline to extract content from Excel files,
+providing a consistent interface with other document reader tools (PDF, DOCX).
+
+Authors:
+    Douglas Raevan Faisal (douglas.raevan.faisal@gdplabs.id)
+"""
+
+import zipfile
+from pathlib import Path
+
+from gllm_docproc.loader.xlsx import OpenpyxlLoader
+
+from aip_agents.tools.document_loader.base_reader import BaseDocumentReaderTool
+from aip_agents.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class ExcelReaderTool(BaseDocumentReaderTool):
+    """Tool to read and extract content from Excel files.
+
+    This tool reads Excel files (.xlsx, .xlsm) and extracts their content using
+    the gllm_docproc loader pipeline. The content is formatted as Markdown tables
+    for easy readability.
+
+    Features:
+    - Supports .xlsx and .xlsm formats
+    - Extracts all sheets or specific sheets
+    - Formats output as Markdown tables
+    - Configurable row limits and file size limits
+
+    Examples:
+        >>> tool = ExcelReaderTool()
+        >>> result = tool._run("/tmp/data.xlsx")
+        >>> print(result)
+    """
+
+    name: str = "excel_reader_tool"
+    description: str = (
+        "Read an Excel file and extract its content as Markdown tables. "
+        "Input should be the path to the Excel file (.xlsx or .xlsm format). "
+        "Each sheet will be formatted as a Markdown table with the sheet name as a header."
+    )
+
+    def _setup_loader(self) -> None:
+        """Set up the XLSX loader for the pipeline.
+
+        This method initializes the OpenpyxlLoader which handles extraction
+        of content from Excel files and formatting as Markdown tables.
+        """
+        self.loader.add_loader(OpenpyxlLoader())
+
+    def _run_with_batching(self, file_path: str, batch_size: int) -> str:
+        """Run without batching until Excel splitting is implemented.
+
+        Args:
+            file_path: Path to the Excel file to be processed
+            batch_size: Number of sheets to include in each batch (not used currently)
+
+        Returns:
+            Extracted text content from the document
+        """
+        # TODO: implement Excel batching with real splits before enabling batching flow.
+        return self._run_standard(file_path)
+
+    def _split_file(self, file_path: str, batch_size: int) -> list[str]:
+        """Split Excel file for batch processing.
+
+        Note: This is a placeholder implementation. Excel batching by sheets
+        could be implemented in the future, but for now we process the entire
+        file at once as Excel files are typically smaller than PDFs.
+
+        Args:
+            file_path: Path to the Excel file to be split
+            batch_size: Number of sheets to include in each split file (not used currently)
+
+        Returns:
+            List containing the original file path (no splitting performed)
+        """
+        # Placeholder implementation - Excel batching not implemented
+        # Excel files are typically smaller and batching by sheets would require
+        # more complex logic to split workbooks
+        logger.info(f"Excel batching not implemented, processing entire file: {file_path}")
+        return [file_path]
+
+    def _validate_excel_file(self, file_path: str) -> str | None:
+        """Validate Excel file and check for potential issues.
+
+        Args:
+            file_path: Path to the Excel file
+
+        Returns:
+            Error message if validation fails, None if successful
+        """
+        path = Path(file_path)
+
+        # Check file extension
+        if path.suffix.lower() not in [".xlsx", ".xlsm"]:
+            return f"Invalid file extension: {path.suffix}. Only .xlsx and .xlsm files are supported."
+
+        # Check for macro-enabled files and warn
+        if path.suffix.lower() == ".xlsm":
+            logger.warning(f"Macro-enabled file detected: {file_path}")
+            # Check if file actually contains macros
+            try:
+                with zipfile.ZipFile(file_path, "r") as zip_file:
+                    if "xl/vbaProject.bin" in zip_file.namelist():
+                        logger.warning(
+                            f"VBA macros found in {file_path}. "
+                            f"Macros will not be executed; only data will be extracted."
+                        )
+            except Exception as e:
+                logger.debug(f"Could not check for macros: {e}")
+
+        return None
+
+    def _run_standard(self, file_path: str) -> str:
+        """Override standard processing to add Excel-specific error handling.
+
+        Args:
+            file_path: Path to the Excel file to be read
+
+        Returns:
+            Extracted text content from the document
+        """
+        # Validate Excel file
+        validation_error = self._validate_excel_file(file_path)
+        if validation_error:
+            return f"Validation Error: {validation_error}"
+
+        try:
+            return self._process_single_file(file_path)
+
+        except PermissionError:
+            return f"Error: Permission denied accessing file: {file_path}"
+
+        except zipfile.BadZipFile:
+            return (
+                "Error: File appears to be corrupted or is not a valid Excel format. "
+                "Please check the file and try again."
+            )
+
+        except Exception as e:
+            error_msg = str(e).lower()
+
+            # Check for specific Excel errors
+            if "password" in error_msg or "encrypted" in error_msg:
+                return "Error: File is password-protected or encrypted. Please provide an unprotected file."
+
+            elif "invalid" in error_msg or "corrupt" in error_msg:
+                return "Error: File appears to be corrupted or invalid Excel format. Please verify the file integrity."
+
+            elif "not supported" in error_msg or "unsupported" in error_msg:
+                return f"Error: File contains unsupported features: {str(e)}"
+
+            else:
+                logger.exception(f"Unexpected error processing Excel file: {file_path}")
+                return f"Error processing Excel file: {str(e)}"