aip-agents-binary 0.5.20__py3-none-manylinux_2_31_x86_64.whl

aip_agents/tools/constants.pyi

@@ -0,0 +1,135 @@
+from _typeshed import Incomplete
+from enum import Enum, StrEnum
+
+BOSA_API_BASE_URL: Incomplete
+BOSA_API_KEY: Incomplete
+BOSA_FETCH_MAX_RETRIES: int
+
+class ToolType(StrEnum):
+    """Tool types for BOSA Connector."""
+    GLLM: str
+    LANGCHAIN: str
+
+class Action(Enum):
+    """Actions for BOSA Connector."""
+    GITHUB: str
+    GOOGLE: str
+    GOOGLE_DRIVE: str
+    GOOGLE_MAIL: str
+    TWITTER: str
+
+class GitHubEndpoint(Enum):
+    """GitHub endpoints for BOSA Connector."""
+    INTEGRATIONS: str
+    USER_HAS_INTEGRATION: str
+    SUCCESS_AUTHORIZE_CALLBACK: str
+    CREATE_ISSUE_HANDLER: str
+    GET_ISSUE_HANDLER: str
+    LIST_ISSUES: str
+    LIST_ISSUES_COMMENTS: str
+    SEARCH_ALL_ISSUES: str
+    GET_COMMITS: str
+    SEARCH_COMMITS: str
+    GET_COLLABORATORS: str
+    GET_RELEASES: str
+    GET_CONTRIBUTORS: str
+    GET_LANGUAGES: str
+    SEARCH_CONTRIBUTIONS: str
+    LIST_PULL_REQUESTS: str
+    SEARCH_PR: str
+    GET_PULL: str
+    GET_ALL_CONTRIBUTOR_COMMIT_ACTIVITIES: str
+    GET_COMMIT_ACTIVITY: str
+    GET_WEEKLY_COMMIT_COUNTS: str
+    GET_USER_CONTRIBUTION_STATISTICS: str
+    LIST_PROJECT_ITEMS: str
+    LIST_PROJECTS: str
+
+class GoogleDriveEndpoint(Enum):
+    """Google Drive endpoints for BOSA Connector."""
+    INTEGRATIONS: str
+    USER_HAS_INTEGRATION: str
+    SUCCESS_AUTHORIZE_CALLBACK: str
+    SEARCH_FILES: str
+    GET_FILE: str
+    CREATE_FILE: str
+    CREATE_FOLDER: str
+    UPDATE_FILE: str
+    UPDATE_FOLDER: str
+    COPY_FILE: str
+    DELETE_FILE: str
+    SUMMARIZE_FOLDER_FILES_BY_TYPE: str
+    SUMMARIZE_TOTAL_FILES_BY_TYPE: str
+    RECENT_FILES: str
+    CREATE_PERMISSION: str
+    LIST_PERMISSIONS: str
+    GET_PERMISSION: str
+    UPDATE_PERMISSION: str
+    DELETE_PERMISSION: str
+    DOWNLOAD_FILE: str
+
+class GoogleDocsEndpoint(Enum):
+    """Google Docs endpoints for BOSA Connector."""
+    INTEGRATIONS: str
+    USER_HAS_INTEGRATION: str
+    SUCCESS_AUTHORIZE_CALLBACK: str
+    GET_DOCUMENT: str
+    LIST_DOCUMENTS: str
+    CREATE_DOCUMENT: str
+    UPDATE_DOCUMENT: str
+    COPY_CONTENT: str
+    UPDATE_DOCUMENT_MARKDOWN: str
+    LIST_COMMENTS: str
+    SUMMARIZE_COMMENTS: str
+
+class GoogleEndpoint(Enum):
+    """Google endpoints for BOSA Connector."""
+    INTEGRATIONS: str
+    USER_HAS_INTEGRATION: str
+    SUCCESS_AUTHORIZE_CALLBACK: str
+    USERINFO: str
+
+class TwitterEndpoint(Enum):
+    """Twitter endpoints for BOSA Connector."""
+    INTEGRATIONS: str
+    USER_HAS_INTEGRATION: str
+    SUCCESS_AUTHORIZE_CALLBACK: str
+    SEARCH: str
+    GET_TWEETS: str
+    GET_THREAD: str
+    GET_USERS: str
+
+class GoogleMailEndpoint(Enum):
+    """Google Mail endpoints for BOSA Connector."""
+    INTEGRATIONS: str
+    USER_HAS_INTEGRATION: str
+    SUCCESS_AUTHORIZE_CALLBACK: str
+    CREATE_DRAFT: str
+    LIST_DRAFTS: str
+    SEND_DRAFT: str
+    GET_DRAFT: str
+    MODIFY_DRAFT: str
+    LIST_LABELS: str
+    LABEL_STATS: str
+    GET_LABEL_DETAILS: str
+    CREATE_LABELS: str
+    MODIFY_LABELS: str
+    DELETE_LABELS: str
+    SEND_EMAIL: str
+    LIST_EMAILS: str
+    GET_EMAIL_DETAILS: str
+    MODIFY_EMAIL: str
+    DELETE_EMAIL: str
+    TRASH_EMAIL: str
+    UNTRASH_EMAIL: str
+    LIST_THREADS: str
+    THREAD_DETAILS: str
+    MODIFY_THREAD: str
+    GET_AUTO_REPLY: str
+    SET_AUTO_REPLY: str
+    GET_ATTACHMENT: str
+    USERINFO: str
+
+class ActionEndpointMap:
+    """Maps Action enums to their corresponding Endpoint enums."""
+    MAP: dict[Action, type[Enum]]

aip_agents/tools/document_loader/__init__.py

@@ -0,0 +1,44 @@
+"""Document loader tools package.
+
+This package provides tools for reading and extracting content from various document formats.
+
+Authors:
+    Christian Trisno Sen Long Chen (christian.t.s.l.chen@gdplabs.id)
+    Douglas Raevan Faisal (douglas.raevan.faisal@gdplabs.id)
+
+References:
+    NONE
+"""
+
+try:
+    from aip_agents.tools.document_loader.base_reader import (  # noqa: F401
+        DOCPROC_AVAILABLE,
+        DOCPROC_MISSING_MESSAGE,
+        BaseDocumentReaderTool,
+        DocumentReaderInput,
+    )
+    from aip_agents.tools.document_loader.docx_reader_tool import DocxReaderTool  # noqa: F401
+    from aip_agents.tools.document_loader.excel_reader_tool import ExcelReaderTool  # noqa: F401
+    from aip_agents.tools.document_loader.pdf_reader_tool import PDFReaderTool  # noqa: F401
+    from aip_agents.tools.document_loader.pdf_splitter import PDFSplitter  # noqa: F401
+
+    __all__ = [
+        "BaseDocumentReaderTool",
+        "DocumentReaderInput",
+        "PDFReaderTool",
+        "DocxReaderTool",
+        "ExcelReaderTool",
+        "PDFSplitter",
+    ]
+    if not DOCPROC_AVAILABLE:
+        import warnings
+
+        warnings.warn(DOCPROC_MISSING_MESSAGE, ImportWarning)
+except ImportError:
+    import warnings
+
+    warnings.warn(
+        "Document loader tools not available. Install gllm-docproc to enable them.",
+        ImportWarning,
+    )
+    __all__ = []

aip_agents/tools/document_loader/__init__.pyi

@@ -0,0 +1,7 @@
+from aip_agents.tools.document_loader.base_reader import BaseDocumentReaderTool as BaseDocumentReaderTool, DocumentReaderInput as DocumentReaderInput
+from aip_agents.tools.document_loader.docx_reader_tool import DocxReaderTool as DocxReaderTool
+from aip_agents.tools.document_loader.excel_reader_tool import ExcelReaderTool as ExcelReaderTool
+from aip_agents.tools.document_loader.pdf_reader_tool import PDFReaderTool as PDFReaderTool
+from aip_agents.tools.document_loader.pdf_splitter import PDFSplitter as PDFSplitter
+
+__all__ = ['BaseDocumentReaderTool', 'DocumentReaderInput', 'PDFReaderTool', 'DocxReaderTool', 'ExcelReaderTool', 'PDFSplitter']

aip_agents/tools/document_loader/base_reader.py

@@ -0,0 +1,302 @@
+"""Base document reader tool.
+
+Authors:
+    Christian Trisno Sen Long Chen (christian.t.s.l.chen@gdplabs.id)
+
+References:
+    https://github.com/GDP-ADMIN/gdplabs-exploration/blob/ai-agent-app/backend/aip_agents/tools/reader/base_reader.py
+"""
+
+import gc
+import os
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING, Any
+
+from langchain_core.runnables import RunnableConfig
+from langchain_core.tools import BaseTool
+from pydantic import BaseModel, Field
+from unidecode import unidecode
+
+from aip_agents.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+DOCPROC_MISSING_MESSAGE = (
+    "gllm-docproc is required for document loader tools but is not installed. "
+    "Install it from your internal registry to enable document processing."
+)
+
+try:
+    from gllm_docproc.loader.pipeline_loader import PipelineLoader
+except ImportError:
+    PipelineLoader = None  # type: ignore[assignment]
+
+if TYPE_CHECKING:
+    from gllm_docproc.loader.pipeline_loader import PipelineLoader as PipelineLoaderType
+else:
+    PipelineLoaderType = Any
+
+
+class _MissingDocprocLoader:
+    """Fallback loader that errors when document processing is attempted."""
+
+    def __init__(self) -> None:
+        self.loaders: list[object] = []
+
+    def add_loader(self, loader: object) -> None:
+        self.loaders.append(loader)
+
+    def load(self, *_args: object, **_kwargs: object) -> list[dict[str, str]]:
+        raise ImportError(DOCPROC_MISSING_MESSAGE)
+
+    def clear_cache(self) -> None:
+        self.loaders.clear()
+
+
+def _build_pipeline_loader() -> "PipelineLoader":
+    if PipelineLoader is None:
+        return _MissingDocprocLoader()
+    return PipelineLoader()  # type: ignore[misc]
+
+
+DOCPROC_AVAILABLE = PipelineLoader is not None
+
+
+class BaseDocumentConfig(BaseModel):
+    """Base tool configuration schema for document processing with batching functionality.
+
+    This configuration enables page-by-page batching to optimize memory usage when
+    processing large document files. When batching is enabled, documents are processed
+    sequentially by pages rather than loading the entire document into memory at once.
+
+    Attributes:
+        batching (bool): Enable page-by-page batching to reduce memory usage.
+            When True, documents are processed page by page sequentially.
+            When False, maintains current behavior of loading entire document.
+            Defaults to False for backward compatibility.
+        batch_size (int): Number of pages to process in each batch.
+            Must be between 1 and 100 pages inclusive.
+            Larger batch sizes may use more memory but could be more efficient.
+            Smaller batch sizes use less memory but may have more overhead.
+            Defaults to 10 for balanced memory usage and efficiency.
+
+    Examples:
+        >>> # Default configuration (no batching)
+        >>> config = BaseDocumentConfig()
+        >>> print(config.batching)  # False
+        >>> print(config.batch_size)  # 10
+
+        >>> # Enable batching with single page processing
+        >>> config = BaseDocumentConfig(batching=True, batch_size=1)
+
+        >>> # Enable batching with multi-page batches
+        >>> config = BaseDocumentConfig(batching=True, batch_size=3)
+    """
+
+    batching: bool = Field(
+        default=False,
+        description="Enable page-by-page batching to reduce memory usage when processing large documents",
+    )
+    batch_size: int = Field(
+        default=10,
+        ge=1,
+        le=100,
+        description="Number of pages to process per batch (1-100 pages)",
+    )
+
+
+class DocumentReaderInput(BaseModel):
+    """Input schema for the DocumentReader tool."""
+
+    file_path: str = Field(..., description="Path to the document file to be read")
+
+
+class BaseDocumentReaderTool(BaseTool, ABC):
+    """Base tool to read and extract text from document files."""
+
+    name: str = "base_document_reader_tool"
+    description: str = "Read a document file and extract its text content."
+    args_schema: type[BaseModel] = DocumentReaderInput
+    tool_config_schema: type[BaseModel] = BaseDocumentConfig
+    loader: PipelineLoaderType = Field(default_factory=_build_pipeline_loader)
+
+    def __init__(self):
+        """Initialize the base document reader tool."""
+        super().__init__()
+        self._setup_loader()
+
+    @abstractmethod
+    def _setup_loader(self):
+        """Set up the specific loaders for each document type."""
+        pass  # pragma: no cover
+
+    def _run(self, file_path: str, config: RunnableConfig | None = None) -> str:
+        """Run with optional batching based on configuration.
+
+        Args:
+            file_path: Path to the document file to be read
+            config: Optional RunnableConfig containing tool configuration
+
+        Returns:
+            Extracted text content from the document
+        """
+        tool_config = None
+        if hasattr(self, "get_tool_config"):
+            tool_config = self.get_tool_config(config)
+
+        tool_config = tool_config or BaseDocumentConfig()
+
+        logger.info(f"Batching: {tool_config.batching}, Batch size: {tool_config.batch_size}")
+        if tool_config.batching:
+            return self._run_with_batching(file_path, tool_config.batch_size)
+        else:
+            return self._run_standard(file_path)
+
+    def _run_standard(self, file_path: str) -> str:
+        """Standard processing (existing behavior).
+
+        Args:
+            file_path: Path to the document file to be read
+
+        Returns:
+            Extracted text content from the document
+        """
+        try:
+            return self._process_single_file(file_path)
+        except Exception as e:
+            return f"Error reading file: {str(e)}"
+
+    def _run_with_batching(self, file_path: str, batch_size: int) -> str:
+        """Process file using batching with existing loader.
+
+        Args:
+            file_path: Path to the document file to be read
+            batch_size: Number of pages to process per batch
+
+        Returns:
+            Extracted text content from the document
+        """
+        try:
+            logger.info(f"Splitting file: {file_path} into {batch_size} pages per batch")
+            split_files = self._split_file(file_path, batch_size)
+            return self._process_file_batch(split_files)
+        except Exception as e:
+            return f"Error reading file: {str(e)}"
+
+    def _process_single_file(self, file_path: str) -> str:
+        """Process a single file and return extracted text.
+
+        Args:
+            file_path: Path to the document file to be read
+
+        Returns:
+            Extracted text content from the document (ASCII-normalized)
+        """
+        try:
+            # Load and process the file
+            loaded_elements = self.loader.load(file_path)
+            full_text = "\n".join(element["text"] for element in loaded_elements)
+
+            # Apply unidecode to convert non-ASCII characters to ASCII equivalents
+            # This prevents encoding errors by transliterating characters like:
+            # "Café" -> "Cafe", "François" -> "Francois", "北京" -> "Bei Jing"
+            result = unidecode(full_text).strip()
+
+            # Explicit memory cleanup
+            del loaded_elements
+            del full_text
+
+            # Force garbage collection to free memory immediately
+            gc.collect()
+
+            return result
+
+        except Exception:
+            # Ensure cleanup even on error
+            gc.collect()
+            raise
+
+    def _process_file_batch(self, split_files: list[str]) -> str:
+        """Process a batch of split files and return combined text.
+
+        Args:
+            split_files: List of temporary file paths to process
+
+        Returns:
+            Combined extracted text content from all files
+        """
+        results = []
+        errors = []
+
+        for split_file in split_files:
+            try:
+                text = self._process_single_file(split_file)
+                results.append(text)
+
+                # Clear the text variable to free memory immediately
+                del text
+
+            except Exception as e:
+                error_msg = f"Error processing batch: {str(e)}"
+                errors.append(error_msg)
+            finally:
+                self._cleanup_temp_file(split_file)
+                # Force garbage collection after each file to minimize memory usage
+                gc.collect()
+
+        # Combine results
+        full_text = "\n".join(results).strip()
+
+        # Clear intermediate results to free memory
+        del results
+        gc.collect()
+
+        if errors:
+            error_summary = f"\n\nProcessing completed with {len(errors)} errors:\n" + "\n".join(errors)
+            full_text += error_summary
+
+        return full_text
+
+    def _cleanup_temp_file(self, file_path: str) -> None:
+        """Clean up temporary file.
+
+        Args:
+            file_path: Path to the temporary file to clean up
+        """
+        try:
+            os.unlink(file_path)
+        except OSError:
+            pass
+
+    def cleanup_memory(self) -> None:
+        """Explicitly clean up memory and force garbage collection.
+
+        This method can be called after processing to minimize memory usage.
+        While it won't reset memory to exactly 0, it will free up as much
+        memory as possible by clearing internal caches and forcing garbage collection.
+        """
+        # Clear any cached data in the loader if it has a cleanup method
+        if hasattr(self.loader, "clear_cache"):
+            self.loader.clear_cache()
+
+        # Force garbage collection multiple times to ensure cleanup
+        for _ in range(3):
+            gc.collect()
+
+        # Optional: Clear loader entirely and reinitialize (more aggressive)
+        # This is commented out as it may affect performance for subsequent calls
+        # self.loader = PipelineLoader()
+        # self._setup_loader()
+
+    @abstractmethod
+    def _split_file(self, file_path: str, batch_size: int) -> list[str]:
+        """Split file into temporary files for batch processing.
+
+        Args:
+            file_path: Path to the document file to be split
+            batch_size: Number of pages to include in each split file
+
+        Returns:
+            List of temporary file paths containing the split content
+        """
+        pass  # pragma: no cover

aip_agents/tools/document_loader/base_reader.pyi

@@ -0,0 +1,75 @@
+from _typeshed import Incomplete
+from abc import ABC
+from aip_agents.utils.logger import get_logger as get_logger
+from gllm_docproc.loader.pipeline_loader import PipelineLoader as PipelineLoaderType
+from langchain_core.tools import BaseTool
+from pydantic import BaseModel
+from typing import Any
+
+logger: Incomplete
+DOCPROC_MISSING_MESSAGE: str
+PipelineLoaderType = Any
+
+class _MissingDocprocLoader:
+    """Fallback loader that errors when document processing is attempted."""
+    loaders: list[object]
+    def __init__(self) -> None: ...
+    def add_loader(self, loader: object) -> None: ...
+    def load(self, *_args: object, **_kwargs: object) -> list[dict[str, str]]: ...
+    def clear_cache(self) -> None: ...
+
+DOCPROC_AVAILABLE: Incomplete
+
+class BaseDocumentConfig(BaseModel):
+    """Base tool configuration schema for document processing with batching functionality.
+
+    This configuration enables page-by-page batching to optimize memory usage when
+    processing large document files. When batching is enabled, documents are processed
+    sequentially by pages rather than loading the entire document into memory at once.
+
+    Attributes:
+        batching (bool): Enable page-by-page batching to reduce memory usage.
+            When True, documents are processed page by page sequentially.
+            When False, maintains current behavior of loading entire document.
+            Defaults to False for backward compatibility.
+        batch_size (int): Number of pages to process in each batch.
+            Must be between 1 and 100 pages inclusive.
+            Larger batch sizes may use more memory but could be more efficient.
+            Smaller batch sizes use less memory but may have more overhead.
+            Defaults to 10 for balanced memory usage and efficiency.
+
+    Examples:
+        >>> # Default configuration (no batching)
+        >>> config = BaseDocumentConfig()
+        >>> print(config.batching)  # False
+        >>> print(config.batch_size)  # 10
+
+        >>> # Enable batching with single page processing
+        >>> config = BaseDocumentConfig(batching=True, batch_size=1)
+
+        >>> # Enable batching with multi-page batches
+        >>> config = BaseDocumentConfig(batching=True, batch_size=3)
+    """
+    batching: bool
+    batch_size: int
+
+class DocumentReaderInput(BaseModel):
+    """Input schema for the DocumentReader tool."""
+    file_path: str
+
+class BaseDocumentReaderTool(BaseTool, ABC):
+    """Base tool to read and extract text from document files."""
+    name: str
+    description: str
+    args_schema: type[BaseModel]
+    tool_config_schema: type[BaseModel]
+    loader: PipelineLoaderType
+    def __init__(self) -> None:
+        """Initialize the base document reader tool."""
+    def cleanup_memory(self) -> None:
+        """Explicitly clean up memory and force garbage collection.
+
+        This method can be called after processing to minimize memory usage.
+        While it won't reset memory to exactly 0, it will free up as much
+        memory as possible by clearing internal caches and forcing garbage collection.
+        """

aip_agents/tools/document_loader/docx_reader_tool.py

@@ -0,0 +1,68 @@
+"""Base tool to read and extract text from document files.
+
+Authors:
+    Christian Trisno Sen Long Chen (christian.t.s.l.chen@gdplabs.id)
+
+References:
+    https://github.com/GDP-ADMIN/gdplabs-exploration/blob/ai-agent-app/backend/aip_agents/tools/
+      reader/docx_reader_tool.py
+"""
+
+try:
+    from gllm_docproc.loader.docx import DOCX2PythonLoader, PythonDOCXTableLoader
+except ImportError as exc:
+    DOCX2PythonLoader = None  # type: ignore[assignment]
+    PythonDOCXTableLoader = None  # type: ignore[assignment]
+    _DOCPROC_IMPORT_ERROR: Exception | None = exc
+else:
+    _DOCPROC_IMPORT_ERROR = None
+
+from aip_agents.tools.document_loader.base_reader import DOCPROC_MISSING_MESSAGE, BaseDocumentReaderTool
+from aip_agents.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class DocxReaderTool(BaseDocumentReaderTool):
+    """Tool to read and extract text from Word documents."""
+
+    name: str = "docx_reader_tool"
+    description: str = "Read a Word document and extract its text content. Input should be the path to the Word file."
+
+    def _setup_loader(self):
+        if DOCX2PythonLoader is None or PythonDOCXTableLoader is None:
+            logger.warning(DOCPROC_MISSING_MESSAGE)
+            if _DOCPROC_IMPORT_ERROR is not None:
+                logger.debug("gllm_docproc import failed: %s", _DOCPROC_IMPORT_ERROR)
+            return
+        self.loader.add_loader(DOCX2PythonLoader())
+        self.loader.add_loader(PythonDOCXTableLoader())
+
+    def _run_with_batching(self, file_path: str, batch_size: int) -> str:
+        """Run without batching until DOCX splitting is implemented.
+
+        Args:
+            file_path: Path to the DOCX file to be processed
+            batch_size: Number of pages to include in each batch (not used currently)
+
+        Returns:
+            Extracted text content from the document
+        """
+        # TODO: implement DOCX batching with real splits before enabling batching flow.
+        return self._run_standard(file_path)
+
+    def _split_file(self, file_path: str, batch_size: int) -> list[str]:
+        """Split DOCX file into temporary files for batch processing.
+
+        Note: This is a placeholder implementation. DOCX batching is not
+        implemented in this feature but the method is required by the abstract base class.
+
+        Args:
+            file_path: Path to the DOCX file to be split
+            batch_size: Number of pages to include in each split file
+
+        Returns:
+            List of temporary file paths containing the split content
+        """
+        # Placeholder implementation - DOCX batching not implemented in this feature
+        return [file_path]

aip_agents/tools/document_loader/docx_reader_tool.pyi

@@ -0,0 +1,10 @@
+from _typeshed import Incomplete
+from aip_agents.tools.document_loader.base_reader import BaseDocumentReaderTool as BaseDocumentReaderTool, DOCPROC_MISSING_MESSAGE as DOCPROC_MISSING_MESSAGE
+from aip_agents.utils.logger import get_logger as get_logger
+
+logger: Incomplete
+
+class DocxReaderTool(BaseDocumentReaderTool):
+    """Tool to read and extract text from Word documents."""
+    name: str
+    description: str