aip-agents-binary 0.0.0b2__py3-none-any.whl → 0.5.11__py3-none-any.whl

aip_agents/storage/config.pyi

@@ -1,42 +1,8 @@
-from aip_agents.storage.base import BaseObjectStorageClient as BaseObjectStorageClient
-from aip_agents.storage.clients.minio_client import MinioConfig as MinioConfig, MinioObjectStorage as MinioObjectStorage
-from aip_agents.storage.providers.base import BaseStorageProvider as BaseStorageProvider
-from aip_agents.storage.providers.memory import InMemoryStorageProvider as InMemoryStorageProvider
-from aip_agents.storage.providers.object_storage import ObjectStorageProvider as ObjectStorageProvider
-from dataclasses import dataclass
-from enum import StrEnum
+from aip_agents.schema.storage import OBJECT_STORAGE_PREFIX as OBJECT_STORAGE_PREFIX, StorageConfig as StorageConfig, StorageType as StorageType
+from aip_agents.storage.base import BaseObjectStorageClient
+from aip_agents.storage.providers.base import BaseStorageProvider
 
-OBJECT_STORAGE_PREFIX: str
-
-class StorageType(StrEnum):
-    """Supported storage types."""
-    MEMORY = 'memory'
-    OBJECT_STORAGE = 'object_storage'
-
-@dataclass
-class StorageConfig:
-    """Configuration for storage providers.
-
-    Attributes:
-        storage_type: Type of storage to use
-        object_prefix: Prefix for object storage keys (like a directory path)
-        object_use_json: Use JSON serialization instead of pickle for object storage
-    """
-    storage_type: StorageType = ...
-    object_prefix: str = ...
-    object_use_json: bool = ...
-    @classmethod
-    def from_env(cls) -> StorageConfig:
-        '''Create StorageConfig from environment variables.
-
-        Environment variables:
-        - TOOL_OUTPUT_STORAGE_TYPE: "memory" or "object_storage"
-        - TOOL_OUTPUT_OBJECT_PREFIX: Prefix for object storage keys
-        - TOOL_OUTPUT_USE_JSON: "true" to use JSON serialization
-
-        Returns:
-            StorageConfig instance
-        '''
+__all__ = ['OBJECT_STORAGE_PREFIX', 'StorageConfig', 'StorageType', 'StorageProviderFactory']
 
 class StorageProviderFactory:
     """Factory for creating storage providers based on configuration."""
@@ -45,22 +11,19 @@ class StorageProviderFactory:
         """Create storage provider based on configuration.
 
         Args:
-            config: Storage configuration
-            object_storage_client: Object storage client (required for object storage)
+            config (StorageConfig): Storage configuration object.
+            object_storage_client (BaseObjectStorageClient | None, optional): Optional object storage client for object storage type.
 
         Returns:
-            Configured storage provider
-
-        Raises:
-            ValueError: If configuration is invalid
+            BaseStorageProvider: The created storage provider instance.
         """
     @staticmethod
     def create_from_env(object_storage_client: BaseObjectStorageClient | None = None) -> BaseStorageProvider:
         """Create storage provider from environment variables.
 
         Args:
-            object_storage_client: Optional object storage client
+            object_storage_client (BaseObjectStorageClient | None, optional): Optional object storage client for object storage type.
 
         Returns:
-            Configured storage provider
+            BaseStorageProvider: The created storage provider instance.
         """

aip_agents/storage/providers/__init__.py

@@ -0,0 +1,15 @@
+"""Storage providers for tool output management.
+
+This module provides different storage backends for tool outputs including
+in-memory, file-based, and object storage providers.
+
+Authors:
+    Christian Trisno Sen Long Chen (christian.t.s.l.chen@gdplabs.id)
+    Fachriza Adhiatma (fachriza.d.adhiatma@gdplabs.id)
+"""
+
+from aip_agents.storage.providers.base import BaseStorageProvider, StorageError
+from aip_agents.storage.providers.memory import InMemoryStorageProvider
+from aip_agents.storage.providers.object_storage import ObjectStorageProvider
+
+__all__ = ["BaseStorageProvider", "StorageError", "InMemoryStorageProvider", "ObjectStorageProvider"]

aip_agents/storage/providers/base.py

@@ -0,0 +1,106 @@
+"""Base storage provider interface.
+
+Authors:
+    Christian Trisno Sen Long Chen (christian.t.s.l.chen@gdplabs.id)
+    Fachriza Adhiatma (fachriza.d.adhiatma@gdplabs.id)
+"""
+
+from abc import ABC, abstractmethod
+from typing import Any
+
+from aip_agents.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class StorageError(Exception):
+    """Base exception for storage operations."""
+
+
+class BaseStorageProvider(ABC):
+    """Base interface for storage providers.
+
+    This abstract class defines the contract that all storage providers
+    must implement to store and retrieve tool outputs.
+    """
+
+    @abstractmethod
+    def store(self, key: str, data: Any) -> None:
+        """Store data with the given key.
+
+        Args:
+            key: Unique identifier for the data
+            data: Data to store (must be serializable)
+
+        Raises:
+            StorageError: If storage operation fails
+        """
+
+    @abstractmethod
+    def retrieve(self, key: str) -> Any:
+        """Retrieve data by key.
+
+        Args:
+            key: Unique identifier for the data
+
+        Returns:
+            The stored data
+
+        Raises:
+            KeyError: If key not found
+            StorageError: If retrieval operation fails
+        """
+
+    @abstractmethod
+    def exists(self, key: str) -> bool:
+        """Check if key exists.
+
+        Args:
+            key: Unique identifier to check
+
+        Returns:
+            True if key exists, False otherwise
+        """
+
+    @abstractmethod
+    def delete(self, key: str) -> None:
+        """Delete data by key.
+
+        Args:
+            key: Unique identifier for the data
+
+        Note:
+            Should not raise error if key doesn't exist
+        """
+
+    @abstractmethod
+    def list_keys(self, prefix: str = "") -> list[str]:
+        """List all keys with optional prefix filter.
+
+        Args:
+            prefix: Optional prefix to filter keys
+
+        Returns:
+            List of matching keys
+        """
+
+    @abstractmethod
+    def clear(self) -> None:
+        """Clear all stored data.
+
+        Warning:
+            This operation is irreversible
+        """
+
+    def get_presigned_url(self, key: str, expires_hours: int = 24) -> str | None:
+        """Generate presigned URL for direct access (optional).
+
+        Args:
+            key: Storage key
+            expires_hours: URL expiration in hours
+
+        Returns:
+            Presigned URL if supported, None otherwise
+        """
+        logger.warning("This class get_presigned_url not implemented for this storage provider")
+        return None

aip_agents/storage/providers/base.pyi

@@ -1,7 +1,6 @@
-import abc
 from _typeshed import Incomplete
 from abc import ABC, abstractmethod
-from aip_agents.utils.logger_manager import LoggerManager as LoggerManager
+from aip_agents.utils.logger import get_logger as get_logger
 from typing import Any
 
 logger: Incomplete
@@ -9,7 +8,7 @@ logger: Incomplete
 class StorageError(Exception):
     """Base exception for storage operations."""
 
-class BaseStorageProvider(ABC, metaclass=abc.ABCMeta):
+class BaseStorageProvider(ABC):
     """Base interface for storage providers.
 
     This abstract class defines the contract that all storage providers

aip_agents/storage/providers/memory.py

@@ -0,0 +1,114 @@
+"""In-memory storage provider implementation.
+
+Authors:
+    Christian Trisno Sen Long Chen (christian.t.s.l.chen@gdplabs.id)
+    Fachriza Adhiatma (fachriza.d.adhiatma@gdplabs.id)
+"""
+
+import sys
+from typing import Any
+
+from aip_agents.storage.providers.base import BaseStorageProvider, StorageError
+
+
+class InMemoryStorageProvider(BaseStorageProvider):
+    """In-memory storage provider for fast access to small data.
+
+    This provider stores all data in memory, providing the fastest
+    access times but limited by available RAM.
+
+    Best for:
+    - Small to medium datasets
+    - Temporary storage
+    - Development and testing
+    - High-frequency access patterns
+    """
+
+    def __init__(self):
+        """Initialize in-memory storage."""
+        self._storage: dict[str, Any] = {}
+
+    def store(self, key: str, data: Any) -> None:
+        """Store data in memory.
+
+        Args:
+            key: Unique identifier for the data
+            data: Data to store
+
+        Raises:
+            StorageError: If storage operation fails
+        """
+        try:
+            self._storage[key] = data
+        except Exception as e:
+            raise StorageError(f"Failed to store data in memory: {e}") from e
+
+    def retrieve(self, key: str) -> Any:
+        """Retrieve data from memory.
+
+        Args:
+            key: Unique identifier for the data
+
+        Returns:
+            The stored data
+
+        Raises:
+            KeyError: If key not found
+        """
+        if key not in self._storage:
+            raise KeyError(f"Key '{key}' not found in memory storage")
+        return self._storage[key]
+
+    def exists(self, key: str) -> bool:
+        """Check if key exists in memory.
+
+        Args:
+            key: Unique identifier to check
+
+        Returns:
+            True if key exists, False otherwise
+        """
+        return key in self._storage
+
+    def delete(self, key: str) -> None:
+        """Delete data from memory.
+
+        Args:
+            key: Unique identifier for the data
+        """
+        self._storage.pop(key, None)
+
+    def list_keys(self, prefix: str = "") -> list[str]:
+        """List all keys with optional prefix.
+
+        Args:
+            prefix: Optional prefix to filter keys
+
+        Returns:
+            List of matching keys
+        """
+        if prefix:
+            return [k for k in self._storage.keys() if k.startswith(prefix)]
+        return list(self._storage.keys())
+
+    def clear(self) -> None:
+        """Clear all data from memory."""
+        self._storage.clear()
+
+    @property
+    def size_bytes(self) -> int:
+        """Get approximate memory usage in bytes.
+
+        Returns:
+            Approximate memory usage in bytes
+        """
+        return sum(sys.getsizeof(v) for v in self._storage.values())
+
+    @property
+    def count(self) -> int:
+        """Get number of stored items.
+
+        Returns:
+            Number of items in storage
+        """
+        return len(self._storage)

aip_agents/storage/providers/object_storage.py

@@ -0,0 +1,214 @@
+"""Object storage provider implementation.
+
+Authors:
+    Christian Trisno Sen Long Chen (christian.t.s.l.chen@gdplabs.id)
+    Fachriza Adhiatma (fachriza.d.adhiatma@gdplabs.id)
+"""
+
+import json
+import pickle
+from typing import Any
+
+from aip_agents.storage.base import BaseObjectStorageClient
+from aip_agents.storage.providers.base import BaseStorageProvider, StorageError
+from aip_agents.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class ObjectStorageProvider(BaseStorageProvider):
+    """Object storage provider for S3-compatible storage.
+
+    Works with any S3-compatible storage including AWS S3, MinIO,
+    Google Cloud Storage (with S3 compatibility), etc.
+
+    Best for:
+    - Very large datasets
+    - Distributed systems
+    - Cloud deployments
+    - Long-term storage
+    - Multi-region access
+    """
+
+    def __init__(self, client: BaseObjectStorageClient, prefix: str = "", use_json: bool = False):
+        """Initialize object storage provider.
+
+        Args:
+            client: Object storage client instance
+            prefix: Prefix for all keys (like a directory)
+            use_json: Use JSON format (True) or pickle (False)
+        """
+        self.client = client
+        self.prefix = prefix.rstrip("/") + "/" if prefix else ""
+        self.use_json = use_json
+
+    def _get_object_key(self, key: str) -> str:
+        """Get full object key with prefix.
+
+        Args:
+            key: Storage key
+
+        Returns:
+            Full object key with prefix
+        """
+        return f"{self.prefix}{key}"
+
+    def _serialize_data(self, data: Any) -> bytes:
+        """Serialize data to bytes.
+
+        Args:
+            data: Data to serialize
+
+        Returns:
+            Serialized data as bytes
+
+        Raises:
+            StorageError: If serialization fails
+        """
+        try:
+            if self.use_json:
+                return json.dumps(data, default=str).encode("utf-8")
+            return pickle.dumps(data)
+        except Exception as e:
+            raise StorageError(f"Failed to serialize data: {e}") from e
+
+    def _deserialize_data(self, data_bytes: bytes) -> Any:
+        """Deserialize data from bytes.
+
+        Args:
+            data_bytes: Serialized data
+
+        Returns:
+            Deserialized data
+
+        Raises:
+            StorageError: If deserialization fails
+        """
+        try:
+            if self.use_json:
+                return json.loads(data_bytes.decode("utf-8"))
+            return pickle.loads(data_bytes)
+        except Exception as e:
+            raise StorageError(f"Failed to deserialize data: {e}") from e
+
+    def store(self, key: str, data: Any) -> None:
+        """Store data in object storage.
+
+        Args:
+            key: Unique identifier for the data
+            data: Data to store
+
+        Raises:
+            StorageError: If storage operation fails
+        """
+        object_key = self._get_object_key(key)
+
+        try:
+            data_bytes = self._serialize_data(data)
+
+            self.client.upload(
+                object_key=object_key,
+                file_stream=data_bytes,
+                content_type="application/json" if self.use_json else "application/octet-stream",
+                metadata={"storage_key": key, "format": "json" if self.use_json else "pickle"},
+            )
+
+            logger.debug(f"Stored data to object storage: {object_key}")
+
+        except Exception as e:
+            raise StorageError(f"Failed to store data in object storage: {e}") from e
+
+    def retrieve(self, key: str) -> Any:
+        """Retrieve data from object storage.
+
+        Args:
+            key: Unique identifier for the data
+
+        Returns:
+            The stored data
+
+        Raises:
+            KeyError: If key not found
+            StorageError: If retrieval operation fails
+        """
+        object_key = self._get_object_key(key)
+
+        try:
+            data_bytes = self.client.get(object_key)
+            return self._deserialize_data(data_bytes)
+
+        except KeyError as e:
+            raise KeyError(f"Key '{key}' not found in object storage: {e}") from e
+        except Exception as e:
+            raise StorageError(f"Failed to retrieve data from object storage: {e}") from e
+
+    def exists(self, key: str) -> bool:
+        """Check if object exists.
+
+        Args:
+            key: Unique identifier to check
+
+        Returns:
+            True if key exists, False otherwise
+        """
+        object_key = self._get_object_key(key)
+
+        try:
+            return self.client.object_exists(object_key)
+        except Exception as e:
+            logger.warning(f"Failed to check object existence: {e}")
+            return False
+
+    def delete(self, key: str) -> None:
+        """Delete object.
+
+        Args:
+            key: Unique identifier for the data
+        """
+        object_key = self._get_object_key(key)
+
+        try:
+            self.client.delete(object_key)
+            logger.debug(f"Deleted object: {object_key}")
+        except Exception as e:
+            logger.warning(f"Failed to delete object {object_key}: {e}")
+
+    def list_keys(self, prefix: str = "") -> list[str]:
+        """List all keys with optional prefix.
+
+        Args:
+            prefix: Optional prefix to filter keys
+
+        Raises:
+            NotImplementedError: list_keys is not implemented
+        """
+        raise NotImplementedError("list_keys not yet implemented for object storage")
+
+    def clear(self) -> None:
+        """Clear all objects with the configured prefix.
+
+        Warning:
+            This is a dangerous operation!
+        """
+        logger.warning("clear() not implemented for object storage - too dangerous")
+        raise NotImplementedError("Batch clear not implemented for safety reasons")
+
+    def get_presigned_url(self, key: str, expires_hours: int = 24) -> str | None:
+        """Generate presigned URL for direct access.
+
+        Args:
+            key: Storage key
+            expires_hours: URL expiration in hours
+
+        Returns:
+            Presigned URL for direct access
+
+        Raises:
+            StorageError: If URL generation fails
+        """
+        object_key = self._get_object_key(key)
+
+        try:
+            return self.client.generate_presigned_url(object_key=object_key, expires=expires_hours)
+        except Exception as e:
+            raise StorageError(f"Failed to generate presigned URL: {e}") from e

aip_agents/storage/providers/object_storage.pyi

@@ -1,7 +1,7 @@
 from _typeshed import Incomplete
 from aip_agents.storage.base import BaseObjectStorageClient as BaseObjectStorageClient
 from aip_agents.storage.providers.base import BaseStorageProvider as BaseStorageProvider, StorageError as StorageError
-from aip_agents.utils.logger_manager import LoggerManager as LoggerManager
+from aip_agents.utils.logger import get_logger as get_logger
 from typing import Any
 
 logger: Incomplete

aip_agents/tools/__init__.py

@@ -0,0 +1,6 @@
+"""Initialize Tools Module for AIP Agents."""
+
+from aip_agents.tools.bosa_tools import BOSA_AUTOMATED_TOOLS
+from aip_agents.tools.gl_connector import GLConnectorTool
+
+__all__ = ["BOSA_AUTOMATED_TOOLS", "GLConnectorTool"]

aip_agents/tools/__init__.pyi

@@ -1,3 +1,4 @@
-from .bosa_tools import BOSA_AUTOMATED_TOOLS as BOSA_AUTOMATED_TOOLS
+from aip_agents.tools.bosa_tools import BOSA_AUTOMATED_TOOLS as BOSA_AUTOMATED_TOOLS
+from aip_agents.tools.gl_connector import GLConnectorTool as GLConnectorTool
 
-__all__ = ['BOSA_AUTOMATED_TOOLS']
+__all__ = ['BOSA_AUTOMATED_TOOLS', 'GLConnectorTool']

aip_agents/tools/bosa_tools.py

@@ -0,0 +1,105 @@
+"""Auto-generated tools from BOSA connector.
+
+Authors:
+    Saul Sayers (saul.sayers@gdplabs.id)
+"""
+
+from bosa_connectors import BosaConnector, BOSAConnectorToolGenerator
+from langchain_core.tools import BaseTool
+
+from aip_agents.tools.constants import BOSA_API_BASE_URL, BOSA_API_KEY, BOSA_FETCH_MAX_RETRIES, ToolType
+from aip_agents.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+def get_bosa_modules_with_retry() -> list[str]:
+    """Try to get available modules with retries.
+
+    Returns:
+        List of available modules.
+    """
+    if not BOSA_API_BASE_URL or not BOSA_API_KEY:
+        logger.warning("BOSA credentials missing (base_url or api_key); returning empty modules list")
+        return []
+
+    connector = BosaConnector(api_base_url=BOSA_API_BASE_URL, api_key=BOSA_API_KEY)
+    modules = []
+    for attempt in range(BOSA_FETCH_MAX_RETRIES):
+        try:
+            modules = list(connector.get_available_modules())
+            if modules:
+                return modules
+            logger.warning(
+                f"Failed to get BOSA available modules, retrying... (attempt {attempt + 1} / {BOSA_FETCH_MAX_RETRIES})"
+            )
+        except Exception as e:
+            logger.exception(
+                f"Exception when getting BOSA available modules (attempt {attempt + 1} / {BOSA_FETCH_MAX_RETRIES}): {e}"
+            )
+    logger.error("Failed to get BOSA available modules after maximum retries")
+    return modules
+
+
+class LazyBosaToolsDict(dict):
+    """Lazy dictionary for BOSA tools."""
+
+    def __missing__(self, app):
+        """When a key is missing, create the tools and store them in the dictionary.
+
+        Args:
+            app: Name of the BOSA connector.
+
+        Returns:
+            List of tools generated by BOSAConnectorToolGenerator.
+        """
+        if app not in get_bosa_modules():
+            return []
+        tools = []
+        for attempt in range(BOSA_FETCH_MAX_RETRIES):
+            try:
+                tools = BOSAConnectorToolGenerator(
+                    api_base_url=BOSA_API_BASE_URL,
+                    api_key=BOSA_API_KEY,
+                    app_name=app,
+                ).generate_tools(tool_type=ToolType.LANGCHAIN)
+                if tools:
+                    self[app] = tools
+                    return tools
+                logger.warning(
+                    f"Failed to create BOSA tools, retrying... (attempt {attempt + 1} / {BOSA_FETCH_MAX_RETRIES})"
+                )
+            except Exception as e:
+                logger.exception(
+                    f"Exception when creating BOSA tools for app '{app}' "
+                    f"(attempt {attempt + 1} / {BOSA_FETCH_MAX_RETRIES}): {e}"
+                )
+        logger.error("Failed to create BOSA tools after maximum retries")
+        return tools
+
+
+# Supported modules (dynamic)
+def get_bosa_modules() -> list[str]:
+    """Lazily fetch and cache BOSA modules.
+
+    This is for backwards compatibility with the old BOSA modules.
+
+    Returns:
+        List of BOSA modules.
+    """
+    if not hasattr(get_bosa_modules, "_cache"):
+        get_bosa_modules._cache = get_bosa_modules_with_retry()
+    return get_bosa_modules._cache
+
+
+# FOR BACKWARDS COMPATIBILITY
+BOSA_MODULES = [
+    "github",
+    "twitter",
+    "google",
+    "google_drive",
+    "google_mail",
+    "google_docs",
+]
+
+BOSA_AUTOMATED_TOOLS: dict[str, list[BaseTool]] = LazyBosaToolsDict()

aip_agents/tools/bosa_tools.pyi

@@ -1,6 +1,6 @@
 from _typeshed import Incomplete
-from aip_agents.tools.constants import BOSA_API_BASE_URL as BOSA_API_BASE_URL, BOSA_API_KEY as BOSA_API_KEY, BOSA_FETCH_MAX_RETRIES as BOSA_FETCH_MAX_RETRIES
-from aip_agents.utils.logger_manager import LoggerManager as LoggerManager
+from aip_agents.tools.constants import BOSA_API_BASE_URL as BOSA_API_BASE_URL, BOSA_API_KEY as BOSA_API_KEY, BOSA_FETCH_MAX_RETRIES as BOSA_FETCH_MAX_RETRIES, ToolType as ToolType
+from aip_agents.utils.logger import get_logger as get_logger
 from langchain_core.tools import BaseTool as BaseTool
 
 logger: Incomplete