agno 2.4.6__py3-none-any.whl → 2.4.8__py3-none-any.whl

agno/knowledge/loaders/__init__.py

@@ -0,0 +1,29 @@
+"""Remote content loaders for Knowledge.
+
+This module provides loaders for various cloud storage providers:
+- S3Loader: AWS S3
+- GCSLoader: Google Cloud Storage
+- SharePointLoader: Microsoft SharePoint
+- GitHubLoader: GitHub repositories
+- AzureBlobLoader: Azure Blob Storage
+
+All loaders inherit from BaseLoader which provides common utilities for
+computing content names, creating content entries, and merging metadata.
+"""
+
+from agno.knowledge.loaders.azure_blob import AzureBlobLoader
+from agno.knowledge.loaders.base import BaseLoader, FileToProcess
+from agno.knowledge.loaders.gcs import GCSLoader
+from agno.knowledge.loaders.github import GitHubLoader
+from agno.knowledge.loaders.s3 import S3Loader
+from agno.knowledge.loaders.sharepoint import SharePointLoader
+
+__all__ = [
+    "BaseLoader",
+    "FileToProcess",
+    "S3Loader",
+    "GCSLoader",
+    "SharePointLoader",
+    "GitHubLoader",
+    "AzureBlobLoader",
+]

agno/knowledge/loaders/azure_blob.py

@@ -0,0 +1,423 @@
+"""Azure Blob Storage content loader for Knowledge.
+
+Provides methods for loading content from Azure Blob Storage.
+"""
+
+# mypy: disable-error-code="attr-defined"
+
+from io import BytesIO
+from typing import Any, Dict, List, Optional, cast
+
+from agno.knowledge.content import Content, ContentStatus
+from agno.knowledge.loaders.base import BaseLoader
+from agno.knowledge.reader import Reader
+from agno.knowledge.remote_content.config import AzureBlobConfig, RemoteContentConfig
+from agno.knowledge.remote_content.remote_content import AzureBlobContent
+from agno.utils.log import log_debug, log_error, log_info, log_warning
+from agno.utils.string import generate_id
+
+
+class AzureBlobLoader(BaseLoader):
+    """Loader for Azure Blob Storage content."""
+
+    # ==========================================
+    # AZURE BLOB HELPERS (shared between sync/async)
+    # ==========================================
+
+    def _validate_azure_config(
+        self,
+        content: Content,
+        config: Optional[RemoteContentConfig],
+    ) -> Optional[AzureBlobConfig]:
+        """Validate and extract Azure Blob config.
+
+        Returns:
+            AzureBlobConfig if valid, None otherwise
+        """
+        remote_content: AzureBlobContent = cast(AzureBlobContent, content.remote_content)
+        azure_config = cast(AzureBlobConfig, config) if isinstance(config, AzureBlobConfig) else None
+
+        if azure_config is None:
+            log_error(f"Azure Blob config not found for config_id: {remote_content.config_id}")
+            return None
+
+        return azure_config
+
+    def _get_azure_blob_client(self, azure_config: AzureBlobConfig):
+        """Get a sync Azure Blob Service Client using client credentials flow.
+
+        Requires the `azure-identity` and `azure-storage-blob` packages.
+        """
+        try:
+            from azure.identity import ClientSecretCredential  # type: ignore
+            from azure.storage.blob import BlobServiceClient  # type: ignore
+        except ImportError:
+            raise ImportError(
+                "The `azure-identity` and `azure-storage-blob` packages are not installed. "
+                "Please install them via `pip install azure-identity azure-storage-blob`."
+            )
+
+        credential = ClientSecretCredential(
+            tenant_id=azure_config.tenant_id,
+            client_id=azure_config.client_id,
+            client_secret=azure_config.client_secret,
+        )
+
+        blob_service = BlobServiceClient(
+            account_url=f"https://{azure_config.storage_account}.blob.core.windows.net",
+            credential=credential,
+        )
+
+        return blob_service
+
+    def _get_azure_blob_client_async(self, azure_config: AzureBlobConfig):
+        """Get an async Azure Blob Service Client using client credentials flow.
+
+        Requires the `azure-identity` and `azure-storage-blob` packages.
+        Uses the async versions from azure.storage.blob.aio and azure.identity.aio.
+        """
+        try:
+            from azure.identity.aio import ClientSecretCredential  # type: ignore
+            from azure.storage.blob.aio import BlobServiceClient  # type: ignore
+        except ImportError:
+            raise ImportError(
+                "The `azure-identity` and `azure-storage-blob` packages are not installed. "
+                "Please install them via `pip install azure-identity azure-storage-blob`."
+            )
+
+        credential = ClientSecretCredential(
+            tenant_id=azure_config.tenant_id,
+            client_id=azure_config.client_id,
+            client_secret=azure_config.client_secret,
+        )
+
+        blob_service = BlobServiceClient(
+            account_url=f"https://{azure_config.storage_account}.blob.core.windows.net",
+            credential=credential,
+        )
+
+        return blob_service
+
+    def _build_azure_metadata(
+        self,
+        azure_config: AzureBlobConfig,
+        blob_name: str,
+        file_name: str,
+    ) -> Dict[str, str]:
+        """Build Azure Blob-specific metadata dictionary."""
+        return {
+            "source_type": "azure_blob",
+            "source_config_id": azure_config.id,
+            "source_config_name": azure_config.name,
+            "azure_storage_account": azure_config.storage_account,
+            "azure_container": azure_config.container,
+            "azure_blob_name": blob_name,
+            "azure_filename": file_name,
+        }
+
+    def _build_azure_virtual_path(
+        self,
+        storage_account: str,
+        container: str,
+        blob_name: str,
+    ) -> str:
+        """Build virtual path for Azure Blob content."""
+        return f"azure://{storage_account}/{container}/{blob_name}"
+
+    def _get_azure_root_path(self, remote_content: AzureBlobContent) -> str:
+        """Get the root path for computing relative paths."""
+        return remote_content.prefix or ""
+
+    # ==========================================
+    # AZURE BLOB LOADERS
+    # ==========================================
+
+    async def _aload_from_azure_blob(
+        self,
+        content: Content,
+        upsert: bool,
+        skip_if_exists: bool,
+        config: Optional[RemoteContentConfig] = None,
+    ):
+        """Load content from Azure Blob Storage (async).
+
+        Requires the AzureBlobConfig to contain tenant_id, client_id, client_secret,
+        storage_account, and container.
+
+        Uses the async Azure SDK to avoid blocking the event loop.
+        """
+        remote_content: AzureBlobContent = cast(AzureBlobContent, content.remote_content)
+        azure_config = self._validate_azure_config(content, config)
+        if azure_config is None:
+            return
+
+        # Get async blob service client
+        try:
+            blob_service = self._get_azure_blob_client_async(azure_config)
+        except ImportError as e:
+            log_error(str(e))
+            return
+        except Exception as e:
+            log_error(f"Error creating Azure Blob client: {e}")
+            return
+
+        # Use async context manager for proper resource cleanup
+        async with blob_service:
+            container_client = blob_service.get_container_client(azure_config.container)
+
+            # Helper to list blobs with a given prefix (async)
+            async def list_blobs_with_prefix(prefix: str) -> List[Dict[str, Any]]:
+                """List all blobs under a given prefix (folder)."""
+                results: List[Dict[str, Any]] = []
+                normalized_prefix = prefix.rstrip("/") + "/" if not prefix.endswith("/") else prefix
+                async for blob in container_client.list_blobs(name_starts_with=normalized_prefix):
+                    if not blob.name.endswith("/"):
+                        results.append(
+                            {
+                                "name": blob.name,
+                                "size": blob.size,
+                                "content_type": blob.content_settings.content_type if blob.content_settings else None,
+                            }
+                        )
+                return results
+
+            # Identify blobs to process
+            blobs_to_process: List[Dict[str, Any]] = []
+
+            try:
+                if remote_content.blob_name:
+                    blob_client = container_client.get_blob_client(remote_content.blob_name)
+                    try:
+                        props = await blob_client.get_blob_properties()
+                        blobs_to_process.append(
+                            {
+                                "name": remote_content.blob_name,
+                                "size": props.size,
+                                "content_type": props.content_settings.content_type if props.content_settings else None,
+                            }
+                        )
+                    except Exception:
+                        log_debug(f"Blob {remote_content.blob_name} not found, checking if it's a folder...")
+                        blobs_to_process = await list_blobs_with_prefix(remote_content.blob_name)
+                        if not blobs_to_process:
+                            log_error(
+                                f"No blob or folder found at path: {remote_content.blob_name}. "
+                                "If this is a folder, ensure files exist inside it."
+                            )
+                            return
+                elif remote_content.prefix:
+                    blobs_to_process = await list_blobs_with_prefix(remote_content.prefix)
+            except Exception as e:
+                log_error(f"Error listing Azure blobs: {e}")
+                return
+
+            if not blobs_to_process:
+                log_warning(f"No blobs found in Azure container: {azure_config.container}")
+                return
+
+            log_info(f"Processing {len(blobs_to_process)} file(s) from Azure Blob Storage")
+            is_folder_upload = len(blobs_to_process) > 1
+            root_path = self._get_azure_root_path(remote_content)
+
+            for blob_info in blobs_to_process:
+                blob_name = blob_info["name"]
+                file_name = blob_name.split("/")[-1]
+
+                # Build metadata and virtual path using helpers
+                virtual_path = self._build_azure_virtual_path(
+                    azure_config.storage_account, azure_config.container, blob_name
+                )
+                azure_metadata = self._build_azure_metadata(azure_config, blob_name, file_name)
+                merged_metadata = self._merge_metadata(azure_metadata, content.metadata)
+
+                # Compute content name using base helper
+                content_name = self._compute_content_name(
+                    blob_name, file_name, content.name, root_path, is_folder_upload
+                )
+
+                # Create content entry using base helper
+                content_entry = self._create_content_entry(
+                    content, content_name, virtual_path, merged_metadata, "azure_blob", is_folder_upload
+                )
+
+                await self._ainsert_contents_db(content_entry)
+
+                if self._should_skip(content_entry.content_hash, skip_if_exists):
+                    content_entry.status = ContentStatus.COMPLETED
+                    await self._aupdate_content(content_entry)
+                    continue
+
+                # Download blob (async)
+                try:
+                    blob_client = container_client.get_blob_client(blob_name)
+                    download_stream = await blob_client.download_blob()
+                    blob_data = await download_stream.readall()
+                    file_content = BytesIO(blob_data)
+                except Exception as e:
+                    log_error(f"Error downloading Azure blob {blob_name}: {e}")
+                    content_entry.status = ContentStatus.FAILED
+                    content_entry.status_message = str(e)
+                    await self._aupdate_content(content_entry)
+                    continue
+
+                # Select reader and read content
+                reader = self._select_reader_by_uri(file_name, content.reader)
+                if reader is None:
+                    log_warning(f"No reader found for file: {file_name}")
+                    content_entry.status = ContentStatus.FAILED
+                    content_entry.status_message = "No suitable reader found"
+                    await self._aupdate_content(content_entry)
+                    continue
+
+                reader = cast(Reader, reader)
+                read_documents = await reader.async_read(file_content, name=file_name)
+
+                # Prepare and insert into vector database
+                if not content_entry.id:
+                    content_entry.id = generate_id(content_entry.content_hash or "")
+                self._prepare_documents_for_insert(read_documents, content_entry.id)
+                await self._ahandle_vector_db_insert(content_entry, read_documents, upsert)
+
+    def _load_from_azure_blob(
+        self,
+        content: Content,
+        upsert: bool,
+        skip_if_exists: bool,
+        config: Optional[RemoteContentConfig] = None,
+    ):
+        """Load content from Azure Blob Storage (sync).
+
+        Requires the AzureBlobConfig to contain tenant_id, client_id, client_secret,
+        storage_account, and container.
+        """
+        remote_content: AzureBlobContent = cast(AzureBlobContent, content.remote_content)
+        azure_config = self._validate_azure_config(content, config)
+        if azure_config is None:
+            return
+
+        # Get blob service client
+        try:
+            blob_service = self._get_azure_blob_client(azure_config)
+        except ImportError as e:
+            log_error(str(e))
+            return
+        except Exception as e:
+            log_error(f"Error creating Azure Blob client: {e}")
+            return
+
+        # Use context manager for proper resource cleanup
+        with blob_service:
+            container_client = blob_service.get_container_client(azure_config.container)
+
+            # Helper to list blobs with a given prefix
+            def list_blobs_with_prefix(prefix: str) -> List[Dict[str, Any]]:
+                """List all blobs under a given prefix (folder)."""
+                results: List[Dict[str, Any]] = []
+                normalized_prefix = prefix.rstrip("/") + "/" if not prefix.endswith("/") else prefix
+                blobs = container_client.list_blobs(name_starts_with=normalized_prefix)
+                for blob in blobs:
+                    if not blob.name.endswith("/"):
+                        results.append(
+                            {
+                                "name": blob.name,
+                                "size": blob.size,
+                                "content_type": blob.content_settings.content_type if blob.content_settings else None,
+                            }
+                        )
+                return results
+
+            # Identify blobs to process
+            blobs_to_process: List[Dict[str, Any]] = []
+
+            try:
+                if remote_content.blob_name:
+                    blob_client = container_client.get_blob_client(remote_content.blob_name)
+                    try:
+                        props = blob_client.get_blob_properties()
+                        blobs_to_process.append(
+                            {
+                                "name": remote_content.blob_name,
+                                "size": props.size,
+                                "content_type": props.content_settings.content_type if props.content_settings else None,
+                            }
+                        )
+                    except Exception:
+                        log_debug(f"Blob {remote_content.blob_name} not found, checking if it's a folder...")
+                        blobs_to_process = list_blobs_with_prefix(remote_content.blob_name)
+                        if not blobs_to_process:
+                            log_error(
+                                f"No blob or folder found at path: {remote_content.blob_name}. "
+                                "If this is a folder, ensure files exist inside it."
+                            )
+                            return
+                elif remote_content.prefix:
+                    blobs_to_process = list_blobs_with_prefix(remote_content.prefix)
+            except Exception as e:
+                log_error(f"Error listing Azure blobs: {e}")
+                return
+
+            if not blobs_to_process:
+                log_warning(f"No blobs found in Azure container: {azure_config.container}")
+                return
+
+            log_info(f"Processing {len(blobs_to_process)} file(s) from Azure Blob Storage")
+            is_folder_upload = len(blobs_to_process) > 1
+            root_path = self._get_azure_root_path(remote_content)
+
+            for blob_info in blobs_to_process:
+                blob_name = blob_info["name"]
+                file_name = blob_name.split("/")[-1]
+
+                # Build metadata and virtual path using helpers
+                virtual_path = self._build_azure_virtual_path(
+                    azure_config.storage_account, azure_config.container, blob_name
+                )
+                azure_metadata = self._build_azure_metadata(azure_config, blob_name, file_name)
+                merged_metadata = self._merge_metadata(azure_metadata, content.metadata)
+
+                # Compute content name using base helper
+                content_name = self._compute_content_name(
+                    blob_name, file_name, content.name, root_path, is_folder_upload
+                )
+
+                # Create content entry using base helper
+                content_entry = self._create_content_entry(
+                    content, content_name, virtual_path, merged_metadata, "azure_blob", is_folder_upload
+                )
+
+                self._insert_contents_db(content_entry)
+
+                if self._should_skip(content_entry.content_hash, skip_if_exists):
+                    content_entry.status = ContentStatus.COMPLETED
+                    self._update_content(content_entry)
+                    continue
+
+                # Download blob
+                try:
+                    blob_client = container_client.get_blob_client(blob_name)
+                    download_stream = blob_client.download_blob()
+                    file_content = BytesIO(download_stream.readall())
+                except Exception as e:
+                    log_error(f"Error downloading Azure blob {blob_name}: {e}")
+                    content_entry.status = ContentStatus.FAILED
+                    content_entry.status_message = str(e)
+                    self._update_content(content_entry)
+                    continue
+
+                # Select reader and read content
+                reader = self._select_reader_by_uri(file_name, content.reader)
+                if reader is None:
+                    log_warning(f"No reader found for file: {file_name}")
+                    content_entry.status = ContentStatus.FAILED
+                    content_entry.status_message = "No suitable reader found"
+                    self._update_content(content_entry)
+                    continue
+
+                reader = cast(Reader, reader)
+                read_documents = reader.read(file_content, name=file_name)
+
+                # Prepare and insert into vector database
+                if not content_entry.id:
+                    content_entry.id = generate_id(content_entry.content_hash or "")
+                self._prepare_documents_for_insert(read_documents, content_entry.id)
+                self._handle_vector_db_insert(content_entry, read_documents, upsert)

agno/knowledge/loaders/base.py

@@ -0,0 +1,187 @@
+"""Base loader class with shared utilities for all content loaders.
+
+Provides common helpers for:
+- Computing content names for files
+- Creating Content entries
+- Building metadata dictionaries
+"""
+
+from dataclasses import dataclass
+from typing import Any, Dict, List, Optional
+
+from agno.knowledge.content import Content, ContentStatus
+from agno.utils.string import generate_id
+
+
+@dataclass
+class FileToProcess:
+    """Represents a file identified for processing."""
+
+    path: str
+    name: str
+    size: Optional[int] = None
+    content_type: Optional[str] = None
+
+
+class BaseLoader:
+    """Base class with shared loader utilities.
+
+    This class provides common methods used by all content loaders to reduce
+    code duplication between sync and async implementations.
+
+    Methods that call self._build_content_hash() assume they are mixed into
+    a class that provides this method (e.g., Knowledge via RemoteKnowledge).
+    """
+
+    def _compute_content_name(
+        self,
+        file_path: str,
+        file_name: str,
+        base_name: Optional[str],
+        root_path: str,
+        is_folder_upload: bool,
+    ) -> str:
+        """Compute the content name for a file.
+
+        Args:
+            file_path: Full path to the file
+            file_name: Name of the file
+            base_name: User-provided base name for the content
+            root_path: Root path of the upload (for computing relative paths)
+            is_folder_upload: Whether this is part of a folder upload
+
+        Returns:
+            The computed content name
+        """
+        if is_folder_upload:
+            relative_path = file_path
+            if root_path and file_path.startswith(root_path + "/"):
+                relative_path = file_path[len(root_path) + 1 :]
+            return f"{base_name}/{relative_path}" if base_name else file_path
+        return base_name or file_name
+
+    def _create_content_entry_for_folder(
+        self,
+        content: Content,
+        content_name: str,
+        virtual_path: str,
+        metadata: Dict[str, Any],
+        file_type: str,
+    ) -> Content:
+        """Create a new Content entry for a file in a folder upload.
+
+        Args:
+            content: Original content object (used for description)
+            content_name: Name for the new content entry
+            virtual_path: Virtual path for hashing
+            metadata: Metadata dictionary
+            file_type: Type of file (e.g., 'github', 'azure_blob')
+
+        Returns:
+            New Content entry with hash and ID set
+        """
+        entry = Content(
+            name=content_name,
+            description=content.description,
+            path=virtual_path,
+            status=ContentStatus.PROCESSING,
+            metadata=metadata,
+            file_type=file_type,
+        )
+        entry.content_hash = self._build_content_hash(entry)  # type: ignore[attr-defined]
+        entry.id = generate_id(entry.content_hash)
+        return entry
+
+    def _update_content_entry_for_single_file(
+        self,
+        content: Content,
+        virtual_path: str,
+        metadata: Dict[str, Any],
+        file_type: str,
+    ) -> Content:
+        """Update an existing Content entry for a single file upload.
+
+        Args:
+            content: Original content object to update
+            virtual_path: Virtual path for hashing
+            metadata: Metadata dictionary
+            file_type: Type of file (e.g., 'github', 'azure_blob')
+
+        Returns:
+            Updated Content entry with hash and ID set if not already present
+        """
+        content.path = virtual_path
+        content.status = ContentStatus.PROCESSING
+        content.metadata = metadata
+        content.file_type = file_type
+        if not content.content_hash:
+            content.content_hash = self._build_content_hash(content)  # type: ignore[attr-defined]
+        if not content.id:
+            content.id = generate_id(content.content_hash)
+        return content
+
+    def _create_content_entry(
+        self,
+        content: Content,
+        content_name: str,
+        virtual_path: str,
+        metadata: Dict[str, Any],
+        file_type: str,
+        is_folder_upload: bool,
+    ) -> Content:
+        """Create or update a Content entry for a file.
+
+        For folder uploads, creates a new Content entry.
+        For single file uploads, updates the original Content object.
+
+        Args:
+            content: Original content object
+            content_name: Name for the content entry
+            virtual_path: Virtual path for hashing
+            metadata: Metadata dictionary
+            file_type: Type of file (e.g., 'github', 'azure_blob')
+            is_folder_upload: Whether this is part of a folder upload
+
+        Returns:
+            Content entry with hash and ID set
+        """
+        if is_folder_upload:
+            return self._create_content_entry_for_folder(content, content_name, virtual_path, metadata, file_type)
+        return self._update_content_entry_for_single_file(content, virtual_path, metadata, file_type)
+
+    def _merge_metadata(
+        self,
+        provider_metadata: Dict[str, str],
+        user_metadata: Optional[Dict[str, Any]],
+    ) -> Dict[str, Any]:
+        """Merge provider metadata with user-provided metadata.
+
+        User metadata takes precedence over provider metadata.
+
+        Args:
+            provider_metadata: Metadata from the provider (e.g., GitHub, Azure)
+            user_metadata: User-provided metadata
+
+        Returns:
+            Merged metadata dictionary
+        """
+        return {**provider_metadata, **(user_metadata or {})}
+
+    def _files_to_dict_list(self, files: List[FileToProcess]) -> List[Dict[str, Any]]:
+        """Convert FileToProcess objects to dict list for compatibility.
+
+        Args:
+            files: List of FileToProcess objects
+
+        Returns:
+            List of dictionaries with file info
+        """
+        return [
+            {
+                "path": f.path,
+                "name": f.name,
+                "size": f.size,
+                "content_type": f.content_type,
+            }
+            for f in files
+        ]