gnosisllm-knowledge 0.2.0__py3-none-any.whl

gnosisllm_knowledge/loaders/base.py

@@ -0,0 +1,399 @@
+"""Base loader with common functionality (Template Method Pattern)."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from abc import ABC, abstractmethod
+from collections.abc import AsyncIterator, Callable
+from typing import Any
+
+from gnosisllm_knowledge.core.domain.document import Document
+from gnosisllm_knowledge.core.domain.result import LoadResult, ValidationResult
+from gnosisllm_knowledge.core.events.emitter import EventEmitter
+from gnosisllm_knowledge.core.events.types import DocumentLoadedEvent, EventType
+from gnosisllm_knowledge.core.interfaces.chunker import ITextChunker
+from gnosisllm_knowledge.core.interfaces.fetcher import IContentFetcher
+
+# Default concurrency for parallel URL fetching
+DEFAULT_MAX_CONCURRENT = 10
+
+
+class BaseLoader(ABC):
+    """Base class for content loaders (Template Method Pattern).
+
+    This class provides common functionality for loading content from
+    various sources. Subclasses implement the `_get_urls` method to
+    define how URLs are discovered from the source.
+
+    Features:
+    - Parallel URL fetching with configurable concurrency
+    - Streaming support for memory-efficient processing
+    - Event emission for progress tracking
+    - Automatic chunking of fetched content
+    - Configurable options per source type
+
+    Example:
+        ```python
+        class MyLoader(BaseLoader):
+            @property
+            def name(self) -> str:
+                return "my-loader"
+
+            def supports(self, source: str) -> bool:
+                return source.startswith("my://")
+
+            async def _get_urls(self, source: str, **options) -> list[str]:
+                # Return list of URLs to fetch
+                return [source.replace("my://", "https://")]
+        ```
+    """
+
+    def __init__(
+        self,
+        fetcher: IContentFetcher,
+        chunker: ITextChunker,
+        config: dict[str, Any] | None = None,
+        event_emitter: EventEmitter | None = None,
+    ) -> None:
+        """Initialize the loader with dependencies.
+
+        Args:
+            fetcher: Content fetcher for retrieving URL content.
+            chunker: Text chunker for splitting content into documents.
+            config: Optional configuration dictionary.
+            event_emitter: Optional event emitter for progress events.
+        """
+        self._fetcher = fetcher
+        self._chunker = chunker
+        self._config = config or {}
+        self._events = event_emitter or EventEmitter()
+        self._logger = logging.getLogger(self.__class__.__name__)
+        self._max_concurrent = self._config.get("max_concurrent", DEFAULT_MAX_CONCURRENT)
+
+    @property
+    @abstractmethod
+    def name(self) -> str:
+        """Return the loader name for registry identification."""
+        ...
+
+    @abstractmethod
+    def supports(self, source: str) -> bool:
+        """Check if this loader supports the given source.
+
+        Args:
+            source: The source URL or path.
+
+        Returns:
+            True if this loader can handle the source.
+        """
+        ...
+
+    @abstractmethod
+    async def _get_urls(self, source: str, **options: Any) -> list[str]:
+        """Get list of URLs to process from the source.
+
+        This method is implemented by subclasses to define how
+        URLs are discovered from the source.
+
+        Args:
+            source: The source URL or path.
+            **options: Loader-specific options.
+
+        Returns:
+            List of URLs to fetch and process.
+        """
+        ...
+
+    async def validate_source(self, source: str) -> ValidationResult:
+        """Validate that the source is accessible and valid.
+
+        Default implementation checks if fetcher can reach the source.
+        Subclasses can override for specific validation logic.
+
+        Args:
+            source: The source URL or path.
+
+        Returns:
+            ValidationResult with validation status.
+        """
+        try:
+            # Check if we can get URLs from source
+            urls = await self._get_urls(source)
+            if not urls:
+                return ValidationResult(
+                    valid=False,
+                    message=f"No URLs found in source: {source}",
+                )
+            return ValidationResult(
+                valid=True,
+                message=f"Source valid with {len(urls)} URLs",
+                metadata={"url_count": len(urls)},
+            )
+        except Exception as e:
+            return ValidationResult(
+                valid=False,
+                message=str(e),
+                errors=[str(e)],
+            )
+
+    async def load(self, source: str, **options: Any) -> LoadResult:
+        """Load all documents from source with parallel URL fetching.
+
+        Args:
+            source: The source URL or path.
+            **options: Loader-specific options.
+
+        Returns:
+            LoadResult with loaded documents and metadata.
+        """
+        import time
+
+        start_time = time.time()
+        documents: list[Document] = []
+        urls_processed = 0
+        urls_failed = 0
+
+        try:
+            urls = await self._get_urls(source, **options)
+
+            if not urls:
+                return LoadResult(
+                    source=source,
+                    source_type=self.name,
+                    document_count=0,
+                    success=True,
+                    duration_ms=(time.time() - start_time) * 1000,
+                )
+
+            self._logger.info(
+                f"Loading {len(urls)} URLs with concurrency={self._max_concurrent}"
+            )
+
+            # Use semaphore to limit concurrent fetches
+            semaphore = asyncio.Semaphore(self._max_concurrent)
+
+            async def fetch_with_semaphore(url: str) -> list[Document]:
+                async with semaphore:
+                    return await self._load_url(url, source, **options)
+
+            # Process all URLs in parallel (limited by semaphore)
+            results = await asyncio.gather(
+                *[fetch_with_semaphore(url) for url in urls],
+                return_exceptions=True,
+            )
+
+            # Flatten results, handling exceptions gracefully
+            for i, result in enumerate(results):
+                if isinstance(result, Exception):
+                    self._logger.error(f"Failed to load URL {urls[i]}: {result}")
+                    urls_failed += 1
+                elif isinstance(result, list):
+                    documents.extend(result)
+                    urls_processed += 1
+
+            duration_ms = (time.time() - start_time) * 1000
+
+            return LoadResult(
+                source=source,
+                source_type=self.name,
+                document_count=len(documents),
+                success=True,
+                duration_ms=duration_ms,
+                urls_processed=urls_processed,
+                urls_failed=urls_failed,
+                bytes_loaded=sum(len(d.content) for d in documents),
+                metadata={"documents": documents},
+            )
+
+        except Exception as e:
+            self._logger.error(f"Load failed: {e}")
+            return LoadResult(
+                source=source,
+                source_type=self.name,
+                document_count=0,
+                success=False,
+                error_message=str(e),
+                duration_ms=(time.time() - start_time) * 1000,
+            )
+
+    async def load_streaming(
+        self,
+        source: str,
+        **options: Any,
+    ) -> AsyncIterator[Document]:
+        """Stream documents from source for memory-efficient processing.
+
+        Yields documents as they are loaded, allowing for immediate
+        processing without waiting for all documents to load.
+
+        Args:
+            source: The source URL or path.
+            **options: Loader-specific options.
+
+        Yields:
+            Document objects as they are loaded.
+        """
+        urls = await self._get_urls(source, **options)
+
+        if not urls:
+            return
+
+        self._logger.info(
+            f"Streaming {len(urls)} URLs with concurrency={self._max_concurrent}"
+        )
+
+        semaphore = asyncio.Semaphore(self._max_concurrent)
+
+        async def fetch_with_semaphore(url: str) -> tuple[str, list[Document]]:
+            async with semaphore:
+                docs = await self._load_url(url, source, **options)
+                return (url, docs)
+
+        # Create tasks for all URLs
+        tasks = [asyncio.create_task(fetch_with_semaphore(url)) for url in urls]
+
+        # Process results as they complete (streaming behavior)
+        for coro in asyncio.as_completed(tasks):
+            try:
+                url, url_docs = await coro
+                for doc in url_docs:
+                    yield doc
+            except Exception as e:
+                self._logger.error(f"Failed to load URL: {e}")
+
+    async def load_with_callback(
+        self,
+        source: str,
+        callback: Callable[[list[Document]], Any],
+        batch_size: int = 5,
+        **options: Any,
+    ) -> int:
+        """Load documents with a callback for batch processing.
+
+        Args:
+            source: The source URL or path.
+            callback: Callback function called with each batch.
+            batch_size: Number of documents per batch.
+            **options: Loader-specific options.
+
+        Returns:
+            Total number of documents loaded.
+        """
+        urls = await self._get_urls(source, **options)
+
+        if not urls:
+            return 0
+
+        self._logger.info(
+            f"Streaming {len(urls)} URLs with concurrency={self._max_concurrent}"
+        )
+
+        total = 0
+        batch: list[Document] = []
+        semaphore = asyncio.Semaphore(self._max_concurrent)
+
+        async def fetch_with_semaphore(url: str) -> tuple[str, list[Document]]:
+            async with semaphore:
+                docs = await self._load_url(url, source, **options)
+                return (url, docs)
+
+        # Create tasks for all URLs
+        tasks = [asyncio.create_task(fetch_with_semaphore(url)) for url in urls]
+
+        # Process results as they complete
+        for coro in asyncio.as_completed(tasks):
+            try:
+                url, url_docs = await coro
+                batch.extend(url_docs)
+
+                # Stream batches as they fill up
+                while len(batch) >= batch_size:
+                    await self._invoke_callback(callback, batch[:batch_size])
+                    total += batch_size
+                    batch = batch[batch_size:]
+
+            except Exception as e:
+                self._logger.error(f"Failed to load URL: {e}")
+
+        # Flush remaining batch
+        if batch:
+            await self._invoke_callback(callback, batch)
+            total += len(batch)
+
+        return total
+
+    async def _invoke_callback(
+        self, callback: Callable[[list[Document]], Any], documents: list[Document]
+    ) -> None:
+        """Invoke callback, handling both sync and async callbacks.
+
+        Args:
+            callback: The callback function to invoke.
+            documents: List of documents to pass to callback.
+        """
+        import inspect
+
+        result = callback(documents)
+        if inspect.iscoroutine(result):
+            await result
+
+    async def _load_url(
+        self,
+        url: str,
+        source: str,
+        **options: Any,
+    ) -> list[Document]:
+        """Load and chunk content from a single URL.
+
+        Args:
+            url: The URL to fetch.
+            source: The original source identifier.
+            **options: Loader-specific options.
+
+        Returns:
+            List of Document objects from the URL.
+        """
+        try:
+            result = await self._fetcher.fetch(url, **options)
+            chunks = self._chunker.chunk(result.content)
+
+            documents: list[Document] = []
+            for chunk in chunks:
+                doc = Document(
+                    content=chunk.content,
+                    source=source,
+                    url=url,
+                    title=result.title,
+                    chunk_index=chunk.index,
+                    total_chunks=len(chunks),
+                    metadata={
+                        "loader": self.name,
+                        "fetch_url": url,
+                        "content_type": result.content_type,
+                    },
+                )
+                documents.append(doc)
+
+            # Emit event
+            self._events.emit(
+                DocumentLoadedEvent(
+                    url=url,
+                    source=source,
+                    chunks_count=len(chunks),
+                    content_length=len(result.content),
+                )
+            )
+
+            return documents
+
+        except Exception as e:
+            self._logger.error(f"Failed to load URL {url}: {e}")
+            self._events.emit(
+                type(
+                    "Event",
+                    (),
+                    {"event_type": EventType.ERROR, "data": {"url": url, "error": str(e)}},
+                )()  # type: ignore[arg-type]
+            )
+            return []

gnosisllm_knowledge/loaders/factory.py

@@ -0,0 +1,202 @@
+"""Loader factory with registry pattern."""
+
+from __future__ import annotations
+
+import logging
+from collections.abc import Callable
+from typing import Any
+
+from gnosisllm_knowledge.core.events.emitter import EventEmitter
+from gnosisllm_knowledge.core.interfaces.chunker import ITextChunker
+from gnosisllm_knowledge.core.interfaces.fetcher import IContentFetcher
+from gnosisllm_knowledge.loaders.base import BaseLoader
+from gnosisllm_knowledge.loaders.sitemap import SitemapLoader
+from gnosisllm_knowledge.loaders.website import WebsiteLoader
+
+# Type for loader factory functions
+LoaderCreator = Callable[
+    [IContentFetcher, ITextChunker, dict[str, Any] | None, EventEmitter | None],
+    BaseLoader,
+]
+
+
+class LoaderFactory:
+    """Factory for creating content loaders (Registry Pattern).
+
+    The factory maintains a registry of loader types and can create
+    the appropriate loader based on source URL or explicit type.
+
+    Built-in loaders:
+    - website: Single URL loading
+    - sitemap: Sitemap XML with recursive discovery
+
+    Example:
+        ```python
+        factory = LoaderFactory(fetcher, chunker)
+
+        # Auto-detect source type
+        loader = factory.create_for_source("https://example.com/sitemap.xml")
+
+        # Explicit type
+        loader = factory.create("sitemap", config={"max_urls": 500})
+
+        # Register custom loader
+        factory.register("custom", MyCustomLoader)
+        ```
+    """
+
+    def __init__(
+        self,
+        fetcher: IContentFetcher,
+        chunker: ITextChunker,
+        default_config: dict[str, Any] | None = None,
+        event_emitter: EventEmitter | None = None,
+    ) -> None:
+        """Initialize the factory with dependencies.
+
+        Args:
+            fetcher: Content fetcher for all loaders.
+            chunker: Text chunker for all loaders.
+            default_config: Default configuration for all loaders.
+            event_emitter: Event emitter for all loaders.
+        """
+        self._fetcher = fetcher
+        self._chunker = chunker
+        self._default_config = default_config or {}
+        self._events = event_emitter or EventEmitter()
+        self._logger = logging.getLogger(__name__)
+
+        # Registry of loader creators
+        self._registry: dict[str, LoaderCreator] = {}
+
+        # Register built-in loaders
+        self._register_builtin_loaders()
+
+    def _register_builtin_loaders(self) -> None:
+        """Register built-in loader types."""
+        self.register("website", lambda f, c, cfg, e: WebsiteLoader(f, c, cfg, e))
+        self.register("sitemap", lambda f, c, cfg, e: SitemapLoader(f, c, cfg, e))
+
+    def register(self, name: str, creator: LoaderCreator) -> None:
+        """Register a loader type.
+
+        Args:
+            name: Loader type name.
+            creator: Factory function that creates the loader.
+
+        Example:
+            ```python
+            factory.register("custom", lambda f, c, cfg, e: MyLoader(f, c, cfg, e))
+            ```
+        """
+        self._registry[name.lower()] = creator
+        self._logger.debug(f"Registered loader type: {name}")
+
+    def unregister(self, name: str) -> bool:
+        """Unregister a loader type.
+
+        Args:
+            name: Loader type name to remove.
+
+        Returns:
+            True if removed, False if not found.
+        """
+        name = name.lower()
+        if name in self._registry:
+            del self._registry[name]
+            return True
+        return False
+
+    def list_types(self) -> list[str]:
+        """List all registered loader types.
+
+        Returns:
+            List of loader type names.
+        """
+        return list(self._registry.keys())
+
+    def create(
+        self,
+        loader_type: str,
+        config: dict[str, Any] | None = None,
+    ) -> BaseLoader:
+        """Create a loader by type name.
+
+        Args:
+            loader_type: Type of loader to create.
+            config: Optional configuration (merged with defaults).
+
+        Returns:
+            Configured loader instance.
+
+        Raises:
+            ValueError: If loader type is not registered.
+        """
+        loader_type = loader_type.lower()
+
+        if loader_type not in self._registry:
+            available = ", ".join(self._registry.keys())
+            raise ValueError(
+                f"Unknown loader type: {loader_type}. Available: {available}"
+            )
+
+        # Merge default config with provided config
+        merged_config = {**self._default_config, **(config or {})}
+
+        creator = self._registry[loader_type]
+        return creator(self._fetcher, self._chunker, merged_config, self._events)
+
+    def create_for_source(
+        self,
+        source: str,
+        config: dict[str, Any] | None = None,
+    ) -> BaseLoader:
+        """Create the appropriate loader for a source URL.
+
+        Auto-detects the loader type based on the source URL.
+
+        Args:
+            source: The source URL or path.
+            config: Optional configuration.
+
+        Returns:
+            Loader that supports the source.
+
+        Raises:
+            ValueError: If no loader supports the source.
+        """
+        # Check each registered loader
+        for loader_type, creator in self._registry.items():
+            merged_config = {**self._default_config, **(config or {})}
+            loader = creator(self._fetcher, self._chunker, merged_config, self._events)
+            if loader.supports(source):
+                self._logger.debug(f"Auto-selected loader type '{loader_type}' for {source}")
+                return loader
+
+        raise ValueError(f"No loader supports source: {source}")
+
+    def detect_type(self, source: str) -> str | None:
+        """Detect the loader type for a source URL.
+
+        Args:
+            source: The source URL or path.
+
+        Returns:
+            Loader type name or None if not detected.
+        """
+        for loader_type, creator in self._registry.items():
+            loader = creator(self._fetcher, self._chunker, {}, None)
+            if loader.supports(source):
+                return loader_type
+        return None
+
+    def supports(self, source: str) -> bool:
+        """Check if any loader supports the source.
+
+        Args:
+            source: The source URL or path.
+
+        Returns:
+            True if at least one loader supports it.
+        """
+        return self.detect_type(source) is not None