gnosisllm-knowledge 0.2.0__py3-none-any.whl → 0.3.0__py3-none-any.whl

gnosisllm_knowledge/core/interfaces/streaming.py

@@ -0,0 +1,127 @@
+"""Streaming interfaces for memory-efficient processing.
+
+These protocols define contracts for streaming operations that process
+data in bounded batches rather than loading everything into memory.
+"""
+
+from __future__ import annotations
+
+from collections.abc import AsyncIterator, Awaitable, Callable
+from typing import TYPE_CHECKING, Any, Protocol
+
+if TYPE_CHECKING:
+    from gnosisllm_knowledge.core.domain.document import Document
+    from gnosisllm_knowledge.core.domain.result import IndexResult
+
+
+class IStreamingUrlDiscoverer(Protocol):
+    """Protocol for streaming URL discovery.
+
+    Implementations yield URLs as they are discovered rather than
+    collecting all URLs first. This enables processing to begin
+    immediately and keeps memory usage bounded.
+
+    Example:
+        ```python
+        discoverer: IStreamingUrlDiscoverer = StreamingSitemapDiscoverer()
+
+        async for url_batch in discoverer.discover_urls_streaming(
+            source="https://example.com/sitemap.xml",
+            batch_size=50,
+        ):
+            for url in url_batch:
+                await process(url)
+        ```
+    """
+
+    async def discover_urls_streaming(
+        self,
+        source: str,
+        batch_size: int = 100,
+        **options: Any,
+    ) -> AsyncIterator[list[str]]:
+        """Yield batches of discovered URLs.
+
+        Args:
+            source: The sitemap or source URL.
+            batch_size: Number of URLs per batch.
+            **options: Discoverer-specific options (max_urls, patterns, etc.)
+
+        Yields:
+            Batches of discovered URLs as they're found.
+        """
+        ...
+
+
+class IStreamingLoader(Protocol):
+    """Protocol for streaming content loading.
+
+    Processes URLs in bounded batches with immediate indexing,
+    preventing memory accumulation.
+    """
+
+    async def load_streaming_with_indexing(
+        self,
+        source: str,
+        index_callback: Callable[[list[Document]], Awaitable[IndexResult]],
+        url_batch_size: int = 50,
+        doc_batch_size: int = 100,
+        **options: Any,
+    ) -> IndexResult:
+        """Load and index with streaming, calling callback for each batch.
+
+        This method:
+        1. Discovers URLs in batches (not all at once)
+        2. Fetches content for each URL batch
+        3. Indexes documents immediately after fetching
+        4. Moves to next batch only after indexing completes
+
+        Memory usage is bounded by:
+        - url_batch_size * avg_url_length (URL strings)
+        - doc_batch_size * avg_doc_size (document content)
+        - fetch_concurrency * avg_page_size (in-flight fetches)
+
+        Args:
+            source: Source URL.
+            index_callback: Called with each batch of documents to index.
+            url_batch_size: URLs to process per iteration.
+            doc_batch_size: Documents per index batch.
+            **options: Additional options.
+
+        Returns:
+            Aggregated index result.
+        """
+        ...
+
+
+class IStreamingPipeline(Protocol):
+    """Protocol for streaming indexing pipelines.
+
+    Orchestrates the full streaming load -> index pipeline with
+    bounded memory guarantees.
+    """
+
+    async def execute(
+        self,
+        source: str,
+        index_name: str,
+        *,
+        account_id: str | None = None,
+        collection_id: str | None = None,
+        source_id: str | None = None,
+        **options: Any,
+    ) -> IndexResult:
+        """Execute the streaming pipeline.
+
+        Args:
+            source: Sitemap URL.
+            index_name: Target OpenSearch index.
+            account_id: For multi-tenancy filtering.
+            collection_id: Collection within account.
+            source_id: Source identifier.
+            **options: Additional loader options.
+
+        Returns:
+            Aggregated index result.
+        """
+        ...

gnosisllm_knowledge/core/streaming/__init__.py

@@ -0,0 +1,36 @@
+"""Streaming pipeline configuration and utilities.
+
+This module provides infrastructure for memory-efficient streaming
+pipelines with bounded queues and backpressure support.
+
+Example:
+    ```python
+    from gnosisllm_knowledge.core.streaming import (
+        PipelineConfig,
+        BoundedQueue,
+        BatchCollector,
+    )
+
+    # Configure pipeline for large sitemap processing
+    config = PipelineConfig(
+        url_batch_size=50,
+        fetch_concurrency=10,
+        index_batch_size=100,
+    )
+
+    # Use bounded queue for backpressure
+    queue: BoundedQueue[str] = BoundedQueue(maxsize=100)
+    ```
+"""
+
+from gnosisllm_knowledge.core.streaming.pipeline import (
+    BatchCollector,
+    BoundedQueue,
+    PipelineConfig,
+)
+
+__all__ = [
+    "BatchCollector",
+    "BoundedQueue",
+    "PipelineConfig",
+]

gnosisllm_knowledge/core/streaming/pipeline.py

@@ -0,0 +1,228 @@
+"""Bounded streaming pipeline with backpressure support.
+
+This module provides infrastructure for memory-efficient streaming pipelines
+with bounded queues that apply backpressure when downstream processing is slow.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import AsyncIterator
+from dataclasses import dataclass
+from typing import Generic, TypeVar
+
+T = TypeVar("T")
+
+
+@dataclass
+class PipelineConfig:
+    """Configuration for streaming pipeline stages.
+
+    Attributes:
+        url_batch_size: Number of URLs to discover before yielding a batch.
+        fetch_concurrency: Maximum parallel URL fetches.
+        fetch_queue_size: Maximum URLs waiting to be fetched.
+        index_batch_size: Documents per index batch.
+        index_queue_size: Maximum docs waiting to be indexed.
+        fetch_timeout_seconds: Timeout for each URL fetch.
+        index_timeout_seconds: Timeout for each index batch.
+    """
+
+    # URL discovery
+    url_batch_size: int = 100
+
+    # Content fetching
+    fetch_concurrency: int = 10
+    fetch_queue_size: int = 50
+
+    # Indexing
+    index_batch_size: int = 100
+    index_queue_size: int = 200
+
+    # Timeouts
+    fetch_timeout_seconds: float = 30.0
+    index_timeout_seconds: float = 60.0
+
+
+class BoundedQueue(Generic[T]):
+    """Async queue with bounded size and backpressure.
+
+    This queue provides backpressure: when full, put() blocks until space
+    is available. This prevents memory from growing unboundedly when
+    producers are faster than consumers.
+
+    Example:
+        ```python
+        queue: BoundedQueue[str] = BoundedQueue(maxsize=10)
+
+        # Producer task
+        async def producer():
+            for url in urls:
+                await queue.put(url)  # Blocks if queue is full
+            queue.close()
+
+        # Consumer task
+        async def consumer():
+            async for item in queue:
+                await process(item)
+        ```
+    """
+
+    def __init__(self, maxsize: int = 0) -> None:
+        """Initialize the bounded queue.
+
+        Args:
+            maxsize: Maximum queue size. 0 means unlimited (no backpressure).
+        """
+        self._queue: asyncio.Queue[T | None] = asyncio.Queue(maxsize=maxsize)
+        self._closed = False
+        self._consumer_count = 0
+
+    async def put(self, item: T) -> None:
+        """Put an item in the queue, blocking if full (backpressure).
+
+        Args:
+            item: The item to add to the queue.
+
+        Raises:
+            RuntimeError: If the queue has been closed.
+        """
+        if self._closed:
+            raise RuntimeError("Queue is closed")
+        await self._queue.put(item)
+
+    def put_nowait(self, item: T) -> None:
+        """Put an item without waiting (raises if full).
+
+        Args:
+            item: The item to add to the queue.
+
+        Raises:
+            RuntimeError: If the queue has been closed.
+            asyncio.QueueFull: If the queue is full.
+        """
+        if self._closed:
+            raise RuntimeError("Queue is closed")
+        self._queue.put_nowait(item)
+
+    async def get(self) -> T | None:
+        """Get an item from the queue.
+
+        Returns:
+            The next item, or None if queue is closed and empty.
+        """
+        item = await self._queue.get()
+        self._queue.task_done()
+        return item
+
+    def close(self) -> None:
+        """Signal that no more items will be added.
+
+        After closing, consumers will receive None when the queue
+        is empty, signaling them to stop.
+        """
+        if not self._closed:
+            self._closed = True
+            # Put sentinel to unblock any waiting consumers
+            try:
+                self._queue.put_nowait(None)
+            except asyncio.QueueFull:
+                # Queue is full, consumer will eventually get the items
+                pass
+
+    @property
+    def is_closed(self) -> bool:
+        """Check if the queue has been closed."""
+        return self._closed
+
+    def qsize(self) -> int:
+        """Return the current queue size."""
+        return self._queue.qsize()
+
+    def empty(self) -> bool:
+        """Return True if the queue is empty."""
+        return self._queue.empty()
+
+    def full(self) -> bool:
+        """Return True if the queue is full."""
+        return self._queue.full()
+
+    def __aiter__(self) -> AsyncIterator[T]:
+        """Return async iterator for consuming items."""
+        return self
+
+    async def __anext__(self) -> T:
+        """Get next item from queue.
+
+        Raises:
+            StopAsyncIteration: When queue is closed and empty.
+        """
+        item = await self.get()
+        if item is None:
+            raise StopAsyncIteration
+        return item
+
+
+class BatchCollector(Generic[T]):
+    """Collects items into batches of a specified size.
+
+    Useful for grouping streaming items into batches for
+    efficient bulk processing.
+
+    Example:
+        ```python
+        collector = BatchCollector[Document](batch_size=100)
+
+        async for doc in document_stream:
+            batch = collector.add(doc)
+            if batch:
+                await index_batch(batch)
+
+        # Flush remaining items
+        final_batch = collector.flush()
+        if final_batch:
+            await index_batch(final_batch)
+        ```
+    """
+
+    def __init__(self, batch_size: int) -> None:
+        """Initialize the batch collector.
+
+        Args:
+            batch_size: Number of items per batch.
+        """
+        self._batch_size = batch_size
+        self._buffer: list[T] = []
+
+    def add(self, item: T) -> list[T] | None:
+        """Add an item to the current batch.
+
+        Args:
+            item: The item to add.
+
+        Returns:
+            A complete batch if batch_size is reached, otherwise None.
+        """
+        self._buffer.append(item)
+        if len(self._buffer) >= self._batch_size:
+            batch = self._buffer
+            self._buffer = []
+            return batch
+        return None
+
+    def flush(self) -> list[T] | None:
+        """Flush any remaining items as a partial batch.
+
+        Returns:
+            The remaining items, or None if empty.
+        """
+        if self._buffer:
+            batch = self._buffer
+            self._buffer = []
+            return batch
+        return None
+
+    @property
+    def pending_count(self) -> int:
+        """Return the number of items waiting in the buffer."""
+        return len(self._buffer)

gnosisllm_knowledge/loaders/base.py

@@ -161,8 +161,8 @@ class BaseLoader(ABC):
                 return LoadResult(
                     source=source,
                     source_type=self.name,
-                    document_count=0,
                     success=True,
+                    documents=[],
                     duration_ms=(time.time() - start_time) * 1000,
                 )
 
@@ -197,13 +197,12 @@ class BaseLoader(ABC):
             return LoadResult(
                 source=source,
                 source_type=self.name,
-                document_count=len(documents),
                 success=True,
+                documents=documents,
                 duration_ms=duration_ms,
                 urls_processed=urls_processed,
                 urls_failed=urls_failed,
                 bytes_loaded=sum(len(d.content) for d in documents),
-                metadata={"documents": documents},
             )
 
         except Exception as e:
@@ -211,8 +210,8 @@ class BaseLoader(ABC):
             return LoadResult(
                 source=source,
                 source_type=self.name,
-                document_count=0,
                 success=False,
+                documents=[],
                 error_message=str(e),
                 duration_ms=(time.time() - start_time) * 1000,
             )

gnosisllm_knowledge/loaders/sitemap.py

@@ -6,13 +6,18 @@ import asyncio
 import fnmatch
 import logging
 import re
+from collections.abc import Awaitable, Callable
 from typing import Any
 from xml.etree import ElementTree
 
 import httpx
 
-from gnosisllm_knowledge.core.events.types import SitemapDiscoveryEvent
+from gnosisllm_knowledge.core.domain.document import Document
+from gnosisllm_knowledge.core.domain.result import IndexResult
+from gnosisllm_knowledge.core.events.types import SitemapDiscoveryEvent, UrlBatchProcessedEvent
+from gnosisllm_knowledge.core.streaming.pipeline import PipelineConfig
 from gnosisllm_knowledge.loaders.base import BaseLoader
+from gnosisllm_knowledge.loaders.sitemap_streaming import StreamingSitemapDiscoverer
 
 # XML namespace for sitemaps
 SITEMAP_NS = {"sm": "http://www.sitemaps.org/schemas/sitemap/0.9"}
@@ -283,3 +288,126 @@ class SitemapLoader(BaseLoader):
         except re.error:
             # Invalid regex, try substring match
             return pattern in url
+
+    async def load_streaming_with_indexing(
+        self,
+        source: str,
+        index_callback: Callable[[list[Document]], Awaitable[IndexResult]],
+        url_batch_size: int = 50,
+        doc_batch_size: int = 100,
+        config: PipelineConfig | None = None,
+        **options: Any,
+    ) -> IndexResult:
+        """Load sitemap with streaming URL discovery and progressive indexing.
+
+        This method:
+        1. Discovers URLs in batches (not all at once)
+        2. Fetches content for each URL batch
+        3. Indexes documents immediately after fetching
+        4. Moves to next batch only after indexing completes
+
+        Memory usage is bounded by:
+        - url_batch_size * avg_url_length (URL strings)
+        - doc_batch_size * avg_doc_size (document content)
+        - fetch_concurrency * avg_page_size (in-flight fetches)
+
+        Args:
+            source: Sitemap URL.
+            index_callback: Called with each batch of documents to index.
+            url_batch_size: URLs to process per iteration.
+            doc_batch_size: Documents per index batch.
+            config: Pipeline configuration.
+            **options: Additional loader options.
+
+        Returns:
+            Aggregated IndexResult.
+        """
+        config = config or PipelineConfig(url_batch_size=url_batch_size)
+        discoverer = StreamingSitemapDiscoverer(config=config)
+
+        total_indexed = 0
+        total_failed = 0
+        all_errors: list[dict[str, Any]] = []
+        batch_index = 0
+        total_urls_processed = 0
+
+        async for url_batch in discoverer.discover_urls_streaming(
+            sitemap_url=source,
+            batch_size=url_batch_size,
+            max_urls=options.get("max_urls", 10000),
+            max_depth=options.get("max_depth", 3),
+            allowed_patterns=options.get("allowed_patterns", []),
+            blocked_patterns=options.get("blocked_patterns", []),
+        ):
+            # Fetch content for this batch of URLs
+            documents = await self._fetch_url_batch(url_batch, source, **options)
+            total_urls_processed += len(url_batch)
+
+            # Emit URL batch processed event
+            self._events.emit(
+                UrlBatchProcessedEvent(
+                    batch_index=batch_index,
+                    urls_in_batch=len(url_batch),
+                    documents_created=len(documents),
+                    total_urls_processed=total_urls_processed,
+                )
+            )
+
+            # Index in sub-batches
+            for i in range(0, len(documents), doc_batch_size):
+                doc_batch = documents[i : i + doc_batch_size]
+                result = await index_callback(doc_batch)
+                total_indexed += result.indexed_count
+                total_failed += result.failed_count
+                if result.errors:
+                    all_errors.extend(result.errors)
+
+            # Memory freed: url_batch and documents go out of scope
+            self._sitemap_logger.info(
+                f"Processed URL batch {batch_index}: {len(url_batch)} URLs, "
+                f"{len(documents)} docs, {total_indexed} total indexed"
+            )
+            batch_index += 1
+
+        return IndexResult(
+            success=total_failed == 0,
+            indexed_count=total_indexed,
+            failed_count=total_failed,
+            errors=all_errors,
+        )
+
+    async def _fetch_url_batch(
+        self,
+        urls: list[str],
+        source: str,
+        **options: Any,
+    ) -> list[Document]:
+        """Fetch and chunk content for a batch of URLs.
+
+        Args:
+            urls: List of URLs to fetch.
+            source: Original source identifier.
+            **options: Loader options.
+
+        Returns:
+            List of Document objects from all URLs.
+        """
+        semaphore = asyncio.Semaphore(self._max_concurrent)
+
+        async def fetch_one(url: str) -> list[Document]:
+            async with semaphore:
+                return await self._load_url(url, source, **options)
+
+        results = await asyncio.gather(
+            *[fetch_one(url) for url in urls],
+            return_exceptions=True,
+        )
+
+        documents: list[Document] = []
+        for i, result in enumerate(results):
+            if isinstance(result, Exception):
+                self._sitemap_logger.error(f"Failed to fetch {urls[i]}: {result}")
+            elif isinstance(result, list):
+                documents.extend(result)
+
+        return documents