gnosisllm-knowledge 0.2.0__py3-none-any.whl → 0.4.0__py3-none-any.whl

gnosisllm_knowledge/core/streaming/pipeline.py

@@ -0,0 +1,228 @@
+"""Bounded streaming pipeline with backpressure support.
+
+This module provides infrastructure for memory-efficient streaming pipelines
+with bounded queues that apply backpressure when downstream processing is slow.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import AsyncIterator
+from dataclasses import dataclass
+from typing import Generic, TypeVar
+
+T = TypeVar("T")
+
+
+@dataclass
+class PipelineConfig:
+    """Configuration for streaming pipeline stages.
+
+    Attributes:
+        url_batch_size: Number of URLs to discover before yielding a batch.
+        fetch_concurrency: Maximum parallel URL fetches.
+        fetch_queue_size: Maximum URLs waiting to be fetched.
+        index_batch_size: Documents per index batch.
+        index_queue_size: Maximum docs waiting to be indexed.
+        fetch_timeout_seconds: Timeout for each URL fetch.
+        index_timeout_seconds: Timeout for each index batch.
+    """
+
+    # URL discovery
+    url_batch_size: int = 100
+
+    # Content fetching
+    fetch_concurrency: int = 10
+    fetch_queue_size: int = 50
+
+    # Indexing
+    index_batch_size: int = 100
+    index_queue_size: int = 200
+
+    # Timeouts
+    fetch_timeout_seconds: float = 30.0
+    index_timeout_seconds: float = 60.0
+
+
+class BoundedQueue(Generic[T]):
+    """Async queue with bounded size and backpressure.
+
+    This queue provides backpressure: when full, put() blocks until space
+    is available. This prevents memory from growing unboundedly when
+    producers are faster than consumers.
+
+    Example:
+        ```python
+        queue: BoundedQueue[str] = BoundedQueue(maxsize=10)
+
+        # Producer task
+        async def producer():
+            for url in urls:
+                await queue.put(url)  # Blocks if queue is full
+            queue.close()
+
+        # Consumer task
+        async def consumer():
+            async for item in queue:
+                await process(item)
+        ```
+    """
+
+    def __init__(self, maxsize: int = 0) -> None:
+        """Initialize the bounded queue.
+
+        Args:
+            maxsize: Maximum queue size. 0 means unlimited (no backpressure).
+        """
+        self._queue: asyncio.Queue[T | None] = asyncio.Queue(maxsize=maxsize)
+        self._closed = False
+        self._consumer_count = 0
+
+    async def put(self, item: T) -> None:
+        """Put an item in the queue, blocking if full (backpressure).
+
+        Args:
+            item: The item to add to the queue.
+
+        Raises:
+            RuntimeError: If the queue has been closed.
+        """
+        if self._closed:
+            raise RuntimeError("Queue is closed")
+        await self._queue.put(item)
+
+    def put_nowait(self, item: T) -> None:
+        """Put an item without waiting (raises if full).
+
+        Args:
+            item: The item to add to the queue.
+
+        Raises:
+            RuntimeError: If the queue has been closed.
+            asyncio.QueueFull: If the queue is full.
+        """
+        if self._closed:
+            raise RuntimeError("Queue is closed")
+        self._queue.put_nowait(item)
+
+    async def get(self) -> T | None:
+        """Get an item from the queue.
+
+        Returns:
+            The next item, or None if queue is closed and empty.
+        """
+        item = await self._queue.get()
+        self._queue.task_done()
+        return item
+
+    def close(self) -> None:
+        """Signal that no more items will be added.
+
+        After closing, consumers will receive None when the queue
+        is empty, signaling them to stop.
+        """
+        if not self._closed:
+            self._closed = True
+            # Put sentinel to unblock any waiting consumers
+            try:
+                self._queue.put_nowait(None)
+            except asyncio.QueueFull:
+                # Queue is full, consumer will eventually get the items
+                pass
+
+    @property
+    def is_closed(self) -> bool:
+        """Check if the queue has been closed."""
+        return self._closed
+
+    def qsize(self) -> int:
+        """Return the current queue size."""
+        return self._queue.qsize()
+
+    def empty(self) -> bool:
+        """Return True if the queue is empty."""
+        return self._queue.empty()
+
+    def full(self) -> bool:
+        """Return True if the queue is full."""
+        return self._queue.full()
+
+    def __aiter__(self) -> AsyncIterator[T]:
+        """Return async iterator for consuming items."""
+        return self
+
+    async def __anext__(self) -> T:
+        """Get next item from queue.
+
+        Raises:
+            StopAsyncIteration: When queue is closed and empty.
+        """
+        item = await self.get()
+        if item is None:
+            raise StopAsyncIteration
+        return item
+
+
+class BatchCollector(Generic[T]):
+    """Collects items into batches of a specified size.
+
+    Useful for grouping streaming items into batches for
+    efficient bulk processing.
+
+    Example:
+        ```python
+        collector = BatchCollector[Document](batch_size=100)
+
+        async for doc in document_stream:
+            batch = collector.add(doc)
+            if batch:
+                await index_batch(batch)
+
+        # Flush remaining items
+        final_batch = collector.flush()
+        if final_batch:
+            await index_batch(final_batch)
+        ```
+    """
+
+    def __init__(self, batch_size: int) -> None:
+        """Initialize the batch collector.
+
+        Args:
+            batch_size: Number of items per batch.
+        """
+        self._batch_size = batch_size
+        self._buffer: list[T] = []
+
+    def add(self, item: T) -> list[T] | None:
+        """Add an item to the current batch.
+
+        Args:
+            item: The item to add.
+
+        Returns:
+            A complete batch if batch_size is reached, otherwise None.
+        """
+        self._buffer.append(item)
+        if len(self._buffer) >= self._batch_size:
+            batch = self._buffer
+            self._buffer = []
+            return batch
+        return None
+
+    def flush(self) -> list[T] | None:
+        """Flush any remaining items as a partial batch.
+
+        Returns:
+            The remaining items, or None if empty.
+        """
+        if self._buffer:
+            batch = self._buffer
+            self._buffer = []
+            return batch
+        return None
+
+    @property
+    def pending_count(self) -> int:
+        """Return the number of items waiting in the buffer."""
+        return len(self._buffer)

gnosisllm_knowledge/fetchers/__init__.py

@@ -1,12 +1,20 @@
 """Content fetchers for retrieving content from URLs."""
 
+from gnosisllm_knowledge.core.exceptions import (
+    DiscoveryJobFailedError,
+    DiscoveryTimeoutError,
+)
 from gnosisllm_knowledge.fetchers.config import FetcherConfig, NeoreaderConfig
 from gnosisllm_knowledge.fetchers.http import HTTPContentFetcher
 from gnosisllm_knowledge.fetchers.neoreader import NeoreaderContentFetcher
+from gnosisllm_knowledge.fetchers.neoreader_discovery import NeoreaderDiscoveryClient
 
 __all__ = [
     "HTTPContentFetcher",
     "NeoreaderContentFetcher",
+    "NeoreaderDiscoveryClient",
     "FetcherConfig",
     "NeoreaderConfig",
+    "DiscoveryTimeoutError",
+    "DiscoveryJobFailedError",
 ]

gnosisllm_knowledge/fetchers/config.py

@@ -40,6 +40,11 @@ class NeoreaderConfig:
         remove_selector: CSS selector for elements to remove.
         with_images: Whether to include image references.
         with_links: Whether to include link references.
+        discovery_enabled: Whether discovery loader is enabled.
+        discovery_poll_interval: Interval between status polls in seconds.
+        discovery_timeout: Maximum time to wait for discovery completion in seconds.
+        discovery_max_depth: Default maximum crawl depth for discovery.
+        discovery_max_pages: Default maximum pages to discover.
     """
 
     host: str = "http://localhost:3000"
@@ -50,6 +55,13 @@ class NeoreaderConfig:
     with_images: bool = False
     with_links: bool = True
 
+    # Discovery settings
+    discovery_enabled: bool = True
+    discovery_poll_interval: float = 2.0
+    discovery_timeout: float = 600.0
+    discovery_max_depth: int = 3
+    discovery_max_pages: int = 100
+
     @classmethod
     def from_env(cls) -> NeoreaderConfig:
         """Create configuration from environment variables.
@@ -62,6 +74,11 @@ class NeoreaderConfig:
         - NEOREADER_REMOVE_SELECTOR: CSS selector for removal
         - NEOREADER_WITH_IMAGES: Include images (true/false)
         - NEOREADER_WITH_LINKS: Include links (true/false)
+        - NEOREADER_DISCOVERY_ENABLED: Enable discovery loader (true/false)
+        - NEOREADER_DISCOVERY_POLL_INTERVAL: Discovery poll interval in seconds
+        - NEOREADER_DISCOVERY_TIMEOUT: Discovery timeout in seconds
+        - NEOREADER_DISCOVERY_MAX_DEPTH: Default max crawl depth
+        - NEOREADER_DISCOVERY_MAX_PAGES: Default max pages to discover
 
         Returns:
             NeoreaderConfig populated from environment.
@@ -74,4 +91,14 @@ class NeoreaderConfig:
             remove_selector=os.getenv("NEOREADER_REMOVE_SELECTOR"),
             with_images=os.getenv("NEOREADER_WITH_IMAGES", "").lower() == "true",
             with_links=os.getenv("NEOREADER_WITH_LINKS", "true").lower() == "true",
+            discovery_enabled=os.getenv("NEOREADER_DISCOVERY_ENABLED", "true").lower()
+            == "true",
+            discovery_poll_interval=float(
+                os.getenv("NEOREADER_DISCOVERY_POLL_INTERVAL", "2.0")
+            ),
+            discovery_timeout=float(
+                os.getenv("NEOREADER_DISCOVERY_TIMEOUT", "600.0")
+            ),
+            discovery_max_depth=int(os.getenv("NEOREADER_DISCOVERY_MAX_DEPTH", "3")),
+            discovery_max_pages=int(os.getenv("NEOREADER_DISCOVERY_MAX_PAGES", "100")),
         )

gnosisllm_knowledge/fetchers/neoreader.py

@@ -43,6 +43,15 @@ class NeoreaderContentFetcher:
         self._config = config or NeoreaderConfig.from_env()
         self._logger = logging.getLogger(__name__)
 
+    @property
+    def config(self) -> NeoreaderConfig:
+        """Expose configuration for reuse by discovery client.
+
+        Returns:
+            The Neo Reader configuration used by this fetcher.
+        """
+        return self._config
+
     async def fetch(self, url: str, **options: Any) -> FetchResult:
         """Fetch content from a URL using Neoreader.
 
@@ -181,7 +190,7 @@ class NeoreaderContentFetcher:
     def _extract_title(self, content: str) -> str | None:
         """Extract title from markdown content.
 
-        Looks for the first H1 heading in the markdown.
+        Looks for the first H1 heading in various formats.
 
         Args:
             content: Markdown content.
@@ -189,14 +198,33 @@ class NeoreaderContentFetcher:
         Returns:
             Title string or None.
         """
-        # Look for first H1 heading
         lines = content.split("\n")
+
+        # Look for ATX-style H1 heading (# Title)
         for line in lines:
             line = line.strip()
             if line.startswith("# "):
                 return line[2:].strip()
 
-        # Try regex for H1
+        # Look for "Title: ..." prefix format (common in Neoreader output)
+        for line in lines:
+            line = line.strip()
+            if line.startswith("Title:"):
+                title = line[6:].strip()
+                # Stop at "URL" or "Source" if present on same line
+                for stop in [" URL", " Source"]:
+                    if stop in title:
+                        title = title[:title.index(stop)]
+                return title.strip() if title else None
+
+        # Look for Setext-style H1 (Title followed by === line)
+        for i, line in enumerate(lines[:-1]):
+            line = line.strip()
+            next_line = lines[i + 1].strip() if i + 1 < len(lines) else ""
+            if line and next_line and all(c == "=" for c in next_line) and len(next_line) >= 3:
+                return line
+
+        # Try regex for ATX H1
         match = re.search(r"^#\s+(.+)$", content, re.MULTILINE)
         if match:
             return match.group(1).strip()