gnosisllm-knowledge 0.2.0__py3-none-any.whl → 0.4.0__py3-none-any.whl

gnosisllm_knowledge/loaders/factory.py

@@ -9,7 +9,11 @@ from typing import Any
 from gnosisllm_knowledge.core.events.emitter import EventEmitter
 from gnosisllm_knowledge.core.interfaces.chunker import ITextChunker
 from gnosisllm_knowledge.core.interfaces.fetcher import IContentFetcher
+from gnosisllm_knowledge.fetchers.config import NeoreaderConfig
+from gnosisllm_knowledge.fetchers.neoreader import NeoreaderContentFetcher
+from gnosisllm_knowledge.fetchers.neoreader_discovery import NeoreaderDiscoveryClient
 from gnosisllm_knowledge.loaders.base import BaseLoader
+from gnosisllm_knowledge.loaders.discovery import DiscoveryLoader
 from gnosisllm_knowledge.loaders.sitemap import SitemapLoader
 from gnosisllm_knowledge.loaders.website import WebsiteLoader
 
@@ -20,6 +24,43 @@ LoaderCreator = Callable[
 ]
 
 
+def _create_discovery_loader(
+    fetcher: IContentFetcher,
+    chunker: ITextChunker,
+    config: dict[str, Any] | None,
+    event_emitter: EventEmitter | None,
+) -> DiscoveryLoader:
+    """Factory function for creating DiscoveryLoader instances.
+
+    Creates a DiscoveryLoader with a NeoreaderDiscoveryClient. If the fetcher
+    is a NeoreaderContentFetcher, reuses its config to ensure consistency.
+    Otherwise, creates config from environment variables.
+
+    Args:
+        fetcher: Content fetcher for retrieving URL content.
+        chunker: Text chunker for splitting content.
+        config: Optional configuration dictionary.
+        event_emitter: Optional event emitter for progress events.
+
+    Returns:
+        Configured DiscoveryLoader instance.
+    """
+    # Get config from fetcher if it's NeoreaderContentFetcher, otherwise use env
+    if isinstance(fetcher, NeoreaderContentFetcher):
+        neoreader_config = fetcher.config
+    else:
+        neoreader_config = NeoreaderConfig.from_env()
+
+    discovery_client = NeoreaderDiscoveryClient(neoreader_config)
+    return DiscoveryLoader(
+        fetcher=fetcher,
+        chunker=chunker,
+        discovery_client=discovery_client,
+        config=config,
+        event_emitter=event_emitter,
+    )
+
+
 class LoaderFactory:
     """Factory for creating content loaders (Registry Pattern).
 
@@ -29,6 +70,7 @@ class LoaderFactory:
     Built-in loaders:
     - website: Single URL loading
     - sitemap: Sitemap XML with recursive discovery
+    - discovery: Website crawling via Neo Reader Discovery API
 
     Example:
         ```python
@@ -40,6 +82,9 @@ class LoaderFactory:
         # Explicit type
         loader = factory.create("sitemap", config={"max_urls": 500})
 
+        # Discovery loader for full website crawling
+        loader = factory.create("discovery", config={"max_depth": 3, "max_pages": 100})
+
         # Register custom loader
         factory.register("custom", MyCustomLoader)
         ```
@@ -76,6 +121,7 @@ class LoaderFactory:
         """Register built-in loader types."""
         self.register("website", lambda f, c, cfg, e: WebsiteLoader(f, c, cfg, e))
         self.register("sitemap", lambda f, c, cfg, e: SitemapLoader(f, c, cfg, e))
+        self.register("discovery", _create_discovery_loader)
 
     def register(self, name: str, creator: LoaderCreator) -> None:
         """Register a loader type.

gnosisllm_knowledge/loaders/sitemap.py

@@ -6,13 +6,18 @@ import asyncio
 import fnmatch
 import logging
 import re
+from collections.abc import Awaitable, Callable
 from typing import Any
 from xml.etree import ElementTree
 
 import httpx
 
-from gnosisllm_knowledge.core.events.types import SitemapDiscoveryEvent
+from gnosisllm_knowledge.core.domain.document import Document
+from gnosisllm_knowledge.core.domain.result import IndexResult
+from gnosisllm_knowledge.core.events.types import SitemapDiscoveryEvent, UrlBatchProcessedEvent
+from gnosisllm_knowledge.core.streaming.pipeline import PipelineConfig
 from gnosisllm_knowledge.loaders.base import BaseLoader
+from gnosisllm_knowledge.loaders.sitemap_streaming import StreamingSitemapDiscoverer
 
 # XML namespace for sitemaps
 SITEMAP_NS = {"sm": "http://www.sitemaps.org/schemas/sitemap/0.9"}
@@ -283,3 +288,126 @@ class SitemapLoader(BaseLoader):
         except re.error:
             # Invalid regex, try substring match
             return pattern in url
+
+    async def load_streaming_with_indexing(
+        self,
+        source: str,
+        index_callback: Callable[[list[Document]], Awaitable[IndexResult]],
+        url_batch_size: int = 50,
+        doc_batch_size: int = 100,
+        config: PipelineConfig | None = None,
+        **options: Any,
+    ) -> IndexResult:
+        """Load sitemap with streaming URL discovery and progressive indexing.
+
+        This method:
+        1. Discovers URLs in batches (not all at once)
+        2. Fetches content for each URL batch
+        3. Indexes documents immediately after fetching
+        4. Moves to next batch only after indexing completes
+
+        Memory usage is bounded by:
+        - url_batch_size * avg_url_length (URL strings)
+        - doc_batch_size * avg_doc_size (document content)
+        - fetch_concurrency * avg_page_size (in-flight fetches)
+
+        Args:
+            source: Sitemap URL.
+            index_callback: Called with each batch of documents to index.
+            url_batch_size: URLs to process per iteration.
+            doc_batch_size: Documents per index batch.
+            config: Pipeline configuration.
+            **options: Additional loader options.
+
+        Returns:
+            Aggregated IndexResult.
+        """
+        config = config or PipelineConfig(url_batch_size=url_batch_size)
+        discoverer = StreamingSitemapDiscoverer(config=config)
+
+        total_indexed = 0
+        total_failed = 0
+        all_errors: list[dict[str, Any]] = []
+        batch_index = 0
+        total_urls_processed = 0
+
+        async for url_batch in discoverer.discover_urls_streaming(
+            sitemap_url=source,
+            batch_size=url_batch_size,
+            max_urls=options.get("max_urls", 10000),
+            max_depth=options.get("max_depth", 3),
+            allowed_patterns=options.get("allowed_patterns", []),
+            blocked_patterns=options.get("blocked_patterns", []),
+        ):
+            # Fetch content for this batch of URLs
+            documents = await self._fetch_url_batch(url_batch, source, **options)
+            total_urls_processed += len(url_batch)
+
+            # Emit URL batch processed event
+            self._events.emit(
+                UrlBatchProcessedEvent(
+                    batch_index=batch_index,
+                    urls_in_batch=len(url_batch),
+                    documents_created=len(documents),
+                    total_urls_processed=total_urls_processed,
+                )
+            )
+
+            # Index in sub-batches
+            for i in range(0, len(documents), doc_batch_size):
+                doc_batch = documents[i : i + doc_batch_size]
+                result = await index_callback(doc_batch)
+                total_indexed += result.indexed_count
+                total_failed += result.failed_count
+                if result.errors:
+                    all_errors.extend(result.errors)
+
+            # Memory freed: url_batch and documents go out of scope
+            self._sitemap_logger.info(
+                f"Processed URL batch {batch_index}: {len(url_batch)} URLs, "
+                f"{len(documents)} docs, {total_indexed} total indexed"
+            )
+            batch_index += 1
+
+        return IndexResult(
+            success=total_failed == 0,
+            indexed_count=total_indexed,
+            failed_count=total_failed,
+            errors=all_errors,
+        )
+
+    async def _fetch_url_batch(
+        self,
+        urls: list[str],
+        source: str,
+        **options: Any,
+    ) -> list[Document]:
+        """Fetch and chunk content for a batch of URLs.
+
+        Args:
+            urls: List of URLs to fetch.
+            source: Original source identifier.
+            **options: Loader options.
+
+        Returns:
+            List of Document objects from all URLs.
+        """
+        semaphore = asyncio.Semaphore(self._max_concurrent)
+
+        async def fetch_one(url: str) -> list[Document]:
+            async with semaphore:
+                return await self._load_url(url, source, **options)
+
+        results = await asyncio.gather(
+            *[fetch_one(url) for url in urls],
+            return_exceptions=True,
+        )
+
+        documents: list[Document] = []
+        for i, result in enumerate(results):
+            if isinstance(result, Exception):
+                self._sitemap_logger.error(f"Failed to fetch {urls[i]}: {result}")
+            elif isinstance(result, list):
+                documents.extend(result)
+
+        return documents

gnosisllm_knowledge/loaders/sitemap_streaming.py

@@ -0,0 +1,258 @@
+"""Streaming sitemap discovery with bounded memory.
+
+This module provides streaming URL discovery from sitemaps, yielding
+batches of URLs as they're discovered rather than collecting all URLs
+first. This enables immediate processing and keeps memory bounded.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import fnmatch
+import logging
+import re
+from collections.abc import AsyncIterator
+from typing import Any
+from xml.etree import ElementTree
+
+import httpx
+
+from gnosisllm_knowledge.core.streaming.pipeline import BoundedQueue, PipelineConfig
+
+# XML namespace for sitemaps
+SITEMAP_NS = {"sm": "http://www.sitemaps.org/schemas/sitemap/0.9"}
+
+
+class StreamingSitemapDiscoverer:
+    """Discovers sitemap URLs in a streaming fashion.
+
+    Instead of collecting all URLs before processing, this yields
+    batches of URLs as they're discovered, enabling immediate processing.
+
+    Key differences from SitemapLoader._get_urls():
+    - Yields batches instead of returning complete list
+    - Uses bounded queue for backpressure
+    - Memory usage is O(batch_size) not O(total_urls)
+
+    Example:
+        ```python
+        discoverer = StreamingSitemapDiscoverer()
+
+        async for url_batch in discoverer.discover_urls_streaming(
+            sitemap_url="https://example.com/sitemap.xml",
+            batch_size=50,
+            max_urls=1000,
+        ):
+            # Process batch immediately
+            for url in url_batch:
+                await fetch_and_process(url)
+        ```
+    """
+
+    def __init__(
+        self,
+        config: PipelineConfig | None = None,
+    ) -> None:
+        """Initialize the streaming sitemap discoverer.
+
+        Args:
+            config: Pipeline configuration with batch sizes and concurrency.
+        """
+        self._config = config or PipelineConfig()
+        self._logger = logging.getLogger(__name__)
+
+    async def discover_urls_streaming(
+        self,
+        sitemap_url: str,
+        batch_size: int | None = None,
+        max_urls: int = 10000,
+        max_depth: int = 3,
+        allowed_patterns: list[str] | None = None,
+        blocked_patterns: list[str] | None = None,
+        **options: Any,
+    ) -> AsyncIterator[list[str]]:
+        """Yield batches of URLs as they're discovered.
+
+        Args:
+            sitemap_url: Root sitemap URL.
+            batch_size: URLs per batch (default from config).
+            max_urls: Maximum total URLs to discover.
+            max_depth: Maximum sitemap recursion depth.
+            allowed_patterns: URL patterns to include.
+            blocked_patterns: URL patterns to exclude.
+            **options: Additional options (unused, for compatibility).
+
+        Yields:
+            Lists of discovered URLs, batch_size at a time.
+        """
+        batch_size = batch_size or self._config.url_batch_size
+        allowed_patterns = allowed_patterns or []
+        blocked_patterns = blocked_patterns or []
+
+        # Use bounded queue for discovered URLs
+        # Queue size is 2x batch_size to allow producer to stay ahead
+        url_queue: BoundedQueue[str] = BoundedQueue(maxsize=batch_size * 2)
+
+        # Tracking state
+        discovered_count = 0
+        seen_urls: set[str] = set()
+
+        async def discover_recursive(url: str, depth: int) -> None:
+            """Recursively discover URLs, pushing to queue."""
+            nonlocal discovered_count
+
+            if depth > max_depth or discovered_count >= max_urls:
+                return
+
+            content = await self._fetch_sitemap(url)
+            if not content:
+                return
+
+            try:
+                root = ElementTree.fromstring(content)
+            except ElementTree.ParseError as e:
+                self._logger.error(f"Failed to parse sitemap {url}: {e}")
+                return
+
+            # Check for sitemap index
+            sitemap_refs = root.findall(".//sm:sitemap/sm:loc", SITEMAP_NS)
+            if sitemap_refs:
+                self._logger.info(
+                    f"Found sitemap index with {len(sitemap_refs)} sitemaps at depth {depth}"
+                )
+                # Process nested sitemaps (limited parallelism to avoid overwhelming)
+                tasks = []
+                for ref in sitemap_refs[:10]:  # Limit parallel sitemap fetches
+                    if ref.text and discovered_count < max_urls:
+                        tasks.append(discover_recursive(ref.text.strip(), depth + 1))
+                await asyncio.gather(*tasks, return_exceptions=True)
+                return
+
+            # Process URL entries
+            url_elements = root.findall(".//sm:url/sm:loc", SITEMAP_NS)
+            for url_elem in url_elements:
+                if url_elem.text and discovered_count < max_urls:
+                    page_url = url_elem.text.strip()
+
+                    if page_url in seen_urls:
+                        continue
+
+                    if not self._should_include_url(
+                        page_url, allowed_patterns, blocked_patterns
+                    ):
+                        continue
+
+                    seen_urls.add(page_url)
+                    await url_queue.put(page_url)  # Backpressure if queue full
+                    discovered_count += 1
+
+                    if discovered_count % 100 == 0:
+                        self._logger.debug(f"Discovered {discovered_count} URLs so far")
+
+        async def discover_and_close() -> None:
+            """Run discovery and close queue when done."""
+            try:
+                await discover_recursive(sitemap_url, depth=0)
+            except Exception as e:
+                self._logger.error(f"Discovery error: {e}")
+            finally:
+                url_queue.close()
+
+        # Start discovery in background task
+        discovery_task = asyncio.create_task(discover_and_close())
+
+        # Yield batches from queue
+        batch: list[str] = []
+        try:
+            async for url in url_queue:
+                batch.append(url)
+                if len(batch) >= batch_size:
+                    yield batch
+                    batch = []
+
+            # Yield remaining URLs
+            if batch:
+                yield batch
+
+        finally:
+            # Ensure discovery task is complete or cancelled
+            if not discovery_task.done():
+                discovery_task.cancel()
+                try:
+                    await discovery_task
+                except asyncio.CancelledError:
+                    pass
+
+    async def _fetch_sitemap(self, url: str) -> str | None:
+        """Fetch sitemap XML content.
+
+        Args:
+            url: The sitemap URL to fetch.
+
+        Returns:
+            Sitemap XML content or None if fetch failed.
+        """
+        try:
+            async with httpx.AsyncClient(
+                timeout=self._config.fetch_timeout_seconds
+            ) as client:
+                response = await client.get(
+                    url,
+                    headers={"Accept": "application/xml, text/xml, */*"},
+                    follow_redirects=True,
+                )
+                response.raise_for_status()
+                return response.text
+        except Exception as e:
+            self._logger.error(f"Failed to fetch sitemap {url}: {e}")
+            return None
+
+    def _should_include_url(
+        self,
+        url: str,
+        allowed_patterns: list[str],
+        blocked_patterns: list[str],
+    ) -> bool:
+        """Check if a URL should be included based on patterns.
+
+        Args:
+            url: The URL to check.
+            allowed_patterns: Patterns that must match (if any).
+            blocked_patterns: Patterns that must not match.
+
+        Returns:
+            True if URL should be included.
+        """
+        # Check blocked patterns first
+        for pattern in blocked_patterns:
+            if self._matches_pattern(url, pattern):
+                return False
+
+        # If allowed patterns specified, at least one must match
+        if allowed_patterns:
+            return any(self._matches_pattern(url, p) for p in allowed_patterns)
+
+        return True
+
+    def _matches_pattern(self, url: str, pattern: str) -> bool:
+        """Check if URL matches a pattern.
+
+        Supports both glob patterns (with *) and regex patterns.
+
+        Args:
+            url: The URL to check.
+            pattern: The pattern to match against.
+
+        Returns:
+            True if URL matches the pattern.
+        """
+        # Try fnmatch for glob patterns
+        if "*" in pattern or "?" in pattern:
+            return fnmatch.fnmatch(url, pattern)
+
+        # Try regex
+        try:
+            return bool(re.search(pattern, url))
+        except re.error:
+            # Invalid regex, try substring match
+            return pattern in url