gnosisllm-knowledge 0.3.0__py3-none-any.whl → 0.4.0__py3-none-any.whl

gnosisllm_knowledge/loaders/discovery.py

@@ -0,0 +1,338 @@
+"""Discovery loader using Neo Reader Discovery API for website crawling."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from typing import TYPE_CHECKING, Any
+
+from gnosisllm_knowledge.core.domain.discovery import DiscoveryConfig, DiscoveryProgress
+from gnosisllm_knowledge.core.events.types import (
+    DiscoveryCompletedEvent,
+    DiscoveryFailedEvent,
+    DiscoveryProgressEvent,
+    DiscoveryStartedEvent,
+)
+from gnosisllm_knowledge.core.exceptions import DiscoveryError
+from gnosisllm_knowledge.loaders.base import BaseLoader
+
+if TYPE_CHECKING:
+    from gnosisllm_knowledge.core.events.emitter import EventEmitter
+    from gnosisllm_knowledge.core.interfaces.chunker import ITextChunker
+    from gnosisllm_knowledge.core.interfaces.fetcher import IContentFetcher
+    from gnosisllm_knowledge.fetchers.neoreader_discovery import NeoreaderDiscoveryClient
+
+
+class DiscoveryLoader(BaseLoader):
+    """Loader that discovers all URLs from a website using Neo Reader Discovery API.
+
+    Similar to SitemapLoader but uses Neo Reader's crawler instead of sitemap XML.
+    Supports full website crawling with configurable depth, filters, and limits.
+
+    The loader follows the Template Method Pattern from BaseLoader:
+    1. _get_urls() performs URL discovery via Neo Reader Discovery API
+    2. Returns all discovered URLs
+    3. BaseLoader handles parallel fetching of each URL's content
+
+    Features:
+    - Async website crawling via Neo Reader Discovery API
+    - Configurable crawl depth, page limits, and domain restrictions
+    - URL filtering with include/exclude patterns
+    - Progress events during discovery
+    - Job cancellation on interruption
+    - Partial failure handling (logs warnings, continues with available URLs)
+
+    Example:
+        ```python
+        from gnosisllm_knowledge.fetchers import NeoreaderContentFetcher
+        from gnosisllm_knowledge.fetchers.neoreader_discovery import NeoreaderDiscoveryClient
+        from gnosisllm_knowledge.chunking import SentenceChunker
+
+        fetcher = NeoreaderContentFetcher.from_env()
+        discovery_client = NeoreaderDiscoveryClient.from_env()
+        chunker = SentenceChunker()
+
+        loader = DiscoveryLoader(
+            fetcher=fetcher,
+            chunker=chunker,
+            discovery_client=discovery_client,
+        )
+
+        result = await loader.load(
+            "https://docs.example.com",
+            max_depth=3,
+            max_pages=100,
+        )
+        ```
+    """
+
+    def __init__(
+        self,
+        fetcher: IContentFetcher,
+        chunker: ITextChunker,
+        discovery_client: NeoreaderDiscoveryClient,
+        config: dict[str, Any] | None = None,
+        event_emitter: EventEmitter | None = None,
+    ) -> None:
+        """Initialize the discovery loader.
+
+        Args:
+            fetcher: Content fetcher for retrieving URL content.
+            chunker: Text chunker for splitting content into documents.
+            discovery_client: Neo Reader Discovery API client for URL discovery.
+            config: Optional configuration dictionary.
+            event_emitter: Optional event emitter for progress events.
+        """
+        super().__init__(fetcher, chunker, config, event_emitter)
+        self._discovery_client = discovery_client
+        self._discovery_logger = logging.getLogger(f"{__name__}.DiscoveryLoader")
+
+    @property
+    def name(self) -> str:
+        """Return the loader name for registry identification.
+
+        Returns:
+            The string "discovery" for factory registration.
+        """
+        return "discovery"
+
+    def supports(self, source: str) -> bool:
+        """Check if this loader supports the given source.
+
+        Supports any HTTP/HTTPS URL that is not a sitemap. Sitemap URLs
+        should be handled by SitemapLoader instead.
+
+        Args:
+            source: The source URL.
+
+        Returns:
+            True if source is HTTP/HTTPS and not a sitemap URL.
+        """
+        source_lower = source.lower()
+        is_http = source_lower.startswith(("http://", "https://"))
+        is_sitemap = "sitemap" in source_lower or source_lower.endswith(".xml")
+        return is_http and not is_sitemap
+
+    async def _get_urls(self, source: str, **options: Any) -> list[str]:
+        """Discover URLs using Neo Reader Discovery API.
+
+        Creates a discovery job, polls for completion with progress updates,
+        and returns the discovered URLs. Handles job cancellation if
+        interrupted.
+
+        Args:
+            source: The starting URL for discovery.
+            **options: Discovery options:
+                - max_depth: Maximum crawl depth (default: 3)
+                - max_pages: Maximum pages to crawl (default: 100)
+                - same_domain: Only crawl same domain (default: True)
+                - include_subdomains: Include subdomains (default: True)
+                - respect_robots: Respect robots.txt (default: True)
+                - parse_sitemap: Also parse sitemap if found (default: False)
+                - with_metadata: Include page metadata (default: True)
+                - crawl_timeout: Crawl timeout in seconds (default: 300)
+                - concurrent_requests: Concurrent crawl requests (default: 5)
+                - request_delay: Delay between requests in ms (default: 100)
+                - include_pattern: Regex pattern for URLs to include
+                - exclude_pattern: Regex pattern for URLs to exclude
+                - path_prefix: Only crawl URLs with this path prefix
+
+        Returns:
+            List of discovered URL strings.
+
+        Raises:
+            DiscoveryError: If discovery fails.
+        """
+        discovery_config = self._build_discovery_config(options)
+        job_id = await self._discovery_client.create_job(source, discovery_config)
+
+        self._emit_discovery_started(source, job_id, discovery_config)
+
+        try:
+            result = await self._wait_for_job_completion(job_id)
+        except (asyncio.CancelledError, Exception) as e:
+            await self._handle_job_cancellation(job_id, e)
+            raise
+
+        return self._process_discovery_result(job_id, result, source)
+
+    def _build_discovery_config(self, options: dict[str, Any]) -> DiscoveryConfig:
+        """Build DiscoveryConfig from options dictionary.
+
+        Args:
+            options: Discovery options passed to _get_urls.
+
+        Returns:
+            Configured DiscoveryConfig instance.
+        """
+        return DiscoveryConfig(
+            max_depth=options.get("max_depth", 3),
+            max_pages=options.get("max_pages", 100),
+            same_domain=options.get("same_domain", True),
+            include_subdomains=options.get("include_subdomains", True),
+            respect_robots=options.get("respect_robots", True),
+            parse_sitemap=options.get("parse_sitemap", False),
+            with_metadata=options.get("with_metadata", True),
+            crawl_timeout=options.get("crawl_timeout", 300),
+            concurrent_requests=options.get("concurrent_requests", 5),
+            request_delay=options.get("request_delay", 100),
+            include_pattern=options.get("include_pattern"),
+            exclude_pattern=options.get("exclude_pattern"),
+            path_prefix=options.get("path_prefix"),
+        )
+
+    def _emit_discovery_started(
+        self,
+        source: str,
+        job_id: str,
+        discovery_config: DiscoveryConfig,
+    ) -> None:
+        """Emit discovery started event.
+
+        Args:
+            source: The starting URL.
+            job_id: The discovery job ID.
+            discovery_config: The discovery configuration.
+        """
+        self._events.emit(
+            DiscoveryStartedEvent(
+                url=source,
+                job_id=job_id,
+                config={
+                    "max_depth": discovery_config.max_depth,
+                    "max_pages": discovery_config.max_pages,
+                    "same_domain": discovery_config.same_domain,
+                    "include_subdomains": discovery_config.include_subdomains,
+                    "respect_robots": discovery_config.respect_robots,
+                    "parse_sitemap": discovery_config.parse_sitemap,
+                },
+            )
+        )
+        self._discovery_logger.info(
+            "Started discovery job %s for %s (max_depth=%d, max_pages=%d)",
+            job_id,
+            source,
+            discovery_config.max_depth,
+            discovery_config.max_pages,
+        )
+
+    async def _wait_for_job_completion(self, job_id: str) -> Any:
+        """Wait for discovery job to complete with progress updates.
+
+        Args:
+            job_id: The discovery job ID.
+
+        Returns:
+            The final DiscoveryJobStatus.
+        """
+
+        async def on_progress(progress: DiscoveryProgress) -> None:
+            """Emit progress event for each update."""
+            self._events.emit(
+                DiscoveryProgressEvent(
+                    job_id=job_id,
+                    percent=progress.percent,
+                    pages_crawled=progress.pages_crawled,
+                    urls_discovered=progress.urls_discovered,
+                    current_depth=progress.current_depth,
+                    message=progress.message,
+                )
+            )
+            self._discovery_logger.debug(
+                "Discovery progress: %d%% (%d pages, %d URLs)",
+                progress.percent,
+                progress.pages_crawled,
+                progress.urls_discovered,
+            )
+
+        return await self._discovery_client.wait_for_completion(
+            job_id,
+            on_progress=on_progress,
+        )
+
+    async def _handle_job_cancellation(self, job_id: str, error: Exception) -> None:
+        """Handle job cancellation on error or interruption.
+
+        Args:
+            job_id: The discovery job ID to cancel.
+            error: The exception that caused cancellation.
+        """
+        self._discovery_logger.warning(
+            "Cancelling discovery job %s due to: %s",
+            job_id,
+            error,
+        )
+        try:
+            await self._discovery_client.cancel_job(job_id)
+            self._discovery_logger.info("Successfully cancelled job %s", job_id)
+        except Exception as cancel_err:
+            self._discovery_logger.error(
+                "Failed to cancel job %s: %s",
+                job_id,
+                cancel_err,
+            )
+
+    def _process_discovery_result(
+        self,
+        job_id: str,
+        result: Any,
+        source: str,
+    ) -> list[str]:
+        """Process discovery result and extract URLs.
+
+        Handles completed, failed, and partial failure cases.
+
+        Args:
+            job_id: The discovery job ID.
+            result: The DiscoveryJobStatus from wait_for_completion.
+            source: The original source URL.
+
+        Returns:
+            List of discovered URL strings.
+
+        Raises:
+            DiscoveryError: If the job failed or was cancelled.
+        """
+        if result.status != "completed":
+            error_msg = result.error or "Unknown error"
+            self._events.emit(
+                DiscoveryFailedEvent(
+                    job_id=job_id,
+                    error=error_msg,
+                )
+            )
+            raise DiscoveryError(
+                f"Discovery failed: {error_msg}",
+                job_id=job_id,
+                source=source,
+            )
+
+        # Handle partial failures (log warning if errors occurred)
+        if result.stats and result.stats.errors > 0:
+            self._discovery_logger.warning(
+                "Discovery completed with %d errors (%d URLs returned)",
+                result.stats.errors,
+                result.stats.urls_returned,
+            )
+
+        # Extract URLs
+        urls = [u.url for u in (result.urls or [])]
+
+        # Emit completion event
+        self._events.emit(
+            DiscoveryCompletedEvent(
+                job_id=job_id,
+                urls_count=len(urls),
+                pages_crawled=result.stats.pages_crawled if result.stats else 0,
+                duration_seconds=result.stats.duration_seconds if result.stats else 0.0,
+                errors=result.stats.errors if result.stats else 0,
+            )
+        )
+
+        self._discovery_logger.info(
+            "Discovery completed: %d URLs discovered in %.1fs",
+            len(urls),
+            result.stats.duration_seconds if result.stats else 0.0,
+        )
+
+        return urls

gnosisllm_knowledge/loaders/discovery_streaming.py

@@ -0,0 +1,343 @@
+"""Streaming discovery with bounded memory for large websites.
+
+This module provides streaming URL discovery from websites using the Neo Reader
+Discovery API, yielding batches of URLs as they're discovered rather than
+waiting for the full discovery to complete. This enables immediate processing
+and keeps memory bounded for sites with many URLs (>500).
+"""
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import fnmatch
+import logging
+import re
+from collections.abc import AsyncIterator, Awaitable, Callable
+from typing import TYPE_CHECKING, Any
+
+from gnosisllm_knowledge.core.domain.discovery import DiscoveryConfig, DiscoveryProgress
+from gnosisllm_knowledge.core.streaming.pipeline import BoundedQueue, PipelineConfig
+
+if TYPE_CHECKING:
+    from gnosisllm_knowledge.fetchers.neoreader_discovery import NeoreaderDiscoveryClient
+
+
+class StreamingDiscoveryDiscoverer:
+    """Discovers website URLs in a streaming fashion using Neo Reader Discovery API.
+
+    Instead of collecting all URLs before processing, this yields batches of URLs
+    as they're discovered from the polling response. This enables immediate
+    processing while discovery continues in the background.
+
+    Key differences from DiscoveryLoader._get_urls():
+    - Yields batches instead of returning complete list
+    - Uses bounded queue for backpressure
+    - Memory usage is O(batch_size) not O(total_urls)
+    - Starts yielding URLs while discovery is still running
+
+    Recommended for sites with >500 expected URLs to start content loading
+    before full discovery completes.
+
+    Example:
+        ```python
+        from gnosisllm_knowledge.fetchers.neoreader_discovery import NeoreaderDiscoveryClient
+        from gnosisllm_knowledge.loaders.discovery_streaming import StreamingDiscoveryDiscoverer
+
+        client = NeoreaderDiscoveryClient.from_env()
+        discoverer = StreamingDiscoveryDiscoverer(discovery_client=client)
+
+        async for url_batch in discoverer.discover_urls_streaming(
+            source="https://docs.example.com",
+            batch_size=50,
+            max_pages=1000,
+        ):
+            # Process batch immediately - content loading can start
+            # while discovery continues
+            for url in url_batch:
+                await fetch_and_process(url)
+        ```
+    """
+
+    def __init__(
+        self,
+        discovery_client: NeoreaderDiscoveryClient,
+        config: PipelineConfig | None = None,
+    ) -> None:
+        """Initialize the streaming discovery discoverer.
+
+        Args:
+            discovery_client: Neo Reader Discovery API client.
+            config: Pipeline configuration with batch sizes and concurrency.
+        """
+        self._discovery_client = discovery_client
+        self._config = config or PipelineConfig()
+        self._logger = logging.getLogger(__name__)
+
+    async def discover_urls_streaming(
+        self,
+        source: str,
+        batch_size: int | None = None,
+        max_pages: int = 1000,
+        max_depth: int = 3,
+        allowed_patterns: list[str] | None = None,
+        blocked_patterns: list[str] | None = None,
+        poll_interval: float = 2.0,
+        discovery_timeout: float = 600.0,
+        on_progress: Callable[[DiscoveryProgress], Awaitable[None] | None] | None = None,
+        **options: Any,
+    ) -> AsyncIterator[list[str]]:
+        """Yield batches of URLs as they're discovered.
+
+        Starts a discovery job and polls for new URLs, yielding batches
+        as they become available. This allows content fetching to begin
+        while discovery is still running.
+
+        Args:
+            source: Starting URL for discovery.
+            batch_size: URLs per batch (default from config).
+            max_pages: Maximum pages to discover.
+            max_depth: Maximum crawl depth.
+            allowed_patterns: URL patterns to include (glob or regex).
+            blocked_patterns: URL patterns to exclude (glob or regex).
+            poll_interval: Seconds between status polls.
+            discovery_timeout: Maximum time for discovery in seconds.
+            on_progress: Optional callback for progress updates (sync or async).
+            **options: Additional discovery options passed to DiscoveryConfig.
+
+        Yields:
+            Lists of discovered URLs, batch_size at a time.
+        """
+        batch_size = batch_size or self._config.url_batch_size
+        allowed_patterns = allowed_patterns or []
+        blocked_patterns = blocked_patterns or []
+
+        # Use bounded queue for discovered URLs
+        # Queue size is 2x batch_size to allow producer to stay ahead
+        url_queue: BoundedQueue[str] = BoundedQueue(maxsize=batch_size * 2)
+
+        # Tracking state
+        seen_urls: set[str] = set()
+        last_yielded_count = 0
+
+        # Build discovery config from options
+        discovery_config = self._build_discovery_config(
+            max_pages=max_pages,
+            max_depth=max_depth,
+            **options,
+        )
+
+        async def poll_and_queue_urls() -> None:
+            """Poll discovery job and push new URLs to queue."""
+            nonlocal last_yielded_count
+
+            # Create discovery job
+            job_id = await self._discovery_client.create_job(source, discovery_config)
+            self._logger.info(
+                "Started streaming discovery job %s for %s",
+                job_id,
+                source,
+            )
+
+            try:
+                loop = asyncio.get_event_loop()
+                start_time = loop.time()
+
+                while True:
+                    # Get job status with URLs
+                    status = await self._discovery_client.get_job_status(
+                        job_id,
+                        include_urls=True,
+                    )
+
+                    # Process any new URLs from this poll
+                    if status.urls:
+                        new_urls_added = 0
+                        for discovered_url in status.urls:
+                            url = discovered_url.url
+
+                            if url in seen_urls:
+                                continue
+
+                            if not self._should_include_url(
+                                url, allowed_patterns, blocked_patterns
+                            ):
+                                continue
+
+                            seen_urls.add(url)
+                            await url_queue.put(url)  # Backpressure if queue full
+                            new_urls_added += 1
+
+                        if new_urls_added > 0:
+                            self._logger.debug(
+                                "Queued %d new URLs (total seen: %d)",
+                                new_urls_added,
+                                len(seen_urls),
+                            )
+
+                    # Check if job is complete
+                    if status.is_terminal():
+                        if status.status == "completed":
+                            self._logger.info(
+                                "Discovery completed: %d URLs discovered in %.1fs",
+                                len(seen_urls),
+                                status.stats.duration_seconds if status.stats else 0.0,
+                            )
+                        elif status.status == "failed":
+                            self._logger.error(
+                                "Discovery failed: %s",
+                                status.error or "Unknown error",
+                            )
+                        break
+
+                    # Check timeout
+                    elapsed = loop.time() - start_time
+                    if elapsed >= discovery_timeout:
+                        self._logger.warning(
+                            "Discovery timeout after %.1fs, cancelling job %s",
+                            elapsed,
+                            job_id,
+                        )
+                        await self._discovery_client.cancel_job(job_id)
+                        break
+
+                    # Call progress callback if provided and we have progress
+                    if status.progress and on_progress:
+                        result = on_progress(status.progress)
+                        # Handle async callbacks
+                        if asyncio.iscoroutine(result):
+                            await result
+
+                    # Log progress
+                    if status.progress:
+                        self._logger.debug(
+                            "Discovery progress: %d%% (%d pages, %d URLs)",
+                            status.progress.percent,
+                            status.progress.pages_crawled,
+                            status.progress.urls_discovered,
+                        )
+
+                    # Wait before next poll
+                    await asyncio.sleep(poll_interval)
+
+            except asyncio.CancelledError:
+                self._logger.warning("Discovery cancelled, cleaning up job %s", job_id)
+                try:
+                    await self._discovery_client.cancel_job(job_id)
+                except Exception as e:
+                    self._logger.error("Failed to cancel job %s: %s", job_id, e)
+                raise
+            except Exception as e:
+                self._logger.error("Discovery error: %s", e)
+                try:
+                    await self._discovery_client.cancel_job(job_id)
+                except Exception as cancel_e:
+                    self._logger.error("Failed to cancel job %s: %s", job_id, cancel_e)
+                raise
+            finally:
+                url_queue.close()
+
+        # Start discovery in background task
+        discovery_task = asyncio.create_task(poll_and_queue_urls())
+
+        # Yield batches from queue
+        batch: list[str] = []
+        try:
+            async for url in url_queue:
+                batch.append(url)
+                if len(batch) >= batch_size:
+                    yield batch
+                    batch = []
+
+            # Yield remaining URLs
+            if batch:
+                yield batch
+
+        finally:
+            # Ensure discovery task is complete or cancelled
+            if not discovery_task.done():
+                discovery_task.cancel()
+                with contextlib.suppress(asyncio.CancelledError):
+                    await discovery_task
+
+    def _build_discovery_config(
+        self,
+        max_pages: int,
+        max_depth: int,
+        **options: Any,
+    ) -> DiscoveryConfig:
+        """Build DiscoveryConfig from parameters and options.
+
+        Args:
+            max_pages: Maximum pages to crawl.
+            max_depth: Maximum crawl depth.
+            **options: Additional discovery options.
+
+        Returns:
+            Configured DiscoveryConfig instance.
+        """
+        return DiscoveryConfig(
+            max_depth=max_depth,
+            max_pages=max_pages,
+            same_domain=options.get("same_domain", True),
+            include_subdomains=options.get("include_subdomains", True),
+            respect_robots=options.get("respect_robots", True),
+            parse_sitemap=options.get("parse_sitemap", False),
+            with_metadata=options.get("with_metadata", True),
+            crawl_timeout=options.get("crawl_timeout", 300),
+            concurrent_requests=options.get("concurrent_requests", 5),
+            request_delay=options.get("request_delay", 100),
+            include_pattern=options.get("include_pattern"),
+            exclude_pattern=options.get("exclude_pattern"),
+            path_prefix=options.get("path_prefix"),
+        )
+
+    def _should_include_url(
+        self,
+        url: str,
+        allowed_patterns: list[str],
+        blocked_patterns: list[str],
+    ) -> bool:
+        """Check if a URL should be included based on patterns.
+
+        Args:
+            url: The URL to check.
+            allowed_patterns: Patterns that must match (if any).
+            blocked_patterns: Patterns that must not match.
+
+        Returns:
+            True if URL should be included.
+        """
+        # Check blocked patterns first
+        for pattern in blocked_patterns:
+            if self._matches_pattern(url, pattern):
+                return False
+
+        # If allowed patterns specified, at least one must match
+        if allowed_patterns:
+            return any(self._matches_pattern(url, p) for p in allowed_patterns)
+
+        return True
+
+    def _matches_pattern(self, url: str, pattern: str) -> bool:
+        """Check if URL matches a pattern.
+
+        Supports both glob patterns (with *) and regex patterns.
+
+        Args:
+            url: The URL to check.
+            pattern: The pattern to match against.
+
+        Returns:
+            True if URL matches the pattern.
+        """
+        # Try fnmatch for glob patterns
+        if "*" in pattern or "?" in pattern:
+            return fnmatch.fnmatch(url, pattern)
+
+        # Try regex
+        try:
+            return bool(re.search(pattern, url))
+        except re.error:
+            # Invalid regex, try substring match
+            return pattern in url