gnosisllm-knowledge 0.2.0__py3-none-any.whl

gnosisllm_knowledge/loaders/sitemap.py

@@ -0,0 +1,285 @@
+"""Sitemap loader with recursive discovery and URL filtering."""
+
+from __future__ import annotations
+
+import asyncio
+import fnmatch
+import logging
+import re
+from typing import Any
+from xml.etree import ElementTree
+
+import httpx
+
+from gnosisllm_knowledge.core.events.types import SitemapDiscoveryEvent
+from gnosisllm_knowledge.loaders.base import BaseLoader
+
+# XML namespace for sitemaps
+SITEMAP_NS = {"sm": "http://www.sitemaps.org/schemas/sitemap/0.9"}
+
+# Default limits
+DEFAULT_MAX_URLS = 1000
+DEFAULT_MAX_DEPTH = 3
+
+
+class SitemapLoader(BaseLoader):
+    """Loader for sitemap XML files with recursive discovery.
+
+    Features:
+    - Recursive sitemap discovery (sitemap index files)
+    - URL filtering with allow/block patterns
+    - Configurable max URLs and depth limits
+    - Parallel processing of nested sitemaps
+    - Deduplication of discovered URLs
+
+    Example:
+        ```python
+        loader = SitemapLoader(
+            fetcher, chunker,
+            config={
+                "max_urls": 500,
+                "max_depth": 2,
+                "allowed_patterns": ["*/docs/*", "*/blog/*"],
+                "blocked_patterns": ["*/admin/*"],
+            }
+        )
+        result = await loader.load("https://example.com/sitemap.xml")
+        ```
+    """
+
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        """Initialize the sitemap loader."""
+        super().__init__(*args, **kwargs)
+        self._sitemap_logger = logging.getLogger(f"{__name__}.SitemapLoader")
+
+    @property
+    def name(self) -> str:
+        """Return the loader name."""
+        return "sitemap"
+
+    def supports(self, source: str) -> bool:
+        """Check if this loader supports the given source.
+
+        Supports URLs that look like sitemaps (contain 'sitemap' or end with .xml).
+
+        Args:
+            source: The source URL.
+
+        Returns:
+            True if this looks like a sitemap URL.
+        """
+        source_lower = source.lower()
+        is_http = source_lower.startswith(("http://", "https://"))
+        is_sitemap = "sitemap" in source_lower or source_lower.endswith(".xml")
+        return is_http and is_sitemap
+
+    async def _get_urls(self, source: str, **options: Any) -> list[str]:
+        """Get list of URLs to process from the sitemap.
+
+        Recursively discovers URLs from sitemap and sitemap index files.
+
+        Args:
+            source: The sitemap URL.
+            **options: Loader-specific options:
+                - max_urls: Maximum URLs to return
+                - max_depth: Maximum recursion depth
+                - allowed_patterns: URL patterns to include
+                - blocked_patterns: URL patterns to exclude
+
+        Returns:
+            List of discovered and filtered URLs.
+        """
+        max_urls = options.get("max_urls", self._config.get("max_urls", DEFAULT_MAX_URLS))
+        max_depth = options.get("max_depth", self._config.get("max_depth", DEFAULT_MAX_DEPTH))
+        allowed_patterns = options.get(
+            "allowed_patterns", self._config.get("allowed_patterns", [])
+        )
+        blocked_patterns = options.get(
+            "blocked_patterns", self._config.get("blocked_patterns", [])
+        )
+
+        discovered_urls: set[str] = set()
+
+        await self._discover_urls(
+            sitemap_url=source,
+            depth=0,
+            max_depth=max_depth,
+            max_urls=max_urls,
+            discovered=discovered_urls,
+            allowed_patterns=allowed_patterns,
+            blocked_patterns=blocked_patterns,
+        )
+
+        # Convert to list and limit
+        urls = list(discovered_urls)[:max_urls]
+        self._sitemap_logger.info(f"Discovered {len(urls)} URLs from sitemap")
+
+        return urls
+
+    async def _discover_urls(
+        self,
+        sitemap_url: str,
+        depth: int,
+        max_depth: int,
+        max_urls: int,
+        discovered: set[str],
+        allowed_patterns: list[str],
+        blocked_patterns: list[str],
+    ) -> None:
+        """Recursively discover URLs from a sitemap.
+
+        Args:
+            sitemap_url: The sitemap URL to process.
+            depth: Current recursion depth.
+            max_depth: Maximum recursion depth.
+            max_urls: Maximum URLs to discover.
+            discovered: Set of already discovered URLs (modified in place).
+            allowed_patterns: URL patterns to include.
+            blocked_patterns: URL patterns to exclude.
+        """
+        if depth > max_depth:
+            self._sitemap_logger.debug(f"Max depth {max_depth} reached, skipping {sitemap_url}")
+            return
+
+        if len(discovered) >= max_urls:
+            self._sitemap_logger.debug(f"Max URLs {max_urls} reached")
+            return
+
+        try:
+            content = await self._fetch_sitemap(sitemap_url)
+            if not content:
+                return
+
+            root = ElementTree.fromstring(content)
+
+            # Check if this is a sitemap index
+            sitemap_refs = root.findall(".//sm:sitemap/sm:loc", SITEMAP_NS)
+            if sitemap_refs:
+                # This is a sitemap index - recursively process each sitemap
+                self._sitemap_logger.info(
+                    f"Found sitemap index with {len(sitemap_refs)} sitemaps at depth {depth}"
+                )
+
+                # Process nested sitemaps in parallel
+                tasks = []
+                for sitemap_ref in sitemap_refs:
+                    if sitemap_ref.text and len(discovered) < max_urls:
+                        tasks.append(
+                            self._discover_urls(
+                                sitemap_url=sitemap_ref.text.strip(),
+                                depth=depth + 1,
+                                max_depth=max_depth,
+                                max_urls=max_urls,
+                                discovered=discovered,
+                                allowed_patterns=allowed_patterns,
+                                blocked_patterns=blocked_patterns,
+                            )
+                        )
+
+                if tasks:
+                    await asyncio.gather(*tasks, return_exceptions=True)
+
+                return
+
+            # Process regular sitemap URLs
+            url_elements = root.findall(".//sm:url/sm:loc", SITEMAP_NS)
+            urls_added = 0
+
+            for url_elem in url_elements:
+                if url_elem.text and len(discovered) < max_urls:
+                    url = url_elem.text.strip()
+
+                    # Apply filters
+                    if self._should_include_url(url, allowed_patterns, blocked_patterns):
+                        if url not in discovered:
+                            discovered.add(url)
+                            urls_added += 1
+
+            # Emit discovery event
+            self._events.emit(
+                SitemapDiscoveryEvent(
+                    sitemap_url=sitemap_url,
+                    urls_discovered=urls_added,
+                    depth=depth,
+                    total_urls=len(discovered),
+                )
+            )
+
+            self._sitemap_logger.debug(
+                f"Discovered {urls_added} URLs from {sitemap_url} at depth {depth}"
+            )
+
+        except Exception as e:
+            self._sitemap_logger.error(f"Failed to process sitemap {sitemap_url}: {e}")
+
+    async def _fetch_sitemap(self, url: str) -> str | None:
+        """Fetch sitemap XML content.
+
+        Args:
+            url: The sitemap URL to fetch.
+
+        Returns:
+            Sitemap XML content or None if fetch failed.
+        """
+        try:
+            async with httpx.AsyncClient(timeout=30.0) as client:
+                response = await client.get(
+                    url,
+                    headers={"Accept": "application/xml, text/xml, */*"},
+                    follow_redirects=True,
+                )
+                response.raise_for_status()
+                return response.text
+        except Exception as e:
+            self._sitemap_logger.error(f"Failed to fetch sitemap {url}: {e}")
+            return None
+
+    def _should_include_url(
+        self,
+        url: str,
+        allowed_patterns: list[str],
+        blocked_patterns: list[str],
+    ) -> bool:
+        """Check if a URL should be included based on patterns.
+
+        Args:
+            url: The URL to check.
+            allowed_patterns: Patterns that must match (if any).
+            blocked_patterns: Patterns that must not match.
+
+        Returns:
+            True if URL should be included.
+        """
+        # Check blocked patterns first
+        for pattern in blocked_patterns:
+            if self._matches_pattern(url, pattern):
+                return False
+
+        # If allowed patterns specified, at least one must match
+        if allowed_patterns:
+            return any(self._matches_pattern(url, p) for p in allowed_patterns)
+
+        return True
+
+    def _matches_pattern(self, url: str, pattern: str) -> bool:
+        """Check if URL matches a pattern.
+
+        Supports both glob patterns (with *) and regex patterns.
+
+        Args:
+            url: The URL to check.
+            pattern: The pattern to match against.
+
+        Returns:
+            True if URL matches the pattern.
+        """
+        # Try fnmatch for glob patterns
+        if "*" in pattern or "?" in pattern:
+            return fnmatch.fnmatch(url, pattern)
+
+        # Try regex
+        try:
+            return bool(re.search(pattern, url))
+        except re.error:
+            # Invalid regex, try substring match
+            return pattern in url

gnosisllm_knowledge/loaders/website.py

@@ -0,0 +1,57 @@
+"""Website loader for single URL loading."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from gnosisllm_knowledge.loaders.base import BaseLoader
+
+
+class WebsiteLoader(BaseLoader):
+    """Loader for single website URLs.
+
+    This is the simplest loader that handles loading content from
+    a single URL. For loading multiple URLs from a sitemap, use
+    SitemapLoader instead.
+
+    Example:
+        ```python
+        loader = WebsiteLoader(fetcher, chunker)
+        result = await loader.load("https://example.com/page")
+        ```
+    """
+
+    @property
+    def name(self) -> str:
+        """Return the loader name."""
+        return "website"
+
+    def supports(self, source: str) -> bool:
+        """Check if this loader supports the given source.
+
+        Supports HTTP and HTTPS URLs that don't look like sitemaps.
+
+        Args:
+            source: The source URL.
+
+        Returns:
+            True if this is a regular website URL.
+        """
+        source_lower = source.lower()
+        is_http = source_lower.startswith(("http://", "https://"))
+        is_sitemap = "sitemap" in source_lower or source_lower.endswith(".xml")
+        return is_http and not is_sitemap
+
+    async def _get_urls(self, source: str, **options: Any) -> list[str]:
+        """Get list of URLs to process.
+
+        For website loader, this simply returns the source URL.
+
+        Args:
+            source: The source URL.
+            **options: Loader-specific options (ignored).
+
+        Returns:
+            List containing just the source URL.
+        """
+        return [source]

gnosisllm_knowledge/services/__init__.py

@@ -0,0 +1,9 @@
+"""Service layer for knowledge orchestration."""
+
+from gnosisllm_knowledge.services.indexing import KnowledgeIndexingService
+from gnosisllm_knowledge.services.search import KnowledgeSearchService
+
+__all__ = [
+    "KnowledgeIndexingService",
+    "KnowledgeSearchService",
+]