gnosisllm-knowledge 0.2.0__py3-none-any.whl

gnosisllm_knowledge/core/interfaces/setup.py

@@ -0,0 +1,164 @@
+"""Setup adapter protocol - Interface for backend setup operations."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from enum import Enum
+from typing import Any, Protocol, runtime_checkable
+
+
+class HealthStatus(Enum):
+    """Health status enumeration."""
+
+    HEALTHY = "healthy"
+    DEGRADED = "degraded"
+    UNHEALTHY = "unhealthy"
+    UNKNOWN = "unknown"
+
+
+@dataclass
+class SetupResult:
+    """Result of a setup operation.
+
+    Attributes:
+        success: Whether setup completed successfully.
+        steps_completed: List of completed step descriptions.
+        errors: List of errors encountered.
+        warnings: List of warnings.
+        duration_ms: Duration in milliseconds.
+        data: Additional result data (e.g., model_id).
+    """
+
+    success: bool
+    steps_completed: list[str] | None = None
+    errors: list[str] | None = None
+    warnings: list[str] | None = None
+    duration_ms: float = 0.0
+    data: dict[str, Any] | None = None
+
+
+@dataclass
+class HealthReport:
+    """Comprehensive health report.
+
+    Attributes:
+        healthy: Overall health status.
+        status: HealthStatus enum value.
+        components: Component health details.
+        checked_at: When the check was performed.
+    """
+
+    healthy: bool
+    status: HealthStatus = HealthStatus.UNKNOWN
+    components: dict[str, Any] = field(default_factory=dict)
+    checked_at: datetime = field(default_factory=datetime.utcnow)
+
+
+@dataclass
+class DiagnosticReport:
+    """Diagnostic report with recommendations.
+
+    Attributes:
+        health: Health report.
+        issues: List of issues found.
+        warnings: List of warnings.
+        recommendations: List of recommendations.
+        cluster_info: Cluster-specific information.
+    """
+
+    health: HealthReport
+    issues: list[str] = field(default_factory=list)
+    warnings: list[str] = field(default_factory=list)
+    recommendations: list[str] = field(default_factory=list)
+    cluster_info: dict[str, Any] = field(default_factory=dict)
+
+
+@runtime_checkable
+class ISetupAdapter(Protocol):
+    """Protocol for backend setup operations.
+
+    Setup adapters are responsible for:
+    - Setting up the search backend (indices, pipelines, etc.)
+    - Health checking the backend
+    - Running diagnostics and providing recommendations
+    - Cleaning up resources
+
+    Implementations should handle all backend-specific setup
+    requirements transparently.
+    """
+
+    @property
+    def name(self) -> str:
+        """Human-readable backend name.
+
+        Returns:
+            Backend name (e.g., "OpenSearch", "Elasticsearch").
+        """
+        ...
+
+    async def health_check(self) -> bool:
+        """Quick health check.
+
+        Returns:
+            True if backend is healthy and responding.
+        """
+        ...
+
+    async def deep_health_check(self) -> HealthReport:
+        """Comprehensive health check with component status.
+
+        Checks all components (cluster, indices, pipelines, etc.)
+        and returns detailed health information.
+
+        Returns:
+            HealthReport with component-level health status.
+        """
+        ...
+
+    async def setup(self, **options: Any) -> SetupResult:
+        """Run complete setup.
+
+        Creates indices, pipelines, templates, and any other
+        required backend resources.
+
+        Args:
+            **options: Setup options like:
+                - force: Recreate existing resources
+                - skip_pipelines: Skip pipeline creation
+                - index_prefix: Custom index prefix
+
+        Returns:
+            SetupResult with completion status.
+        """
+        ...
+
+    async def cleanup(self) -> SetupResult:
+        """Clean up all resources.
+
+        Removes all resources created by setup. Use with caution
+        as this will delete all data.
+
+        Returns:
+            SetupResult with cleanup status.
+        """
+        ...
+
+    async def diagnose(self) -> DiagnosticReport:
+        """Run diagnostics and return recommendations.
+
+        Analyzes the backend configuration and state,
+        identifies issues, and provides recommendations.
+
+        Returns:
+            DiagnosticReport with issues and recommendations.
+        """
+        ...
+
+    def get_setup_steps(self) -> list[tuple[str, str]]:
+        """Get list of setup steps.
+
+        Returns:
+            List of (step_name, step_description) tuples.
+        """
+        ...

gnosisllm_knowledge/fetchers/__init__.py

@@ -0,0 +1,12 @@
+"""Content fetchers for retrieving content from URLs."""
+
+from gnosisllm_knowledge.fetchers.config import FetcherConfig, NeoreaderConfig
+from gnosisllm_knowledge.fetchers.http import HTTPContentFetcher
+from gnosisllm_knowledge.fetchers.neoreader import NeoreaderContentFetcher
+
+__all__ = [
+    "HTTPContentFetcher",
+    "NeoreaderContentFetcher",
+    "FetcherConfig",
+    "NeoreaderConfig",
+]

gnosisllm_knowledge/fetchers/config.py

@@ -0,0 +1,77 @@
+"""Fetcher configuration."""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass, field
+
+
+@dataclass
+class FetcherConfig:
+    """Base configuration for content fetchers.
+
+    Attributes:
+        timeout: Request timeout in seconds.
+        user_agent: User-Agent header value.
+        headers: Additional HTTP headers.
+        max_retries: Maximum retry attempts.
+        retry_delay: Delay between retries in seconds.
+    """
+
+    timeout: float = 30.0
+    user_agent: str = "gnosisllm-knowledge/0.1.0"
+    headers: dict[str, str] = field(default_factory=dict)
+    max_retries: int = 3
+    retry_delay: float = 1.0
+
+
+@dataclass
+class NeoreaderConfig:
+    """Configuration for Neoreader content fetcher.
+
+    Neoreader is a service that converts web pages to clean markdown,
+    making content extraction easier for RAG systems.
+
+    Attributes:
+        host: Neoreader API host URL.
+        api_key: API key for authentication.
+        timeout: Request timeout in seconds.
+        target_selector: CSS selector for main content extraction.
+        remove_selector: CSS selector for elements to remove.
+        with_images: Whether to include image references.
+        with_links: Whether to include link references.
+    """
+
+    host: str = "http://localhost:3000"
+    api_key: str | None = None
+    timeout: float = 30.0
+    target_selector: str | None = None
+    remove_selector: str | None = None
+    with_images: bool = False
+    with_links: bool = True
+
+    @classmethod
+    def from_env(cls) -> NeoreaderConfig:
+        """Create configuration from environment variables.
+
+        Environment variables:
+        - NEOREADER_HOST: API host URL
+        - NEOREADER_API_KEY: API key
+        - NEOREADER_TIMEOUT: Request timeout
+        - NEOREADER_TARGET_SELECTOR: CSS selector for content
+        - NEOREADER_REMOVE_SELECTOR: CSS selector for removal
+        - NEOREADER_WITH_IMAGES: Include images (true/false)
+        - NEOREADER_WITH_LINKS: Include links (true/false)
+
+        Returns:
+            NeoreaderConfig populated from environment.
+        """
+        return cls(
+            host=os.getenv("NEOREADER_HOST", "http://localhost:3000"),
+            api_key=os.getenv("NEOREADER_API_KEY"),
+            timeout=float(os.getenv("NEOREADER_TIMEOUT", "30")),
+            target_selector=os.getenv("NEOREADER_TARGET_SELECTOR"),
+            remove_selector=os.getenv("NEOREADER_REMOVE_SELECTOR"),
+            with_images=os.getenv("NEOREADER_WITH_IMAGES", "").lower() == "true",
+            with_links=os.getenv("NEOREADER_WITH_LINKS", "true").lower() == "true",
+        )

gnosisllm_knowledge/fetchers/http.py

@@ -0,0 +1,167 @@
+"""Generic HTTP content fetcher."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import re
+from typing import Any
+
+import httpx
+
+from gnosisllm_knowledge.core.exceptions import FetchError, TimeoutError
+from gnosisllm_knowledge.core.interfaces.fetcher import FetchResult
+from gnosisllm_knowledge.fetchers.config import FetcherConfig
+
+
+class HTTPContentFetcher:
+    """Generic HTTP content fetcher.
+
+    Fetches raw content from URLs using HTTP requests. For better
+    content extraction (converting HTML to markdown), use
+    NeoreaderContentFetcher instead.
+
+    Example:
+        ```python
+        fetcher = HTTPContentFetcher()
+        result = await fetcher.fetch("https://example.com/page")
+        print(result.content)
+        ```
+    """
+
+    def __init__(self, config: FetcherConfig | None = None) -> None:
+        """Initialize the fetcher.
+
+        Args:
+            config: Optional fetcher configuration.
+        """
+        self._config = config or FetcherConfig()
+        self._logger = logging.getLogger(__name__)
+
+    async def fetch(self, url: str, **options: Any) -> FetchResult:
+        """Fetch content from a URL.
+
+        Args:
+            url: The URL to fetch.
+            **options: Additional options:
+                - timeout: Override default timeout
+                - headers: Additional headers
+
+        Returns:
+            FetchResult with content and metadata.
+
+        Raises:
+            FetchError: If the fetch fails.
+            TimeoutError: If the request times out.
+        """
+        timeout = options.get("timeout", self._config.timeout)
+        extra_headers = options.get("headers", {})
+
+        headers = {
+            "User-Agent": self._config.user_agent,
+            **self._config.headers,
+            **extra_headers,
+        }
+
+        try:
+            async with httpx.AsyncClient(
+                timeout=timeout,
+                follow_redirects=True,
+            ) as client:
+                response = await client.get(url, headers=headers)
+                response.raise_for_status()
+
+                content = response.text
+                content_type = response.headers.get("content-type", "text/html")
+                title = self._extract_title(content, content_type)
+
+                return FetchResult(
+                    content=content,
+                    status_code=response.status_code,
+                    content_type=content_type,
+                    url=str(response.url),  # Final URL after redirects
+                    title=title,
+                    encoding=response.encoding,
+                    headers=dict(response.headers),
+                )
+
+        except httpx.TimeoutException as e:
+            raise TimeoutError(
+                f"Request timed out after {timeout}s",
+                timeout=timeout,
+                operation="fetch",
+                cause=e,
+            ) from e
+        except httpx.HTTPStatusError as e:
+            raise FetchError(
+                f"HTTP {e.response.status_code}",
+                source=url,
+                status_code=e.response.status_code,
+                cause=e,
+            ) from e
+        except Exception as e:
+            raise FetchError(str(e), source=url, cause=e) from e
+
+    async def health_check(self) -> bool:
+        """Check if HTTP requests can be made.
+
+        Returns:
+            True (HTTP fetcher is always "healthy").
+        """
+        return True
+
+    async def fetch_batch(
+        self,
+        urls: list[str],
+        max_concurrent: int = 10,
+        **options: Any,
+    ) -> list[FetchResult | Exception]:
+        """Fetch multiple URLs concurrently.
+
+        Args:
+            urls: List of URLs to fetch.
+            max_concurrent: Maximum concurrent requests.
+            **options: Options passed to each fetch call.
+
+        Returns:
+            List of FetchResult objects or Exception for failed fetches.
+        """
+        semaphore = asyncio.Semaphore(max_concurrent)
+
+        async def fetch_with_limit(url: str) -> FetchResult | Exception:
+            async with semaphore:
+                try:
+                    return await self.fetch(url, **options)
+                except Exception as e:
+                    return e
+
+        results = await asyncio.gather(
+            *[fetch_with_limit(url) for url in urls],
+        )
+
+        return list(results)
+
+    def _extract_title(self, content: str, content_type: str) -> str | None:
+        """Extract title from content.
+
+        Args:
+            content: The fetched content.
+            content_type: Content MIME type.
+
+        Returns:
+            Extracted title or None.
+        """
+        if "html" not in content_type.lower():
+            return None
+
+        # Try to extract from <title> tag
+        title_match = re.search(r"<title[^>]*>([^<]+)</title>", content, re.IGNORECASE)
+        if title_match:
+            return title_match.group(1).strip()
+
+        # Try to extract from <h1> tag
+        h1_match = re.search(r"<h1[^>]*>([^<]+)</h1>", content, re.IGNORECASE)
+        if h1_match:
+            return h1_match.group(1).strip()
+
+        return None

gnosisllm_knowledge/fetchers/neoreader.py

@@ -0,0 +1,204 @@
+"""Neoreader content fetcher for clean markdown extraction."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import re
+from typing import Any
+
+import httpx
+
+from gnosisllm_knowledge.core.exceptions import (
+    ConnectionError,
+    FetchError,
+    TimeoutError,
+)
+from gnosisllm_knowledge.core.interfaces.fetcher import FetchResult
+from gnosisllm_knowledge.fetchers.config import NeoreaderConfig
+
+
+class NeoreaderContentFetcher:
+    """Content fetcher using Neoreader for clean markdown extraction.
+
+    Neoreader converts web pages to clean markdown, removing navigation,
+    ads, and other noise. This produces much better content for RAG
+    systems compared to raw HTML.
+
+    Example:
+        ```python
+        config = NeoreaderConfig.from_env()
+        fetcher = NeoreaderContentFetcher(config)
+        result = await fetcher.fetch("https://example.com/page")
+        print(result.content)  # Clean markdown
+        ```
+    """
+
+    def __init__(self, config: NeoreaderConfig | None = None) -> None:
+        """Initialize the fetcher.
+
+        Args:
+            config: Neoreader configuration. Uses environment variables if not provided.
+        """
+        self._config = config or NeoreaderConfig.from_env()
+        self._logger = logging.getLogger(__name__)
+
+    async def fetch(self, url: str, **options: Any) -> FetchResult:
+        """Fetch content from a URL using Neoreader.
+
+        Args:
+            url: The URL to fetch.
+            **options: Additional options:
+                - target_selector: CSS selector for content
+                - remove_selector: CSS selector for removal
+                - timeout: Override default timeout
+
+        Returns:
+            FetchResult with markdown content.
+
+        Raises:
+            FetchError: If the fetch fails.
+            TimeoutError: If the request times out.
+            ConnectionError: If Neoreader is not available.
+        """
+        timeout = options.get("timeout", self._config.timeout)
+        target_selector = options.get("target_selector", self._config.target_selector)
+        remove_selector = options.get("remove_selector", self._config.remove_selector)
+
+        headers = {
+            "Accept": "text/markdown",
+            "X-Respond-With": "markdown",
+        }
+
+        if self._config.api_key:
+            headers["Authorization"] = f"Bearer {self._config.api_key}"
+
+        if target_selector:
+            headers["X-Target-Selector"] = target_selector
+        if remove_selector:
+            headers["X-Remove-Selector"] = remove_selector
+        if timeout:
+            headers["X-Timeout"] = str(int(timeout * 1000))
+
+        try:
+            async with httpx.AsyncClient(
+                timeout=timeout,
+                follow_redirects=True,
+            ) as client:
+                response = await client.get(
+                    f"{self._config.host}/{url}",
+                    headers=headers,
+                )
+                response.raise_for_status()
+
+                content = response.text
+                title = self._extract_title(content)
+
+                return FetchResult(
+                    content=content,
+                    status_code=response.status_code,
+                    content_type="text/markdown",
+                    url=url,
+                    title=title,
+                    encoding="utf-8",
+                    headers=dict(response.headers),
+                )
+
+        except httpx.TimeoutException as e:
+            raise TimeoutError(
+                f"Request timed out after {timeout}s",
+                timeout=timeout,
+                operation="fetch",
+                cause=e,
+            ) from e
+        except httpx.ConnectError as e:
+            raise ConnectionError(
+                f"Cannot connect to Neoreader at {self._config.host}",
+                host=self._config.host,
+                cause=e,
+            ) from e
+        except httpx.HTTPStatusError as e:
+            raise FetchError(
+                f"HTTP {e.response.status_code}",
+                source=url,
+                status_code=e.response.status_code,
+                cause=e,
+            ) from e
+        except Exception as e:
+            raise FetchError(str(e), source=url, cause=e) from e
+
+    async def health_check(self) -> bool:
+        """Check if Neoreader service is available.
+
+        Returns:
+            True if Neoreader is responding, False otherwise.
+        """
+        try:
+            async with httpx.AsyncClient(timeout=5.0) as client:
+                # Try to reach the Neoreader health endpoint or root
+                response = await client.get(f"{self._config.host}/health")
+                return response.status_code < 500
+        except Exception:
+            try:
+                # Fallback to root endpoint
+                async with httpx.AsyncClient(timeout=5.0) as client:
+                    response = await client.get(self._config.host)
+                    return response.status_code < 500
+            except Exception:
+                return False
+
+    async def fetch_batch(
+        self,
+        urls: list[str],
+        max_concurrent: int = 10,
+        **options: Any,
+    ) -> list[FetchResult | Exception]:
+        """Fetch multiple URLs concurrently.
+
+        Args:
+            urls: List of URLs to fetch.
+            max_concurrent: Maximum concurrent requests.
+            **options: Options passed to each fetch call.
+
+        Returns:
+            List of FetchResult objects or Exception for failed fetches.
+        """
+        semaphore = asyncio.Semaphore(max_concurrent)
+
+        async def fetch_with_limit(url: str) -> FetchResult | Exception:
+            async with semaphore:
+                try:
+                    return await self.fetch(url, **options)
+                except Exception as e:
+                    return e
+
+        results = await asyncio.gather(
+            *[fetch_with_limit(url) for url in urls],
+        )
+
+        return list(results)
+
+    def _extract_title(self, content: str) -> str | None:
+        """Extract title from markdown content.
+
+        Looks for the first H1 heading in the markdown.
+
+        Args:
+            content: Markdown content.
+
+        Returns:
+            Title string or None.
+        """
+        # Look for first H1 heading
+        lines = content.split("\n")
+        for line in lines:
+            line = line.strip()
+            if line.startswith("# "):
+                return line[2:].strip()
+
+        # Try regex for H1
+        match = re.search(r"^#\s+(.+)$", content, re.MULTILINE)
+        if match:
+            return match.group(1).strip()
+
+        return None

gnosisllm_knowledge/loaders/__init__.py

@@ -0,0 +1,13 @@
+"""Content loaders for various source types."""
+
+from gnosisllm_knowledge.loaders.base import BaseLoader
+from gnosisllm_knowledge.loaders.factory import LoaderFactory
+from gnosisllm_knowledge.loaders.sitemap import SitemapLoader
+from gnosisllm_knowledge.loaders.website import WebsiteLoader
+
+__all__ = [
+    "BaseLoader",
+    "LoaderFactory",
+    "WebsiteLoader",
+    "SitemapLoader",
+]