gnosisllm-knowledge 0.3.0__py3-none-any.whl → 0.4.3__py3-none-any.whl

gnosisllm_knowledge/core/domain/discovery.py

@@ -0,0 +1,166 @@
+"""Domain models for website discovery."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+
+@dataclass
+class DiscoveryConfig:
+    """Configuration for website discovery crawl.
+
+    Controls how the Neo Reader Discovery API crawls and discovers URLs.
+
+    Attributes:
+        max_depth: Maximum crawl depth from start URL.
+        max_pages: Maximum number of pages to crawl.
+        same_domain: Only crawl URLs on the same domain.
+        include_subdomains: Include subdomains when same_domain is True.
+        respect_robots: Respect robots.txt rules.
+        parse_sitemap: Also parse sitemap if available.
+        with_metadata: Include page metadata (title, etc.) in results.
+        crawl_timeout: Overall timeout for the crawl in seconds.
+        concurrent_requests: Number of concurrent crawl requests.
+        request_delay: Delay between requests in milliseconds.
+        include_pattern: Regex pattern for URLs to include.
+        exclude_pattern: Regex pattern for URLs to exclude.
+        path_prefix: Only crawl URLs with this path prefix.
+    """
+
+    max_depth: int = 3
+    max_pages: int = 100
+    same_domain: bool = True
+    include_subdomains: bool = True
+    respect_robots: bool = True
+    parse_sitemap: bool = False
+    with_metadata: bool = True
+    crawl_timeout: int = 300
+    concurrent_requests: int = 5
+    request_delay: int = 100
+    include_pattern: str | None = None
+    exclude_pattern: str | None = None
+    path_prefix: str | None = None
+
+    def to_headers(self) -> dict[str, str]:
+        """Convert config to HTTP headers for Neo Reader API.
+
+        Returns:
+            Dictionary of header name to value.
+        """
+        headers = {
+            "X-Max-Depth": str(self.max_depth),
+            "X-Max-Pages": str(self.max_pages),
+            "X-Same-Domain": str(self.same_domain).lower(),
+            "X-Include-Subdomains": str(self.include_subdomains).lower(),
+            "X-Respect-Robots": str(self.respect_robots).lower(),
+            "X-Parse-Sitemap": str(self.parse_sitemap).lower(),
+            "X-With-Metadata": str(self.with_metadata).lower(),
+            "X-Crawl-Timeout": str(self.crawl_timeout),
+            "X-Concurrent-Requests": str(self.concurrent_requests),
+            "X-Request-Delay": str(self.request_delay),
+        }
+        if self.include_pattern:
+            headers["X-Include-Pattern"] = self.include_pattern
+        if self.exclude_pattern:
+            headers["X-Exclude-Pattern"] = self.exclude_pattern
+        if self.path_prefix:
+            headers["X-Path-Prefix"] = self.path_prefix
+        return headers
+
+
+@dataclass
+class DiscoveryProgress:
+    """Progress information for a running discovery job.
+
+    Attributes:
+        percent: Completion percentage (0-100).
+        pages_crawled: Number of pages crawled so far.
+        urls_discovered: Number of URLs discovered so far.
+        current_depth: Current crawl depth.
+        message: Human-readable progress message.
+    """
+
+    percent: int = 0
+    pages_crawled: int = 0
+    urls_discovered: int = 0
+    current_depth: int = 0
+    message: str = ""
+
+
+@dataclass
+class DiscoveryStats:
+    """Statistics for a completed discovery job.
+
+    Attributes:
+        pages_crawled: Total pages crawled.
+        urls_found: Total URLs found during crawl.
+        urls_returned: URLs returned in results (after filtering).
+        urls_filtered: URLs excluded by filters.
+        errors: Number of errors during crawl.
+        duration_seconds: Total crawl duration.
+    """
+
+    pages_crawled: int = 0
+    urls_found: int = 0
+    urls_returned: int = 0
+    urls_filtered: int = 0
+    errors: int = 0
+    duration_seconds: float = 0.0
+
+
+@dataclass
+class DiscoveredURL:
+    """A URL discovered during crawl.
+
+    Attributes:
+        url: The discovered URL.
+        depth: Crawl depth at which URL was found.
+        title: Page title if available.
+        is_internal: Whether URL is internal to the domain.
+    """
+
+    url: str
+    depth: int = 0
+    title: str | None = None
+    is_internal: bool = True
+
+
+@dataclass
+class DiscoveryJobStatus:
+    """Status of a discovery job.
+
+    Represents the current state of an async discovery job.
+
+    Attributes:
+        job_id: Unique job identifier.
+        status: Job status (pending, queued, running, completed, failed, cancelled).
+        start_url: The URL that started the discovery.
+        progress: Progress information if job is running.
+        stats: Statistics if job is completed.
+        urls: Discovered URLs if job is completed.
+        error: Error message if job failed.
+    """
+
+    job_id: str
+    status: str
+    start_url: str
+    progress: DiscoveryProgress | None = None
+    stats: DiscoveryStats | None = None
+    urls: list[DiscoveredURL] = field(default_factory=list)
+    error: str | None = None
+
+    def is_terminal(self) -> bool:
+        """Check if job is in a terminal state.
+
+        Returns:
+            True if job is completed, failed, or cancelled.
+        """
+        return self.status in ("completed", "failed", "cancelled")
+
+    def is_running(self) -> bool:
+        """Check if job is currently running.
+
+        Returns:
+            True if job is pending, queued, or running.
+        """
+        return self.status in ("pending", "queued", "running")

gnosisllm_knowledge/core/domain/document.py

@@ -26,17 +26,22 @@ class Document:
     This is the core domain object that flows through the knowledge pipeline.
     Documents are created by loaders, processed by chunkers, and stored by indexers.
 
+    Note:
+        This library is tenant-agnostic. Multi-tenancy is achieved through index
+        isolation (e.g., `knowledge-{account_id}`). Tenant information like account_id
+        should be passed in the metadata dictionary if needed for audit purposes.
+
     Attributes:
         content: The main text content of the document.
         source: Source identifier (URL, file path, etc.).
         doc_id: Unique identifier. Auto-generated from content hash if not provided.
         title: Optional document title.
         url: URL where the document was fetched from.
-        metadata: Arbitrary metadata dictionary.
+        metadata: Arbitrary metadata dictionary (can include tenant info for audit).
 
-        Multi-tenancy fields:
-            account_id: Account/tenant identifier.
+        Collection fields:
             collection_id: Collection the document belongs to.
+            collection_name: Collection name for display in aggregations.
             source_id: Source identifier within the collection.
 
         Chunking info:
@@ -70,8 +75,7 @@ class Document:
     url: str | None = None
     metadata: dict[str, Any] = field(default_factory=dict)
 
-    # Multi-tenancy fields
-    account_id: str | None = None
+    # Collection fields
     collection_id: str | None = None
     collection_name: str | None = None  # For display in aggregations
     source_id: str | None = None
@@ -139,7 +143,6 @@ class Document:
             title=self.title,
             url=self.url,
             metadata=self.metadata.copy(),
-            account_id=self.account_id,
             collection_id=self.collection_id,
             collection_name=self.collection_name,
             source_id=self.source_id,
@@ -154,23 +157,21 @@ class Document:
             created_at=self.created_at,
         )
 
-    def with_tenant(
+    def with_collection(
         self,
-        account_id: str,
-        collection_id: str | None = None,
+        collection_id: str,
         collection_name: str | None = None,
         source_id: str | None = None,
     ) -> Document:
-        """Create a new document with tenant information.
+        """Create a new document with collection information.
 
         Args:
-            account_id: Account/tenant identifier.
             collection_id: Collection identifier.
             collection_name: Collection name for display.
             source_id: Source identifier.
 
         Returns:
-            New Document instance with tenant information set.
+            New Document instance with collection information set.
         """
         return Document(
             content=self.content,
@@ -179,8 +180,7 @@ class Document:
             title=self.title,
             url=self.url,
             metadata=self.metadata.copy(),
-            account_id=account_id,
-            collection_id=collection_id or self.collection_id,
+            collection_id=collection_id,
             collection_name=collection_name or self.collection_name,
             source_id=source_id or self.source_id,
             chunk_index=self.chunk_index,
@@ -203,11 +203,6 @@ class Document:
         """Check if this document is a chunk of a larger document."""
         return self.chunk_index is not None and self.total_chunks is not None
 
-    @property
-    def is_multi_tenant(self) -> bool:
-        """Check if this document has tenant information."""
-        return self.account_id is not None
-
 
 @dataclass
 class TextChunk:

gnosisllm_knowledge/core/domain/search.py

@@ -39,6 +39,11 @@ class AgentType(str, Enum):
 class SearchQuery:
     """Search query with filters and options.
 
+    Note:
+        This library is tenant-agnostic. Multi-tenancy is achieved through index
+        isolation (e.g., `knowledge-{account_id}`). Callers should ensure they're
+        searching the correct tenant-specific index.
+
     Attributes:
         text: The search query text.
         mode: Search mode to use.
@@ -49,7 +54,6 @@ class SearchQuery:
         Filters:
             collection_ids: Filter by collection IDs.
             source_ids: Filter by source IDs.
-            account_id: Multi-tenant account filter.
             metadata_filters: Custom metadata filters.
 
         Advanced options:
@@ -69,7 +73,6 @@ class SearchQuery:
     # Filters
     collection_ids: list[str] | None = None
     source_ids: list[str] | None = None
-    account_id: str | None = None
     metadata_filters: dict[str, Any] = field(default_factory=dict)
 
     # Advanced options
@@ -89,26 +92,6 @@ class SearchQuery:
             min_score=self.min_score,
             collection_ids=self.collection_ids,
             source_ids=self.source_ids,
-            account_id=self.account_id,
-            metadata_filters=self.metadata_filters.copy(),
-            field_boosts=self.field_boosts.copy() if self.field_boosts else None,
-            include_highlights=self.include_highlights,
-            include_fields=self.include_fields,
-            exclude_fields=self.exclude_fields,
-            explain=self.explain,
-        )
-
-    def with_tenant(self, account_id: str) -> SearchQuery:
-        """Create a copy with tenant information."""
-        return SearchQuery(
-            text=self.text,
-            mode=self.mode,
-            limit=self.limit,
-            offset=self.offset,
-            min_score=self.min_score,
-            collection_ids=self.collection_ids,
-            source_ids=self.source_ids,
-            account_id=account_id,
             metadata_filters=self.metadata_filters.copy(),
             field_boosts=self.field_boosts.copy() if self.field_boosts else None,
             include_highlights=self.include_highlights,
@@ -216,13 +199,17 @@ class ReasoningStep:
 class AgenticSearchQuery:
     """Query for agentic search with conversation support.
 
+    Note:
+        This library is tenant-agnostic. Multi-tenancy is achieved through index
+        isolation (e.g., `knowledge-{account_id}`). Callers should ensure they're
+        searching the correct tenant-specific index.
+
     Attributes:
         text: The search query text.
         agent_type: Type of agent to use.
         conversation_id: ID for continuing a conversation.
         collection_ids: Filter by collection IDs.
         source_ids: Filter by source IDs.
-        account_id: Multi-tenant account filter.
         limit: Maximum number of source documents to retrieve.
         include_reasoning: Whether to include reasoning steps.
         metadata_filters: Custom metadata filters.
@@ -235,7 +222,6 @@ class AgenticSearchQuery:
     conversation_id: str | None = None
     collection_ids: list[str] | None = None
     source_ids: list[str] | None = None
-    account_id: str | None = None
     limit: int = 10
     include_reasoning: bool = True
     metadata_filters: dict[str, Any] = field(default_factory=dict)
@@ -250,7 +236,6 @@ class AgenticSearchQuery:
             limit=self.limit,
             collection_ids=self.collection_ids,
             source_ids=self.source_ids,
-            account_id=self.account_id,
             metadata_filters=self.metadata_filters.copy(),
         )
 

gnosisllm_knowledge/core/domain/source.py

@@ -10,6 +10,11 @@ from typing import Any
 class SourceConfig:
     """Configuration for a content source.
 
+    Note:
+        This library is tenant-agnostic. Multi-tenancy is achieved through index
+        isolation (e.g., `knowledge-{account_id}`). Tenant information should be
+        managed by the caller, not embedded in source configuration.
+
     Attributes:
         url: The source URL or path.
         source_type: Type of source (website, sitemap, file, etc.).
@@ -26,8 +31,7 @@ class SourceConfig:
             remove_selector: CSS selector for elements to remove.
             timeout: Request timeout in seconds.
 
-        Multi-tenancy:
-            account_id: Account/tenant identifier.
+        Collection:
             collection_id: Collection identifier.
             source_id: Source identifier within collection.
     """
@@ -47,8 +51,7 @@ class SourceConfig:
     remove_selector: str | None = None
     timeout: int | None = None
 
-    # Multi-tenancy
-    account_id: str | None = None
+    # Collection
     collection_id: str | None = None
     source_id: str | None = None
 
@@ -73,26 +76,23 @@ class SourceConfig:
             target_selector=self.target_selector,
             remove_selector=self.remove_selector,
             timeout=self.timeout,
-            account_id=self.account_id,
             collection_id=self.collection_id,
             source_id=self.source_id,
         )
 
-    def with_tenant(
+    def with_collection(
         self,
-        account_id: str,
-        collection_id: str | None = None,
+        collection_id: str,
         source_id: str | None = None,
     ) -> SourceConfig:
-        """Create a copy with tenant information.
+        """Create a copy with collection information.
 
         Args:
-            account_id: Account/tenant identifier.
             collection_id: Collection identifier.
             source_id: Source identifier.
 
         Returns:
-            New SourceConfig with tenant information.
+            New SourceConfig with collection information.
         """
         return SourceConfig(
             url=self.url,
@@ -105,7 +105,6 @@ class SourceConfig:
             target_selector=self.target_selector,
             remove_selector=self.remove_selector,
             timeout=self.timeout,
-            account_id=account_id,
             collection_id=collection_id,
             source_id=source_id,
         )

gnosisllm_knowledge/core/events/__init__.py

@@ -4,6 +4,10 @@ from gnosisllm_knowledge.core.events.emitter import EventEmitter
 from gnosisllm_knowledge.core.events.types import (
     BatchCompletedEvent,
     BatchStartedEvent,
+    DiscoveryCompletedEvent,
+    DiscoveryFailedEvent,
+    DiscoveryProgressEvent,
+    DiscoveryStartedEvent,
     DocumentIndexedEvent,
     DocumentLoadedEvent,
     Event,
@@ -20,4 +24,8 @@ __all__ = [
     "SitemapDiscoveryEvent",
     "BatchStartedEvent",
     "BatchCompletedEvent",
+    "DiscoveryStartedEvent",
+    "DiscoveryProgressEvent",
+    "DiscoveryCompletedEvent",
+    "DiscoveryFailedEvent",
 ]

gnosisllm_knowledge/core/events/types.py

@@ -34,6 +34,12 @@ class EventType(str, Enum):
     LOAD_FAILED = "load_failed"
     SITEMAP_DISCOVERED = "sitemap_discovered"
 
+    # Discovery events
+    DISCOVERY_STARTED = "discovery_started"
+    DISCOVERY_PROGRESS = "discovery_progress"
+    DISCOVERY_COMPLETED = "discovery_completed"
+    DISCOVERY_FAILED = "discovery_failed"
+
     # Streaming events
     STREAMING_PROGRESS = "streaming_progress"
     URL_BATCH_PROCESSED = "url_batch_processed"
@@ -91,11 +97,14 @@ class EventType(str, Enum):
 class Event:
     """Base event class.
 
+    Note:
+        This library is tenant-agnostic. Multi-tenancy is achieved through index
+        isolation. Any tenant-specific context should be passed in the data dict.
+
     Attributes:
         event_type: The type of event.
         timestamp: When the event occurred.
-        data: Additional event data.
-        account_id: Account ID for multi-tenant context.
+        data: Additional event data (can include tenant context for audit).
         user_id: User ID if applicable.
         request_id: Request ID for tracing.
         trace_id: Distributed trace ID.
@@ -107,7 +116,6 @@ class Event:
     data: dict[str, Any] = field(default_factory=dict)
 
     # Context
-    account_id: str | None = None
     user_id: str | None = None
     request_id: str | None = None
 
@@ -117,7 +125,6 @@ class Event:
 
     def with_context(
         self,
-        account_id: str | None = None,
         user_id: str | None = None,
         request_id: str | None = None,
     ) -> Event:
@@ -126,7 +133,6 @@ class Event:
             event_type=self.event_type,
             timestamp=self.timestamp,
             data=self.data.copy(),
-            account_id=account_id or self.account_id,
             user_id=user_id or self.user_id,
             request_id=request_id or self.request_id,
             trace_id=self.trace_id,
@@ -300,3 +306,114 @@ class StreamingCompletedEvent(Event):
             "failed_count": self.failed_count,
             "duration_ms": self.duration_ms,
         }
+
+
+# === Discovery Events ===
+
+
+@dataclass
+class DiscoveryStartedEvent(Event):
+    """Event emitted when a discovery job starts.
+
+    Attributes:
+        url: The starting URL for discovery.
+        job_id: The discovery job ID.
+        config: Discovery configuration as dictionary.
+    """
+
+    url: str = ""
+    job_id: str = ""
+    config: dict[str, Any] = field(default_factory=dict)
+
+    def __post_init__(self) -> None:
+        """Set event type."""
+        self.event_type = EventType.DISCOVERY_STARTED
+        self.data = {
+            "url": self.url,
+            "job_id": self.job_id,
+            "config": self.config,
+        }
+
+
+@dataclass
+class DiscoveryProgressEvent(Event):
+    """Event emitted during discovery progress updates.
+
+    Attributes:
+        job_id: The discovery job ID.
+        percent: Progress percentage (0-100).
+        pages_crawled: Number of pages crawled so far.
+        urls_discovered: Number of URLs discovered so far.
+        current_depth: Current crawl depth.
+        message: Human-readable progress message.
+    """
+
+    job_id: str = ""
+    percent: int = 0
+    pages_crawled: int = 0
+    urls_discovered: int = 0
+    current_depth: int = 0
+    message: str = ""
+
+    def __post_init__(self) -> None:
+        """Set event type."""
+        self.event_type = EventType.DISCOVERY_PROGRESS
+        self.data = {
+            "job_id": self.job_id,
+            "percent": self.percent,
+            "pages_crawled": self.pages_crawled,
+            "urls_discovered": self.urls_discovered,
+            "current_depth": self.current_depth,
+            "message": self.message,
+        }
+
+
+@dataclass
+class DiscoveryCompletedEvent(Event):
+    """Event emitted when discovery completes successfully.
+
+    Attributes:
+        job_id: The discovery job ID.
+        urls_count: Total number of URLs discovered.
+        pages_crawled: Total number of pages crawled.
+        duration_seconds: Total discovery duration.
+        errors: Number of errors encountered during discovery.
+    """
+
+    job_id: str = ""
+    urls_count: int = 0
+    pages_crawled: int = 0
+    duration_seconds: float = 0.0
+    errors: int = 0
+
+    def __post_init__(self) -> None:
+        """Set event type."""
+        self.event_type = EventType.DISCOVERY_COMPLETED
+        self.data = {
+            "job_id": self.job_id,
+            "urls_count": self.urls_count,
+            "pages_crawled": self.pages_crawled,
+            "duration_seconds": self.duration_seconds,
+            "errors": self.errors,
+        }
+
+
+@dataclass
+class DiscoveryFailedEvent(Event):
+    """Event emitted when discovery fails.
+
+    Attributes:
+        job_id: The discovery job ID.
+        error: Error message describing the failure.
+    """
+
+    job_id: str = ""
+    error: str = ""
+
+    def __post_init__(self) -> None:
+        """Set event type."""
+        self.event_type = EventType.DISCOVERY_FAILED
+        self.data = {
+            "job_id": self.job_id,
+            "error": self.error,
+        }

gnosisllm_knowledge/core/exceptions.py

@@ -539,3 +539,96 @@ class MemoryConfigurationError(MemoryError):
         self.missing_config = missing_config
         if missing_config:
             self.details["missing_config"] = missing_config
+
+
+# === Discovery Exceptions ===
+
+
+class DiscoveryError(KnowledgeError):
+    """Base exception for discovery operations.
+
+    Raised when website discovery fails.
+    All discovery-related exceptions inherit from this class.
+    """
+
+    def __init__(
+        self,
+        message: str = "Discovery error",
+        *,
+        job_id: str | None = None,
+        source: str | None = None,
+        **kwargs: Any,
+    ) -> None:
+        """Initialize the exception.
+
+        Args:
+            message: Human-readable error message.
+            job_id: The discovery job ID if available.
+            source: The source URL being discovered.
+            **kwargs: Additional arguments for parent class.
+        """
+        super().__init__(message, **kwargs)
+        self.job_id = job_id
+        self.source = source
+        if job_id:
+            self.details["job_id"] = job_id
+        if source:
+            self.details["source"] = source
+
+
+class DiscoveryTimeoutError(DiscoveryError):
+    """Discovery job timed out.
+
+    Raised when a discovery job exceeds its configured timeout
+    while waiting for completion.
+    """
+
+    def __init__(
+        self,
+        message: str = "Discovery job timed out",
+        *,
+        elapsed: float | None = None,
+        timeout: float | None = None,
+        **kwargs: Any,
+    ) -> None:
+        """Initialize the exception.
+
+        Args:
+            message: Human-readable error message.
+            elapsed: Time elapsed before timeout.
+            timeout: The timeout value that was exceeded.
+            **kwargs: Additional arguments for parent class.
+        """
+        super().__init__(message, **kwargs)
+        self.elapsed = elapsed
+        self.timeout = timeout
+        if elapsed is not None:
+            self.details["elapsed"] = elapsed
+        if timeout is not None:
+            self.details["timeout"] = timeout
+
+
+class DiscoveryJobFailedError(DiscoveryError):
+    """Discovery job failed on the server.
+
+    Raised when a discovery job completes with a failed or cancelled status.
+    """
+
+    def __init__(
+        self,
+        message: str = "Discovery job failed",
+        *,
+        status: str | None = None,
+        **kwargs: Any,
+    ) -> None:
+        """Initialize the exception.
+
+        Args:
+            message: Human-readable error message.
+            status: The final job status.
+            **kwargs: Additional arguments for parent class.
+        """
+        super().__init__(message, **kwargs)
+        self.status = status
+        if status:
+            self.details["status"] = status