gnosisllm-knowledge 0.2.0__py3-none-any.whl → 0.4.0__py3-none-any.whl

gnosisllm_knowledge/core/domain/source.py

@@ -10,6 +10,11 @@ from typing import Any
 class SourceConfig:
     """Configuration for a content source.
 
+    Note:
+        This library is tenant-agnostic. Multi-tenancy is achieved through index
+        isolation (e.g., `knowledge-{account_id}`). Tenant information should be
+        managed by the caller, not embedded in source configuration.
+
     Attributes:
         url: The source URL or path.
         source_type: Type of source (website, sitemap, file, etc.).
@@ -26,8 +31,7 @@ class SourceConfig:
             remove_selector: CSS selector for elements to remove.
             timeout: Request timeout in seconds.
 
-        Multi-tenancy:
-            account_id: Account/tenant identifier.
+        Collection:
             collection_id: Collection identifier.
             source_id: Source identifier within collection.
     """
@@ -47,8 +51,7 @@ class SourceConfig:
     remove_selector: str | None = None
     timeout: int | None = None
 
-    # Multi-tenancy
-    account_id: str | None = None
+    # Collection
     collection_id: str | None = None
     source_id: str | None = None
 
@@ -73,26 +76,23 @@ class SourceConfig:
             target_selector=self.target_selector,
             remove_selector=self.remove_selector,
             timeout=self.timeout,
-            account_id=self.account_id,
             collection_id=self.collection_id,
             source_id=self.source_id,
         )
 
-    def with_tenant(
+    def with_collection(
         self,
-        account_id: str,
-        collection_id: str | None = None,
+        collection_id: str,
         source_id: str | None = None,
     ) -> SourceConfig:
-        """Create a copy with tenant information.
+        """Create a copy with collection information.
 
         Args:
-            account_id: Account/tenant identifier.
             collection_id: Collection identifier.
             source_id: Source identifier.
 
         Returns:
-            New SourceConfig with tenant information.
+            New SourceConfig with collection information.
         """
         return SourceConfig(
             url=self.url,
@@ -105,7 +105,6 @@ class SourceConfig:
             target_selector=self.target_selector,
             remove_selector=self.remove_selector,
             timeout=self.timeout,
-            account_id=account_id,
             collection_id=collection_id,
             source_id=source_id,
         )

gnosisllm_knowledge/core/events/__init__.py

@@ -4,6 +4,10 @@ from gnosisllm_knowledge.core.events.emitter import EventEmitter
 from gnosisllm_knowledge.core.events.types import (
     BatchCompletedEvent,
     BatchStartedEvent,
+    DiscoveryCompletedEvent,
+    DiscoveryFailedEvent,
+    DiscoveryProgressEvent,
+    DiscoveryStartedEvent,
     DocumentIndexedEvent,
     DocumentLoadedEvent,
     Event,
@@ -20,4 +24,8 @@ __all__ = [
     "SitemapDiscoveryEvent",
     "BatchStartedEvent",
     "BatchCompletedEvent",
+    "DiscoveryStartedEvent",
+    "DiscoveryProgressEvent",
+    "DiscoveryCompletedEvent",
+    "DiscoveryFailedEvent",
 ]

gnosisllm_knowledge/core/events/types.py

@@ -14,6 +14,7 @@ class EventType(str, Enum):
     Events are organized by category:
     - Loading events: Document and content loading
     - Indexing events: Document indexing operations
+    - Streaming events: Streaming pipeline progress
     - Search events: Search and retrieval operations
     - Agentic events: AI-powered operations
     - Setup events: Backend setup operations
@@ -33,6 +34,17 @@ class EventType(str, Enum):
     LOAD_FAILED = "load_failed"
     SITEMAP_DISCOVERED = "sitemap_discovered"
 
+    # Discovery events
+    DISCOVERY_STARTED = "discovery_started"
+    DISCOVERY_PROGRESS = "discovery_progress"
+    DISCOVERY_COMPLETED = "discovery_completed"
+    DISCOVERY_FAILED = "discovery_failed"
+
+    # Streaming events
+    STREAMING_PROGRESS = "streaming_progress"
+    URL_BATCH_PROCESSED = "url_batch_processed"
+    STREAMING_COMPLETED = "streaming_completed"
+
     # Indexing events
     INDEX_STARTED = "index_started"
     DOCUMENT_INDEXED = "document_indexed"
@@ -85,11 +97,14 @@ class EventType(str, Enum):
 class Event:
     """Base event class.
 
+    Note:
+        This library is tenant-agnostic. Multi-tenancy is achieved through index
+        isolation. Any tenant-specific context should be passed in the data dict.
+
     Attributes:
         event_type: The type of event.
         timestamp: When the event occurred.
-        data: Additional event data.
-        account_id: Account ID for multi-tenant context.
+        data: Additional event data (can include tenant context for audit).
         user_id: User ID if applicable.
         request_id: Request ID for tracing.
         trace_id: Distributed trace ID.
@@ -101,7 +116,6 @@ class Event:
     data: dict[str, Any] = field(default_factory=dict)
 
     # Context
-    account_id: str | None = None
     user_id: str | None = None
     request_id: str | None = None
 
@@ -111,7 +125,6 @@ class Event:
 
     def with_context(
         self,
-        account_id: str | None = None,
         user_id: str | None = None,
         request_id: str | None = None,
     ) -> Event:
@@ -120,7 +133,6 @@ class Event:
             event_type=self.event_type,
             timestamp=self.timestamp,
             data=self.data.copy(),
-            account_id=account_id or self.account_id,
             user_id=user_id or self.user_id,
             request_id=request_id or self.request_id,
             trace_id=self.trace_id,
@@ -224,3 +236,184 @@ class BatchCompletedEvent(Event):
             "failure_count": self.failure_count,
             "duration_ms": self.duration_ms,
         }
+
+
+@dataclass
+class StreamingProgressEvent(Event):
+    """Progress event for streaming operations.
+
+    Emitted periodically during streaming pipeline execution to
+    provide visibility into progress.
+    """
+
+    urls_discovered: int = 0
+    urls_processed: int = 0
+    documents_indexed: int = 0
+    documents_failed: int = 0
+    phase: str = "unknown"
+    memory_mb: float | None = None
+
+    def __post_init__(self) -> None:
+        """Set event type."""
+        self.event_type = EventType.STREAMING_PROGRESS
+        self.data = {
+            "urls_discovered": self.urls_discovered,
+            "urls_processed": self.urls_processed,
+            "documents_indexed": self.documents_indexed,
+            "documents_failed": self.documents_failed,
+            "phase": self.phase,
+            "memory_mb": self.memory_mb,
+        }
+
+
+@dataclass
+class UrlBatchProcessedEvent(Event):
+    """Event emitted when a batch of URLs is processed."""
+
+    batch_index: int = 0
+    urls_in_batch: int = 0
+    documents_created: int = 0
+    total_urls_processed: int = 0
+
+    def __post_init__(self) -> None:
+        """Set event type."""
+        self.event_type = EventType.URL_BATCH_PROCESSED
+        self.data = {
+            "batch_index": self.batch_index,
+            "urls_in_batch": self.urls_in_batch,
+            "documents_created": self.documents_created,
+            "total_urls_processed": self.total_urls_processed,
+        }
+
+
+@dataclass
+class StreamingCompletedEvent(Event):
+    """Event emitted when streaming pipeline completes."""
+
+    total_urls: int = 0
+    total_documents: int = 0
+    indexed_count: int = 0
+    failed_count: int = 0
+    duration_ms: float = 0.0
+
+    def __post_init__(self) -> None:
+        """Set event type."""
+        self.event_type = EventType.STREAMING_COMPLETED
+        self.data = {
+            "total_urls": self.total_urls,
+            "total_documents": self.total_documents,
+            "indexed_count": self.indexed_count,
+            "failed_count": self.failed_count,
+            "duration_ms": self.duration_ms,
+        }
+
+
+# === Discovery Events ===
+
+
+@dataclass
+class DiscoveryStartedEvent(Event):
+    """Event emitted when a discovery job starts.
+
+    Attributes:
+        url: The starting URL for discovery.
+        job_id: The discovery job ID.
+        config: Discovery configuration as dictionary.
+    """
+
+    url: str = ""
+    job_id: str = ""
+    config: dict[str, Any] = field(default_factory=dict)
+
+    def __post_init__(self) -> None:
+        """Set event type."""
+        self.event_type = EventType.DISCOVERY_STARTED
+        self.data = {
+            "url": self.url,
+            "job_id": self.job_id,
+            "config": self.config,
+        }
+
+
+@dataclass
+class DiscoveryProgressEvent(Event):
+    """Event emitted during discovery progress updates.
+
+    Attributes:
+        job_id: The discovery job ID.
+        percent: Progress percentage (0-100).
+        pages_crawled: Number of pages crawled so far.
+        urls_discovered: Number of URLs discovered so far.
+        current_depth: Current crawl depth.
+        message: Human-readable progress message.
+    """
+
+    job_id: str = ""
+    percent: int = 0
+    pages_crawled: int = 0
+    urls_discovered: int = 0
+    current_depth: int = 0
+    message: str = ""
+
+    def __post_init__(self) -> None:
+        """Set event type."""
+        self.event_type = EventType.DISCOVERY_PROGRESS
+        self.data = {
+            "job_id": self.job_id,
+            "percent": self.percent,
+            "pages_crawled": self.pages_crawled,
+            "urls_discovered": self.urls_discovered,
+            "current_depth": self.current_depth,
+            "message": self.message,
+        }
+
+
+@dataclass
+class DiscoveryCompletedEvent(Event):
+    """Event emitted when discovery completes successfully.
+
+    Attributes:
+        job_id: The discovery job ID.
+        urls_count: Total number of URLs discovered.
+        pages_crawled: Total number of pages crawled.
+        duration_seconds: Total discovery duration.
+        errors: Number of errors encountered during discovery.
+    """
+
+    job_id: str = ""
+    urls_count: int = 0
+    pages_crawled: int = 0
+    duration_seconds: float = 0.0
+    errors: int = 0
+
+    def __post_init__(self) -> None:
+        """Set event type."""
+        self.event_type = EventType.DISCOVERY_COMPLETED
+        self.data = {
+            "job_id": self.job_id,
+            "urls_count": self.urls_count,
+            "pages_crawled": self.pages_crawled,
+            "duration_seconds": self.duration_seconds,
+            "errors": self.errors,
+        }
+
+
+@dataclass
+class DiscoveryFailedEvent(Event):
+    """Event emitted when discovery fails.
+
+    Attributes:
+        job_id: The discovery job ID.
+        error: Error message describing the failure.
+    """
+
+    job_id: str = ""
+    error: str = ""
+
+    def __post_init__(self) -> None:
+        """Set event type."""
+        self.event_type = EventType.DISCOVERY_FAILED
+        self.data = {
+            "job_id": self.job_id,
+            "error": self.error,
+        }

gnosisllm_knowledge/core/exceptions.py

@@ -405,3 +405,230 @@ class DocumentNotFoundError(KnowledgeError):
             self.details["doc_id"] = doc_id
         if index_name:
             self.details["index_name"] = index_name
+
+
+# === Memory Exceptions ===
+
+
+class MemoryError(KnowledgeError):
+    """Base exception for memory operations.
+
+    Raised when memory operations fail.
+    """
+
+    pass
+
+
+class ContainerNotFoundError(MemoryError):
+    """Container does not exist.
+
+    Raised when a memory container cannot be found.
+    """
+
+    def __init__(
+        self,
+        message: str = "Container not found",
+        *,
+        container_id: str | None = None,
+        **kwargs: Any,
+    ) -> None:
+        super().__init__(message, **kwargs)
+        self.container_id = container_id
+        if container_id:
+            self.details["container_id"] = container_id
+
+
+class ContainerExistsError(MemoryError):
+    """Container already exists.
+
+    Raised when attempting to create a container that already exists.
+    """
+
+    def __init__(
+        self,
+        message: str = "Container already exists",
+        *,
+        container_name: str | None = None,
+        **kwargs: Any,
+    ) -> None:
+        super().__init__(message, **kwargs)
+        self.container_name = container_name
+        if container_name:
+            self.details["container_name"] = container_name
+
+
+class SessionNotFoundError(MemoryError):
+    """Session does not exist.
+
+    Raised when a session cannot be found.
+    """
+
+    def __init__(
+        self,
+        message: str = "Session not found",
+        *,
+        session_id: str | None = None,
+        container_id: str | None = None,
+        **kwargs: Any,
+    ) -> None:
+        super().__init__(message, **kwargs)
+        self.session_id = session_id
+        self.container_id = container_id
+        if session_id:
+            self.details["session_id"] = session_id
+        if container_id:
+            self.details["container_id"] = container_id
+
+
+class InferenceError(MemoryError):
+    """LLM inference failed.
+
+    Raised when LLM inference for memory extraction fails.
+    """
+
+    def __init__(
+        self,
+        message: str = "LLM inference failed",
+        *,
+        model_id: str | None = None,
+        strategy: str | None = None,
+        **kwargs: Any,
+    ) -> None:
+        super().__init__(message, **kwargs)
+        self.model_id = model_id
+        self.strategy = strategy
+        if model_id:
+            self.details["model_id"] = model_id
+        if strategy:
+            self.details["strategy"] = strategy
+
+
+class InferenceTimeoutError(InferenceError):
+    """LLM inference timed out.
+
+    Raised when LLM inference exceeds the configured timeout.
+    """
+
+    def __init__(
+        self,
+        message: str = "LLM inference timed out",
+        *,
+        timeout_seconds: float | None = None,
+        **kwargs: Any,
+    ) -> None:
+        super().__init__(message, **kwargs)
+        self.timeout_seconds = timeout_seconds
+        if timeout_seconds:
+            self.details["timeout_seconds"] = timeout_seconds
+
+
+class MemoryConfigurationError(MemoryError):
+    """Memory is not properly configured.
+
+    Raised when memory configuration is missing or invalid.
+    """
+
+    def __init__(
+        self,
+        message: str = "Memory configuration error",
+        *,
+        missing_config: list[str] | None = None,
+        **kwargs: Any,
+    ) -> None:
+        super().__init__(message, **kwargs)
+        self.missing_config = missing_config
+        if missing_config:
+            self.details["missing_config"] = missing_config
+
+
+# === Discovery Exceptions ===
+
+
+class DiscoveryError(KnowledgeError):
+    """Base exception for discovery operations.
+
+    Raised when website discovery fails.
+    All discovery-related exceptions inherit from this class.
+    """
+
+    def __init__(
+        self,
+        message: str = "Discovery error",
+        *,
+        job_id: str | None = None,
+        source: str | None = None,
+        **kwargs: Any,
+    ) -> None:
+        """Initialize the exception.
+
+        Args:
+            message: Human-readable error message.
+            job_id: The discovery job ID if available.
+            source: The source URL being discovered.
+            **kwargs: Additional arguments for parent class.
+        """
+        super().__init__(message, **kwargs)
+        self.job_id = job_id
+        self.source = source
+        if job_id:
+            self.details["job_id"] = job_id
+        if source:
+            self.details["source"] = source
+
+
+class DiscoveryTimeoutError(DiscoveryError):
+    """Discovery job timed out.
+
+    Raised when a discovery job exceeds its configured timeout
+    while waiting for completion.
+    """
+
+    def __init__(
+        self,
+        message: str = "Discovery job timed out",
+        *,
+        elapsed: float | None = None,
+        timeout: float | None = None,
+        **kwargs: Any,
+    ) -> None:
+        """Initialize the exception.
+
+        Args:
+            message: Human-readable error message.
+            elapsed: Time elapsed before timeout.
+            timeout: The timeout value that was exceeded.
+            **kwargs: Additional arguments for parent class.
+        """
+        super().__init__(message, **kwargs)
+        self.elapsed = elapsed
+        self.timeout = timeout
+        if elapsed is not None:
+            self.details["elapsed"] = elapsed
+        if timeout is not None:
+            self.details["timeout"] = timeout
+
+
+class DiscoveryJobFailedError(DiscoveryError):
+    """Discovery job failed on the server.
+
+    Raised when a discovery job completes with a failed or cancelled status.
+    """
+
+    def __init__(
+        self,
+        message: str = "Discovery job failed",
+        *,
+        status: str | None = None,
+        **kwargs: Any,
+    ) -> None:
+        """Initialize the exception.
+
+        Args:
+            message: Human-readable error message.
+            status: The final job status.
+            **kwargs: Additional arguments for parent class.
+        """
+        super().__init__(message, **kwargs)
+        self.status = status
+        if status:
+            self.details["status"] = status

gnosisllm_knowledge/core/interfaces/__init__.py

@@ -5,16 +5,33 @@ from gnosisllm_knowledge.core.interfaces.chunker import ITextChunker
 from gnosisllm_knowledge.core.interfaces.fetcher import FetchResult, IContentFetcher
 from gnosisllm_knowledge.core.interfaces.indexer import IDocumentIndexer
 from gnosisllm_knowledge.core.interfaces.loader import IContentLoader
+from gnosisllm_knowledge.core.interfaces.memory import (
+    IHistoryRetriever,
+    IMemoryContainerManager,
+    IMemoryRetriever,
+    IMemoryStats,
+    IMemoryStore,
+    ISessionManager,
+)
 from gnosisllm_knowledge.core.interfaces.searcher import IKnowledgeSearcher
 from gnosisllm_knowledge.core.interfaces.setup import ISetupAdapter
 
 __all__ = [
+    # Content loading
     "IContentLoader",
     "IContentFetcher",
     "FetchResult",
     "ITextChunker",
+    # Indexing and search
     "IDocumentIndexer",
     "IKnowledgeSearcher",
     "IAgenticSearcher",
     "ISetupAdapter",
+    # Memory
+    "IMemoryContainerManager",
+    "IMemoryStore",
+    "IMemoryRetriever",
+    "IHistoryRetriever",
+    "ISessionManager",
+    "IMemoryStats",
 ]

gnosisllm_knowledge/core/interfaces/agentic.py

@@ -1,4 +1,11 @@
-"""Agentic searcher protocol - Interface for AI-powered search operations."""
+"""Agentic searcher protocol - Interface for AI-powered search operations.
+
+Note:
+    This library is tenant-agnostic. Multi-tenancy is achieved through index
+    isolation (e.g., `knowledge-{account_id}`). Agentic searcher implementations
+    should not include tenant filtering logic - callers should use tenant-specific
+    indices.
+"""
 
 from __future__ import annotations
 
@@ -15,6 +22,9 @@ if TYPE_CHECKING:
 class IAgenticSearcher(Protocol):
     """Protocol for agentic search operations using AI agents.
 
+    This protocol is tenant-agnostic. Multi-tenancy is achieved through index
+    isolation by using tenant-specific index names.
+
     Agentic searchers are responsible for:
     - Understanding natural language queries
     - Automatically constructing optimal search strategies
@@ -107,13 +117,11 @@ class IAgenticSearcher(Protocol):
 
     async def list_conversations(
         self,
-        account_id: str | None = None,
         limit: int = 100,
     ) -> list[dict[str, Any]]:
         """List active conversations.
 
         Args:
-            account_id: Filter by account (multi-tenant).
             limit: Maximum number of conversations.
 
         Returns:

gnosisllm_knowledge/core/interfaces/indexer.py

@@ -1,4 +1,10 @@
-"""Document indexer protocol - Interface Segregation Principle."""
+"""Document indexer protocol - Interface Segregation Principle.
+
+Note:
+    This library is tenant-agnostic. Multi-tenancy is achieved through index
+    isolation (e.g., `knowledge-{account_id}`). Indexer implementations should
+    not include tenant filtering logic - callers should use tenant-specific indices.
+"""
 
 from __future__ import annotations
 
@@ -14,6 +20,9 @@ if TYPE_CHECKING:
 class IDocumentIndexer(Protocol):
     """Protocol for indexing documents into a search backend.
 
+    This protocol is tenant-agnostic. Multi-tenancy is achieved through index
+    isolation by using tenant-specific index names.
+
     Document indexers are responsible for:
     - Generating embeddings for documents
     - Storing documents in the search backend