gnosisllm-knowledge 0.3.0__py3-none-any.whl → 0.4.3__py3-none-any.whl

gnosisllm_knowledge/api/knowledge.py

@@ -1,4 +1,39 @@
-"""High-level Knowledge API facade."""
+"""High-level Knowledge API facade.
+
+This module provides the main entry point for the gnosisllm-knowledge library.
+The Knowledge class is a high-level facade that abstracts the complexity of
+loading, indexing, and searching knowledge documents.
+
+Note:
+    This library is tenant-agnostic. Multi-tenancy should be handled at the
+    API layer by using separate indices per account (e.g.,
+    `knowledge-{account_id}`) rather than filtering by account_id.
+
+Example:
+    ```python
+    # Create Knowledge instance for a specific tenant
+    knowledge = Knowledge.from_opensearch(
+        host="localhost",
+        port=9200,
+    )
+
+    # Use a tenant-specific index
+    tenant_index = f"knowledge-{account_id}"
+
+    # Load content
+    await knowledge.load(
+        "https://docs.example.com/sitemap.xml",
+        index_name=tenant_index,
+        collection_id="docs",
+    )
+
+    # Search (tenant isolation via index name)
+    results = await knowledge.search(
+        "how to configure",
+        index_name=tenant_index,
+    )
+    ```
+"""
 
 from __future__ import annotations
 
@@ -84,6 +119,7 @@ class Knowledge:
         loader_factory: LoaderFactory | None = None,
         default_index: str | None = None,
         events: EventEmitter | None = None,
+        config: OpenSearchConfig | None = None,
     ) -> None:
         """Initialize Knowledge with components.
 
@@ -96,6 +132,7 @@ class Knowledge:
             loader_factory: Optional loader factory.
             default_index: Default index name.
             events: Optional event emitter.
+            config: Optional OpenSearch config for settings like batch sizes.
 
         Note:
             Embeddings are generated automatically by OpenSearch ingest pipeline.
@@ -109,6 +146,7 @@ class Knowledge:
         self._loader_factory = loader_factory
         self._default_index = default_index
         self._events = events or EventEmitter()
+        self._config = config
 
         # Initialize services lazily
         self._indexing_service: KnowledgeIndexingService | None = None
@@ -130,6 +168,10 @@ class Knowledge:
     ) -> Knowledge:
         """Create Knowledge instance with OpenSearch backend.
 
+        This factory creates a Knowledge instance configured for OpenSearch.
+        The returned instance is tenant-agnostic - multi-tenancy should be
+        handled by using separate indices per account.
+
         Args:
             host: OpenSearch host.
             port: OpenSearch port.
@@ -147,6 +189,19 @@ class Knowledge:
         Note:
             Embeddings are generated automatically by OpenSearch ingest pipeline.
             Run 'gnosisllm-knowledge setup' to configure the ML model.
+
+        Example:
+            ```python
+            # Create a Knowledge instance
+            knowledge = Knowledge.from_opensearch(
+                host="localhost",
+                port=9200,
+            )
+
+            # Use tenant-specific index for isolation
+            tenant_index = f"gnosisllm-{account_id}-knowledge"
+            await knowledge.load(source, index_name=tenant_index)
+            ```
         """
         # Import OpenSearch client
         try:
@@ -210,21 +265,36 @@ class Knowledge:
             fetcher=fetcher,
             loader_factory=loader_factory,
             default_index=config.knowledge_index_name,
+            config=config,
         )
 
     @classmethod
     def from_env(cls) -> Knowledge:
         """Create Knowledge instance from environment variables.
 
+        This factory creates a Knowledge instance using configuration from
+        environment variables. The returned instance is tenant-agnostic -
+        multi-tenancy should be handled by using separate indices per account.
+
         Returns:
             Configured Knowledge instance.
+
+        Example:
+            ```python
+            # Create from environment
+            knowledge = Knowledge.from_env()
+
+            # Use tenant-specific index for isolation
+            tenant_index = f"gnosisllm-{account_id}-knowledge"
+            await knowledge.search("query", index_name=tenant_index)
+            ```
         """
         config = OpenSearchConfig.from_env()
         neoreader_config = NeoreaderConfig.from_env()
 
         return cls.from_opensearch(
             config=config,
-            neoreader_url=neoreader_config.base_url if neoreader_config.base_url else None,
+            neoreader_url=neoreader_config.host if neoreader_config.host else None,
         )
 
     @property
@@ -318,7 +388,6 @@ class Knowledge:
         source: str,
         *,
         index_name: str | None = None,
-        account_id: str | None = None,
         collection_id: str | None = None,
         source_id: str | None = None,
         source_type: str | None = None,
@@ -329,10 +398,13 @@ class Knowledge:
 
         Automatically detects source type (sitemap, website, etc.).
 
+        Note:
+            This method is tenant-agnostic. Multi-tenancy should be handled
+            by using separate indices per account.
+
         Args:
             source: Source URL or path.
-            index_name: Target index (uses default if not provided).
-            account_id: Account ID for multi-tenancy.
+            index_name: Target index (use tenant-specific name for isolation).
             collection_id: Collection ID.
             source_id: Source ID (auto-generated if not provided).
             source_type: Explicit source type (auto-detected if not provided).
@@ -363,12 +435,15 @@ class Knowledge:
             events=self._events,
         )
 
+        # Get batch size from config or use default
+        batch_size = self._config.indexing_batch_size if self._config else 10
+
         return await service.load_and_index(
             source=source,
             index_name=index,
-            account_id=account_id,
             collection_id=collection_id,
             source_id=source_id,
+            batch_size=batch_size,
             **options,
         )
 
@@ -377,7 +452,6 @@ class Knowledge:
         source: str,
         *,
         index_name: str | None = None,
-        account_id: str | None = None,
         collection_id: str | None = None,
         collection_name: str | None = None,
         source_id: str | None = None,
@@ -398,10 +472,13 @@ class Knowledge:
         - Document storage: O(index_batch_size)
         - In-flight fetches: O(fetch_concurrency * avg_page_size)
 
+        Note:
+            This method is tenant-agnostic. Multi-tenancy should be handled
+            by using separate indices per account.
+
         Args:
             source: Sitemap URL.
-            index_name: Target index (uses default if not provided).
-            account_id: Account ID for multi-tenancy.
+            index_name: Target index (use tenant-specific name for isolation).
             collection_id: Collection ID.
             collection_name: Collection name for display.
             source_id: Source ID (auto-generated if not provided).
@@ -419,6 +496,7 @@ class Knowledge:
             # Efficiently load 100k+ URL sitemap
             result = await knowledge.load_streaming(
                 "https://large-site.com/sitemap.xml",
+                index_name="knowledge-account123",  # Tenant-specific
                 url_batch_size=100,
                 fetch_concurrency=20,
                 max_urls=50000,
@@ -454,7 +532,6 @@ class Knowledge:
         return await pipeline.execute(
             source=source,
             index_name=index,
-            account_id=account_id,
             collection_id=collection_id,
             collection_name=collection_name,
             source_id=source_id,
@@ -471,7 +548,6 @@ class Knowledge:
         mode: SearchMode = SearchMode.HYBRID,
         limit: int = 10,
         offset: int = 0,
-        account_id: str | None = None,
         collection_ids: list[str] | None = None,
         source_ids: list[str] | None = None,
         min_score: float | None = None,
@@ -479,13 +555,16 @@ class Knowledge:
     ) -> SearchResult:
         """Search for knowledge documents.
 
+        Note:
+            This method is tenant-agnostic. Multi-tenancy should be handled
+            by using separate indices per account.
+
         Args:
             query: Search query text.
-            index_name: Index to search (uses default if not provided).
+            index_name: Index to search (use tenant-specific name for isolation).
             mode: Search mode (semantic, keyword, hybrid).
             limit: Maximum results.
             offset: Result offset for pagination.
-            account_id: Account ID for multi-tenancy.
             collection_ids: Filter by collection IDs.
             source_ids: Filter by source IDs.
             min_score: Minimum score threshold.
@@ -500,7 +579,6 @@ class Knowledge:
             mode=mode,
             limit=limit,
             offset=offset,
-            account_id=account_id,
             collection_ids=collection_ids,
             source_ids=source_ids,
             min_score=min_score,
@@ -578,19 +656,73 @@ class Knowledge:
 
     # === Management Methods ===
 
+    async def get_document(
+        self,
+        document_id: str,
+        *,
+        index_name: str | None = None,
+    ) -> dict[str, Any] | None:
+        """Get a single document by ID.
+
+        Note:
+            This method is tenant-agnostic. Multi-tenancy should be handled
+            by using separate indices per account.
+
+        Args:
+            document_id: Document ID to retrieve.
+            index_name: Index name (use tenant-specific name for isolation).
+                Uses default index if not provided.
+
+        Returns:
+            Document dict with all fields (excluding embeddings) or None if not found.
+        """
+        index = index_name or self._default_index
+        if not index:
+            raise ValueError("No index specified and no default index configured")
+
+        return await self._indexer.get(document_id, index)
+
+    async def delete_document(
+        self,
+        document_id: str,
+        *,
+        index_name: str | None = None,
+    ) -> bool:
+        """Delete a single document by ID.
+
+        Note:
+            This method is tenant-agnostic. Multi-tenancy should be handled
+            by using separate indices per account.
+
+        Args:
+            document_id: Document ID to delete.
+            index_name: Index name (use tenant-specific name for isolation).
+                Uses default index if not provided.
+
+        Returns:
+            True if deleted, False if not found.
+        """
+        index = index_name or self._default_index
+        if not index:
+            raise ValueError("No index specified and no default index configured")
+
+        return await self._indexer.delete(document_id, index)
+
     async def delete_source(
         self,
         source_id: str,
         *,
         index_name: str | None = None,
-        account_id: str | None = None,
     ) -> int:
         """Delete all documents from a source.
 
+        Note:
+            This method is tenant-agnostic. Multi-tenancy should be handled
+            by using separate indices per account.
+
         Args:
             source_id: Source ID to delete.
-            index_name: Index name.
-            account_id: Account ID for multi-tenancy.
+            index_name: Index name (use tenant-specific name for isolation).
 
         Returns:
             Count of deleted documents.
@@ -599,21 +731,23 @@ class Knowledge:
         if not index:
             raise ValueError("No index specified")
 
-        return await self.indexing.delete_source(source_id, index, account_id)
+        return await self.indexing.delete_source(source_id, index)
 
     async def delete_collection(
         self,
         collection_id: str,
         *,
         index_name: str | None = None,
-        account_id: str | None = None,
     ) -> int:
         """Delete all documents from a collection.
 
+        Note:
+            This method is tenant-agnostic. Multi-tenancy should be handled
+            by using separate indices per account.
+
         Args:
             collection_id: Collection ID to delete.
-            index_name: Index name.
-            account_id: Account ID for multi-tenancy.
+            index_name: Index name (use tenant-specific name for isolation).
 
         Returns:
             Count of deleted documents.
@@ -622,54 +756,85 @@ class Knowledge:
         if not index:
             raise ValueError("No index specified")
 
-        return await self.indexing.delete_collection(collection_id, index, account_id)
+        return await self.indexing.delete_collection(collection_id, index)
 
     async def count(
         self,
         *,
         index_name: str | None = None,
-        account_id: str | None = None,
         collection_id: str | None = None,
+        source_id: str | None = None,
     ) -> int:
         """Count documents.
 
+        Note:
+            This method is tenant-agnostic. Multi-tenancy should be handled
+            by using separate indices per account.
+
         Args:
-            index_name: Index to count.
-            account_id: Filter by account.
+            index_name: Index to count (use tenant-specific name for isolation).
             collection_id: Filter by collection.
+            source_id: Filter by source (for source deletion confirmation).
 
         Returns:
             Document count.
         """
         return await self.search_service.count(
             index_name=index_name,
-            account_id=account_id,
             collection_id=collection_id,
+            source_id=source_id,
         )
 
     # === Collection and Stats Methods ===
 
-    async def get_collections(self) -> list[dict[str, Any]]:
+    async def get_collections(
+        self,
+        *,
+        index_name: str | None = None,
+    ) -> list[dict[str, Any]]:
         """Get all collections with document counts.
 
         Aggregates unique collection_ids from indexed documents.
 
+        Note:
+            This method is tenant-agnostic. Multi-tenancy should be handled
+            by using separate indices per account.
+
+        Args:
+            index_name: Index to query (use tenant-specific name for isolation).
+                Uses default index if not provided.
+
         Returns:
             List of collection dictionaries with id, name, and document_count.
         """
-        return await self.search_service.get_collections()
+        index = index_name or self._default_index
+        return await self.search_service.get_collections(index_name=index)
 
-    async def get_stats(self) -> dict[str, Any]:
+    async def get_stats(
+        self,
+        *,
+        index_name: str | None = None,
+    ) -> dict[str, Any]:
         """Get index statistics.
 
+        Note:
+            This method is tenant-agnostic. Multi-tenancy should be handled
+            by using separate indices per account.
+
+        Args:
+            index_name: Index to query (use tenant-specific name for isolation).
+                Uses default index if not provided.
+
         Returns:
             Dictionary with document_count, index_name, and other stats.
         """
-        return await self.search_service.get_stats()
+        index = index_name or self._default_index
+        return await self.search_service.get_stats(index_name=index)
 
     async def list_documents(
         self,
         *,
+        index_name: str | None = None,
         source_id: str | None = None,
         collection_id: str | None = None,
         limit: int = 50,
@@ -677,7 +842,13 @@ class Knowledge:
     ) -> dict[str, Any]:
         """List documents with optional filters.
 
+        Note:
+            This method is tenant-agnostic. Multi-tenancy should be handled
+            by using separate indices per account.
+
         Args:
+            index_name: Index to query (use tenant-specific name for isolation).
+                Uses default index if not provided.
             source_id: Optional source ID filter.
             collection_id: Optional collection ID filter.
             limit: Maximum documents to return (max 100).
@@ -686,9 +857,9 @@ class Knowledge:
         Returns:
             Dictionary with documents, total, limit, offset.
         """
-        index = self._default_index
+        index = index_name or self._default_index
         if not index:
-            raise ValueError("No default index configured")
+            raise ValueError("No index specified and no default index configured")
 
         # Clamp limit to reasonable bounds
         limit = min(max(1, limit), 100)
@@ -823,6 +994,33 @@ class Knowledge:
         return await agentic_searcher.agentic_search(agentic_query, index, **options)
 
     async def close(self) -> None:
-        """Close connections and clean up resources."""
-        # Subclasses or future implementations can override this
-        pass
+        """Close connections and clean up resources.
+
+        Closes the underlying AsyncOpenSearch client to prevent
+        unclosed aiohttp session warnings. Properly handles
+        CancelledError during event loop shutdown.
+        """
+        import asyncio
+
+        # Close the OpenSearch client via the searcher
+        # Note: indexer, searcher, and setup share the same client instance,
+        # so closing via searcher is sufficient
+        if hasattr(self._searcher, '_client') and self._searcher._client is not None:
+            client = self._searcher._client
+            try:
+                await client.close()
+                logger.debug("Closed OpenSearch client connection")
+            except asyncio.CancelledError:
+                # Event loop is shutting down - this is expected during cleanup
+                logger.debug("OpenSearch client close cancelled (event loop shutting down)")
+            except Exception as e:
+                logger.warning(f"Error closing OpenSearch client: {e}")
+            finally:
+                # Clear client reference on all components that share it
+                # This prevents any accidental reuse after close
+                if hasattr(self._searcher, '_client'):
+                    self._searcher._client = None
+                if hasattr(self._indexer, '_client'):
+                    self._indexer._client = None
+                if self._setup and hasattr(self._setup, '_client'):
+                    self._setup._client = None

gnosisllm_knowledge/backends/memory/indexer.py

@@ -1,4 +1,10 @@
-"""In-memory document indexer for testing."""
+"""In-memory document indexer for testing.
+
+Note:
+    This library is tenant-agnostic. Multi-tenancy is achieved through index
+    isolation (e.g., `knowledge-{account_id}`). The memory indexer does not
+    include tenant filtering logic - use separate index names per tenant.
+"""
 
 from __future__ import annotations
 
@@ -14,6 +20,9 @@ from gnosisllm_knowledge.core.domain.result import BatchResult, IndexResult
 class MemoryIndexer:
     """In-memory document indexer for testing.
 
+    This indexer is tenant-agnostic. Multi-tenancy is achieved through index
+    isolation by using tenant-specific index names.
+
     Stores documents in a dictionary for fast testing without
     requiring an external OpenSearch instance.
 
@@ -185,6 +194,22 @@ class MemoryIndexer:
         """
         return await self.index(document, index_name)
 
+    async def get(
+        self,
+        doc_id: str,
+        index_name: str,
+    ) -> dict[str, Any] | None:
+        """Get a document by ID (async interface).
+
+        Args:
+            doc_id: Document ID.
+            index_name: Index name.
+
+        Returns:
+            Document dictionary or None if not found.
+        """
+        return self._indices.get(index_name, {}).get(doc_id)
+
     async def delete(
         self,
         doc_id: str,
@@ -365,8 +390,8 @@ class MemoryIndexer:
             "url": document.url,
             "title": document.title,
             "source": document.source,
-            "account_id": document.account_id,
             "collection_id": document.collection_id,
+            "collection_name": document.collection_name,
             "source_id": document.source_id,
             "chunk_index": document.chunk_index,
             "total_chunks": document.total_chunks,