gnosisllm-knowledge 0.3.0__py3-none-any.whl → 0.4.0__py3-none-any.whl

gnosisllm_knowledge/backends/opensearch/agentic.py

@@ -2,6 +2,12 @@
 
 Uses OpenSearch ML agents for AI-powered search with reasoning capabilities.
 Supports flow agents (fast RAG) and conversational agents (multi-turn with memory).
+
+Note:
+    This module is **tenant-agnostic**. Multi-tenancy is achieved through index isolation:
+    each tenant's data resides in a separate OpenSearch index. The caller (e.g., gnosisllm-api)
+    is responsible for constructing the appropriate index name (e.g., `knowledge-{account_id}`).
+    The library operates on the provided index without any tenant-specific filtering logic.
 """
 
 from __future__ import annotations
@@ -9,7 +15,6 @@ from __future__ import annotations
 import asyncio
 import json
 import logging
-import uuid
 from datetime import UTC, datetime
 from typing import TYPE_CHECKING, Any
 
@@ -297,13 +302,15 @@ class OpenSearchAgenticSearcher:
 
     async def list_conversations(
         self,
-        account_id: str | None = None,
         limit: int = 100,
     ) -> list[dict[str, Any]]:
         """List active conversations.
 
+        Note:
+            This library is tenant-agnostic. Multi-tenancy is achieved through
+            index isolation (separate index per account).
+
         Args:
-            account_id: Filter by account (multi-tenant).
             limit: Maximum number of conversations.
 
         Returns:
@@ -311,8 +318,6 @@ class OpenSearchAgenticSearcher:
         """
         try:
             body: dict[str, Any] = {"size": limit}
-            if account_id:
-                body["query"] = {"term": {"account_id": account_id}}
 
             response = await self._client.transport.perform_request(
                 "POST",
@@ -365,16 +370,18 @@ class OpenSearchAgenticSearcher:
     async def create_conversation(
         self,
         name: str | None = None,
-        account_id: str | None = None,
     ) -> str | None:
         """Create a new conversation memory.
 
         Uses the OpenSearch Memory API to create a conversation memory.
         The endpoint is POST /_plugins/_ml/memory (introduced in 2.12).
 
+        Note:
+            This library is tenant-agnostic. Multi-tenancy is achieved through
+            index isolation (separate index per account).
+
         Args:
             name: Optional name for the conversation.
-            account_id: Optional account ID for multi-tenancy.
 
         Returns:
             The new conversation/memory ID, or None if creation fails.
@@ -382,8 +389,6 @@ class OpenSearchAgenticSearcher:
         body: dict[str, Any] = {}
         if name:
             body["name"] = name
-        if account_id:
-            body["account_id"] = account_id
 
         try:
             # POST /_plugins/_ml/memory creates a new memory (OpenSearch 2.12+)

gnosisllm_knowledge/backends/opensearch/indexer.py

@@ -87,13 +87,15 @@ class OpenSearchIndexer:
             # Embeddings are generated by OpenSearch ingest pipeline
             doc_body = self._prepare_document(document)
 
-            # Index the document
+            # Index the document with ingest pipeline for embedding generation
             refresh = options.get("refresh", False)
+            pipeline = self._config.ingest_pipeline_name
             await self._client.index(
                 index=index_name,
                 id=document.doc_id,
                 body=doc_body,
                 refresh=refresh,
+                pipeline=pipeline,
             )
 
             return IndexResult(
@@ -272,6 +274,43 @@ class OpenSearchIndexer:
             failed_count=0,
         )
 
+    async def get(
+        self,
+        doc_id: str,
+        index_name: str,
+    ) -> dict[str, Any] | None:
+        """Get a document by ID.
+
+        Uses OpenSearch client's direct get() API (CRUD operation, not search).
+
+        Args:
+            doc_id: Document ID to retrieve.
+            index_name: Index name.
+
+        Returns:
+            Document dict (source fields) or None if not found.
+            Excludes embeddings from response for efficiency.
+        """
+        try:
+            response = await self._client.get(
+                index=index_name,
+                id=doc_id,
+                _source_excludes=["content_embedding"],
+            )
+            source = response.get("_source", {})
+            # Include the document ID in the response
+            source["id"] = response.get("_id", doc_id)
+            return source
+        except Exception as e:
+            if "not_found" in str(e).lower():
+                return None
+            logger.error(f"Failed to get document {doc_id}: {e}")
+            raise IndexError(
+                message=f"Failed to get document: {e}",
+                details={"document_id": doc_id},
+                cause=e,
+            ) from e
+
     async def delete(
         self,
         doc_id: str,
@@ -434,7 +473,9 @@ class OpenSearchIndexer:
         if not actions:
             return IndexResult(success=True, index_name=index_name, indexed_count=0, failed_count=0)
 
-        response = await self._client.bulk(body=actions)
+        # Use ingest pipeline for embedding generation
+        pipeline = self._config.ingest_pipeline_name
+        response = await self._client.bulk(body=actions, pipeline=pipeline)
 
         indexed = 0
         failed = 0
@@ -460,6 +501,11 @@ class OpenSearchIndexer:
     def _prepare_document(self, document: Document) -> dict[str, Any]:
         """Prepare document for indexing.
 
+        Note:
+            This library is tenant-agnostic. Multi-tenancy is achieved through index
+            isolation. Tenant information should be passed in document.metadata if
+            needed for audit purposes.
+
         Args:
             document: Document to prepare.
 
@@ -479,7 +525,6 @@ class OpenSearchIndexer:
             "url": document.url,
             "title": document.title,
             "source": document.source,
-            "account_id": document.account_id,
             "collection_id": document.collection_id,
             "collection_name": document.collection_name,
             "source_id": document.source_id,

gnosisllm_knowledge/backends/opensearch/mappings.py

@@ -1,4 +1,10 @@
-"""OpenSearch index mappings for knowledge documents."""
+"""OpenSearch index mappings for knowledge documents.
+
+Note:
+    This library is tenant-agnostic. Multi-tenancy is achieved through index
+    isolation (e.g., `knowledge-{account_id}`). Index mappings do not include
+    tenant-specific fields like account_id.
+"""
 
 from __future__ import annotations
 
@@ -56,8 +62,7 @@ def get_knowledge_index_mappings(config: OpenSearchConfig) -> dict[str, Any]:
                 "fields": {"keyword": {"type": "keyword", "ignore_above": 512}},
             },
             "source": {"type": "keyword"},
-            # === Multi-tenant Fields ===
-            "account_id": {"type": "keyword"},
+            # === Collection Fields ===
             "collection_id": {"type": "keyword"},
             "collection_name": {"type": "keyword"},  # For aggregation display
             "source_id": {"type": "keyword"},
@@ -129,13 +134,16 @@ def get_memory_index_settings(config: OpenSearchConfig) -> dict[str, Any]:
 def get_memory_index_mappings() -> dict[str, Any]:
     """Get index mappings for conversation memory.
 
+    Note:
+        This library is tenant-agnostic. Multi-tenancy is achieved through index
+        isolation. Use tenant-specific index names for conversation memory.
+
     Returns:
         Index mappings dictionary.
     """
     return {
         "properties": {
             "conversation_id": {"type": "keyword"},
-            "account_id": {"type": "keyword"},
             "user_id": {"type": "keyword"},
             "message_index": {"type": "integer"},
             "role": {"type": "keyword"},  # user, assistant, system

gnosisllm_knowledge/backends/opensearch/queries.py

@@ -2,6 +2,10 @@
 
 Uses OpenSearch neural search - embeddings are generated automatically
 via the deployed model. No Python-side embedding generation needed.
+
+Note: This module is tenant-agnostic. Multi-tenancy should be handled
+at the API layer by using separate indices per account (e.g.,
+`knowledge-{account_id}`) rather than filtering by account_id.
 """
 
 from __future__ import annotations
@@ -18,9 +22,13 @@ class QueryBuilder:
     model handles embedding generation automatically via ingest and
     search pipelines.
 
+    Note:
+        This builder is tenant-agnostic. Multi-tenancy should be handled
+        by using separate indices per account.
+
     Example:
         ```python
-        query = SearchQuery(text="how to configure", account_id="acc123")
+        query = SearchQuery(text="how to configure", collection_ids=["col-1"])
         builder = QueryBuilder(query, model_id="abc123")
         os_query = builder.build_hybrid_query()
         ```
@@ -204,12 +212,12 @@ class QueryBuilder:
             },
         }
 
-        # Apply filters at top level for hybrid
+        # Apply filters using post_filter for hybrid queries
+        # Hybrid queries cannot be wrapped in bool - they must be top-level
         filters = self._build_filters()
         if filters:
-            query["query"] = {
+            query["post_filter"] = {
                 "bool": {
-                    "must": [query["query"]],
                     "filter": filters,
                 }
             }
@@ -270,15 +278,15 @@ class QueryBuilder:
     def _build_filters(self) -> list[dict[str, Any]]:
         """Build filter clauses from query parameters.
 
+        Note:
+            This method is tenant-agnostic. Multi-tenancy should be handled
+            at the API layer by using separate indices per account.
+
         Returns:
-            List of filter clauses.
+            List of filter clauses for collection, source, and metadata filters.
         """
         filters: list[dict[str, Any]] = []
 
-        # Multi-tenant filter (required for security)
-        if self._query.account_id:
-            filters.append({"term": {"account_id": self._query.account_id}})
-
         # Collection filter
         if self._query.collection_ids:
             filters.append({"terms": {"collection_id": self._query.collection_ids}})
@@ -357,67 +365,61 @@ class QueryBuilder:
         ]
 
 
-def build_delete_by_source_query(
-    source_id: str,
-    account_id: str | None = None,
-) -> dict[str, Any]:
+def build_delete_by_source_query(source_id: str) -> dict[str, Any]:
     """Build query to delete documents by source.
 
+    Note:
+        This function is tenant-agnostic. Multi-tenancy should be handled
+        at the API layer by using separate indices per account.
+
     Args:
         source_id: Source ID to delete.
-        account_id: Optional account filter for multi-tenancy.
 
     Returns:
         Delete-by-query dictionary.
     """
-    filters = [{"term": {"source_id": source_id}}]
-    if account_id:
-        filters.append({"term": {"account_id": account_id}})
-
     return {
         "query": {
             "bool": {
-                "filter": filters,
+                "filter": [{"term": {"source_id": source_id}}],
             }
         }
     }
 
 
-def build_delete_by_collection_query(
-    collection_id: str,
-    account_id: str | None = None,
-) -> dict[str, Any]:
+def build_delete_by_collection_query(collection_id: str) -> dict[str, Any]:
     """Build query to delete documents by collection.
 
+    Note:
+        This function is tenant-agnostic. Multi-tenancy should be handled
+        at the API layer by using separate indices per account.
+
     Args:
         collection_id: Collection ID to delete.
-        account_id: Optional account filter for multi-tenancy.
 
     Returns:
         Delete-by-query dictionary.
     """
-    filters = [{"term": {"collection_id": collection_id}}]
-    if account_id:
-        filters.append({"term": {"account_id": account_id}})
-
     return {
         "query": {
             "bool": {
-                "filter": filters,
+                "filter": [{"term": {"collection_id": collection_id}}],
             }
         }
     }
 
 
 def build_count_query(
-    account_id: str | None = None,
     collection_id: str | None = None,
     source_id: str | None = None,
 ) -> dict[str, Any]:
     """Build query to count documents.
 
+    Note:
+        This function is tenant-agnostic. Multi-tenancy should be handled
+        at the API layer by using separate indices per account.
+
     Args:
-        account_id: Optional account filter.
         collection_id: Optional collection filter.
         source_id: Optional source filter.
 
@@ -426,8 +428,6 @@ def build_count_query(
     """
     filters: list[dict[str, Any]] = []
 
-    if account_id:
-        filters.append({"term": {"account_id": account_id}})
     if collection_id:
         filters.append({"term": {"collection_id": collection_id}})
     if source_id:

gnosisllm_knowledge/backends/opensearch/searcher.py

@@ -2,6 +2,10 @@
 
 Uses OpenSearch neural search - embeddings are generated automatically
 by the deployed ML model. No Python-side embedding generation needed.
+
+Note: This module is tenant-agnostic. Multi-tenancy should be handled
+at the API layer by using separate indices per account (e.g.,
+`knowledge-{account_id}`) rather than filtering by account_id.
 """
 
 from __future__ import annotations
@@ -506,7 +510,6 @@ class OpenSearchKnowledgeSearcher:
         self,
         index_name: str,
         *,
-        account_id: str | None = None,
         source_id: str | None = None,
         collection_id: str | None = None,
         limit: int = 50,
@@ -514,9 +517,12 @@ class OpenSearchKnowledgeSearcher:
     ) -> dict[str, Any]:
         """List documents with optional filters.
 
+        Note:
+            This method is tenant-agnostic. Multi-tenancy should be handled
+            at the API layer by using separate indices per account.
+
         Args:
-            index_name: Index to query.
-            account_id: Optional account ID filter.
+            index_name: Index to query (use tenant-specific name for isolation).
             source_id: Optional source ID filter.
             collection_id: Optional collection ID filter.
             limit: Maximum documents to return.
@@ -540,9 +546,6 @@ class OpenSearchKnowledgeSearcher:
             # Build filter clauses
             filters: list[dict[str, Any]] = []
 
-            if account_id:
-                filters.append({"term": {"account_id": account_id}})
-
             if source_id:
                 filters.append({"term": {"source_id": source_id}})
 

gnosisllm_knowledge/cli/app.py

@@ -1,6 +1,11 @@
 """GnosisLLM Knowledge CLI Application.
 
 Main entry point assembling all CLI commands with enterprise-grade UX.
+
+Note:
+    This library is tenant-agnostic. Multi-tenancy is achieved through index
+    isolation - each tenant should use a separate index (e.g., "knowledge-{account_id}").
+    Use --index to target tenant-specific indices.
 """
 
 from __future__ import annotations
@@ -147,17 +152,13 @@ def load(
         typer.Option(
             "--type",
             "-t",
-            help="Source type: website, sitemap (auto-detects if not specified).",
+            help="Source type: website, sitemap, discovery (auto-detects if not specified).",
         ),
     ] = None,
     index: Annotated[
         str,
-        typer.Option("--index", "-i", help="Target index name."),
+        typer.Option("--index", "-i", help="Target index name (use tenant-specific name for multi-tenancy)."),
     ] = "knowledge",
-    account_id: Annotated[
-        Optional[str],
-        typer.Option("--account-id", "-a", help="Multi-tenant account ID."),
-    ] = None,
     collection_id: Annotated[
         Optional[str],
         typer.Option("--collection-id", "-c", help="Collection grouping ID."),
@@ -186,16 +187,50 @@ def load(
         bool,
         typer.Option("--verbose", "-V", help="Show per-document progress."),
     ] = False,
+    discovery: Annotated[
+        bool,
+        typer.Option(
+            "--discovery",
+            "-D",
+            help="Use discovery loader to crawl and discover all URLs from the website.",
+        ),
+    ] = False,
+    max_depth: Annotated[
+        int,
+        typer.Option("--max-depth", help="Maximum crawl depth for discovery (default: 3)."),
+    ] = 3,
+    max_pages: Annotated[
+        int,
+        typer.Option("--max-pages", help="Maximum pages to discover (default: 100)."),
+    ] = 100,
+    same_domain: Annotated[
+        bool,
+        typer.Option(
+            "--same-domain/--any-domain",
+            help="Only crawl URLs on the same domain (default: same domain only).",
+        ),
+    ] = True,
 ) -> None:
     """Load and index content from URLs or sitemaps.
 
     Fetches content, chunks it for optimal embedding, and indexes
     into OpenSearch with automatic embedding generation.
 
+    [bold]Multi-tenancy:[/bold]
+    Use --index with tenant-specific index names for isolation
+    (e.g., --index knowledge-{account_id}). Each tenant's data
+    is stored in a separate index for complete isolation.
+
+    [bold]Discovery Mode:[/bold]
+    Use --discovery to crawl and discover all URLs from a website
+    before loading. This is useful for sites without a sitemap.
+
     [bold]Example:[/bold]
         $ gnosisllm-knowledge load https://docs.example.com/intro
         $ gnosisllm-knowledge load https://example.com/sitemap.xml --type sitemap
         $ gnosisllm-knowledge load https://docs.example.com/sitemap.xml --max-urls 500
+        $ gnosisllm-knowledge load https://docs.example.com --discovery --max-depth 5
+        $ gnosisllm-knowledge load https://docs.example.com --index knowledge-tenant-123
     """
     from gnosisllm_knowledge.cli.commands.load import load_command
 
@@ -205,7 +240,6 @@ def load(
             source=source,
             source_type=source_type,
             index_name=index,
-            account_id=account_id,
             collection_id=collection_id,
             source_id=source_id,
             batch_size=batch_size,
@@ -213,6 +247,10 @@ def load(
             force=force,
             dry_run=dry_run,
             verbose=verbose,
+            discovery=discovery,
+            max_depth=max_depth,
+            max_pages=max_pages,
+            same_domain=same_domain,
         )
     )
 
@@ -238,7 +276,7 @@ def search(
     ] = "hybrid",
     index: Annotated[
         str,
-        typer.Option("--index", "-i", help="Index to search."),
+        typer.Option("--index", "-i", help="Index to search (use tenant-specific name for multi-tenancy)."),
     ] = "knowledge",
     limit: Annotated[
         int,
@@ -248,10 +286,6 @@ def search(
         int,
         typer.Option("--offset", "-o", help="Pagination offset."),
     ] = 0,
-    account_id: Annotated[
-        Optional[str],
-        typer.Option("--account-id", "-a", help="Filter by account ID."),
-    ] = None,
     collection_ids: Annotated[
         Optional[str],
         typer.Option("--collection-ids", "-c", help="Filter by collection IDs (comma-separated)."),
@@ -289,10 +323,16 @@ def search(
     - [cyan]hybrid[/cyan]: Combined semantic + keyword (default, best results)
     - [cyan]agentic[/cyan]: AI-powered search with reasoning
 
+    [bold]Multi-tenancy:[/bold]
+    Use --index with tenant-specific index names for isolation
+    (e.g., --index knowledge-{account_id}). Each tenant's data
+    is stored in a separate index for complete isolation.
+
     [bold]Example:[/bold]
         $ gnosisllm-knowledge search "how to configure auth"
         $ gnosisllm-knowledge search "API reference" --mode semantic --limit 10
         $ gnosisllm-knowledge search --interactive
+        $ gnosisllm-knowledge search "query" --index knowledge-tenant-123
     """
     from gnosisllm_knowledge.cli.commands.search import search_command
 
@@ -304,7 +344,6 @@ def search(
             index_name=index,
             limit=limit,
             offset=offset,
-            account_id=account_id,
             collection_ids=collection_ids,
             source_ids=source_ids,
             min_score=min_score,
@@ -451,7 +490,7 @@ def agentic_setup(
 def agentic_chat(
     index: Annotated[
         str,
-        typer.Option("--index", "-i", help="Index to search."),
+        typer.Option("--index", "-i", help="Index to search (use tenant-specific name for multi-tenancy)."),
     ] = "knowledge",
     agent_type: Annotated[
         str,
@@ -461,10 +500,6 @@ def agentic_chat(
             help="Agent type: flow or conversational (default).",
         ),
     ] = "conversational",
-    account_id: Annotated[
-        Optional[str],
-        typer.Option("--account-id", "-a", help="Filter by account ID."),
-    ] = None,
     collection_ids: Annotated[
         Optional[str],
         typer.Option("--collection-ids", "-c", help="Filter by collection IDs (comma-separated)."),
@@ -479,10 +514,15 @@ def agentic_chat(
     Start a conversation with the AI-powered knowledge assistant.
     The agent remembers context for multi-turn dialogue.
 
+    [bold]Multi-tenancy:[/bold]
+    Use --index with tenant-specific index names for isolation
+    (e.g., --index knowledge-{account_id}).
+
     [bold]Example:[/bold]
         $ gnosisllm-knowledge agentic chat
         $ gnosisllm-knowledge agentic chat --type flow
         $ gnosisllm-knowledge agentic chat --verbose
+        $ gnosisllm-knowledge agentic chat --index knowledge-tenant-123
     """
     from gnosisllm_knowledge.cli.commands.agentic import agentic_chat_command
 
@@ -491,7 +531,6 @@ def agentic_chat(
             display=display,
             index_name=index,
             agent_type=agent_type,
-            account_id=account_id,
             collection_ids=collection_ids,
             verbose=verbose,
         )

gnosisllm_knowledge/cli/commands/agentic.py

@@ -4,6 +4,10 @@ Commands:
 - setup: Configure agents in OpenSearch
 - chat: Interactive agentic chat session
 - status: Show agent configuration status
+
+Note:
+    This library is tenant-agnostic. Multi-tenancy is achieved through index
+    isolation - each tenant should use a separate index (e.g., "knowledge-{account_id}").
 """
 
 from __future__ import annotations
@@ -202,17 +206,19 @@ async def agentic_chat_command(
     display: RichDisplayService,
     index_name: str = "knowledge",
     agent_type: str = "conversational",
-    account_id: str | None = None,
     collection_ids: str | None = None,
     verbose: bool = False,
 ) -> None:
     """Interactive agentic chat session.
 
+    Note:
+        Multi-tenancy is achieved through index isolation. Use tenant-specific
+        index names instead (e.g., --index knowledge-tenant-123).
+
     Args:
         display: Display service for output.
-        index_name: Index to search.
+        index_name: Index to search (use tenant-specific name for isolation).
         agent_type: Agent type ('flow' or 'conversational').
-        account_id: Filter by account ID.
         collection_ids: Filter by collection IDs (comma-separated).
         verbose: Show reasoning steps.
     """
@@ -242,7 +248,6 @@ async def agentic_chat_command(
         if agent_type == "conversational":
             return await searcher.create_conversation(
                 name="CLI Chat Session",
-                account_id=account_id,
             )
         return None
 
@@ -291,7 +296,6 @@ async def agentic_chat_command(
                     agent_type=AgentType.CONVERSATIONAL if agent_type == "conversational" else AgentType.FLOW,
                     conversation_id=conversation_id,
                     collection_ids=collection_list,
-                    account_id=account_id,
                     include_reasoning=verbose,
                 )
 
@@ -395,7 +399,6 @@ async def agentic_search_command(
     query: str,
     index_name: str = "knowledge",
     agent_type: str = "flow",
-    account_id: str | None = None,
     collection_ids: str | None = None,
     source_ids: str | None = None,
     limit: int = 5,
@@ -404,12 +407,15 @@ async def agentic_search_command(
 ) -> dict[str, Any] | None:
     """Execute agentic search.
 
+    Note:
+        Multi-tenancy is achieved through index isolation. Use tenant-specific
+        index names instead (e.g., --index knowledge-tenant-123).
+
     Args:
         display: Display service for output.
         query: Search query text.
-        index_name: Index to search.
+        index_name: Index to search (use tenant-specific name for isolation).
         agent_type: Agent type ('flow' or 'conversational').
-        account_id: Filter by account ID.
         collection_ids: Filter by collection IDs (comma-separated).
         source_ids: Filter by source IDs (comma-separated).
         limit: Maximum source documents to retrieve.
@@ -447,12 +453,12 @@ async def agentic_search_command(
             )
 
         # Build query
+        # Note: account_id is deprecated and ignored - use index isolation instead
         agentic_query = AgenticSearchQuery(
             text=query,
             agent_type=AgentType.CONVERSATIONAL if agent_type == "conversational" else AgentType.FLOW,
             collection_ids=collection_list,
             source_ids=source_list,
-            account_id=account_id,
             limit=limit,
             include_reasoning=verbose,
         )