gnosisllm-knowledge 0.2.0__py3-none-any.whl → 0.4.0__py3-none-any.whl

gnosisllm_knowledge/backends/opensearch/config.py

@@ -74,7 +74,7 @@ class OpenSearchConfig:
 
     # === k-NN Settings ===
     knn_engine: str = "lucene"  # lucene (recommended for OpenSearch 2.9+), faiss
-    knn_space_type: str = "l2"  # l2, cosinesimil, innerproduct
+    knn_space_type: str = "cosinesimil"  # cosinesimil (recommended), l2, innerproduct
     knn_algo_param_ef_search: int = 512
     knn_algo_param_ef_construction: int = 512
     knn_algo_param_m: int = 16
@@ -85,10 +85,13 @@ class OpenSearchConfig:
     model_group_id: str | None = None
     embedding_field: str = "content_embedding"  # Field name for embeddings
 
-    # === Agentic Search ===
+    # === Agentic Search (OpenSearch 3.2+) ===
+    # Uses QueryPlanningTool for LLM-generated DSL queries
     # Agent IDs from 'gnosisllm-knowledge agentic setup'
     flow_agent_id: str | None = None
     conversational_agent_id: str | None = None
+    # Agentic search pipeline (created during agentic setup)
+    agentic_pipeline_name: str | None = None
     # LLM for agent reasoning (OpenAI model ID)
     agentic_llm_model: str = "gpt-4o"
     # Agent execution limits
@@ -139,57 +142,75 @@ class OpenSearchConfig:
     def from_env(cls) -> OpenSearchConfig:
         """Create config from environment variables.
 
-        Environment variables:
-            OPENSEARCH_HOST: Host (default: localhost)
-            OPENSEARCH_PORT: Port (default: 9200)
-            OPENSEARCH_USE_SSL: Use SSL (default: false)
-            OPENSEARCH_VERIFY_CERTS: Verify certificates (default: true)
-            OPENSEARCH_USERNAME: Username
-            OPENSEARCH_PASSWORD: Password
-            OPENSEARCH_USE_AWS_SIGV4: Use AWS Sig v4 auth (default: false)
-            AWS_REGION: AWS region for Sig v4
-            OPENSEARCH_NODES: Comma-separated list of nodes
-            EMBEDDING_MODEL: OpenAI embedding model
-            EMBEDDING_DIMENSION: Embedding vector dimension
-            OPENAI_API_KEY: OpenAI API key
-            OPENSEARCH_INDEX_PREFIX: Index name prefix
-            OPENSEARCH_SHARDS: Number of shards
-            OPENSEARCH_REPLICAS: Number of replicas
-            OPENSEARCH_FLOW_AGENT_ID: Flow agent ID for agentic search
-            OPENSEARCH_CONVERSATIONAL_AGENT_ID: Conversational agent ID
-            AGENTIC_LLM_MODEL: LLM model for agent reasoning (default: gpt-4o)
-            AGENTIC_MAX_ITERATIONS: Maximum agent iterations (default: 5)
-            AGENTIC_TIMEOUT_SECONDS: Agent execution timeout (default: 60)
+        All configuration options can be set via environment variables.
+        See .env.example for a complete list with descriptions.
 
         Returns:
             Configuration from environment.
         """
+        # Parse nodes list
         nodes_str = os.getenv("OPENSEARCH_NODES", "")
-        nodes = tuple(nodes_str.split(",")) if nodes_str else None
+        nodes = tuple(n.strip() for n in nodes_str.split(",") if n.strip()) or None
 
         return cls(
+            # === Connection ===
             host=os.getenv("OPENSEARCH_HOST", "localhost"),
             port=int(os.getenv("OPENSEARCH_PORT", "9200")),
-            use_ssl=os.getenv("OPENSEARCH_USE_SSL", "").lower() == "true",
+            use_ssl=os.getenv("OPENSEARCH_USE_SSL", "false").lower() == "true",
             verify_certs=os.getenv("OPENSEARCH_VERIFY_CERTS", "true").lower() == "true",
+            ca_certs=os.getenv("OPENSEARCH_CA_CERTS"),
+            # Authentication
             username=os.getenv("OPENSEARCH_USERNAME"),
             password=os.getenv("OPENSEARCH_PASSWORD"),
-            use_aws_sigv4=os.getenv("OPENSEARCH_USE_AWS_SIGV4", "").lower() == "true",
+            # AWS OpenSearch Service
+            use_aws_sigv4=os.getenv("OPENSEARCH_USE_AWS_SIGV4", "false").lower() == "true",
             aws_region=os.getenv("AWS_REGION"),
+            aws_service=os.getenv("OPENSEARCH_AWS_SERVICE", "es"),
+            # === Cluster (High Availability) ===
             nodes=nodes,
+            sniff_on_start=os.getenv("OPENSEARCH_SNIFF_ON_START", "false").lower() == "true",
+            sniff_on_node_failure=os.getenv("OPENSEARCH_SNIFF_ON_NODE_FAILURE", "true").lower()
+            == "true",
+            sniff_timeout=float(os.getenv("OPENSEARCH_SNIFF_TIMEOUT", "10.0")),
+            sniffer_timeout=float(os.getenv("OPENSEARCH_SNIFFER_TIMEOUT", "60.0")),
+            # === Embedding ===
             embedding_model=os.getenv("EMBEDDING_MODEL", "text-embedding-3-small"),
             embedding_dimension=int(os.getenv("EMBEDDING_DIMENSION", "1536")),
             openai_api_key=os.getenv("OPENAI_API_KEY"),
+            embedding_batch_size=int(os.getenv("EMBEDDING_BATCH_SIZE", "100")),
+            # === Index Settings ===
             index_prefix=os.getenv("OPENSEARCH_INDEX_PREFIX", "gnosisllm"),
             number_of_shards=int(os.getenv("OPENSEARCH_SHARDS", "5")),
             number_of_replicas=int(os.getenv("OPENSEARCH_REPLICAS", "1")),
-            model_id=os.getenv("OPENSEARCH_MODEL_ID"),
+            refresh_interval=os.getenv("OPENSEARCH_REFRESH_INTERVAL", "1s"),
+            # Pipeline names
             ingest_pipeline_name=os.getenv("OPENSEARCH_INGEST_PIPELINE"),
             search_pipeline_name=os.getenv("OPENSEARCH_SEARCH_PIPELINE"),
-            # Agentic search configuration
+            # === k-NN Settings ===
+            knn_engine=os.getenv("OPENSEARCH_KNN_ENGINE", "lucene"),
+            knn_space_type=os.getenv("OPENSEARCH_KNN_SPACE_TYPE", "cosinesimil"),
+            knn_algo_param_ef_search=int(os.getenv("OPENSEARCH_KNN_EF_SEARCH", "512")),
+            knn_algo_param_ef_construction=int(
+                os.getenv("OPENSEARCH_KNN_EF_CONSTRUCTION", "512")
+            ),
+            knn_algo_param_m=int(os.getenv("OPENSEARCH_KNN_M", "16")),
+            # === Neural Search ===
+            model_id=os.getenv("OPENSEARCH_MODEL_ID"),
+            model_group_id=os.getenv("OPENSEARCH_MODEL_GROUP_ID"),
+            embedding_field=os.getenv("OPENSEARCH_EMBEDDING_FIELD", "content_embedding"),
+            # === Agentic Search ===
             flow_agent_id=os.getenv("OPENSEARCH_FLOW_AGENT_ID"),
             conversational_agent_id=os.getenv("OPENSEARCH_CONVERSATIONAL_AGENT_ID"),
+            agentic_pipeline_name=os.getenv("OPENSEARCH_AGENTIC_PIPELINE"),
             agentic_llm_model=os.getenv("AGENTIC_LLM_MODEL", "gpt-4o"),
             agentic_max_iterations=int(os.getenv("AGENTIC_MAX_ITERATIONS", "5")),
             agentic_timeout_seconds=int(os.getenv("AGENTIC_TIMEOUT_SECONDS", "60")),
+            memory_window_size=int(os.getenv("AGENTIC_MEMORY_WINDOW_SIZE", "10")),
+            # === Timeouts ===
+            connect_timeout=float(os.getenv("OPENSEARCH_CONNECT_TIMEOUT", "5.0")),
+            read_timeout=float(os.getenv("OPENSEARCH_READ_TIMEOUT", "30.0")),
+            bulk_timeout=float(os.getenv("OPENSEARCH_BULK_TIMEOUT", "120.0")),
+            # === Bulk Indexing ===
+            bulk_batch_size=int(os.getenv("OPENSEARCH_BULK_BATCH_SIZE", "500")),
+            bulk_max_concurrent=int(os.getenv("OPENSEARCH_BULK_MAX_CONCURRENT", "3")),
         )

gnosisllm_knowledge/backends/opensearch/indexer.py

@@ -87,13 +87,15 @@ class OpenSearchIndexer:
             # Embeddings are generated by OpenSearch ingest pipeline
             doc_body = self._prepare_document(document)
 
-            # Index the document
+            # Index the document with ingest pipeline for embedding generation
             refresh = options.get("refresh", False)
+            pipeline = self._config.ingest_pipeline_name
             await self._client.index(
                 index=index_name,
                 id=document.doc_id,
                 body=doc_body,
                 refresh=refresh,
+                pipeline=pipeline,
             )
 
             return IndexResult(
@@ -272,6 +274,43 @@ class OpenSearchIndexer:
             failed_count=0,
         )
 
+    async def get(
+        self,
+        doc_id: str,
+        index_name: str,
+    ) -> dict[str, Any] | None:
+        """Get a document by ID.
+
+        Uses OpenSearch client's direct get() API (CRUD operation, not search).
+
+        Args:
+            doc_id: Document ID to retrieve.
+            index_name: Index name.
+
+        Returns:
+            Document dict (source fields) or None if not found.
+            Excludes embeddings from response for efficiency.
+        """
+        try:
+            response = await self._client.get(
+                index=index_name,
+                id=doc_id,
+                _source_excludes=["content_embedding"],
+            )
+            source = response.get("_source", {})
+            # Include the document ID in the response
+            source["id"] = response.get("_id", doc_id)
+            return source
+        except Exception as e:
+            if "not_found" in str(e).lower():
+                return None
+            logger.error(f"Failed to get document {doc_id}: {e}")
+            raise IndexError(
+                message=f"Failed to get document: {e}",
+                details={"document_id": doc_id},
+                cause=e,
+            ) from e
+
     async def delete(
         self,
         doc_id: str,
@@ -434,7 +473,9 @@ class OpenSearchIndexer:
         if not actions:
             return IndexResult(success=True, index_name=index_name, indexed_count=0, failed_count=0)
 
-        response = await self._client.bulk(body=actions)
+        # Use ingest pipeline for embedding generation
+        pipeline = self._config.ingest_pipeline_name
+        response = await self._client.bulk(body=actions, pipeline=pipeline)
 
         indexed = 0
         failed = 0
@@ -460,6 +501,11 @@ class OpenSearchIndexer:
     def _prepare_document(self, document: Document) -> dict[str, Any]:
         """Prepare document for indexing.
 
+        Note:
+            This library is tenant-agnostic. Multi-tenancy is achieved through index
+            isolation. Tenant information should be passed in document.metadata if
+            needed for audit purposes.
+
         Args:
             document: Document to prepare.
 
@@ -479,8 +525,8 @@ class OpenSearchIndexer:
             "url": document.url,
             "title": document.title,
             "source": document.source,
-            "account_id": document.account_id,
             "collection_id": document.collection_id,
+            "collection_name": document.collection_name,
             "source_id": document.source_id,
             "chunk_index": document.chunk_index,
             "total_chunks": document.total_chunks,

gnosisllm_knowledge/backends/opensearch/mappings.py

@@ -1,4 +1,10 @@
-"""OpenSearch index mappings for knowledge documents."""
+"""OpenSearch index mappings for knowledge documents.
+
+Note:
+    This library is tenant-agnostic. Multi-tenancy is achieved through index
+    isolation (e.g., `knowledge-{account_id}`). Index mappings do not include
+    tenant-specific fields like account_id.
+"""
 
 from __future__ import annotations
 
@@ -56,9 +62,9 @@ def get_knowledge_index_mappings(config: OpenSearchConfig) -> dict[str, Any]:
                 "fields": {"keyword": {"type": "keyword", "ignore_above": 512}},
             },
             "source": {"type": "keyword"},
-            # === Multi-tenant Fields ===
-            "account_id": {"type": "keyword"},
+            # === Collection Fields ===
             "collection_id": {"type": "keyword"},
+            "collection_name": {"type": "keyword"},  # For aggregation display
             "source_id": {"type": "keyword"},
             # === Content ===
             "content": {
@@ -128,13 +134,16 @@ def get_memory_index_settings(config: OpenSearchConfig) -> dict[str, Any]:
 def get_memory_index_mappings() -> dict[str, Any]:
     """Get index mappings for conversation memory.
 
+    Note:
+        This library is tenant-agnostic. Multi-tenancy is achieved through index
+        isolation. Use tenant-specific index names for conversation memory.
+
     Returns:
         Index mappings dictionary.
     """
     return {
         "properties": {
             "conversation_id": {"type": "keyword"},
-            "account_id": {"type": "keyword"},
             "user_id": {"type": "keyword"},
             "message_index": {"type": "integer"},
             "role": {"type": "keyword"},  # user, assistant, system
@@ -168,7 +177,7 @@ def get_index_template(
             "settings": get_knowledge_index_settings(config),
             "mappings": get_knowledge_index_mappings(config),
         },
-        "priority": 100,
+        "priority": 200,  # Higher than default gnosisllm-template (100)
         "version": 1,
     }
 

gnosisllm_knowledge/backends/opensearch/memory/__init__.py

@@ -0,0 +1,12 @@
+"""OpenSearch Agentic Memory backend."""
+
+from gnosisllm_knowledge.backends.opensearch.memory.client import OpenSearchMemoryClient
+from gnosisllm_knowledge.backends.opensearch.memory.config import MemoryConfig
+from gnosisllm_knowledge.backends.opensearch.memory.setup import MemorySetup, SetupStatus
+
+__all__ = [
+    "MemoryConfig",
+    "MemorySetup",
+    "OpenSearchMemoryClient",
+    "SetupStatus",
+]