gnosisllm-knowledge 0.4.0__tar.gz → 0.4.4__tar.gz

{gnosisllm_knowledge-0.4.0 → gnosisllm_knowledge-0.4.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gnosisllm-knowledge
-Version: 0.4.0
+Version: 0.4.4
 Summary: Enterprise-grade knowledge loading, indexing, and search for Python
 License: MIT
 Keywords: knowledge-base,rag,semantic-search,vector-search,opensearch,llm,embeddings,enterprise

{gnosisllm_knowledge-0.4.0 → gnosisllm_knowledge-0.4.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "gnosisllm-knowledge"
-version = "0.4.0"
+version = "0.4.4"
 description = "Enterprise-grade knowledge loading, indexing, and search for Python"
 authors = [
     {name = "David Marsa", email = "david.marsa@neomanex.com"},

{gnosisllm_knowledge-0.4.0 → gnosisllm_knowledge-0.4.4}/src/gnosisllm_knowledge/api/knowledge.py

@@ -119,6 +119,7 @@ class Knowledge:
         loader_factory: LoaderFactory | None = None,
         default_index: str | None = None,
         events: EventEmitter | None = None,
+        config: OpenSearchConfig | None = None,
     ) -> None:
         """Initialize Knowledge with components.
 
@@ -131,6 +132,7 @@ class Knowledge:
             loader_factory: Optional loader factory.
             default_index: Default index name.
             events: Optional event emitter.
+            config: Optional OpenSearch config for settings like batch sizes.
 
         Note:
             Embeddings are generated automatically by OpenSearch ingest pipeline.
@@ -144,6 +146,7 @@ class Knowledge:
         self._loader_factory = loader_factory
         self._default_index = default_index
         self._events = events or EventEmitter()
+        self._config = config
 
         # Initialize services lazily
         self._indexing_service: KnowledgeIndexingService | None = None
@@ -262,6 +265,7 @@ class Knowledge:
             fetcher=fetcher,
             loader_factory=loader_factory,
             default_index=config.knowledge_index_name,
+            config=config,
         )
 
     @classmethod
@@ -431,11 +435,15 @@ class Knowledge:
             events=self._events,
         )
 
+        # Get batch size from config or use default
+        batch_size = self._config.indexing_batch_size if self._config else 10
+
         return await service.load_and_index(
             source=source,
             index_name=index,
             collection_id=collection_id,
             source_id=source_id,
+            batch_size=batch_size,
             **options,
         )
 

{gnosisllm_knowledge-0.4.0 → gnosisllm_knowledge-0.4.4}/src/gnosisllm_knowledge/backends/memory/searcher.py

@@ -395,16 +395,37 @@ class MemorySearcher:
             "offset": offset,
         }
 
-    def count(self, index_name: str) -> int:
-        """Count documents in index.
+    async def count(
+        self,
+        index_name: str,
+        collection_id: str | None = None,
+        source_id: str | None = None,
+    ) -> int:
+        """Count documents in index with optional filters.
 
         Args:
             index_name: Index to count.
+            collection_id: Filter by collection.
+            source_id: Filter by source.
 
         Returns:
             Document count.
         """
-        return self._indexer.count(index_name)
+        # Use efficient O(1) count when no filters
+        if not collection_id and not source_id:
+            return self._indexer.count(index_name)
+
+        # With filters, iterate over index values (memory backend is for testing only)
+        index_data = self._indexer._indices.get(index_name, {})
+        count = 0
+        for doc in index_data.values():
+            if collection_id and doc.get("collection_id") != collection_id:
+                continue
+            if source_id and doc.get("source_id") != source_id:
+                continue
+            count += 1
+
+        return count
 
     async def get_collections(self, index_name: str) -> list[dict[str, Any]]:
         """Get unique collections with document counts.

{gnosisllm_knowledge-0.4.0 → gnosisllm_knowledge-0.4.4}/src/gnosisllm_knowledge/backends/opensearch/config.py

@@ -109,6 +109,11 @@ class OpenSearchConfig:
     bulk_batch_size: int = 500
     bulk_max_concurrent: int = 3
 
+    # === Indexing Service ===
+    # Batch size for progressive indexing during load operations
+    # Documents are indexed in batches of this size as they stream in
+    indexing_batch_size: int = 10
+
     @property
     def url(self) -> str:
         """Get the full OpenSearch URL."""
@@ -213,4 +218,6 @@ class OpenSearchConfig:
             # === Bulk Indexing ===
             bulk_batch_size=int(os.getenv("OPENSEARCH_BULK_BATCH_SIZE", "500")),
             bulk_max_concurrent=int(os.getenv("OPENSEARCH_BULK_MAX_CONCURRENT", "3")),
+            # === Indexing Service ===
+            indexing_batch_size=int(os.getenv("GNOSISLLM_INDEXING_BATCH_SIZE", "10")),
         )

{gnosisllm_knowledge-0.4.0 → gnosisllm_knowledge-0.4.4}/src/gnosisllm_knowledge/backends/opensearch/searcher.py

@@ -236,12 +236,14 @@ class OpenSearchKnowledgeSearcher:
         self,
         queries: list[SearchQuery],
         index_name: str,
+        **options: Any,
     ) -> list[SearchResult]:
         """Execute multiple searches in a single request.
 
         Args:
             queries: List of search queries.
             index_name: Index to search.
+            **options: Additional options (text_field, embedding_field, model_id).
 
         Returns:
             List of search results.
@@ -252,11 +254,15 @@ class OpenSearchKnowledgeSearcher:
         start_time = time.perf_counter()
         msearch_body = []
 
+        # Allow overriding embedding field and model_id via options
+        embedding_field = options.get("embedding_field", self._config.embedding_field)
+        model_id = options.get("model_id", self._config.model_id)
+
         for query in queries:
             builder = QueryBuilder(
                 query,
-                model_id=self._config.model_id,
-                embedding_field=self._config.embedding_field,
+                model_id=model_id,
+                embedding_field=embedding_field,
             )
 
             if query.mode == SearchMode.SEMANTIC:
@@ -506,6 +512,61 @@ class OpenSearchKnowledgeSearcher:
                 "error": str(e),
             }
 
+    async def count(
+        self,
+        index_name: str,
+        collection_id: str | None = None,
+        source_id: str | None = None,
+    ) -> int:
+        """Count documents in index with optional filters.
+
+        Uses native _count API instead of search for efficiency and to avoid
+        hybrid search issues with empty queries.
+
+        Args:
+            index_name: Index to query.
+            collection_id: Filter by collection.
+            source_id: Filter by source.
+
+        Returns:
+            Document count.
+        """
+        try:
+            # Check if index exists first
+            exists = await self._client.indices.exists(index=index_name)
+            if not exists:
+                logger.debug(f"Index {index_name} does not exist, returning count 0")
+                return 0
+
+            # Build query with optional filters
+            query: dict[str, Any] = {"match_all": {}}
+
+            filters = []
+            if collection_id:
+                filters.append({"term": {"collection_id": collection_id}})
+            if source_id:
+                filters.append({"term": {"source_id": source_id}})
+
+            if filters:
+                query = {"bool": {"filter": filters}}
+
+            # Use native _count API
+            response = await self._client.count(
+                index=index_name,
+                body={"query": query},
+            )
+
+            count = response.get("count", 0)
+            logger.debug(f"Count for {index_name}: {count} (collection={collection_id}, source={source_id})")
+            return count
+
+        except Exception as e:
+            logger.error(f"Failed to count documents in {index_name}: {e}")
+            raise SearchError(
+                message=f"Count failed: {e}",
+                details={"index": index_name, "collection_id": collection_id, "source_id": source_id},
+            ) from e
+
     async def list_documents(
         self,
         index_name: str,

{gnosisllm_knowledge-0.4.0 → gnosisllm_knowledge-0.4.4}/src/gnosisllm_knowledge/backends/opensearch/setup.py

@@ -249,10 +249,9 @@ class OpenSearchSetupAdapter:
             self._model_id = self._config.model_id
 
         # Step 4: Create ingest pipeline
-        # Only create ingest pipeline for global setup (not per-account)
-        # Account indices should use the global pipeline to ensure consistent model
-        is_global_setup = self._config.index_prefix == "gnosisllm"
-        if self._model_id and is_global_setup:
+        # Create pipeline for any setup that has a model deployed
+        # Each index_prefix namespace gets its own pipeline
+        if self._model_id:
             try:
                 await self._create_ingest_pipeline()
                 pipeline_name = self._config.ingest_pipeline_name or f"{self._config.index_prefix}-ingest-pipeline"
@@ -261,35 +260,33 @@ class OpenSearchSetupAdapter:
                 errors.append(f"Failed to create ingest pipeline: {e}")
                 logger.error(f"Failed to create ingest pipeline: {e}")
 
-        # Step 5: Create search pipeline (only for global setup)
-        if is_global_setup:
-            try:
-                await self._create_search_pipeline()
-                pipeline_name = self._config.search_pipeline_name or f"{self._config.index_prefix}-search-pipeline"
-                steps_completed.append(f"Created search pipeline: {pipeline_name}")
-            except Exception as e:
-                errors.append(f"Failed to create search pipeline: {e}")
-                logger.error(f"Failed to create search pipeline: {e}")
+        # Step 5: Create search pipeline for hybrid search
+        try:
+            await self._create_search_pipeline()
+            pipeline_name = self._config.search_pipeline_name or f"{self._config.index_prefix}-search-pipeline"
+            steps_completed.append(f"Created search pipeline: {pipeline_name}")
+        except Exception as e:
+            errors.append(f"Failed to create search pipeline: {e}")
+            logger.error(f"Failed to create search pipeline: {e}")
 
-        # Step 6: Create index template (only for global setup)
-        # Template covers all gnosisllm-* indices including per-account indices
-        if is_global_setup:
-            try:
-                template_name = f"{self._config.index_prefix}-template"
-                template_body = get_index_template(self._config)
+        # Step 6: Create index template for this namespace
+        # Template covers all {index_prefix}-* indices
+        try:
+            template_name = f"{self._config.index_prefix}-template"
+            template_body = get_index_template(self._config)
 
-                # Ensure template has global pipeline for auto-index creation
-                global_pipeline = self._config.ingest_pipeline_name or "gnosisllm-ingest-pipeline"
-                template_body["template"]["settings"]["index"]["default_pipeline"] = global_pipeline
+            # Set default pipeline for auto-index creation within this namespace
+            default_pipeline = self._config.ingest_pipeline_name or f"{self._config.index_prefix}-ingest-pipeline"
+            template_body["template"]["settings"]["index"]["default_pipeline"] = default_pipeline
 
-                await self._client.indices.put_index_template(
-                    name=template_name,
-                    body=template_body,
-                )
-                steps_completed.append(f"Created index template: {template_name}")
-            except Exception as e:
-                errors.append(f"Failed to create index template: {e}")
-                logger.error(f"Failed to create index template: {e}")
+            await self._client.indices.put_index_template(
+                name=template_name,
+                body=template_body,
+            )
+            steps_completed.append(f"Created index template: {template_name}")
+        except Exception as e:
+            errors.append(f"Failed to create index template: {e}")
+            logger.error(f"Failed to create index template: {e}")
 
         # Step 7: Create knowledge index
         try:
@@ -298,9 +295,8 @@ class OpenSearchSetupAdapter:
 
             if not exists:
                 settings = get_knowledge_index_settings(self._config)
-                # Add default pipeline - always use global pipeline for consistency
-                # This ensures all accounts use the same embedding model
-                pipeline_name = self._config.ingest_pipeline_name or "gnosisllm-ingest-pipeline"
+                # Add default pipeline for this namespace
+                pipeline_name = self._config.ingest_pipeline_name or f"{self._config.index_prefix}-ingest-pipeline"
                 settings["index"]["default_pipeline"] = pipeline_name
 
                 await self._client.indices.create(

{gnosisllm_knowledge-0.4.0 → gnosisllm_knowledge-0.4.4}/src/gnosisllm_knowledge/core/interfaces/searcher.py

@@ -185,3 +185,23 @@ class IKnowledgeSearcher(Protocol):
             List of SearchResults in same order as queries.
         """
         ...
+
+    async def count(
+        self,
+        index_name: str,
+        collection_id: str | None = None,
+        source_id: str | None = None,
+    ) -> int:
+        """Count documents in index with optional filters.
+
+        Uses native count API instead of search for efficiency.
+
+        Args:
+            index_name: Target index name.
+            collection_id: Filter by collection.
+            source_id: Filter by source.
+
+        Returns:
+            Document count.
+        """
+        ...

{gnosisllm_knowledge-0.4.0 → gnosisllm_knowledge-0.4.4}/src/gnosisllm_knowledge/services/indexing.py

@@ -119,6 +119,16 @@ class KnowledgeIndexingService:
         source_id = source_id or str(uuid.uuid4())
         document_defaults = options.pop("document_defaults", {})
 
+        # Extract metadata from document_defaults to merge with doc.metadata later
+        # This allows callers to pass custom metadata (e.g., parent_collection_id)
+        # without conflicting with the explicit metadata= parameter
+        extra_metadata = document_defaults.pop("metadata", {})
+
+        # Ensure index exists with correct mappings before indexing
+        # This prevents OpenSearch from auto-creating the index with dynamic mapping
+        # which would map keyword fields (like collection_id) as text fields
+        await self._indexer.ensure_index(index_name)
+
         # Emit batch started event
         await self._events.emit_async(
             BatchStartedEvent(
@@ -139,6 +149,8 @@ class KnowledgeIndexingService:
             # Note: Loader already chunks content, so we don't re-chunk here
             async for doc in self._loader.load_streaming(source, **options):
                 # Enrich document with collection info
+                # Merge doc.metadata with extra_metadata from document_defaults
+                merged_metadata = {**doc.metadata, **extra_metadata}
                 enriched_doc = Document(
                     content=doc.content,
                     source=source,
@@ -151,7 +163,7 @@ class KnowledgeIndexingService:
                     total_chunks=doc.total_chunks,
                     parent_doc_id=doc.parent_doc_id,
                     status=DocumentStatus.INDEXED,
-                    metadata=doc.metadata,
+                    metadata=merged_metadata,
                     **document_defaults,
                 )
 
@@ -223,6 +235,9 @@ class KnowledgeIndexingService:
         Returns:
             Index result.
         """
+        # Ensure index exists with correct mappings before indexing
+        await self._indexer.ensure_index(index_name)
+
         total_indexed = 0
         total_failed = 0
         errors: list[str] = []

{gnosisllm_knowledge-0.4.0 → gnosisllm_knowledge-0.4.4}/src/gnosisllm_knowledge/services/search.py

@@ -328,6 +328,9 @@ class KnowledgeSearchService:
     ) -> int:
         """Count documents in index.
 
+        Uses native count API instead of search for efficiency and to avoid
+        hybrid search issues with empty queries.
+
         Note:
             This method is tenant-agnostic. Multi-tenancy should be handled
             at the API layer by using separate indices per account.
@@ -344,18 +347,12 @@ class KnowledgeSearchService:
         if not index:
             raise SearchError(message="No index specified")
 
-        # Build count query with optional filters
-        query = SearchQuery(
-            text="",
-            limit=0,
-            collection_ids=[collection_id] if collection_id else None,
-            source_ids=[source_id] if source_id else None,
+        return await self._searcher.count(
+            index_name=index,
+            collection_id=collection_id,
+            source_id=source_id,
         )
 
-        # Use a simple match_all to get total count
-        result = await self._searcher.search(query, index)
-        return result.total_hits
-
     async def get_collections(
         self,
         index_name: str | None = None,

{gnosisllm_knowledge-0.4.0 → gnosisllm_knowledge-0.4.4}/src/gnosisllm_knowledge/services/streaming_pipeline.py

@@ -173,6 +173,12 @@ class StreamingIndexingPipeline:
                 DeprecationWarning,
                 stacklevel=2,
             )
+
+        # Ensure index exists with correct mappings before indexing
+        # This prevents OpenSearch from auto-creating the index with dynamic mapping
+        # which would map keyword fields (like collection_id) as text fields
+        await self._indexer.ensure_index(index_name)
+
         start_time = time.time()
         self._progress = StreamingProgress(current_phase="starting")
         await self._emit_progress()