gnosisllm-knowledge 0.4.0__py3-none-any.whl → 0.4.3__py3-none-any.whl

gnosisllm_knowledge/api/knowledge.py

@@ -119,6 +119,7 @@ class Knowledge:
         loader_factory: LoaderFactory | None = None,
         default_index: str | None = None,
         events: EventEmitter | None = None,
+        config: OpenSearchConfig | None = None,
     ) -> None:
         """Initialize Knowledge with components.
 
@@ -131,6 +132,7 @@ class Knowledge:
             loader_factory: Optional loader factory.
             default_index: Default index name.
             events: Optional event emitter.
+            config: Optional OpenSearch config for settings like batch sizes.
 
         Note:
             Embeddings are generated automatically by OpenSearch ingest pipeline.
@@ -144,6 +146,7 @@ class Knowledge:
         self._loader_factory = loader_factory
         self._default_index = default_index
         self._events = events or EventEmitter()
+        self._config = config
 
         # Initialize services lazily
         self._indexing_service: KnowledgeIndexingService | None = None
@@ -262,6 +265,7 @@ class Knowledge:
             fetcher=fetcher,
             loader_factory=loader_factory,
             default_index=config.knowledge_index_name,
+            config=config,
         )
 
     @classmethod
@@ -431,11 +435,15 @@ class Knowledge:
             events=self._events,
         )
 
+        # Get batch size from config or use default
+        batch_size = self._config.indexing_batch_size if self._config else 10
+
         return await service.load_and_index(
             source=source,
             index_name=index,
             collection_id=collection_id,
             source_id=source_id,
+            batch_size=batch_size,
             **options,
         )
 

gnosisllm_knowledge/backends/memory/searcher.py

@@ -395,16 +395,37 @@ class MemorySearcher:
             "offset": offset,
         }
 
-    def count(self, index_name: str) -> int:
-        """Count documents in index.
+    async def count(
+        self,
+        index_name: str,
+        collection_id: str | None = None,
+        source_id: str | None = None,
+    ) -> int:
+        """Count documents in index with optional filters.
 
         Args:
             index_name: Index to count.
+            collection_id: Filter by collection.
+            source_id: Filter by source.
 
         Returns:
             Document count.
         """
-        return self._indexer.count(index_name)
+        # Use efficient O(1) count when no filters
+        if not collection_id and not source_id:
+            return self._indexer.count(index_name)
+
+        # With filters, iterate over index values (memory backend is for testing only)
+        index_data = self._indexer._indices.get(index_name, {})
+        count = 0
+        for doc in index_data.values():
+            if collection_id and doc.get("collection_id") != collection_id:
+                continue
+            if source_id and doc.get("source_id") != source_id:
+                continue
+            count += 1
+
+        return count
 
     async def get_collections(self, index_name: str) -> list[dict[str, Any]]:
         """Get unique collections with document counts.

gnosisllm_knowledge/backends/opensearch/config.py

@@ -109,6 +109,11 @@ class OpenSearchConfig:
     bulk_batch_size: int = 500
     bulk_max_concurrent: int = 3
 
+    # === Indexing Service ===
+    # Batch size for progressive indexing during load operations
+    # Documents are indexed in batches of this size as they stream in
+    indexing_batch_size: int = 10
+
     @property
     def url(self) -> str:
         """Get the full OpenSearch URL."""
@@ -213,4 +218,6 @@ class OpenSearchConfig:
             # === Bulk Indexing ===
             bulk_batch_size=int(os.getenv("OPENSEARCH_BULK_BATCH_SIZE", "500")),
             bulk_max_concurrent=int(os.getenv("OPENSEARCH_BULK_MAX_CONCURRENT", "3")),
+            # === Indexing Service ===
+            indexing_batch_size=int(os.getenv("GNOSISLLM_INDEXING_BATCH_SIZE", "10")),
         )

gnosisllm_knowledge/backends/opensearch/searcher.py

@@ -506,6 +506,61 @@ class OpenSearchKnowledgeSearcher:
                 "error": str(e),
             }
 
+    async def count(
+        self,
+        index_name: str,
+        collection_id: str | None = None,
+        source_id: str | None = None,
+    ) -> int:
+        """Count documents in index with optional filters.
+
+        Uses native _count API instead of search for efficiency and to avoid
+        hybrid search issues with empty queries.
+
+        Args:
+            index_name: Index to query.
+            collection_id: Filter by collection.
+            source_id: Filter by source.
+
+        Returns:
+            Document count.
+        """
+        try:
+            # Check if index exists first
+            exists = await self._client.indices.exists(index=index_name)
+            if not exists:
+                logger.debug(f"Index {index_name} does not exist, returning count 0")
+                return 0
+
+            # Build query with optional filters
+            query: dict[str, Any] = {"match_all": {}}
+
+            filters = []
+            if collection_id:
+                filters.append({"term": {"collection_id": collection_id}})
+            if source_id:
+                filters.append({"term": {"source_id": source_id}})
+
+            if filters:
+                query = {"bool": {"filter": filters}}
+
+            # Use native _count API
+            response = await self._client.count(
+                index=index_name,
+                body={"query": query},
+            )
+
+            count = response.get("count", 0)
+            logger.debug(f"Count for {index_name}: {count} (collection={collection_id}, source={source_id})")
+            return count
+
+        except Exception as e:
+            logger.error(f"Failed to count documents in {index_name}: {e}")
+            raise SearchError(
+                message=f"Count failed: {e}",
+                details={"index": index_name, "collection_id": collection_id, "source_id": source_id},
+            ) from e
+
     async def list_documents(
         self,
         index_name: str,

gnosisllm_knowledge/backends/opensearch/setup.py

@@ -249,10 +249,9 @@ class OpenSearchSetupAdapter:
             self._model_id = self._config.model_id
 
         # Step 4: Create ingest pipeline
-        # Only create ingest pipeline for global setup (not per-account)
-        # Account indices should use the global pipeline to ensure consistent model
-        is_global_setup = self._config.index_prefix == "gnosisllm"
-        if self._model_id and is_global_setup:
+        # Create pipeline for any setup that has a model deployed
+        # Each index_prefix namespace gets its own pipeline
+        if self._model_id:
             try:
                 await self._create_ingest_pipeline()
                 pipeline_name = self._config.ingest_pipeline_name or f"{self._config.index_prefix}-ingest-pipeline"
@@ -261,35 +260,33 @@ class OpenSearchSetupAdapter:
                 errors.append(f"Failed to create ingest pipeline: {e}")
                 logger.error(f"Failed to create ingest pipeline: {e}")
 
-        # Step 5: Create search pipeline (only for global setup)
-        if is_global_setup:
-            try:
-                await self._create_search_pipeline()
-                pipeline_name = self._config.search_pipeline_name or f"{self._config.index_prefix}-search-pipeline"
-                steps_completed.append(f"Created search pipeline: {pipeline_name}")
-            except Exception as e:
-                errors.append(f"Failed to create search pipeline: {e}")
-                logger.error(f"Failed to create search pipeline: {e}")
+        # Step 5: Create search pipeline for hybrid search
+        try:
+            await self._create_search_pipeline()
+            pipeline_name = self._config.search_pipeline_name or f"{self._config.index_prefix}-search-pipeline"
+            steps_completed.append(f"Created search pipeline: {pipeline_name}")
+        except Exception as e:
+            errors.append(f"Failed to create search pipeline: {e}")
+            logger.error(f"Failed to create search pipeline: {e}")
 
-        # Step 6: Create index template (only for global setup)
-        # Template covers all gnosisllm-* indices including per-account indices
-        if is_global_setup:
-            try:
-                template_name = f"{self._config.index_prefix}-template"
-                template_body = get_index_template(self._config)
+        # Step 6: Create index template for this namespace
+        # Template covers all {index_prefix}-* indices
+        try:
+            template_name = f"{self._config.index_prefix}-template"
+            template_body = get_index_template(self._config)
 
-                # Ensure template has global pipeline for auto-index creation
-                global_pipeline = self._config.ingest_pipeline_name or "gnosisllm-ingest-pipeline"
-                template_body["template"]["settings"]["index"]["default_pipeline"] = global_pipeline
+            # Set default pipeline for auto-index creation within this namespace
+            default_pipeline = self._config.ingest_pipeline_name or f"{self._config.index_prefix}-ingest-pipeline"
+            template_body["template"]["settings"]["index"]["default_pipeline"] = default_pipeline
 
-                await self._client.indices.put_index_template(
-                    name=template_name,
-                    body=template_body,
-                )
-                steps_completed.append(f"Created index template: {template_name}")
-            except Exception as e:
-                errors.append(f"Failed to create index template: {e}")
-                logger.error(f"Failed to create index template: {e}")
+            await self._client.indices.put_index_template(
+                name=template_name,
+                body=template_body,
+            )
+            steps_completed.append(f"Created index template: {template_name}")
+        except Exception as e:
+            errors.append(f"Failed to create index template: {e}")
+            logger.error(f"Failed to create index template: {e}")
 
         # Step 7: Create knowledge index
         try:
@@ -298,9 +295,8 @@ class OpenSearchSetupAdapter:
 
             if not exists:
                 settings = get_knowledge_index_settings(self._config)
-                # Add default pipeline - always use global pipeline for consistency
-                # This ensures all accounts use the same embedding model
-                pipeline_name = self._config.ingest_pipeline_name or "gnosisllm-ingest-pipeline"
+                # Add default pipeline for this namespace
+                pipeline_name = self._config.ingest_pipeline_name or f"{self._config.index_prefix}-ingest-pipeline"
                 settings["index"]["default_pipeline"] = pipeline_name
 
                 await self._client.indices.create(

gnosisllm_knowledge/core/interfaces/searcher.py

@@ -185,3 +185,23 @@ class IKnowledgeSearcher(Protocol):
             List of SearchResults in same order as queries.
         """
         ...
+
+    async def count(
+        self,
+        index_name: str,
+        collection_id: str | None = None,
+        source_id: str | None = None,
+    ) -> int:
+        """Count documents in index with optional filters.
+
+        Uses native count API instead of search for efficiency.
+
+        Args:
+            index_name: Target index name.
+            collection_id: Filter by collection.
+            source_id: Filter by source.
+
+        Returns:
+            Document count.
+        """
+        ...

gnosisllm_knowledge/services/indexing.py

@@ -119,6 +119,16 @@ class KnowledgeIndexingService:
         source_id = source_id or str(uuid.uuid4())
         document_defaults = options.pop("document_defaults", {})
 
+        # Extract metadata from document_defaults to merge with doc.metadata later
+        # This allows callers to pass custom metadata (e.g., parent_collection_id)
+        # without conflicting with the explicit metadata= parameter
+        extra_metadata = document_defaults.pop("metadata", {})
+
+        # Ensure index exists with correct mappings before indexing
+        # This prevents OpenSearch from auto-creating the index with dynamic mapping
+        # which would map keyword fields (like collection_id) as text fields
+        await self._indexer.ensure_index(index_name)
+
         # Emit batch started event
         await self._events.emit_async(
             BatchStartedEvent(
@@ -139,6 +149,8 @@ class KnowledgeIndexingService:
             # Note: Loader already chunks content, so we don't re-chunk here
             async for doc in self._loader.load_streaming(source, **options):
                 # Enrich document with collection info
+                # Merge doc.metadata with extra_metadata from document_defaults
+                merged_metadata = {**doc.metadata, **extra_metadata}
                 enriched_doc = Document(
                     content=doc.content,
                     source=source,
@@ -151,7 +163,7 @@ class KnowledgeIndexingService:
                     total_chunks=doc.total_chunks,
                     parent_doc_id=doc.parent_doc_id,
                     status=DocumentStatus.INDEXED,
-                    metadata=doc.metadata,
+                    metadata=merged_metadata,
                     **document_defaults,
                 )
 
@@ -223,6 +235,9 @@ class KnowledgeIndexingService:
         Returns:
             Index result.
         """
+        # Ensure index exists with correct mappings before indexing
+        await self._indexer.ensure_index(index_name)
+
         total_indexed = 0
         total_failed = 0
         errors: list[str] = []

gnosisllm_knowledge/services/search.py

@@ -328,6 +328,9 @@ class KnowledgeSearchService:
     ) -> int:
         """Count documents in index.
 
+        Uses native count API instead of search for efficiency and to avoid
+        hybrid search issues with empty queries.
+
         Note:
             This method is tenant-agnostic. Multi-tenancy should be handled
             at the API layer by using separate indices per account.
@@ -344,18 +347,12 @@ class KnowledgeSearchService:
         if not index:
             raise SearchError(message="No index specified")
 
-        # Build count query with optional filters
-        query = SearchQuery(
-            text="",
-            limit=0,
-            collection_ids=[collection_id] if collection_id else None,
-            source_ids=[source_id] if source_id else None,
+        return await self._searcher.count(
+            index_name=index,
+            collection_id=collection_id,
+            source_id=source_id,
         )
 
-        # Use a simple match_all to get total count
-        result = await self._searcher.search(query, index)
-        return result.total_hits
-
     async def get_collections(
         self,
         index_name: str | None = None,

gnosisllm_knowledge/services/streaming_pipeline.py

@@ -173,6 +173,12 @@ class StreamingIndexingPipeline:
                 DeprecationWarning,
                 stacklevel=2,
             )
+
+        # Ensure index exists with correct mappings before indexing
+        # This prevents OpenSearch from auto-creating the index with dynamic mapping
+        # which would map keyword fields (like collection_id) as text fields
+        await self._indexer.ensure_index(index_name)
+
         start_time = time.time()
         self._progress = StreamingProgress(current_phase="starting")
         await self._emit_progress()

{gnosisllm_knowledge-0.4.0.dist-info → gnosisllm_knowledge-0.4.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gnosisllm-knowledge
-Version: 0.4.0
+Version: 0.4.3
 Summary: Enterprise-grade knowledge loading, indexing, and search for Python
 License: MIT
 Keywords: knowledge-base,rag,semantic-search,vector-search,opensearch,llm,embeddings,enterprise

{gnosisllm_knowledge-0.4.0.dist-info → gnosisllm_knowledge-0.4.3.dist-info}/RECORD

@@ -1,14 +1,14 @@
 gnosisllm_knowledge/__init__.py,sha256=Egfn6ROa9OttWweCBVocNCUivKwWfdeFEFgpwAoo_ww,4809
 gnosisllm_knowledge/api/__init__.py,sha256=xVG6wMjbFa7_qDVRKlp43O4f0EQmPr9w3pAOGmmh32U,200
-gnosisllm_knowledge/api/knowledge.py,sha256=sjBXBe__wRdGoawmLfDKw9W2o8efOTy0aDVPbt4SMyQ,33718
+gnosisllm_knowledge/api/knowledge.py,sha256=Hb6kTrZDUwHC1OCmOWLHcAhIHfF96cpJpmYSkK7prv4,34067
 gnosisllm_knowledge/api/memory.py,sha256=BrYWS3fCPaymwjzlnLqAKiCrUuMlUDMod-DcrTaarkQ,28926
 gnosisllm_knowledge/backends/__init__.py,sha256=QkVcZbr-u_vDyGiOMs2jrV1oA21VyCLd6g6aJZcSJxk,927
 gnosisllm_knowledge/backends/memory/__init__.py,sha256=Q_2Fh0-Xs1f7CsNn9V7z8NLW0P7F1owmxZQ4k3fTSXo,238
 gnosisllm_knowledge/backends/memory/indexer.py,sha256=4BotG-jveyEFjw7F8K-Co-to9NlydvyDA67JncRtD_E,11284
-gnosisllm_knowledge/backends/memory/searcher.py,sha256=wCnt_ssfA4YxASjSQ7QnXidTqb0Fk0cYYoqKEuiZBMQ,18591
+gnosisllm_knowledge/backends/memory/searcher.py,sha256=ITdLIxdgw8V789zVQvkBJfRghNsGU9Vz1vnqhRkFzE0,19360
 gnosisllm_knowledge/backends/opensearch/__init__.py,sha256=wwg3QPVVMLkq5qHUGw_9m6gvXO3hxa-xaBj3_BRJlN4,679
 gnosisllm_knowledge/backends/opensearch/agentic.py,sha256=VT9ubsgNacgHEIibKtC1mOEAFKV8VjZmf1NhH_exMvk,36861
-gnosisllm_knowledge/backends/opensearch/config.py,sha256=9xHMvr5-vnCcGFOPoO2_bbKuB70Gm979BqrhvcKsKm0,8640
+gnosisllm_knowledge/backends/opensearch/config.py,sha256=rA5uojJwlkDZJ-x9-HlO7nm_3SI_zcZ6IJZKQvDS_is,8967
 gnosisllm_knowledge/backends/opensearch/indexer.py,sha256=0KGagbZ6lMXat9DMSDQgBM-hckNZtDn_eStSlUxQpGo,16803
 gnosisllm_knowledge/backends/opensearch/mappings.py,sha256=zyUBzSPanGEDMUCDs98X_4uwLqLfRikLIO8ut03oc-w,8279
 gnosisllm_knowledge/backends/opensearch/memory/__init__.py,sha256=9EganuADfP6-1lky1Mk_oogr2d9pCmOFcNUjO7UMVZo,403
@@ -16,8 +16,8 @@ gnosisllm_knowledge/backends/opensearch/memory/client.py,sha256=ho0Sn1lXK7EDy0RR
 gnosisllm_knowledge/backends/opensearch/memory/config.py,sha256=astoJqofwhl4KUZ6vTUG-yiG1h5hfpJjYdsBzhKzx_E,4633
 gnosisllm_knowledge/backends/opensearch/memory/setup.py,sha256=0ILimioesxx2GLQ4I3uCA-Nu9Wy09PI-rSA85t8ssLw,10372
 gnosisllm_knowledge/backends/opensearch/queries.py,sha256=cubU6fUCeSA6Ag6uDCgrF_pZAj2BgVVmAot2t5sgbxY,13590
-gnosisllm_knowledge/backends/opensearch/searcher.py,sha256=3WvvIjjrEr4iZbDoBGdEGsgARJagHEe2CDfJak0_Gbs,20192
-gnosisllm_knowledge/backends/opensearch/setup.py,sha256=G9G0KZkz_ZynJjRIRWJImLDEFvQQE2wWRJfq_mlfpIs,59483
+gnosisllm_knowledge/backends/opensearch/searcher.py,sha256=c0lYiyi4g_9Y0MbrY_2OgHi1C5u2M97CwG1YWWb8o9g,22071
+gnosisllm_knowledge/backends/opensearch/setup.py,sha256=7WcykXNfPVI1yidRBf2ZVnshNBrvDAXfAgLoDH7n_6Q,59137
 gnosisllm_knowledge/chunking/__init__.py,sha256=XKrk04DqwhhPxxP_SLGnQrVUo2fexmvzuEN9YzO4Jxk,225
 gnosisllm_knowledge/chunking/fixed.py,sha256=LXZhRbYbtg4hQzqrAKyGrM_zmDhqJWXDi6cHOiJlGrM,3905
 gnosisllm_knowledge/chunking/sentence.py,sha256=EqY2Y1dpfNK_qukXQC9QYrKN8b1dqyyX9TZN5bUYGqs,7271
@@ -52,7 +52,7 @@ gnosisllm_knowledge/core/interfaces/fetcher.py,sha256=19u2TKE6rDjX-8l7xIRKUqa6DL
 gnosisllm_knowledge/core/interfaces/indexer.py,sha256=PMm2cPJhkVKuNcyLA35S9yYBIk-bBGI2KlQiEv6bKcc,6546
 gnosisllm_knowledge/core/interfaces/loader.py,sha256=NKq5Yj1Pmbv1I8AUMDtleUCZO5URZFw6kCdx6uLG7Bk,2903
 gnosisllm_knowledge/core/interfaces/memory.py,sha256=xrgPsqSfmj1-Snmyd35b-HOKipI_9N_FWRQklHj6x8I,13057
-gnosisllm_knowledge/core/interfaces/searcher.py,sha256=iuFZ5moiMCRhkWty8171BlpP7Jf-NjYqeArenD-L_4E,5015
+gnosisllm_knowledge/core/interfaces/searcher.py,sha256=5Itba4fHKodLV1eGTL74pYC-9bXwo2-a6-0w6o-CFPc,5512
 gnosisllm_knowledge/core/interfaces/setup.py,sha256=9DMKVqoNUrWKbS58QCnfwlGk4sD9R5iGe-WLwlW7z-Q,4457
 gnosisllm_knowledge/core/interfaces/streaming.py,sha256=sQuWTch8gMandFG1AY4xUVbVC-gwijzktq_Pj5QexWc,4160
 gnosisllm_knowledge/core/streaming/__init__.py,sha256=-PKIWbSu9bI3X3H8bucjuoqBGBiH0UOvM9OSn9dh1Rs,814
@@ -72,10 +72,10 @@ gnosisllm_knowledge/loaders/sitemap_streaming.py,sha256=KkkIg_oJxJrjQP88kS_jvqQv
 gnosisllm_knowledge/loaders/website.py,sha256=i54S7X0wTYNw9jJQfaoWFqMkGJkbAJeBv-OFBNjPQ_g,1587
 gnosisllm_knowledge/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 gnosisllm_knowledge/services/__init__.py,sha256=qOn1CjsB25UQ2K1TQIGMBe9-CUrGxAKhBrNh9Awo8n8,273
-gnosisllm_knowledge/services/indexing.py,sha256=xZDsULtEuQBYyJ8EZvK_Kp-Gpr6oD87DeURtg-CY3A0,13000
-gnosisllm_knowledge/services/search.py,sha256=WcoO9LJIqdpmThrv0iKvs0CvNWnB3fBh8kdOKaBLO9I,11992
-gnosisllm_knowledge/services/streaming_pipeline.py,sha256=WaLAoDLdgfiBCoqh6A_Im-m4NNwDvCVNtd6f5WrKJPs,11500
-gnosisllm_knowledge-0.4.0.dist-info/METADATA,sha256=I1hufeDnDOl_UMdIlPmDgjt_4bv63oDfbmmuQv-grXE,16311
-gnosisllm_knowledge-0.4.0.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
-gnosisllm_knowledge-0.4.0.dist-info/entry_points.txt,sha256=-4wtWCTX7bT8WessWJyqfkQ99-2aCVuSoTL1S_QpaOo,68
-gnosisllm_knowledge-0.4.0.dist-info/RECORD,,
+gnosisllm_knowledge/services/indexing.py,sha256=MWoe7A_J5bzzSil7rOq363ianBriu3iAuZiy2qnaUKQ,13856
+gnosisllm_knowledge/services/search.py,sha256=ekhD8DrmQ4mmj9pmvzCextFGyn85JWT0HaMviqgi-fo,11869
+gnosisllm_knowledge/services/streaming_pipeline.py,sha256=nztvlgCtK0okHY5Eowk0627a45lts_PkzSTXkeB67xA,11785
+gnosisllm_knowledge-0.4.3.dist-info/METADATA,sha256=iX4v_d9e9aV-n8uo-b8CJSzuMiOm1eiiQlx0btLiQ68,16311
+gnosisllm_knowledge-0.4.3.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
+gnosisllm_knowledge-0.4.3.dist-info/entry_points.txt,sha256=-4wtWCTX7bT8WessWJyqfkQ99-2aCVuSoTL1S_QpaOo,68
+gnosisllm_knowledge-0.4.3.dist-info/RECORD,,