nucliadb 6.2.1.post2972__py3-none-any.whl → 6.2.1.post2977__py3-none-any.whl

nucliadb/common/cluster/settings.py

@@ -90,6 +90,9 @@ class Settings(BaseSettings):
     nidx_searcher_address: Optional[str] = Field(
         default=None, description="NIDX gRPC searcher API address"
     )
+    nidx_indexer_address: Optional[str] = Field(
+        default=None, description="NIDX gRPC indexer API address"
+    )
 
 
 settings = Settings()

nucliadb/common/nidx.py

@@ -21,7 +21,7 @@
 import os
 from typing import Optional
 
-from nidx_protos.nidx_pb2_grpc import NidxApiStub, NidxSearcherStub
+from nidx_protos.nidx_pb2_grpc import NidxApiStub, NidxIndexerStub, NidxSearcherStub
 
 from nucliadb.common.cluster.base import AbstractIndexNode
 from nucliadb.common.cluster.settings import settings
@@ -124,16 +124,8 @@ class NidxBindingUtility(NidxUtility):
         self.binding.wait_for_sync()
 
 
-class NidxServiceUtility(NidxUtility):
-    """Implements Nidx utility connecting to the network service"""
-
+class NidxNatsIndexer:
     def __init__(self):
-        if indexing_settings.index_nidx_subject is None:
-            raise ValueError("INDEX_NIDX_SUBJECT needed for nidx utility")
-
-        if not settings.nidx_api_address or not settings.nidx_searcher_address:
-            raise ValueError("NIDX_API_ADDRESS and NIDX_SEARCHER_ADDRESS are required")
-
         self.nats_connection_manager = NatsConnectionManager(
             service_name="NidxIndexer",
             nats_servers=indexing_settings.index_jetstream_servers,
@@ -143,10 +135,6 @@ class NidxServiceUtility(NidxUtility):
 
     async def initialize(self):
         await self.nats_connection_manager.initialize()
-        self.api_client = NidxApiStub(get_traced_grpc_channel(settings.nidx_api_address, "nidx_api"))
-        self.searcher_client = NidxSearcherStub(
-            get_traced_grpc_channel(settings.nidx_searcher_address, "nidx_searcher")
-        )
 
     async def finalize(self):
         await self.nats_connection_manager.finalize()
@@ -159,15 +147,68 @@ class NidxServiceUtility(NidxUtility):
         return res.seq
 
 
-async def start_nidx_utility() -> NidxUtility:
+class NidxGrpcIndexer:
+    def __init__(self, address):
+        self.address = address
+
+    async def initialize(self):
+        self.client = NidxIndexerStub(get_traced_grpc_channel(self.address, "nidx_indexer"))
+
+    async def finalize(self):
+        pass
+
+    async def index(self, writer: IndexMessage) -> int:
+        await self.client.Index(writer)
+        return 0
+
+
+class NidxServiceUtility(NidxUtility):
+    """Implements Nidx utility connecting to the network service"""
+
+    def __init__(self):
+        if not settings.nidx_api_address or not settings.nidx_searcher_address:
+            raise ValueError("NIDX_API_ADDRESS and NIDX_SEARCHER_ADDRESS are required")
+
+        if indexing_settings.index_nidx_subject:
+            self.indexer = NidxNatsIndexer()
+        elif settings.nidx_indexer_address is not None:
+            self.indexer = NidxGrpcIndexer(settings.nidx_indexer_address)
+        else:
+            raise ValueError("NIDX_INDEXER_ADDRESS or INDEX_NIDX_SUBJECT are required")
+
+    async def initialize(self):
+        await self.indexer.initialize()
+        self.api_client = NidxApiStub(get_traced_grpc_channel(settings.nidx_api_address, "nidx_api"))
+        self.searcher_client = NidxSearcherStub(
+            get_traced_grpc_channel(settings.nidx_searcher_address, "nidx_searcher")
+        )
+
+    async def finalize(self):
+        await self.indexer.finalize()
+
+    async def index(self, writer: IndexMessage) -> int:
+        return await self.indexer.index(writer)
+
+
+async def start_nidx_utility() -> Optional[NidxUtility]:
     nidx = get_utility(Utility.NIDX)
     if nidx:
         return nidx
 
     nidx_utility: NidxUtility
     if settings.standalone_mode:
-        nidx_utility = NidxBindingUtility()
+        if (
+            settings.nidx_api_address is not None
+            and settings.nidx_searcher_address is not None
+            and settings.nidx_indexer_address is not None
+        ):
+            # Standalone with nidx service (via grpc). This is used in clustered standalone mode
+            nidx_utility = NidxServiceUtility()
+        else:
+            # Normal standalone mode with binding
+            nidx_utility = NidxBindingUtility()
     else:
+        # Component deploy with nidx service via grpc & nats (cloud)
         nidx_utility = NidxServiceUtility()
 
     await nidx_utility.initialize()

nucliadb/train/lifecycle.py

@@ -22,6 +22,7 @@ from contextlib import asynccontextmanager
 
 from fastapi import FastAPI
 
+from nucliadb.common.nidx import start_nidx_utility, stop_nidx_utility
 from nucliadb.train import SERVICE_NAME
 from nucliadb.train.utils import (
     start_shard_manager,
@@ -36,6 +37,7 @@ from nucliadb_utils.utilities import start_audit_utility, stop_audit_utility
 @asynccontextmanager
 async def lifespan(app: FastAPI):
     await setup_telemetry(SERVICE_NAME)
+    await start_nidx_utility()
     await start_shard_manager()
     await start_train_grpc(SERVICE_NAME)
     await start_audit_utility(SERVICE_NAME)
@@ -45,4 +47,5 @@ async def lifespan(app: FastAPI):
     await stop_audit_utility()
     await stop_train_grpc()
     await stop_shard_manager()
+    await stop_nidx_utility()
     await clean_telemetry(SERVICE_NAME)

{nucliadb-6.2.1.post2972.dist-info → nucliadb-6.2.1.post2977.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: nucliadb
-Version: 6.2.1.post2972
+Version: 6.2.1.post2977
 Home-page: https://docs.nuclia.dev/docs/management/nucliadb/intro
 Author: NucliaDB Community
 Author-email: nucliadb@nuclia.com
@@ -22,10 +22,10 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: >=3.9, <4
 Description-Content-Type: text/markdown
-Requires-Dist: nucliadb-telemetry[all]>=6.2.1.post2972
-Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.2.1.post2972
-Requires-Dist: nucliadb-protos>=6.2.1.post2972
-Requires-Dist: nucliadb-models>=6.2.1.post2972
+Requires-Dist: nucliadb-telemetry[all]>=6.2.1.post2977
+Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.2.1.post2977
+Requires-Dist: nucliadb-protos>=6.2.1.post2977
+Requires-Dist: nucliadb-models>=6.2.1.post2977
 Requires-Dist: nucliadb-admin-assets>=1.0.0.post1224
 Requires-Dist: nuclia-models>=0.24.2
 Requires-Dist: uvicorn

{nucliadb-6.2.1.post2972.dist-info → nucliadb-6.2.1.post2977.dist-info}/RECORD

@@ -41,7 +41,7 @@ nucliadb/common/constants.py,sha256=QpigxJh_CtD85Evy0PtV5cVq6x0U_f9xfIcXz1ymkUg,
 nucliadb/common/counters.py,sha256=yhJEmmrglTSrDmB8OjaFLkZ__TwhTxayyQrtacnB55I,957
 nucliadb/common/ids.py,sha256=HMb213Kz9HaY4IsBwaQJFhUErntKWV-29s0UHaGcf1E,8004
 nucliadb/common/locking.py,sha256=RL0CabZVPzxHZyUjYeUyLvsJTm7W3J9o4fEgsY_ufNc,5896
-nucliadb/common/nidx.py,sha256=56fimZcR-__SOfbgU72GEZqPPjYhEsz5DHurAef72XE,8823
+nucliadb/common/nidx.py,sha256=_LoU8D4afEtlW0c3vGUCoatDZvMr0-2l_GtIGap7VxA,10185
 nucliadb/common/cluster/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/common/cluster/base.py,sha256=kklDqyvsubNX0W494ttl9f3E58lGaX6AXqAd8XX8ZHE,5522
 nucliadb/common/cluster/exceptions.py,sha256=V3c_fgH00GyJ-a5CaGLhwTuhwhUNR9YAGvS5jaRuc_Y,1495
@@ -50,7 +50,7 @@ nucliadb/common/cluster/index_node.py,sha256=g38H1kiAliF3Y6et_CWYInpn_xPxf7THAFJ
 nucliadb/common/cluster/manager.py,sha256=3UnYwVb-ZykYfLndxM7TLw7-2T_vxqoFXMu0Pzxh5-A,15327
 nucliadb/common/cluster/rebalance.py,sha256=jSEYsPgs_Dobv3FOaKl5arBko4s8JlWkahm8LOzgNnE,9135
 nucliadb/common/cluster/rollover.py,sha256=dx6AF9ywKP10iBNlcoJgRV40921fOPpVWaCUU54hztE,25823
-nucliadb/common/cluster/settings.py,sha256=RqIMJNyNcn5aV5PifUflkn4zDVX-Ruo36PXGP1CV1mc,3263
+nucliadb/common/cluster/settings.py,sha256=TMoym-cZsQ2soWfLAce0moSa2XncttQyhahL43LrWTo,3384
 nucliadb/common/cluster/utils.py,sha256=Vu0f6026EBELe-ff3d5B0ihD0HtjSWKDCr4dy7LmKqg,5848
 nucliadb/common/cluster/standalone/__init__.py,sha256=itSI7dtTwFP55YMX4iK7JzdMHS5CQVUiB1XzQu4UBh8,833
 nucliadb/common/cluster/standalone/utils.py,sha256=af3r-x_GF7A6dwIAhZLR-r-SZQEVxsFrDKeMfUTA6G0,1908
@@ -262,7 +262,7 @@ nucliadb/tests/vectors.py,sha256=CcNKx-E8LPpyvRyljbmb-Tn_wST9Juw2CBoogWrKiTk,628
 nucliadb/train/__init__.py,sha256=NVwe5yULoHXb80itIJT8YJYEz2xbiOPQ7_OMys6XJw8,1301
 nucliadb/train/app.py,sha256=TiRttTvekLuZdIvi46E4HyuumDTkR4G4Luqq3fEdjes,2824
 nucliadb/train/generator.py,sha256=0_zqWsLUHmJZl0lXhGorO5CWSkl42-k78dqb1slZ5h0,3904
-nucliadb/train/lifecycle.py,sha256=dWnFuo-nYaWd6uayregmakQaZbQFdtVJReJONhYAQYI,1581
+nucliadb/train/lifecycle.py,sha256=7rW2Be9cMUg7-fY9JhC5krXBB7MDOM4BpfNRRO5IAec,1713
 nucliadb/train/models.py,sha256=BmgmMjDsu_1Ih5JDAqo6whhume90q0ASJcDP9dkMQm8,1198
 nucliadb/train/nodes.py,sha256=HROQMRw2g5sJTnuBagh3B0id3iWonRJ68tg3skOme9k,5748
 nucliadb/train/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -330,9 +330,9 @@ nucliadb/writer/tus/local.py,sha256=7jYa_w9b-N90jWgN2sQKkNcomqn6JMVBOVeDOVYJHto,
 nucliadb/writer/tus/s3.py,sha256=vF0NkFTXiXhXq3bCVXXVV-ED38ECVoUeeYViP8uMqcU,8357
 nucliadb/writer/tus/storage.py,sha256=ToqwjoYnjI4oIcwzkhha_MPxi-k4Jk3Lt55zRwaC1SM,2903
 nucliadb/writer/tus/utils.py,sha256=MSdVbRsRSZVdkaum69_0wku7X3p5wlZf4nr6E0GMKbw,2556
-nucliadb-6.2.1.post2972.dist-info/METADATA,sha256=YnrbW19LMC5vJsf6nRuyDwUNVBFivf8KQJnvl2xeMro,4603
-nucliadb-6.2.1.post2972.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-nucliadb-6.2.1.post2972.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
-nucliadb-6.2.1.post2972.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
-nucliadb-6.2.1.post2972.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-nucliadb-6.2.1.post2972.dist-info/RECORD,,
+nucliadb-6.2.1.post2977.dist-info/METADATA,sha256=FSC36qYxtNo-I5n-eTSSAh-_WqYCp4i1EvkIcqpWGWM,4603
+nucliadb-6.2.1.post2977.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+nucliadb-6.2.1.post2977.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
+nucliadb-6.2.1.post2977.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
+nucliadb-6.2.1.post2977.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+nucliadb-6.2.1.post2977.dist-info/RECORD,,