nucliadb 6.2.1.post2954__py3-none-any.whl → 6.2.1.post2972__py3-none-any.whl

nucliadb/writer/back_pressure.py

@@ -30,7 +30,6 @@ from cachetools import TTLCache
 from fastapi import HTTPException, Request
 
 from nucliadb.common import datamanagers
-from nucliadb.common.cluster.manager import get_index_nodes
 from nucliadb.common.context import ApplicationContext
 from nucliadb.common.context.fastapi import get_app_context
 from nucliadb.common.http_clients.processing import ProcessingHTTPClient
@@ -168,7 +167,7 @@ class Materializer:
         self.ingest_check_interval = ingest_check_interval
 
         self.ingest_pending: int = 0
-        self.indexing_pending: dict[str, int] = {}
+        self.indexing_pending: int = 0
 
         self._tasks: list[asyncio.Task] = []
         self._running = False
@@ -232,7 +231,7 @@ class Materializer:
         response = await self.processing_http_client.stats(kbid=kbid, timeout=0.5)
         return response.incomplete
 
-    def get_indexing_pending(self) -> dict[str, int]:
+    def get_indexing_pending(self) -> int:
         return self.indexing_pending
 
     def get_ingest_pending(self) -> int:
@@ -241,20 +240,18 @@ class Materializer:
     async def _get_indexing_pending_task(self):
         try:
             while True:
-                for node in get_index_nodes():
-                    try:
-                        with back_pressure_observer({"type": "get_indexing_pending"}):
-                            self.indexing_pending[node.id] = await get_nats_consumer_pending_messages(
-                                self.nats_manager,
-                                stream=const.Streams.INDEX.name,
-                                consumer=const.Streams.INDEX.group.format(node=node.id),
-                            )
-                    except Exception:
-                        logger.exception(
-                            "Error getting pending messages to index",
-                            exc_info=True,
-                            extra={"node_id": node.id},
+                try:
+                    with back_pressure_observer({"type": "get_indexing_pending"}):
+                        self.indexing_pending = await get_nats_consumer_pending_messages(
+                            self.nats_manager,
+                            stream="nidx",
+                            consumer="nidx",
                         )
+                except Exception:
+                    logger.exception(
+                        "Error getting pending messages to index",
+                        exc_info=True,
+                    )
                 await asyncio.sleep(self.indexing_check_interval)
         except asyncio.CancelledError:
             pass
@@ -386,7 +383,7 @@ async def check_indexing_behind(
     context: ApplicationContext,
     kbid: str,
     resource_uuid: Optional[str],
-    pending_by_node: dict[str, int],
+    pending: int,
 ):
     """
     If a resource uuid is provided, it will check the nodes that have the replicas
@@ -398,36 +395,10 @@ async def check_indexing_behind(
         # Indexing back pressure is disabled
         return
 
-    if len(pending_by_node) == 0:
-        logger.warning("No nodes found to check for pending messages")
-        return
-
-    # Get nodes that are involved in the indexing of the request
-    if resource_uuid is not None:
-        nodes_to_check = await get_nodes_for_resource_shard(context, kbid, resource_uuid)
-    else:
-        nodes_to_check = await get_nodes_for_kb_active_shards(context, kbid)
-
-    if len(nodes_to_check) == 0:
-        logger.warning(
-            "No nodes found to check for pending messages",
-            extra={"kbid": kbid, "resource_uuid": resource_uuid},
-        )
-        return
-
-    # Get the highest pending value
-    highest_pending = 0
-    for node in nodes_to_check:
-        if node not in pending_by_node:
-            logger.warning("Node not found in pending messages", extra={"node": node})
-            continue
-        if pending_by_node[node] > highest_pending:
-            highest_pending = pending_by_node[node]
-
-    if highest_pending > max_pending:
+    if pending > max_pending:
         try_after = estimate_try_after(
             rate=settings.indexing_rate,
-            pending=highest_pending,
+            pending=pending,
             max_wait=settings.max_wait_time,
         )
         data = BackPressureData(type="indexing", try_after=try_after)
@@ -437,7 +408,7 @@ async def check_indexing_behind(
                 "kbid": kbid,
                 "resource_uuid": resource_uuid,
                 "try_after": try_after,
-                "pending": highest_pending,
+                "pending": pending,
             },
         )
         raise BackPressureException(data)

nucliadb/writer/settings.py

@@ -36,7 +36,7 @@ class BackPressureSettings(BaseSettings):
         alias="back_pressure_enabled",
     )
     indexing_rate: float = Field(
-        default=4,
+        default=10,
         description="Estimation of the indexing rate in messages per second. This is used to calculate the try again in time",  # noqa
     )
     ingest_rate: float = Field(
@@ -48,7 +48,7 @@ class BackPressureSettings(BaseSettings):
         description="Estimation of the processing rate in messages per second. This is used to calculate the try again in time",  # noqa
     )
     max_indexing_pending: int = Field(
-        default=200,
+        default=1000,
         description="Max number of messages pending to index in a node queue before rate limiting writes. Set to 0 to disable indexing back pressure checks",  # noqa
         alias="back_pressure_max_indexing_pending",
     )

{nucliadb-6.2.1.post2954.dist-info → nucliadb-6.2.1.post2972.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: nucliadb
-Version: 6.2.1.post2954
+Version: 6.2.1.post2972
 Home-page: https://docs.nuclia.dev/docs/management/nucliadb/intro
 Author: NucliaDB Community
 Author-email: nucliadb@nuclia.com
@@ -22,12 +22,11 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: >=3.9, <4
 Description-Content-Type: text/markdown
-Requires-Dist: nucliadb-telemetry[all]>=6.2.1.post2954
-Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.2.1.post2954
-Requires-Dist: nucliadb-protos>=6.2.1.post2954
-Requires-Dist: nucliadb-models>=6.2.1.post2954
+Requires-Dist: nucliadb-telemetry[all]>=6.2.1.post2972
+Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.2.1.post2972
+Requires-Dist: nucliadb-protos>=6.2.1.post2972
+Requires-Dist: nucliadb-models>=6.2.1.post2972
 Requires-Dist: nucliadb-admin-assets>=1.0.0.post1224
-Requires-Dist: nucliadb-node-binding>=2.26.0
 Requires-Dist: nuclia-models>=0.24.2
 Requires-Dist: uvicorn
 Requires-Dist: argdantic
@@ -78,7 +77,6 @@ Requires-Dist: async_lru>=2.0.4
 Requires-Dist: async-timeout>=4.0.3
 Requires-Dist: cachetools>=5.3.2
 Requires-Dist: types-cachetools>=5.3.0.5
-Requires-Dist: kubernetes_asyncio<30.0.0
 Provides-Extra: redis
 Requires-Dist: redis>=4.3.4; extra == "redis"
 Dynamic: author

{nucliadb-6.2.1.post2954.dist-info → nucliadb-6.2.1.post2972.dist-info}/RECORD

@@ -31,7 +31,7 @@ migrations/pg/0002_catalog.py,sha256=Rsleecu351Ty19kYZgOpqX5G3MEAY8nMxCJrAeuS2Mw
 migrations/pg/0003_catalog_kbid_index.py,sha256=uKq_vtnuf73GVf0mtl2rhzdk_czAoEU1UdiVKVZpA0M,1044
 migrations/pg/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/__init__.py,sha256=_abCmDJ_0ku483Os4UAjPX7Nywm39cQgAV_DiyjsKeQ,891
-nucliadb/health.py,sha256=f2UqyxTcziOOPe9oAlB0XVAKEhYWJM6wDPVJ0cFEhzE,3695
+nucliadb/health.py,sha256=UIxxA4oms4HIsCRZM_SZsdkIZIlgzmOxw-qSHLlWuak,3465
 nucliadb/learning_proxy.py,sha256=LxsGbYD-kwCY6wlZWOhGv2kiDJKGz623J7WDfL38yHw,19359
 nucliadb/metrics_exporter.py,sha256=Rz6G7V_C_GTZCFzd0xEtIfixtZgUuffnr4rDKCbXXWM,5595
 nucliadb/openapi.py,sha256=wDiw0dVEvTpJvbatkJ0JZLkKm9RItZT5PWRHjqRfqTA,2272
@@ -41,29 +41,19 @@ nucliadb/common/constants.py,sha256=QpigxJh_CtD85Evy0PtV5cVq6x0U_f9xfIcXz1ymkUg,
 nucliadb/common/counters.py,sha256=yhJEmmrglTSrDmB8OjaFLkZ__TwhTxayyQrtacnB55I,957
 nucliadb/common/ids.py,sha256=HMb213Kz9HaY4IsBwaQJFhUErntKWV-29s0UHaGcf1E,8004
 nucliadb/common/locking.py,sha256=RL0CabZVPzxHZyUjYeUyLvsJTm7W3J9o4fEgsY_ufNc,5896
-nucliadb/common/nidx.py,sha256=D74oNdniWjbc6gBBDwZP74NH-egTIORHhbfzgIto8DE,8667
+nucliadb/common/nidx.py,sha256=56fimZcR-__SOfbgU72GEZqPPjYhEsz5DHurAef72XE,8823
 nucliadb/common/cluster/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/common/cluster/base.py,sha256=kklDqyvsubNX0W494ttl9f3E58lGaX6AXqAd8XX8ZHE,5522
 nucliadb/common/cluster/exceptions.py,sha256=V3c_fgH00GyJ-a5CaGLhwTuhwhUNR9YAGvS5jaRuc_Y,1495
 nucliadb/common/cluster/grpc_node_dummy.py,sha256=L85wBnfab7Rev0CfsfUjPxQC6DiHPsETKrZAOLx9XHg,3510
 nucliadb/common/cluster/index_node.py,sha256=g38H1kiAliF3Y6et_CWYInpn_xPxf7THAFJ7RtgLNZo,3246
-nucliadb/common/cluster/manager.py,sha256=cKC7ZZdI-abD9Ha5wIRlLxm4cVaF2xeZ0exMaY0s75c,27365
-nucliadb/common/cluster/rebalance.py,sha256=ZlMuGRYfWQeGAAH6vbt8R9Yg80sJ_epVNPRj2cCEaM4,9167
-nucliadb/common/cluster/rollover.py,sha256=Z_KILOi0uC_qEQ7Nc-9u2veoYkonL7nqQZmIB9gtG_w,28458
+nucliadb/common/cluster/manager.py,sha256=3UnYwVb-ZykYfLndxM7TLw7-2T_vxqoFXMu0Pzxh5-A,15327
+nucliadb/common/cluster/rebalance.py,sha256=jSEYsPgs_Dobv3FOaKl5arBko4s8JlWkahm8LOzgNnE,9135
+nucliadb/common/cluster/rollover.py,sha256=dx6AF9ywKP10iBNlcoJgRV40921fOPpVWaCUU54hztE,25823
 nucliadb/common/cluster/settings.py,sha256=RqIMJNyNcn5aV5PifUflkn4zDVX-Ruo36PXGP1CV1mc,3263
-nucliadb/common/cluster/utils.py,sha256=tDYx7owJrIKWsaRFSdw2YPSZF82CQIS0eFyY0AJJWEo,6402
-nucliadb/common/cluster/discovery/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
-nucliadb/common/cluster/discovery/base.py,sha256=-iKxLF8CSDO9dyT0DGiR_Hi8MiwxTN_0hqfNAcHNc1c,6473
-nucliadb/common/cluster/discovery/k8s.py,sha256=etU8uSlgr6UwniWC0BeOfJvYL4iJkLbzl4PPyQM45OM,12282
-nucliadb/common/cluster/discovery/manual.py,sha256=ZryMjHD45WdIJlEPIoyouAljkLw3ZO_SGGmv3oJnBf4,1919
-nucliadb/common/cluster/discovery/single.py,sha256=2BhcencPKQQIfVitmTPJZm3TkBHyY9ZMcr-Clh8k2tM,1737
-nucliadb/common/cluster/discovery/types.py,sha256=vs-K790rofjZ4FWYaMkgjkZZqMvIvd_0eSw3shuiLwA,1139
-nucliadb/common/cluster/discovery/utils.py,sha256=OWQ3NewGX7PlIGOWuS7M5wxDMb96kQj-Ll53qiv0QcE,2526
+nucliadb/common/cluster/utils.py,sha256=Vu0f6026EBELe-ff3d5B0ihD0HtjSWKDCr4dy7LmKqg,5848
 nucliadb/common/cluster/standalone/__init__.py,sha256=itSI7dtTwFP55YMX4iK7JzdMHS5CQVUiB1XzQu4UBh8,833
-nucliadb/common/cluster/standalone/grpc_node_binding.py,sha256=oTqzHgowO3A-N891Th8P--0VHcE0gXj2GltjGpATUCo,12221
-nucliadb/common/cluster/standalone/index_node.py,sha256=QuxAjx3NguLeK8X91f8xtl54Q7-ynrqJEoB-KgssTIQ,4578
-nucliadb/common/cluster/standalone/service.py,sha256=A9VSTkBW0scGuNIQi1JiGTzFXhHuEK7KJ1SqPcgO0gM,3416
-nucliadb/common/cluster/standalone/utils.py,sha256=T6pJ7pWt14i0cVzJ1upw2texM6pophOYJYTdekJppLI,3501
+nucliadb/common/cluster/standalone/utils.py,sha256=af3r-x_GF7A6dwIAhZLR-r-SZQEVxsFrDKeMfUTA6G0,1908
 nucliadb/common/context/__init__.py,sha256=gPwX1cZHPV9r-fLl3ZCZMYBTAwo6Q_xvSlQtd4qPd4c,3592
 nucliadb/common/context/fastapi.py,sha256=j3HZ3lne6mIfw1eEar2het8RWzv6UruUZpXaKieSLOs,1527
 nucliadb/common/datamanagers/__init__.py,sha256=U1cg-KvqbfzN5AnL_tFFrERmPb81w_0MNiTmxObmla4,2062
@@ -110,7 +100,7 @@ nucliadb/export_import/models.py,sha256=dbjScNkiMRv4X3Ktudy1JRliD25bfoDTy3JmEZgQ
 nucliadb/export_import/tasks.py,sha256=fpCBeFYPReyLIdk38LDM9Tpnw_VczeMrobT4n1RAIp4,2507
 nucliadb/export_import/utils.py,sha256=zrNrkkc9i3uT-R6Ju4J_0WNrzayln3KuQFCz-_qIaIA,19613
 nucliadb/ingest/__init__.py,sha256=fsw3C38VP50km3R-nHL775LNGPpJ4JxqXJ2Ib1f5SqE,1011
-nucliadb/ingest/app.py,sha256=L8MDbURnSdD6yI4yCnSbmnNccsnha-zJEkKSaGk1xMg,7612
+nucliadb/ingest/app.py,sha256=kZ8RgZNumzVEssVGnSZIh8jyCYKYotGhxBYz3EWx6Cc,7361
 nucliadb/ingest/cache.py,sha256=w7jMMzamOmQ7gwXna6Dqm6isRNBVv6l5BTBlTxaYWjE,1005
 nucliadb/ingest/partitions.py,sha256=2NIhMYbNT0TNBL6bX1UMSi7vxFGICstCKEqsB0TXHOE,2410
 nucliadb/ingest/processing.py,sha256=gg1DqbMFwqdOsmCSGsZc2abRdYz86xOZJun9vrHOCzs,20618
@@ -119,13 +109,13 @@ nucliadb/ingest/serialize.py,sha256=13NcAP0Tw0zxMsjdudnSnedsg1YpiqFrA-TTIxQA6Ww,
 nucliadb/ingest/settings.py,sha256=0B-wQNa8FLqtNcQgRzh-fuIuGptM816XHcbH1NQKfmE,3050
 nucliadb/ingest/utils.py,sha256=l1myURu3r8oA11dx3GpHw-gNTUc1AFX8xdPm9Lgl2rA,2275
 nucliadb/ingest/consumer/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
-nucliadb/ingest/consumer/auditing.py,sha256=EJoqRRr4dk2eUMK0GOY6b9xHO0YLQ0LjoP_xZBLACZo,7280
+nucliadb/ingest/consumer/auditing.py,sha256=QLffHz49oz9h9P080oBB7eTBL4cqWI-JTTDeg3SmFQ8,7264
 nucliadb/ingest/consumer/consumer.py,sha256=Lej1d6jqmaeR3vjzD0mnfKcVzZTT4TQ3lb1DOfyNWM4,14117
 nucliadb/ingest/consumer/materializer.py,sha256=7ofLbwjldJA8TWXDRZRM4U5EviZt3qNSQ8oadmkzS0Y,3840
 nucliadb/ingest/consumer/metrics.py,sha256=ji1l_4cKiHJthQd8YNem1ft4iMbw9KThmVvJmLcv3Xg,1075
 nucliadb/ingest/consumer/pull.py,sha256=EYT0ImngMQgatStG68p2GSrPQBbJxeuq8nFm8DdAbwk,9280
 nucliadb/ingest/consumer/service.py,sha256=EZM1sABW_7bj6j2UgKUHUuK-EGIEYnLdtPAn8agfWz0,7110
-nucliadb/ingest/consumer/shard_creator.py,sha256=19wf-Bu_9hb_muCDVblamWuvLr09e5dMu9Id5I4-rGw,4324
+nucliadb/ingest/consumer/shard_creator.py,sha256=8SotMc-o_G8XZU52gR4Aay7tcigTdIXgz8YtxqHmJ1Q,4309
 nucliadb/ingest/consumer/utils.py,sha256=jpX8D4lKzuPCpArQLZeX_Zczq3pfen_zAf8sPJfOEZU,2642
 nucliadb/ingest/fields/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/ingest/fields/base.py,sha256=b6QpVPsCiDirDiYG3-yOCMaSNznJSHmQB0z6J_eDIyw,20657
@@ -138,7 +128,7 @@ nucliadb/ingest/fields/text.py,sha256=tFvSQJAe0W7ePpp2_WDfLiE2yglR1OTU0Zht9acvOF
 nucliadb/ingest/orm/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/ingest/orm/brain.py,sha256=UND5EsNUdd7XdjScYqRqg4r_xCx3l-My8alGw5M9CWg,28398
 nucliadb/ingest/orm/broker_message.py,sha256=ZEMueoGuuRKO4tHgzc0P0AM1Ls1TTYey_4UvRQf0BpY,6915
-nucliadb/ingest/orm/entities.py,sha256=2PslT1FZ6yCvJtjR0UpKTSzxJrtS-C_gZx4ZTWHunTc,15759
+nucliadb/ingest/orm/entities.py,sha256=5d6Gfo-Yz-rns_mNJeRqiGaPeWpUMgSKZnmWIGMLCKo,15537
 nucliadb/ingest/orm/exceptions.py,sha256=k4Esv4NtL4TrGTcsQpwrSfDhPQpiYcRbB1SpYmBX5MY,1432
 nucliadb/ingest/orm/knowledgebox.py,sha256=XefDz6YsA0DLMS6T5W8P3VMFTkwr13bIae2ot7wAFpE,25259
 nucliadb/ingest/orm/metrics.py,sha256=OkwMSPKLZcKba0ZTwtTiIxwBgaLMX5ydhGieKvi2y7E,1096
@@ -166,7 +156,7 @@ nucliadb/migrator/utils.py,sha256=NgUreUvON8_nWEzTxELBMWlfV7E6-6qi-g0DMEbVEz4,28
 nucliadb/models/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/models/responses.py,sha256=qnuOoc7TrVSUnpikfTwHLKez47_DE4mSFzpxrwtqijA,1599
 nucliadb/purge/__init__.py,sha256=ijcigiWz38ohXmVVwDU87aCki1BkmAIQRjDoNQ3LPRM,11647
-nucliadb/purge/orphan_shards.py,sha256=fA5yqRRN-M50OIk8dkAi1_ShFVjwDYEYqzMA9dYP0eU,9227
+nucliadb/purge/orphan_shards.py,sha256=7Qm5PFscGr5ihcn5YZ9BaDh6shagkYouv8Z_tTzLuM8,9222
 nucliadb/reader/__init__.py,sha256=C5Efic7WlGm2U2C5WOyquMFbIj2Pojwe_8mwzVYnOzE,1304
 nucliadb/reader/app.py,sha256=Se-BFTE6d1v1msLzQn4q5XIhjnSxa2ckDSHdvm7NRf8,3096
 nucliadb/reader/lifecycle.py,sha256=5jYyzMD1tpIh-OYbQoNMjKZ0-3D9KFnULa3B_Vf2xyY,1740
@@ -189,7 +179,7 @@ nucliadb/search/__init__.py,sha256=tnypbqcH4nBHbGpkINudhKgdLKpwXQCvDtPchUlsyY4,1
 nucliadb/search/app.py,sha256=6UV7rO0f3w5bNFXLdQM8bwUwXayMGnM4hF6GGv7WPv4,4260
 nucliadb/search/lifecycle.py,sha256=DW8v4WUi4rZqc7xTOi3rE67W7877WG7fH9oTZbolHdE,2099
 nucliadb/search/openapi.py,sha256=t3Wo_4baTrfPftg2BHsyLWNZ1MYn7ZRdW7ht-wFOgRs,1016
-nucliadb/search/predict.py,sha256=Jan9hMsiIE5WEcyjHMA79Gz8-2pMbHbb8Kb3SgoXw8M,20897
+nucliadb/search/predict.py,sha256=EWOiWVUX9U_TE19Cl6bpCr6Mjs7hjuvCcG26C7e6KnQ,20919
 nucliadb/search/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nucliadb/search/run.py,sha256=aFb-CXRi_C8YMpP_ivNj8KW1BYhADj88y8K9Lr_nUPI,1402
 nucliadb/search/settings.py,sha256=vem3EcyYlTPSim0kEK-xe-erF4BZg0CT_LAb8ZRQAE8,1684
@@ -200,7 +190,7 @@ nucliadb/search/api/v1/ask.py,sha256=Od2U_gaOZK6dJZ1eDGQQJ3xUVnbBih58VPYVAsQErOw
 nucliadb/search/api/v1/catalog.py,sha256=ubYPS1wmPHzOgH9LR0qJmmV-9ELZPtHRSs5TYJ1pA9A,7117
 nucliadb/search/api/v1/feedback.py,sha256=kNLc4dHz2SXHzV0PwC1WiRAwY88fDptPcP-kO0q-FrQ,2620
 nucliadb/search/api/v1/find.py,sha256=DsnWkySu_cFajDWJIxN8DYvLL_Rm2yiCjHD8TsqPfRk,9304
-nucliadb/search/api/v1/knowledgebox.py,sha256=uPaMF5c1168Owd6fy_x7IUlcTBzDp2Qm6i_aiz6pa-s,8805
+nucliadb/search/api/v1/knowledgebox.py,sha256=Hrt2h-28DDlwN3AdjMZPTYI6om0RMy9bmJvqVHvw8sE,8620
 nucliadb/search/api/v1/predict_proxy.py,sha256=QrGzo0hKjtmyGZ6pjlJHYAh4hxwVUIOTcVcerRCw7eE,3047
 nucliadb/search/api/v1/router.py,sha256=mtT07rBZcVfpa49doaw9b1tj3sdi3qLH0gn9Io6NYM0,988
 nucliadb/search/api/v1/search.py,sha256=vCj5V9kozoti0JrgU_XJhTcBucWzI4SY1B0yCSj9EQw,13638
@@ -211,7 +201,7 @@ nucliadb/search/api/v1/resource/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV
 nucliadb/search/api/v1/resource/ask.py,sha256=XMEP9_Uwy37yaXLcIYKMXGiZYNASD8RTByzQGjd9LPQ,3847
 nucliadb/search/api/v1/resource/search.py,sha256=X0rQU14r_s4_CPpoE2sc84AJPX68gvCftcP4bosWHhA,4812
 nucliadb/search/requesters/__init__.py,sha256=itSI7dtTwFP55YMX4iK7JzdMHS5CQVUiB1XzQu4UBh8,833
-nucliadb/search/requesters/utils.py,sha256=7ovWSGzhLpZGTMi9x9nMOi7QNCgt2qah-7Kam-cIvUg,8468
+nucliadb/search/requesters/utils.py,sha256=ZTiWDkDihJ7rcvs7itCe8hr6OclVcvu_2EAPFeyGhF4,8389
 nucliadb/search/search/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/search/search/cache.py,sha256=n9vkN6Y6Xnr2RBJyoH0WzjzGTJOMfKekU9tfPTWWCPc,6810
 nucliadb/search/search/cut.py,sha256=ytY0_GY7ocNjfxTb4aosxEp4ZfhQNDP--JkhEMGD298,1153
@@ -230,15 +220,15 @@ nucliadb/search/search/predict_proxy.py,sha256=xBlh6kjuQpWRq7KsBx4pEl2PtnwljjQIi
 nucliadb/search/search/query.py,sha256=CbCLdkBgD1XRVXN5qgT1rforf28cLNXvKbjbZzQYmUA,38051
 nucliadb/search/search/rank_fusion.py,sha256=tRGo_KlsFsVx1CQEy1iqQ6f0T1Dq1kf0axDXHuuzvvM,6946
 nucliadb/search/search/rerankers.py,sha256=0kAHES9X_FKkP7KSN9NRETFmRPKzwrFAo_54MbyvM7Q,9051
-nucliadb/search/search/shards.py,sha256=mM2aCHWhl_gwkCENXDShPukS-_qnB5tFS3UAJuzM9qA,2182
+nucliadb/search/search/shards.py,sha256=JSRSrHgHcF4sXyuZZoJdMfK0v_LHpoSRf1lCr5-K5ko,2742
 nucliadb/search/search/summarize.py,sha256=ksmYPubEQvAQgfPdZHfzB_rR19B2ci4IYZ6jLdHxZo8,4996
 nucliadb/search/search/utils.py,sha256=iF2tbBA56gRMJH1TlE2hMrqeXqjoeOPt4KgRdp2m9Ek,3313
 nucliadb/search/search/chat/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
-nucliadb/search/search/chat/ask.py,sha256=sopRYv2yyNUr-rSILpRm4rWI6exBQoGTaONhW1VBiVY,36008
+nucliadb/search/search/chat/ask.py,sha256=tE1Q5V58oLMCo-T9s0N6Kko-1RWn1e4kHfbbPBsD2uU,36266
 nucliadb/search/search/chat/exceptions.py,sha256=Siy4GXW2L7oPhIR86H3WHBhE9lkV4A4YaAszuGGUf54,1356
 nucliadb/search/search/chat/images.py,sha256=PA8VWxT5_HUGfW1ULhKTK46UBsVyINtWWqEM1ulzX1E,3095
 nucliadb/search/search/chat/prompt.py,sha256=r2JTiRWH3YHPdeRAG5w6gD0g0fWVxdTjYIR86qAVa7k,47106
-nucliadb/search/search/chat/query.py,sha256=EfZVJ_Nxo2Jwrbl4ICUL61srKv22PCPt1tZDXMZZQq4,15418
+nucliadb/search/search/chat/query.py,sha256=y7W5VuKl1XiZuNsxZIcxxHcFXSG6It2W5CoftZ-ekAc,15428
 nucliadb/search/search/query_parser/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/search/search/query_parser/exceptions.py,sha256=tuzl7ZyvVsRz6u0_3zMe60vx39nd3pi641prs-5nC0E,872
 nucliadb/search/search/query_parser/models.py,sha256=-VlCDXUCgOroAZw1Leqhj2VMgRv_CD2w40PXXOBLaUM,2332
@@ -248,7 +238,7 @@ nucliadb/standalone/api_router.py,sha256=zR03TQ-Pd2kXx1jeV83Puw19112Z8Jhln7p1cAn
 nucliadb/standalone/app.py,sha256=mAApNK_iVsQgJyd-mtwCeZq5csSimwnXmlQGH9a70pE,5586
 nucliadb/standalone/auth.py,sha256=UwMv-TywhMZabvVg3anQLeCRdoHDnWf2o3luvnoNBjs,7670
 nucliadb/standalone/config.py,sha256=g9JBJQfyw87TYZ3yuy0O9WFVLd_MmCJxSRSI0E8FwZE,5396
-nucliadb/standalone/introspect.py,sha256=KJ91JYGaxMJ62lNe6Ji9NFfyEyks6lVJcGTmqfPjtp0,6986
+nucliadb/standalone/introspect.py,sha256=xHdHV-CB0Vy5cp1MQAodu0Pc8izpzl_lX2ARJJwL3RI,6083
 nucliadb/standalone/lifecycle.py,sha256=rdKLG-oOLN4rfd2VGG_2vlDUWYneWSCiuEhoeiFKfnM,2343
 nucliadb/standalone/migrations.py,sha256=s9-3RSZ-O3bjEw2TnBe_YWLUEKbub0bARUxi1gA3yuY,1950
 nucliadb/standalone/purge.py,sha256=ZY-cebb214FFiPG7OFmXZGg0G3CK5Amw0FLLm9WJhKE,1343
@@ -272,9 +262,9 @@ nucliadb/tests/vectors.py,sha256=CcNKx-E8LPpyvRyljbmb-Tn_wST9Juw2CBoogWrKiTk,628
 nucliadb/train/__init__.py,sha256=NVwe5yULoHXb80itIJT8YJYEz2xbiOPQ7_OMys6XJw8,1301
 nucliadb/train/app.py,sha256=TiRttTvekLuZdIvi46E4HyuumDTkR4G4Luqq3fEdjes,2824
 nucliadb/train/generator.py,sha256=0_zqWsLUHmJZl0lXhGorO5CWSkl42-k78dqb1slZ5h0,3904
-nucliadb/train/lifecycle.py,sha256=aCNaRURu0ZOUJaWLTZuEjwTstnB9MuLtzxOMztQoGxc,1773
+nucliadb/train/lifecycle.py,sha256=dWnFuo-nYaWd6uayregmakQaZbQFdtVJReJONhYAQYI,1581
 nucliadb/train/models.py,sha256=BmgmMjDsu_1Ih5JDAqo6whhume90q0ASJcDP9dkMQm8,1198
-nucliadb/train/nodes.py,sha256=_89ZIpBb0HnR2jejvuO6aPsgHVSGbasPWz0lkGmVnvU,5925
+nucliadb/train/nodes.py,sha256=HROQMRw2g5sJTnuBagh3B0id3iWonRJ68tg3skOme9k,5748
 nucliadb/train/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nucliadb/train/resource.py,sha256=3qQ_9Zdt5JAbtD-wpmt7OeDGRNKS-fQdKAuIQfznZm0,16219
 nucliadb/train/run.py,sha256=evz6CKVfJOzkbHMoaYz2mTMlKjJnNOb1O8zBBWMpeBw,1400
@@ -302,13 +292,13 @@ nucliadb/train/generators/token_classifier.py,sha256=Vl14aaWoqrgYPijmvM62OjxDdAN
 nucliadb/train/generators/utils.py,sha256=1uSELmM4CpKy9jWp6j_u7_n_KR-udRNkes4UmPMOCcI,3907
 nucliadb/writer/__init__.py,sha256=S298mrZL3vr62OrBqi97mdLxgR5cReMlRJgnaQHZV7s,1304
 nucliadb/writer/app.py,sha256=ABBO8-u4pDAa61b3mCdD0TFhuHAYcxMkgpZSGgWARuE,2736
-nucliadb/writer/back_pressure.py,sha256=s3Z8ZLD8s0xApUkt3BRZmOFzgVJ5FDqljJFpAYeI5LU,19194
+nucliadb/writer/back_pressure.py,sha256=JaiC2JAugVA92gDHzABZFiuQexiOKZC9C-3Jn9VF-M0,17898
 nucliadb/writer/exceptions.py,sha256=-Z7LW--eid7PNeKFuzo9kAlbLEBMUosxE-UVIgGD3SA,929
 nucliadb/writer/lifecycle.py,sha256=OYyhUZ1ejlybPzO-O_EsInjdifKiPiEzooy2d_2DW3k,2550
 nucliadb/writer/openapi.py,sha256=thqCO1ht_RJgOkXs-aIsv8aXJrU5z8wo2n05l2_LqMs,1032
 nucliadb/writer/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nucliadb/writer/run.py,sha256=euVZ_rtHDXs-O1kB-Pt1Id8eft9CYVpWH3zJzEoEqls,1448
-nucliadb/writer/settings.py,sha256=32Umt2SqeIL8PW4_C6hkuq01QT1YmcROiWpmoy1D5Wk,3286
+nucliadb/writer/settings.py,sha256=pA9aMAvY8H6zvsxAOdGY8SZLrThDvJ8KLhluGI0GxnQ,3288
 nucliadb/writer/utilities.py,sha256=AZ5qEny1Xm0IDsFtH13oJa2usvJZK8f0FdgF1LrnLCw,1036
 nucliadb/writer/vectorsets.py,sha256=18XJvsyi0-tePQWig8dl5qaNPaufEZb0-uD22IAOTa0,5648
 nucliadb/writer/api/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
@@ -340,9 +330,9 @@ nucliadb/writer/tus/local.py,sha256=7jYa_w9b-N90jWgN2sQKkNcomqn6JMVBOVeDOVYJHto,
 nucliadb/writer/tus/s3.py,sha256=vF0NkFTXiXhXq3bCVXXVV-ED38ECVoUeeYViP8uMqcU,8357
 nucliadb/writer/tus/storage.py,sha256=ToqwjoYnjI4oIcwzkhha_MPxi-k4Jk3Lt55zRwaC1SM,2903
 nucliadb/writer/tus/utils.py,sha256=MSdVbRsRSZVdkaum69_0wku7X3p5wlZf4nr6E0GMKbw,2556
-nucliadb-6.2.1.post2954.dist-info/METADATA,sha256=uqKCYU8MpuEtw2VE0oX1sX-WQTRD-ACMAf1jRBOO7P0,4689
-nucliadb-6.2.1.post2954.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-nucliadb-6.2.1.post2954.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
-nucliadb-6.2.1.post2954.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
-nucliadb-6.2.1.post2954.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-nucliadb-6.2.1.post2954.dist-info/RECORD,,
+nucliadb-6.2.1.post2972.dist-info/METADATA,sha256=YnrbW19LMC5vJsf6nRuyDwUNVBFivf8KQJnvl2xeMro,4603
+nucliadb-6.2.1.post2972.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+nucliadb-6.2.1.post2972.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
+nucliadb-6.2.1.post2972.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
+nucliadb-6.2.1.post2972.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+nucliadb-6.2.1.post2972.dist-info/RECORD,,

nucliadb/common/cluster/discovery/__init__.py

@@ -1,19 +0,0 @@
-# Copyright (C) 2021 Bosutech XXI S.L.
-#
-# nucliadb is offered under the AGPL v3.0 and as commercial software.
-# For commercial licensing, contact us at info@nuclia.com.
-#
-# AGPL:
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-#

nucliadb/common/cluster/discovery/base.py

@@ -1,178 +0,0 @@
-# Copyright (C) 2021 Bosutech XXI S.L.
-#
-# nucliadb is offered under the AGPL v3.0 and as commercial software.
-# For commercial licensing, contact us at info@nuclia.com.
-#
-# AGPL:
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-#
-import abc
-import asyncio
-import logging
-
-import backoff
-from grpc.aio import AioRpcError
-
-from nucliadb.common.cluster import manager
-from nucliadb.common.cluster.discovery.types import IndexNodeMetadata
-from nucliadb.common.cluster.exceptions import NodeConnectionError
-from nucliadb.common.cluster.settings import Settings
-from nucliadb_protos import (
-    noderesources_pb2,
-    nodewriter_pb2,
-    nodewriter_pb2_grpc,
-    replication_pb2_grpc,
-    standalone_pb2,
-    standalone_pb2_grpc,
-)
-from nucliadb_telemetry import metrics
-from nucliadb_utils.grpc import get_traced_grpc_channel
-
-logger = logging.getLogger(__name__)
-
-AVAILABLE_NODES = metrics.Gauge("nucliadb_nodes_available")
-
-
-def update_members(members: list[IndexNodeMetadata]) -> None:
-    # First add new nodes or update existing ones
-    valid_ids = []
-    for member in members:
-        valid_ids.append(member.node_id)
-
-        shard_count = member.shard_count
-        if shard_count is None:
-            shard_count = 0
-            logger.warning(f"Node {member.node_id} has no shard_count")
-
-        node = manager.get_index_node(member.node_id)
-        if node is None:
-            logger.debug(f"{member.node_id} add {member.address}")
-            manager.add_index_node(
-                id=member.node_id,
-                address=member.address,
-                shard_count=shard_count,
-                available_disk=member.available_disk,
-                primary_id=member.primary_id,
-            )
-            logger.debug("Node added")
-        else:
-            logger.debug(f"{member.node_id} update")
-            node.address = member.address
-            node.shard_count = shard_count
-            node.available_disk = member.available_disk
-            logger.debug("Node updated")
-
-    # Then cleanup nodes that are no longer reported
-    node_ids = [x.id for x in manager.get_index_nodes()]
-    removed_node_ids = []
-    for key in node_ids:
-        if key not in valid_ids:
-            node = manager.get_index_node(key)
-            if node is not None:
-                removed_node_ids.append(key)
-                logger.warning(f"{key} remove {node.address}")
-                manager.remove_index_node(key, node.primary_id)
-
-    if len(removed_node_ids) > 1:
-        logger.warning(
-            f"{len(removed_node_ids)} nodes are down simultaneously. This should never happen!"
-        )
-
-    AVAILABLE_NODES.set(len(manager.get_index_nodes()))
-
-
-@backoff.on_exception(backoff.constant, (Exception,), interval=0.5, max_tries=2)
-async def _get_index_node_metadata(
-    settings: Settings, address: str, read_replica: bool = False
-) -> IndexNodeMetadata:
-    """
-    Get node metadata directly from the writer.
-
-    Establishes a new connection on every try on purpose to avoid long lived connections
-    and dns caching issues.
-
-    This method should be used carefully and results should be cached.
-    """
-    if address in settings.writer_port_map:
-        # test wiring
-        port = settings.writer_port_map[address]
-        grpc_address = f"localhost:{port}"
-    else:
-        grpc_address = f"{address}:{settings.node_writer_port}"
-    channel = get_traced_grpc_channel(grpc_address, "discovery", variant="_writer")
-    if read_replica:
-        # on a read replica, we need to use the replication service
-        stub = replication_pb2_grpc.ReplicationServiceStub(channel)
-    else:
-        stub = nodewriter_pb2_grpc.NodeWriterStub(channel)  # type: ignore
-    try:
-        metadata: nodewriter_pb2.NodeMetadata = await stub.GetMetadata(noderesources_pb2.EmptyQuery())  # type: ignore
-    except AioRpcError as exc:
-        raise NodeConnectionError() from exc
-
-    primary_id = (
-        getattr(metadata, "primary_node_id", None)
-        # the or None here is important because the proto returns an empty string
-        or None
-    )
-    if read_replica and primary_id is None:
-        raise Exception("Primary node id not found when it is expected to be a read replica")
-
-    return IndexNodeMetadata(
-        node_id=metadata.node_id,
-        name=metadata.node_id,
-        address=address,
-        shard_count=metadata.shard_count,
-        primary_id=primary_id,
-        available_disk=metadata.available_disk,
-    )
-
-
-@backoff.on_exception(backoff.expo, (Exception,), jitter=backoff.random_jitter, max_tries=4)
-async def _get_standalone_index_node_metadata(settings: Settings, address: str) -> IndexNodeMetadata:
-    if ":" not in address:
-        grpc_address = f"{address}:{settings.standalone_node_port}"
-    else:
-        grpc_address = address
-    channel = get_traced_grpc_channel(grpc_address, "standalone_proxy")
-    stub = standalone_pb2_grpc.StandaloneClusterServiceStub(channel)
-    resp: standalone_pb2.NodeInfoResponse = await stub.NodeInfo(standalone_pb2.NodeInfoRequest())  # type: ignore
-    return IndexNodeMetadata(
-        node_id=resp.id,
-        name=resp.id,
-        address=address,
-        shard_count=resp.shard_count,
-        available_disk=resp.available_disk,
-    )
-
-
-class AbstractClusterDiscovery(abc.ABC):
-    task: asyncio.Task
-
-    def __init__(self, settings: Settings):
-        self.settings = settings
-
-    @abc.abstractmethod
-    async def initialize(self) -> None:
-        """ """
-
-    @abc.abstractmethod
-    async def finalize(self) -> None:
-        """ """
-
-    async def _query_node_metadata(self, address: str, read_replica: bool = False) -> IndexNodeMetadata:
-        if self.settings.standalone_mode:
-            return await _get_standalone_index_node_metadata(self.settings, address)
-        else:
-            return await _get_index_node_metadata(self.settings, address, read_replica)