nucliadb 6.2.1.post3332__py3-none-any.whl → 6.2.1.post3365__py3-none-any.whl

nucliadb/common/cluster/rebalance.py

@@ -31,7 +31,7 @@ from nucliadb_telemetry.utils import setup_telemetry
 from nucliadb_utils.fastapi.run import serve_metrics
 
 from .settings import settings
-from .utils import delete_resource_from_shard, index_resource_to_shard, wait_for_node
+from .utils import delete_resource_from_shard, index_resource_to_shard
 
 logger = logging.getLogger(__name__)
 
@@ -151,14 +151,6 @@ async def move_set_of_kb_resources(
                     extra={"kbid": kbid, "resource_id": resource_id},
                 )
 
-    node_ids = set()
-    for replica in from_shard.replicas:
-        node_ids.add(replica.node)
-    for replica in to_shard.replicas:
-        node_ids.add(replica.node)
-    for node_id in node_ids:
-        await wait_for_node(context, node_id)
-
 
 async def rebalance_kb(context: ApplicationContext, kbid: str) -> None:
     await maybe_add_shard(kbid)

nucliadb/common/cluster/rollover.py

@@ -39,7 +39,6 @@ from .utils import (
     get_resource,
     get_resource_index_message,
     index_resource_to_shard,
-    wait_for_node,
 )
 
 logger = logging.getLogger(__name__)
@@ -232,7 +231,6 @@ async def index_to_rollover_index(
         return
 
     logger.info("Indexing to rollover index", extra=extra)
-    wait_index_batch: list[writer_pb2.ShardObject] = []
     # now index on all new shards only
     while True:
         async with datamanagers.with_transaction() as txn:
@@ -288,16 +286,6 @@ async def index_to_rollover_index(
                 modification_time=_to_ts(resource.basic.modified.ToDatetime()),  # type: ignore
             )
             await txn.commit()
-        wait_index_batch.append(shard)
-
-        if len(wait_index_batch) > 10:
-            node_ids = set()
-            for shard_batch in wait_index_batch:
-                for replica in shard_batch.replicas:
-                    node_ids.add(replica.node)
-            for node_id in node_ids:
-                await wait_for_node(app_context, node_id)
-            wait_index_batch = []
 
     async with datamanagers.with_transaction() as txn:
         state.resources_indexed = True

nucliadb/common/cluster/utils.py

@@ -30,8 +30,6 @@ from nucliadb.common.cluster.manager import (
 from nucliadb.common.cluster.settings import settings
 from nucliadb.ingest.orm.resource import Resource
 from nucliadb_protos import nodereader_pb2, writer_pb2
-from nucliadb_utils import const
-from nucliadb_utils.settings import is_onprem_nucliadb
 from nucliadb_utils.utilities import Utility, clean_utility, get_utility, set_utility
 
 if TYPE_CHECKING:  # pragma: no cover
@@ -76,34 +74,6 @@ def get_shard_manager() -> KBShardManager:
     return get_utility(Utility.SHARD_MANAGER)  # type: ignore
 
 
-async def wait_for_node(app_context: ApplicationContext, node_id: str) -> None:
-    if is_onprem_nucliadb():
-        # On onprem deployments indexing is synchronous right now, so we don't need to wait
-        return
-
-    logged = False
-    while True:
-        # get raw js client
-        js = app_context.nats_manager.js
-        consumer_info = await js.consumer_info(
-            const.Streams.INDEX.name, const.Streams.INDEX.group.format(node=node_id)
-        )
-        if consumer_info.num_pending < 5:
-            return
-
-        if not logged:
-            logger.info(
-                f"Waiting for node to consume messages. {consumer_info.num_pending} messages left.",
-                extra={"node": node_id},
-            )
-            logged = True
-        # usually we consume around 3-4 messages/s with some eventual peaks of
-        # 10-30. If there are too many pending messages, we can wait more.
-        # We suppose 5 messages/s and don't wait more than 60s
-        sleep = min(max(2, consumer_info.num_pending / 5), 60)
-        await asyncio.sleep(sleep)
-
-
 async def get_resource(kbid: str, resource_id: str) -> Optional[Resource]:
     async with datamanagers.with_ro_transaction() as txn:
         return await datamanagers.resources.get_resource(txn, kbid=kbid, rid=resource_id)

nucliadb/common/context/__init__.py

@@ -25,18 +25,15 @@ from nucliadb.common.cluster.utils import setup_cluster, teardown_cluster
 from nucliadb.common.maindb.driver import Driver
 from nucliadb.common.maindb.utils import setup_driver, teardown_driver
 from nucliadb.common.nidx import start_nidx_utility, stop_nidx_utility
-from nucliadb_utils.indexing import IndexingUtility
 from nucliadb_utils.nats import NatsConnectionManager
 from nucliadb_utils.partition import PartitionUtility
 from nucliadb_utils.settings import indexing_settings
 from nucliadb_utils.storages.storage import Storage
 from nucliadb_utils.utilities import (
     get_storage,
-    start_indexing_utility,
     start_nats_manager,
     start_partitioning_utility,
     start_transaction_utility,
-    stop_indexing_utility,
     stop_nats_manager,
     stop_partitioning_utility,
     stop_transaction_utility,
@@ -49,7 +46,6 @@ class ApplicationContext:
     shard_manager: KBShardManager
     blob_storage: Storage
     partitioning: PartitionUtility
-    indexing: IndexingUtility
     nats_manager: NatsConnectionManager
 
     def __init__(self, service_name: str = "service") -> None:
@@ -77,7 +73,6 @@ class ApplicationContext:
                 indexing_settings.index_jetstream_servers,
                 indexing_settings.index_jetstream_auth,
             )
-            self.indexing = await start_indexing_utility()
         self.transaction = await start_transaction_utility(self.service_name)
         self.nidx = await start_nidx_utility()
 
@@ -88,7 +83,6 @@ class ApplicationContext:
         await stop_nidx_utility()
         await stop_transaction_utility()
         if not in_standalone_mode():
-            await stop_indexing_utility()
             await stop_nats_manager()
 
         stop_partitioning_utility()

nucliadb/ingest/app.py

@@ -22,7 +22,6 @@ import importlib.metadata
 from typing import Awaitable, Callable
 
 from nucliadb import health
-from nucliadb.common.cluster.settings import settings as cluster_settings
 from nucliadb.common.cluster.utils import setup_cluster, teardown_cluster
 from nucliadb.common.context import ApplicationContext
 from nucliadb.common.nidx import start_nidx_utility
@@ -38,15 +37,13 @@ from nucliadb_telemetry.logs import setup_logging
 from nucliadb_telemetry.utils import setup_telemetry
 from nucliadb_utils.fastapi.run import serve_metrics
 from nucliadb_utils.run import run_until_exit
-from nucliadb_utils.settings import indexing_settings, transaction_settings
+from nucliadb_utils.settings import transaction_settings
 from nucliadb_utils.utilities import (
     start_audit_utility,
-    start_indexing_utility,
     start_nats_manager,
     start_partitioning_utility,
     start_transaction_utility,
     stop_audit_utility,
-    stop_indexing_utility,
     stop_nats_manager,
     stop_partitioning_utility,
     stop_transaction_utility,
@@ -58,8 +55,6 @@ async def initialize() -> list[Callable[[], Awaitable[None]]]:
 
     await setup_cluster()
     await start_transaction_utility(SERVICE_NAME)
-    if not cluster_settings.standalone_mode and indexing_settings.index_jetstream_servers is not None:
-        await start_indexing_utility(SERVICE_NAME)
 
     start_partitioning_utility()
 
@@ -70,7 +65,6 @@ async def initialize() -> list[Callable[[], Awaitable[None]]]:
     finalizers = [
         stop_partitioning_utility,
         stop_transaction_utility,
-        stop_indexing_utility,
         stop_audit_utility,
         teardown_cluster,
     ]

{nucliadb-6.2.1.post3332.dist-info → nucliadb-6.2.1.post3365.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: nucliadb
-Version: 6.2.1.post3332
+Version: 6.2.1.post3365
 Summary: NucliaDB
 Author-email: Nuclia <nucliadb@nuclia.com>
 License: AGPL
@@ -20,11 +20,11 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: <4,>=3.9
 Description-Content-Type: text/markdown
-Requires-Dist: nucliadb-telemetry[all]>=6.2.1.post3332
-Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.2.1.post3332
-Requires-Dist: nucliadb-protos>=6.2.1.post3332
-Requires-Dist: nucliadb-models>=6.2.1.post3332
-Requires-Dist: nidx-protos>=6.2.1.post3332
+Requires-Dist: nucliadb-telemetry[all]>=6.2.1.post3365
+Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.2.1.post3365
+Requires-Dist: nucliadb-protos>=6.2.1.post3365
+Requires-Dist: nucliadb-models>=6.2.1.post3365
+Requires-Dist: nidx-protos>=6.2.1.post3365
 Requires-Dist: nucliadb-admin-assets>=1.0.0.post1224
 Requires-Dist: nuclia-models>=0.24.2
 Requires-Dist: uvicorn

{nucliadb-6.2.1.post3332.dist-info → nucliadb-6.2.1.post3365.dist-info}/RECORD

@@ -49,13 +49,13 @@ nucliadb/common/cluster/exceptions.py,sha256=V3c_fgH00GyJ-a5CaGLhwTuhwhUNR9YAGvS
 nucliadb/common/cluster/grpc_node_dummy.py,sha256=L85wBnfab7Rev0CfsfUjPxQC6DiHPsETKrZAOLx9XHg,3510
 nucliadb/common/cluster/index_node.py,sha256=g38H1kiAliF3Y6et_CWYInpn_xPxf7THAFJ7RtgLNZo,3246
 nucliadb/common/cluster/manager.py,sha256=-y5PCUzxrvvUCBopmi8JztPiSBYYbUBeLer0XtO4Ukk,15383
-nucliadb/common/cluster/rebalance.py,sha256=P3P45m390ImOYBmjuufNpBRXijWvcNpkYqPiDPcdbHg,8960
-nucliadb/common/cluster/rollover.py,sha256=dx6AF9ywKP10iBNlcoJgRV40921fOPpVWaCUU54hztE,25823
+nucliadb/common/cluster/rebalance.py,sha256=cGrqc6GQ6Y2X1moW7NzIdLLsij0cbKySiu-qUa3fA-o,8700
+nucliadb/common/cluster/rollover.py,sha256=iTJ9EQmHbzXL34foNFto-hqdC0Kq1pF1mNxqv0jqhBs,25362
 nucliadb/common/cluster/settings.py,sha256=TMoym-cZsQ2soWfLAce0moSa2XncttQyhahL43LrWTo,3384
-nucliadb/common/cluster/utils.py,sha256=Vu0f6026EBELe-ff3d5B0ihD0HtjSWKDCr4dy7LmKqg,5848
+nucliadb/common/cluster/utils.py,sha256=7nQvnVFxM4XV7J560R8hUA-GPzrgD19UlQxHrl4mZUc,4687
 nucliadb/common/cluster/standalone/__init__.py,sha256=itSI7dtTwFP55YMX4iK7JzdMHS5CQVUiB1XzQu4UBh8,833
 nucliadb/common/cluster/standalone/utils.py,sha256=af3r-x_GF7A6dwIAhZLR-r-SZQEVxsFrDKeMfUTA6G0,1908
-nucliadb/common/context/__init__.py,sha256=gPwX1cZHPV9r-fLl3ZCZMYBTAwo6Q_xvSlQtd4qPd4c,3592
+nucliadb/common/context/__init__.py,sha256=ZLUvKuIPaolKeA3aeZa2JcHwCIaEauNu8WpdKsiINXo,3354
 nucliadb/common/context/fastapi.py,sha256=j3HZ3lne6mIfw1eEar2het8RWzv6UruUZpXaKieSLOs,1527
 nucliadb/common/datamanagers/__init__.py,sha256=U1cg-KvqbfzN5AnL_tFFrERmPb81w_0MNiTmxObmla4,2062
 nucliadb/common/datamanagers/atomic.py,sha256=DU7RihO8WaGNuh_GTEpQ-8hkoinY5GSpNSzVpLsZh30,3225
@@ -101,7 +101,7 @@ nucliadb/export_import/models.py,sha256=dbjScNkiMRv4X3Ktudy1JRliD25bfoDTy3JmEZgQ
 nucliadb/export_import/tasks.py,sha256=fpCBeFYPReyLIdk38LDM9Tpnw_VczeMrobT4n1RAIp4,2507
 nucliadb/export_import/utils.py,sha256=zrNrkkc9i3uT-R6Ju4J_0WNrzayln3KuQFCz-_qIaIA,19613
 nucliadb/ingest/__init__.py,sha256=fsw3C38VP50km3R-nHL775LNGPpJ4JxqXJ2Ib1f5SqE,1011
-nucliadb/ingest/app.py,sha256=kZ8RgZNumzVEssVGnSZIh8jyCYKYotGhxBYz3EWx6Cc,7361
+nucliadb/ingest/app.py,sha256=n5PlZyrf-bJsrc6jMRSO4DTjGyZLb4q6EA6Y4x6TYfQ,7028
 nucliadb/ingest/cache.py,sha256=w7jMMzamOmQ7gwXna6Dqm6isRNBVv6l5BTBlTxaYWjE,1005
 nucliadb/ingest/partitions.py,sha256=2NIhMYbNT0TNBL6bX1UMSi7vxFGICstCKEqsB0TXHOE,2410
 nucliadb/ingest/processing.py,sha256=gg1DqbMFwqdOsmCSGsZc2abRdYz86xOZJun9vrHOCzs,20618
@@ -336,8 +336,8 @@ nucliadb/writer/tus/local.py,sha256=7jYa_w9b-N90jWgN2sQKkNcomqn6JMVBOVeDOVYJHto,
 nucliadb/writer/tus/s3.py,sha256=vF0NkFTXiXhXq3bCVXXVV-ED38ECVoUeeYViP8uMqcU,8357
 nucliadb/writer/tus/storage.py,sha256=ToqwjoYnjI4oIcwzkhha_MPxi-k4Jk3Lt55zRwaC1SM,2903
 nucliadb/writer/tus/utils.py,sha256=MSdVbRsRSZVdkaum69_0wku7X3p5wlZf4nr6E0GMKbw,2556
-nucliadb-6.2.1.post3332.dist-info/METADATA,sha256=Gj3sfBbSi6LtLdhJofzr_eR6V_AGI7LBFYuxDkNcfac,4291
-nucliadb-6.2.1.post3332.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-nucliadb-6.2.1.post3332.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
-nucliadb-6.2.1.post3332.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
-nucliadb-6.2.1.post3332.dist-info/RECORD,,
+nucliadb-6.2.1.post3365.dist-info/METADATA,sha256=g_650xn-beF0Qeo5H3cP5BlhSObncN0TJz-QOgA8VDk,4291
+nucliadb-6.2.1.post3365.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+nucliadb-6.2.1.post3365.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
+nucliadb-6.2.1.post3365.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
+nucliadb-6.2.1.post3365.dist-info/RECORD,,