nucliadb 6.3.5.post3865__py3-none-any.whl → 6.3.5.post3886__py3-none-any.whl

nucliadb/common/datamanagers/cluster.py

@@ -38,6 +38,16 @@ async def get_kb_shards(
     return await get_kv_pb(txn, key, writer_pb2.Shards, for_update=for_update)
 
 
+async def is_kb_shard(txn: Transaction, *, kbid: str, shard_id: str, for_update: bool = False) -> bool:
+    shards = await get_kb_shards(txn, kbid=kbid, for_update=for_update)
+    if shards is None:
+        return False
+    for shard in shards.shards:
+        if shard.shard == shard_id:
+            return True
+    return False
+
+
 async def update_kb_shards(txn: Transaction, *, kbid: str, shards: writer_pb2.Shards) -> None:
     key = KB_SHARDS.format(kbid=kbid)
     await txn.set(key, shards.SerializeToString())

nucliadb/purge/orphan_shards.py

@@ -20,7 +20,6 @@
 import argparse
 import asyncio
 import importlib.metadata
-from dataclasses import dataclass
 from typing import Optional
 
 from grpc.aio import AioRpcError
@@ -28,8 +27,6 @@ from grpc.aio import AioRpcError
 from nucliadb.common import datamanagers
 from nucliadb.common.cluster import manager
 from nucliadb.common.cluster.base import AbstractIndexNode
-from nucliadb.common.cluster.exceptions import ShardsNotFound
-from nucliadb.common.cluster.manager import KBShardManager
 from nucliadb.common.cluster.utils import setup_cluster, teardown_cluster
 from nucliadb.common.maindb.driver import Driver
 from nucliadb.common.maindb.utils import setup_driver, teardown_driver
@@ -38,17 +35,13 @@ from nucliadb.ingest import logger
 from nucliadb_telemetry import errors
 from nucliadb_telemetry.logs import setup_logging
 
-
-@dataclass
-class ShardLocation:
-    kbid: str
-    node_id: str
+ShardKb = str
 
 
 UNKNOWN_KB = "unknown"
 
 
-async def detect_orphan_shards(driver: Driver) -> dict[str, ShardLocation]:
+async def detect_orphan_shards(driver: Driver) -> dict[str, ShardKb]:
     """Detect orphan shards in the system. An orphan shard is one indexed but
     not referenced for any stored KB.
 
@@ -58,59 +51,63 @@ async def detect_orphan_shards(driver: Driver) -> dict[str, ShardLocation]:
     indexed_shards = await _get_indexed_shards()
     stored_shards = await _get_stored_shards(driver)
 
-    # Log an error in case we found a shard stored but not indexed, this should
-    # never happen as shards are created in the index node and then stored in
-    # maindb
+    # In normal conditions, this should never happen as shards are created first
+    # in the index and then in maindb. However, if a new shard has been created
+    # between index and maindb scans, we can also see it here
     not_indexed_shards = stored_shards.keys() - indexed_shards.keys()
     for shard_id in not_indexed_shards:
-        location = stored_shards[shard_id]
-        logger.error(
-            "Found a shard on maindb not indexed in the index nodes",
+        kbid = stored_shards[shard_id]
+        logger.info(
+            "Found a shard on maindb not indexed in the index nodes. "
+            "This can be either a shard not indexed (error) or a brand new shard. "
+            "If you run purge and find it again, it's probably an error",
             extra={
                 "shard_id": shard_id,
-                "kbid": location.kbid,
-                "node_id": location.node_id,
+                "kbid": kbid,
             },
         )
 
     orphan_shard_ids = indexed_shards.keys() - stored_shards.keys()
-    orphan_shards: dict[str, ShardLocation] = {}
+    orphan_shards: dict[str, ShardKb] = {}
     node = manager.get_nidx_fake_node()
-    async with datamanagers.with_ro_transaction() as txn:
-        for shard_id in orphan_shard_ids:
-            kbid = await _get_kbid(node, shard_id) or UNKNOWN_KB
-            # Shards with knwon KB ids can be checked and ignore those comming from
-            # an ongoing migration/rollover
-            if kbid != UNKNOWN_KB:
-                skip = await datamanagers.rollover.is_rollover_shard(txn, kbid=kbid, shard_id=shard_id)
+    for shard_id in orphan_shard_ids:
+        kbid = await _get_kbid(node, shard_id) or UNKNOWN_KB
+        # Shards with knwon KB ids can be checked and ignore those comming from
+        # an ongoing migration/rollover (ongoing or finished)
+        if kbid != UNKNOWN_KB:
+            async with datamanagers.with_ro_transaction() as txn:
+                skip = await datamanagers.rollover.is_rollover_shard(
+                    txn, kbid=kbid, shard_id=shard_id
+                ) or await datamanagers.cluster.is_kb_shard(txn, kbid=kbid, shard_id=shard_id)
                 if skip:
                     continue
-            orphan_shards[shard_id] = ShardLocation(kbid=kbid, node_id="nidx")
+        orphan_shards[shard_id] = kbid
+
+    for shard_id in orphan_shard_ids:
+        kbid = await _get_kbid(node, shard_id) or UNKNOWN_KB
+        orphan_shards[shard_id] = kbid
     return orphan_shards
 
 
-async def _get_indexed_shards() -> dict[str, ShardLocation]:
+async def _get_indexed_shards() -> dict[str, ShardKb]:
     nidx = manager.get_nidx_fake_node()
     shards = await nidx.list_shards()
-    return {shard_id: ShardLocation(kbid=UNKNOWN_KB, node_id="nidx") for shard_id in shards}
+    return {shard_id: UNKNOWN_KB for shard_id in shards}
 
 
-async def _get_stored_shards(driver: Driver) -> dict[str, ShardLocation]:
-    stored_shards: dict[str, ShardLocation] = {}
-    shards_manager = KBShardManager()
+async def _get_stored_shards(driver: Driver) -> dict[str, ShardKb]:
+    stored_shards: dict[str, ShardKb] = {}
 
     async with driver.transaction(read_only=True) as txn:
         async for kbid, _ in datamanagers.kb.get_kbs(txn):
-            try:
-                kb_shards = await shards_manager.get_shards_by_kbid(kbid)
-            except ShardsNotFound:
+            kb_shards = await datamanagers.cluster.get_kb_shards(txn, kbid=kbid)
+            if kb_shards is None:
                 logger.warning("KB not found while looking for orphan shards", extra={"kbid": kbid})
                 continue
-            else:
-                for shard_object_pb in kb_shards:
-                    stored_shards[shard_object_pb.nidx_shard_id] = ShardLocation(
-                        kbid=kbid, node_id="nidx"
-                    )
+
+            for shard_object_pb in kb_shards.shards:
+                stored_shards[shard_object_pb.nidx_shard_id] = kbid
+
     return stored_shards
 
 
@@ -120,7 +117,7 @@ async def _get_kbid(node: AbstractIndexNode, shard_id: str) -> Optional[str]:
         shard_pb = await node.get_shard(shard_id)
     except AioRpcError as grpc_error:
         logger.error(
-            "Can't get shard while looking for orphans in index nodes, is it broken?",
+            "Can't get shard while looking for orphans in nidx, is there something broken?",
             exc_info=grpc_error,
             extra={
                 "node_id": node.id,
@@ -136,11 +133,11 @@ async def report_orphan_shards(driver: Driver):
     orphan_shards = await detect_orphan_shards(driver)
     logger.info(f"Found {len(orphan_shards)} orphan shards")
     async with driver.transaction(read_only=True) as txn:
-        for shard_id, location in orphan_shards.items():
-            if location.kbid == UNKNOWN_KB:
+        for shard_id, kbid in orphan_shards.items():
+            if kbid == UNKNOWN_KB:
                 msg = "Found orphan shard but could not get KB info"
             else:
-                kb_exists = await datamanagers.kb.exists_kb(txn, kbid=location.kbid)
+                kb_exists = await datamanagers.kb.exists_kb(txn, kbid=kbid)
                 if kb_exists:
                     msg = "Found orphan shard for existing KB"
                 else:
@@ -150,8 +147,7 @@ async def report_orphan_shards(driver: Driver):
                 msg,
                 extra={
                     "shard_id": shard_id,
-                    "kbid": location.kbid,
-                    "node_id": location.node_id,
+                    "kbid": kbid,
                 },
             )
 
@@ -161,13 +157,12 @@ async def purge_orphan_shards(driver: Driver):
     logger.info(f"Found {len(orphan_shards)} orphan shards. Purge starts...")
 
     node = manager.get_nidx_fake_node()
-    for shard_id, location in orphan_shards.items():
+    for shard_id, kbid in orphan_shards.items():
         logger.info(
             "Deleting orphan shard from index node",
             extra={
                 "shard_id": shard_id,
-                "kbid": location.kbid,
-                "node_id": location.node_id,
+                "kbid": kbid,
             },
         )
         await node.delete_shard(shard_id)

{nucliadb-6.3.5.post3865.dist-info → nucliadb-6.3.5.post3886.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nucliadb
-Version: 6.3.5.post3865
+Version: 6.3.5.post3886
 Summary: NucliaDB
 Author-email: Nuclia <nucliadb@nuclia.com>
 License: AGPL
@@ -20,14 +20,14 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: <4,>=3.9
 Description-Content-Type: text/markdown
-Requires-Dist: nucliadb-telemetry[all]>=6.3.5.post3865
-Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.3.5.post3865
-Requires-Dist: nucliadb-protos>=6.3.5.post3865
-Requires-Dist: nucliadb-models>=6.3.5.post3865
-Requires-Dist: nidx-protos>=6.3.5.post3865
+Requires-Dist: nucliadb-telemetry[all]>=6.3.5.post3886
+Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.3.5.post3886
+Requires-Dist: nucliadb-protos>=6.3.5.post3886
+Requires-Dist: nucliadb-models>=6.3.5.post3886
+Requires-Dist: nidx-protos>=6.3.5.post3886
 Requires-Dist: nucliadb-admin-assets>=1.0.0.post1224
 Requires-Dist: nuclia-models>=0.24.2
-Requires-Dist: uvicorn
+Requires-Dist: uvicorn[standard]
 Requires-Dist: argdantic
 Requires-Dist: aiohttp>=3.11.11
 Requires-Dist: lru-dict>=1.1.7

{nucliadb-6.3.5.post3865.dist-info → nucliadb-6.3.5.post3886.dist-info}/RECORD

@@ -72,7 +72,7 @@ nucliadb/common/context/__init__.py,sha256=ZLUvKuIPaolKeA3aeZa2JcHwCIaEauNu8WpdK
 nucliadb/common/context/fastapi.py,sha256=j3HZ3lne6mIfw1eEar2het8RWzv6UruUZpXaKieSLOs,1527
 nucliadb/common/datamanagers/__init__.py,sha256=jksw4pXyXb05SG3EN-BPBrhc1u1Ge_m21PYqD7NYQEs,2118
 nucliadb/common/datamanagers/atomic.py,sha256=WihdtBWQIAuElZQjh1xQ--q5dJowwlkovqsW-OB_t2k,3230
-nucliadb/common/datamanagers/cluster.py,sha256=psTwAWSLj83vhFnC1iJJ6holrolAI4nKos9PuEWspYY,1500
+nucliadb/common/datamanagers/cluster.py,sha256=iU0b7AESm1Yi8Wp3pIKgqixZGNMjeBrxSpvEKsaZKgY,1831
 nucliadb/common/datamanagers/entities.py,sha256=gI-0mbMlqrr9FiyhexEh6czhgYcMxE2s9m4o866EK9o,5340
 nucliadb/common/datamanagers/exceptions.py,sha256=Atz_PP_GGq4jgJaWcAkcRbHBoBaGcC9yJvFteylKtTE,883
 nucliadb/common/datamanagers/fields.py,sha256=9KqBzTssAT68FR5hd17Xu_CSwAYdKFuYic1ITnrfFNc,3971
@@ -170,7 +170,7 @@ nucliadb/migrator/utils.py,sha256=NgUreUvON8_nWEzTxELBMWlfV7E6-6qi-g0DMEbVEz4,28
 nucliadb/models/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/models/responses.py,sha256=qnuOoc7TrVSUnpikfTwHLKez47_DE4mSFzpxrwtqijA,1599
 nucliadb/purge/__init__.py,sha256=UXbto56EWYLwZj6uEc-flQVe3gDDNFtM6EV-aIkryPU,12353
-nucliadb/purge/orphan_shards.py,sha256=AU1Jfc4qtQFasB6dkuGz0a_Zjs-i7liUTf7Xpl8R8ng,7939
+nucliadb/purge/orphan_shards.py,sha256=WSQmVQJQ-rAoQAmypOUJBpDP-9VJchWao3ZLTzHdJ1U,7764
 nucliadb/reader/__init__.py,sha256=C5Efic7WlGm2U2C5WOyquMFbIj2Pojwe_8mwzVYnOzE,1304
 nucliadb/reader/app.py,sha256=Se-BFTE6d1v1msLzQn4q5XIhjnSxa2ckDSHdvm7NRf8,3096
 nucliadb/reader/lifecycle.py,sha256=5jYyzMD1tpIh-OYbQoNMjKZ0-3D9KFnULa3B_Vf2xyY,1740
@@ -355,8 +355,8 @@ nucliadb/writer/tus/local.py,sha256=7jYa_w9b-N90jWgN2sQKkNcomqn6JMVBOVeDOVYJHto,
 nucliadb/writer/tus/s3.py,sha256=vF0NkFTXiXhXq3bCVXXVV-ED38ECVoUeeYViP8uMqcU,8357
 nucliadb/writer/tus/storage.py,sha256=ToqwjoYnjI4oIcwzkhha_MPxi-k4Jk3Lt55zRwaC1SM,2903
 nucliadb/writer/tus/utils.py,sha256=MSdVbRsRSZVdkaum69_0wku7X3p5wlZf4nr6E0GMKbw,2556
-nucliadb-6.3.5.post3865.dist-info/METADATA,sha256=Rc22hxCQhijcEqGOWxspP2qpzON-fTJFhZjRg9VAxzY,4291
-nucliadb-6.3.5.post3865.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-nucliadb-6.3.5.post3865.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
-nucliadb-6.3.5.post3865.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
-nucliadb-6.3.5.post3865.dist-info/RECORD,,
+nucliadb-6.3.5.post3886.dist-info/METADATA,sha256=fshbhRod9yW4otVjd60tvgPUwOAklTc0Wh8VTcp-lfo,4301
+nucliadb-6.3.5.post3886.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+nucliadb-6.3.5.post3886.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
+nucliadb-6.3.5.post3886.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
+nucliadb-6.3.5.post3886.dist-info/RECORD,,