nucliadb 6.3.1.post3546__py3-none-any.whl → 6.3.1.post3557__py3-none-any.whl

migrations/0020_drain_nodes_from_cluster.py

@@ -28,9 +28,6 @@ create new shards in the remaining nodes.
 
 import logging
 
-from nucliadb.common import datamanagers
-from nucliadb.common.cluster.rollover import rollover_kb_index
-from nucliadb.common.cluster.settings import settings as cluster_settings
 from nucliadb.migrator.context import ExecutionContext
 
 logger = logging.getLogger(__name__)
@@ -39,45 +36,4 @@ logger = logging.getLogger(__name__)
 async def migrate(context: ExecutionContext) -> None: ...
 
 
-async def migrate_kb(context: ExecutionContext, kbid: str) -> None:
-    """
-    Rollover KB shards if any of the shards are on the nodes to drain
-    """
-    drain_node_ids = cluster_settings.drain_nodes
-    if len(drain_node_ids) == 0:
-        logger.info("Skipping migration because no drain_nodes are set")
-        return
-
-    if not await kb_has_shards_on_drain_nodes(kbid, drain_node_ids):
-        logger.info(
-            "KB does not have shards on the nodes to drain, skipping rollover",
-            extra={"kbid": kbid},
-        )
-        return
-
-    logger.info("Rolling over affected KB", extra={"kbid": kbid})
-    await rollover_kb_index(context, kbid, drain_nodes=drain_node_ids)
-
-
-async def kb_has_shards_on_drain_nodes(kbid: str, drain_node_ids: list[str]) -> bool:
-    async with datamanagers.with_ro_transaction() as txn:
-        shards = await datamanagers.cluster.get_kb_shards(txn, kbid=kbid)
-        if not shards:
-            logger.warning("Shards object not found", extra={"kbid": kbid})
-            return False
-        shard_in_drain_nodes = False
-        for shard in shards.shards:
-            for replica in shard.replicas:
-                if replica.node in drain_node_ids:
-                    logger.info(
-                        "Shard found in drain nodes, will rollover it",
-                        extra={
-                            "kbid": kbid,
-                            "logical_shard": shard.shard,
-                            "replica_shard_id": replica.shard.id,
-                            "node": replica.node,
-                            "drain_node_ids": drain_node_ids,
-                        },
-                    )
-                    shard_in_drain_nodes = True
-        return shard_in_drain_nodes
+async def migrate_kb(context: ExecutionContext, kbid: str) -> None: ...

nucliadb/backups/const.py

@@ -19,6 +19,9 @@
 #
 
 
+from nucliadb.tasks.utils import NatsConsumer, NatsStream
+
+
 class MaindbKeys:
     METADATA = "kbs/{kbid}/backups/{backup_id}"
     LAST_RESTORED = "kbs/{kbid}/backup/{backup_id}/last_restored"
@@ -41,9 +44,8 @@ class BackupFinishedStream:
     subject = "backups.creation_finished"
 
 
-class BackupsNatsStream:
-    name = "ndb-backups"
-    stream_subjects = ["ndb-backups.>"]
-    create_subject = "ndb-backups.create"
-    delete_subject = "ndb-backups.delete"
-    restore_subject = "ndb-backups.restore"
+class BackupsNatsConfig:
+    stream = NatsStream(name="ndb-backups", subjects=["ndb-backups.>"])
+    create_consumer = NatsConsumer(subject="ndb-backups.create", group="ndb-backups-create")
+    delete_consumer = NatsConsumer(subject="ndb-backups.delete", group="ndb-backups-delete")
+    restore_consumer = NatsConsumer(subject="ndb-backups.restore", group="ndb-backups-restore")

nucliadb/backups/tasks.py

@@ -19,7 +19,7 @@
 #
 from typing import Awaitable, Callable
 
-from nucliadb.backups.const import BackupsNatsStream
+from nucliadb.backups.const import BackupsNatsConfig
 from nucliadb.backups.create import backup_kb_task
 from nucliadb.backups.delete import delete_backup
 from nucliadb.backups.models import CreateBackupRequest, DeleteBackupRequest, RestoreBackupRequest
@@ -33,9 +33,8 @@ from nucliadb.tasks.producer import NatsTaskProducer
 def creator_consumer() -> NatsTaskConsumer[CreateBackupRequest]:
     consumer: NatsTaskConsumer = create_consumer(
         name="backup_creator",
-        stream=BackupsNatsStream.name,
-        stream_subjects=BackupsNatsStream.stream_subjects,
-        consumer_subject=BackupsNatsStream.create_subject,
+        stream=BackupsNatsConfig.stream,
+        consumer=BackupsNatsConfig.create_consumer,
         callback=backup_kb_task,
         msg_type=CreateBackupRequest,
         max_concurrent_messages=10,
@@ -46,9 +45,8 @@ def creator_consumer() -> NatsTaskConsumer[CreateBackupRequest]:
 async def create(kbid: str, backup_id: str) -> None:
     producer: NatsTaskProducer[CreateBackupRequest] = create_producer(
         name="backup_creator",
-        stream=BackupsNatsStream.name,
-        stream_subjects=BackupsNatsStream.stream_subjects,
-        producer_subject=BackupsNatsStream.create_subject,
+        stream=BackupsNatsConfig.stream,
+        producer_subject=BackupsNatsConfig.create_consumer.subject,
         msg_type=CreateBackupRequest,
     )
     msg = CreateBackupRequest(
@@ -61,9 +59,8 @@ async def create(kbid: str, backup_id: str) -> None:
 def restorer_consumer() -> NatsTaskConsumer[RestoreBackupRequest]:
     consumer: NatsTaskConsumer = create_consumer(
         name="backup_restorer",
-        stream=BackupsNatsStream.name,
-        stream_subjects=BackupsNatsStream.stream_subjects,
-        consumer_subject=BackupsNatsStream.restore_subject,
+        stream=BackupsNatsConfig.stream,
+        consumer=BackupsNatsConfig.restore_consumer,
         callback=restore_kb_task,
         msg_type=RestoreBackupRequest,
         max_concurrent_messages=10,
@@ -74,9 +71,8 @@ def restorer_consumer() -> NatsTaskConsumer[RestoreBackupRequest]:
 async def restore(kbid: str, backup_id: str) -> None:
     producer: NatsTaskProducer[RestoreBackupRequest] = create_producer(
         name="backup_restorer",
-        stream=BackupsNatsStream.name,
-        stream_subjects=BackupsNatsStream.stream_subjects,
-        producer_subject=BackupsNatsStream.restore_subject,
+        stream=BackupsNatsConfig.stream,
+        producer_subject=BackupsNatsConfig.restore_consumer.subject,
         msg_type=RestoreBackupRequest,
     )
     msg = RestoreBackupRequest(
@@ -89,9 +85,8 @@ async def restore(kbid: str, backup_id: str) -> None:
 def deleter_consumer() -> NatsTaskConsumer[DeleteBackupRequest]:
     consumer: NatsTaskConsumer = create_consumer(
         name="backup_deleter",
-        stream=BackupsNatsStream.name,
-        stream_subjects=BackupsNatsStream.stream_subjects,
-        consumer_subject=BackupsNatsStream.delete_subject,
+        stream=BackupsNatsConfig.stream,
+        consumer=BackupsNatsConfig.delete_consumer,
         callback=delete_backup,
         msg_type=DeleteBackupRequest,
         max_concurrent_messages=2,
@@ -102,9 +97,8 @@ def deleter_consumer() -> NatsTaskConsumer[DeleteBackupRequest]:
 async def delete(backup_id: str) -> None:
     producer: NatsTaskProducer[DeleteBackupRequest] = create_producer(
         name="backup_deleter",
-        stream=BackupsNatsStream.name,
-        stream_subjects=BackupsNatsStream.stream_subjects,
-        producer_subject=BackupsNatsStream.delete_subject,
+        stream=BackupsNatsConfig.stream,
+        producer_subject=BackupsNatsConfig.delete_consumer.subject,
         msg_type=DeleteBackupRequest,
     )
     msg = DeleteBackupRequest(

nucliadb/common/cluster/settings.py

@@ -24,12 +24,6 @@ from pydantic import Field
 from pydantic_settings import BaseSettings
 
 
-class ClusterDiscoveryMode(str, enum.Enum):
-    MANUAL = "manual"
-    KUBERNETES = "kubernetes"
-    SINGLE_NODE = "single_node"
-
-
 class StandaloneNodeRole(enum.Enum):
     ALL = "all"
     INDEX = "index"
@@ -39,53 +33,20 @@ class StandaloneNodeRole(enum.Enum):
 class Settings(BaseSettings):
     data_path: str = "./data/node"
     standalone_mode: bool = False
-    standalone_node_port: int = Field(
-        default=10009,
-        title="Standalone node port",
-        description="Port to use for standalone nodes to communication with each other through",
-    )
     standalone_node_role: StandaloneNodeRole = StandaloneNodeRole.ALL
 
-    node_replicas: int = 2
-
-    node_writer_port: int = 10000
-    node_reader_port: int = 10001
-
-    # Only for testing purposes
-    writer_port_map: dict[str, int] = {}
-    reader_port_map: dict[str, int] = {}
-
-    # Node limits
+    # Index limits
     max_shard_paragraphs: int = Field(
         default=500_000,
         title="Max shard paragraphs",
         description="Maximum number of paragraphs to target per shard",
     )
-    max_node_replicas: int = Field(
-        default=800,
-        title="Max node replicas",
-        description="Maximum number of shard replicas a single node will manage",
-    )
     max_resource_paragraphs: int = Field(
         default=50_000,
         title="Max paragraphs per resource",
         description="Maximum number of paragraphs allowed on a single resource",
     )
 
-    drain_nodes: list[str] = Field(
-        default=[],
-        title="Drain nodes",
-        description="List of node IDs to ignore when creating new shards. It is used for draining nodes from a cluster. Example: ['1bf3bfe7-e164-4a19-a4d9-41372fc15aca',]",  # noqa: E501
-    )
-
-    local_reader_threads: int = 5
-    local_writer_threads: int = 5
-
-    cluster_discovery_mode: ClusterDiscoveryMode = ClusterDiscoveryMode.KUBERNETES
-    cluster_discovery_kubernetes_namespace: str = "nucliadb"
-    cluster_discovery_kubernetes_selector: str = "appType=node"
-    cluster_discovery_manual_addresses: list[str] = []
-
     nidx_api_address: Optional[str] = Field(default=None, description="NIDX gRPC API address")
     nidx_searcher_address: Optional[str] = Field(
         default=None, description="NIDX gRPC searcher API address"

nucliadb/export_import/tasks.py

@@ -24,15 +24,36 @@ from nucliadb.export_import.models import NatsTaskMessage
 from nucliadb.tasks import create_consumer, create_producer
 from nucliadb.tasks.consumer import NatsTaskConsumer
 from nucliadb.tasks.producer import NatsTaskProducer
-from nucliadb_utils import const
+from nucliadb.tasks.utils import NatsConsumer, NatsStream
+
+
+class ExportsNatsConfig:
+    stream = NatsStream(
+        name="ndb-exports",
+        subjects=["ndb-exports"],
+    )
+    consumer = NatsConsumer(
+        subject="ndb-exports",
+        group="ndb-exports",
+    )
+
+
+class ImportsNatsConfig:
+    stream = NatsStream(
+        name="ndb-imports",
+        subjects=["ndb-imports"],
+    )
+    consumer = NatsConsumer(
+        subject="ndb-imports",
+        group="ndb-imports",
+    )
 
 
 def get_exports_consumer() -> NatsTaskConsumer[NatsTaskMessage]:
     return create_consumer(
         name="exports_consumer",
-        stream=const.Streams.KB_EXPORTS.name,
-        stream_subjects=[const.Streams.KB_EXPORTS.subject],
-        consumer_subject=const.Streams.KB_EXPORTS.subject,
+        stream=ExportsNatsConfig.stream,
+        consumer=ExportsNatsConfig.consumer,
         callback=export_kb_to_blob_storage,
         msg_type=NatsTaskMessage,
         max_concurrent_messages=10,
@@ -42,9 +63,8 @@ def get_exports_consumer() -> NatsTaskConsumer[NatsTaskMessage]:
 async def get_exports_producer(context: ApplicationContext) -> NatsTaskProducer[NatsTaskMessage]:
     producer = create_producer(
         name="exports_producer",
-        stream=const.Streams.KB_EXPORTS.name,
-        stream_subjects=[const.Streams.KB_EXPORTS.subject],
-        producer_subject=const.Streams.KB_EXPORTS.subject,
+        stream=ExportsNatsConfig.stream,
+        producer_subject=ExportsNatsConfig.consumer.subject,
         msg_type=NatsTaskMessage,
     )
     await producer.initialize(context)
@@ -54,9 +74,8 @@ async def get_exports_producer(context: ApplicationContext) -> NatsTaskProducer[
 def get_imports_consumer() -> NatsTaskConsumer[NatsTaskMessage]:
     return create_consumer(
         name="imports_consumer",
-        stream=const.Streams.KB_IMPORTS.name,
-        stream_subjects=[const.Streams.KB_IMPORTS.subject],
-        consumer_subject=const.Streams.KB_IMPORTS.subject,
+        stream=ImportsNatsConfig.stream,
+        consumer=ImportsNatsConfig.consumer,
         callback=import_kb_from_blob_storage,
         msg_type=NatsTaskMessage,
         max_concurrent_messages=10,
@@ -66,9 +85,8 @@ def get_imports_consumer() -> NatsTaskConsumer[NatsTaskMessage]:
 async def get_imports_producer(context: ApplicationContext) -> NatsTaskProducer[NatsTaskMessage]:
     producer = create_producer(
         name="imports_producer",
-        stream=const.Streams.KB_IMPORTS.name,
-        stream_subjects=[const.Streams.KB_IMPORTS.subject],
-        producer_subject=const.Streams.KB_IMPORTS.subject,
+        stream=ImportsNatsConfig.stream,
+        producer_subject=ImportsNatsConfig.consumer.subject,
         msg_type=NatsTaskMessage,
     )
     await producer.initialize(context)

nucliadb/standalone/config.py

@@ -22,7 +22,7 @@ import logging
 import os
 
 from nucliadb.common.cluster.settings import StandaloneNodeRole
-from nucliadb.standalone.settings import Settings, StandaloneDiscoveryMode
+from nucliadb.standalone.settings import Settings
 
 logger = logging.getLogger(__name__)
 
@@ -76,7 +76,6 @@ def config_nucliadb(nucliadb_args: Settings):
     use some specific settings.
     """
 
-    from nucliadb.common.cluster.settings import ClusterDiscoveryMode
     from nucliadb.common.cluster.settings import settings as cluster_settings
     from nucliadb.ingest.settings import settings as ingest_settings
     from nucliadb.train.settings import settings as train_settings
@@ -91,14 +90,8 @@ def config_nucliadb(nucliadb_args: Settings):
 
     cluster_settings.standalone_mode = True
     cluster_settings.data_path = nucliadb_args.data_path
-    cluster_settings.standalone_node_port = nucliadb_args.standalone_node_port
     cluster_settings.standalone_node_role = nucliadb_args.standalone_node_role
 
-    if nucliadb_args.cluster_discovery_mode == StandaloneDiscoveryMode.DEFAULT:
-        # default for standalone is single node
-        cluster_settings.cluster_discovery_mode = ClusterDiscoveryMode.SINGLE_NODE
-        cluster_settings.node_replicas = 1
-
     ingest_settings.nuclia_partitions = 1
     ingest_settings.replica_number = 0
     ingest_settings.partitions = ["1"]

nucliadb/standalone/run.py

@@ -99,10 +99,7 @@ def run():
         "Admin UI": f"http://{settings.http_host}:{settings.http_port}/admin",
         "Key-value backend": ingest_settings.driver.value,
         "Blob storage backend": storage_settings.file_backend.value,
-        "Cluster discovery mode": cluster_settings.cluster_discovery_mode.value,
-        "Node replicas": cluster_settings.node_replicas,
         "Index data path": os.path.realpath(cluster_settings.data_path),
-        "Node port": cluster_settings.standalone_node_port,
         "Auth policy": settings.auth_policy.value,
         "Node role": cluster_settings.standalone_node_role.value,
     }

nucliadb/standalone/settings.py

@@ -30,13 +30,6 @@ from nucliadb_utils.settings import StorageSettings
 from nucliadb_utils.storages.settings import Settings as ExtendedStorageSettings
 
 
-class StandaloneDiscoveryMode(Enum):
-    DEFAULT = "default"
-    MANUAL = "manual"
-    KUBERNETES = "kubernetes"
-    SINGLE_NODE = "single_node"
-
-
 class AuthPolicy(Enum):
     UPSTREAM_NAIVE = "upstream_naive"
     UPSTREAM_AUTH_HEADER = "upstream_auth_header"
@@ -60,8 +53,6 @@ class Settings(DriverSettings, StorageSettings, ExtendedStorageSettings):
     http_port: int = pydantic.Field(default=8080, description="HTTP Port")
     ingest_grpc_port: int = pydantic.Field(default=8030, description="Ingest GRPC Port")
     train_grpc_port: int = pydantic.Field(default=8031, description="Train GRPC Port")
-    standalone_node_port: int = pydantic.Field(default=10009, description="Node GRPC Port")
-
     auth_policy: AuthPolicy = pydantic.Field(
         default=AuthPolicy.UPSTREAM_NAIVE,
         description="""Auth policy to use for http requests.
@@ -111,8 +102,6 @@ Examples:
         description="JWK key used for temporary token generation and validation.",
     )
 
-    cluster_discovery_mode: StandaloneDiscoveryMode = StandaloneDiscoveryMode.DEFAULT
-
     fork: bool = pydantic.Field(default=False, description="Fork process on startup")
 
     # Standalone logging settings

nucliadb/tasks/consumer.py

@@ -28,7 +28,7 @@ from nats.aio.client import Msg
 from nucliadb.common.context import ApplicationContext
 from nucliadb.tasks.logger import logger
 from nucliadb.tasks.models import Callback, MsgType
-from nucliadb.tasks.utils import create_nats_stream_if_not_exists
+from nucliadb.tasks.utils import NatsConsumer, NatsStream, create_nats_stream_if_not_exists
 from nucliadb_telemetry import errors
 from nucliadb_utils.nats import MessageProgressUpdater
 from nucliadb_utils.settings import nats_consumer_settings
@@ -40,17 +40,15 @@ class NatsTaskConsumer(Generic[MsgType]):
     def __init__(
         self,
         name: str,
-        stream: str,
-        stream_subjects: list[str],
-        consumer_subject: str,
+        stream: NatsStream,
+        consumer: NatsConsumer,
         callback: Callback,
         msg_type: Type[MsgType],
         max_concurrent_messages: Optional[int] = None,
     ):
         self.name = name
         self.stream = stream
-        self.stream_subjects = stream_subjects
-        self.consumer_subject = consumer_subject
+        self.consumer = consumer
         self.callback = callback
         self.msg_type = msg_type
         self.max_concurrent_messages = max_concurrent_messages
@@ -61,7 +59,7 @@ class NatsTaskConsumer(Generic[MsgType]):
     async def initialize(self, context: ApplicationContext):
         self.context = context
         await create_nats_stream_if_not_exists(
-            context, stream_name=self.stream, subjects=self.stream_subjects
+            context, stream_name=self.stream.name, subjects=self.stream.subjects
         )
         await self._setup_nats_subscription()
         self.initialized = True
@@ -80,17 +78,15 @@ class NatsTaskConsumer(Generic[MsgType]):
 
     async def _setup_nats_subscription(self):
         # Nats push consumer
-        stream = self.stream
-        subject = group = self.consumer_subject
         max_ack_pending = (
             self.max_concurrent_messages
             if self.max_concurrent_messages
             else nats_consumer_settings.nats_max_ack_pending
         )
         self.subscription = await self.context.nats_manager.subscribe(
-            subject=subject,
-            queue=group,
-            stream=stream,
+            subject=self.consumer.subject,
+            queue=self.consumer.group,
+            stream=self.stream.name,
             cb=self._subscription_worker_as_task,
             subscription_lost_cb=self._setup_nats_subscription,
             manual_ack=True,
@@ -103,7 +99,7 @@ class NatsTaskConsumer(Generic[MsgType]):
             ),
         )
         logger.info(
-            f"Subscribed to {subject} on stream {stream}",
+            f"Subscribed {self.consumer.group} to {self.consumer.subject} on stream {self.stream.name}",
             extra={"consumer_name": self.name},
         )
 
@@ -178,9 +174,8 @@ class NatsTaskConsumer(Generic[MsgType]):
 
 def create_consumer(
     name: str,
-    stream: str,
-    stream_subjects: list[str],
-    consumer_subject: str,
+    stream: NatsStream,
+    consumer: NatsConsumer,
     callback: Callback,
     msg_type: Type[MsgType],
     max_concurrent_messages: Optional[int] = None,
@@ -188,13 +183,11 @@ def create_consumer(
     """
     Returns a non-initialized consumer
     """
-    consumer = NatsTaskConsumer(
+    return NatsTaskConsumer(
         name=name,
         stream=stream,
-        stream_subjects=stream_subjects,
-        consumer_subject=consumer_subject,
+        consumer=consumer,
         callback=callback,
         msg_type=msg_type,
         max_concurrent_messages=max_concurrent_messages,
     )
-    return consumer

nucliadb/tasks/producer.py

@@ -22,7 +22,7 @@ from typing import Generic, Optional, Type
 from nucliadb.common.context import ApplicationContext
 from nucliadb.tasks.logger import logger
 from nucliadb.tasks.models import MsgType
-from nucliadb.tasks.utils import create_nats_stream_if_not_exists
+from nucliadb.tasks.utils import NatsStream, create_nats_stream_if_not_exists
 from nucliadb_telemetry import errors
 
 
@@ -30,14 +30,12 @@ class NatsTaskProducer(Generic[MsgType]):
     def __init__(
         self,
         name: str,
-        stream: str,
-        stream_subjects: list[str],
+        stream: NatsStream,
         producer_subject: str,
         msg_type: Type[MsgType],
     ):
         self.name = name
         self.stream = stream
-        self.stream_subjects = stream_subjects
         self.producer_subject = producer_subject
         self.msg_type = msg_type
         self.context: Optional[ApplicationContext] = None
@@ -47,8 +45,8 @@ class NatsTaskProducer(Generic[MsgType]):
         self.context = context
         await create_nats_stream_if_not_exists(
             self.context,
-            self.stream,
-            subjects=self.stream_subjects,
+            self.stream.name,
+            subjects=self.stream.subjects,
         )
         self.initialized = True
 
@@ -81,8 +79,7 @@ class NatsTaskProducer(Generic[MsgType]):
 
 def create_producer(
     name: str,
-    stream: str,
-    stream_subjects: list[str],
+    stream: NatsStream,
     producer_subject: str,
     msg_type: Type[MsgType],
 ) -> NatsTaskProducer[MsgType]:
@@ -92,7 +89,6 @@ def create_producer(
     producer = NatsTaskProducer[MsgType](
         name=name,
         stream=stream,
-        stream_subjects=stream_subjects,
         producer_subject=producer_subject,
         msg_type=msg_type,
     )

nucliadb/tasks/utils.py

@@ -18,6 +18,8 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 
+from dataclasses import dataclass
+
 import nats
 
 from nucliadb.common.context import ApplicationContext
@@ -31,3 +33,19 @@ async def create_nats_stream_if_not_exists(
         await js.stream_info(stream_name)
     except nats.js.errors.NotFoundError:
         await js.add_stream(name=stream_name, subjects=subjects)
+
+
+@dataclass
+class NatsStream:
+    name: str
+    subjects: list[str]
+
+
+@dataclass
+class NatsConsumer:
+    """
+    NOTE: groups can't contain '.', '*' or '>' characters.
+    """
+
+    subject: str
+    group: str

{nucliadb-6.3.1.post3546.dist-info → nucliadb-6.3.1.post3557.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: nucliadb
-Version: 6.3.1.post3546
+Version: 6.3.1.post3557
 Summary: NucliaDB
 Author-email: Nuclia <nucliadb@nuclia.com>
 License: AGPL
@@ -20,11 +20,11 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: <4,>=3.9
 Description-Content-Type: text/markdown
-Requires-Dist: nucliadb-telemetry[all]>=6.3.1.post3546
-Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.3.1.post3546
-Requires-Dist: nucliadb-protos>=6.3.1.post3546
-Requires-Dist: nucliadb-models>=6.3.1.post3546
-Requires-Dist: nidx-protos>=6.3.1.post3546
+Requires-Dist: nucliadb-telemetry[all]>=6.3.1.post3557
+Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.3.1.post3557
+Requires-Dist: nucliadb-protos>=6.3.1.post3557
+Requires-Dist: nucliadb-models>=6.3.1.post3557
+Requires-Dist: nidx-protos>=6.3.1.post3557
 Requires-Dist: nucliadb-admin-assets>=1.0.0.post1224
 Requires-Dist: nuclia-models>=0.24.2
 Requires-Dist: uvicorn

{nucliadb-6.3.1.post3546.dist-info → nucliadb-6.3.1.post3557.dist-info}/RECORD

@@ -16,7 +16,7 @@ migrations/0016_upgrade_to_paragraphs_v2.py,sha256=9eepvzme-nb_mw6rwIdjjJzbFcpOq
 migrations/0017_multiple_writable_shards.py,sha256=HSi-eXXI0kO9sLgunUMuVpFnYMjVhWTVfDS_lIjlkuM,2095
 migrations/0018_purge_orphan_kbslugs.py,sha256=ztEOAjqlWVagv1UMg_sOm8HaW6S9FoDOIg7-rZ05tro,2177
 migrations/0019_upgrade_to_paragraphs_v3.py,sha256=zP13_IKE7u4ox2gyc493L2_ewyiPlr7Csn5K6n5eylI,2479
-migrations/0020_drain_nodes_from_cluster.py,sha256=cgm_72kH57QiBbPx17Judn7Wp5hQnKn6UW_1Z37_8s8,3269
+migrations/0020_drain_nodes_from_cluster.py,sha256=BeECAI0T8u14M2U5USl1fFNcsfmdMerNhisolYQN_eA,1411
 migrations/0021_overwrite_vectorsets_key.py,sha256=O6nb2a7kDFX9I3XFrVtudCUyKptpGyv2_GYvcvbQOI8,1583
 migrations/0022_fix_paragraph_deletion_bug.py,sha256=-tH342VXF-8xwc_h3P1cYaUtTT1wHSGf7ZoeVEpnaYs,1422
 migrations/0023_backfill_pg_catalog.py,sha256=gw22pU5cAtg2a7n7xVaVqT2itjAoDMNtzWwTllwqIvg,2993
@@ -40,13 +40,13 @@ nucliadb/metrics_exporter.py,sha256=6u0geEYFxgE5I2Fhl_sxsvGN-ZkaFZNGutSXwrzrsVs,
 nucliadb/openapi.py,sha256=wDiw0dVEvTpJvbatkJ0JZLkKm9RItZT5PWRHjqRfqTA,2272
 nucliadb/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nucliadb/backups/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
-nucliadb/backups/const.py,sha256=llb5TaC53Ce6BMmlPKPUrVhVGl7uQrqv_Vle-P4GET4,1673
+nucliadb/backups/const.py,sha256=9vPAhLxQO_gNAjSdPxWuv3V66s9WcdpjOQ89CZlfmuk,1894
 nucliadb/backups/create.py,sha256=AM_nC7TgHOX0EFGaTXClS28jBSK28fHrKNZi14z2wek,10442
 nucliadb/backups/delete.py,sha256=1rnBhVUGYYZJXSZUrrgYMDZ5NyswEWkIA-G-crRCyHk,2404
 nucliadb/backups/models.py,sha256=-hITU4Mv6AxePu12toBu_fjpEv6vVGcwNVxV22O9jQA,1273
 nucliadb/backups/restore.py,sha256=xhslVvTf4H8VmDucZpjrEFpKj6csPIWBadCPMVJYKQ8,9703
 nucliadb/backups/settings.py,sha256=SyzsInj1BRbBI0atg5IXWbMbOZ_eVg4eSQ3IcnUhCxQ,1357
-nucliadb/backups/tasks.py,sha256=QgGDBBWsTpD-jLcrRP-C897Zo1qvc_ux1b4L-SlCnd8,4530
+nucliadb/backups/tasks.py,sha256=4_kOVJ2yCwMvDEpzJgTuTt75TNlpq5woyw9sTAcaSkw,4194
 nucliadb/backups/utils.py,sha256=ayDaxfWP5cPnAkQH-tF4M6cnowsPQgU2ljYz_iL1CbE,1249
 nucliadb/common/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/common/constants.py,sha256=QpigxJh_CtD85Evy0PtV5cVq6x0U_f9xfIcXz1ymkUg,869
@@ -58,11 +58,10 @@ nucliadb/common/cluster/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIX
 nucliadb/common/cluster/base.py,sha256=kklDqyvsubNX0W494ttl9f3E58lGaX6AXqAd8XX8ZHE,5522
 nucliadb/common/cluster/exceptions.py,sha256=t7v_l93t44l2tQpdQXgO_w-c4YZRcaayOz1A2i0w4RQ,1258
 nucliadb/common/cluster/grpc_node_dummy.py,sha256=LxONv0mhDFhx7mI91qqGfQlQ-R0qOGDYaxhXoBHLXaE,3548
-nucliadb/common/cluster/index_node.py,sha256=g38H1kiAliF3Y6et_CWYInpn_xPxf7THAFJ7RtgLNZo,3246
 nucliadb/common/cluster/manager.py,sha256=KIzqAYGgdVK3GicJ9LdLoei8arWZ7H60imbc32USPj4,12754
 nucliadb/common/cluster/rebalance.py,sha256=cLUlR08SsqmnoA_9GDflV6k2tXmkAPpyFxZErzp45vo,8754
 nucliadb/common/cluster/rollover.py,sha256=iTJ9EQmHbzXL34foNFto-hqdC0Kq1pF1mNxqv0jqhBs,25362
-nucliadb/common/cluster/settings.py,sha256=TMoym-cZsQ2soWfLAce0moSa2XncttQyhahL43LrWTo,3384
+nucliadb/common/cluster/settings.py,sha256=JPwV_0U_i618Tn66GWUq6qCKNjy4TWkGEGld9GwH5uk,2048
 nucliadb/common/cluster/utils.py,sha256=7nQvnVFxM4XV7J560R8hUA-GPzrgD19UlQxHrl4mZUc,4687
 nucliadb/common/cluster/standalone/__init__.py,sha256=itSI7dtTwFP55YMX4iK7JzdMHS5CQVUiB1XzQu4UBh8,833
 nucliadb/common/cluster/standalone/utils.py,sha256=af3r-x_GF7A6dwIAhZLR-r-SZQEVxsFrDKeMfUTA6G0,1908
@@ -109,11 +108,10 @@ nucliadb/export_import/exceptions.py,sha256=Dw8WqfG4r6MPJc5TPfbjMvCgXXWTcTOecGHR
 nucliadb/export_import/exporter.py,sha256=k2QVx1EjqFlDYiggriWiEJzwtMXzHbldsqWdpGQM3_U,7074
 nucliadb/export_import/importer.py,sha256=v5cq9Nn8c2zrY_K_00mydR52f8mdFxR7tLdtNLQ0qvk,4229
 nucliadb/export_import/models.py,sha256=dbjScNkiMRv4X3Ktudy1JRliD25bfoDTy3JmEZgQSCc,2121
-nucliadb/export_import/tasks.py,sha256=yPNdBdvTD7eGc7zvV9Rp7UZ0-mDhA34OOsLqHvns_v0,2975
+nucliadb/export_import/tasks.py,sha256=4JX3bygyLCLSuGxMCStYyoclh_CL8rPxrVVWuGqvcmM,3146
 nucliadb/export_import/utils.py,sha256=iAQAjYuNx0dhM2b5-1A0NEs8tSRsznuT-izysUrTwS0,19986
 nucliadb/ingest/__init__.py,sha256=fsw3C38VP50km3R-nHL775LNGPpJ4JxqXJ2Ib1f5SqE,1011
 nucliadb/ingest/app.py,sha256=rX1KE5vsAzG9hlArBk8WE2SOlvdYylcb-jNkMQNPJdQ,7407
-nucliadb/ingest/cache.py,sha256=w7jMMzamOmQ7gwXna6Dqm6isRNBVv6l5BTBlTxaYWjE,1005
 nucliadb/ingest/partitions.py,sha256=2NIhMYbNT0TNBL6bX1UMSi7vxFGICstCKEqsB0TXHOE,2410
 nucliadb/ingest/processing.py,sha256=8OggvuxNzktTTKDTUwsIuazhDParEWhn46CBZaMYAy8,20659
 nucliadb/ingest/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -258,24 +256,24 @@ nucliadb/standalone/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20
 nucliadb/standalone/api_router.py,sha256=hgq9FXpihzgjHkwcVGfGCSwyXy67fqXTfLFHuINzIi0,5567
 nucliadb/standalone/app.py,sha256=mAApNK_iVsQgJyd-mtwCeZq5csSimwnXmlQGH9a70pE,5586
 nucliadb/standalone/auth.py,sha256=UwMv-TywhMZabvVg3anQLeCRdoHDnWf2o3luvnoNBjs,7670
-nucliadb/standalone/config.py,sha256=g9JBJQfyw87TYZ3yuy0O9WFVLd_MmCJxSRSI0E8FwZE,5396
+nucliadb/standalone/config.py,sha256=hJ3p4dBRSsj5FOmIgAiEX9ZsAGUYd1W-_UJIol5LCCg,4967
 nucliadb/standalone/lifecycle.py,sha256=rdKLG-oOLN4rfd2VGG_2vlDUWYneWSCiuEhoeiFKfnM,2343
 nucliadb/standalone/migrations.py,sha256=s9-3RSZ-O3bjEw2TnBe_YWLUEKbub0bARUxi1gA3yuY,1950
 nucliadb/standalone/purge.py,sha256=ZY-cebb214FFiPG7OFmXZGg0G3CK5Amw0FLLm9WJhKE,1343
 nucliadb/standalone/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-nucliadb/standalone/run.py,sha256=d4lmi9ePObbHeDqwcoZnp5JIkBp9iIPSw9uOwxeQMOU,5623
-nucliadb/standalone/settings.py,sha256=nPJ8R8yNUQD4G92zXBi5KiD6QWhXZdtLoQQUTXfuaHE,6041
+nucliadb/standalone/run.py,sha256=0QKEAT6pCaLvnuxTG3RltTlhE2g5-HI21KbOWfusBGE,5425
+nucliadb/standalone/settings.py,sha256=fbgqVT37XB2cJHJARnR19MO_dz6NLbkuIC2okH7J80o,5714
 nucliadb/standalone/versions.py,sha256=8CxNMNt2NgWM8ct50UsR4d44-ae7wtQI-sV-yGiFqyI,3508
 nucliadb/standalone/static/favicon.ico,sha256=96pKGp6Sx457JkTfjy1dtApMhkitixfU6invCUGAYOU,2285
 nucliadb/standalone/static/index.html,sha256=PEZfuEQFYnYACAL1ceN8xC0im8lBrUx838RkE8tbvgA,3833
 nucliadb/standalone/static/logo.svg,sha256=-wQqSvPGTdlKjUP6pHE6kiq005pgYjDzp9nPl0X71Mk,2639
 nucliadb/tasks/__init__.py,sha256=oFJ3A8HD7w11mBu-IixYE_KxA7juMGlYQb7YD_y6WPM,975
-nucliadb/tasks/consumer.py,sha256=x-999Nsw6lBcKvyGyCGPiGP_naANVYMfl9M-u0U3mhY,7052
+nucliadb/tasks/consumer.py,sha256=xc0Ql3N1Iq52dJ3t4YYGJFj1NCQAly0J5W_brfLa_F8,6894
 nucliadb/tasks/logger.py,sha256=C7keOEO_mjLVp5VbqAZ2QXfqVB2Hot7NgBlUP_SDSMw,924
 nucliadb/tasks/models.py,sha256=qrZKi5DNDQ07waMsp5L4_Fi7WRs57YiO-kmXlrBzEAA,1168
-nucliadb/tasks/producer.py,sha256=w4R1YXgXtmCPGcoNNOr3qkqJYcHJtSmix-xjt7vsPqk,3261
+nucliadb/tasks/producer.py,sha256=JRGnATkALyr_iLHq0OAjzVbfxZ_SOUa6sx-smU5p6SQ,3136
 nucliadb/tasks/retries.py,sha256=Zv-3Hys-SKayG9VQ7_7EIflkegE5j-xPGrf-nwaxsfY,5075
-nucliadb/tasks/utils.py,sha256=6tQVckqyzxv8PhVAd3ZqcMYpGcn73ZY6p1cpm1FxagA,1214
+nucliadb/tasks/utils.py,sha256=tV1AbWdFc3qfIULX44Veqj41FCD1B6XYjG6brULBeiw,1459
 nucliadb/tests/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/tests/config.py,sha256=JN_Jhgj-fwM9_8IeO9pwxr6C1PiwRDrXxm67Y38rU30,2080
 nucliadb/tests/vectors.py,sha256=CcNKx-E8LPpyvRyljbmb-Tn_wST9Juw2CBoogWrKiTk,62843
@@ -349,8 +347,8 @@ nucliadb/writer/tus/local.py,sha256=7jYa_w9b-N90jWgN2sQKkNcomqn6JMVBOVeDOVYJHto,
 nucliadb/writer/tus/s3.py,sha256=vF0NkFTXiXhXq3bCVXXVV-ED38ECVoUeeYViP8uMqcU,8357
 nucliadb/writer/tus/storage.py,sha256=ToqwjoYnjI4oIcwzkhha_MPxi-k4Jk3Lt55zRwaC1SM,2903
 nucliadb/writer/tus/utils.py,sha256=MSdVbRsRSZVdkaum69_0wku7X3p5wlZf4nr6E0GMKbw,2556
-nucliadb-6.3.1.post3546.dist-info/METADATA,sha256=DNgc6YIrIj8zGIewzwiHRTsrQL_kYcQVR8VddQdzKLw,4291
-nucliadb-6.3.1.post3546.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-nucliadb-6.3.1.post3546.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
-nucliadb-6.3.1.post3546.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
-nucliadb-6.3.1.post3546.dist-info/RECORD,,
+nucliadb-6.3.1.post3557.dist-info/METADATA,sha256=z3g4U4gIG0vYmw55DjZgSN2PJJUziqy_qvdWFaAxDbA,4291
+nucliadb-6.3.1.post3557.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+nucliadb-6.3.1.post3557.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
+nucliadb-6.3.1.post3557.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
+nucliadb-6.3.1.post3557.dist-info/RECORD,,

nucliadb/common/cluster/index_node.py

@@ -1,77 +0,0 @@
-# Copyright (C) 2021 Bosutech XXI S.L.
-#
-# nucliadb is offered under the AGPL v3.0 and as commercial software.
-# For commercial licensing, contact us at info@nuclia.com.
-#
-# AGPL:
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-#
-from typing import Optional
-
-from lru import LRU
-
-from nucliadb.common.cluster.base import AbstractIndexNode
-from nucliadb.common.cluster.grpc_node_dummy import DummyReaderStub, DummyWriterStub
-from nucliadb.ingest import SERVICE_NAME
-from nucliadb_protos.nodereader_pb2_grpc import NodeReaderStub
-from nucliadb_protos.nodewriter_pb2_grpc import NodeWriterStub
-from nucliadb_utils.grpc import get_traced_grpc_channel
-
-from .settings import settings
-
-READ_CONNECTIONS = LRU(50)  # type: ignore
-WRITE_CONNECTIONS = LRU(50)  # type: ignore
-
-
-class IndexNode(AbstractIndexNode):
-    _writer: Optional[NodeWriterStub] = None
-    _reader: Optional[NodeReaderStub] = None
-
-    def _get_service_address(self, port_map: dict[str, int], port: Optional[int]) -> str:
-        hostname = self.address.split(":")[0]
-        if port is None:
-            # For testing purposes we need to be able to have a writing port
-            port = port_map[hostname]
-            grpc_address = f"localhost:{port}"
-        else:
-            grpc_address = f"{hostname}:{port}"
-        return grpc_address
-
-    @property
-    def writer(self) -> NodeWriterStub:
-        if self._writer is None or self.address not in WRITE_CONNECTIONS:
-            if not self.dummy:
-                grpc_address = self._get_service_address(
-                    settings.writer_port_map, settings.node_writer_port
-                )
-                channel = get_traced_grpc_channel(grpc_address, SERVICE_NAME, variant="_writer")
-                WRITE_CONNECTIONS[self.address] = NodeWriterStub(channel)
-            else:
-                WRITE_CONNECTIONS[self.address] = DummyWriterStub()
-            self._writer = WRITE_CONNECTIONS[self.address]
-        return self._writer  # type: ignore
-
-    @property
-    def reader(self) -> NodeReaderStub:
-        if self._reader is None or self.address not in READ_CONNECTIONS:
-            if not self.dummy:
-                grpc_address = self._get_service_address(
-                    settings.reader_port_map, settings.node_reader_port
-                )
-                channel = get_traced_grpc_channel(grpc_address, SERVICE_NAME, variant="_reader")
-                READ_CONNECTIONS[self.address] = NodeReaderStub(channel)
-            else:
-                READ_CONNECTIONS[self.address] = DummyReaderStub()
-            self._reader = READ_CONNECTIONS[self.address]
-        return self._reader  # type: ignore

nucliadb/ingest/cache.py

@@ -1,25 +0,0 @@
-# Copyright (C) 2021 Bosutech XXI S.L.
-#
-# nucliadb is offered under the AGPL v3.0 and as commercial software.
-# For commercial licensing, contact us at info@nuclia.com.
-#
-# AGPL:
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-#
-from nucliadb.common.cluster.index_node import READ_CONNECTIONS, WRITE_CONNECTIONS
-
-
-def clear_ingest_cache():
-    READ_CONNECTIONS.clear()
-    WRITE_CONNECTIONS.clear()