nucliadb 6.3.1.post3546__py3-none-any.whl → 6.3.1.post3549__py3-none-any.whl

migrations/0020_drain_nodes_from_cluster.py

@@ -28,9 +28,6 @@ create new shards in the remaining nodes.
 
 import logging
 
-from nucliadb.common import datamanagers
-from nucliadb.common.cluster.rollover import rollover_kb_index
-from nucliadb.common.cluster.settings import settings as cluster_settings
 from nucliadb.migrator.context import ExecutionContext
 
 logger = logging.getLogger(__name__)
@@ -39,45 +36,4 @@ logger = logging.getLogger(__name__)
 async def migrate(context: ExecutionContext) -> None: ...
 
 
-async def migrate_kb(context: ExecutionContext, kbid: str) -> None:
-    """
-    Rollover KB shards if any of the shards are on the nodes to drain
-    """
-    drain_node_ids = cluster_settings.drain_nodes
-    if len(drain_node_ids) == 0:
-        logger.info("Skipping migration because no drain_nodes are set")
-        return
-
-    if not await kb_has_shards_on_drain_nodes(kbid, drain_node_ids):
-        logger.info(
-            "KB does not have shards on the nodes to drain, skipping rollover",
-            extra={"kbid": kbid},
-        )
-        return
-
-    logger.info("Rolling over affected KB", extra={"kbid": kbid})
-    await rollover_kb_index(context, kbid, drain_nodes=drain_node_ids)
-
-
-async def kb_has_shards_on_drain_nodes(kbid: str, drain_node_ids: list[str]) -> bool:
-    async with datamanagers.with_ro_transaction() as txn:
-        shards = await datamanagers.cluster.get_kb_shards(txn, kbid=kbid)
-        if not shards:
-            logger.warning("Shards object not found", extra={"kbid": kbid})
-            return False
-        shard_in_drain_nodes = False
-        for shard in shards.shards:
-            for replica in shard.replicas:
-                if replica.node in drain_node_ids:
-                    logger.info(
-                        "Shard found in drain nodes, will rollover it",
-                        extra={
-                            "kbid": kbid,
-                            "logical_shard": shard.shard,
-                            "replica_shard_id": replica.shard.id,
-                            "node": replica.node,
-                            "drain_node_ids": drain_node_ids,
-                        },
-                    )
-                    shard_in_drain_nodes = True
-        return shard_in_drain_nodes
+async def migrate_kb(context: ExecutionContext, kbid: str) -> None: ...

nucliadb/common/cluster/settings.py

@@ -24,12 +24,6 @@ from pydantic import Field
 from pydantic_settings import BaseSettings
 
 
-class ClusterDiscoveryMode(str, enum.Enum):
-    MANUAL = "manual"
-    KUBERNETES = "kubernetes"
-    SINGLE_NODE = "single_node"
-
-
 class StandaloneNodeRole(enum.Enum):
     ALL = "all"
     INDEX = "index"
@@ -39,53 +33,20 @@ class StandaloneNodeRole(enum.Enum):
 class Settings(BaseSettings):
     data_path: str = "./data/node"
     standalone_mode: bool = False
-    standalone_node_port: int = Field(
-        default=10009,
-        title="Standalone node port",
-        description="Port to use for standalone nodes to communication with each other through",
-    )
     standalone_node_role: StandaloneNodeRole = StandaloneNodeRole.ALL
 
-    node_replicas: int = 2
-
-    node_writer_port: int = 10000
-    node_reader_port: int = 10001
-
-    # Only for testing purposes
-    writer_port_map: dict[str, int] = {}
-    reader_port_map: dict[str, int] = {}
-
-    # Node limits
+    # Index limits
     max_shard_paragraphs: int = Field(
         default=500_000,
         title="Max shard paragraphs",
         description="Maximum number of paragraphs to target per shard",
     )
-    max_node_replicas: int = Field(
-        default=800,
-        title="Max node replicas",
-        description="Maximum number of shard replicas a single node will manage",
-    )
     max_resource_paragraphs: int = Field(
         default=50_000,
         title="Max paragraphs per resource",
         description="Maximum number of paragraphs allowed on a single resource",
     )
 
-    drain_nodes: list[str] = Field(
-        default=[],
-        title="Drain nodes",
-        description="List of node IDs to ignore when creating new shards. It is used for draining nodes from a cluster. Example: ['1bf3bfe7-e164-4a19-a4d9-41372fc15aca',]",  # noqa: E501
-    )
-
-    local_reader_threads: int = 5
-    local_writer_threads: int = 5
-
-    cluster_discovery_mode: ClusterDiscoveryMode = ClusterDiscoveryMode.KUBERNETES
-    cluster_discovery_kubernetes_namespace: str = "nucliadb"
-    cluster_discovery_kubernetes_selector: str = "appType=node"
-    cluster_discovery_manual_addresses: list[str] = []
-
     nidx_api_address: Optional[str] = Field(default=None, description="NIDX gRPC API address")
     nidx_searcher_address: Optional[str] = Field(
         default=None, description="NIDX gRPC searcher API address"

nucliadb/standalone/config.py

@@ -22,7 +22,7 @@ import logging
 import os
 
 from nucliadb.common.cluster.settings import StandaloneNodeRole
-from nucliadb.standalone.settings import Settings, StandaloneDiscoveryMode
+from nucliadb.standalone.settings import Settings
 
 logger = logging.getLogger(__name__)
 
@@ -76,7 +76,6 @@ def config_nucliadb(nucliadb_args: Settings):
     use some specific settings.
     """
 
-    from nucliadb.common.cluster.settings import ClusterDiscoveryMode
     from nucliadb.common.cluster.settings import settings as cluster_settings
     from nucliadb.ingest.settings import settings as ingest_settings
     from nucliadb.train.settings import settings as train_settings
@@ -91,14 +90,8 @@ def config_nucliadb(nucliadb_args: Settings):
 
     cluster_settings.standalone_mode = True
     cluster_settings.data_path = nucliadb_args.data_path
-    cluster_settings.standalone_node_port = nucliadb_args.standalone_node_port
     cluster_settings.standalone_node_role = nucliadb_args.standalone_node_role
 
-    if nucliadb_args.cluster_discovery_mode == StandaloneDiscoveryMode.DEFAULT:
-        # default for standalone is single node
-        cluster_settings.cluster_discovery_mode = ClusterDiscoveryMode.SINGLE_NODE
-        cluster_settings.node_replicas = 1
-
     ingest_settings.nuclia_partitions = 1
     ingest_settings.replica_number = 0
     ingest_settings.partitions = ["1"]

nucliadb/standalone/run.py

@@ -99,10 +99,7 @@ def run():
         "Admin UI": f"http://{settings.http_host}:{settings.http_port}/admin",
         "Key-value backend": ingest_settings.driver.value,
         "Blob storage backend": storage_settings.file_backend.value,
-        "Cluster discovery mode": cluster_settings.cluster_discovery_mode.value,
-        "Node replicas": cluster_settings.node_replicas,
         "Index data path": os.path.realpath(cluster_settings.data_path),
-        "Node port": cluster_settings.standalone_node_port,
         "Auth policy": settings.auth_policy.value,
         "Node role": cluster_settings.standalone_node_role.value,
     }

nucliadb/standalone/settings.py

@@ -30,13 +30,6 @@ from nucliadb_utils.settings import StorageSettings
 from nucliadb_utils.storages.settings import Settings as ExtendedStorageSettings
 
 
-class StandaloneDiscoveryMode(Enum):
-    DEFAULT = "default"
-    MANUAL = "manual"
-    KUBERNETES = "kubernetes"
-    SINGLE_NODE = "single_node"
-
-
 class AuthPolicy(Enum):
     UPSTREAM_NAIVE = "upstream_naive"
     UPSTREAM_AUTH_HEADER = "upstream_auth_header"
@@ -60,8 +53,6 @@ class Settings(DriverSettings, StorageSettings, ExtendedStorageSettings):
     http_port: int = pydantic.Field(default=8080, description="HTTP Port")
     ingest_grpc_port: int = pydantic.Field(default=8030, description="Ingest GRPC Port")
     train_grpc_port: int = pydantic.Field(default=8031, description="Train GRPC Port")
-    standalone_node_port: int = pydantic.Field(default=10009, description="Node GRPC Port")
-
     auth_policy: AuthPolicy = pydantic.Field(
         default=AuthPolicy.UPSTREAM_NAIVE,
         description="""Auth policy to use for http requests.
@@ -111,8 +102,6 @@ Examples:
         description="JWK key used for temporary token generation and validation.",
     )
 
-    cluster_discovery_mode: StandaloneDiscoveryMode = StandaloneDiscoveryMode.DEFAULT
-
     fork: bool = pydantic.Field(default=False, description="Fork process on startup")
 
     # Standalone logging settings

{nucliadb-6.3.1.post3546.dist-info → nucliadb-6.3.1.post3549.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: nucliadb
-Version: 6.3.1.post3546
+Version: 6.3.1.post3549
 Summary: NucliaDB
 Author-email: Nuclia <nucliadb@nuclia.com>
 License: AGPL
@@ -20,11 +20,11 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: <4,>=3.9
 Description-Content-Type: text/markdown
-Requires-Dist: nucliadb-telemetry[all]>=6.3.1.post3546
-Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.3.1.post3546
-Requires-Dist: nucliadb-protos>=6.3.1.post3546
-Requires-Dist: nucliadb-models>=6.3.1.post3546
-Requires-Dist: nidx-protos>=6.3.1.post3546
+Requires-Dist: nucliadb-telemetry[all]>=6.3.1.post3549
+Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.3.1.post3549
+Requires-Dist: nucliadb-protos>=6.3.1.post3549
+Requires-Dist: nucliadb-models>=6.3.1.post3549
+Requires-Dist: nidx-protos>=6.3.1.post3549
 Requires-Dist: nucliadb-admin-assets>=1.0.0.post1224
 Requires-Dist: nuclia-models>=0.24.2
 Requires-Dist: uvicorn

{nucliadb-6.3.1.post3546.dist-info → nucliadb-6.3.1.post3549.dist-info}/RECORD

@@ -16,7 +16,7 @@ migrations/0016_upgrade_to_paragraphs_v2.py,sha256=9eepvzme-nb_mw6rwIdjjJzbFcpOq
 migrations/0017_multiple_writable_shards.py,sha256=HSi-eXXI0kO9sLgunUMuVpFnYMjVhWTVfDS_lIjlkuM,2095
 migrations/0018_purge_orphan_kbslugs.py,sha256=ztEOAjqlWVagv1UMg_sOm8HaW6S9FoDOIg7-rZ05tro,2177
 migrations/0019_upgrade_to_paragraphs_v3.py,sha256=zP13_IKE7u4ox2gyc493L2_ewyiPlr7Csn5K6n5eylI,2479
-migrations/0020_drain_nodes_from_cluster.py,sha256=cgm_72kH57QiBbPx17Judn7Wp5hQnKn6UW_1Z37_8s8,3269
+migrations/0020_drain_nodes_from_cluster.py,sha256=BeECAI0T8u14M2U5USl1fFNcsfmdMerNhisolYQN_eA,1411
 migrations/0021_overwrite_vectorsets_key.py,sha256=O6nb2a7kDFX9I3XFrVtudCUyKptpGyv2_GYvcvbQOI8,1583
 migrations/0022_fix_paragraph_deletion_bug.py,sha256=-tH342VXF-8xwc_h3P1cYaUtTT1wHSGf7ZoeVEpnaYs,1422
 migrations/0023_backfill_pg_catalog.py,sha256=gw22pU5cAtg2a7n7xVaVqT2itjAoDMNtzWwTllwqIvg,2993
@@ -58,11 +58,10 @@ nucliadb/common/cluster/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIX
 nucliadb/common/cluster/base.py,sha256=kklDqyvsubNX0W494ttl9f3E58lGaX6AXqAd8XX8ZHE,5522
 nucliadb/common/cluster/exceptions.py,sha256=t7v_l93t44l2tQpdQXgO_w-c4YZRcaayOz1A2i0w4RQ,1258
 nucliadb/common/cluster/grpc_node_dummy.py,sha256=LxONv0mhDFhx7mI91qqGfQlQ-R0qOGDYaxhXoBHLXaE,3548
-nucliadb/common/cluster/index_node.py,sha256=g38H1kiAliF3Y6et_CWYInpn_xPxf7THAFJ7RtgLNZo,3246
 nucliadb/common/cluster/manager.py,sha256=KIzqAYGgdVK3GicJ9LdLoei8arWZ7H60imbc32USPj4,12754
 nucliadb/common/cluster/rebalance.py,sha256=cLUlR08SsqmnoA_9GDflV6k2tXmkAPpyFxZErzp45vo,8754
 nucliadb/common/cluster/rollover.py,sha256=iTJ9EQmHbzXL34foNFto-hqdC0Kq1pF1mNxqv0jqhBs,25362
-nucliadb/common/cluster/settings.py,sha256=TMoym-cZsQ2soWfLAce0moSa2XncttQyhahL43LrWTo,3384
+nucliadb/common/cluster/settings.py,sha256=JPwV_0U_i618Tn66GWUq6qCKNjy4TWkGEGld9GwH5uk,2048
 nucliadb/common/cluster/utils.py,sha256=7nQvnVFxM4XV7J560R8hUA-GPzrgD19UlQxHrl4mZUc,4687
 nucliadb/common/cluster/standalone/__init__.py,sha256=itSI7dtTwFP55YMX4iK7JzdMHS5CQVUiB1XzQu4UBh8,833
 nucliadb/common/cluster/standalone/utils.py,sha256=af3r-x_GF7A6dwIAhZLR-r-SZQEVxsFrDKeMfUTA6G0,1908
@@ -113,7 +112,6 @@ nucliadb/export_import/tasks.py,sha256=yPNdBdvTD7eGc7zvV9Rp7UZ0-mDhA34OOsLqHvns_
 nucliadb/export_import/utils.py,sha256=iAQAjYuNx0dhM2b5-1A0NEs8tSRsznuT-izysUrTwS0,19986
 nucliadb/ingest/__init__.py,sha256=fsw3C38VP50km3R-nHL775LNGPpJ4JxqXJ2Ib1f5SqE,1011
 nucliadb/ingest/app.py,sha256=rX1KE5vsAzG9hlArBk8WE2SOlvdYylcb-jNkMQNPJdQ,7407
-nucliadb/ingest/cache.py,sha256=w7jMMzamOmQ7gwXna6Dqm6isRNBVv6l5BTBlTxaYWjE,1005
 nucliadb/ingest/partitions.py,sha256=2NIhMYbNT0TNBL6bX1UMSi7vxFGICstCKEqsB0TXHOE,2410
 nucliadb/ingest/processing.py,sha256=8OggvuxNzktTTKDTUwsIuazhDParEWhn46CBZaMYAy8,20659
 nucliadb/ingest/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -258,13 +256,13 @@ nucliadb/standalone/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20
 nucliadb/standalone/api_router.py,sha256=hgq9FXpihzgjHkwcVGfGCSwyXy67fqXTfLFHuINzIi0,5567
 nucliadb/standalone/app.py,sha256=mAApNK_iVsQgJyd-mtwCeZq5csSimwnXmlQGH9a70pE,5586
 nucliadb/standalone/auth.py,sha256=UwMv-TywhMZabvVg3anQLeCRdoHDnWf2o3luvnoNBjs,7670
-nucliadb/standalone/config.py,sha256=g9JBJQfyw87TYZ3yuy0O9WFVLd_MmCJxSRSI0E8FwZE,5396
+nucliadb/standalone/config.py,sha256=hJ3p4dBRSsj5FOmIgAiEX9ZsAGUYd1W-_UJIol5LCCg,4967
 nucliadb/standalone/lifecycle.py,sha256=rdKLG-oOLN4rfd2VGG_2vlDUWYneWSCiuEhoeiFKfnM,2343
 nucliadb/standalone/migrations.py,sha256=s9-3RSZ-O3bjEw2TnBe_YWLUEKbub0bARUxi1gA3yuY,1950
 nucliadb/standalone/purge.py,sha256=ZY-cebb214FFiPG7OFmXZGg0G3CK5Amw0FLLm9WJhKE,1343
 nucliadb/standalone/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-nucliadb/standalone/run.py,sha256=d4lmi9ePObbHeDqwcoZnp5JIkBp9iIPSw9uOwxeQMOU,5623
-nucliadb/standalone/settings.py,sha256=nPJ8R8yNUQD4G92zXBi5KiD6QWhXZdtLoQQUTXfuaHE,6041
+nucliadb/standalone/run.py,sha256=0QKEAT6pCaLvnuxTG3RltTlhE2g5-HI21KbOWfusBGE,5425
+nucliadb/standalone/settings.py,sha256=fbgqVT37XB2cJHJARnR19MO_dz6NLbkuIC2okH7J80o,5714
 nucliadb/standalone/versions.py,sha256=8CxNMNt2NgWM8ct50UsR4d44-ae7wtQI-sV-yGiFqyI,3508
 nucliadb/standalone/static/favicon.ico,sha256=96pKGp6Sx457JkTfjy1dtApMhkitixfU6invCUGAYOU,2285
 nucliadb/standalone/static/index.html,sha256=PEZfuEQFYnYACAL1ceN8xC0im8lBrUx838RkE8tbvgA,3833
@@ -349,8 +347,8 @@ nucliadb/writer/tus/local.py,sha256=7jYa_w9b-N90jWgN2sQKkNcomqn6JMVBOVeDOVYJHto,
 nucliadb/writer/tus/s3.py,sha256=vF0NkFTXiXhXq3bCVXXVV-ED38ECVoUeeYViP8uMqcU,8357
 nucliadb/writer/tus/storage.py,sha256=ToqwjoYnjI4oIcwzkhha_MPxi-k4Jk3Lt55zRwaC1SM,2903
 nucliadb/writer/tus/utils.py,sha256=MSdVbRsRSZVdkaum69_0wku7X3p5wlZf4nr6E0GMKbw,2556
-nucliadb-6.3.1.post3546.dist-info/METADATA,sha256=DNgc6YIrIj8zGIewzwiHRTsrQL_kYcQVR8VddQdzKLw,4291
-nucliadb-6.3.1.post3546.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-nucliadb-6.3.1.post3546.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
-nucliadb-6.3.1.post3546.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
-nucliadb-6.3.1.post3546.dist-info/RECORD,,
+nucliadb-6.3.1.post3549.dist-info/METADATA,sha256=zRXMEsuY63780s274oEzv-I_wLynIyYEfW8iLptlP-U,4291
+nucliadb-6.3.1.post3549.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+nucliadb-6.3.1.post3549.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
+nucliadb-6.3.1.post3549.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
+nucliadb-6.3.1.post3549.dist-info/RECORD,,

nucliadb/common/cluster/index_node.py

@@ -1,77 +0,0 @@
-# Copyright (C) 2021 Bosutech XXI S.L.
-#
-# nucliadb is offered under the AGPL v3.0 and as commercial software.
-# For commercial licensing, contact us at info@nuclia.com.
-#
-# AGPL:
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-#
-from typing import Optional
-
-from lru import LRU
-
-from nucliadb.common.cluster.base import AbstractIndexNode
-from nucliadb.common.cluster.grpc_node_dummy import DummyReaderStub, DummyWriterStub
-from nucliadb.ingest import SERVICE_NAME
-from nucliadb_protos.nodereader_pb2_grpc import NodeReaderStub
-from nucliadb_protos.nodewriter_pb2_grpc import NodeWriterStub
-from nucliadb_utils.grpc import get_traced_grpc_channel
-
-from .settings import settings
-
-READ_CONNECTIONS = LRU(50)  # type: ignore
-WRITE_CONNECTIONS = LRU(50)  # type: ignore
-
-
-class IndexNode(AbstractIndexNode):
-    _writer: Optional[NodeWriterStub] = None
-    _reader: Optional[NodeReaderStub] = None
-
-    def _get_service_address(self, port_map: dict[str, int], port: Optional[int]) -> str:
-        hostname = self.address.split(":")[0]
-        if port is None:
-            # For testing purposes we need to be able to have a writing port
-            port = port_map[hostname]
-            grpc_address = f"localhost:{port}"
-        else:
-            grpc_address = f"{hostname}:{port}"
-        return grpc_address
-
-    @property
-    def writer(self) -> NodeWriterStub:
-        if self._writer is None or self.address not in WRITE_CONNECTIONS:
-            if not self.dummy:
-                grpc_address = self._get_service_address(
-                    settings.writer_port_map, settings.node_writer_port
-                )
-                channel = get_traced_grpc_channel(grpc_address, SERVICE_NAME, variant="_writer")
-                WRITE_CONNECTIONS[self.address] = NodeWriterStub(channel)
-            else:
-                WRITE_CONNECTIONS[self.address] = DummyWriterStub()
-            self._writer = WRITE_CONNECTIONS[self.address]
-        return self._writer  # type: ignore
-
-    @property
-    def reader(self) -> NodeReaderStub:
-        if self._reader is None or self.address not in READ_CONNECTIONS:
-            if not self.dummy:
-                grpc_address = self._get_service_address(
-                    settings.reader_port_map, settings.node_reader_port
-                )
-                channel = get_traced_grpc_channel(grpc_address, SERVICE_NAME, variant="_reader")
-                READ_CONNECTIONS[self.address] = NodeReaderStub(channel)
-            else:
-                READ_CONNECTIONS[self.address] = DummyReaderStub()
-            self._reader = READ_CONNECTIONS[self.address]
-        return self._reader  # type: ignore

nucliadb/ingest/cache.py

@@ -1,25 +0,0 @@
-# Copyright (C) 2021 Bosutech XXI S.L.
-#
-# nucliadb is offered under the AGPL v3.0 and as commercial software.
-# For commercial licensing, contact us at info@nuclia.com.
-#
-# AGPL:
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-#
-from nucliadb.common.cluster.index_node import READ_CONNECTIONS, WRITE_CONNECTIONS
-
-
-def clear_ingest_cache():
-    READ_CONNECTIONS.clear()
-    WRITE_CONNECTIONS.clear()