sbcli-dev 3.9.3__zip → 3.9.5__zip

{sbcli_dev-3.9.3 → sbcli_dev-3.9.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sbcli-dev
-Version: 3.9.3
+Version: 3.9.5
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

{sbcli_dev-3.9.3 → sbcli_dev-3.9.5}/env_var

@@ -1,5 +1,5 @@
 SIMPLY_BLOCK_COMMAND_NAME=sbcli-dev
-SIMPLY_BLOCK_VERSION=3.9.3
+SIMPLY_BLOCK_VERSION=3.9.5
 
 
 SIMPLY_BLOCK_DOCKER_IMAGE=simplyblock/simplyblock:main

{sbcli_dev-3.9.3 → sbcli_dev-3.9.5}/sbcli_dev.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sbcli-dev
-Version: 3.9.3
+Version: 3.9.5
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

{sbcli_dev-3.9.3 → sbcli_dev-3.9.5}/sbcli_dev.egg-info/SOURCES.txt

@@ -90,7 +90,6 @@ simplyblock_core/services/caching_node_monitor.py
 simplyblock_core/services/cap_monitor.py
 simplyblock_core/services/capacity_and_stats_collector.py
 simplyblock_core/services/device_monitor.py
-simplyblock_core/services/distr_event_collector.py
 simplyblock_core/services/health_check_service.py
 simplyblock_core/services/install_service.sh
 simplyblock_core/services/log_agg_service.py

{sbcli_dev-3.9.3 → sbcli_dev-3.9.5}/simplyblock_core/cluster_ops.py

@@ -215,8 +215,9 @@ def create_cluster(blk_size, page_size_in_blocks, cli_pass,
     shutil.rmtree(temp_dir)
 
     logger.info("Deploying swarm stack ...")
+    log_level = "DEBUG" if constants.LOG_WEB_DEBUG else "INFO"
     ret = scripts.deploy_stack(cli_pass, DEV_IP, constants.SIMPLY_BLOCK_DOCKER_IMAGE, c.secret, c.uuid,
-                               log_del_interval, metrics_retention_period)
+                               log_del_interval, metrics_retention_period, log_level=log_level)
     logger.info("Deploying swarm stack > Done")
 
     if ret == 0:

{sbcli_dev-3.9.3 → sbcli_dev-3.9.5}/simplyblock_core/controllers/device_controller.py

@@ -80,6 +80,19 @@ def device_set_read_only(device_id):
 
 
 def device_set_online(device_id):
+    db_controller = DBController()
+    dev = db_controller.get_storage_devices(device_id)
+    snode = db_controller.get_storage_node_by_id(dev.node_id)
+    logger.info("Make other nodes connect to the node devices")
+    snodes = db_controller.get_storage_nodes_by_cluster_id(snode.cluster_id)
+    for node in snodes:
+        if node.get_id() == snode.get_id() or node.status != StorageNode.STATUS_ONLINE:
+            continue
+        node.remote_devices = storage_node_ops._connect_to_remote_devs(node)
+        if node.enable_ha_jm:
+            node.remote_jm_devices = storage_node_ops._connect_to_remote_jm_devs(node)
+        node.write_to_db()
+
     ret = device_set_state(device_id, NVMeDevice.STATUS_ONLINE)
     if ret:
         logger.info("Adding task to device data migration")

{sbcli_dev-3.9.3 → sbcli_dev-3.9.5}/simplyblock_core/controllers/lvol_controller.py

@@ -186,7 +186,7 @@ def _get_next_3_nodes(cluster_id, lvol_size=0):
             # node_stat_list = db_controller.get_node_stats(node, limit=1000)
             # combined_record = utils.sum_records(node_stat_list)
             node_st = {
-                "lvol": len(node.lvols) or 1,
+                "lvol": len(node.lvols)+1,
                 # "cpu": 1 + (node.cpu * node.cpu_hz),
                 # "r_io": combined_record.read_io_ps,
                 # "w_io": combined_record.write_io_ps,

{sbcli_dev-3.9.3 → sbcli_dev-3.9.5}/simplyblock_core/distr_controller.py

@@ -34,7 +34,6 @@ def send_node_status_event(node, node_status):
 def send_dev_status_event(device, dev_status):
     db_controller = DBController()
     storage_ID = device.cluster_device_order
-    logging.info(f"Sending event updates, device: {storage_ID}, status: {dev_status}")
     node_status_event = {
         "timestamp": datetime.datetime.now().isoformat("T", "seconds") + 'Z',
         "event_type": "device_status",
@@ -46,8 +45,8 @@ def send_dev_status_event(device, dev_status):
     for node in snodes:
         if node.status != node.STATUS_ONLINE:
             continue
-        logger.info(f"Sending to: {node.get_id()}")
-        rpc_client = RPCClient(node.mgmt_ip, node.rpc_port, node.rpc_username, node.rpc_password, timeout=10)
+        logging.debug(f"Sending event updates, device: {storage_ID}, status: {dev_status}, node: {node.get_id()}")
+        rpc_client = RPCClient(node.mgmt_ip, node.rpc_port, node.rpc_username, node.rpc_password, timeout=3, retry=1)
         ret = rpc_client.distr_status_events_update(events)
         if not ret:
             logger.warning("Failed to send event update")
@@ -82,7 +81,6 @@ def get_distr_cluster_map(snodes, target_node):
         dev_w_map = []
         node_w = 0
         for i, dev in enumerate(snode.nvme_devices):
-            logger.debug(f"Device: {dev.get_id()}, status: {dev.status}")
             if dev.status in [NVMeDevice.STATUS_JM, NVMeDevice.STATUS_NEW]:
                 continue
             dev_w = int(dev.size/(1024*1024*1024)) or 1
@@ -100,6 +98,8 @@ def get_distr_cluster_map(snodes, target_node):
                         break
             if not name:
                 name = f"remote_{dev.alceml_bdev}n1"
+                dev_status = NVMeDevice.STATUS_UNAVAILABLE
+            logger.debug(f"Device: {dev.get_id()}, status: {dev_status}, bdev_name: {name}")
             dev_map[dev.cluster_device_order] = {
                 "UUID": dev.get_id(),
                 "bdev_name": name,

{sbcli_dev-3.9.3 → sbcli_dev-3.9.5}/simplyblock_core/scripts/__init__.py

@@ -30,10 +30,12 @@ def configure_docker(docker_ip):
     return __run_script(['bash', '-x', os.path.join(DIR_PATH, 'config_docker.sh'), docker_ip])
 
 
-def deploy_stack(cli_pass, dev_ip, image_name, graylog_password, cluster_id, log_del_interval, metrics_retention_period):
+def deploy_stack(cli_pass, dev_ip, image_name, graylog_password, cluster_id,
+                 log_del_interval, metrics_retention_period, log_level):
     pass_hash = hashlib.sha256(graylog_password.encode('utf-8')).hexdigest()
     return __run_script(
-        ['sudo', 'bash', '-x', os.path.join(DIR_PATH, 'deploy_stack.sh'), cli_pass, dev_ip, image_name, pass_hash, graylog_password, cluster_id, log_del_interval, metrics_retention_period])
+        ['sudo', 'bash', '-x', os.path.join(DIR_PATH, 'deploy_stack.sh'), cli_pass, dev_ip, image_name, pass_hash,
+         graylog_password, cluster_id, log_del_interval, metrics_retention_period, log_level])
 
 
 def deploy_cleaner():

{sbcli_dev-3.9.3 → sbcli_dev-3.9.5}/simplyblock_core/scripts/deploy_stack.sh

@@ -11,6 +11,7 @@ export CLUSTER_SECRET=$5
 export CLUSTER_ID=$6
 export LOG_DELETION_INTERVAL=$7
 export RETENTION_PERIOD=$8
+export LOG_LEVEL=$9
 export DIR="$(dirname "$(realpath "$0")")"
 
 if [ -s "/etc/foundationdb/fdb.cluster" ]

{sbcli_dev-3.9.3 → sbcli_dev-3.9.5}/simplyblock_core/scripts/docker-compose-swarm.yml

@@ -6,6 +6,7 @@ services:
     networks:
       - hostnet
     environment:
+      SIMPLYBLOCK_LOG_LEVEL: '$LOG_LEVEL'
       FDB_CLUSTER_FILE_CONTENTS: '$FDB_CLUSTER_FILE_CONTENTS'
       FDB_NETWORKING_MODE: 'container'
       FDB_CLUSTER_FILE: '/etc/foundationdb/fdb.cluster'
@@ -32,8 +33,9 @@ services:
     volumes:
       - "/etc/foundationdb:/etc/foundationdb"
     environment:
-      - FLASK_DEBUG=False
-      - FLASK_ENV=production
+      SIMPLYBLOCK_LOG_LEVEL: "$LOG_LEVEL"
+      FLASK_DEBUG: "False"
+      FLASK_ENV: "production"
 
   StorageNodeMonitor:
     image: $SIMPLYBLOCK_DOCKER_IMAGE
@@ -45,6 +47,8 @@ services:
       - "/etc/foundationdb:/etc/foundationdb"
     networks:
       - hostnet
+    environment:
+      SIMPLYBLOCK_LOG_LEVEL: "$LOG_LEVEL"
 
   MgmtNodeMonitor:
     image: $SIMPLYBLOCK_DOCKER_IMAGE
@@ -56,6 +60,8 @@ services:
       - "/etc/foundationdb:/etc/foundationdb"
     networks:
       - hostnet
+    environment:
+      SIMPLYBLOCK_LOG_LEVEL: "$LOG_LEVEL"
 
   CachingNodeMonitor:
     image: $SIMPLYBLOCK_DOCKER_IMAGE
@@ -67,6 +73,8 @@ services:
       - "/etc/foundationdb:/etc/foundationdb"
     networks:
       - hostnet
+    environment:
+      SIMPLYBLOCK_LOG_LEVEL: "$LOG_LEVEL"
 
   LVolStatsCollector:
     image: $SIMPLYBLOCK_DOCKER_IMAGE
@@ -78,6 +86,8 @@ services:
       - "/etc/foundationdb:/etc/foundationdb"
     networks:
       - hostnet
+    environment:
+      SIMPLYBLOCK_LOG_LEVEL: "$LOG_LEVEL"
 
   CachedLVolStatsCollector:
     image: $SIMPLYBLOCK_DOCKER_IMAGE
@@ -89,18 +99,8 @@ services:
       - "/etc/foundationdb:/etc/foundationdb"
     networks:
       - hostnet
-
-  PortStatsCollector:
-    image: $SIMPLYBLOCK_DOCKER_IMAGE
-    command: "python simplyblock_core/services/port_stat_collector.py"
-    networks:
-      - hostnet
-    deploy:
-      mode: global
-      placement:
-        constraints: [node.role == worker]
-    volumes:
-      - "/etc/foundationdb:/etc/foundationdb"
+    environment:
+      SIMPLYBLOCK_LOG_LEVEL: "$LOG_LEVEL"
 
   MainDistrEventCollector:
     image: $SIMPLYBLOCK_DOCKER_IMAGE
@@ -112,6 +112,8 @@ services:
         constraints: [node.role == manager]
     volumes:
       - "/etc/foundationdb:/etc/foundationdb"
+    environment:
+      SIMPLYBLOCK_LOG_LEVEL: "$LOG_LEVEL"
 
   HAProxy:
     image: haproxytech/haproxy-debian:latest
@@ -127,6 +129,8 @@ services:
       - monitoring-net
     volumes:
       - "$DIR/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg"
+    environment:
+      SIMPLYBLOCK_LOG_LEVEL: "$LOG_LEVEL"
 
   CapacityAndStatsCollector:
     image: $SIMPLYBLOCK_DOCKER_IMAGE
@@ -138,6 +142,8 @@ services:
       - "/etc/foundationdb:/etc/foundationdb"
     networks:
       - hostnet
+    environment:
+      SIMPLYBLOCK_LOG_LEVEL: "$LOG_LEVEL"
 
   CapacityMonitor:
     image: $SIMPLYBLOCK_DOCKER_IMAGE
@@ -149,6 +155,8 @@ services:
       - "/etc/foundationdb:/etc/foundationdb"
     networks:
       - hostnet
+    environment:
+      SIMPLYBLOCK_LOG_LEVEL: "$LOG_LEVEL"
 
   HealthCheck:
     image: $SIMPLYBLOCK_DOCKER_IMAGE
@@ -160,6 +168,8 @@ services:
       - "/etc/foundationdb:/etc/foundationdb"
     networks:
       - hostnet
+    environment:
+      SIMPLYBLOCK_LOG_LEVEL: "$LOG_LEVEL"
 
   DeviceMonitor:
     image: $SIMPLYBLOCK_DOCKER_IMAGE
@@ -171,6 +181,8 @@ services:
       - "/etc/foundationdb:/etc/foundationdb"
     networks:
       - hostnet
+    environment:
+      SIMPLYBLOCK_LOG_LEVEL: "$LOG_LEVEL"
 
   LVolMonitor:
     image: $SIMPLYBLOCK_DOCKER_IMAGE
@@ -182,10 +194,13 @@ services:
       - "/etc/foundationdb:/etc/foundationdb"
     networks:
       - hostnet
+    environment:
+      SIMPLYBLOCK_LOG_LEVEL: "$LOG_LEVEL"
 
   CleanupFDB:
     image: $SIMPLYBLOCK_DOCKER_IMAGE
     environment:
+      SIMPLYBLOCK_LOG_LEVEL: "$LOG_LEVEL"
       LOG_DELETION_INTERVAL: "${LOG_DELETION_INTERVAL}"
     command: "python simplyblock_core/workers/cleanup_foundationdb.py"
     deploy:
@@ -206,6 +221,8 @@ services:
       - "/etc/foundationdb:/etc/foundationdb"
     networks:
       - hostnet
+    environment:
+      SIMPLYBLOCK_LOG_LEVEL: "$LOG_LEVEL"
 
   TasksRunnerMigration:
     image: $SIMPLYBLOCK_DOCKER_IMAGE
@@ -217,6 +234,8 @@ services:
       - "/etc/foundationdb:/etc/foundationdb"
     networks:
       - hostnet
+    environment:
+      SIMPLYBLOCK_LOG_LEVEL: "$LOG_LEVEL"
 
   TasksRunnerFailedMigration:
     image: $SIMPLYBLOCK_DOCKER_IMAGE
@@ -228,6 +247,8 @@ services:
       - "/etc/foundationdb:/etc/foundationdb"
     networks:
       - hostnet
+    environment:
+      SIMPLYBLOCK_LOG_LEVEL: "$LOG_LEVEL"
 
   TasksRunnerNewDeviceMigration:
     image: $SIMPLYBLOCK_DOCKER_IMAGE
@@ -239,6 +260,8 @@ services:
       - "/etc/foundationdb:/etc/foundationdb"
     networks:
       - hostnet
+    environment:
+      SIMPLYBLOCK_LOG_LEVEL: "$LOG_LEVEL"
 
   NewDeviceDiscovery:
     image: $SIMPLYBLOCK_DOCKER_IMAGE
@@ -250,6 +273,8 @@ services:
       - "/etc/foundationdb:/etc/foundationdb"
     networks:
       - hostnet
+    environment:
+      SIMPLYBLOCK_LOG_LEVEL: "$LOG_LEVEL"
 
   TasksNodeAddRunner:
     image: $SIMPLYBLOCK_DOCKER_IMAGE
@@ -261,6 +286,8 @@ services:
       - "/etc/foundationdb:/etc/foundationdb"
     networks:
       - hostnet
+    environment:
+      SIMPLYBLOCK_LOG_LEVEL: "$LOG_LEVEL"
 
 
 volumes:

{sbcli_dev-3.9.3 → sbcli_dev-3.9.5}/simplyblock_core/services/health_check_service.py

@@ -68,10 +68,6 @@ while True:
             node_api_check = health_controller._check_node_api(snode.mgmt_ip)
             logger.info(f"Check: node API {snode.mgmt_ip}:5000 ... {node_api_check}")
 
-            if snode.status == StorageNode.STATUS_OFFLINE:
-                set_node_health_check(snode, ping_check & node_api_check)
-                continue
-
             # 3- check node RPC
             node_rpc_check = health_controller._check_node_rpc(
                 snode.mgmt_ip, snode.rpc_port, snode.rpc_username, snode.rpc_password)

{sbcli_dev-3.9.3 → sbcli_dev-3.9.5}/simplyblock_core/services/main_distr_event_collector.py

@@ -56,7 +56,7 @@ def process_device_event(event):
                     new_remote_devices.append(rem_dev)
             node.remote_devices = new_remote_devices
             node.write_to_db(db_controller.kv_store)
-            distr_controller.send_cluster_map_to_node(node)
+            device_controller.device_set_unavailable(device_id)
 
         else:
             if event.message == 'SPDK_BDEV_EVENT_REMOVE':

{sbcli_dev-3.9.3 → sbcli_dev-3.9.5}/simplyblock_web/templates/storage_deploy_spdk.yaml.j2

@@ -90,9 +90,9 @@ spec:
         resources:
           limits:
             hugepages-2Mi: {{ MEM_GEGA }}Gi
-            memory: 10Gi
+            memory: 4Gi
           requests:
-            memory: 10Gi
+            memory: 4Gi
 
       - name: spdk-proxy-container
         image: {{ SIMPLYBLOCK_DOCKER_IMAGE }}
@@ -110,33 +110,3 @@ spec:
           value: "{{ RPC_USERNAME }}"
         - name: RPC_PASSWORD
           value: "{{ RPC_PASSWORD }}"
-
-{#      - name: node-exporter#}
-{#        image: prom/node-exporter:v1.7.0#}
-{#        user: root#}
-{#        command:#}
-{#          - '--path.procfs=/host/proc'#}
-{#          - '--path.sysfs=/host/sys'#}
-{#          - '--path.rootfs=/host'#}
-{#          - '--collector.filesystem.ignored-mount-points="^(/rootfs|/host|)/(sys|proc|dev|host|etc)($$|/)"'#}
-{#          - '--collector.filesystem.ignored-fs-types="^(sys|proc|auto|cgroup|devpts|ns|au|fuse\.lxc|mqueue)(fs|)$$"'#}
-{#          - '--no-collector.ipvs'#}
-{#        volumeMounts:#}
-{#        - name: host-sys#}
-{#          mountPath: /host/sys#}
-{#          readOnly: true#}
-{#        - name: host-proc#}
-{#          mountPath: /host/proc#}
-{#          readOnly: true#}
-{#        - name: host-rootfs#}
-{#          mountPath: /rootfs#}
-{#          readOnly: true#}
-
-      - name: distr-event-collector
-        image: {{ SIMPLYBLOCK_DOCKER_IMAGE }}
-        imagePullPolicy: "Always"
-        command: ["python", "simplyblock_core/services/distr_event_collector.py"]
-        volumeMounts:
-        - name: foundationdb
-          mountPath: /etc/foundationdb
-          readOnly: true

sbcli_dev-3.9.3/simplyblock_core/services/distr_event_collector.py

@@ -1,157 +0,0 @@
-# coding=utf-8
-import time
-
-
-from simplyblock_core import constants, kv_store, utils, rpc_client
-from simplyblock_core.controllers import events_controller, device_controller, lvol_events
-from simplyblock_core.models.lvol_model import LVol
-
-
-from simplyblock_core.models.nvme_device import NVMeDevice
-from simplyblock_core.rpc_client import RPCClient
-
-
-logger = utils.get_logger(__name__)
-
-
-# get DB controller
-db_controller = kv_store.DBController()
-
-
-def process_device_event(event):
-    if event.message in ['SPDK_BDEV_EVENT_REMOVE', "error_open", 'error_read', "error_write", "error_unmap"]:
-        node_id = event.node_id
-        storage_id = event.storage_id
-
-        device = None
-        for node in db_controller.get_storage_nodes():
-            for dev in node.nvme_devices:
-                if dev.cluster_device_order == storage_id:
-                    if dev.status not in [NVMeDevice.STATUS_ONLINE, NVMeDevice.STATUS_READONLY]:
-                        logger.info(f"The storage device is not online, skipping. status: {dev.status}")
-                        # event.status = 'skipped'
-                        # return
-                    device = dev
-                    break
-
-        if not device:
-            logger.info(f"Device not found!, storage id: {storage_id} from node: {node_id}")
-            event.status = 'device_not_found'
-            return
-
-        device_id = device.get_id()
-        if event.message == 'SPDK_BDEV_EVENT_REMOVE':
-            if device.node_id == node_id:
-                logger.info(f"Removing storage id: {storage_id} from node: {node_id}")
-                device_controller.device_remove(device_id)
-            else:
-                logger.info(f"Removing remote storage id: {storage_id} from node: {node_id}")
-                new_remote_devices = []
-                device_node = db_controller.get_storage_node_by_id(node_id)
-                rpc_client = RPCClient(device_node.mgmt_ip, device_node.rpc_port,
-                                       device_node.rpc_username, device_node.rpc_password)
-                for rem_dev in device_node.remote_devices:
-                    if rem_dev.get_id() == device.get_id():
-                        ctrl_name = rem_dev.remote_bdev[:-2]
-                        rpc_client.bdev_nvme_detach_controller(ctrl_name)
-                    else:
-                        new_remote_devices.append(rem_dev)
-                device_node.remote_devices = new_remote_devices
-                device_node.write_to_db(db_controller.kv_store)
-
-        elif event.message in ['error_write', 'error_unmap']:
-            logger.info(f"Setting device to read-only")
-            device_controller.device_set_io_error(device_id, True)
-            device_controller.device_set_read_only(device_id)
-        else:
-            logger.info(f"Setting device to unavailable")
-            device_controller.device_set_io_error(device_id, True)
-            device_controller.device_set_unavailable(device_id)
-
-        event.status = 'processed'
-
-
-def process_lvol_event(event):
-    if event.message in ["error_open", 'error_read', "error_write", "error_unmap"]:
-        # vuid = event.object_dict['vuid']
-        node_id = event.node_id
-        lvols = []
-        for lv in db_controller.get_lvols():  # pass
-            if lv.node_id == node_id:
-                lvols.append(lv)
-
-        if not lvols:
-            logger.error(f"LVols on node {node_id} not found")
-            event.status = 'lvols_not_found'
-        else:
-            for lvol in lvols:
-                if lvol.status == LVol.STATUS_ONLINE:
-                    logger.info("Setting LVol to offline")
-                    lvol.io_error = True
-                    old_status = lvol.status
-                    lvol.status = LVol.STATUS_OFFLINE
-                    lvol.write_to_db(db_controller.kv_store)
-                    lvol_events.lvol_status_change(lvol, lvol.status, old_status, caused_by="monitor")
-                    lvol_events.lvol_io_error_change(lvol, True, False, caused_by="monitor")
-            event.status = 'processed'
-    else:
-        logger.error(f"Unknown LVol event message: {event.message}")
-        event.status = "event_unknown"
-
-
-def process_event(event_id):
-    event = db_controller.get_events(event_id)[0]
-    if event.event == "device_status":
-        if event.storage_id >= 0:
-            process_device_event(event)
-
-        if event.vuid >= 0:
-            process_lvol_event(event)
-
-    event.write_to_db(db_controller.kv_store)
-
-
-hostname = utils.get_hostname()
-logger.info("Getting node info...")
-while True:
-    time.sleep(constants.DISTR_EVENT_COLLECTOR_INTERVAL_SEC)
-
-    snode = db_controller.get_storage_node_by_hostname(hostname)
-    if not snode:
-        logger.error("This node is not part of the cluster, hostname: %s, Retrying..." % hostname)
-        continue
-
-    logger.info(f"Starting Distr event collector on node: {hostname}")
-
-    client = rpc_client.RPCClient(
-        snode.mgmt_ip,
-        snode.rpc_port,
-        snode.rpc_username,
-        snode.rpc_password,
-        timeout=10, retry=2)
-
-    try:
-        events = client.distr_status_events_discard_then_get(0, constants.DISTR_EVENT_COLLECTOR_NUM_OF_EVENTS)
-
-        if not events:
-            logger.debug("no events found")
-            continue
-
-        logger.info(f"Found events: {len(events)}")
-        event_ids = []
-        for ev in events:
-            logger.debug(ev)
-            ev_id = events_controller.log_distr_event(snode.cluster_id, snode.get_id(), ev)
-            event_ids.append(ev_id)
-
-        for eid in event_ids:
-            logger.info(f"Processing event: {eid}")
-            process_event(eid)
-
-        logger.info(f"Discarding events: {len(events)}")
-        client.distr_status_events_discard_then_get(len(events), 0)
-
-    except Exception as e:
-        logger.error("Failed to process distr events")
-        logger.exception(e)
-        continue