sbcli-pre 1.2.5__zip → 1.2.6__zip

{sbcli_pre-1.2.5 → sbcli_pre-1.2.6}/simplyblock_core/models/nvme_device.py

@@ -28,7 +28,7 @@ class NVMeDevice(BaseModel):
         "serial_number": {"type": str, 'default': ""},
         "overload_percentage": {"type": int, 'default': 0},
         "nvme_bdev": {"type": str, 'default': ""},
-        "alloc_bdev": {"type": str, 'default': ""},
+        "nvme_controller": {"type": str, 'default': ""},
         "alceml_bdev": {"type": str, 'default': ""},
         "node_id": {"type": str, 'default': ""},
         "pt_bdev": {"type": str, 'default': ""},
@@ -37,7 +37,6 @@ class NVMeDevice(BaseModel):
         "nvmf_port": {"type": int, 'default': 0},
         "remote_bdev": {"type": str, 'default': ""},
         "testing_bdev": {"type": str, 'default': ""},
-        "jm_bdev": {"type": str, 'default': ""},
         "cluster_device_order": {"type": int, 'default': 0},
         "health_check": {"type": bool, "default": True},
         "cluster_id": {"type": str, 'default': ""},
@@ -47,6 +46,13 @@ class NVMeDevice(BaseModel):
         "io_error": {"type": bool, 'default': False},
         "retries_exhausted": {"type": bool, 'default': False},
 
+        "partition_main_bdev": {"type": str, 'default': ""},
+        "partition_main_size": {"type": int, 'default': 0},
+        "partition_jm_bdev": {"type": str, 'default': ""},
+        "partition_jm_size": {"type": int, 'default': 0},
+
+        "physical_label": {"type": int, 'default': 0},
+
     }
 
     def __init__(self, data=None):
@@ -57,5 +63,37 @@ class NVMeDevice(BaseModel):
     def get_id(self):
         return self.uuid
 
-    def get_capacity_percentage(self):
-        return ((self.size - self.capacity) / self.size) * 100
+
+class JMDevice(BaseModel):
+
+    STATUS_ONLINE = 'online'
+    STATUS_UNAVAILABLE = 'unavailable'
+    STATUS_REMOVED = 'removed'
+    STATUS_FAILED = 'failed'
+    STATUS_READONLY = 'read_only'
+
+    attributes = {
+        "uuid": {"type": str, 'default': ""},
+        "device_name": {"type": str, 'default': ""},
+        "status": {"type": str, 'default': ""},
+        "size": {"type": int, 'default': -1},
+
+        "jm_nvme_bdev_list": {"type": List[str], 'default': []},
+        "raid_bdev": {"type": str, 'default': ""},
+        "nvme_bdev": {"type": str, 'default': ""},
+        "alceml_bdev": {"type": str, 'default': ""},
+        "jm_bdev": {"type": str, 'default': ""},
+
+
+        "health_check": {"type": bool, "default": True},
+        "io_error": {"type": bool, 'default': False},
+    }
+
+    def __init__(self, data=None):
+        super(JMDevice, self).__init__()
+        self.set_attrs(self.attributes, data)
+        self.object_type = "object"
+
+    def get_id(self):
+        return self.uuid
+

{sbcli_pre-1.2.5 → sbcli_pre-1.2.6}/simplyblock_core/models/storage_node.py

@@ -5,7 +5,7 @@ from typing import List
 
 from simplyblock_core.models.base_model import BaseModel
 from simplyblock_core.models.iface import IFace
-from simplyblock_core.models.nvme_device import NVMeDevice
+from simplyblock_core.models.nvme_device import NVMeDevice, JMDevice
 
 
 class StorageNode(BaseModel):
@@ -70,6 +70,10 @@ class StorageNode(BaseModel):
 
         # spdk params
         "spdk_cpu_mask": {"type": str, "default": ""},
+        "app_thread_mask": {"type": str, "default": ""},
+        "pollers_mask": {"type": str, "default": ""},
+        "os_cores": {"type": str, "default": []},
+        "dev_cpu_mask": {"type": str, "default": ""},
         "spdk_mem": {"type": int, "default": 0},
         "spdk_image": {"type": str, "default": ""},
         "spdk_debug": {"type": bool, "default": False},
@@ -84,6 +88,10 @@ class StorageNode(BaseModel):
         "iobuf_small_bufsize": {"type": int, "default": 0},
         "iobuf_large_bufsize": {"type": int, "default": 0},
 
+        "num_partitions_per_dev": {"type": int, "default": 1},
+        "jm_percent": {"type": int, "default": 3},
+        "jm_device": {"type": JMDevice, "default": None},
+
     }
 
     def __init__(self, data=None):

{sbcli_pre-1.2.5 → sbcli_pre-1.2.6}/simplyblock_core/rpc_client.py

@@ -306,9 +306,9 @@ class RPCClient:
         params = {"name": name}
         return self._request("bdev_compress_delete", params)
 
-    def ultra21_bdev_pass_create(self, alloc_bdev, vuid, pt_name):
+    def ultra21_bdev_pass_create(self, base_bdev, vuid, pt_name):
         params = {
-            "base_bdev": alloc_bdev,
+            "base_bdev": base_bdev,
             "vuid": vuid,
             "pt_bdev": pt_name
         }
@@ -318,7 +318,8 @@ class RPCClient:
         params = {"name": name}
         return self._request2("ultra21_bdev_pass_delete", params)
 
-    def bdev_alceml_create(self, alceml_name, nvme_name, uuid, pba_init_mode=3):
+    def bdev_alceml_create(self, alceml_name, nvme_name, uuid, pba_init_mode=3,
+                           dev_cpu_mask=""):
         params = {
             "name": alceml_name,
             "cntr_path": nvme_name,
@@ -334,10 +335,13 @@ class RPCClient:
             "use_optimized": True,
             "pba_nbalign": 4096
         }
+        if dev_cpu_mask:
+            params["bdb_lcpu_mask"] = int(dev_cpu_mask,16)
         return self._request("bdev_alceml_create", params)
 
     def bdev_distrib_create(self, name, vuid, ndcs, npcs, num_blocks, block_size, jm_names,
-                            chunk_size, ha_comm_addrs=None, ha_inode_self=None, pba_page_size=2097152):
+                            chunk_size, ha_comm_addrs=None, ha_inode_self=None, pba_page_size=2097152,
+                            dev_cpu_mask=""):
         """"
             // Optional (not specified = no HA)
             // Comma-separated communication addresses, for each node, e.g. "192.168.10.1:45001,192.168.10.1:32768".
@@ -363,6 +367,8 @@ class RPCClient:
         if ha_comm_addrs:
             params['ha_comm_addrs'] = ha_comm_addrs
             params['ha_inode_self'] = ha_inode_self
+        if dev_cpu_mask:
+            params["bdb_lcpu_mask"] = int(dev_cpu_mask, 16)
 
         return self._request("bdev_distrib_create", params)
 
@@ -386,7 +392,7 @@ class RPCClient:
         params = {
             "name": name,
             "raid_level": "0",
-            "strip_size_kb": 4 * len(bdevs_list),
+            "strip_size_kb": 4,
             "base_bdevs": bdevs_list
         }
         return self._request("bdev_raid_create", params)
@@ -428,8 +434,9 @@ class RPCClient:
             "trsvcid": str(port),
             "subnqn": nqn,
             "fabrics_connect_timeout_us": 100000,
-            "fast_io_fail_timeout_sec": 0,
+            "fast_io_fail_timeout_sec": 1,
             "num_io_queues": 16384,
+            "ctrlr_loss_timeout_sec": 2,
         }
         return self._request("bdev_nvme_attach_controller", params)
 
@@ -477,9 +484,9 @@ class RPCClient:
         params = {
             "bdev_retry_count": 0,
             "transport_retry_count": 0,
-            "ctrlr_loss_timeout_sec": -1,
-            "fast_io_fail_timeout_sec": 5,
-            "reconnect_delay_sec": 5,
+            "ctrlr_loss_timeout_sec": 2,
+            "fast_io_fail_timeout_sec": 1,
+            "reconnect_delay_sec": 1,
             "keep_alive_timeout_ms": 200,
             "transport_ack_timeout": 7,
             "timeout_us": 100000
@@ -583,12 +590,14 @@ class RPCClient:
         }
         return self._request("ultra21_lvol_dismount", params)
 
-    def bdev_jm_create(self, name, name_storage1, block_size=4096):
+    def bdev_jm_create(self, name, name_storage1, block_size=4096, dev_cpu_mask=""):
         params = {
             "name": name,
             "name_storage1": name_storage1,
             "block_size": block_size
         }
+        if dev_cpu_mask:
+            params["bdb_lcpu_mask"] = int(dev_cpu_mask, 16)
         return self._request("bdev_jm_create", params)
 
     def bdev_jm_delete(self, name):
@@ -620,6 +629,42 @@ class RPCClient:
     def framework_start_init(self):
         return self._request("framework_start_init")
 
+    def bdev_examine(self, name):
+        params = {"name": name}
+        return self._request("bdev_examine", params)
+
+    def nbd_start_disk(self, bdev_name, nbd_device="/dev/nbd0"):
+        params = {
+            "bdev_name": bdev_name,
+            "nbd_device": nbd_device,
+        }
+        return self._request("nbd_start_disk", params)
+
+    def nbd_stop_disk(self, nbd_device):
+        params = {
+            "nbd_device": nbd_device
+        }
+        return self._request("nbd_stop_disk", params)
+
+
     def bdev_jm_unmap_vuid(self, name, vuid):
         params = {"name": name, "vuid": vuid}
         return self._request("bdev_jm_unmap_vuid", params)
+
+    def sock_impl_set_options(self):
+        method = "sock_impl_set_options"
+        params = {"impl_name": "posix", "enable_quickack": True,
+                  "enable_zerocopy_send_server": True,
+                  "enable_zerocopy_send_client": True}
+        return self._request(method, params)
+
+    def nvmf_set_config(self, poll_groups_mask):
+        params = {"poll_groups_mask": poll_groups_mask}
+        return self._request("nvmf_set_config", params)
+
+    def thread_get_stats(self):
+        return self._request("thread_get_stats")
+
+    def thread_set_cpumask(self, app_thread_process_id, app_thread_mask):
+        params = {"id": app_thread_process_id, "cpumask": app_thread_mask}
+        return self._request("thread_set_cpumask", params)

{sbcli_pre-1.2.5 → sbcli_pre-1.2.6}/simplyblock_core/scripts/__init__.py

@@ -35,10 +35,6 @@ def deploy_stack(cli_pass, dev_ip, image_name, graylog_password, cluster_id, log
     return __run_script(
         ['sudo', 'bash', '-x', os.path.join(DIR_PATH, 'deploy_stack.sh'), cli_pass, dev_ip, image_name, pass_hash, graylog_password, cluster_id, log_del_interval, metrics_retention_period])
 
-def apply_dashboard(grafanaPassword):
-    return __run_script(
-        ['sudo', 'bash', '-x', os.path.join(DIR_PATH, 'apply_dashboard.sh'), grafanaPassword])
-
 
 def deploy_cleaner():
     return __run_script(['sudo', 'bash', '-x', os.path.join(DIR_PATH, 'clean_local_storage_deploy.sh')])

sbcli_pre-1.2.5/simplyblock_core/scripts/alerting/alert_resources.yaml → sbcli_pre-1.2.6/simplyblock_core/scripts/alerting/alert_resources.yaml.j2

@@ -12,15 +12,26 @@ contactPoints:
     name: grafana-alerts
     receivers:
       - uid: grafana
-        type: slack
+        type: {{ ALERT_TYPE }}
+        {% if ALERT_TYPE == 'slack' %}
         settings:
           username: grafana_bot
-          url: 'https://hooks.slack.com/services/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'
+          url: '{{ CONTACT_POINT }}'
           title: |
-            {{ template "slack.title" . }}
+            {{ '{{' }} template "slack.title" . {{ '}}' }}
           text: |
-            {{ template "slack.message" . }}
+            {{ '{{' }} template "slack.message" . {{ '}}' }}
+        {% else %}
+        settings:
+          addresses: '{{ CONTACT_POINT }}'
+          subject: |
+            {{ '{{' }} template "email.subject" . {{ '}}' }}
+          body: |
+            {{ '{{' }} template "email.body" . {{ '}}' }}
+        {% endif %}
 
+{% if ALERT_TYPE == 'slack' %}
+{% raw %}
 templates:
   - orgId: 1
     name: slack.title
@@ -38,7 +49,9 @@ templates:
       *Description*: {{ .Annotations.description }}
       {{ end -}}
       *Log message*: {{ index .Labels "message" }}
-      *Explore logs:* https://grafanaURL.com/explore?orgId=1
+      {% endraw %}
+      *Explore logs:* {{ GRAFANA_ENDPOINT }}
+      {% raw %}
       {{ if .DashboardURL -}}
       *Go to dashboard:* {{ .DashboardURL }}
       {{- end }}
@@ -65,3 +78,39 @@ templates:
       {{ end }}
 
       {{- end }}
+{% endraw %}
+{% else %}
+{% raw %}
+  - orgId: 1
+    name: email.subject
+    template: |-
+      {{ define "email.subject" -}}
+      [{{ .Status | toUpper }}] Grafana Alert
+      {{- end -}}
+  - orgId: 1
+    name: email.body
+    template: |-
+      {{ define "email.body" -}}
+      Alert: {{ .Labels.alertname }}
+      {{ if .Annotations -}}
+      Summary: {{ .Annotations.summary}}
+      Description: {{ .Annotations.description }}
+      {{ end -}}
+      Log message: {{ index .Labels "message" }}
+      Explore logs: {{ GRAFANA_ENDPOINT }}
+      {{ if .DashboardURL -}}
+      Go to dashboard: {{ .DashboardURL }}
+      {{- end }}
+      {{ if .PanelURL -}}
+      Go to panel: {{ .PanelURL }}
+      {{- end }}
+      Details:
+      {{ range .Labels.SortedPairs -}}
+      - {{ .Name }}: `{{ .Value }}`
+      {{ end -}}
+      {{ if .SilenceURL -}}
+      Silence this alert: {{ .SilenceURL }}
+      {{- end }}
+      {{- end }}
+{% endraw %}
+{% endif %}

{sbcli_pre-1.2.5 → sbcli_pre-1.2.6}/simplyblock_core/scripts/dashboards/cluster.json

@@ -2354,4 +2354,4 @@
         "version": 5,
         "weekStart": ""
     }
-}
+}

{sbcli_pre-1.2.5 → sbcli_pre-1.2.6}/simplyblock_core/scripts/deploy_stack.sh

@@ -19,6 +19,15 @@ then
    export FDB_CLUSTER_FILE_CONTENTS=$FDB_CLUSTER_FILE_CONTENTS
 fi
 
+docker network create monitoring-net -d overlay --attachable
+
+INSTANCE_ID=$(curl -s http://169.254.169.254/latest/meta-data/instance-id)
+
+if [ -n "$INSTANCE_ID" ] 
+then
+  export USE_EFS="rexray/efs"
+fi
+
 docker stack deploy --compose-file="$DIR"/docker-compose-swarm-monitoring.yml monitoring
 
 # wait for the services to become online

{sbcli_pre-1.2.5 → sbcli_pre-1.2.6}/simplyblock_core/scripts/docker-compose-swarm-monitoring.yml

@@ -9,6 +9,8 @@ services:
     deploy:
       placement:
         constraints: [node.role == manager]
+    networks:
+      - monitoring-net
 
   opensearch:
     image: "opensearchproject/opensearch:2.4.0"
@@ -24,6 +26,8 @@ services:
     deploy:
       placement:
         constraints: [node.role == manager]
+    networks:
+      - monitoring-net
 
   graylog:
     hostname: "server"
@@ -34,16 +38,16 @@ services:
       GRAYLOG_PASSWORD_SECRET: "${GRAYLOG_PASSWORD_SECRET}"
       GRAYLOG_ROOT_PASSWORD_SHA2: "${GRAYLOG_ROOT_PASSWORD_SHA2}"
       GRAYLOG_HTTP_BIND_ADDRESS: "0.0.0.0:9000"
-      GRAYLOG_HTTP_EXTERNAL_URI: "http://localhost:9000/"
+      GRAYLOG_HTTP_EXTERNAL_URI: "http://localhost/graylog/"
       GRAYLOG_ELASTICSEARCH_HOSTS: "http://opensearch:9200"
       GRAYLOG_MONGODB_URI: "mongodb://mongodb:27017/graylog"
+      GRAYLOG_SKIP_PREFLIGHT_CHECKS: "true"
     ports:
     - "5044:5044/tcp"   # Beats
     - "5140:5140/udp"   # Syslog
     - "5140:5140/tcp"   # Syslog
     - "5555:5555/tcp"   # RAW TCP
     - "5555:5555/udp"   # RAW TCP
-    - "9000:9000/tcp"   # Server API
     - "12201:12201/tcp" # GELF TCP
     - "12201:12201/udp" # GELF UDP
     - "13301:13301/tcp" # Forwarder data
@@ -54,22 +58,28 @@ services:
     deploy:
       placement:
         constraints: [node.role == manager]
+    networks:
+      - monitoring-net
 
   promagent:
     image: simplyblock/promagent
     environment:
       ClusterID: "${CLUSTER_ID}"
-      ClusterIP: "${CLUSTER_IP}"
+      ClusterIP: "HAProxy"
       ClusterSecret: "${CLUSTER_SECRET}"
     deploy:
       placement:
         constraints: [node.role == manager]
+    networks:
+      - monitoring-net
 
   pushgateway:
     image: prom/pushgateway
     deploy:
       placement:
         constraints: [node.role == manager]
+    networks:
+      - monitoring-net
 
   prometheus:
     image: prom/prometheus:v2.44.0
@@ -85,6 +95,8 @@ services:
     deploy:
       placement:
         constraints: [node.role == manager]
+    networks:
+      - monitoring-net
 
   node-exporter:
     image: prom/node-exporter:v1.7.0
@@ -105,7 +117,9 @@ services:
       mode: global
       placement:
         constraints: [node.role == worker]
-        
+    networks:
+      - monitoring-net
+
   grafana:
     image: grafana/grafana:10.0.12
     environment:
@@ -114,19 +128,16 @@ services:
       GF_ALERTING_ENABLED: "true"
       GF_PATHS_PROVISIONING: "/etc/grafana/provisioning"
       GF_INSTALL_PLUGINS: "grafana-opensearch-datasource"
+      GF_SERVER_ROOT_URL: "http://localhost/grafana/"
     volumes:
       - ./datasource.yml:/etc/grafana/provisioning/datasources/datasource.yaml
       - grafana_data:/var/lib/grafana
       - ./alerting:/etc/grafana/provisioning/alerting
-    restart: "always"
-    ports:
-      - target: 3000
-        published: 3000
-        protocol: tcp
-        mode: host
     deploy:
       placement:
         constraints: [node.role == manager]
+    networks:
+      - monitoring-net
 
   CleanupGraylog:
     image: $SIMPLYBLOCK_DOCKER_IMAGE
@@ -136,19 +147,25 @@ services:
     deploy:
       placement:
         constraints: [node.role == manager]
-
-  ### monitoring ###
+    networks:
+      - monitoring-net
 
 volumes:
   mongodb_data:
+    driver: ${USE_EFS:-local}
   os_data:
+    driver: ${USE_EFS:-local}
   graylog_data:
+    driver: ${USE_EFS:-local}
   graylog_journal:
+    driver: ${USE_EFS:-local}
   grafana_data:
+    driver: ${USE_EFS:-local}
+  graylog_config:
+    driver: ${USE_EFS:-local}  
   prometheus_data:
-  alertmanager_data:
+    driver: ${USE_EFS:-local}
 
 networks:
-  hostnet:
+  monitoring-net:
     external: true
-    name: host

{sbcli_pre-1.2.5 → sbcli_pre-1.2.6}/simplyblock_core/scripts/docker-compose-swarm.yml

@@ -114,6 +114,7 @@ services:
       - 8404:8404
     networks:
       - localnet
+      - monitoring-net
     volumes:
       - "$DIR/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg"
 
@@ -185,9 +186,20 @@ services:
     networks:
       - hostnet
 
-  TasksRunner:
+  TasksRunnerRestart:
     image: $SIMPLYBLOCK_DOCKER_IMAGE
-    command: "python simplyblock_core/services/job_tasks.py"
+    command: "python simplyblock_core/services/tasks_runner_restart.py"
+    deploy:
+      placement:
+        constraints: [node.role == manager]
+    volumes:
+      - "/etc/foundationdb:/etc/foundationdb"
+    networks:
+      - hostnet
+
+  TasksRunnerMigration:
+    image: $SIMPLYBLOCK_DOCKER_IMAGE
+    command: "python simplyblock_core/services/tasks_runner_migration.py"
     deploy:
       placement:
         constraints: [node.role == manager]
@@ -200,6 +212,9 @@ volumes:
   os_data:
 
 networks:
+  monitoring-net:
+    external: true
+
   hostnet:
     external: true
     name: host

{sbcli_pre-1.2.5 → sbcli_pre-1.2.6}/simplyblock_core/scripts/haproxy.cfg

@@ -42,6 +42,16 @@ backend wep_api_services
     balance roundrobin
     server-template webapi- 3 WebAppAPI:5000 check resolvers docker init-addr libc,none
 
+backend grafana_services
+    balance roundrobin
+    http-request set-path %[path,regsub(^/grafana/?,/)]
+    server-template grafana- 1 grafana:3000 check resolvers docker init-addr libc,none
+
+backend graylog_services
+    balance roundrobin
+    http-request set-path %[path,regsub(^/graylog/?,/)]
+    server-template graylog- 1 graylog:9000 check resolvers docker init-addr libc,none
+
 frontend stats_front
     bind *:8404
     stats enable
@@ -52,4 +62,9 @@ frontend stats_front
 
 frontend web_api_front
     bind *:80
+
+    use_backend grafana_services if { path /grafana } || { path_beg /grafana/ }
+    use_backend graylog_services if { path /graylog } || { path_beg /graylog/ }
+
     default_backend wep_api_services
+

{sbcli_pre-1.2.5 → sbcli_pre-1.2.6}/simplyblock_core/scripts/install_deps.sh

@@ -15,6 +15,8 @@ sudo yum install hostname pkg-config git wget python3-pip yum-utils docker-ce do
 sudo systemctl enable docker
 sudo systemctl start docker
 
+sudo docker plugin install --grant-all-permissions rexray/efs REXRAY_PREEMPT=true
+
 wget https://github.com/apple/foundationdb/releases/download/7.3.3/foundationdb-clients-7.3.3-1.el7.x86_64.rpm -q
 sudo rpm -U foundationdb-clients-7.3.3-1.el7.x86_64.rpm --quiet --reinstall
 rm -f foundationdb-clients-7.3.3-1.el7.x86_64.rpm
@@ -29,6 +31,7 @@ sudo sed -i 's/#X11UseLocalhost yes/X11UseLocalhost no/g' /etc/ssh/sshd_config
 
 sudo service sshd restart
 sudo modprobe nvme-tcp
+sudo modprobe nbd
 
 sudo sysctl -w net.ipv6.conf.all.disable_ipv6=1
 

{sbcli_pre-1.2.5 → sbcli_pre-1.2.6}/simplyblock_core/scripts/stack_deploy_wait.sh

@@ -6,7 +6,7 @@ opt_h=0
 opt_l=""
 opt_r=0
 opt_s=5
-opt_t=600 # 10 min
+opt_t=900 # 15 min
 
 start_epoc=$(date +%s)
 cmd_min_timeout=15

{sbcli_pre-1.2.5 → sbcli_pre-1.2.6}/simplyblock_core/services/capacity_and_stats_collector.py

@@ -193,7 +193,7 @@ while True:
                     logger.info(f"Device is skipped: {device.get_id()} status: {device.status}")
                     continue
                 capacity_dict = rpc_client.alceml_get_capacity(device.alceml_bdev)
-                stats_dict = rpc_client.get_device_stats(device.alloc_bdev)
+                stats_dict = rpc_client.get_device_stats(device.nvme_bdev)
                 record = add_device_stats(cl, device, capacity_dict, stats_dict)
                 if record:
                     devices_records.append(record)

{sbcli_pre-1.2.5 → sbcli_pre-1.2.6}/simplyblock_core/services/device_monitor.py

@@ -5,7 +5,7 @@ import sys
 import uuid
 
 from simplyblock_core import constants, kv_store
-from simplyblock_core.models.job_schedule import JobSchedule
+from simplyblock_core.controllers import tasks_controller
 from simplyblock_core.models.nvme_device import NVMeDevice
 from simplyblock_core.models.storage_node import StorageNode
 
@@ -27,47 +27,6 @@ db_store = kv_store.KVStore()
 db_controller = kv_store.DBController()
 
 
-def add_device_to_auto_restart(device):
-    tasks = db_controller.get_job_tasks(device.cluster_id)
-    for task in tasks:
-        if task.device_id == device.get_id():
-            if task.status != JobSchedule.STATUS_DONE:
-                logger.info(f"Task found, skip adding new task: {task.get_id()}")
-                return
-
-    ds = JobSchedule()
-    ds.uuid = str(uuid.uuid4())
-    ds.cluster_id = device.cluster_id
-    ds.node_id = device.node_id
-    ds.device_id = device.get_id()
-    ds.date = int(time.time())
-    ds.function_name = "device_restart"
-    ds.status = 'new'
-
-    ds.write_to_db(db_store)
-    return ds.get_id()
-
-
-def add_node_to_auto_restart(node):
-    tasks = db_controller.get_job_tasks(node.cluster_id)
-    for task in tasks:
-        if task.node_id == node.get_id():
-            if task.status != JobSchedule.STATUS_DONE:
-                logger.info(f"Task found, skip adding new task: {task.get_id()}")
-                return
-
-    ds = JobSchedule()
-    ds.uuid = str(uuid.uuid4())
-    ds.cluster_id = node.cluster_id
-    ds.node_id = node.get_id()
-    ds.date = int(time.time())
-    ds.function_name = "node_restart"
-    ds.status = 'new'
-
-    ds.write_to_db(db_store)
-    return ds.get_id()
-
-
 logger.info("Starting Device monitor...")
 while True:
     nodes = db_controller.get_storage_nodes()
@@ -89,9 +48,9 @@ while True:
                 logger.info("Adding device to auto restart")
                 auto_restart_devices.append(dev)
 
-        if len(auto_restart_devices) == 1:
-            add_device_to_auto_restart(auto_restart_devices[0])
-        elif len(auto_restart_devices) >= 2 and len(online_devices) == 0:
-            add_node_to_auto_restart(node)
+        if len(auto_restart_devices) >= 2 or len(online_devices) == 0:
+            tasks_controller.add_node_to_auto_restart(node)
+        elif len(auto_restart_devices) == 1:
+            tasks_controller.add_device_to_auto_restart(auto_restart_devices[0])
 
     time.sleep(constants.DEV_MONITOR_INTERVAL_SEC)