sbcli-dev 3.8.51__zip → 3.8.53__zip

{sbcli_dev-3.8.51 → sbcli_dev-3.8.53}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sbcli-dev
-Version: 3.8.51
+Version: 3.8.53
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

{sbcli_dev-3.8.51 → sbcli_dev-3.8.53}/env_var

@@ -1,5 +1,5 @@
 SIMPLY_BLOCK_COMMAND_NAME=sbcli-dev
-SIMPLY_BLOCK_VERSION=3.8.51
+SIMPLY_BLOCK_VERSION=3.8.53
 
 
 SIMPLY_BLOCK_DOCKER_IMAGE=simplyblock/simplyblock:main

{sbcli_dev-3.8.51 → sbcli_dev-3.8.53}/sbcli_dev.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sbcli-dev
-Version: 3.8.51
+Version: 3.8.53
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

{sbcli_dev-3.8.51 → sbcli_dev-3.8.53}/sbcli_dev.egg-info/SOURCES.txt

@@ -102,6 +102,7 @@ simplyblock_core/services/new_device_discovery.py
 simplyblock_core/services/port_stat_collector.py
 simplyblock_core/services/remove_service.sh
 simplyblock_core/services/service_template.service
+simplyblock_core/services/spdk_http_proxy_server.py
 simplyblock_core/services/storage_node_monitor.py
 simplyblock_core/services/tasks_runner_failed_migration.py
 simplyblock_core/services/tasks_runner_migration.py

{sbcli_dev-3.8.51 → sbcli_dev-3.8.53}/simplyblock_core/constants.py

@@ -57,7 +57,6 @@ SIMPLY_BLOCK_CLI_NAME = "sbcli-dev"
 TASK_EXEC_INTERVAL_SEC = 30
 TASK_EXEC_RETRY_COUNT = 8
 
-SIMPLY_BLOCK_SPDK_CORE_IMAGE = "simplyblock/spdk-core:latest"
 SIMPLY_BLOCK_SPDK_ULTRA_IMAGE = "simplyblock/spdk:main-latest"
 SIMPLY_BLOCK_SPDK_ULTRA_IMAGE_JM = "simplyblock/spdk:feature-journal-ha-latest"
 

{sbcli_dev-3.8.51 → sbcli_dev-3.8.53}/simplyblock_core/controllers/caching_node_controller.py

@@ -626,9 +626,7 @@ def deploy(ifname):
         ]
     )
     logger.info("Pulling SPDK images")
-    spdk_core = constants.SIMPLY_BLOCK_SPDK_CORE_IMAGE
-
-    node_docker.images.pull(spdk_core)
+    node_docker.images.pull(constants.SIMPLY_BLOCK_DOCKER_IMAGE)
     return f"{dev_ip}:5000"
 
 

{sbcli_dev-3.8.51 → sbcli_dev-3.8.53}/simplyblock_core/distr_controller.py

@@ -86,9 +86,9 @@ def get_distr_cluster_map(snodes, target_node):
             dev_w = int(dev.size/(1024*1024*1024)) or 1
             node_w += dev_w
             name = None
+            dev_status = dev.status
             if snode.get_id() == target_node.get_id():
                 name = dev.alceml_bdev
-                dev_status = dev.status
                 local_node_index = index
             else:
                 for dev2 in target_node.remote_devices:
@@ -98,7 +98,6 @@ def get_distr_cluster_map(snodes, target_node):
                         break
             if not name:
                 name = f"remote_{dev.alceml_bdev}n1"
-                dev_status = NVMeDevice.STATUS_UNAVAILABLE
             dev_map[dev.cluster_device_order] = {
                 "UUID": dev.get_id(),
                 "bdev_name": name,

sbcli_dev-3.8.53/simplyblock_core/services/spdk_http_proxy_server.py

@@ -0,0 +1,151 @@
+#!/usr/bin/env python3
+
+import base64
+import json
+import logging
+import os
+import socket
+import ssl
+import sys
+
+from http.server import ThreadingHTTPServer
+from http.server import BaseHTTPRequestHandler
+
+
+TIMEOUT = 5*60  # 5 min
+rpc_sock = '/var/tmp/spdk.sock'
+logger_handler = logging.StreamHandler(stream=sys.stdout)
+logger_handler.setFormatter(logging.Formatter('%(asctime)s: %(levelname)s: %(message)s'))
+logger = logging.getLogger()
+logger.addHandler(logger_handler)
+logger.setLevel(logging.DEBUG)
+
+
+def get_env_var(name, default=None, is_required=False):
+    if not name:
+        logger.warning("Invalid env var name %s", name)
+        return False
+    if name not in os.environ and is_required:
+        logger.error("env value is required: %s" % name)
+        raise Exception("env value is required: %s" % name)
+    return os.environ.get(name, default)
+
+
+def rpc_call(req):
+    req_data = json.loads(req.decode('ascii'))
+    print(req_data)
+    sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+    sock.settimeout(TIMEOUT)
+    sock.connect(rpc_sock)
+    sock.sendall(req)
+
+    if 'id' not in req_data:
+        sock.close()
+        return None
+
+    buf = ''
+    closed = False
+    response = None
+
+    while not closed:
+        newdata = sock.recv(1024)
+        if (newdata == b''):
+            closed = True
+        buf += newdata.decode('ascii')
+        try:
+            response = json.loads(buf)
+        except ValueError:
+            continue  # incomplete response; keep buffering
+        break
+
+    sock.close()
+
+    if not response and len(buf) > 0:
+        raise
+
+    print(buf)
+
+    return buf
+
+
+class ServerHandler(BaseHTTPRequestHandler):
+
+    key = ""
+
+    def do_HEAD(self):
+        self.send_response(200)
+        self.send_header('Content-type', 'text/html')
+        self.end_headers()
+
+    def do_AUTHHEAD(self):
+        self.send_response(401)
+        self.send_header('WWW-Authenticate', 'text/html')
+        self.send_header('Content-type', 'text/html')
+        self.end_headers()
+
+    def do_INTERNALERROR(self):
+        self.send_response(500)
+        self.send_header('Content-type', 'text/html')
+        self.end_headers()
+
+    def do_POST(self):
+        if self.headers['Authorization'] != 'Basic ' + self.key:
+            self.do_AUTHHEAD()
+        else:
+            if "Content-Length" in self.headers:
+                data_string = self.rfile.read(int(self.headers['Content-Length']))
+            elif "chunked" in self.headers.get("Transfer-Encoding", ""):
+                data_string = b''
+                while True:
+                    line = self.rfile.readline().strip()
+                    chunk_length = int(line, 16)
+
+                    if chunk_length != 0:
+                        chunk = self.rfile.read(chunk_length)
+                        data_string += chunk
+
+                    # Each chunk is followed by an additional empty newline
+                    # that we have to consume.
+                    self.rfile.readline()
+
+                    # Finally, a chunk size of 0 is an end indication
+                    if chunk_length == 0:
+                        break
+
+            try:
+                response = rpc_call(data_string)
+                if response is not None:
+                    self.do_HEAD()
+                    self.wfile.write(bytes(response.encode(encoding='ascii')))
+            except ValueError:
+                self.do_INTERNALERROR()
+
+
+def run_server(host, port, user, password, cert=None):
+    # encoding user and password
+    key = base64.b64encode((user+':'+password).encode(encoding='ascii')).decode('ascii')
+
+    try:
+        ServerHandler.key = key
+        httpd = ThreadingHTTPServer((host, port), ServerHandler)
+        httpd.timeout = TIMEOUT
+        if cert is not None:
+            httpd.socket = ssl.wrap_socket(httpd.socket, certfile=cert, server_side=True)
+        print('Started RPC http proxy server')
+        httpd.serve_forever()
+    except KeyboardInterrupt:
+        print('Shutting down server')
+        httpd.socket.close()
+
+
+server_ip = get_env_var("SERVER_IP", is_required=True)
+rpc_port = get_env_var("RPC_PORT", is_required=True)
+rpc_username = get_env_var("RPC_USERNAME", is_required=True)
+rpc_password = get_env_var("RPC_PASSWORD", is_required=True)
+
+try:
+    rpc_port = int(rpc_port)
+except Exception:
+    rpc_port = 8080
+
+run_server(server_ip, rpc_port, rpc_username, rpc_password)

{sbcli_dev-3.8.51 → sbcli_dev-3.8.53}/simplyblock_core/services/tasks_runner_restart.py

@@ -186,7 +186,7 @@ def task_runner_node(task):
         logger.info(f"Node restart succeeded")
 
     time.sleep(5)
-    if _get_node_unavailable_devices_count(node.get_id()) == 0 and node.status == StorageNode.STATUS_ONLINE:
+    if node.status == StorageNode.STATUS_ONLINE:
         logger.info(f"Node is online: {node.get_id()}")
         task.function_result = "done"
         task.status = JobSchedule.STATUS_DONE

{sbcli_dev-3.8.51 → sbcli_dev-3.8.53}/simplyblock_core/storage_node_ops.py

@@ -2357,9 +2357,7 @@ def deploy(ifname):
         ]
     )
     logger.info("Pulling SPDK images")
-    logger.debug(constants.SIMPLY_BLOCK_SPDK_CORE_IMAGE)
     logger.debug(constants.SIMPLY_BLOCK_SPDK_ULTRA_IMAGE)
-    node_docker.images.pull(constants.SIMPLY_BLOCK_SPDK_CORE_IMAGE)
     node_docker.images.pull(constants.SIMPLY_BLOCK_SPDK_ULTRA_IMAGE)
     return f"{dev_ip}:5000"
 

{sbcli_dev-3.8.51 → sbcli_dev-3.8.53}/simplyblock_web/blueprints/caching_node_ops.py

@@ -176,7 +176,7 @@ def spdk_process_start():
             node.remove(force=True)
             time.sleep(2)
 
-    spdk_image = constants.SIMPLY_BLOCK_SPDK_CORE_IMAGE
+    spdk_image = constants.SIMPLY_BLOCK_SPDK_ULTRA_IMAGE
 
     if 'spdk_image' in data and data['spdk_image']:
         spdk_image = data['spdk_image']
@@ -205,8 +205,8 @@ def spdk_process_start():
     rpc_password = data['rpc_password']
 
     container2 = node_docker.containers.run(
-        spdk_image,
-        "python /root/scripts/spdk_http_proxy_server.py",
+        constants.SIMPLY_BLOCK_DOCKER_IMAGE,
+        "python simplyblock_core/services/spdk_http_proxy_server.py",
         name="spdk_proxy",
         detach=True,
         network_mode="host",

{sbcli_dev-3.8.51 → sbcli_dev-3.8.53}/simplyblock_web/blueprints/caching_node_ops_k8s.py

@@ -112,7 +112,7 @@ def spdk_process_start():
 
     spdk_mem_gega = int(spdk_mem / 1024)
 
-    spdk_image = constants.SIMPLY_BLOCK_SPDK_CORE_IMAGE
+    spdk_image = constants.SIMPLY_BLOCK_SPDK_ULTRA_IMAGE
 
     if 'spdk_image' in data and data['spdk_image']:
         spdk_image = data['spdk_image']
@@ -138,6 +138,7 @@ def spdk_process_start():
             'RPC_PASSWORD': data['rpc_password'],
             'HOSTNAME': node_name,
             'NAMESPACE': namespace,
+            'SIMPLYBLOCK_DOCKER_IMAGE': constants.SIMPLY_BLOCK_DOCKER_IMAGE,
         }
         dep = yaml.safe_load(template.render(values))
         logger.debug(dep)

{sbcli_dev-3.8.51 → sbcli_dev-3.8.53}/simplyblock_web/blueprints/node_api_caching_docker.py

@@ -64,7 +64,7 @@ def spdk_process_start():
             node.remove(force=True)
             time.sleep(2)
 
-    spdk_image = constants.SIMPLY_BLOCK_SPDK_CORE_IMAGE
+    spdk_image = constants.SIMPLY_BLOCK_SPDK_ULTRA_IMAGE
     if 'spdk_image' in data and data['spdk_image']:
         spdk_image = data['spdk_image']
         node_docker.images.pull(spdk_image)
@@ -92,8 +92,8 @@ def spdk_process_start():
     rpc_password = data['rpc_password']
 
     container2 = node_docker.containers.run(
-        constants.SIMPLY_BLOCK_SPDK_CORE_IMAGE,
-        "python /root/scripts/spdk_http_proxy_server.py",
+        constants.SIMPLY_BLOCK_DOCKER_IMAGE,
+        "python simplyblock_core/services/spdk_http_proxy_server.py",
         name="spdk_proxy",
         detach=True,
         network_mode="host",

{sbcli_dev-3.8.51 → sbcli_dev-3.8.53}/simplyblock_web/blueprints/node_api_caching_ks.py

@@ -63,7 +63,7 @@ def spdk_process_start():
     else:
         spdk_mem = 64096
 
-    spdk_image = constants.SIMPLY_BLOCK_SPDK_CORE_IMAGE
+    spdk_image = constants.SIMPLY_BLOCK_SPDK_ULTRA_IMAGE
     if 'spdk_image' in data and data['spdk_image']:
         spdk_image = data['spdk_image']
         # node_docker.images.pull(spdk_image)
@@ -82,6 +82,7 @@ def spdk_process_start():
         'RPC_PORT': data['rpc_port'],
         'RPC_USERNAME': data['rpc_username'],
         'RPC_PASSWORD': data['rpc_password'],
+        'SIMPLYBLOCK_DOCKER_IMAGE': constants.SIMPLY_BLOCK_DOCKER_IMAGE,
     }
     dep = yaml.safe_load(template.render(values))
     resp = k8s_apps_v1.create_namespaced_deployment(body=dep, namespace=namespace)

{sbcli_dev-3.8.51 → sbcli_dev-3.8.53}/simplyblock_web/blueprints/snode_ops.py

@@ -143,8 +143,8 @@ def spdk_process_start():
         # restart_policy={"Name": "on-failure", "MaximumRetryCount": 99}
     )
     container2 = node_docker.containers.run(
-        constants.SIMPLY_BLOCK_SPDK_CORE_IMAGE,
-        "python /root/scripts/spdk_http_proxy_server.py",
+        constants.SIMPLY_BLOCK_DOCKER_IMAGE,
+        "python simplyblock_core/services/spdk_http_proxy_server.py",
         name="spdk_proxy",
         detach=True,
         network_mode="host",

{sbcli_dev-3.8.51 → sbcli_dev-3.8.53}/simplyblock_web/blueprints/snode_ops_k8s.py

@@ -100,100 +100,6 @@ def scan_devices():
     }
     return utils.get_response(out)
 
-#
-# @bp.route('/spdk_process_start', methods=['POST'])
-# def spdk_process_start():
-#     try:
-#         data = request.get_json()
-#     except:
-#         data = {}
-#
-#     set_debug = None
-#     if 'spdk_debug' in data and data['spdk_debug']:
-#         set_debug = data['spdk_debug']
-#
-#     spdk_cpu_mask = None
-#     if 'spdk_cpu_mask' in data:
-#         spdk_cpu_mask = data['spdk_cpu_mask']
-#
-#     spdk_mem = None
-#     if 'spdk_mem' in data:
-#         spdk_mem = data['spdk_mem']
-#
-#     if spdk_mem:
-#         spdk_mem = int(utils.parse_size(spdk_mem) / (1000 * 1000))
-#     else:
-#         spdk_mem = 4000
-#
-#     node_docker = get_docker_client()
-#     nodes = node_docker.containers.list(all=True)
-#     for node in nodes:
-#         if node.attrs["Name"] in ["/spdk", "/spdk_proxy"]:
-#             logger.info(f"{node.attrs['Name']} container found, removing...")
-#             node.stop()
-#             node.remove(force=True)
-#             time.sleep(2)
-#
-#     spdk_debug = ""
-#     if set_debug:
-#         spdk_debug = "1"
-#
-#     spdk_image = constants.SIMPLY_BLOCK_SPDK_ULTRA_IMAGE
-#     if 'spdk_image' in data and data['spdk_image']:
-#         spdk_image = data['spdk_image']
-#         # node_docker.images.pull(spdk_image)
-#
-#     if "cluster_ip" in data and data['cluster_ip']:
-#         cluster_ip = data['cluster_ip']
-#         log_config = LogConfig(type=LogConfig.types.GELF, config={"gelf-address": f"udp://{cluster_ip}:12201"})
-#     else:
-#         log_config = LogConfig(type=LogConfig.types.JOURNALD)
-#
-#     container = node_docker.containers.run(
-#         spdk_image,
-#         f"/root/scripts/run_distr.sh {spdk_cpu_mask} {spdk_mem} {spdk_debug}",
-#         name="spdk",
-#         detach=True,
-#         privileged=True,
-#         network_mode="host",
-#         log_config=log_config,
-#         volumes=[
-#             '/etc/simplyblock:/etc/simplyblock',
-#             '/var/tmp:/var/tmp',
-#             '/dev:/dev',
-#             '/lib/modules/:/lib/modules/',
-#             '/var/lib/systemd/coredump/:/var/lib/systemd/coredump/',
-#             '/sys:/sys'],
-#         # restart_policy={"Name": "on-failure", "MaximumRetryCount": 99}
-#     )
-#     container2 = node_docker.containers.run(
-#         constants.SIMPLY_BLOCK_SPDK_CORE_IMAGE,
-#         "python /root/scripts/spdk_http_proxy.py",
-#         name="spdk_proxy",
-#         detach=True,
-#         network_mode="host",
-#         log_config=log_config,
-#         volumes=[
-#             '/var/tmp:/var/tmp',
-#             '/etc/foundationdb:/etc/foundationdb'],
-#         restart_policy={"Name": "always"}
-#     )
-#     retries = 10
-#     while retries > 0:
-#         info = node_docker.containers.get(container.attrs['Id'])
-#         status = info.attrs['State']["Status"]
-#         is_running = info.attrs['State']["Running"]
-#         if not is_running:
-#             logger.info("Container is not running, waiting...")
-#             time.sleep(3)
-#             retries -= 1
-#         else:
-#             logger.info(f"Container status: {status}, Is Running: {is_running}")
-#             return utils.get_response(True)
-#
-#     return utils.get_response(
-#         False, f"Container create max retries reached, Container status: {status}, Is Running: {is_running}")
-#
 
 def get_cluster_id():
     out, _, _ = node_utils.run_command(f"cat {cluster_id_file}")

{sbcli_dev-3.8.51 → sbcli_dev-3.8.53}/simplyblock_web/templates/caching_deploy_spdk.yaml.j2

@@ -92,9 +92,9 @@ spec:
             memory: 4096Mi
 
       - name: spdk-proxy-container
-        image: {{ SPDK_IMAGE }}
+        image: {{ SIMPLYBLOCK_DOCKER_IMAGE }}
         imagePullPolicy: "Always"
-        command: ["python", "/root/scripts/spdk_http_proxy_server.py"]
+        command: ["python", "simplyblock_core/services/spdk_http_proxy_server.py"]
         volumeMounts:
         - name: socket-dir
           mountPath: /var/tmp

{sbcli_dev-3.8.51 → sbcli_dev-3.8.53}/simplyblock_web/templates/storage_deploy_spdk.yaml.j2

@@ -95,9 +95,9 @@ spec:
             memory: 4096Mi
 
       - name: spdk-proxy-container
-        image: {{ SPDK_IMAGE }}
+        image: {{ SIMPLYBLOCK_DOCKER_IMAGE }}
         imagePullPolicy: "Always"
-        command: ["python", "/root/scripts/spdk_http_proxy_server.py"]
+        command: ["python", "simplyblock_core/services/spdk_http_proxy_server.py"]
         volumeMounts:
         - name: socket-dir
           mountPath: /var/tmp