sbcli-dev 4.0.50__zip → 4.0.51__zip

{sbcli_dev-4.0.50 → sbcli_dev-4.0.51}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sbcli-dev
-Version: 4.0.50
+Version: 4.0.51
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

sbcli_dev-4.0.51/env_var

@@ -0,0 +1,7 @@
+SIMPLY_BLOCK_COMMAND_NAME=sbcli-dev
+SIMPLY_BLOCK_VERSION=4.0.51
+
+
+SIMPLY_BLOCK_DOCKER_IMAGE=simplyblock/simplyblock:main
+
+DOCKER_USER=hamdysimplyblock

{sbcli_dev-4.0.50 → sbcli_dev-4.0.51}/sbcli_dev.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sbcli-dev
-Version: 4.0.50
+Version: 4.0.51
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy

{sbcli_dev-4.0.50 → sbcli_dev-4.0.51}/simplyblock_core/cluster_ops.py

@@ -77,7 +77,7 @@ def _add_graylog_input(cluster_ip, password):
     url = f"http://{cluster_ip}/graylog/api/system/inputs"
     payload = json.dumps({
         "title": "spdk log input",
-        "type": "org.graylog2.inputs.gelf.udp.GELFUDPInput",
+        "type": "org.graylog2.inputs.gelf.tcp.GELFTCPInput",
         "configuration": {
             "bind_address": "0.0.0.0",
             "port": 12201,

{sbcli_dev-4.0.50 → sbcli_dev-4.0.51}/simplyblock_core/constants.py

@@ -59,7 +59,7 @@ TASK_EXEC_RETRY_COUNT = 8
 
 SIMPLY_BLOCK_SPDK_ULTRA_IMAGE = "simplyblock/spdk:main-latest"
 
-GELF_PORT = 12201
+GELF_PORT = 12202
 
 MIN_HUGE_PAGE_MEMORY_FOR_LVOL = 209715200
 MIN_SYS_MEMORY_FOR_LVOL = 524288000

{sbcli_dev-4.0.50 → sbcli_dev-4.0.51}/simplyblock_core/controllers/events_controller.py

@@ -7,11 +7,11 @@ from simplyblock_core.models.events import EventObj
 from simplyblock_core.kv_store import DBController
 from simplyblock_core import constants
 
-from graypy import GELFUDPHandler
+from graypy import GELFTCPHandler
 
 # configure logging
 logging.captureWarnings(True)
-gelf_handler = GELFUDPHandler('0.0.0.0', constants.GELF_PORT)
+gelf_handler = GELFTCPHandler('0.0.0.0', constants.GELF_PORT)
 logger = logging.getLogger()
 logger.setLevel(logging.DEBUG)
 logger.addHandler(gelf_handler)

{sbcli_dev-4.0.50 → sbcli_dev-4.0.51}/simplyblock_core/scripts/docker-compose-swarm.yml

@@ -123,7 +123,7 @@ services:
         constraints: [node.role == manager]
     ports:
       - 80:80
-      - 8404:8404
+      - 12202:12202
     networks:
       - localnet
       - monitoring-net

{sbcli_dev-4.0.50 → sbcli_dev-4.0.51}/simplyblock_core/scripts/haproxy.cfg

@@ -52,24 +52,30 @@ backend graylog_services
     http-request set-path %[path,regsub(^/graylog/?,/)]
     server-template graylog- 1 graylog:9000 check resolvers docker init-addr libc,none
 
-backend graylog_input_services
-    balance roundrobin
-    http-request set-path %[path,regsub(^/graylog_input/?,/)]
-    server-template graylog_input- 1 graylog:12201 check resolvers docker init-addr libc,none
-
-frontend stats_front
-    bind *:8404
+backend haproxy_services
+    http-request set-path %[path,regsub(^/haproxy/?,/)]
     stats enable
     stats uri /
     stats refresh 10s
     stats show-legends
     stats show-node
 
+backend graylog_input_services
+    mode tcp
+    balance roundrobin
+    server-template graylog_input- 1 graylog:12201 check resolvers docker init-addr libc,none
+
+
 frontend web_api_front
     bind *:80
 
     use_backend grafana_services if { path /grafana } || { path_beg /grafana/ }
     use_backend graylog_services if { path /graylog } || { path_beg /graylog/ }
-    use_backend graylog_input_services if { path /graylog_input } || { path_beg /graylog_input/ }
+    use_backend haproxy_services if { path /haproxy } || { path_beg /haproxy/ }
 
     default_backend wep_api_services
+
+frontend graylog_input_front
+    bind *:12202
+    mode tcp
+    default_backend graylog_input_services

{sbcli_dev-4.0.50 → sbcli_dev-4.0.51}/simplyblock_core/services/log_agg_service.py

@@ -9,12 +9,12 @@ from simplyblock_core import constants, kv_store, utils
 from simplyblock_core.models.stats import DeviceStatObject, NodeStatObject, ClusterStatObject
 
 # Import the GELF logger
-from graypy import GELFUDPHandler
+from graypy import GELFTCPHandler
 
 # configure logging
 logger_handler = logging.StreamHandler(stream=sys.stdout)
 logger_handler.setFormatter(logging.Formatter('%(asctime)s: %(levelname)s: %(message)s'))
-gelf_handler = GELFUDPHandler('0.0.0.0', constants.GELF_PORT)
+gelf_handler = GELFTCPHandler('0.0.0.0', constants.GELF_PORT)
 logger = logging.getLogger()
 logger.addHandler(gelf_handler)
 logger.addHandler(logger_handler)

{sbcli_dev-4.0.50 → sbcli_dev-4.0.51}/simplyblock_core/services/new_device_discovery.py

@@ -8,14 +8,14 @@ from simplyblock_core.models.nvme_device import NVMeDevice
 from simplyblock_core.models.storage_node import StorageNode
 
 # Import the GELF logger
-from graypy import GELFUDPHandler
+from graypy import GELFTCPHandler
 
 from simplyblock_core.snode_client import SNodeClient
 
 # configure logging
 logger_handler = logging.StreamHandler(stream=sys.stdout)
 logger_handler.setFormatter(logging.Formatter('%(asctime)s: %(levelname)s: %(message)s'))
-gelf_handler = GELFUDPHandler('0.0.0.0', constants.GELF_PORT)
+gelf_handler = GELFTCPHandler('0.0.0.0', constants.GELF_PORT)
 logger = logging.getLogger()
 logger.addHandler(gelf_handler)
 logger.addHandler(logger_handler)

{sbcli_dev-4.0.50 → sbcli_dev-4.0.51}/simplyblock_core/services/tasks_runner_failed_migration.py

@@ -10,7 +10,7 @@ from simplyblock_core.models.job_schedule import JobSchedule
 
 
 # Import the GELF logger
-from graypy import GELFUDPHandler
+from graypy import GELFTCPHandler
 
 from simplyblock_core.models.nvme_device import NVMeDevice
 from simplyblock_core.models.storage_node import StorageNode
@@ -111,7 +111,7 @@ def task_runner(task):
 # configure logging
 logger_handler = logging.StreamHandler(stream=sys.stdout)
 logger_handler.setFormatter(logging.Formatter('%(asctime)s: %(levelname)s: %(message)s'))
-gelf_handler = GELFUDPHandler('0.0.0.0', constants.GELF_PORT)
+gelf_handler = GELFTCPHandler('0.0.0.0', constants.GELF_PORT)
 logger = logging.getLogger()
 logger.addHandler(gelf_handler)
 logger.addHandler(logger_handler)

{sbcli_dev-4.0.50 → sbcli_dev-4.0.51}/simplyblock_core/services/tasks_runner_new_dev_migration.py

@@ -10,7 +10,7 @@ from simplyblock_core.models.job_schedule import JobSchedule
 
 
 # Import the GELF logger
-from graypy import GELFUDPHandler
+from graypy import GELFTCPHandler
 
 from simplyblock_core.models.nvme_device import NVMeDevice
 from simplyblock_core.models.storage_node import StorageNode
@@ -123,7 +123,7 @@ def task_runner(task):
 # configure logging
 logger_handler = logging.StreamHandler(stream=sys.stdout)
 logger_handler.setFormatter(logging.Formatter('%(asctime)s: %(levelname)s: %(message)s'))
-gelf_handler = GELFUDPHandler('0.0.0.0', constants.GELF_PORT)
+gelf_handler = GELFTCPHandler('0.0.0.0', constants.GELF_PORT)
 logger = logging.getLogger()
 logger.addHandler(gelf_handler)
 logger.addHandler(logger_handler)

{sbcli_dev-4.0.50 → sbcli_dev-4.0.51}/simplyblock_core/services/tasks_runner_restart.py

@@ -141,13 +141,6 @@ def task_runner_node(task):
         task.write_to_db(db_controller.kv_store)
         return True
 
-    if node.status == StorageNode.STATUS_RESTARTING:
-        logger.info(f"Node is restarting, stopping task")
-        task.function_result = f"Node is restarting"
-        task.status = JobSchedule.STATUS_DONE
-        task.write_to_db(db_controller.kv_store)
-        return True
-
     if node.status == StorageNode.STATUS_SCHEDULABLE:
         logger.info(f"Node is schedulable, stopping task")
         task.function_result = f"Node is schedulable"
@@ -169,6 +162,12 @@ def task_runner_node(task):
         return True
 
     if task.status != JobSchedule.STATUS_RUNNING:
+        if node.status == StorageNode.STATUS_RESTARTING:
+            logger.info(f"Node is restarting, stopping task")
+            task.function_result = f"Node is restarting"
+            task.status = JobSchedule.STATUS_DONE
+            task.write_to_db(db_controller.kv_store)
+            return True
         task.status = JobSchedule.STATUS_RUNNING
         task.write_to_db(db_controller.kv_store)
         tasks_events.task_updated(task)

{sbcli_dev-4.0.50 → sbcli_dev-4.0.51}/simplyblock_core/utils.py

@@ -11,7 +11,7 @@ import sys
 
 import docker
 from prettytable import PrettyTable
-from graypy import GELFUDPHandler
+from graypy import GELFTCPHandler
 
 from simplyblock_core import constants
 from simplyblock_core import shell_utils
@@ -562,7 +562,7 @@ def get_logger(name):
     logger_handler.setFormatter(logging.Formatter('%(asctime)s: %(levelname)s: %(message)s'))
     logger.addHandler(logger_handler)
 
-    gelf_handler = GELFUDPHandler('0.0.0.0', constants.GELF_PORT)
+    gelf_handler = GELFTCPHandler('0.0.0.0', constants.GELF_PORT)
     logger.addHandler(gelf_handler)
     return logging.getLogger(name)
 

{sbcli_dev-4.0.50 → sbcli_dev-4.0.51}/simplyblock_web/blueprints/snode_ops.py

@@ -155,7 +155,7 @@ def spdk_process_start():
 
     if "cluster_ip" in data and data['cluster_ip']:
         cluster_ip = data['cluster_ip']
-        log_config = LogConfig(type=LogConfig.types.GELF, config={"gelf-address": f"udp://{cluster_ip}:12201"})
+        log_config = LogConfig(type=LogConfig.types.GELF, config={"gelf-address": f"tcp://{cluster_ip}:12202"})
     else:
         log_config = LogConfig(type=LogConfig.types.JOURNALD)
 
@@ -182,7 +182,7 @@ def spdk_process_start():
         name="spdk_proxy",
         detach=True,
         network_mode="host",
-        log_config=LogConfig(type=LogConfig.types.JOURNALD),
+        log_config=log_config,
         volumes=[
             '/var/tmp:/var/tmp'
         ],

{sbcli_dev-4.0.50 → sbcli_dev-4.0.51}/simplyblock_web/blueprints/snode_ops_k8s.py

@@ -387,6 +387,7 @@ def spdk_process_start():
             'NAMESPACE': namespace,
             'FDB_CONNECTION': fdb_connection,
             'SIMPLYBLOCK_DOCKER_IMAGE': constants.SIMPLY_BLOCK_DOCKER_IMAGE,
+            'GRAYLOG_SERVER_IP': data['cluster_ip']
         }
         dep = yaml.safe_load(template.render(values))
         logger.debug(dep)

{sbcli_dev-4.0.50 → sbcli_dev-4.0.51}/simplyblock_web/templates/storage_deploy_spdk.yaml.j2

@@ -13,6 +13,7 @@ spec:
       labels:
         app: spdk-app-{{ HOSTNAME }}
     spec:
+      serviceAccountName: storage-node-sa
       nodeSelector:
         kubernetes.io/hostname: {{ HOSTNAME }}
       hostNetwork: true
@@ -43,6 +44,12 @@ spec:
             medium: HugePages
         - name: script-volume
           emptyDir: {}
+        - name: varlog
+          hostPath:
+            path: /var/log
+        - name: dockercontainerlogdirectory
+          hostPath:
+            path: /var/log/pods
       tolerations:
         - effect: NoSchedule
           key: dedicated
@@ -110,3 +117,37 @@ spec:
           value: "{{ RPC_USERNAME }}"
         - name: RPC_PASSWORD
           value: "{{ RPC_PASSWORD }}"
+
+      - name: fluentd
+        image: fluent/fluentd-kubernetes-daemonset:v1.17.1-debian-graylog-1.2
+        imagePullPolicy: "Always"
+        env:
+        - name: K8S_NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        - name:  FLUENT_GRAYLOG_HOST
+          value: "{{ GRAYLOG_SERVER_IP }}"
+        - name:  FLUENT_GRAYLOG_PORT
+          value: "12202"
+        - name:  FLUENT_GRAYLOG_PROTOCOL
+          value: "tcp"
+        - name:  FLUENTD_SYSTEMD_CONF
+          value: "disable"
+        - name: FLUENT_CONTAINER_TAIL_EXCLUDE_PATH
+          value: /var/log/pods/fluent*
+        - name: FLUENT_CONTAINER_TAIL_PARSER_TYPE
+          value: /^(?<time>.+) (?<stream>stdout|stderr)( (?<logtag>.))? (?<log>.*)$/
+        resources:
+          requests:
+            cpu: 200m
+            memory: 0.5Gi
+          limits:
+            cpu: 1000m
+            memory: 1Gi
+        volumeMounts:
+        - name: varlog
+          mountPath: /var/log
+        - name: dockercontainerlogdirectory
+          mountPath: /var/log/pods
+          readOnly: true

sbcli_dev-4.0.50/env_var

@@ -1,7 +0,0 @@
-SIMPLY_BLOCK_COMMAND_NAME=sbcli-dev
-SIMPLY_BLOCK_VERSION=4.0.50
-
-
-SIMPLY_BLOCK_DOCKER_IMAGE=simplyblock/simplyblock:main-dev
-
-DOCKER_USER=hamdysimplyblock