sbcli-pre 1.2.4__zip → 1.2.6__zip

{sbcli_pre-1.2.4 → sbcli_pre-1.2.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sbcli-pre
-Version: 1.2.4
+Version: 1.2.6
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy
@@ -21,7 +21,7 @@ Requires-Dist: graypy
 
 [![Python Unit Testing](https://github.com/simplyblock-io/sbcli/actions/workflows/python-testing.yml/badge.svg)](https://github.com/simplyblock-io/sbcli/actions/workflows/python-testing.yml)
 
- 
+
 ## Install
 Add the package repo from AWS CodeArtifact using [awscli](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html)
 
@@ -33,19 +33,34 @@ Install package
 pip install --extra-index-url https://pypi.org/simple sbcli-dev
 ```
 
-# Components 
+# Components
 
 ## Simply Block Core
 Contains core logic and controllers for the simplyblock cluster
 
 ## Simply Block CLI
-Please see this document 
+Please see this document
 [README.md](../main/simplyblock_cli/README.md)
 
 
 ## Simply Block Web API
-Please see this document 
+Please see this document
 [README.md](../main/simplyblock_web/README.md)
 
 
 
+### local development
+
+FoundationDB requires a client library (libfdb_c.dylib) for the Python bindings to interact with the database.
+Depending on the OS architecture, please install the appropriate version from the official github repo
+
+```
+wget https://github.com/apple/foundationdb/releases/download/7.3.3/FoundationDB-7.3.3_arm64.pkg
+```
+
+setup the code on a management node and the webApp code can be developed by building the `docker-compose-dev.yml` file.
+
+
+```
+sudo docker compose -f docker-compose-dev.yml up --build -d
+```

{sbcli_pre-1.2.4 → sbcli_pre-1.2.6}/README.md

@@ -4,7 +4,7 @@
 
 [![Python Unit Testing](https://github.com/simplyblock-io/sbcli/actions/workflows/python-testing.yml/badge.svg)](https://github.com/simplyblock-io/sbcli/actions/workflows/python-testing.yml)
 
- 
+
 ## Install
 Add the package repo from AWS CodeArtifact using [awscli](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html)
 
@@ -16,19 +16,34 @@ Install package
 pip install --extra-index-url https://pypi.org/simple sbcli-dev
 ```
 
-# Components 
+# Components
 
 ## Simply Block Core
 Contains core logic and controllers for the simplyblock cluster
 
 ## Simply Block CLI
-Please see this document 
+Please see this document
 [README.md](../main/simplyblock_cli/README.md)
 
 
 ## Simply Block Web API
-Please see this document 
+Please see this document
 [README.md](../main/simplyblock_web/README.md)
 
 
 
+### local development
+
+FoundationDB requires a client library (libfdb_c.dylib) for the Python bindings to interact with the database.
+Depending on the OS architecture, please install the appropriate version from the official github repo
+
+```
+wget https://github.com/apple/foundationdb/releases/download/7.3.3/FoundationDB-7.3.3_arm64.pkg
+```
+
+setup the code on a management node and the webApp code can be developed by building the `docker-compose-dev.yml` file.
+
+
+```
+sudo docker compose -f docker-compose-dev.yml up --build -d
+```

{sbcli_pre-1.2.4 → sbcli_pre-1.2.6}/env_var

@@ -1,5 +1,5 @@
 SIMPLY_BLOCK_COMMAND_NAME=sbcli-pre
-SIMPLY_BLOCK_VERSION=1.2.4
+SIMPLY_BLOCK_VERSION=1.2.6
 
 SIMPLY_BLOCK_DOCKER_IMAGE=simplyblock/simplyblock:pre-release
 

{sbcli_pre-1.2.4 → sbcli_pre-1.2.6}/sbcli_pre.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sbcli-pre
-Version: 1.2.4
+Version: 1.2.6
 Summary: CLI for managing SimplyBlock cluster
 Home-page: https://www.simplyblock.io/
 Author: Hamdy
@@ -21,7 +21,7 @@ Requires-Dist: graypy
 
 [![Python Unit Testing](https://github.com/simplyblock-io/sbcli/actions/workflows/python-testing.yml/badge.svg)](https://github.com/simplyblock-io/sbcli/actions/workflows/python-testing.yml)
 
- 
+
 ## Install
 Add the package repo from AWS CodeArtifact using [awscli](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html)
 
@@ -33,19 +33,34 @@ Install package
 pip install --extra-index-url https://pypi.org/simple sbcli-dev
 ```
 
-# Components 
+# Components
 
 ## Simply Block Core
 Contains core logic and controllers for the simplyblock cluster
 
 ## Simply Block CLI
-Please see this document 
+Please see this document
 [README.md](../main/simplyblock_cli/README.md)
 
 
 ## Simply Block Web API
-Please see this document 
+Please see this document
 [README.md](../main/simplyblock_web/README.md)
 
 
 
+### local development
+
+FoundationDB requires a client library (libfdb_c.dylib) for the Python bindings to interact with the database.
+Depending on the OS architecture, please install the appropriate version from the official github repo
+
+```
+wget https://github.com/apple/foundationdb/releases/download/7.3.3/FoundationDB-7.3.3_arm64.pkg
+```
+
+setup the code on a management node and the webApp code can be developed by building the `docker-compose-dev.yml` file.
+
+
+```
+sudo docker compose -f docker-compose-dev.yml up --build -d
+```

{sbcli_pre-1.2.4 → sbcli_pre-1.2.6}/sbcli_pre.egg-info/SOURCES.txt

@@ -46,6 +46,7 @@ simplyblock_core/models/base_model.py
 simplyblock_core/models/caching_node.py
 simplyblock_core/models/cluster.py
 simplyblock_core/models/compute_node.py
+simplyblock_core/models/deployer.py
 simplyblock_core/models/events.py
 simplyblock_core/models/global_settings.py
 simplyblock_core/models/iface.py
@@ -117,6 +118,7 @@ simplyblock_web/blueprints/node_api_caching_ks.py
 simplyblock_web/blueprints/snode_ops.py
 simplyblock_web/blueprints/web_api_caching_node.py
 simplyblock_web/blueprints/web_api_cluster.py
+simplyblock_web/blueprints/web_api_deployer.py
 simplyblock_web/blueprints/web_api_device.py
 simplyblock_web/blueprints/web_api_lvol.py
 simplyblock_web/blueprints/web_api_mgmt_node.py

{sbcli_pre-1.2.4 → sbcli_pre-1.2.6}/simplyblock_core/cluster_ops.py

@@ -589,7 +589,7 @@ def update_cluster(cl_id):
     logger.info("Done")
     return True
 
- 
+
 def cluster_grace_startup(cl_id):
     db_controller = DBController()
     cluster = db_controller.get_cluster_by_id(cl_id)
@@ -607,7 +607,7 @@ def cluster_grace_startup(cl_id):
         get_node = db_controller.get_storage_node_by_id(node.get_id())
         if get_node.status != StorageNode.STATUS_ONLINE:
             logger.error("failed to restart node")
-    
+
     return True
 
 
@@ -624,7 +624,7 @@ def cluster_grace_shutdown(cl_id):
         storage_node_ops.suspend_storage_node(node.get_id())
         logger.info(f"Shutting down node: {node.get_id()}")
         storage_node_ops.shutdown_storage_node(node.get_id())
-       
+
     logger.info(f"Suspending cluster: {cl_id}")
     suspend_cluster(cl_id)
     return True

{sbcli_pre-1.2.4 → sbcli_pre-1.2.6}/simplyblock_core/kv_store.py

@@ -9,6 +9,7 @@ from simplyblock_core.models.caching_node import CachingNode
 from simplyblock_core.models.cluster import ClusterMap
 
 from simplyblock_core.models.cluster import Cluster
+from simplyblock_core.models.deployer import Deployer
 from simplyblock_core.models.compute_node import ComputeNode
 from simplyblock_core.models.job_schedule import JobSchedule
 from simplyblock_core.models.port_stat import PortStat
@@ -265,6 +266,14 @@ class DBController:
         if ret:
             return ret[0]
 
+    def get_deployers(self):
+        return Deployer().read_from_db(self.kv_store)
+
+    def get_deployer_by_id(self, deployer_id):
+        ret = Deployer().read_from_db(self.kv_store, id=deployer_id)
+        if ret:
+            return ret[0]
+
     def get_port_stats(self, node_id, port_id, limit=20):
         stats = PortStat().read_from_db(self.kv_store, id="%s/%s" % (node_id, port_id), limit=limit, reverse=True)
         return stats

sbcli_pre-1.2.6/simplyblock_core/models/deployer.py

@@ -0,0 +1,62 @@
+# coding=utf-8
+from simplyblock_core.models.base_model import BaseModel
+
+class Deployer(BaseModel):
+
+    STATUS_ACTIVE = "active"
+    STATUS_READONLY = 'read_only'
+    STATUS_INACTIVE = "inactive"
+    STATUS_SUSPENDED = "suspended"
+    STATUS_DEGRADED = "degraded"
+
+    DOCKER_PULL = "dockerpull"
+    TF_STATE_INIT = "tfstate_init"
+    TF_PLAN = "tfplan"
+    TF_APPLY = "tfapply"
+
+    STATUS_CODE_MAP = {
+        DOCKER_PULL: 0,
+        TF_STATE_INIT: 1,
+        TF_PLAN: 2,
+        TF_APPLY: 3,
+    }
+
+    attributes = {
+        "uuid": {"type": str, 'default': ""},
+        "region": {"type": str, 'default': ""},
+        "availability_zone": {"type": str, 'default': ""},
+        "sbcli_cmd": {"type": str, 'default': ""},
+        "sbcli_pkg_version": {"type": str, 'default': ""},
+        "whitelist_ips": {"type": str, 'default': ""},  # todo: make this a list
+        "mgmt_nodes": {"type": int, 'default': 0},
+        "storage_nodes": {"type": int, 'default': 0},
+        "extra_nodes": {"type": int, 'default': 0},
+        "mgmt_nodes_instance_type": {"type": str, 'default': ""},
+        "storage_nodes_instance_type": {"type": str, 'default': ""},
+        "extra_nodes_instance_type": {"type": str, 'default': ""},
+        "storage_nodes_ebs_size1": {"type": int, 'default': 0},  # size in GB
+        "storage_nodes_ebs_size2": {"type": int, 'default': 0},  # size in GB
+        "volumes_per_storage_nodes": {"type": int, 'default': 0},
+        "nr_hugepages": {"type": int, 'default': 0},
+        "tf_state_bucket_name": {"type": str, 'default': ""},
+        "tf_state_bucket_region": {"type": str, 'default': ""},
+        "tf_workspace": {"type": str, 'default': ""},
+        "status": {"type": str, 'default': ""},
+        "tf_logs_bucket_name": {"type": str, 'default': ""},
+        "ecr_account_id": {"type": str, 'default': ""},
+        "ecr_region": {"type": str, 'default': ""},
+        "ecr_repository_name": {"type": str, 'default': ""},
+        "ecr_image_tag": {"type": str, 'default': ""},
+        "tf_output": {"type": str, 'default': ""},
+    }
+
+    def __init__(self, data=None):
+        super(Deployer, self).__init__()
+        self.set_attrs(self.attributes, data)
+        self.object_type = "object"
+
+    def get_id(self):
+        return self.uuid
+
+    def get_status_code(self):
+        return self.status

{sbcli_pre-1.2.4 → sbcli_pre-1.2.6}/simplyblock_core/scripts/deploy_stack.sh

@@ -21,6 +21,13 @@ fi
 
 docker network create monitoring-net -d overlay --attachable
 
+INSTANCE_ID=$(curl -s http://169.254.169.254/latest/meta-data/instance-id)
+
+if [ -n "$INSTANCE_ID" ] 
+then
+  export USE_EFS="rexray/efs"
+fi
+
 docker stack deploy --compose-file="$DIR"/docker-compose-swarm-monitoring.yml monitoring
 
 # wait for the services to become online

{sbcli_pre-1.2.4 → sbcli_pre-1.2.6}/simplyblock_core/scripts/docker-compose-swarm-monitoring.yml

@@ -41,6 +41,7 @@ services:
       GRAYLOG_HTTP_EXTERNAL_URI: "http://localhost/graylog/"
       GRAYLOG_ELASTICSEARCH_HOSTS: "http://opensearch:9200"
       GRAYLOG_MONGODB_URI: "mongodb://mongodb:27017/graylog"
+      GRAYLOG_SKIP_PREFLIGHT_CHECKS: "true"
     ports:
     - "5044:5044/tcp"   # Beats
     - "5140:5140/udp"   # Syslog
@@ -64,7 +65,7 @@ services:
     image: simplyblock/promagent
     environment:
       ClusterID: "${CLUSTER_ID}"
-      ClusterIP: "${CLUSTER_IP}"
+      ClusterIP: "HAProxy"
       ClusterSecret: "${CLUSTER_SECRET}"
     deploy:
       placement:
@@ -151,12 +152,19 @@ services:
 
 volumes:
   mongodb_data:
+    driver: ${USE_EFS:-local}
   os_data:
+    driver: ${USE_EFS:-local}
   graylog_data:
+    driver: ${USE_EFS:-local}
   graylog_journal:
+    driver: ${USE_EFS:-local}
   grafana_data:
+    driver: ${USE_EFS:-local}
+  graylog_config:
+    driver: ${USE_EFS:-local}  
   prometheus_data:
-  alertmanager_data:
+    driver: ${USE_EFS:-local}
 
 networks:
   monitoring-net:

{sbcli_pre-1.2.4 → sbcli_pre-1.2.6}/simplyblock_core/scripts/install_deps.sh

@@ -15,6 +15,8 @@ sudo yum install hostname pkg-config git wget python3-pip yum-utils docker-ce do
 sudo systemctl enable docker
 sudo systemctl start docker
 
+sudo docker plugin install --grant-all-permissions rexray/efs REXRAY_PREEMPT=true
+
 wget https://github.com/apple/foundationdb/releases/download/7.3.3/foundationdb-clients-7.3.3-1.el7.x86_64.rpm -q
 sudo rpm -U foundationdb-clients-7.3.3-1.el7.x86_64.rpm --quiet --reinstall
 rm -f foundationdb-clients-7.3.3-1.el7.x86_64.rpm

{sbcli_pre-1.2.4 → sbcli_pre-1.2.6}/simplyblock_core/scripts/stack_deploy_wait.sh

@@ -6,7 +6,7 @@ opt_h=0
 opt_l=""
 opt_r=0
 opt_s=5
-opt_t=600 # 10 min
+opt_t=900 # 15 min
 
 start_epoc=$(date +%s)
 cmd_min_timeout=15

{sbcli_pre-1.2.4 → sbcli_pre-1.2.6}/simplyblock_web/app.py

@@ -7,7 +7,7 @@ from flask import Flask
 
 import utils
 from blueprints import web_api_cluster, web_api_mgmt_node, web_api_device, \
-    web_api_lvol, web_api_storage_node, web_api_pool, web_api_caching_node, web_api_snapshot
+    web_api_lvol, web_api_storage_node, web_api_pool, web_api_caching_node, web_api_snapshot, web_api_deployer
 from auth_middleware import token_required
 from simplyblock_core import constants
 
@@ -31,6 +31,7 @@ app.register_blueprint(web_api_snapshot.bp)
 app.register_blueprint(web_api_storage_node.bp)
 app.register_blueprint(web_api_pool.bp)
 app.register_blueprint(web_api_caching_node.bp)
+app.register_blueprint(web_api_deployer.bp)
 
 
 @app.before_request