sbcli-pre 1.2.4__zip → 1.2.5__zip

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (141) hide show
  1. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/PKG-INFO +20 -5
  2. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/README.md +19 -4
  3. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/env_var +1 -1
  4. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/sbcli_pre.egg-info/PKG-INFO +20 -5
  5. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/sbcli_pre.egg-info/SOURCES.txt +5 -5
  6. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_cli/cli.py +115 -113
  7. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/cluster_ops.py +238 -141
  8. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/constants.py +7 -5
  9. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/controllers/caching_node_controller.py +6 -8
  10. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/controllers/cluster_events.py +0 -9
  11. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/controllers/device_controller.py +63 -56
  12. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/controllers/events_controller.py +3 -5
  13. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/controllers/health_controller.py +40 -30
  14. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/controllers/lvol_controller.py +38 -51
  15. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/controllers/pool_controller.py +4 -8
  16. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/controllers/snapshot_controller.py +3 -9
  17. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/distr_controller.py +9 -13
  18. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/kv_store.py +29 -47
  19. sbcli_pre-1.2.5/simplyblock_core/mgmt_node_ops.py +80 -0
  20. sbcli_pre-1.2.5/simplyblock_core/models/deployer.py +62 -0
  21. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/models/events.py +1 -9
  22. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/models/job_schedule.py +0 -6
  23. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/models/nvme_device.py +4 -42
  24. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/models/storage_node.py +1 -9
  25. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/rpc_client.py +10 -55
  26. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/scripts/__init__.py +4 -0
  27. sbcli_pre-1.2.4/simplyblock_core/scripts/alerting/alert_resources.yaml.j2 → sbcli_pre-1.2.5/simplyblock_core/scripts/alerting/alert_resources.yaml +5 -54
  28. sbcli_pre-1.2.5/simplyblock_core/scripts/apply_dashboard.sh +22 -0
  29. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/scripts/dashboards/cluster.json +1 -1
  30. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/scripts/deploy_stack.sh +0 -2
  31. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/scripts/docker-compose-swarm-monitoring.yml +13 -22
  32. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/scripts/docker-compose-swarm.yml +2 -17
  33. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/scripts/haproxy.cfg +0 -15
  34. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/scripts/install_deps.sh +0 -1
  35. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/services/capacity_and_stats_collector.py +1 -1
  36. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/services/device_monitor.py +46 -5
  37. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/services/distr_event_collector.py +11 -10
  38. sbcli_pre-1.2.5/simplyblock_core/services/health_check_service.py +136 -0
  39. sbcli_pre-1.2.4/simplyblock_core/services/tasks_runner_restart.py → sbcli_pre-1.2.5/simplyblock_core/services/job_tasks.py +46 -95
  40. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/services/lvol_monitor.py +1 -1
  41. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/services/lvol_stat_collector.py +1 -1
  42. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/services/port_stat_collector.py +1 -0
  43. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/services/storage_node_monitor.py +44 -49
  44. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/snode_client.py +0 -12
  45. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/storage_node_ops.py +336 -525
  46. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/utils.py +1 -46
  47. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_web/app.py +2 -1
  48. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_web/blueprints/snode_ops.py +25 -103
  49. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_web/blueprints/web_api_cluster.py +43 -20
  50. sbcli_pre-1.2.5/simplyblock_web/blueprints/web_api_deployer.py +394 -0
  51. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_web/blueprints/web_api_device.py +7 -10
  52. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_web/blueprints/web_api_lvol.py +5 -9
  53. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_web/blueprints/web_api_pool.py +5 -14
  54. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_web/blueprints/web_api_storage_node.py +10 -3
  55. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_web/node_utils.py +2 -0
  56. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_web/utils.py +0 -8
  57. sbcli_pre-1.2.4/simplyblock_core/controllers/tasks_controller.py +0 -103
  58. sbcli_pre-1.2.4/simplyblock_core/controllers/tasks_events.py +0 -37
  59. sbcli_pre-1.2.4/simplyblock_core/mgmt_node_ops.py +0 -205
  60. sbcli_pre-1.2.4/simplyblock_core/services/health_check_service.py +0 -134
  61. sbcli_pre-1.2.4/simplyblock_core/services/tasks_runner_migration.py +0 -61
  62. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/pyproject.toml +0 -0
  63. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/sbcli_pre.egg-info/dependency_links.txt +0 -0
  64. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/sbcli_pre.egg-info/entry_points.txt +0 -0
  65. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/sbcli_pre.egg-info/requires.txt +0 -0
  66. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/sbcli_pre.egg-info/top_level.txt +0 -0
  67. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/setup.cfg +0 -0
  68. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/setup.py +0 -0
  69. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_cli/main.py +0 -0
  70. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/__init__.py +0 -0
  71. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/cnode_client.py +0 -0
  72. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/compute_node_ops.py +0 -0
  73. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/controllers/__init__.py +0 -0
  74. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/controllers/device_events.py +0 -0
  75. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/controllers/lvol_events.py +0 -0
  76. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/controllers/mgmt_events.py +0 -0
  77. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/controllers/pool_events.py +0 -0
  78. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/controllers/snapshot_events.py +0 -0
  79. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/controllers/storage_events.py +0 -0
  80. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/models/__init__.py +0 -0
  81. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/models/base_model.py +0 -0
  82. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/models/caching_node.py +0 -0
  83. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/models/cluster.py +0 -0
  84. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/models/compute_node.py +0 -0
  85. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/models/global_settings.py +0 -0
  86. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/models/iface.py +0 -0
  87. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/models/lvol_model.py +0 -0
  88. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/models/mgmt_node.py +0 -0
  89. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/models/pool.py +0 -0
  90. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/models/port_stat.py +0 -0
  91. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/models/snapshot.py +0 -0
  92. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/models/stats.py +0 -0
  93. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/pci_utils.py +0 -0
  94. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/scripts/alerting/alert_rules.yaml +0 -0
  95. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/scripts/clean_local_storage_deploy.sh +0 -0
  96. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/scripts/config_docker.sh +0 -0
  97. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/scripts/dashboards/devices.json +0 -0
  98. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/scripts/dashboards/lvols.json +0 -0
  99. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/scripts/dashboards/node-exporter.json +0 -0
  100. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/scripts/dashboards/nodes.json +0 -0
  101. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/scripts/dashboards/pools.json +0 -0
  102. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/scripts/datasource.yml +0 -0
  103. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/scripts/db_config_double.sh +0 -0
  104. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/scripts/db_config_single.sh +0 -0
  105. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/scripts/prometheus.yml +0 -0
  106. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/scripts/run_ssh.sh +0 -0
  107. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/scripts/set_db_config.sh +0 -0
  108. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/scripts/stack_deploy_wait.sh +0 -0
  109. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/services/__init__.py +0 -0
  110. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/services/caching_node_monitor.py +0 -0
  111. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/services/cap_monitor.py +0 -0
  112. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/services/install_service.sh +0 -0
  113. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/services/log_agg_service.py +0 -0
  114. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/services/mgmt_node_monitor.py +0 -0
  115. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/services/remove_service.sh +0 -0
  116. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/services/service_template.service +0 -0
  117. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_core/shell_utils.py +0 -0
  118. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_web/__init__.py +0 -0
  119. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_web/auth_middleware.py +0 -0
  120. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_web/blueprints/__init__.py +0 -0
  121. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_web/blueprints/caching_node_ops.py +0 -0
  122. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_web/blueprints/caching_node_ops_k8s.py +0 -0
  123. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_web/blueprints/node_api_basic.py +0 -0
  124. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_web/blueprints/node_api_caching_docker.py +0 -0
  125. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_web/blueprints/node_api_caching_ks.py +0 -0
  126. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_web/blueprints/web_api_caching_node.py +0 -0
  127. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_web/blueprints/web_api_mgmt_node.py +0 -0
  128. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_web/blueprints/web_api_snapshot.py +0 -0
  129. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_web/caching_node_app.py +0 -0
  130. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_web/caching_node_app_k8s.py +0 -0
  131. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_web/node_webapp.py +0 -0
  132. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_web/snode_app.py +0 -0
  133. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_web/static/delete.py +0 -0
  134. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_web/static/deploy.py +0 -0
  135. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_web/static/deploy_cnode.yaml +0 -0
  136. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_web/static/deploy_spdk.yaml +0 -0
  137. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_web/static/is_up.py +0 -0
  138. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_web/static/list_deps.py +0 -0
  139. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_web/static/rpac.yaml +0 -0
  140. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_web/static/tst.py +0 -0
  141. {sbcli_pre-1.2.4 → sbcli_pre-1.2.5}/simplyblock_web/templates/deploy_spdk.yaml.j2 +0 -0
@@ -0,0 +1,394 @@
1
+ #!/usr/bin/env python
2
+ # encoding: utf-8
3
+ import logging
4
+ import boto3
5
+ import time
6
+ from botocore.exceptions import ClientError
7
+ import threading
8
+ import uuid
9
+
10
+ from flask import Blueprint
11
+ from flask import request
12
+
13
+ from simplyblock_web import utils
14
+
15
+ from simplyblock_core import kv_store
16
+ from simplyblock_core.models.deployer import Deployer
17
+
18
+ logger = logging.getLogger(__name__)
19
+ logger.setLevel(logging.DEBUG)
20
+ bp = Blueprint("deployer", __name__)
21
+ db_controller = kv_store.DBController()
22
+
23
+
24
+ ## Terraform variables
25
+ document_name = 'AWS-RunShellScript'
26
+ output_key_prefix = 'ssm-output'
27
+
28
+ # intialise clients
29
+ ssm = boto3.client('ssm', region_name='us-east-1')
30
+ s3 = boto3.client('s3', region_name='us-east-1')
31
+
32
+
33
+ def get_instance_tf_engine_instance_id():
34
+ tag_value = 'tfengine'
35
+ tag_key = 'Name'
36
+
37
+ ec2 = boto3.client('ec2', region_name='us-east-1')
38
+ response = ec2.describe_instances(
39
+ Filters=[
40
+ {
41
+ 'Name': f'tag:{tag_key}',
42
+ 'Values': [tag_value]
43
+ },
44
+ {
45
+ 'Name': 'instance-state-name',
46
+ 'Values': ['running']
47
+ }
48
+ ]
49
+ )
50
+
51
+ # Extract instance IDs
52
+ instance_ids = [
53
+ instance['InstanceId']
54
+ for reservation in response['Reservations']
55
+ for instance in reservation['Instances']
56
+ ]
57
+ return instance_ids
58
+
59
+ def check_command_status(ssm_client, command_id, instance_id):
60
+ # time.sleep(1) # wait for a second to avoid: An error occurred (InvocationDoesNotExist)
61
+ try:
62
+ response = ssm_client.get_command_invocation(
63
+ CommandId=command_id,
64
+ InstanceId=instance_id
65
+ )
66
+ return response['Status']
67
+ except Exception as e:
68
+ print(f"Exeception {e}")
69
+ return "Exeception"
70
+
71
+
72
+ def wait_for_s3_object(s3_client, bucket, key, timeout=300, interval=5):
73
+ elapsed_time = 0
74
+
75
+ while elapsed_time < timeout:
76
+ try:
77
+ s3_client.head_object(Bucket=bucket, Key=key)
78
+ return True
79
+ except ClientError as e:
80
+ if e.response['Error']['Code'] == '404':
81
+ time.sleep(interval)
82
+ elapsed_time += interval
83
+ else:
84
+ raise e
85
+ return False
86
+
87
+ # todo: fetch both STDOUT and STDERR based on
88
+ def display_logs(command_id, instance_id, status, tf_logs_bucket_name):
89
+ output_s3_key = f'{output_key_prefix}/{command_id}/{instance_id}/awsrunShellScript/0.awsrunShellScript/stdout'
90
+ error_s3_key = f'{output_key_prefix}/{command_id}/{instance_id}/awsrunShellScript/0.awsrunShellScript/stderr'
91
+
92
+ stdout_content = ''
93
+ stderr_content = ''
94
+ # if success get STDOUT. else get STDERR
95
+ if status == 'Success':
96
+ wait_for_s3_object(s3, tf_logs_bucket_name, output_s3_key)
97
+ try:
98
+ stdout_object = s3.get_object(Bucket=tf_logs_bucket_name, Key=output_s3_key)
99
+ stdout_content = stdout_object['Body'].read().decode('utf-8')
100
+ print('Output:')
101
+ print(stdout_content)
102
+ except ClientError as ex:
103
+ if ex.response['Error']['Code'] == 'NoSuchKey':
104
+ print('No object found - returning empty')
105
+ else:
106
+ wait_for_s3_object(s3, tf_logs_bucket_name, error_s3_key)
107
+ try:
108
+ stderr_object = s3.get_object(Bucket=tf_logs_bucket_name, Key=error_s3_key)
109
+ stderr_content = stderr_object['Body'].read().decode('utf-8')
110
+ print('Error:')
111
+ print(stderr_content)
112
+ except ClientError as ex:
113
+ if ex.response['Error']['Code'] == 'NoSuchKey':
114
+ print('No object found - returning empty')
115
+
116
+ return stdout_content, stderr_content
117
+
118
+ def wait_for_status(command_id, instance_id):
119
+ status = 'Pending'
120
+ while status not in ['Success', 'Failed', 'Cancelled', 'TimedOut']:
121
+ status = check_command_status(ssm, command_id, instance_id)
122
+ print(f'Current status: {status}')
123
+ if status in ['Success', 'Failed', 'Cancelled', 'TimedOut']:
124
+ break
125
+ time.sleep(5) # Wait for 5 seconds before checking the status again
126
+
127
+ return status
128
+
129
+ def update_cluster(d, kv_store, storage_nodes, availability_zone):
130
+
131
+ print('started update_cluster')
132
+ TFSTATE_BUCKET=d.tf_state_bucket_name
133
+ TFSTATE_KEY='csi'
134
+ TFSTATE_REGION=d.tf_state_bucket_region
135
+ TF_WORKSPACE=d.tf_workspace
136
+
137
+ mgmt_nodes = d.mgmt_nodes
138
+ aws_region = d.region
139
+ ECR_REGION = d.ecr_region
140
+ ECR_REPOSITORY_NAME = d.ecr_repository_name
141
+ ECR_IMAGE_TAG = d.ecr_image_tag
142
+ ECR_ACCOUNT_ID = d.ecr_account_id
143
+ tf_logs_bucket_name = d.tf_logs_bucket_name
144
+
145
+ d.status = "in_progress"
146
+ d.write_to_db(kv_store)
147
+
148
+ instance_ids = get_instance_tf_engine_instance_id()
149
+ if len(instance_ids) == 0:
150
+ # wait for a min and try again before returning error on the API
151
+ print('no instance IDs')
152
+ d.status = "no instance IDs"
153
+ d.write_to_db(kv_store)
154
+ return False, "", "no instance IDs"
155
+
156
+ commands = [
157
+ f"""
158
+ docker pull $ECR_ACCOUNT_ID.dkr.ecr.{ECR_REGION}.amazonaws.com/{ECR_REPOSITORY_NAME}:{ECR_IMAGE_TAG}
159
+ docker volume create terraform
160
+ """,
161
+ f"""
162
+ docker run --rm -v terraform:/app -e TF_LOG=DEBUG -w /app {ECR_ACCOUNT_ID}.dkr.ecr.{ECR_REGION}.amazonaws.com/{ECR_REPOSITORY_NAME}:{ECR_IMAGE_TAG} \
163
+ workspace select -or-create {TF_WORKSPACE}
164
+ """,
165
+ f"""
166
+ docker run --rm -v terraform:/app -w /app {ECR_ACCOUNT_ID}.dkr.ecr.{ECR_REGION}.amazonaws.com/{ECR_REPOSITORY_NAME}:{ECR_IMAGE_TAG} \
167
+ init -reconfigure -input=false \
168
+ -backend-config='bucket={TFSTATE_BUCKET}' \
169
+ -backend-config='key={TFSTATE_KEY}' \
170
+ -backend-config='region={TFSTATE_REGION}'
171
+ """,
172
+ f"""
173
+ docker run --rm -v terraform:/app -w /app {ECR_ACCOUNT_ID}.dkr.ecr.{ECR_REGION}.amazonaws.com/{ECR_REPOSITORY_NAME}:{ECR_IMAGE_TAG} \
174
+ plan -var mgmt_nodes={mgmt_nodes} -var storage_nodes={storage_nodes} -var az={availability_zone} -var region={aws_region}
175
+ """,
176
+ f"""
177
+ docker run --rm -v terraform:/app -w /app {ECR_ACCOUNT_ID}.dkr.ecr.{ECR_REGION}.amazonaws.com/{ECR_REPOSITORY_NAME}:{ECR_IMAGE_TAG} \
178
+ apply -var mgmt_nodes={mgmt_nodes} -var storage_nodes={storage_nodes} -var az={availability_zone} -var region={aws_region} --auto-approve
179
+ """
180
+ ]
181
+
182
+ # Send command with S3 output parameters
183
+ response = ssm.send_command(
184
+ InstanceIds=instance_ids,
185
+ DocumentName=document_name,
186
+ Parameters={
187
+ 'commands': commands
188
+ },
189
+ OutputS3BucketName=tf_logs_bucket_name,
190
+ OutputS3KeyPrefix=output_key_prefix
191
+ )
192
+
193
+ command_id = response['Command']['CommandId']
194
+ print(f'Command ID: {command_id}')
195
+ d.status = f"waiting for status. commandID: {command_id}"
196
+ d.write_to_db(kv_store)
197
+
198
+ d.status = wait_for_status(command_id, instance_ids[0])
199
+ d.write_to_db(kv_store)
200
+
201
+ if d.status == "Success":
202
+ # get terraform outputs
203
+ commands = [
204
+ f"""
205
+ docker run --rm -v terraform:/app -w /app {ECR_ACCOUNT_ID}.dkr.ecr.{ECR_REGION}.amazonaws.com/{ECR_REPOSITORY_NAME}:{ECR_IMAGE_TAG} \
206
+ output --json
207
+ """
208
+ ]
209
+ response = ssm.send_command(
210
+ InstanceIds=instance_ids,
211
+ DocumentName=document_name,
212
+ Parameters={
213
+ 'commands': commands
214
+ },
215
+ OutputS3BucketName=tf_logs_bucket_name,
216
+ OutputS3KeyPrefix=output_key_prefix
217
+ )
218
+
219
+ command_id = response['Command']['CommandId']
220
+ print(f'Command ID: {command_id}')
221
+ d.status = "fetching tfouputs"
222
+ d.write_to_db(kv_store)
223
+
224
+ d.status = wait_for_status(command_id, instance_ids[0])
225
+ d.write_to_db(kv_store)
226
+
227
+ stdout, _ = display_logs(command_id, instance_ids[0], d.status, tf_logs_bucket_name)
228
+ d.tf_output = stdout
229
+ d.storage_nodes = storage_nodes
230
+ d.availability_zone = availability_zone
231
+ d.write_to_db(kv_store)
232
+
233
+
234
+ @bp.route('/deployer', methods=['GET'], defaults={'uuid': None})
235
+ @bp.route('/deployer/<string:uuid>', methods=['GET'])
236
+ def list_deployer(uuid):
237
+ deployers_list = []
238
+ if uuid:
239
+ dpl = db_controller.get_deployer_by_id(uuid)
240
+ if dpl:
241
+ deployers_list.append(dpl)
242
+ else:
243
+ return utils.get_response_error(f"Deployer not found: {uuid}", 404)
244
+ else:
245
+ dpls = db_controller.get_deployers()
246
+ if dpls:
247
+ deployers_list.extend(dpls)
248
+
249
+ data = []
250
+ for deployer in deployers_list:
251
+ d = deployer.get_clean_dict()
252
+ data.append(d)
253
+ return utils.get_response(data)
254
+
255
+ def validate_tf_settings(dpl_data):
256
+ """
257
+ """
258
+ if 'tf_state_bucket_name' not in dpl_data:
259
+ return "missing required param: tf_state_bucket_name"
260
+ if 'tf_state_bucket_region' not in dpl_data:
261
+ return "missing required param: tf_state_bucket_region"
262
+ if 'tf_workspace' not in dpl_data:
263
+ return "missing required param: tf_workspace"
264
+ if 'tf_logs_bucket_name' not in dpl_data:
265
+ return "missing required param: tf_logs_bucket_name"
266
+ if 'ecr_account_id' not in dpl_data:
267
+ return "missing required param: ecr_account_id"
268
+ if 'ecr_region' not in dpl_data:
269
+ return "missing required param: ecr_region"
270
+ if 'ecr_repository_name' not in dpl_data:
271
+ return "missing required param: ecr_repository_name"
272
+ if 'ecr_image_tag' not in dpl_data:
273
+ return "missing required param: ecr_image_tag"
274
+
275
+ return ""
276
+
277
+ def validate_tf_vars(dpl_data):
278
+ """
279
+ """
280
+ if 'region' not in dpl_data:
281
+ return "missing required param: region"
282
+ if 'availability_zone' not in dpl_data:
283
+ return "missing required param: availability_zone"
284
+ if 'sbcli_cmd' not in dpl_data:
285
+ return "missing required param: sbcli_cmd"
286
+ if 'sbcli_pkg_version' not in dpl_data:
287
+ return "missing required param: sbcli_pkg_version"
288
+ if 'mgmt_nodes' not in dpl_data:
289
+ return "missing required param: mgmt_nodes"
290
+ if 'storage_nodes' not in dpl_data:
291
+ return "missing required param: storage_nodes"
292
+ if 'mgmt_nodes_instance_type' not in dpl_data:
293
+ return "missing required param: mgmt_nodes_instance_type"
294
+ if 'storage_nodes_instance_type' not in dpl_data:
295
+ return "missing required param: storage_nodes_instance_type"
296
+
297
+ return ""
298
+
299
+ @bp.route('/deployer/tfvars', methods=['POST'])
300
+ def set_tf_vars():
301
+ """
302
+ Take the terraform variables and replace the existing values
303
+ Used to set the TF vars that are used currently
304
+ Ideally this should be called immediately the first cluster is created. So that for all the subsequent
305
+ storage node add in a different availability zone can be done using the /deployer API
306
+ """
307
+ dpl_data = request.get_json()
308
+ validation_err = validate_tf_vars(dpl_data)
309
+ if validation_err != "":
310
+ return utils.get_response_error(validation_err, 400)
311
+
312
+ validation_err = validate_tf_settings(dpl_data)
313
+ if validation_err != "":
314
+ return utils.get_response_error(validation_err, 400)
315
+
316
+ # if there are no deployments, create a new one.
317
+ # else update the existing one
318
+ d = get_deployer()
319
+ if d is None:
320
+ d = Deployer()
321
+ d.uuid = str(uuid.uuid4())
322
+
323
+ # set TF variables
324
+ d.region = dpl_data['region']
325
+ d.availability_zone = dpl_data['availability_zone']
326
+ d.sbcli_cmd = dpl_data['sbcli_cmd']
327
+ d.sbcli_pkg_version = dpl_data['sbcli_pkg_version']
328
+ d.mgmt_nodes = dpl_data['mgmt_nodes']
329
+ d.storage_nodes = dpl_data['storage_nodes']
330
+ d.mgmt_nodes_instance_type = dpl_data['mgmt_nodes_instance_type']
331
+ d.storage_nodes_instance_type = dpl_data['storage_nodes_instance_type']
332
+
333
+ # set TF settings
334
+ d.tf_state_bucket_name = dpl_data['tf_state_bucket_name']
335
+ d.tf_state_bucket_region = dpl_data['tf_state_bucket_region']
336
+ d.tf_workspace = dpl_data['tf_workspace']
337
+ d.tf_logs_bucket_name = dpl_data['tf_logs_bucket_name']
338
+ d.ecr_account_id = dpl_data['ecr_account_id']
339
+ d.ecr_region = dpl_data['ecr_region']
340
+ d.ecr_repository_name = dpl_data['ecr_repository_name']
341
+ d.ecr_image_tag = dpl_data['ecr_image_tag']
342
+
343
+ d.write_to_db(db_controller.kv_store)
344
+ return utils.get_response(d.to_dict()), 201
345
+
346
+ @bp.route('/deployer', methods=['POST'])
347
+ def add_deployer():
348
+ """
349
+ This API creates the Infrastructure in a different AZ and on the same existing AWS Account
350
+ """
351
+
352
+ # validations
353
+ dpl_data = request.get_json()
354
+
355
+ if "uuid" not in dpl_data:
356
+ return utils.get_response_error("missing required param: uuid", 400)
357
+
358
+ uuid = dpl_data['uuid']
359
+ d = db_controller.get_deployer_by_id(uuid)
360
+
361
+ if "storage_nodes" not in dpl_data:
362
+ return utils.get_response_error("missing required param: storage_nodes", 400)
363
+
364
+ if "availability_zone" not in dpl_data:
365
+ return utils.get_response_error("missing required param: availability_zone", 400)
366
+
367
+ # start the deployment
368
+ d.status = "started"
369
+ d.write_to_db(db_controller.kv_store)
370
+
371
+ storage_nodes = int(dpl_data['storage_nodes'])
372
+ availability_zone = dpl_data['availability_zone']
373
+ d.write_to_db(db_controller.kv_store)
374
+
375
+ t = threading.Thread(
376
+ target=update_cluster,
377
+ args=(d, db_controller.kv_store, d.storage_nodes+storage_nodes, availability_zone))
378
+
379
+ t.start()
380
+
381
+ output = {
382
+ "status": d.status,
383
+ }
384
+
385
+ return utils.get_response(output, http_code=201)
386
+
387
+ def get_deployer():
388
+ """
389
+ get the deployer if it exists. Else returns an None
390
+ """
391
+ dpls = db_controller.get_deployers()
392
+ if len(dpls) > 0:
393
+ return dpls[0]
394
+ return None
@@ -3,7 +3,7 @@
3
3
 
4
4
  import logging
5
5
 
6
- from flask import Blueprint, request
6
+ from flask import Blueprint
7
7
 
8
8
  from simplyblock_core.controllers import device_controller
9
9
  from simplyblock_web import utils
@@ -33,15 +33,12 @@ def list_devices_by_node(uuid):
33
33
  def list_storage_devices(uuid):
34
34
  devices = []
35
35
  if uuid:
36
- dev = db_controller.get_storage_device_by_id(uuid)
36
+ dev = db_controller.get_storage_devices(uuid)
37
37
  if not dev:
38
38
  return utils.get_response_error(f"device not found: {uuid}", 404)
39
39
  devices = [dev]
40
40
  else:
41
- cluster_id = utils.get_cluster_id(request)
42
- nodes = db_controller.get_storage_nodes_by_cluster_id(cluster_id)
43
- for node in nodes:
44
- devices.append(node.nvme_devices)
41
+ devices = db_controller.get_storage_devices()
45
42
  data = []
46
43
  for dev in devices:
47
44
  data.append(dev.get_clean_dict())
@@ -50,7 +47,7 @@ def list_storage_devices(uuid):
50
47
  @bp.route('/device/capacity/<string:uuid>/history/<string:history>', methods=['GET'])
51
48
  @bp.route('/device/capacity/<string:uuid>', methods=['GET'], defaults={'history': None})
52
49
  def device_capacity(uuid, history):
53
- device = db_controller.get_storage_device_by_id(uuid)
50
+ device = db_controller.get_storage_devices(uuid)
54
51
  if not device:
55
52
  return utils.get_response_error(f"devices not found: {uuid}", 404)
56
53
 
@@ -61,7 +58,7 @@ def device_capacity(uuid, history):
61
58
  @bp.route('/device/iostats/<string:uuid>/history/<string:history>', methods=['GET'])
62
59
  @bp.route('/device/iostats/<string:uuid>', methods=['GET'], defaults={'history': None})
63
60
  def device_iostats(uuid, history):
64
- devices = db_controller.get_storage_device_by_id(uuid)
61
+ devices = db_controller.get_storage_devices(uuid)
65
62
  if not devices:
66
63
  return utils.get_response_error(f"devices not found: {uuid}", 404)
67
64
 
@@ -74,7 +71,7 @@ def device_iostats(uuid, history):
74
71
 
75
72
  @bp.route('/device/reset/<string:uuid>', methods=['GET'])
76
73
  def device_reset(uuid):
77
- devices = db_controller.get_storage_device_by_id(uuid)
74
+ devices = db_controller.get_storage_devices(uuid)
78
75
  if not devices:
79
76
  return utils.get_response_error(f"devices not found: {uuid}", 404)
80
77
 
@@ -84,7 +81,7 @@ def device_reset(uuid):
84
81
 
85
82
  @bp.route('/device/remove/<string:uuid>', methods=['GET'])
86
83
  def device_remove(uuid):
87
- devices = db_controller.get_storage_device_by_id(uuid)
84
+ devices = db_controller.get_storage_devices(uuid)
88
85
  if not devices:
89
86
  return utils.get_response_error(f"devices not found: {uuid}", 404)
90
87
 
@@ -19,21 +19,17 @@ bp = Blueprint("lvol", __name__)
19
19
  db_controller = kv_store.DBController()
20
20
 
21
21
 
22
- @bp.route('/lvol', defaults={'uuid': None, "cluster_id": None}, methods=['GET'])
23
- @bp.route('/lvol/<string:uuid>', defaults={"cluster_id": None}, methods=['GET'])
24
- @bp.route('/lvol/cluster_id/<string:cluster_id>', defaults={'uuid': None,}, methods=['GET'])
25
- def list_lvols(uuid, cluster_id):
22
+ @bp.route('/lvol', defaults={'uuid': None}, methods=['GET'])
23
+ @bp.route('/lvol/<string:uuid>', methods=['GET'])
24
+ def list_lvols(uuid):
26
25
  if uuid:
27
26
  lvol = db_controller.get_lvol_by_id(uuid)
28
27
  if lvol:
29
28
  lvols = [lvol]
30
29
  else:
31
30
  return utils.get_response_error(f"LVol not found: {uuid}", 404)
32
- elif cluster_id:
33
- lvols = db_controller.get_lvols(cluster_id)
34
31
  else:
35
- cluster_id = utils.get_cluster_id(request)
36
- lvols = db_controller.get_lvols(cluster_id)
32
+ lvols = db_controller.get_lvols()
37
33
  data = []
38
34
  for lvol in lvols:
39
35
  data.append(lvol.get_clean_dict())
@@ -127,7 +123,7 @@ def add_lvol():
127
123
  if not pool:
128
124
  return utils.get_response(None, f"Pool not found: {pool_id_or_name}", 400)
129
125
 
130
- for lvol in db_controller.get_lvols(): # pass
126
+ for lvol in db_controller.get_lvols():
131
127
  if lvol.pool_uuid == pool.get_id():
132
128
  if lvol.lvol_name == name:
133
129
  return utils.get_response(lvol.get_id())
@@ -17,21 +17,17 @@ bp = Blueprint("pool", __name__)
17
17
  db_controller = kv_store.DBController()
18
18
 
19
19
 
20
- @bp.route('/pool', defaults={'uuid': None, "cluster_id": None}, methods=['GET'])
21
- @bp.route('/pool/<string:uuid>', defaults={"cluster_id": None}, methods=['GET'])
22
- @bp.route('/pool/cluster_id/<string:cluster_id>', defaults={'uuid': None,}, methods=['GET'])
23
- def list_pools(uuid, cluster_id):
20
+ @bp.route('/pool', defaults={'uuid': None}, methods=['GET'])
21
+ @bp.route('/pool/<string:uuid>', methods=['GET'])
22
+ def list_pools(uuid):
24
23
  if uuid:
25
24
  pool = db_controller.get_pool_by_id(uuid)
26
25
  if pool:
27
26
  pools = [pool]
28
27
  else:
29
28
  return utils.get_response_error(f"Pool not found: {uuid}", 404)
30
- elif cluster_id:
31
- pools = db_controller.get_pools(cluster_id)
32
29
  else:
33
- cluster_id = utils.get_cluster_id(request)
34
- pools = db_controller.get_pools(cluster_id)
30
+ pools = db_controller.get_pools()
35
31
  data = []
36
32
  for pool in pools:
37
33
  data.append(pool.get_clean_dict())
@@ -43,7 +39,6 @@ def add_pool():
43
39
  """
44
40
  Params:
45
41
  | name (required) | LVol name or id
46
- | cluster_id (required) | Cluster uuid
47
42
  | pool_max | Pool maximum size: 10M, 10G, 10(bytes)
48
43
  | lvol_max | LVol maximum size: 10M, 10G, 10(bytes)
49
44
  | no_secret | pool is created with a secret
@@ -56,11 +51,7 @@ def add_pool():
56
51
  if 'name' not in pool_data:
57
52
  return utils.get_response_error("missing required param: name", 400)
58
53
 
59
- if 'cluster_id' not in pool_data:
60
- return utils.get_response_error("missing required param: cluster_id", 400)
61
-
62
54
  name = pool_data['name']
63
- cluster_id = pool_data['cluster_id']
64
55
  for p in db_controller.get_pools():
65
56
  if p.pool_name == name:
66
57
  return utils.get_response_error(f"Pool found with the same name: {name}", 400)
@@ -84,7 +75,7 @@ def add_pool():
84
75
 
85
76
  ret = pool_controller.add_pool(
86
77
  name, pool_max_size, lvol_max_size, max_rw_iops, max_rw_mbytes,
87
- max_r_mbytes_per_sec, max_w_mbytes_per_sec, pool_secret, cluster_id)
78
+ max_r_mbytes_per_sec, max_w_mbytes_per_sec, pool_secret)
88
79
 
89
80
  return utils.get_response(ret)
90
81
 
@@ -30,8 +30,7 @@ def list_storage_nodes(uuid):
30
30
  else:
31
31
  return utils.get_response_error(f"node not found: {uuid}", 404)
32
32
  else:
33
- cluster_id = utils.get_cluster_id(request)
34
- nodes = db_controller.get_storage_nodes_by_cluster_id(cluster_id)
33
+ nodes = db_controller.get_storage_nodes()
35
34
  data = []
36
35
  for node in nodes:
37
36
  d = node.get_clean_dict()
@@ -180,6 +179,14 @@ def storage_node_add():
180
179
  data_nics = req_data['data_nics']
181
180
  data_nics = data_nics.split(",")
182
181
 
182
+ spdk_cpu_mask = None
183
+ if 'spdk_cpu_mask' in req_data:
184
+ msk = req_data['spdk_cpu_mask']
185
+ if utils.validate_cpu_mask(msk):
186
+ spdk_cpu_mask = msk
187
+ else:
188
+ return utils.get_response_error(f"Invalid cpu mask value: {msk}", 400)
189
+
183
190
  spdk_mem = None
184
191
  if 'spdk_mem' in req_data:
185
192
  mem = req_data['spdk_mem']
@@ -188,7 +195,7 @@ def storage_node_add():
188
195
  return utils.get_response_error(f"SPDK memory:{mem} must be larger than 1G", 400)
189
196
 
190
197
  out = storage_node_ops.add_node(
191
- cluster_id, node_ip, ifname, data_nics, spdk_mem,
198
+ cluster_id, node_ip, ifname, data_nics, spdk_cpu_mask, spdk_mem,
192
199
  spdk_image=spdk_image, spdk_debug=spdk_debug)
193
200
 
194
201
  return utils.get_response(out)
@@ -87,6 +87,8 @@ def get_nics_data():
87
87
  logger.error(err)
88
88
  return []
89
89
  data = json.loads(out)
90
+ logger.debug("ifaces")
91
+ logger.debug(data)
90
92
 
91
93
  def _get_ip4_address(list_of_addr):
92
94
  if list_of_addr:
@@ -85,11 +85,3 @@ def get_int_value_or_default(data, key, default):
85
85
  return int(get_value_or_default(data, key, default))
86
86
  except Exception:
87
87
  return default
88
-
89
-
90
- def get_cluster_id(request):
91
- au = request.headers["Authorization"]
92
- if len(au.split()) == 2:
93
- cluster_id = au.split()[0]
94
- cluster_secret = au.split()[1]
95
- return cluster_id