sbcli-pre 1.4.3__zip → 1.4.5__zip

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (141) hide show
  1. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/PKG-INFO +11 -1
  2. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/env_var +1 -1
  3. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/sbcli_pre.egg-info/PKG-INFO +11 -1
  4. sbcli_pre-1.4.5/sbcli_pre.egg-info/requires.txt +18 -0
  5. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/setup.py +10 -0
  6. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_cli/cli.py +3 -3
  7. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/cnode_client.py +11 -0
  8. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/constants.py +1 -3
  9. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/controllers/caching_node_controller.py +53 -29
  10. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/models/storage_node.py +3 -4
  11. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/scripts/deploy_stack.sh +9 -0
  12. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/scripts/docker-compose-swarm-monitoring.yml +5 -14
  13. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/services/caching_node_monitor.py +2 -14
  14. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/services/cap_monitor.py +2 -15
  15. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/services/capacity_and_stats_collector.py +3 -13
  16. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/services/device_monitor.py +2 -14
  17. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/services/distr_event_collector.py +3 -14
  18. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/services/health_check_service.py +4 -15
  19. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/services/lvol_monitor.py +5 -17
  20. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/services/lvol_stat_collector.py +3 -15
  21. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/services/mgmt_node_monitor.py +2 -12
  22. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/services/port_stat_collector.py +3 -16
  23. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/services/storage_node_monitor.py +3 -17
  24. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/services/tasks_runner_migration.py +2 -14
  25. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/services/tasks_runner_restart.py +2 -15
  26. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/storage_node_ops.py +91 -87
  27. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/utils.py +15 -0
  28. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_web/app.py +1 -2
  29. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_web/blueprints/caching_node_ops.py +72 -1
  30. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_web/blueprints/caching_node_ops_k8s.py +76 -1
  31. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_web/blueprints/snode_ops.py +42 -42
  32. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_web/templates/deploy_spdk.yaml.j2 +8 -3
  33. sbcli_pre-1.4.3/sbcli_pre.egg-info/requires.txt +0 -8
  34. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/README.md +0 -0
  35. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/pyproject.toml +0 -0
  36. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/sbcli_pre.egg-info/SOURCES.txt +0 -0
  37. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/sbcli_pre.egg-info/dependency_links.txt +0 -0
  38. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/sbcli_pre.egg-info/entry_points.txt +0 -0
  39. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/sbcli_pre.egg-info/top_level.txt +0 -0
  40. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/setup.cfg +0 -0
  41. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_cli/main.py +0 -0
  42. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/__init__.py +0 -0
  43. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/cluster_ops.py +0 -0
  44. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/compute_node_ops.py +0 -0
  45. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/controllers/__init__.py +0 -0
  46. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/controllers/cluster_events.py +0 -0
  47. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/controllers/device_controller.py +0 -0
  48. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/controllers/device_events.py +0 -0
  49. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/controllers/events_controller.py +0 -0
  50. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/controllers/health_controller.py +0 -0
  51. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/controllers/lvol_controller.py +0 -0
  52. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/controllers/lvol_events.py +0 -0
  53. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/controllers/mgmt_events.py +0 -0
  54. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/controllers/pool_controller.py +0 -0
  55. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/controllers/pool_events.py +0 -0
  56. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/controllers/snapshot_controller.py +0 -0
  57. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/controllers/snapshot_events.py +0 -0
  58. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/controllers/storage_events.py +0 -0
  59. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/controllers/tasks_controller.py +0 -0
  60. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/controllers/tasks_events.py +0 -0
  61. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/distr_controller.py +0 -0
  62. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/kv_store.py +0 -0
  63. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/mgmt_node_ops.py +0 -0
  64. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/models/__init__.py +0 -0
  65. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/models/base_model.py +0 -0
  66. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/models/caching_node.py +0 -0
  67. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/models/cluster.py +0 -0
  68. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/models/compute_node.py +0 -0
  69. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/models/deployer.py +0 -0
  70. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/models/events.py +0 -0
  71. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/models/global_settings.py +0 -0
  72. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/models/iface.py +0 -0
  73. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/models/job_schedule.py +0 -0
  74. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/models/lvol_model.py +0 -0
  75. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/models/mgmt_node.py +0 -0
  76. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/models/nvme_device.py +0 -0
  77. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/models/pool.py +0 -0
  78. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/models/port_stat.py +0 -0
  79. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/models/snapshot.py +0 -0
  80. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/models/stats.py +0 -0
  81. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/pci_utils.py +0 -0
  82. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/rpc_client.py +0 -0
  83. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/scripts/__init__.py +0 -0
  84. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/scripts/alerting/alert_resources.yaml.j2 +0 -0
  85. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/scripts/alerting/alert_rules.yaml +0 -0
  86. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/scripts/clean_local_storage_deploy.sh +0 -0
  87. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/scripts/config_docker.sh +0 -0
  88. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/scripts/dashboard.yml +0 -0
  89. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/scripts/dashboards/cluster.json +0 -0
  90. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/scripts/dashboards/devices.json +0 -0
  91. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/scripts/dashboards/lvols.json +0 -0
  92. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/scripts/dashboards/node-exporter.json +0 -0
  93. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/scripts/dashboards/nodes.json +0 -0
  94. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/scripts/dashboards/pools.json +0 -0
  95. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/scripts/datasource.yml +0 -0
  96. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/scripts/db_config_double.sh +0 -0
  97. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/scripts/db_config_single.sh +0 -0
  98. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/scripts/docker-compose-swarm.yml +0 -0
  99. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/scripts/haproxy.cfg +0 -0
  100. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/scripts/install_deps.sh +0 -0
  101. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/scripts/objstore.yml +0 -0
  102. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/scripts/prometheus.yml +0 -0
  103. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/scripts/run_ssh.sh +0 -0
  104. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/scripts/set_db_config.sh +0 -0
  105. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/scripts/stack_deploy_wait.sh +0 -0
  106. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/services/__init__.py +0 -0
  107. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/services/install_service.sh +0 -0
  108. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/services/log_agg_service.py +0 -0
  109. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/services/remove_service.sh +0 -0
  110. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/services/service_template.service +0 -0
  111. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/shell_utils.py +0 -0
  112. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_core/snode_client.py +0 -0
  113. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_web/__init__.py +0 -0
  114. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_web/auth_middleware.py +0 -0
  115. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_web/blueprints/__init__.py +0 -0
  116. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_web/blueprints/node_api_basic.py +0 -0
  117. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_web/blueprints/node_api_caching_docker.py +0 -0
  118. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_web/blueprints/node_api_caching_ks.py +0 -0
  119. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_web/blueprints/web_api_caching_node.py +0 -0
  120. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_web/blueprints/web_api_cluster.py +0 -0
  121. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_web/blueprints/web_api_deployer.py +0 -0
  122. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_web/blueprints/web_api_device.py +0 -0
  123. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_web/blueprints/web_api_lvol.py +0 -0
  124. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_web/blueprints/web_api_mgmt_node.py +0 -0
  125. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_web/blueprints/web_api_pool.py +0 -0
  126. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_web/blueprints/web_api_snapshot.py +0 -0
  127. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_web/blueprints/web_api_storage_node.py +0 -0
  128. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_web/caching_node_app.py +0 -0
  129. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_web/caching_node_app_k8s.py +0 -0
  130. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_web/node_utils.py +0 -0
  131. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_web/node_webapp.py +0 -0
  132. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_web/snode_app.py +0 -0
  133. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_web/static/delete.py +0 -0
  134. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_web/static/deploy.py +0 -0
  135. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_web/static/deploy_cnode.yaml +0 -0
  136. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_web/static/deploy_spdk.yaml +0 -0
  137. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_web/static/is_up.py +0 -0
  138. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_web/static/list_deps.py +0 -0
  139. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_web/static/rpac.yaml +0 -0
  140. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_web/static/tst.py +0 -0
  141. {sbcli_pre-1.4.3 → sbcli_pre-1.4.5}/simplyblock_web/utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: sbcli-pre
3
- Version: 1.4.3
3
+ Version: 1.4.5
4
4
  Summary: CLI for managing SimplyBlock cluster
5
5
  Home-page: https://www.simplyblock.io/
6
6
  Author: Hamdy
@@ -8,12 +8,22 @@ Author-email: hamdy@simplyblock.io
8
8
  Description-Content-Type: text/markdown
9
9
  Requires-Dist: foundationdb
10
10
  Requires-Dist: requests
11
+ Requires-Dist: numpy
11
12
  Requires-Dist: typing
12
13
  Requires-Dist: prettytable
13
14
  Requires-Dist: docker
14
15
  Requires-Dist: psutil
15
16
  Requires-Dist: py-cpuinfo
17
+ Requires-Dist: pytest
18
+ Requires-Dist: mock
19
+ Requires-Dist: setuptools
20
+ Requires-Dist: flask
21
+ Requires-Dist: kubernetes
22
+ Requires-Dist: PyYAML
23
+ Requires-Dist: urllib3
16
24
  Requires-Dist: graypy
25
+ Requires-Dist: boto3
26
+ Requires-Dist: jinja2
17
27
 
18
28
 
19
29
  # Simply Block
@@ -1,5 +1,5 @@
1
1
  SIMPLY_BLOCK_COMMAND_NAME=sbcli-pre
2
- SIMPLY_BLOCK_VERSION=1.4.3
2
+ SIMPLY_BLOCK_VERSION=1.4.5
3
3
 
4
4
 
5
5
  SIMPLY_BLOCK_DOCKER_IMAGE=simplyblock/simplyblock:pre-release
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: sbcli-pre
3
- Version: 1.4.3
3
+ Version: 1.4.5
4
4
  Summary: CLI for managing SimplyBlock cluster
5
5
  Home-page: https://www.simplyblock.io/
6
6
  Author: Hamdy
@@ -8,12 +8,22 @@ Author-email: hamdy@simplyblock.io
8
8
  Description-Content-Type: text/markdown
9
9
  Requires-Dist: foundationdb
10
10
  Requires-Dist: requests
11
+ Requires-Dist: numpy
11
12
  Requires-Dist: typing
12
13
  Requires-Dist: prettytable
13
14
  Requires-Dist: docker
14
15
  Requires-Dist: psutil
15
16
  Requires-Dist: py-cpuinfo
17
+ Requires-Dist: pytest
18
+ Requires-Dist: mock
19
+ Requires-Dist: setuptools
20
+ Requires-Dist: flask
21
+ Requires-Dist: kubernetes
22
+ Requires-Dist: PyYAML
23
+ Requires-Dist: urllib3
16
24
  Requires-Dist: graypy
25
+ Requires-Dist: boto3
26
+ Requires-Dist: jinja2
17
27
 
18
28
 
19
29
  # Simply Block
@@ -0,0 +1,18 @@
1
+ foundationdb
2
+ requests
3
+ numpy
4
+ typing
5
+ prettytable
6
+ docker
7
+ psutil
8
+ py-cpuinfo
9
+ pytest
10
+ mock
11
+ setuptools
12
+ flask
13
+ kubernetes
14
+ PyYAML
15
+ urllib3
16
+ graypy
17
+ boto3
18
+ jinja2
@@ -68,12 +68,22 @@ setup(
68
68
  install_requires=[
69
69
  "foundationdb",
70
70
  "requests",
71
+ "numpy",
71
72
  "typing",
72
73
  "prettytable",
73
74
  "docker",
74
75
  "psutil",
75
76
  "py-cpuinfo",
77
+ "pytest",
78
+ "mock",
79
+ "setuptools",
80
+ "flask",
81
+ "kubernetes",
82
+ "PyYAML",
83
+ "urllib3",
76
84
  "graypy",
85
+ "boto3",
86
+ "jinja2",
77
87
  ],
78
88
  entry_points={
79
89
  'console_scripts': [
@@ -585,8 +585,8 @@ class CLIWrapper:
585
585
  sub_command.add_argument("--memory", help='SPDK huge memory allocation, default is Max hugepages available', dest='spdk_mem')
586
586
  sub_command.add_argument("--spdk-image", help='SPDK image uri', dest='spdk_image')
587
587
  sub_command.add_argument("--namespace", help='k8s namespace to deploy on',)
588
- sub_command.add_argument("--multipathing", help='Enable multipathing for lvol connection, default: True',
589
- type=bool, default=True)
588
+ sub_command.add_argument("--multipathing", help='Enable multipathing for lvol connection, default: on',
589
+ default="on", choices=["on", "off"])
590
590
 
591
591
  self.add_sub_command(subparser, 'list', 'List Caching nodes')
592
592
 
@@ -1069,7 +1069,7 @@ class CLIWrapper:
1069
1069
  data_nics = []
1070
1070
  spdk_image = args.spdk_image
1071
1071
  namespace = args.namespace
1072
- multipathing = args.multipathing
1072
+ multipathing = args.multipathing == "on"
1073
1073
 
1074
1074
  spdk_cpu_mask = None
1075
1075
  if args.spdk_cpu_mask:
@@ -111,3 +111,14 @@ class CNodeClient:
111
111
 
112
112
  def disconnect_all(self):
113
113
  return self._request("POST", "cnode/disconnect_all")
114
+
115
+ def make_gpt_partitions(self, nbd_device, jm_percent):
116
+ params = {
117
+ "nbd_device": nbd_device,
118
+ "jm_percent": jm_percent,
119
+ }
120
+ return self._request("POST", "cnode/make_gpt_partitions", params)
121
+
122
+ def delete_dev_gpt_partitions(self, device_pci):
123
+ params = {"device_pci": device_pci}
124
+ return self._request("POST", "cnode/delete_dev_gpt_partitions", params)
@@ -43,10 +43,8 @@ weights = {
43
43
  "w_b": 10
44
44
  }
45
45
 
46
- # To use 75% of hugepages to calculate ssd size to use for the ocf bdev
47
- CACHING_NODE_MEMORY_FACTOR = 0.75
48
46
 
49
- HEALTH_CHECK_INTERVAL_SEC = 60
47
+ HEALTH_CHECK_INTERVAL_SEC = 10
50
48
 
51
49
  GRAYLOG_CHECK_INTERVAL_SEC = 60
52
50
 
@@ -84,9 +84,9 @@ def add_node(cluster_id, node_ip, iface_name, data_nics_list, spdk_cpu_mask, spd
84
84
  return False
85
85
 
86
86
  logger.info(f"Add Caching node: {node_ip}")
87
- snode_api = CNodeClient(node_ip)
87
+ cnode_api = CNodeClient(node_ip)
88
88
 
89
- node_info, _ = snode_api.info()
89
+ node_info, _ = cnode_api.info()
90
90
  system_id = node_info['system_id']
91
91
  hostname = node_info['hostname']
92
92
  logger.info(f"Node found: {node_info['hostname']}")
@@ -95,8 +95,8 @@ def add_node(cluster_id, node_ip, iface_name, data_nics_list, spdk_cpu_mask, spd
95
95
  logger.error("Node already exists, try remove it first.")
96
96
  return False
97
97
 
98
- node_info, _ = snode_api.info()
99
- results, err = snode_api.join_db(db_connection=cluster.db_connection)
98
+ node_info, _ = cnode_api.info()
99
+ results, err = cnode_api.join_db(db_connection=cluster.db_connection)
100
100
 
101
101
  data_nics = []
102
102
  names = data_nics_list or [iface_name]
@@ -142,16 +142,21 @@ def add_node(cluster_id, node_ip, iface_name, data_nics_list, spdk_cpu_mask, spd
142
142
  logger.info("Node Memory info")
143
143
  logger.info(f"RAM Total: {utils.humanbytes(memory_details['total'])}")
144
144
  logger.info(f"RAM Free: {utils.humanbytes(memory_details['free'])}")
145
- logger.info(f"HP Total: {utils.humanbytes(memory_details['huge_total'])}")
146
- # huge_free = memory_details['huge_free']
147
- logger.info(f"HP Free: {utils.humanbytes(memory_details['huge_free'])}")
145
+ huge_total = memory_details['huge_total']
146
+ logger.info(f"HP Total: {utils.humanbytes(huge_total)}")
147
+ huge_free = memory_details['huge_free']
148
+ logger.info(f"HP Free: {utils.humanbytes(huge_free)}")
148
149
  # if huge_free < 1 * 1024 * 1024:
149
150
  # logger.warning(f"Free hugepages are less than 1G: {utils.humanbytes(huge_free)}")
150
151
  if not spdk_mem:
151
- spdk_mem = memory_details['huge_free']
152
+ if huge_total > 1024 * 1024 * 1024:
153
+ spdk_mem = huge_total
154
+ else:
155
+ logger.error(f"Free hugepages are less than 1G: {utils.humanbytes(huge_total)}")
156
+ return False
152
157
 
153
158
  logger.info(f"Deploying SPDK with HP: {utils.humanbytes(spdk_mem)}")
154
- results, err = snode_api.spdk_process_start(
159
+ results, err = cnode_api.spdk_process_start(
155
160
  spdk_cpu_mask, spdk_mem, spdk_image, snode.mgmt_ip,
156
161
  snode.rpc_port, snode.rpc_username, snode.rpc_password, namespace)
157
162
  if not results:
@@ -160,7 +165,7 @@ def add_node(cluster_id, node_ip, iface_name, data_nics_list, spdk_cpu_mask, spd
160
165
 
161
166
  retries = 20
162
167
  while retries > 0:
163
- resp, _ = snode_api.spdk_process_is_up()
168
+ resp, _ = cnode_api.spdk_process_is_up()
164
169
  if resp:
165
170
  logger.info(f"Pod is up")
166
171
  break
@@ -173,7 +178,7 @@ def add_node(cluster_id, node_ip, iface_name, data_nics_list, spdk_cpu_mask, spd
173
178
  logger.error("Pod is not running, exiting")
174
179
  return False
175
180
 
176
- time.sleep(10)
181
+ time.sleep(20)
177
182
 
178
183
  # creating RPCClient instance
179
184
  rpc_client = RPCClient(
@@ -182,7 +187,7 @@ def add_node(cluster_id, node_ip, iface_name, data_nics_list, spdk_cpu_mask, spd
182
187
  timeout=60*5, retry=5)
183
188
 
184
189
  # # get new node info after starting spdk
185
- # node_info, _ = snode_api.info()
190
+ node_info, _ = cnode_api.info()
186
191
  # mem = node_info['memory_details']['huge_free']
187
192
  # logger.info(f"Free Hugepages detected: {utils.humanbytes(mem)}")
188
193
 
@@ -200,27 +205,46 @@ def add_node(cluster_id, node_ip, iface_name, data_nics_list, spdk_cpu_mask, spd
200
205
  logger.error("Hugepages must be larger than 1G")
201
206
  return False
202
207
 
203
- mem = spdk_mem - 1024*1024
208
+ mem = spdk_mem - 1024*1024*1024
204
209
  snode.hugepages = mem
205
210
  logger.info(f"Hugepages to be used: {utils.humanbytes(mem)}")
206
211
 
207
212
  ssd_size = ssd_dev.size
208
213
  supported_ssd_size = mem * 100 / 2.25
209
- split_factor = math.ceil(ssd_size/supported_ssd_size)
210
214
 
211
215
  logger.info(f"Supported SSD size: {utils.humanbytes(supported_ssd_size)}")
212
216
  logger.info(f"SSD size: {utils.humanbytes(ssd_size)}")
213
217
 
214
- cache_size = 0
215
- cache_bdev = None
216
218
  if supported_ssd_size < ssd_size:
217
- logger.info(f"SSD size is bigger than the supported size, will use split bdev: {split_factor}")
218
- ret = rpc_client.bdev_split(ssd_dev.nvme_bdev, split_factor)
219
- cache_bdev = ret[0]
220
- cache_size = int(ssd_dev.size/split_factor)
221
- snode.cache_split_factor = split_factor
219
+ logger.info(f"SSD size is bigger than the supported size, creating partition")
220
+
221
+ nbd_device = rpc_client.nbd_start_disk(ssd_dev.nvme_bdev)
222
+ time.sleep(3)
223
+ if not nbd_device:
224
+ logger.error(f"Failed to start nbd dev")
225
+ return False
226
+
227
+ jm_percent = int((supported_ssd_size/ssd_size) * 100)
228
+ result, error = cnode_api.make_gpt_partitions(nbd_device, jm_percent)
229
+ if error:
230
+ logger.error(f"Failed to make partitions")
231
+ logger.error(error)
232
+ return False
233
+ time.sleep(3)
234
+ rpc_client.nbd_stop_disk(nbd_device)
235
+ time.sleep(1)
236
+ rpc_client.bdev_nvme_detach_controller(ssd_dev.nvme_controller)
237
+ time.sleep(1)
238
+ rpc_client.bdev_nvme_controller_attach(ssd_dev.nvme_controller, ssd_dev.pcie_address)
239
+ time.sleep(1)
240
+ rpc_client.bdev_examine(ssd_dev.nvme_bdev)
241
+ time.sleep(1)
242
+
243
+ cache_bdev = f"{ssd_dev.nvme_bdev}p1"
244
+ cache_size = int(supported_ssd_size)
245
+
222
246
  else:
223
- snode.cache_split_factor = 0
247
+
224
248
  cache_bdev = ssd_dev.nvme_bdev
225
249
  cache_size = ssd_dev.size
226
250
 
@@ -277,13 +301,13 @@ def recreate(node_id):
277
301
  logger.error("No NVMe devices was found!")
278
302
  return False
279
303
 
280
- snode.nvme_devices = nvme_devs
304
+ # snode.nvme_devices = nvme_devs
281
305
  # snode.write_to_db(db_controller.kv_store)
282
306
 
283
- ssd_dev = nvme_devs[0]
307
+ # ssd_dev = nvme_devs[0]
284
308
 
285
- if snode.cache_split_factor > 1:
286
- ret = rpc_client.bdev_split(ssd_dev.nvme_bdev, snode.cache_split_factor)
309
+ # if snode.cache_split_factor > 1:
310
+ # ret = rpc_client.bdev_split(ssd_dev.nvme_bdev, snode.cache_split_factor)
287
311
 
288
312
  logger.info(f"Cache size: {utils.humanbytes(snode.cache_size)}")
289
313
 
@@ -683,13 +707,13 @@ def remove_node(node_id, force=False):
683
707
 
684
708
  logger.info("Removing node")
685
709
 
686
-
687
710
  try:
688
711
  snode_api = CNodeClient(snode.api_endpoint)
689
712
  results, err = snode_api.spdk_process_kill()
713
+ ret = snode_api.delete_dev_gpt_partitions(snode.nvme_devices[0].pcie_address)
714
+
690
715
  except:
691
716
  pass
692
- snode.remove(db_controller.kv_store)
693
717
 
694
- # storage_events.snode_remove(snode)
718
+ snode.remove(db_controller.kv_store)
695
719
  logger.info("done")
@@ -85,10 +85,9 @@ class StorageNode(BaseModel):
85
85
  "spdk_image": {"type": str, "default": ""},
86
86
  "spdk_debug": {"type": bool, "default": False},
87
87
 
88
-
89
- "ec2_metadata": {"type": dict, "default": {}},
90
- "ec2_instance_id": {"type": str, "default": ""},
91
- "ec2_public_ip": {"type": str, "default": ""},
88
+ "cloud_instance_id": {"type": str, "default": ""},
89
+ "cloud_instance_type": {"type": str, "default": ""},
90
+ "cloud_instance_public_ip": {"type": str, "default": ""},
92
91
 
93
92
  # IO buffer options
94
93
  "iobuf_small_pool_count": {"type": int, "default": 0},
@@ -19,6 +19,15 @@ then
19
19
  export FDB_CLUSTER_FILE_CONTENTS=$FDB_CLUSTER_FILE_CONTENTS
20
20
  fi
21
21
 
22
+ if [[ "$LOG_DELETION_INTERVAL" == *d ]]; then
23
+ export MAX_NUMBER_OF_INDICES=${LOG_DELETION_INTERVAL%d}
24
+ elif [[ "$LOG_DELETION_INTERVAL" == *h || "$LOG_DELETION_INTERVAL" == *m ]]; then
25
+ export MAX_NUMBER_OF_INDICES=1
26
+ else
27
+ echo "Invalid LOG_DELETION_INTERVAL format. Please use a value ending in 'd', 'h', or 'm'."
28
+ exit 1
29
+ fi
30
+
22
31
  docker network create monitoring-net -d overlay --attachable
23
32
 
24
33
  docker stack deploy --compose-file="$DIR"/docker-compose-swarm-monitoring.yml monitoring
@@ -42,6 +42,10 @@ services:
42
42
  GRAYLOG_ELASTICSEARCH_HOSTS: "http://opensearch:9200"
43
43
  GRAYLOG_MONGODB_URI: "mongodb://mongodb:27017/graylog"
44
44
  GRAYLOG_SKIP_PREFLIGHT_CHECKS: "true"
45
+ GRAYLOG_ROTATION_STRATEGY: "time"
46
+ GRAYLOG_RETENTION_STRATEGY: "delete"
47
+ GRAYLOG_ELASTICSEARCH_MAX_NUMBER_OF_INDICES: "${MAX_NUMBER_OF_INDICES}"
48
+ GRAYLOG_ELASTICSEARCH_MAX_TIME_PER_INDEX: "1d"
45
49
  ports:
46
50
  - "5044:5044/tcp" # Beats
47
51
  - "5140:5140/udp" # Syslog
@@ -190,8 +194,6 @@ services:
190
194
  restart: "always"
191
195
  deploy:
192
196
  mode: global
193
- placement:
194
- constraints: [node.role == worker]
195
197
  networks:
196
198
  - monitoring-net
197
199
 
@@ -216,24 +218,13 @@ services:
216
218
  networks:
217
219
  - monitoring-net
218
220
 
219
- CleanupGraylog:
220
- image: $SIMPLYBLOCK_DOCKER_IMAGE
221
- environment:
222
- LOG_DELETION_INTERVAL: "${LOG_DELETION_INTERVAL}"
223
- command: "python simplyblock_core/workers/cleanup_graylog.py"
224
- deploy:
225
- placement:
226
- constraints: [node.role == manager]
227
- networks:
228
- - monitoring-net
229
-
230
221
  volumes:
231
222
  mongodb_data:
232
223
  os_data:
233
224
  graylog_data:
234
225
  graylog_journal:
235
226
  grafana_data:
236
- graylog_config:
227
+ graylog_config:
237
228
  prometheus_data:
238
229
  thanos_compactor_data:
239
230
 
@@ -1,28 +1,16 @@
1
1
  # coding=utf-8
2
- import logging
3
2
  import os
4
3
 
5
4
  import time
6
- import sys
7
5
  from datetime import datetime
8
6
 
9
7
 
10
- from simplyblock_core import constants, kv_store
8
+ from simplyblock_core import constants, kv_store, utils
11
9
  from simplyblock_core.rpc_client import RPCClient
12
10
  from simplyblock_core.models.caching_node import CachingNode
13
11
 
14
- # Import the GELF logger
15
- from graypy import GELFUDPHandler
16
-
17
- # configure logging
18
- logger_handler = logging.StreamHandler(stream=sys.stdout)
19
- logger_handler.setFormatter(logging.Formatter('%(asctime)s: %(levelname)s: %(message)s'))
20
- gelf_handler = GELFUDPHandler('0.0.0.0', constants.GELF_PORT)
21
- logger = logging.getLogger()
22
- logger.addHandler(gelf_handler)
23
- logger.addHandler(logger_handler)
24
- logger.setLevel(logging.DEBUG)
25
12
 
13
+ logger = utils.get_logger(__name__)
26
14
 
27
15
  # get DB controller
28
16
  db_store = kv_store.KVStore()
@@ -1,28 +1,15 @@
1
1
  # coding=utf-8
2
- import logging
3
2
 
4
3
  import time
5
- import sys
6
4
 
7
5
 
8
-
9
- from simplyblock_core import kv_store, constants, cluster_ops
6
+ from simplyblock_core import kv_store, constants, cluster_ops, utils
10
7
  from simplyblock_core.controllers import cluster_events
11
8
  from simplyblock_core.models.cluster import Cluster
12
9
 
13
- # Import the GELF logger
14
- from graypy import GELFUDPHandler
15
10
 
16
- # configure logging
17
- logger_handler = logging.StreamHandler(stream=sys.stdout)
18
- logger_handler.setFormatter(logging.Formatter('%(asctime)s: %(levelname)s: %(message)s'))
19
- gelf_handler = GELFUDPHandler('0.0.0.0', constants.GELF_PORT)
20
- logger = logging.getLogger()
21
- logger.addHandler(gelf_handler)
22
- logger.addHandler(logger_handler)
23
- logger.setLevel(logging.DEBUG)
11
+ logger = utils.get_logger(__name__)
24
12
 
25
- ### script to test connection once connection is ascertain
26
13
  # get DB controller
27
14
  db_controller = kv_store.DBController()
28
15
 
@@ -1,16 +1,14 @@
1
1
  # coding=utf-8
2
- import logging
3
-
4
2
  import time
5
- import sys
6
3
 
7
4
  from simplyblock_core import constants, kv_store, utils
8
5
  from simplyblock_core.models.nvme_device import NVMeDevice
9
6
  from simplyblock_core.rpc_client import RPCClient
10
7
  from simplyblock_core.models.stats import DeviceStatObject, NodeStatObject, ClusterStatObject
11
8
 
12
- # Import the GELF logger
13
- from graypy import GELFUDPHandler
9
+
10
+ logger = utils.get_logger(__name__)
11
+
14
12
 
15
13
  last_object_record = {}
16
14
 
@@ -150,14 +148,6 @@ def add_cluster_stats(cl, records):
150
148
  return stat_obj
151
149
 
152
150
 
153
- # configure logging
154
- logger_handler = logging.StreamHandler(stream=sys.stdout)
155
- logger_handler.setFormatter(logging.Formatter('%(asctime)s: %(levelname)s: %(message)s'))
156
- gelf_handler = GELFUDPHandler('0.0.0.0', constants.GELF_PORT)
157
- logger = logging.getLogger()
158
- logger.addHandler(gelf_handler)
159
- logger.addHandler(logger_handler)
160
- logger.setLevel(logging.DEBUG)
161
151
 
162
152
  # get DB controller
163
153
  db_controller = kv_store.DBController()
@@ -1,26 +1,14 @@
1
1
  # coding=utf-8
2
- import logging
3
2
  import time
4
- import sys
5
- import uuid
6
3
 
7
- from simplyblock_core import constants, kv_store
4
+ from simplyblock_core import constants, kv_store, utils
8
5
  from simplyblock_core.controllers import tasks_controller
9
6
  from simplyblock_core.models.nvme_device import NVMeDevice
10
7
  from simplyblock_core.models.storage_node import StorageNode
11
8
 
12
- # Import the GELF logger
13
- from graypy import GELFUDPHandler
14
9
 
10
+ logger = utils.get_logger(__name__)
15
11
 
16
- # configure logging
17
- logger_handler = logging.StreamHandler(stream=sys.stdout)
18
- logger_handler.setFormatter(logging.Formatter('%(asctime)s: %(levelname)s: %(message)s'))
19
- gelf_handler = GELFUDPHandler('0.0.0.0', constants.GELF_PORT)
20
- logger = logging.getLogger()
21
- logger.addHandler(gelf_handler)
22
- logger.addHandler(logger_handler)
23
- logger.setLevel(logging.DEBUG)
24
12
 
25
13
  # get DB controller
26
14
  db_store = kv_store.KVStore()
@@ -1,29 +1,18 @@
1
1
  # coding=utf-8
2
- import logging
3
- import os
4
-
5
2
  import time
6
- import sys
7
3
 
8
4
 
9
5
  from simplyblock_core import constants, kv_store, utils, rpc_client
10
6
  from simplyblock_core.controllers import events_controller, device_controller, lvol_events
11
7
  from simplyblock_core.models.lvol_model import LVol
12
8
 
13
- # Import the GELF logger
14
- from graypy import GELFUDPHandler
15
9
 
16
10
  from simplyblock_core.models.nvme_device import NVMeDevice
17
11
  from simplyblock_core.rpc_client import RPCClient
18
12
 
19
- # configure logging
20
- logger_handler = logging.StreamHandler(stream=sys.stdout)
21
- logger_handler.setFormatter(logging.Formatter('%(asctime)s: %(levelname)s: %(message)s'))
22
- gelf_handler = GELFUDPHandler('0.0.0.0', constants.GELF_PORT)
23
- logger = logging.getLogger()
24
- logger.addHandler(gelf_handler)
25
- logger.addHandler(logger_handler)
26
- logger.setLevel(logging.DEBUG)
13
+
14
+ logger = utils.get_logger(__name__)
15
+
27
16
 
28
17
  # get DB controller
29
18
  db_controller = kv_store.DBController()
@@ -1,18 +1,16 @@
1
1
  # coding=utf-8
2
- import logging
3
-
4
2
  import time
5
- import sys
6
3
  from datetime import datetime
7
4
 
8
5
 
9
6
  from simplyblock_core.controllers import health_controller, storage_events, device_events
10
7
  from simplyblock_core.models.storage_node import StorageNode
11
8
  from simplyblock_core.rpc_client import RPCClient
12
- from simplyblock_core import constants, kv_store
9
+ from simplyblock_core import constants, kv_store, utils
10
+
11
+
12
+ logger = utils.get_logger(__name__)
13
13
 
14
- # Import the GELF logger
15
- from graypy import GELFUDPHandler
16
14
 
17
15
  def set_node_health_check(snode, health_check_status):
18
16
  snode = db_controller.get_storage_node_by_id(snode.get_id())
@@ -40,15 +38,6 @@ def set_device_health_check(cluster_id, device, health_check_status):
40
38
  dev, dev.health_check, old_status, caused_by="monitor")
41
39
 
42
40
 
43
- # configure logging
44
- logger_handler = logging.StreamHandler(stream=sys.stdout)
45
- logger_handler.setFormatter(logging.Formatter('%(asctime)s: %(levelname)s: %(message)s'))
46
- gelf_handler = GELFUDPHandler('0.0.0.0', constants.GELF_PORT)
47
- logger = logging.getLogger()
48
- logger.addHandler(gelf_handler)
49
- logger.addHandler(logger_handler)
50
- logger.setLevel(logging.DEBUG)
51
-
52
41
  # get DB controller
53
42
  db_store = kv_store.KVStore()
54
43
  db_controller = kv_store.DBController()
@@ -1,17 +1,15 @@
1
1
  # coding=utf-8
2
- import logging
3
- import os
4
-
5
2
  import time
6
- import sys
7
3
  from datetime import datetime
8
4
 
9
- from simplyblock_core import constants, kv_store
5
+
6
+ from simplyblock_core import constants, kv_store, utils
10
7
  from simplyblock_core.models.lvol_model import LVol
11
8
  from simplyblock_core.controllers import health_controller, lvol_events
12
9
 
13
- # Import the GELF logger
14
- from graypy import GELFUDPHandler
10
+
11
+ logger = utils.get_logger(__name__)
12
+
15
13
 
16
14
  def set_lvol_status(lvol, status):
17
15
  if lvol.status != status:
@@ -33,20 +31,10 @@ def set_lvol_health_check(lvol, health_check_status):
33
31
  lvol_events.lvol_health_check_change(lvol, lvol.health_check, old_status, caused_by="monitor")
34
32
 
35
33
 
36
- # configure logging
37
- logger_handler = logging.StreamHandler(stream=sys.stdout)
38
- logger_handler.setFormatter(logging.Formatter('%(asctime)s: %(levelname)s: %(message)s'))
39
- gelf_handler = GELFUDPHandler('0.0.0.0', constants.GELF_PORT)
40
- logger = logging.getLogger()
41
- logger.addHandler(gelf_handler)
42
- logger.addHandler(logger_handler)
43
- logger.setLevel(logging.DEBUG)
44
-
45
34
  # get DB controller
46
35
  db_store = kv_store.KVStore()
47
36
  db_controller = kv_store.DBController()
48
37
 
49
-
50
38
  logger.info("Starting LVol monitor...")
51
39
  while True:
52
40
  lvols = db_controller.get_lvols() # pass