kaqing 2.0.110__py3-none-any.whl → 2.0.214__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of kaqing might be problematic. Click here for more details.

Files changed (251) hide show
  1. adam/__init__.py +0 -2
  2. adam/app_session.py +9 -12
  3. adam/apps.py +18 -4
  4. adam/batch.py +19 -19
  5. adam/checks/check_utils.py +16 -46
  6. adam/checks/cpu.py +7 -1
  7. adam/checks/cpu_metrics.py +52 -0
  8. adam/checks/disk.py +2 -3
  9. adam/columns/columns.py +3 -1
  10. adam/columns/cpu.py +3 -1
  11. adam/columns/cpu_metrics.py +22 -0
  12. adam/columns/memory.py +3 -4
  13. adam/commands/__init__.py +24 -0
  14. adam/commands/app/app.py +38 -0
  15. adam/commands/{app_ping.py → app/app_ping.py} +7 -13
  16. adam/commands/{login.py → app/login.py} +22 -24
  17. adam/commands/app/show_app_actions.py +49 -0
  18. adam/commands/{show → app}/show_app_id.py +8 -11
  19. adam/commands/{show → app}/show_app_queues.py +7 -14
  20. adam/commands/app/show_login.py +56 -0
  21. adam/commands/app/utils_app.py +106 -0
  22. adam/commands/audit/audit.py +22 -40
  23. adam/commands/audit/audit_repair_tables.py +15 -19
  24. adam/commands/audit/audit_run.py +15 -22
  25. adam/commands/audit/completions_l.py +15 -0
  26. adam/commands/audit/show_last10.py +4 -18
  27. adam/commands/audit/show_slow10.py +4 -17
  28. adam/commands/audit/show_top10.py +4 -16
  29. adam/commands/audit/utils_show_top10.py +15 -3
  30. adam/commands/bash/__init__.py +5 -0
  31. adam/commands/bash/bash.py +36 -0
  32. adam/commands/bash/bash_completer.py +93 -0
  33. adam/commands/bash/utils_bash.py +16 -0
  34. adam/commands/cassandra/__init__.py +0 -0
  35. adam/commands/cassandra/download_cassandra_log.py +45 -0
  36. adam/commands/{restart.py → cassandra/restart_cluster.py} +12 -26
  37. adam/commands/cassandra/restart_node.py +51 -0
  38. adam/commands/cassandra/restart_nodes.py +47 -0
  39. adam/commands/{rollout.py → cassandra/rollout.py} +20 -25
  40. adam/commands/cassandra/show_cassandra_repairs.py +37 -0
  41. adam/commands/cassandra/show_cassandra_status.py +117 -0
  42. adam/commands/{show → cassandra}/show_cassandra_version.py +5 -18
  43. adam/commands/cassandra/show_processes.py +50 -0
  44. adam/commands/cassandra/show_storage.py +44 -0
  45. adam/commands/{watch.py → cassandra/watch.py} +26 -29
  46. adam/commands/cli/__init__.py +0 -0
  47. adam/commands/{cli_commands.py → cli/cli_commands.py} +8 -4
  48. adam/commands/cli/clipboard_copy.py +86 -0
  49. adam/commands/cli/show_cli_commands.py +56 -0
  50. adam/commands/code.py +57 -0
  51. adam/commands/command.py +211 -40
  52. adam/commands/commands_utils.py +20 -27
  53. adam/commands/config/__init__.py +0 -0
  54. adam/commands/{param_get.py → config/param_get.py} +11 -14
  55. adam/commands/{param_set.py → config/param_set.py} +8 -12
  56. adam/commands/{show → config}/show_params.py +2 -5
  57. adam/commands/cql/alter_tables.py +66 -0
  58. adam/commands/cql/completions_c.py +29 -0
  59. adam/commands/cql/cqlsh.py +10 -32
  60. adam/commands/cql/utils_cql.py +306 -0
  61. adam/commands/debug/__init__.py +0 -0
  62. adam/commands/debug/debug.py +22 -0
  63. adam/commands/debug/debug_completes.py +35 -0
  64. adam/commands/debug/debug_timings.py +35 -0
  65. adam/commands/debug/show_offloaded_completes.py +45 -0
  66. adam/commands/deploy/code_start.py +7 -10
  67. adam/commands/deploy/code_stop.py +4 -21
  68. adam/commands/deploy/code_utils.py +3 -3
  69. adam/commands/deploy/deploy.py +4 -27
  70. adam/commands/deploy/deploy_frontend.py +14 -17
  71. adam/commands/deploy/deploy_pg_agent.py +3 -6
  72. adam/commands/deploy/deploy_pod.py +65 -73
  73. adam/commands/deploy/deploy_utils.py +14 -24
  74. adam/commands/deploy/undeploy.py +4 -27
  75. adam/commands/deploy/undeploy_frontend.py +4 -7
  76. adam/commands/deploy/undeploy_pg_agent.py +6 -8
  77. adam/commands/deploy/undeploy_pod.py +11 -12
  78. adam/commands/devices/__init__.py +0 -0
  79. adam/commands/devices/device.py +149 -0
  80. adam/commands/devices/device_app.py +163 -0
  81. adam/commands/devices/device_auit_log.py +49 -0
  82. adam/commands/devices/device_cass.py +179 -0
  83. adam/commands/devices/device_export.py +87 -0
  84. adam/commands/devices/device_postgres.py +160 -0
  85. adam/commands/devices/devices.py +25 -0
  86. adam/commands/diag/__init__.py +0 -0
  87. adam/commands/{check.py → diag/check.py} +16 -25
  88. adam/commands/diag/generate_report.py +52 -0
  89. adam/commands/diag/issues.py +43 -0
  90. adam/commands/exit.py +1 -4
  91. adam/commands/export/__init__.py +0 -0
  92. adam/commands/export/clean_up_all_export_sessions.py +37 -0
  93. adam/commands/export/clean_up_export_sessions.py +39 -0
  94. adam/commands/export/completions_x.py +11 -0
  95. adam/commands/export/download_export_session.py +40 -0
  96. adam/commands/export/drop_export_database.py +39 -0
  97. adam/commands/export/drop_export_databases.py +37 -0
  98. adam/commands/export/export.py +37 -0
  99. adam/commands/export/export_databases.py +251 -0
  100. adam/commands/export/export_select.py +34 -0
  101. adam/commands/export/export_sessions.py +210 -0
  102. adam/commands/export/export_use.py +49 -0
  103. adam/commands/export/export_x_select.py +48 -0
  104. adam/commands/export/exporter.py +419 -0
  105. adam/commands/export/import_files.py +44 -0
  106. adam/commands/export/import_session.py +40 -0
  107. adam/commands/export/importer.py +81 -0
  108. adam/commands/export/importer_athena.py +157 -0
  109. adam/commands/export/importer_sqlite.py +78 -0
  110. adam/commands/export/show_column_counts.py +45 -0
  111. adam/commands/export/show_export_databases.py +39 -0
  112. adam/commands/export/show_export_session.py +39 -0
  113. adam/commands/export/show_export_sessions.py +37 -0
  114. adam/commands/export/utils_export.py +366 -0
  115. adam/commands/fs/__init__.py +0 -0
  116. adam/commands/fs/cat.py +36 -0
  117. adam/commands/fs/cat_local.py +42 -0
  118. adam/commands/fs/cd.py +41 -0
  119. adam/commands/fs/download_file.py +47 -0
  120. adam/commands/fs/find_files.py +51 -0
  121. adam/commands/fs/find_processes.py +76 -0
  122. adam/commands/fs/head.py +36 -0
  123. adam/commands/fs/ls.py +41 -0
  124. adam/commands/fs/ls_local.py +40 -0
  125. adam/commands/fs/pwd.py +45 -0
  126. adam/commands/fs/rm.py +18 -0
  127. adam/commands/fs/rm_downloads.py +39 -0
  128. adam/commands/fs/rm_logs.py +38 -0
  129. adam/commands/{shell.py → fs/shell.py} +12 -4
  130. adam/commands/{show → fs}/show_adam.py +3 -3
  131. adam/commands/{show → fs}/show_host.py +1 -1
  132. adam/commands/help.py +5 -3
  133. adam/commands/intermediate_command.py +52 -0
  134. adam/commands/kubectl.py +38 -0
  135. adam/commands/medusa/medusa.py +4 -22
  136. adam/commands/medusa/medusa_backup.py +20 -27
  137. adam/commands/medusa/medusa_restore.py +35 -48
  138. adam/commands/medusa/medusa_show_backupjobs.py +16 -18
  139. adam/commands/medusa/medusa_show_restorejobs.py +13 -18
  140. adam/commands/medusa/utils_medusa.py +15 -0
  141. adam/commands/nodetool/__init__.py +0 -0
  142. adam/commands/{nodetool.py → nodetool/nodetool.py} +9 -20
  143. adam/commands/postgres/completions_p.py +22 -0
  144. adam/commands/postgres/postgres.py +47 -55
  145. adam/commands/postgres/postgres_databases.py +269 -0
  146. adam/commands/postgres/postgres_ls.py +5 -9
  147. adam/commands/postgres/postgres_preview.py +5 -9
  148. adam/commands/postgres/utils_postgres.py +80 -0
  149. adam/commands/preview_table.py +8 -44
  150. adam/commands/reaper/reaper.py +4 -27
  151. adam/commands/reaper/reaper_forward.py +49 -56
  152. adam/commands/reaper/reaper_forward_session.py +6 -0
  153. adam/commands/reaper/reaper_forward_stop.py +10 -16
  154. adam/commands/reaper/reaper_restart.py +7 -14
  155. adam/commands/reaper/reaper_run_abort.py +8 -33
  156. adam/commands/reaper/reaper_runs.py +43 -58
  157. adam/commands/reaper/reaper_runs_abort.py +29 -49
  158. adam/commands/reaper/reaper_schedule_activate.py +14 -33
  159. adam/commands/reaper/reaper_schedule_start.py +9 -33
  160. adam/commands/reaper/reaper_schedule_stop.py +9 -33
  161. adam/commands/reaper/reaper_schedules.py +4 -14
  162. adam/commands/reaper/reaper_status.py +8 -16
  163. adam/commands/reaper/utils_reaper.py +203 -0
  164. adam/commands/repair/repair.py +4 -22
  165. adam/commands/repair/repair_log.py +5 -11
  166. adam/commands/repair/repair_run.py +27 -34
  167. adam/commands/repair/repair_scan.py +32 -40
  168. adam/commands/repair/repair_stop.py +5 -12
  169. adam/commands/show.py +40 -0
  170. adam/config.py +5 -15
  171. adam/embedded_params.py +1 -1
  172. adam/log.py +4 -4
  173. adam/repl.py +83 -116
  174. adam/repl_commands.py +86 -45
  175. adam/repl_session.py +9 -1
  176. adam/repl_state.py +176 -40
  177. adam/sql/async_executor.py +62 -0
  178. adam/sql/lark_completer.py +286 -0
  179. adam/sql/lark_parser.py +604 -0
  180. adam/sql/qingl.lark +1076 -0
  181. adam/sql/sql_completer.py +52 -27
  182. adam/sql/sql_state_machine.py +131 -19
  183. adam/sso/authn_ad.py +6 -8
  184. adam/sso/authn_okta.py +4 -6
  185. adam/sso/cred_cache.py +4 -9
  186. adam/sso/idp.py +9 -12
  187. adam/utils.py +670 -31
  188. adam/utils_athena.py +145 -0
  189. adam/utils_audits.py +12 -103
  190. adam/utils_issues.py +32 -0
  191. adam/utils_k8s/app_clusters.py +35 -0
  192. adam/utils_k8s/app_pods.py +41 -0
  193. adam/utils_k8s/cassandra_clusters.py +35 -20
  194. adam/utils_k8s/cassandra_nodes.py +15 -6
  195. adam/utils_k8s/custom_resources.py +16 -17
  196. adam/utils_k8s/ingresses.py +2 -2
  197. adam/utils_k8s/jobs.py +7 -11
  198. adam/utils_k8s/k8s.py +96 -0
  199. adam/utils_k8s/kube_context.py +3 -6
  200. adam/{pod_exec_result.py → utils_k8s/pod_exec_result.py} +13 -4
  201. adam/utils_k8s/pods.py +159 -89
  202. adam/utils_k8s/secrets.py +4 -4
  203. adam/utils_k8s/service_accounts.py +5 -4
  204. adam/utils_k8s/services.py +2 -2
  205. adam/utils_k8s/statefulsets.py +6 -14
  206. adam/utils_local.py +80 -0
  207. adam/utils_net.py +4 -4
  208. adam/utils_repl/__init__.py +0 -0
  209. adam/utils_repl/appendable_completer.py +6 -0
  210. adam/utils_repl/automata_completer.py +48 -0
  211. adam/utils_repl/repl_completer.py +93 -0
  212. adam/utils_repl/state_machine.py +173 -0
  213. adam/utils_sqlite.py +132 -0
  214. adam/version.py +1 -1
  215. {kaqing-2.0.110.dist-info → kaqing-2.0.214.dist-info}/METADATA +1 -1
  216. kaqing-2.0.214.dist-info/RECORD +272 -0
  217. kaqing-2.0.214.dist-info/top_level.txt +2 -0
  218. teddy/__init__.py +0 -0
  219. teddy/lark_parser.py +436 -0
  220. teddy/lark_parser2.py +618 -0
  221. adam/commands/alter_tables.py +0 -81
  222. adam/commands/app.py +0 -67
  223. adam/commands/bash.py +0 -150
  224. adam/commands/cd.py +0 -125
  225. adam/commands/cp.py +0 -95
  226. adam/commands/cql/cql_completions.py +0 -15
  227. adam/commands/cql/cql_utils.py +0 -112
  228. adam/commands/devices.py +0 -118
  229. adam/commands/issues.py +0 -75
  230. adam/commands/logs.py +0 -40
  231. adam/commands/ls.py +0 -146
  232. adam/commands/postgres/postgres_context.py +0 -239
  233. adam/commands/postgres/postgres_utils.py +0 -31
  234. adam/commands/postgres/psql_completions.py +0 -10
  235. adam/commands/pwd.py +0 -77
  236. adam/commands/reaper/reaper_session.py +0 -159
  237. adam/commands/report.py +0 -63
  238. adam/commands/show/show.py +0 -54
  239. adam/commands/show/show_app_actions.py +0 -56
  240. adam/commands/show/show_cassandra_status.py +0 -128
  241. adam/commands/show/show_commands.py +0 -61
  242. adam/commands/show/show_login.py +0 -63
  243. adam/commands/show/show_processes.py +0 -53
  244. adam/commands/show/show_repairs.py +0 -47
  245. adam/commands/show/show_storage.py +0 -52
  246. kaqing-2.0.110.dist-info/RECORD +0 -187
  247. kaqing-2.0.110.dist-info/top_level.txt +0 -1
  248. /adam/commands/{show → app}/__init__.py +0 -0
  249. /adam/commands/{nodetool_commands.py → nodetool/nodetool_commands.py} +0 -0
  250. {kaqing-2.0.110.dist-info → kaqing-2.0.214.dist-info}/WHEEL +0 -0
  251. {kaqing-2.0.110.dist-info → kaqing-2.0.214.dist-info}/entry_points.txt +0 -0
adam/__init__.py CHANGED
@@ -1,3 +1 @@
1
- import click
2
-
3
1
  from .version import __version__, __release__
adam/app_session.py CHANGED
@@ -1,7 +1,6 @@
1
1
  import json
2
2
  import threading
3
3
  import time
4
- import traceback
5
4
  import requests
6
5
  from urllib.parse import urlparse
7
6
 
@@ -9,7 +8,7 @@ from adam.log import Log
9
8
  from adam.sso.idp import Idp
10
9
  from adam.sso.idp_login import IdpLogin
11
10
  from adam.config import Config
12
- from adam.utils import json_to_csv, lines_to_tabular, log, log2
11
+ from adam.utils import debug, debug_trace, json_to_csv, tabulize, log2, log_exc
13
12
  from adam.apps import Apps
14
13
 
15
14
  class AppLogin:
@@ -62,11 +61,9 @@ class AppSession:
62
61
  if r.status_code >= 200 and r.status_code < 300 or r.status_code == 400:
63
62
  try:
64
63
  js = r.json()
65
- try:
64
+ with log_exc(js):
66
65
  header, lines = json_to_csv(js, delimiter='\t')
67
- log(lines_to_tabular(lines, header=header, separator='\t'))
68
- except:
69
- log(js)
66
+ tabulize(lines, header=header, separator='\t')
70
67
  except:
71
68
  if urlparse(r.url).hostname != urlparse(uri).hostname and not retried:
72
69
  app_login = app_session.login(idp_uri=app_login.idp_uri, forced=forced, use_token_from_env=False, use_cached_creds=False)
@@ -76,7 +73,7 @@ class AppSession:
76
73
 
77
74
  if r.text:
78
75
  log2(f'{r.status_code} {r.url} Failed parsing the results.')
79
- Config().debug(r.text)
76
+ debug(r.text)
80
77
  else:
81
78
  log2(r.status_code)
82
79
  log2(r.text)
@@ -115,7 +112,7 @@ class AppSession:
115
112
  try:
116
113
  # oidc/login may hang
117
114
  timeout = Config().get('app.login.timeout', 5)
118
- Config().debug(f'-> {idp_login.app_login_url}')
115
+ debug(f'-> {idp_login.app_login_url}')
119
116
  session.post(idp_login.app_login_url, headers=headers, data=form_data, timeout=timeout)
120
117
  except Exception:
121
118
  pass
@@ -133,7 +130,7 @@ class AppSession:
133
130
  check_uri = Config().get('app.login.session-check-url', 'https://{host}/{env}/{app}/api/8/C3/userSessionToken')
134
131
  check_uri = check_uri.replace('{host}', self.host).replace('{env}', self.env).replace('{app}', 'c3')
135
132
  r = session.get(check_uri)
136
- Config().debug(f'{r.status_code} {check_uri}')
133
+ debug(f'{r.status_code} {check_uri}')
137
134
 
138
135
  res_text = r.text
139
136
  js = json.loads(res_text)
@@ -142,10 +139,10 @@ class AppSession:
142
139
  break
143
140
 
144
141
  app_access_token = js['signedToken']
145
- Config().debug(f'{r.text}')
142
+ debug(f'{r.text}')
146
143
 
147
144
  self.app_login = AppLogin(session, app_access_token, idp_uri)
148
- except Exception:
145
+ except Exception as e:
149
146
  try:
150
147
  need = urlparse(r.url).hostname
151
148
  if idp_login.idp_uri:
@@ -158,7 +155,7 @@ class AppSession:
158
155
  log2(f"Invalid username/password.")
159
156
  break
160
157
  finally:
161
- Config().debug(traceback.format_exc())
158
+ debug_trace()
162
159
 
163
160
  if 'res_text' in locals():
164
161
  Log.log_to_file(res_text)
adam/apps.py CHANGED
@@ -12,7 +12,8 @@ from adam.utils_k8s.services import Services
12
12
  from adam.utils import copy_config_file
13
13
 
14
14
  class AppAction:
15
- def __init__(self, name: str, payload: str = None, args: dict[str, str] = None, help: str = None):
15
+ def __init__(self, typ: str, name: str, payload: str = None, args: dict[str, str] = None, help: str = None):
16
+ self.typ = typ
16
17
  self.name = name
17
18
  self.payload = payload
18
19
  self.args = args
@@ -33,6 +34,19 @@ class AppAction:
33
34
 
34
35
  return args
35
36
 
37
+ def __str__(self):
38
+ line = None
39
+
40
+ args = ','.join(self.arguments())
41
+ if args:
42
+ line = f'{self.typ}.{self.name},{args}'
43
+ else:
44
+ line = f'{self.typ}.{self.name},'
45
+ if self.help:
46
+ line = f'{line},{self.help}'
47
+
48
+ return line
49
+
36
50
  class AppType:
37
51
  all_types: list['AppType'] = []
38
52
 
@@ -62,12 +76,12 @@ class AppType:
62
76
  aa: AppAction = None
63
77
 
64
78
  if isinstance(v, str):
65
- aa = AppAction(k, help=v)
79
+ aa = AppAction(typ, k, help=v)
66
80
  elif isinstance(v, dict):
67
81
  args = {k: v1 for k, v1 in v.items()}
68
- aa = AppAction(k, payload=v['payload'], args=args, help=v['help'])
82
+ aa = AppAction(typ, k, payload=v['payload'], args=args, help=v['help'])
69
83
  else:
70
- aa = AppAction(k)
84
+ aa = AppAction(typ, k)
71
85
 
72
86
  app_actions.append(aa)
73
87
 
adam/batch.py CHANGED
@@ -1,29 +1,29 @@
1
1
  import click
2
2
 
3
3
  from adam.commands.audit.audit import Audit, AuditCommandHelper
4
- from adam.commands.bash import Bash
5
- from adam.commands.check import Check, CheckCommandHelper
6
- from adam.commands.cp import ClipboardCopy, CopyCommandHelper
4
+ from adam.commands.bash.bash import Bash
5
+ from adam.commands.cassandra.restart_nodes import RestartNodes
6
+ from adam.commands.cassandra.rollout import RollOut
7
+ from adam.commands.cassandra.watch import Watch
8
+ from adam.commands.cli.clipboard_copy import ClipboardCopy, CopyCommandHelper
7
9
  from adam.commands.command import Command
8
10
  from adam.commands.command_helpers import ClusterCommandHelper, ClusterOrPodCommandHelper, PodCommandHelper
9
11
  from adam.commands.cql.cqlsh import CqlCommandHelper, Cqlsh
10
12
  from adam.commands.deploy.deploy import Deploy, DeployCommandHelper
11
13
  from adam.commands.deploy.undeploy import Undeploy, UndeployCommandHelper
12
- from adam.commands.issues import Issues
13
- from adam.commands.login import Login
14
- from adam.commands.logs import Logs
15
- from adam.commands.ls import Ls
14
+ from adam.commands.app.login import Login
15
+ from adam.commands.cassandra.download_cassandra_log import DownloadCassandraLog
16
+ from adam.commands.diag.check import Check, CheckCommandHelper
17
+ from adam.commands.diag.generate_report import GenerateReport
18
+ from adam.commands.diag.issues import Issues
19
+ from adam.commands.fs.ls import Ls
16
20
  from adam.commands.medusa.medusa import Medusa
17
- from adam.commands.nodetool import NodeTool, NodeToolCommandHelper
21
+ from adam.commands.nodetool.nodetool import NodeTool, NodeToolCommandHelper
18
22
  from adam.commands.postgres.postgres import Postgres, PostgresCommandHelper
19
23
  from adam.commands.preview_table import PreviewTable
20
24
  from adam.commands.reaper.reaper import Reaper, ReaperCommandHelper
21
25
  from adam.commands.repair.repair import Repair, RepairCommandHelper
22
- from adam.commands.report import Report
23
- from adam.commands.restart import Restart
24
- from adam.commands.rollout import RollOut
25
- from adam.commands.show.show import Show, ShowCommandHelper
26
- from adam.commands.watch import Watch
26
+ from adam.commands.show import Show, ShowCommandHelper
27
27
  from adam.utils_k8s.kube_context import KubeContext
28
28
  from adam.repl import enter_repl
29
29
  from adam.repl_state import ReplState
@@ -35,7 +35,7 @@ from adam.cli_group import cli
35
35
  @click.option('--param', '-v', multiple=True, metavar='<key>=<value>', help='parameter override')
36
36
  @click.argument('extra_args', nargs=-1, metavar='repair', type=click.UNPROCESSED)
37
37
  def audit(kubeconfig: str, config: str, param: list[str], extra_args):
38
- run_command(Audit(), kubeconfig, config, param, None, None, None, extra_args)
38
+ run_command(Audit(), kubeconfig, config, param, None, None, None, extra_args, device=ReplState.L)
39
39
 
40
40
 
41
41
  @cli.command(context_settings=dict(ignore_unknown_options=True, allow_extra_args=True), cls=ClusterOrPodCommandHelper, help='Run a single bash command.')
@@ -128,7 +128,7 @@ def login(kubeconfig: str, config: str, param: list[str], namespace: str, pod: s
128
128
  @click.option('--pod', '-p', required=False, metavar='pod', help='Kubernetes pod name')
129
129
  @click.argument('extra_args', nargs=-1, metavar='<pod>', type=click.UNPROCESSED)
130
130
  def logs(kubeconfig: str, config: str, param: list[str], namespace: str, pod: str, extra_args):
131
- run_command(Logs(), kubeconfig, config, param, None, namespace, pod, extra_args)
131
+ run_command(DownloadCassandraLog(), kubeconfig, config, param, None, namespace, pod, extra_args)
132
132
 
133
133
 
134
134
  @cli.command(context_settings=dict(ignore_unknown_options=True, allow_extra_args=True), cls=ClusterCommandHelper, help='List statefulset or pods.')
@@ -221,7 +221,7 @@ def repair(kubeconfig: str, config: str, param: list[str], cluster: str, namespa
221
221
  @click.option('--show', '-s', is_flag=True, help='show output from Cassandra nodes')
222
222
  @click.argument('extra_args', nargs=-1, metavar='[cluster|pod]', type=click.UNPROCESSED)
223
223
  def report(kubeconfig: str, config: str, param: list[str], cluster: str, namespace: str, pod: str, show: bool, extra_args):
224
- run_command(Report(), kubeconfig, config, param, cluster, namespace, pod, ('-s',) + extra_args if show else extra_args)
224
+ run_command(GenerateReport(), kubeconfig, config, param, cluster, namespace, pod, ('-s',) + extra_args if show else extra_args)
225
225
 
226
226
 
227
227
  @cli.command(context_settings=dict(ignore_unknown_options=True, allow_extra_args=True), cls=ClusterOrPodCommandHelper, help='Restart Cassandra Cluster or Node')
@@ -234,7 +234,7 @@ def report(kubeconfig: str, config: str, param: list[str], cluster: str, namespa
234
234
  @click.option('--force', is_flag=True, help='need for restarting the whole cluster')
235
235
  @click.argument('extra_args', nargs=-1, metavar='<cluster|pod>', type=click.UNPROCESSED)
236
236
  def restart(kubeconfig: str, config: str, param: list[str], cluster: str, namespace: str, pod: str, force: bool, extra_args):
237
- run_command(Restart(), kubeconfig, config, param, cluster, namespace, pod, ('--force',) + extra_args if force else extra_args)
237
+ run_command(RestartNodes(), kubeconfig, config, param, cluster, namespace, pod, ('--force',) + extra_args if force else extra_args)
238
238
 
239
239
 
240
240
  @cli.command(context_settings=dict(ignore_unknown_options=True, allow_extra_args=True), cls=ClusterOrPodCommandHelper, help='Rolling restart Cassandra Cluster.')
@@ -281,14 +281,14 @@ def watch(kubeconfig: str, config: str, param: list[str], cluster: str, namespac
281
281
  run_command(Watch(), kubeconfig, config, param, cluster, namespace, None, extra_args)
282
282
 
283
283
 
284
- def run_command(cmd: Command, kubeconfig: str, config: str, params: list[str], cluster:str, namespace: str, pod: str, extra_args):
284
+ def run_command(cmd: Command, kubeconfig: str, config: str, params: list[str], cluster:str, namespace: str, pod: str, extra_args, device=ReplState.C):
285
285
  is_user_entry = False
286
286
 
287
287
  KubeContext.init_config(kubeconfig, is_user_entry=is_user_entry)
288
288
  if not KubeContext.init_params(config, params, is_user_entry=is_user_entry):
289
289
  return
290
290
 
291
- state = ReplState(ns_sts=cluster, pod=pod, namespace=namespace)
291
+ state = ReplState(device=device, ns_sts=cluster, pod=pod, namespace=namespace)
292
292
  if cmd.command() == 'pg' and not extra_args:
293
293
  state, _ = state.apply_args(extra_args)
294
294
  state.device = ReplState.P
@@ -1,7 +1,3 @@
1
- from collections.abc import Callable
2
- from concurrent.futures import ThreadPoolExecutor, as_completed
3
- import time
4
-
5
1
  from adam.checks.check import Check
6
2
  from adam.checks.check_context import CheckContext
7
3
  from adam.checks.check_result import CheckResult
@@ -14,10 +10,9 @@ from adam.checks.memory import Memory
14
10
  from adam.checks.status import Status
15
11
  from adam.config import Config
16
12
  from adam.utils_k8s.cassandra_nodes import CassandraNodes
17
- from adam.utils_k8s.kube_context import KubeContext
18
13
  from adam.utils_k8s.secrets import Secrets
19
14
  from adam.utils_k8s.statefulsets import StatefulSets
20
- from adam.utils import elapsed_time, log2
15
+ from adam.utils import parallelize, log2
21
16
 
22
17
  def all_checks() -> list[Check]:
23
18
  return [CompactionStats(), Cpu(), Gossip(), Memory(), Disk(), Status()]
@@ -38,57 +33,32 @@ def checks_from_csv(check_str: str):
38
33
 
39
34
  return checks
40
35
 
41
- def run_checks(cluster: str = None, namespace: str = None, pod: str = None, checks: list[Check] = None, show_output=True):
36
+ def run_checks(cluster: str = None, namespace: str = None, pod: str = None, checks: list[Check] = None, show_out=True):
42
37
  if not checks:
43
38
  checks = all_checks()
44
39
 
45
- sss: list[tuple[str, str]] = StatefulSets.list_sts_name_and_ns()
46
-
47
- action = 'issues'
48
- crs: list[CheckResult] = []
49
-
50
- def on_clusters(f: Callable[[any, list[str]], any]):
51
- for ss, ns in sss:
52
- if (not cluster or cluster == ss) and (not namespace or namespace == ns):
53
- pods = StatefulSets.pods(ss, ns)
54
- for pod_name in [pod.metadata.name for pod in pods]:
55
- if not pod or pod == pod_name:
56
- f(ss, ns, pod_name, show_output)
57
-
58
- max_workers = Config().action_workers(action, 30)
59
- if max_workers < 2:
60
- def serial(ss, ns, pod_name, show_output):
61
- if not pod or pod == pod_name:
62
- crs.append(run_checks_on_pod(checks, ss[0], ns, pod_name, show_output))
63
-
64
- on_clusters(serial)
65
- else:
66
- if KubeContext.show_parallelism():
67
- log2(f'Executing on all nodes from statefulset with {max_workers} workers...')
68
- start_time = time.time()
69
- try:
70
- futures = []
71
- with ThreadPoolExecutor(max_workers=max_workers) as executor:
72
- def submit(ss, ns, pod_name, show_output):
73
- f = executor.submit(run_checks_on_pod, checks, ss, ns, pod_name, show_output,)
74
- if f: futures.append(f)
75
-
76
- on_clusters(submit)
40
+ sts_ns: list[tuple[str, str]] = StatefulSets.list_sts_name_and_ns()
77
41
 
78
- crs = [future.result() for future in as_completed(futures)]
79
- finally:
80
- if KubeContext.show_parallelism():
81
- log2(f"Parallel {action} elapsed time: {elapsed_time(start_time)} with {max_workers} workers")
42
+ sts_ns_pods: list[tuple[str, str, str]] = []
43
+ for sts, ns in sts_ns:
44
+ if (not cluster or cluster == sts) and (not namespace or namespace == ns):
45
+ pods = StatefulSets.pods(sts, ns)
46
+ for pod_name in [pod.metadata.name for pod in pods]:
47
+ if not pod or pod == pod_name:
48
+ sts_ns_pods.append((sts, ns, pod_name))
82
49
 
83
- return crs
50
+ with parallelize(sts_ns_pods,
51
+ Config().action_workers('issues', 30),
52
+ msg='d`Running|Ran checks on {size} pods') as exec:
53
+ return exec.map(lambda sts_ns_pod: run_checks_on_pod(checks, sts_ns_pod[0], sts_ns_pod[1], sts_ns_pod[2], show_out))
84
54
 
85
- def run_checks_on_pod(checks: list[Check], cluster: str = None, namespace: str = None, pod: str = None, show_output=True):
55
+ def run_checks_on_pod(checks: list[Check], cluster: str = None, namespace: str = None, pod: str = None, show_out=True):
86
56
  host_id = CassandraNodes.get_host_id(pod, namespace)
87
57
  user, pw = Secrets.get_user_pass(pod, namespace)
88
58
  results = {}
89
59
  issues: list[Issue] = []
90
60
  for c in checks:
91
- check_results = c.check(CheckContext(cluster, host_id, pod, namespace, user, pw, show_output=show_output))
61
+ check_results = c.check(CheckContext(cluster, host_id, pod, namespace, user, pw, show_output=show_out))
92
62
  if check_results.details:
93
63
  results = results | {check_results.name: check_results.details}
94
64
  if check_results.issues:
adam/checks/cpu.py CHANGED
@@ -7,6 +7,7 @@ from adam.checks.issue import Issue
7
7
  from adam.config import Config
8
8
  from adam.utils_k8s.cassandra_nodes import CassandraNodes
9
9
  from adam.utils_k8s.custom_resources import CustomResources
10
+ from adam.utils_k8s.pods import Pods
10
11
 
11
12
  class Cpu(Check):
12
13
  def name(self):
@@ -20,10 +21,15 @@ class Cpu(Check):
20
21
  'namespace': ctx.namespace,
21
22
  'statefulset': ctx.statefulset,
22
23
  'cpu': 'Unknown',
23
- 'idle': 'Unknown'
24
+ 'idle': 'Unknown',
25
+ 'limit': 'NA'
24
26
  }
25
27
 
26
28
  try:
29
+ container = Pods.get_container(ctx.namespace, ctx.pod, container_name='cassandra')
30
+ if container.resources.limits and "cpu" in container.resources.limits:
31
+ details['limit'] = container.resources.limits["cpu"]
32
+
27
33
  idle = 'Unknown'
28
34
  result = CassandraNodes.exec(ctx.pod, ctx.namespace, "mpstat 5 2 | grep Average | awk '{print $NF}'", show_out=ctx.show_output)
29
35
  lines = result.stdout.strip(' \r\n').split('\n')
@@ -0,0 +1,52 @@
1
+ from kubernetes.utils import parse_quantity
2
+
3
+ from adam.checks.check import Check
4
+ from adam.checks.check_context import CheckContext
5
+ from adam.checks.check_result import CheckResult
6
+ from adam.checks.issue import Issue
7
+ from adam.config import Config
8
+ from adam.utils_k8s.custom_resources import CustomResources
9
+ from adam.utils_k8s.pods import Pods
10
+
11
+ class CpuMetrics(Check):
12
+ def name(self):
13
+ return 'cpu-metrics'
14
+
15
+ def check(self, ctx: CheckContext) -> CheckResult:
16
+ issues: list[Issue] = []
17
+
18
+ details = {
19
+ 'name': ctx.pod,
20
+ 'namespace': ctx.namespace,
21
+ 'statefulset': ctx.statefulset,
22
+ 'cpu': 'Unknown',
23
+ 'limit': 'NA'
24
+ }
25
+
26
+ try:
27
+ container = Pods.get_container(ctx.namespace, ctx.pod, container_name='cassandra')
28
+ if container.resources.limits and "cpu" in container.resources.limits:
29
+ details['limit'] = container.resources.limits["cpu"]
30
+
31
+ metrics = CustomResources.get_metrics(ctx.namespace, ctx.pod, container_name='cassandra')
32
+ usage = 'Unknown'
33
+ if metrics:
34
+ usage = details['cpu'] = metrics["usage"]["cpu"]
35
+
36
+ cpu_threshold = Config().get('checks.cpu-threshold', 0.0)
37
+ if cpu_threshold != 0.0 and usage != "Unknown" and parse_quantity(usage) > cpu_threshold:
38
+ issues.append(Issue(
39
+ statefulset=ctx.statefulset,
40
+ namespace=ctx.namespace,
41
+ pod=ctx.pod,
42
+ category='cpu',
43
+ desc=f'CPU is too busy: {usage}',
44
+ suggestion=f"qing restart {ctx.pod}@{ctx.namespace}"
45
+ ))
46
+ except Exception as e:
47
+ issues.append(self.issue_from_err(sts_name=ctx.statefulset, ns=ctx.namespace, pod_name=ctx.pod, exception=e))
48
+
49
+ return CheckResult(self.name(), details, issues)
50
+
51
+ def help(self):
52
+ return f'{CpuMetrics().name()}: check cpu busy percentage with metrics'
adam/checks/disk.py CHANGED
@@ -6,6 +6,7 @@ from adam.checks.check_context import CheckContext
6
6
  from adam.checks.check_result import CheckResult
7
7
  from adam.checks.issue import Issue
8
8
  from adam.config import Config
9
+ from adam.utils import log_exc
9
10
  from adam.utils_k8s.cassandra_nodes import CassandraNodes
10
11
 
11
12
  class Disk(Check):
@@ -87,10 +88,8 @@ class Disk(Check):
87
88
 
88
89
  ss_size = 0.0
89
90
  if ss_out:
90
- try:
91
+ with log_exc():
91
92
  ss_size = round(float(ss_out.strip(' \r\n')) / 1024 / 1024, 2)
92
- except:
93
- pass
94
93
 
95
94
  def parse_du_out(l: str, default: str = None):
96
95
  groups = re.match(r'^(\S+)\s+(\S+)$', l.strip('\r'))
adam/columns/columns.py CHANGED
@@ -1,6 +1,7 @@
1
1
  from adam.columns.column import Column
2
2
  from adam.columns.compactions import Compactions
3
3
  from adam.columns.cpu import Cpu
4
+ from adam.columns.cpu_metrics import CpuMetrics
4
5
  from adam.columns.dir_data import DataDir
5
6
  from adam.columns.dir_snapshots import SnapshotsDir
6
7
  from adam.columns.gossip import Gossip
@@ -23,7 +24,7 @@ class Columns:
23
24
  COLUMNS_BY_NAME = None
24
25
 
25
26
  def all_columns():
26
- return [Compactions(), Cpu(), DataDir(), SnapshotsDir(), Gossip(), HostId(), Memory(),
27
+ return [Compactions(), Cpu(), CpuMetrics(), DataDir(), SnapshotsDir(), Gossip(), HostId(), Memory(),
27
28
  NodeAddress(), NodeLoad(), NodeOwns(), NodeStatus(),NodeTokens(), PodName(), CassandraVolume(), RootVolume()]
28
29
 
29
30
  def columns_by_name():
@@ -38,6 +39,7 @@ class Columns:
38
39
  name = name.strip(' ')
39
40
  if not name in Columns.COLUMNS_BY_NAME:
40
41
  return None
42
+
41
43
  cols.append(Columns.COLUMNS_BY_NAME[name]())
42
44
 
43
45
  return cols
adam/columns/cpu.py CHANGED
@@ -1,3 +1,5 @@
1
+ from kubernetes.utils.quantity import parse_quantity
2
+
1
3
  from adam.checks.check_result import CheckResult
2
4
  from adam.checks.cpu import Cpu as CpuCheck
3
5
  from adam.columns.column import Column
@@ -14,4 +16,4 @@ class Cpu(Column):
14
16
  cpu = r.details[CpuCheck().name()]
15
17
  busy = 100.0 - float(cpu['idle'])
16
18
 
17
- return f'{round(busy)}%'
19
+ return f'{round(busy)}%/{parse_quantity(cpu["limit"]) * 100}%'
@@ -0,0 +1,22 @@
1
+ from kubernetes.utils.quantity import parse_quantity
2
+
3
+ from adam.checks.check_result import CheckResult
4
+ from adam.checks.cpu_metrics import CpuMetrics as CpuCheck
5
+ from adam.columns.column import Column
6
+
7
+ class CpuMetrics(Column):
8
+ def name(self):
9
+ return 'cpu-metrics'
10
+
11
+ def checks(self):
12
+ return [CpuCheck()]
13
+
14
+ def pod_value(self, check_results: list[CheckResult], pod_name: str):
15
+ r = self.result_by_pod(check_results, pod_name)
16
+ cpu = r.details[CpuCheck().name()]
17
+
18
+ cpu_decimal = parse_quantity(cpu['cpu'])
19
+ cpu_limit = parse_quantity(cpu['limit'])
20
+ business = cpu_decimal * 100 / cpu_limit
21
+
22
+ return f"{business:.2f}%({cpu_decimal}/{cpu_limit})"
adam/columns/memory.py CHANGED
@@ -3,6 +3,7 @@ from kubernetes.utils import parse_quantity
3
3
  from adam.checks.check_result import CheckResult
4
4
  from adam.checks.memory import Memory as MemoryCheck
5
5
  from adam.columns.column import Column
6
+ from adam.utils import log_exc
6
7
 
7
8
  class Memory(Column):
8
9
  def name(self):
@@ -18,7 +19,5 @@ class Memory(Column):
18
19
  return f"{Memory.to_g(mem['used'])}/{Memory.to_g(mem['limit'])}"
19
20
 
20
21
  def to_g(v: str):
21
- try:
22
- return f'{round(parse_quantity(v) / 1024 / 1024 / 1024, 2)}G'
23
- except:
24
- return v
22
+ with log_exc():
23
+ return f'{round(parse_quantity(v) / 1024 / 1024 / 1024, 2)}G'
adam/commands/__init__.py CHANGED
@@ -0,0 +1,24 @@
1
+ from collections.abc import Callable
2
+ from adam.commands.command import ExtractAllOptionsHandler, ExtractOptionsHandler, ExtractSequenceOptionsHandler, ExtractTrailingOptionsHandler, ValidateArgCountHandler
3
+ from adam.repl_state import ReplState
4
+ from adam.commands.app.utils_app import AppHandler
5
+
6
+ def app(state: ReplState) -> AppHandler:
7
+ return AppHandler(state)
8
+
9
+ def extract_options(args: list[str], options: list[str]):
10
+ return ExtractOptionsHandler(args, options = options)
11
+
12
+ def extract_trailing_options(args: list[str], trailing: list[str]):
13
+ return ExtractTrailingOptionsHandler(args, trailing = trailing)
14
+
15
+ def extract_all_options(args: list[str], trailing = None, sequence = None, options = None):
16
+ return ExtractAllOptionsHandler(args, trailing = trailing, sequence = sequence, options = options)
17
+
18
+ def extract_sequence(args: list[str], sequence: list[str]):
19
+ return ExtractSequenceOptionsHandler(args, sequence = sequence)
20
+
21
+ def validate_args(args: list[str], state: ReplState, at_least: int = 1, exactly: int = -1,
22
+ name: str = None, msg: Callable[[], None] = None, default: str = None,
23
+ separator=None):
24
+ return ValidateArgCountHandler(args, state, at_least=at_least, exactly=exactly, name=name, msg=msg, default=default, separator=separator)
@@ -0,0 +1,38 @@
1
+ from adam.commands import app, extract_options
2
+ from adam.commands.command import Command
3
+ from adam.repl_state import ReplState, RequiredState
4
+
5
+ class App(Command):
6
+ COMMAND = 'app'
7
+
8
+ # the singleton pattern
9
+ def __new__(cls, *args, **kwargs):
10
+ if not hasattr(cls, 'instance'): cls.instance = super(App, cls).__new__(cls)
11
+
12
+ return cls.instance
13
+
14
+ def __init__(self, successor: Command=None):
15
+ super().__init__(successor)
16
+
17
+ def command(self):
18
+ return App.COMMAND
19
+
20
+ def required(self):
21
+ return RequiredState.APP_APP
22
+
23
+ def run(self, cmd: str, state: ReplState):
24
+ if not(args := self.args(cmd)):
25
+ return super().run(cmd, state)
26
+
27
+ with self.validate(args, state) as (args, state):
28
+ with extract_options(args, '--force') as (args, forced):
29
+ with app(state) as http:
30
+ http.post(args, forced=forced)
31
+
32
+ return state
33
+
34
+ def completion(self, _: ReplState):
35
+ return {}
36
+
37
+ def help(self, _: ReplState):
38
+ return f"<AppType>.<AppAction> <args> [--force]\t post app action; check with 'show app actions' command"
@@ -1,6 +1,6 @@
1
+ from adam.commands import app, extract_options
1
2
  from adam.commands.command import Command
2
3
  from adam.repl_state import ReplState, RequiredState
3
- from adam.app_session import AppSession
4
4
 
5
5
  class AppPing(Command):
6
6
  COMMAND = 'app ping'
@@ -24,21 +24,15 @@ class AppPing(Command):
24
24
  if not(args := self.args(cmd)):
25
25
  return super().run(cmd, state)
26
26
 
27
- state, args = self.apply_state(args, state)
28
- if not self.validate_state(state):
29
- return state
27
+ with self.validate(args, state) as (args, state):
28
+ with extract_options(args, '--force') as (args, forced):
29
+ with app(state) as http:
30
+ http.post(['Echo.echoStatic'], forced=forced)
30
31
 
31
- _, forced = Command.extract_options(args, '--force')
32
-
33
- AppSession.run(state.app_env, state.app_app, state.namespace, 'Echo', 'echoStatic', forced= forced)
34
-
35
- return state
32
+ return state
36
33
 
37
34
  def completion(self, state: ReplState):
38
- if state.app_app:
39
- return super().completion(state, {'--force': None})
40
-
41
- return {}
35
+ return super().completion(state, {'--force': None})
42
36
 
43
37
  def help(self, _: ReplState):
44
38
  return f"{AppPing.COMMAND} [--force]\t ping app server with Echo.echoStatic()"