kaqing 2.0.98__py3-none-any.whl → 2.0.171__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (194) hide show
  1. adam/__init__.py +0 -2
  2. adam/app_session.py +9 -7
  3. adam/batch.py +4 -18
  4. adam/checks/check_utils.py +14 -46
  5. adam/checks/cpu.py +7 -1
  6. adam/checks/cpu_metrics.py +52 -0
  7. adam/columns/columns.py +3 -1
  8. adam/columns/cpu.py +3 -1
  9. adam/columns/cpu_metrics.py +22 -0
  10. adam/commands/__init__.py +15 -0
  11. adam/commands/alter_tables.py +50 -61
  12. adam/commands/app_cmd.py +38 -0
  13. adam/commands/app_ping.py +8 -14
  14. adam/commands/audit/audit.py +43 -30
  15. adam/commands/audit/audit_repair_tables.py +26 -46
  16. adam/commands/audit/audit_run.py +50 -0
  17. adam/commands/audit/show_last10.py +48 -0
  18. adam/commands/audit/show_slow10.py +47 -0
  19. adam/commands/audit/show_top10.py +45 -0
  20. adam/commands/audit/utils_show_top10.py +59 -0
  21. adam/commands/bash/__init__.py +5 -0
  22. adam/commands/bash/bash.py +36 -0
  23. adam/commands/bash/bash_completer.py +93 -0
  24. adam/commands/bash/utils_bash.py +16 -0
  25. adam/commands/cat.py +50 -0
  26. adam/commands/cd.py +15 -91
  27. adam/commands/check.py +23 -18
  28. adam/commands/cli_commands.py +2 -3
  29. adam/commands/code.py +57 -0
  30. adam/commands/command.py +96 -40
  31. adam/commands/commands_utils.py +9 -19
  32. adam/commands/cp.py +33 -39
  33. adam/commands/cql/cql_completions.py +30 -8
  34. adam/commands/cql/cqlsh.py +12 -27
  35. adam/commands/cql/utils_cql.py +343 -0
  36. adam/commands/deploy/code_start.py +7 -10
  37. adam/commands/deploy/code_stop.py +4 -21
  38. adam/commands/deploy/code_utils.py +3 -3
  39. adam/commands/deploy/deploy.py +4 -21
  40. adam/commands/deploy/deploy_frontend.py +14 -17
  41. adam/commands/deploy/deploy_pg_agent.py +3 -6
  42. adam/commands/deploy/deploy_pod.py +67 -73
  43. adam/commands/deploy/deploy_utils.py +14 -24
  44. adam/commands/deploy/undeploy.py +4 -21
  45. adam/commands/deploy/undeploy_frontend.py +4 -7
  46. adam/commands/deploy/undeploy_pg_agent.py +6 -8
  47. adam/commands/deploy/undeploy_pod.py +11 -12
  48. adam/commands/devices/device.py +118 -0
  49. adam/commands/devices/device_app.py +173 -0
  50. adam/commands/devices/device_auit_log.py +49 -0
  51. adam/commands/devices/device_cass.py +185 -0
  52. adam/commands/devices/device_export.py +86 -0
  53. adam/commands/devices/device_postgres.py +144 -0
  54. adam/commands/devices/devices.py +25 -0
  55. adam/commands/exit.py +1 -4
  56. adam/commands/export/__init__.py +0 -0
  57. adam/commands/export/clean_up_all_export_sessions.py +37 -0
  58. adam/commands/export/clean_up_export_sessions.py +51 -0
  59. adam/commands/export/drop_export_database.py +55 -0
  60. adam/commands/export/drop_export_databases.py +43 -0
  61. adam/commands/export/export.py +53 -0
  62. adam/commands/export/export_databases.py +170 -0
  63. adam/commands/export/export_handlers.py +71 -0
  64. adam/commands/export/export_select.py +81 -0
  65. adam/commands/export/export_select_x.py +54 -0
  66. adam/commands/export/export_use.py +52 -0
  67. adam/commands/export/exporter.py +352 -0
  68. adam/commands/export/import_session.py +40 -0
  69. adam/commands/export/importer.py +67 -0
  70. adam/commands/export/importer_athena.py +80 -0
  71. adam/commands/export/importer_sqlite.py +47 -0
  72. adam/commands/export/show_column_counts.py +54 -0
  73. adam/commands/export/show_export_databases.py +36 -0
  74. adam/commands/export/show_export_session.py +48 -0
  75. adam/commands/export/show_export_sessions.py +44 -0
  76. adam/commands/export/utils_export.py +314 -0
  77. adam/commands/help.py +10 -6
  78. adam/commands/intermediate_command.py +49 -0
  79. adam/commands/issues.py +14 -40
  80. adam/commands/kubectl.py +38 -0
  81. adam/commands/login.py +28 -24
  82. adam/commands/logs.py +4 -6
  83. adam/commands/ls.py +11 -116
  84. adam/commands/medusa/medusa.py +4 -22
  85. adam/commands/medusa/medusa_backup.py +20 -24
  86. adam/commands/medusa/medusa_restore.py +30 -32
  87. adam/commands/medusa/medusa_show_backupjobs.py +16 -17
  88. adam/commands/medusa/medusa_show_restorejobs.py +12 -17
  89. adam/commands/nodetool.py +11 -17
  90. adam/commands/param_get.py +11 -12
  91. adam/commands/param_set.py +9 -10
  92. adam/commands/postgres/postgres.py +43 -36
  93. adam/commands/postgres/{postgres_session.py → postgres_context.py} +80 -46
  94. adam/commands/postgres/postgres_ls.py +4 -8
  95. adam/commands/postgres/postgres_preview.py +5 -9
  96. adam/commands/postgres/psql_completions.py +2 -2
  97. adam/commands/postgres/utils_postgres.py +66 -0
  98. adam/commands/preview_table.py +8 -61
  99. adam/commands/pwd.py +14 -44
  100. adam/commands/reaper/reaper.py +4 -24
  101. adam/commands/reaper/reaper_forward.py +48 -55
  102. adam/commands/reaper/reaper_forward_session.py +6 -0
  103. adam/commands/reaper/reaper_forward_stop.py +10 -16
  104. adam/commands/reaper/reaper_restart.py +7 -14
  105. adam/commands/reaper/reaper_run_abort.py +11 -30
  106. adam/commands/reaper/reaper_runs.py +42 -57
  107. adam/commands/reaper/reaper_runs_abort.py +29 -49
  108. adam/commands/reaper/reaper_schedule_activate.py +11 -30
  109. adam/commands/reaper/reaper_schedule_start.py +10 -29
  110. adam/commands/reaper/reaper_schedule_stop.py +10 -29
  111. adam/commands/reaper/reaper_schedules.py +4 -14
  112. adam/commands/reaper/reaper_status.py +8 -16
  113. adam/commands/reaper/utils_reaper.py +196 -0
  114. adam/commands/repair/repair.py +4 -22
  115. adam/commands/repair/repair_log.py +4 -7
  116. adam/commands/repair/repair_run.py +27 -29
  117. adam/commands/repair/repair_scan.py +31 -34
  118. adam/commands/repair/repair_stop.py +4 -7
  119. adam/commands/report.py +25 -21
  120. adam/commands/restart.py +25 -26
  121. adam/commands/rollout.py +19 -24
  122. adam/commands/shell.py +5 -4
  123. adam/commands/show/show.py +6 -19
  124. adam/commands/show/show_app_actions.py +26 -22
  125. adam/commands/show/show_app_id.py +8 -11
  126. adam/commands/show/show_app_queues.py +7 -10
  127. adam/commands/show/{show_repairs.py → show_cassandra_repairs.py} +8 -17
  128. adam/commands/show/show_cassandra_status.py +29 -33
  129. adam/commands/show/show_cassandra_version.py +4 -14
  130. adam/commands/show/show_commands.py +19 -21
  131. adam/commands/show/show_host.py +1 -1
  132. adam/commands/show/show_login.py +26 -24
  133. adam/commands/show/show_processes.py +16 -18
  134. adam/commands/show/show_storage.py +10 -20
  135. adam/commands/watch.py +26 -29
  136. adam/config.py +5 -14
  137. adam/embedded_params.py +1 -1
  138. adam/pod_exec_result.py +7 -1
  139. adam/repl.py +95 -131
  140. adam/repl_commands.py +48 -20
  141. adam/repl_state.py +270 -61
  142. adam/sql/sql_completer.py +105 -63
  143. adam/sql/sql_state_machine.py +618 -0
  144. adam/sql/term_completer.py +3 -0
  145. adam/sso/authn_ad.py +6 -5
  146. adam/sso/authn_okta.py +3 -3
  147. adam/sso/cred_cache.py +3 -2
  148. adam/sso/idp.py +3 -3
  149. adam/utils.py +439 -3
  150. adam/utils_app.py +98 -0
  151. adam/utils_athena.py +140 -87
  152. adam/utils_audits.py +106 -0
  153. adam/utils_issues.py +32 -0
  154. adam/utils_k8s/app_clusters.py +28 -0
  155. adam/utils_k8s/app_pods.py +33 -0
  156. adam/utils_k8s/cassandra_clusters.py +22 -20
  157. adam/utils_k8s/cassandra_nodes.py +4 -4
  158. adam/utils_k8s/custom_resources.py +5 -0
  159. adam/utils_k8s/ingresses.py +2 -2
  160. adam/utils_k8s/k8s.py +87 -0
  161. adam/utils_k8s/pods.py +77 -68
  162. adam/utils_k8s/secrets.py +4 -4
  163. adam/utils_k8s/service_accounts.py +5 -4
  164. adam/utils_k8s/services.py +2 -2
  165. adam/utils_k8s/statefulsets.py +1 -12
  166. adam/utils_net.py +4 -4
  167. adam/utils_repl/__init__.py +0 -0
  168. adam/utils_repl/automata_completer.py +48 -0
  169. adam/utils_repl/repl_completer.py +46 -0
  170. adam/utils_repl/state_machine.py +173 -0
  171. adam/utils_sqlite.py +109 -0
  172. adam/version.py +1 -1
  173. {kaqing-2.0.98.dist-info → kaqing-2.0.171.dist-info}/METADATA +1 -1
  174. kaqing-2.0.171.dist-info/RECORD +236 -0
  175. adam/commands/app.py +0 -67
  176. adam/commands/bash.py +0 -92
  177. adam/commands/cql/cql_table_completer.py +0 -8
  178. adam/commands/cql/cql_utils.py +0 -115
  179. adam/commands/describe/describe.py +0 -47
  180. adam/commands/describe/describe_keyspace.py +0 -60
  181. adam/commands/describe/describe_keyspaces.py +0 -49
  182. adam/commands/describe/describe_schema.py +0 -49
  183. adam/commands/describe/describe_table.py +0 -60
  184. adam/commands/describe/describe_tables.py +0 -49
  185. adam/commands/devices.py +0 -118
  186. adam/commands/postgres/postgres_utils.py +0 -31
  187. adam/commands/postgres/psql_table_completer.py +0 -11
  188. adam/commands/reaper/reaper_session.py +0 -159
  189. adam/sql/state_machine.py +0 -460
  190. kaqing-2.0.98.dist-info/RECORD +0 -191
  191. /adam/commands/{describe → devices}/__init__.py +0 -0
  192. {kaqing-2.0.98.dist-info → kaqing-2.0.171.dist-info}/WHEEL +0 -0
  193. {kaqing-2.0.98.dist-info → kaqing-2.0.171.dist-info}/entry_points.txt +0 -0
  194. {kaqing-2.0.98.dist-info → kaqing-2.0.171.dist-info}/top_level.txt +0 -0
adam/__init__.py CHANGED
@@ -1,3 +1 @@
1
- import click
2
-
3
1
  from .version import __version__, __release__
adam/app_session.py CHANGED
@@ -6,10 +6,11 @@ import requests
6
6
  from urllib.parse import urlparse
7
7
 
8
8
  from adam.log import Log
9
+ from adam.repl_state import ReplState
9
10
  from adam.sso.idp import Idp
10
11
  from adam.sso.idp_login import IdpLogin
11
12
  from adam.config import Config
12
- from adam.utils import json_to_csv, lines_to_tabular, log, log2
13
+ from adam.utils import debug, json_to_csv, lines_to_tabular, log, log2
13
14
  from adam.apps import Apps
14
15
 
15
16
  class AppLogin:
@@ -65,7 +66,8 @@ class AppSession:
65
66
  try:
66
67
  header, lines = json_to_csv(js, delimiter='\t')
67
68
  log(lines_to_tabular(lines, header=header, separator='\t'))
68
- except:
69
+ except Exception as e:
70
+ # traceback.print_exc(e)
69
71
  log(js)
70
72
  except:
71
73
  if urlparse(r.url).hostname != urlparse(uri).hostname and not retried:
@@ -76,7 +78,7 @@ class AppSession:
76
78
 
77
79
  if r.text:
78
80
  log2(f'{r.status_code} {r.url} Failed parsing the results.')
79
- Config().debug(r.text)
81
+ debug(r.text)
80
82
  else:
81
83
  log2(r.status_code)
82
84
  log2(r.text)
@@ -115,7 +117,7 @@ class AppSession:
115
117
  try:
116
118
  # oidc/login may hang
117
119
  timeout = Config().get('app.login.timeout', 5)
118
- Config().debug(f'-> {idp_login.app_login_url}')
120
+ debug(f'-> {idp_login.app_login_url}')
119
121
  session.post(idp_login.app_login_url, headers=headers, data=form_data, timeout=timeout)
120
122
  except Exception:
121
123
  pass
@@ -133,7 +135,7 @@ class AppSession:
133
135
  check_uri = Config().get('app.login.session-check-url', 'https://{host}/{env}/{app}/api/8/C3/userSessionToken')
134
136
  check_uri = check_uri.replace('{host}', self.host).replace('{env}', self.env).replace('{app}', 'c3')
135
137
  r = session.get(check_uri)
136
- Config().debug(f'{r.status_code} {check_uri}')
138
+ debug(f'{r.status_code} {check_uri}')
137
139
 
138
140
  res_text = r.text
139
141
  js = json.loads(res_text)
@@ -142,7 +144,7 @@ class AppSession:
142
144
  break
143
145
 
144
146
  app_access_token = js['signedToken']
145
- Config().debug(f'{r.text}')
147
+ debug(f'{r.text}')
146
148
 
147
149
  self.app_login = AppLogin(session, app_access_token, idp_uri)
148
150
  except Exception:
@@ -158,7 +160,7 @@ class AppSession:
158
160
  log2(f"Invalid username/password.")
159
161
  break
160
162
  finally:
161
- Config().debug(traceback.format_exc())
163
+ debug(traceback.format_exc())
162
164
 
163
165
  if 'res_text' in locals():
164
166
  Log.log_to_file(res_text)
adam/batch.py CHANGED
@@ -1,7 +1,7 @@
1
1
  import click
2
2
 
3
3
  from adam.commands.audit.audit import Audit, AuditCommandHelper
4
- from adam.commands.bash import Bash
4
+ from adam.commands.bash.bash import Bash
5
5
  from adam.commands.check import Check, CheckCommandHelper
6
6
  from adam.commands.cp import ClipboardCopy, CopyCommandHelper
7
7
  from adam.commands.command import Command
@@ -9,7 +9,6 @@ from adam.commands.command_helpers import ClusterCommandHelper, ClusterOrPodComm
9
9
  from adam.commands.cql.cqlsh import CqlCommandHelper, Cqlsh
10
10
  from adam.commands.deploy.deploy import Deploy, DeployCommandHelper
11
11
  from adam.commands.deploy.undeploy import Undeploy, UndeployCommandHelper
12
- from adam.commands.describe.describe import Describe, DescribeCommandHelper
13
12
  from adam.commands.issues import Issues
14
13
  from adam.commands.login import Login
15
14
  from adam.commands.logs import Logs
@@ -36,7 +35,7 @@ from adam.cli_group import cli
36
35
  @click.option('--param', '-v', multiple=True, metavar='<key>=<value>', help='parameter override')
37
36
  @click.argument('extra_args', nargs=-1, metavar='repair', type=click.UNPROCESSED)
38
37
  def audit(kubeconfig: str, config: str, param: list[str], extra_args):
39
- run_command(Audit(), kubeconfig, config, param, None, None, None, extra_args)
38
+ run_command(Audit(), kubeconfig, config, param, None, None, None, extra_args, device=ReplState.L)
40
39
 
41
40
 
42
41
  @cli.command(context_settings=dict(ignore_unknown_options=True, allow_extra_args=True), cls=ClusterOrPodCommandHelper, help='Run a single bash command.')
@@ -97,19 +96,6 @@ def deploy(kubeconfig: str, config: str, param: list[str], namespace: str, extra
97
96
  run_command(Deploy(), kubeconfig, config, param, None, namespace, None, extra_args)
98
97
 
99
98
 
100
- @cli.command(context_settings=dict(ignore_unknown_options=True, allow_extra_args=True), cls=DescribeCommandHelper, help='Describe keyspace(s) or table(s).')
101
- @click.option('--kubeconfig', '-k', required=False, metavar='path', help='path to kubeconfig file')
102
- @click.option('--config', default='params.yaml', metavar='path', help='path to kaqing parameters file')
103
- @click.option('--param', '-v', multiple=True, metavar='<key>=<value>', help='parameter override')
104
- @click.option('--cluster', '-c', required=False, metavar='statefulset', help='Kubernetes statefulset name')
105
- @click.option('--namespace', '-n', required=False, metavar='namespace', help='Kubernetes namespace')
106
- @click.option('--pod', '-p', required=False, metavar='pod', help='Kubernetes pod name')
107
- @click.option('--all-nodes', '-a', is_flag=True, help='execute on all Cassandra nodes')
108
- @click.argument('extra_args', nargs=-1, metavar='<cluster|pod>', type=click.UNPROCESSED)
109
- def describe(kubeconfig: str, config: str, param: list[str], cluster: str, namespace: str, pod: str, all_nodes: bool, extra_args):
110
- run_command(Describe(), kubeconfig, config, param, cluster, namespace, pod, extra_args + ('&',) if all_nodes else extra_args)
111
-
112
-
113
99
  @cli.command(context_settings=dict(ignore_unknown_options=True, allow_extra_args=True), cls=ClusterOrPodCommandHelper, help="Print Qing's issues.")
114
100
  @click.option('--kubeconfig', '-k', required=False, metavar='path', help='path to kubeconfig file')
115
101
  @click.option('--config', default='params.yaml', metavar='path', help='path to kaqing parameters file')
@@ -295,14 +281,14 @@ def watch(kubeconfig: str, config: str, param: list[str], cluster: str, namespac
295
281
  run_command(Watch(), kubeconfig, config, param, cluster, namespace, None, extra_args)
296
282
 
297
283
 
298
- def run_command(cmd: Command, kubeconfig: str, config: str, params: list[str], cluster:str, namespace: str, pod: str, extra_args):
284
+ def run_command(cmd: Command, kubeconfig: str, config: str, params: list[str], cluster:str, namespace: str, pod: str, extra_args, device=ReplState.C):
299
285
  is_user_entry = False
300
286
 
301
287
  KubeContext.init_config(kubeconfig, is_user_entry=is_user_entry)
302
288
  if not KubeContext.init_params(config, params, is_user_entry=is_user_entry):
303
289
  return
304
290
 
305
- state = ReplState(ns_sts=cluster, pod=pod, namespace=namespace)
291
+ state = ReplState(device=device, ns_sts=cluster, pod=pod, namespace=namespace)
306
292
  if cmd.command() == 'pg' and not extra_args:
307
293
  state, _ = state.apply_args(extra_args)
308
294
  state.device = ReplState.P
@@ -1,7 +1,3 @@
1
- from collections.abc import Callable
2
- from concurrent.futures import ThreadPoolExecutor, as_completed
3
- import time
4
-
5
1
  from adam.checks.check import Check
6
2
  from adam.checks.check_context import CheckContext
7
3
  from adam.checks.check_result import CheckResult
@@ -14,10 +10,9 @@ from adam.checks.memory import Memory
14
10
  from adam.checks.status import Status
15
11
  from adam.config import Config
16
12
  from adam.utils_k8s.cassandra_nodes import CassandraNodes
17
- from adam.utils_k8s.kube_context import KubeContext
18
13
  from adam.utils_k8s.secrets import Secrets
19
14
  from adam.utils_k8s.statefulsets import StatefulSets
20
- from adam.utils import elapsed_time, log2
15
+ from adam.utils import parallelize, log2
21
16
 
22
17
  def all_checks() -> list[Check]:
23
18
  return [CompactionStats(), Cpu(), Gossip(), Memory(), Disk(), Status()]
@@ -38,57 +33,30 @@ def checks_from_csv(check_str: str):
38
33
 
39
34
  return checks
40
35
 
41
- def run_checks(cluster: str = None, namespace: str = None, pod: str = None, checks: list[Check] = None, show_output=True):
36
+ def run_checks(cluster: str = None, namespace: str = None, pod: str = None, checks: list[Check] = None, show_out=True):
42
37
  if not checks:
43
38
  checks = all_checks()
44
39
 
45
- sss: list[tuple[str, str]] = StatefulSets.list_sts_name_and_ns()
46
-
47
- action = 'issues'
48
- crs: list[CheckResult] = []
49
-
50
- def on_clusters(f: Callable[[any, list[str]], any]):
51
- for ss, ns in sss:
52
- if (not cluster or cluster == ss) and (not namespace or namespace == ns):
53
- pods = StatefulSets.pods(ss, ns)
54
- for pod_name in [pod.metadata.name for pod in pods]:
55
- if not pod or pod == pod_name:
56
- f(ss, ns, pod_name, show_output)
57
-
58
- max_workers = Config().action_workers(action, 30)
59
- if max_workers < 2:
60
- def serial(ss, ns, pod_name, show_output):
61
- if not pod or pod == pod_name:
62
- crs.append(run_checks_on_pod(checks, ss[0], ns, pod_name, show_output))
63
-
64
- on_clusters(serial)
65
- else:
66
- if KubeContext.show_parallelism():
67
- log2(f'Executing on all nodes from statefulset with {max_workers} workers...')
68
- start_time = time.time()
69
- try:
70
- futures = []
71
- with ThreadPoolExecutor(max_workers=max_workers) as executor:
72
- def submit(ss, ns, pod_name, show_output):
73
- f = executor.submit(run_checks_on_pod, checks, ss, ns, pod_name, show_output,)
74
- if f: futures.append(f)
75
-
76
- on_clusters(submit)
40
+ sts_ns: list[tuple[str, str]] = StatefulSets.list_sts_name_and_ns()
77
41
 
78
- crs = [future.result() for future in as_completed(futures)]
79
- finally:
80
- if KubeContext.show_parallelism():
81
- log2(f"Parallel {action} elapsed time: {elapsed_time(start_time)} with {max_workers} workers")
42
+ sts_ns_pods: list[tuple[str, str, str]] = []
43
+ for sts, ns in sts_ns:
44
+ if (not cluster or cluster == sts) and (not namespace or namespace == ns):
45
+ pods = StatefulSets.pods(sts, ns)
46
+ for pod_name in [pod.metadata.name for pod in pods]:
47
+ if not pod or pod == pod_name:
48
+ sts_ns_pods.append((sts, ns, pod_name))
82
49
 
83
- return crs
50
+ with parallelize(sts_ns_pods, Config().action_workers('issues', 30), msg='d`Running|Ran checks on {size} pods') as exec:
51
+ return exec.map(lambda sts_ns_pod: run_checks_on_pod(checks, sts_ns_pod[0], sts_ns_pod[1], sts_ns_pod[2], show_out))
84
52
 
85
- def run_checks_on_pod(checks: list[Check], cluster: str = None, namespace: str = None, pod: str = None, show_output=True):
53
+ def run_checks_on_pod(checks: list[Check], cluster: str = None, namespace: str = None, pod: str = None, show_out=True):
86
54
  host_id = CassandraNodes.get_host_id(pod, namespace)
87
55
  user, pw = Secrets.get_user_pass(pod, namespace)
88
56
  results = {}
89
57
  issues: list[Issue] = []
90
58
  for c in checks:
91
- check_results = c.check(CheckContext(cluster, host_id, pod, namespace, user, pw, show_output=show_output))
59
+ check_results = c.check(CheckContext(cluster, host_id, pod, namespace, user, pw, show_output=show_out))
92
60
  if check_results.details:
93
61
  results = results | {check_results.name: check_results.details}
94
62
  if check_results.issues:
adam/checks/cpu.py CHANGED
@@ -7,6 +7,7 @@ from adam.checks.issue import Issue
7
7
  from adam.config import Config
8
8
  from adam.utils_k8s.cassandra_nodes import CassandraNodes
9
9
  from adam.utils_k8s.custom_resources import CustomResources
10
+ from adam.utils_k8s.pods import Pods
10
11
 
11
12
  class Cpu(Check):
12
13
  def name(self):
@@ -20,10 +21,15 @@ class Cpu(Check):
20
21
  'namespace': ctx.namespace,
21
22
  'statefulset': ctx.statefulset,
22
23
  'cpu': 'Unknown',
23
- 'idle': 'Unknown'
24
+ 'idle': 'Unknown',
25
+ 'limit': 'NA'
24
26
  }
25
27
 
26
28
  try:
29
+ container = Pods.get_container(ctx.namespace, ctx.pod, container_name='cassandra')
30
+ if container.resources.limits and "cpu" in container.resources.limits:
31
+ details['limit'] = container.resources.limits["cpu"]
32
+
27
33
  idle = 'Unknown'
28
34
  result = CassandraNodes.exec(ctx.pod, ctx.namespace, "mpstat 5 2 | grep Average | awk '{print $NF}'", show_out=ctx.show_output)
29
35
  lines = result.stdout.strip(' \r\n').split('\n')
@@ -0,0 +1,52 @@
1
+ from kubernetes.utils import parse_quantity
2
+
3
+ from adam.checks.check import Check
4
+ from adam.checks.check_context import CheckContext
5
+ from adam.checks.check_result import CheckResult
6
+ from adam.checks.issue import Issue
7
+ from adam.config import Config
8
+ from adam.utils_k8s.custom_resources import CustomResources
9
+ from adam.utils_k8s.pods import Pods
10
+
11
+ class CpuMetrics(Check):
12
+ def name(self):
13
+ return 'cpu-metrics'
14
+
15
+ def check(self, ctx: CheckContext) -> CheckResult:
16
+ issues: list[Issue] = []
17
+
18
+ details = {
19
+ 'name': ctx.pod,
20
+ 'namespace': ctx.namespace,
21
+ 'statefulset': ctx.statefulset,
22
+ 'cpu': 'Unknown',
23
+ 'limit': 'NA'
24
+ }
25
+
26
+ try:
27
+ container = Pods.get_container(ctx.namespace, ctx.pod, container_name='cassandra')
28
+ if container.resources.limits and "cpu" in container.resources.limits:
29
+ details['limit'] = container.resources.limits["cpu"]
30
+
31
+ metrics = CustomResources.get_metrics(ctx.namespace, ctx.pod, container_name='cassandra')
32
+ usage = 'Unknown'
33
+ if metrics:
34
+ usage = details['cpu'] = metrics["usage"]["cpu"]
35
+
36
+ cpu_threshold = Config().get('checks.cpu-threshold', 0.0)
37
+ if cpu_threshold != 0.0 and usage != "Unknown" and parse_quantity(usage) > cpu_threshold:
38
+ issues.append(Issue(
39
+ statefulset=ctx.statefulset,
40
+ namespace=ctx.namespace,
41
+ pod=ctx.pod,
42
+ category='cpu',
43
+ desc=f'CPU is too busy: {usage}',
44
+ suggestion=f"qing restart {ctx.pod}@{ctx.namespace}"
45
+ ))
46
+ except Exception as e:
47
+ issues.append(self.issue_from_err(sts_name=ctx.statefulset, ns=ctx.namespace, pod_name=ctx.pod, exception=e))
48
+
49
+ return CheckResult(self.name(), details, issues)
50
+
51
+ def help(self):
52
+ return f'{CpuMetrics().name()}: check cpu busy percentage with metrics'
adam/columns/columns.py CHANGED
@@ -1,6 +1,7 @@
1
1
  from adam.columns.column import Column
2
2
  from adam.columns.compactions import Compactions
3
3
  from adam.columns.cpu import Cpu
4
+ from adam.columns.cpu_metrics import CpuMetrics
4
5
  from adam.columns.dir_data import DataDir
5
6
  from adam.columns.dir_snapshots import SnapshotsDir
6
7
  from adam.columns.gossip import Gossip
@@ -23,7 +24,7 @@ class Columns:
23
24
  COLUMNS_BY_NAME = None
24
25
 
25
26
  def all_columns():
26
- return [Compactions(), Cpu(), DataDir(), SnapshotsDir(), Gossip(), HostId(), Memory(),
27
+ return [Compactions(), Cpu(), CpuMetrics(), DataDir(), SnapshotsDir(), Gossip(), HostId(), Memory(),
27
28
  NodeAddress(), NodeLoad(), NodeOwns(), NodeStatus(),NodeTokens(), PodName(), CassandraVolume(), RootVolume()]
28
29
 
29
30
  def columns_by_name():
@@ -38,6 +39,7 @@ class Columns:
38
39
  name = name.strip(' ')
39
40
  if not name in Columns.COLUMNS_BY_NAME:
40
41
  return None
42
+
41
43
  cols.append(Columns.COLUMNS_BY_NAME[name]())
42
44
 
43
45
  return cols
adam/columns/cpu.py CHANGED
@@ -1,3 +1,5 @@
1
+ from kubernetes.utils.quantity import parse_quantity
2
+
1
3
  from adam.checks.check_result import CheckResult
2
4
  from adam.checks.cpu import Cpu as CpuCheck
3
5
  from adam.columns.column import Column
@@ -14,4 +16,4 @@ class Cpu(Column):
14
16
  cpu = r.details[CpuCheck().name()]
15
17
  busy = 100.0 - float(cpu['idle'])
16
18
 
17
- return f'{round(busy)}%'
19
+ return f'{round(busy)}%/{parse_quantity(cpu["limit"]) * 100}%'
@@ -0,0 +1,22 @@
1
+ from kubernetes.utils.quantity import parse_quantity
2
+
3
+ from adam.checks.check_result import CheckResult
4
+ from adam.checks.cpu_metrics import CpuMetrics as CpuCheck
5
+ from adam.columns.column import Column
6
+
7
+ class CpuMetrics(Column):
8
+ def name(self):
9
+ return 'cpu-metrics'
10
+
11
+ def checks(self):
12
+ return [CpuCheck()]
13
+
14
+ def pod_value(self, check_results: list[CheckResult], pod_name: str):
15
+ r = self.result_by_pod(check_results, pod_name)
16
+ cpu = r.details[CpuCheck().name()]
17
+
18
+ cpu_decimal = parse_quantity(cpu['cpu'])
19
+ cpu_limit = parse_quantity(cpu['limit'])
20
+ business = cpu_decimal * 100 / cpu_limit
21
+
22
+ return f"{business:.2f}%({cpu_decimal}/{cpu_limit})"
adam/commands/__init__.py CHANGED
@@ -0,0 +1,15 @@
1
+ from adam.commands.command import ExtractAllOptionsHandler, ExtractOptionsHandler, ExtractTrailingOptionsHandler
2
+ from adam.repl_state import ReplState
3
+ from adam.utils_app import AppHandler
4
+
5
+ def app(state: ReplState) -> AppHandler:
6
+ return AppHandler(state)
7
+
8
+ def extract_options(args: list[str], options = None):
9
+ return ExtractOptionsHandler(args, options = options)
10
+
11
+ def extract_trailing_options(args: list[str], trailing = None):
12
+ return ExtractTrailingOptionsHandler(args, trailing = trailing)
13
+
14
+ def extract_all_options(args: list[str], trailing = None, options = None):
15
+ return ExtractAllOptionsHandler(args, trailing = trailing, options = options)
@@ -1,7 +1,7 @@
1
+ from adam.commands import extract_options
1
2
  from adam.commands.command import Command
2
- from adam.commands.cql.cql_utils import tables as get_tables, run_cql
3
+ from adam.commands.cql.utils_cql import cassandra, cassandra_tables as get_tables
3
4
  from adam.config import Config
4
- from adam.pod_exec_result import PodExecResult
5
5
  from adam.repl_state import ReplState, RequiredState
6
6
  from adam.utils import log2
7
7
 
@@ -27,66 +27,55 @@ class AlterTables(Command):
27
27
  if not(args := self.args(cmd)):
28
28
  return super().run(cmd, state)
29
29
 
30
- state, args = self.apply_state(args, state)
31
- if not self.validate_state(state):
32
- return state
33
-
34
- if not args:
35
- if state.in_repl:
36
- log2('Please enter gc grace in seconds. e.g. alter gc-grace-seconds 3600')
37
- else:
38
- log2('* gc grace second is missing.')
39
- log2()
40
- Command.display_help()
41
-
42
- return 'missing-arg'
43
-
44
- args, include_reaper = Command.extract_options(args, '--include-reaper')
45
- arg_str = ' '.join(args)
46
-
47
- # r: list[PodExecResult] = run_cql(state, 'describe tables', show_out=False, on_any=True)
48
- # if not r:
49
- # log2('No pod is available')
50
- # return 'no-pod'
51
-
52
- excludes = [e.strip(' \r\n') for e in Config().get(
53
- 'cql.alter-tables.excludes',
54
- 'system_auth,system_traces,reaper_db,system_distributed,system_views,system,system_schema,system_virtual_schema').split(',')]
55
- batching = Config().get('cql.alter-tables.batching', True)
56
- tables = get_tables(state, on_any=True)
57
- for k, v in tables.items():
58
- if k not in excludes or k == 'reaper_db' and include_reaper:
59
- if batching:
60
- # alter table <table_name> with GC_GRACE_SECONDS = <timeout>;
61
- cql = ';\n'.join([f'alter table {k}.{t} with {arg_str}' for t in v])
62
- try:
63
- run_cql(state, cql, [], show_out=Config().is_debug(), on_any=True)
64
- except Exception as e:
65
- log2(e)
66
- continue
67
- else:
68
- for t in v:
69
- try:
30
+ with self.validate(args, state) as (args, state):
31
+ with extract_options(args, '--include-reaper') as (args, include_reaper):
32
+ if not args:
33
+ if state.in_repl:
34
+ log2('Please enter gc grace in seconds. e.g. alter gc-grace-seconds 3600')
35
+ else:
36
+ log2('* gc grace second is missing.')
37
+ log2()
38
+ Command.display_help()
39
+
40
+ return 'missing-arg'
41
+
42
+ arg_str = ' '.join(args)
43
+
44
+ excludes = [e.strip(' \r\n') for e in Config().get(
45
+ 'cql.alter-tables.excludes',
46
+ 'system_auth,system_traces,reaper_db,system_distributed,system_views,system,system_schema,system_virtual_schema').split(',')]
47
+ batching = Config().get('cql.alter-tables.batching', True)
48
+ tables = get_tables(state, on_any=True)
49
+ for k, v in tables.items():
50
+ if k not in excludes or k == 'reaper_db' and include_reaper:
51
+ if batching:
70
52
  # alter table <table_name> with GC_GRACE_SECONDS = <timeout>;
71
- cql = f'alter table {k}.{t} with {arg_str}'
72
- run_cql(state, cql, [], show_out=Config().is_debug(), on_any=True)
73
- except Exception as e:
74
- log2(e)
75
- continue
76
-
77
- log2(f'{len(v)} tables altered in {k}.')
78
-
79
- # do not continue to cql route
80
- return state
81
-
82
- def completion(self, state: ReplState) -> dict[str, any]:
83
- if state.sts:
84
- ps = Config().get('cql.alter-tables.gc-grace-periods', '3600,86400,864000,7776000').split(',')
85
- return super().completion(state, {
86
- 'GC_GRACE_SECONDS': {'=': {p.strip(' \r\n'): {'--include-reaper': None} for p in ps}},
87
- })
88
-
53
+ cql = ';\n'.join([f'alter table {k}.{t} with {arg_str}' for t in v])
54
+ try:
55
+ with cassandra(state) as pods:
56
+ pods.cql(cql, show_out=Config().is_debug(), show_query=not Config().is_debug(), on_any=True)
57
+ except Exception as e:
58
+ log2(e)
59
+ continue
60
+ else:
61
+ for t in v:
62
+ try:
63
+ # alter table <table_name> with GC_GRACE_SECONDS = <timeout>;
64
+ cql = f'alter table {k}.{t} with {arg_str}'
65
+ with cassandra(state) as pods:
66
+ pods.cql(show_out=Config().is_debug(), show_query=not Config().is_debug(), on_any=True)
67
+ except Exception as e:
68
+ log2(e)
69
+ continue
70
+
71
+ log2(f'{len(v)} tables altered in {k}.')
72
+
73
+ # do not continue to cql route
74
+ return state
75
+
76
+ def completion(self, _: ReplState) -> dict[str, any]:
77
+ # auto completion is taken care of by sql completer
89
78
  return {}
90
79
 
91
80
  def help(self, _: ReplState) -> str:
92
- return f'{AlterTables.COMMAND} <param = value> [--include-reaper] \t alter on all tables'
81
+ return f'{AlterTables.COMMAND} <param = value> [--include-reaper] \t alter schema on all tables'
@@ -0,0 +1,38 @@
1
+ from adam.commands import app, extract_options
2
+ from adam.commands.command import Command
3
+ from adam.repl_state import ReplState, RequiredState
4
+
5
+ class App(Command):
6
+ COMMAND = 'app'
7
+
8
+ # the singleton pattern
9
+ def __new__(cls, *args, **kwargs):
10
+ if not hasattr(cls, 'instance'): cls.instance = super(App, cls).__new__(cls)
11
+
12
+ return cls.instance
13
+
14
+ def __init__(self, successor: Command=None):
15
+ super().__init__(successor)
16
+
17
+ def command(self):
18
+ return App.COMMAND
19
+
20
+ def required(self):
21
+ return RequiredState.APP_APP
22
+
23
+ def run(self, cmd: str, state: ReplState):
24
+ if not(args := self.args(cmd)):
25
+ return super().run(cmd, state)
26
+
27
+ with self.validate(args, state) as (args, state):
28
+ with extract_options(args, '--force') as (args, forced):
29
+ with app(state) as http:
30
+ http.post(args, forced=forced)
31
+
32
+ return state
33
+
34
+ def completion(self, state: ReplState):
35
+ return super().completion(state, {'--force': None})
36
+
37
+ def help(self, _: ReplState):
38
+ return f"<AppType>.<AppAction> <args> [--force]\t post app action; check with 'show app actions' command"
adam/commands/app_ping.py CHANGED
@@ -1,6 +1,6 @@
1
+ from adam.commands import app, extract_options
1
2
  from adam.commands.command import Command
2
3
  from adam.repl_state import ReplState, RequiredState
3
- from adam.app_session import AppSession
4
4
 
5
5
  class AppPing(Command):
6
6
  COMMAND = 'app ping'
@@ -18,27 +18,21 @@ class AppPing(Command):
18
18
  return AppPing.COMMAND
19
19
 
20
20
  def required(self):
21
- return RequiredState.CLUSTER_OR_POD
21
+ return RequiredState.APP_APP
22
22
 
23
23
  def run(self, cmd: str, state: ReplState):
24
24
  if not(args := self.args(cmd)):
25
25
  return super().run(cmd, state)
26
26
 
27
- state, args = self.apply_state(args, state)
28
- if not self.validate_state(state, app_required=RequiredState.APP_APP):
29
- return state
27
+ with self.validate(args, state) as (args, state):
28
+ with extract_options(args, '--force') as (args, forced):
29
+ with app(state) as http:
30
+ http.post(['Echo.echoStatic'], forced=forced)
30
31
 
31
- _, forced = Command.extract_options(args, '--force')
32
-
33
- AppSession.run(state.app_env, state.app_app, state.namespace, 'Echo', 'echoStatic', forced= forced)
34
-
35
- return state
32
+ return state
36
33
 
37
34
  def completion(self, state: ReplState):
38
- if state.app_app:
39
- return super().completion(state, {'--force': None})
40
-
41
- return {}
35
+ return super().completion(state, {'--force': None})
42
36
 
43
37
  def help(self, _: ReplState):
44
38
  return f"{AppPing.COMMAND} [--force]\t ping app server with Echo.echoStatic()"