kaqing 2.0.184__py3-none-any.whl → 2.0.227__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of kaqing might be problematic. Click here for more details.
- adam/app_session.py +1 -1
- adam/batch.py +15 -15
- adam/checks/compactionstats.py +2 -1
- adam/checks/cpu.py +2 -1
- adam/checks/disk.py +6 -5
- adam/checks/gossip.py +2 -1
- adam/checks/memory.py +2 -1
- adam/checks/status.py +2 -1
- adam/commands/app/app.py +4 -4
- adam/commands/app/app_ping.py +2 -2
- adam/commands/{login.py → app/login.py} +2 -2
- adam/commands/app/show_app_actions.py +3 -3
- adam/commands/app/show_app_id.py +2 -2
- adam/commands/app/show_app_queues.py +2 -2
- adam/commands/{show → app}/show_login.py +3 -3
- adam/commands/app/utils_app.py +9 -1
- adam/commands/audit/audit.py +8 -24
- adam/commands/audit/audit_repair_tables.py +3 -3
- adam/commands/audit/audit_run.py +3 -3
- adam/commands/audit/completions_l.py +15 -0
- adam/commands/audit/show_last10.py +2 -3
- adam/commands/audit/show_slow10.py +2 -2
- adam/commands/audit/show_top10.py +2 -2
- adam/commands/bash/bash.py +3 -3
- adam/commands/bash/utils_bash.py +1 -1
- adam/commands/cassandra/download_cassandra_log.py +45 -0
- adam/commands/cassandra/restart_cluster.py +47 -0
- adam/commands/cassandra/restart_node.py +51 -0
- adam/commands/cassandra/restart_nodes.py +47 -0
- adam/commands/{rollout.py → cassandra/rollout.py} +3 -3
- adam/commands/{show → cassandra}/show_cassandra_repairs.py +7 -5
- adam/commands/{show → cassandra}/show_cassandra_status.py +24 -17
- adam/commands/{show → cassandra}/show_cassandra_version.py +2 -2
- adam/commands/cassandra/show_processes.py +50 -0
- adam/commands/cassandra/show_storage.py +44 -0
- adam/commands/{watch.py → cassandra/watch.py} +2 -2
- adam/commands/cli/__init__.py +0 -0
- adam/commands/{cli_commands.py → cli/cli_commands.py} +6 -1
- adam/commands/{clipboard_copy.py → cli/clipboard_copy.py} +4 -4
- adam/commands/{show/show_commands.py → cli/show_cli_commands.py} +5 -5
- adam/commands/code.py +2 -2
- adam/commands/command.py +54 -14
- adam/commands/commands_utils.py +14 -6
- adam/commands/config/__init__.py +0 -0
- adam/commands/{param_get.py → config/param_get.py} +2 -2
- adam/commands/{param_set.py → config/param_set.py} +2 -2
- adam/commands/{show → config}/show_params.py +3 -3
- adam/commands/{alter_tables.py → cql/alter_tables.py} +3 -3
- adam/commands/cql/completions_c.py +29 -0
- adam/commands/cql/cqlsh.py +4 -8
- adam/commands/cql/utils_cql.py +36 -17
- adam/commands/debug/__init__.py +0 -0
- adam/commands/debug/debug.py +22 -0
- adam/commands/debug/debug_completes.py +35 -0
- adam/commands/debug/debug_timings.py +35 -0
- adam/commands/debug/show_offloaded_completes.py +45 -0
- adam/commands/deploy/code_start.py +2 -2
- adam/commands/deploy/code_stop.py +2 -2
- adam/commands/deploy/deploy_frontend.py +2 -2
- adam/commands/deploy/deploy_pg_agent.py +2 -2
- adam/commands/deploy/deploy_pod.py +2 -2
- adam/commands/deploy/undeploy_frontend.py +2 -2
- adam/commands/deploy/undeploy_pg_agent.py +2 -2
- adam/commands/deploy/undeploy_pod.py +2 -2
- adam/commands/devices/device.py +37 -11
- adam/commands/devices/device_app.py +7 -7
- adam/commands/devices/device_auit_log.py +2 -2
- adam/commands/devices/device_cass.py +6 -6
- adam/commands/devices/device_export.py +7 -4
- adam/commands/devices/device_postgres.py +19 -9
- adam/commands/devices/devices.py +1 -1
- adam/commands/diag/__init__.py +0 -0
- adam/commands/{check.py → diag/check.py} +3 -3
- adam/commands/diag/generate_report.py +52 -0
- adam/commands/{issues.py → diag/issues.py} +3 -2
- adam/commands/exit.py +2 -2
- adam/commands/export/clean_up_all_export_sessions.py +2 -2
- adam/commands/export/clean_up_export_sessions.py +2 -2
- adam/commands/export/completions_x.py +11 -0
- adam/commands/export/download_export_session.py +5 -5
- adam/commands/export/drop_export_database.py +2 -2
- adam/commands/export/drop_export_databases.py +2 -2
- adam/commands/export/export.py +3 -19
- adam/commands/export/export_databases.py +20 -11
- adam/commands/export/export_select.py +9 -34
- adam/commands/export/export_sessions.py +13 -11
- adam/commands/export/export_use.py +6 -6
- adam/commands/export/export_x_select.py +48 -0
- adam/commands/export/exporter.py +140 -53
- adam/commands/export/import_files.py +3 -7
- adam/commands/export/import_session.py +2 -6
- adam/commands/export/importer.py +12 -13
- adam/commands/export/importer_athena.py +15 -35
- adam/commands/export/importer_sqlite.py +19 -8
- adam/commands/export/show_column_counts.py +11 -12
- adam/commands/export/show_export_databases.py +4 -4
- adam/commands/export/show_export_session.py +5 -5
- adam/commands/export/show_export_sessions.py +4 -4
- adam/commands/export/utils_export.py +40 -25
- adam/commands/fs/__init__.py +0 -0
- adam/commands/{cat.py → fs/cat.py} +4 -4
- adam/commands/fs/cat_local.py +42 -0
- adam/commands/{cd.py → fs/cd.py} +4 -4
- adam/commands/{download_file.py → fs/download_file.py} +7 -7
- adam/commands/{find_files.py → fs/find_files.py} +7 -7
- adam/commands/{find_processes.py → fs/find_processes.py} +14 -22
- adam/commands/{head.py → fs/head.py} +5 -5
- adam/commands/fs/head_local.py +46 -0
- adam/commands/{ls.py → fs/ls.py} +4 -4
- adam/commands/fs/ls_local.py +40 -0
- adam/commands/{pwd.py → fs/pwd.py} +2 -2
- adam/commands/fs/rm.py +18 -0
- adam/commands/fs/rm_downloads.py +39 -0
- adam/commands/fs/rm_logs.py +44 -0
- adam/commands/fs/rm_logs_local.py +38 -0
- adam/commands/{shell.py → fs/shell.py} +2 -2
- adam/commands/{show → fs}/show_adam.py +3 -3
- adam/commands/{show → fs}/show_host.py +2 -2
- adam/commands/fs/show_last_results.py +39 -0
- adam/commands/fs/tail.py +36 -0
- adam/commands/fs/tail_local.py +46 -0
- adam/commands/fs/utils_fs.py +192 -0
- adam/commands/help.py +2 -2
- adam/commands/intermediate_command.py +3 -0
- adam/commands/kubectl.py +2 -2
- adam/commands/medusa/medusa_backup.py +2 -2
- adam/commands/medusa/medusa_restore.py +4 -18
- adam/commands/medusa/medusa_show_backupjobs.py +2 -2
- adam/commands/medusa/medusa_show_restorejobs.py +2 -2
- adam/commands/medusa/utils_medusa.py +15 -0
- adam/commands/nodetool/__init__.py +0 -0
- adam/commands/nodetool/nodetool.py +87 -0
- adam/commands/nodetool/utils_nodetool.py +44 -0
- adam/commands/postgres/completions_p.py +22 -0
- adam/commands/postgres/postgres.py +10 -20
- adam/commands/postgres/postgres_databases.py +3 -3
- adam/commands/postgres/postgres_ls.py +3 -3
- adam/commands/postgres/postgres_preview.py +2 -2
- adam/commands/postgres/utils_postgres.py +12 -2
- adam/commands/preview_table.py +3 -4
- adam/commands/reaper/reaper_forward.py +2 -2
- adam/commands/reaper/reaper_forward_stop.py +2 -2
- adam/commands/reaper/reaper_restart.py +2 -2
- adam/commands/reaper/reaper_run_abort.py +2 -2
- adam/commands/reaper/reaper_runs.py +14 -12
- adam/commands/reaper/reaper_runs_abort.py +2 -2
- adam/commands/reaper/reaper_schedule_activate.py +8 -4
- adam/commands/reaper/reaper_schedule_start.py +3 -4
- adam/commands/reaper/reaper_schedule_stop.py +3 -4
- adam/commands/reaper/reaper_schedules.py +2 -2
- adam/commands/reaper/reaper_status.py +2 -2
- adam/commands/reaper/utils_reaper.py +41 -6
- adam/commands/repair/repair_log.py +2 -2
- adam/commands/repair/repair_run.py +2 -2
- adam/commands/repair/repair_scan.py +2 -4
- adam/commands/repair/repair_stop.py +2 -3
- adam/commands/{show/show.py → show.py} +12 -11
- adam/config.py +4 -5
- adam/embedded_params.py +1 -1
- adam/repl.py +24 -10
- adam/repl_commands.py +68 -45
- adam/repl_session.py +16 -1
- adam/repl_state.py +16 -1
- adam/sql/async_executor.py +62 -0
- adam/sql/lark_completer.py +286 -0
- adam/sql/lark_parser.py +604 -0
- adam/sql/qingl.lark +1075 -0
- adam/sso/cred_cache.py +2 -5
- adam/utils.py +259 -82
- adam/utils_async_job.py +73 -0
- adam/utils_k8s/app_clusters.py +11 -4
- adam/utils_k8s/app_pods.py +10 -5
- adam/utils_k8s/cassandra_clusters.py +19 -7
- adam/utils_k8s/cassandra_nodes.py +16 -6
- adam/utils_k8s/k8s.py +9 -0
- adam/utils_k8s/kube_context.py +1 -4
- adam/{pod_exec_result.py → utils_k8s/pod_exec_result.py} +8 -2
- adam/utils_k8s/pods.py +189 -29
- adam/utils_k8s/statefulsets.py +5 -2
- adam/utils_local.py +78 -2
- adam/utils_repl/appendable_completer.py +6 -0
- adam/utils_repl/repl_completer.py +51 -4
- adam/utils_sqlite.py +3 -8
- adam/version.py +1 -1
- {kaqing-2.0.184.dist-info → kaqing-2.0.227.dist-info}/METADATA +1 -1
- kaqing-2.0.227.dist-info/RECORD +280 -0
- kaqing-2.0.227.dist-info/top_level.txt +2 -0
- teddy/__init__.py +0 -0
- teddy/lark_parser.py +436 -0
- teddy/lark_parser2.py +618 -0
- adam/commands/cql/cql_completions.py +0 -32
- adam/commands/export/export_select_x.py +0 -54
- adam/commands/logs.py +0 -37
- adam/commands/nodetool.py +0 -69
- adam/commands/postgres/psql_completions.py +0 -11
- adam/commands/report.py +0 -61
- adam/commands/restart.py +0 -60
- adam/commands/show/show_processes.py +0 -49
- adam/commands/show/show_storage.py +0 -42
- kaqing-2.0.184.dist-info/RECORD +0 -244
- kaqing-2.0.184.dist-info/top_level.txt +0 -1
- /adam/commands/{show → cassandra}/__init__.py +0 -0
- /adam/commands/{nodetool_commands.py → nodetool/nodetool_commands.py} +0 -0
- {kaqing-2.0.184.dist-info → kaqing-2.0.227.dist-info}/WHEEL +0 -0
- {kaqing-2.0.184.dist-info → kaqing-2.0.227.dist-info}/entry_points.txt +0 -0
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
from adam.commands.fs.utils_fs import show_last_pod_results, show_last_results
|
|
2
|
+
from adam.commands.command import Command
|
|
3
|
+
from adam.repl_state import ReplState
|
|
4
|
+
from adam.utils import log_to_pods
|
|
5
|
+
|
|
6
|
+
class ShowLastResults(Command):
|
|
7
|
+
COMMAND = 'show last results'
|
|
8
|
+
|
|
9
|
+
# the singleton pattern
|
|
10
|
+
def __new__(cls, *args, **kwargs):
|
|
11
|
+
if not hasattr(cls, 'instance'): cls.instance = super(ShowLastResults, cls).__new__(cls)
|
|
12
|
+
|
|
13
|
+
return cls.instance
|
|
14
|
+
|
|
15
|
+
def __init__(self, successor: Command=None):
|
|
16
|
+
super().__init__(successor)
|
|
17
|
+
|
|
18
|
+
def command(self):
|
|
19
|
+
return ShowLastResults.COMMAND
|
|
20
|
+
|
|
21
|
+
def aliases(self):
|
|
22
|
+
return [':?']
|
|
23
|
+
|
|
24
|
+
def run(self, cmd: str, state: ReplState):
|
|
25
|
+
if not self.args(cmd):
|
|
26
|
+
return super().run(cmd, state)
|
|
27
|
+
|
|
28
|
+
if log_to_pods():
|
|
29
|
+
show_last_pod_results(state)
|
|
30
|
+
else:
|
|
31
|
+
show_last_results(state)
|
|
32
|
+
|
|
33
|
+
return state
|
|
34
|
+
|
|
35
|
+
def completion(self, state: ReplState):
|
|
36
|
+
return super().completion(state)
|
|
37
|
+
|
|
38
|
+
def help(self, state: ReplState):
|
|
39
|
+
return super().help(state, 'show results of last command')
|
adam/commands/fs/tail.py
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
from adam.commands import validate_args
|
|
2
|
+
from adam.commands.command import Command
|
|
3
|
+
from adam.commands.devices.devices import Devices
|
|
4
|
+
from adam.repl_state import ReplState, RequiredState
|
|
5
|
+
|
|
6
|
+
class Tail(Command):
|
|
7
|
+
COMMAND = 'tail'
|
|
8
|
+
|
|
9
|
+
# the singleton pattern
|
|
10
|
+
def __new__(cls, *args, **kwargs):
|
|
11
|
+
if not hasattr(cls, 'instance'): cls.instance = super(Tail, cls).__new__(cls)
|
|
12
|
+
|
|
13
|
+
return cls.instance
|
|
14
|
+
|
|
15
|
+
def __init__(self, successor: Command=None):
|
|
16
|
+
super().__init__(successor)
|
|
17
|
+
|
|
18
|
+
def command(self):
|
|
19
|
+
return Tail.COMMAND
|
|
20
|
+
|
|
21
|
+
def required(self):
|
|
22
|
+
return [RequiredState.CLUSTER_OR_POD, RequiredState.APP_APP, ReplState.P]
|
|
23
|
+
|
|
24
|
+
def run(self, cmd: str, state: ReplState):
|
|
25
|
+
if not(args := self.args(cmd)):
|
|
26
|
+
return super().run(cmd, state)
|
|
27
|
+
|
|
28
|
+
with self.validate(args, state) as (args, state):
|
|
29
|
+
with validate_args(args, state, name='file') as args:
|
|
30
|
+
return Devices.of(state).bash(state, state, ['tail', '-n', '10', args])
|
|
31
|
+
|
|
32
|
+
def completion(self, state: ReplState):
|
|
33
|
+
return super().completion(state, lambda: {f: None for f in Devices.of(state).files(state)}, pods=Devices.of(state).pods(state, '-'), auto='jit')
|
|
34
|
+
|
|
35
|
+
def help(self, state: ReplState):
|
|
36
|
+
return super().help(state, 'run tail command on the pod', args='<file>')
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import os
|
|
2
|
+
|
|
3
|
+
from adam.commands import validate_args
|
|
4
|
+
from adam.commands.command import Command
|
|
5
|
+
from adam.repl_state import ReplState, RequiredState
|
|
6
|
+
from adam.utils import log2
|
|
7
|
+
from adam.utils_local import find_local_files
|
|
8
|
+
|
|
9
|
+
class TailLocal(Command):
|
|
10
|
+
COMMAND = ':tail'
|
|
11
|
+
|
|
12
|
+
# the singleton pattern
|
|
13
|
+
def __new__(cls, *args, **kwargs):
|
|
14
|
+
if not hasattr(cls, 'instance'): cls.instance = super(TailLocal, cls).__new__(cls)
|
|
15
|
+
|
|
16
|
+
return cls.instance
|
|
17
|
+
|
|
18
|
+
def __init__(self, successor: Command=None):
|
|
19
|
+
super().__init__(successor)
|
|
20
|
+
|
|
21
|
+
def command(self):
|
|
22
|
+
return TailLocal.COMMAND
|
|
23
|
+
|
|
24
|
+
def required(self):
|
|
25
|
+
return [RequiredState.CLUSTER_OR_POD, RequiredState.APP_APP, ReplState.P]
|
|
26
|
+
|
|
27
|
+
def run(self, cmd: str, state: ReplState):
|
|
28
|
+
if not(args := self.args(cmd)):
|
|
29
|
+
return super().run(cmd, state)
|
|
30
|
+
|
|
31
|
+
with self.validate(args, state) as (args, state):
|
|
32
|
+
with validate_args(args, state, name='file') as args:
|
|
33
|
+
cmd = f'tail -n 10 {args}'
|
|
34
|
+
log2(cmd)
|
|
35
|
+
log2()
|
|
36
|
+
|
|
37
|
+
os.system(cmd)
|
|
38
|
+
log2()
|
|
39
|
+
|
|
40
|
+
return state
|
|
41
|
+
|
|
42
|
+
def completion(self, state: ReplState):
|
|
43
|
+
return super().completion(state, lambda: {n: None for n in find_local_files(file_type='f', max_depth=1)}, auto='jit')
|
|
44
|
+
|
|
45
|
+
def help(self, state: ReplState):
|
|
46
|
+
return super().help(state, 'run tail command on local file system', args='<file>')
|
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
import itertools
|
|
2
|
+
import os
|
|
3
|
+
import re
|
|
4
|
+
|
|
5
|
+
from adam.commands.devices.devices import Devices
|
|
6
|
+
from adam.config import Config
|
|
7
|
+
from adam.repl_state import ReplState
|
|
8
|
+
from adam.utils import Color, PodLogFile, log2, log_dir, log_to_pods, pod_log_dir, tabulize
|
|
9
|
+
from adam.utils_k8s.pod_exec_result import PodExecResult
|
|
10
|
+
from adam.utils_k8s.pods import Pods
|
|
11
|
+
from adam.utils_local import find_local_files
|
|
12
|
+
from adam.utils_async_job import AsyncJobs
|
|
13
|
+
|
|
14
|
+
def show_last_results(state: ReplState):
|
|
15
|
+
last_id, last_command = AsyncJobs.last_command()
|
|
16
|
+
logs: list[str] = find_local_files(f'{log_dir()}/{last_id}*')
|
|
17
|
+
|
|
18
|
+
# /tmp/qing-db/q/logs/16145959.repair-0.err
|
|
19
|
+
# /tmp/qing-db/q/logs/16145959.repair-0.log
|
|
20
|
+
|
|
21
|
+
logs_by_n: dict[str, LogLine] = {}
|
|
22
|
+
for l in logs:
|
|
23
|
+
size = str(os.path.getsize(l))
|
|
24
|
+
n = l[len(f'{log_dir()}/{last_id}'):]
|
|
25
|
+
if n.startswith('.'):
|
|
26
|
+
n = n[1:]
|
|
27
|
+
|
|
28
|
+
if n.endswith('.log'):
|
|
29
|
+
n = n[:-4]
|
|
30
|
+
key = 'out'
|
|
31
|
+
else:
|
|
32
|
+
n = n[:-4]
|
|
33
|
+
key = 'err'
|
|
34
|
+
|
|
35
|
+
n = n.split('-')[-1]
|
|
36
|
+
|
|
37
|
+
if n not in logs_by_n:
|
|
38
|
+
logs_by_n[n] = LogLine(n, file=l.replace('.log', '.err'))
|
|
39
|
+
|
|
40
|
+
if key == 'out':
|
|
41
|
+
logs_by_n[n].out = size
|
|
42
|
+
else:
|
|
43
|
+
logs_by_n[n].err = size
|
|
44
|
+
|
|
45
|
+
if last_command:
|
|
46
|
+
keywords = last_command.strip(' &').split(' ')
|
|
47
|
+
if keywords and keywords[0] == 'nodetool':
|
|
48
|
+
# nodetool -u cs-a7b13e29bd-superuser -pw lDed6uXQAQP72kHOYuML repair &
|
|
49
|
+
keywords = keywords[-1:]
|
|
50
|
+
|
|
51
|
+
for ps in find_pids_for_pod(state.pod, Devices.of(state).default_container(state), state.namespace, keywords, match_last_arg=True):
|
|
52
|
+
n = ps.pod.split('-')[-1]
|
|
53
|
+
if n in logs_by_n:
|
|
54
|
+
logs_by_n[n].merge(ps)
|
|
55
|
+
|
|
56
|
+
log2(f'[{last_id}] {last_command}')
|
|
57
|
+
log2()
|
|
58
|
+
tabulize(sorted(logs_by_n.keys()), fn=lambda n: logs_by_n[n].table_line(), header=LogLine.header, separator='\t')
|
|
59
|
+
|
|
60
|
+
def show_last_pod_results(state: ReplState):
|
|
61
|
+
last_id, last_command = AsyncJobs.last_command()
|
|
62
|
+
|
|
63
|
+
container = Devices.of(state).default_container(state)
|
|
64
|
+
|
|
65
|
+
if last_command:
|
|
66
|
+
keywords = last_command.strip(' &').split(' ')
|
|
67
|
+
if keywords and keywords[0] == 'nodetool':
|
|
68
|
+
# nodetool -u cs-a7b13e29bd-superuser -pw lDed6uXQAQP72kHOYuML repair &
|
|
69
|
+
keywords = keywords[-1:]
|
|
70
|
+
|
|
71
|
+
action = 'find-files'
|
|
72
|
+
msg = 'd`Running|Ran ' + action + ' onto {size} pods'
|
|
73
|
+
pods = Devices.of(state).pod_names(state)
|
|
74
|
+
with Pods.parallelize(pods, len(pods), msg=msg, action=action) as exec:
|
|
75
|
+
results: list[LogLine] = exec.map(lambda pod: log_line_for_pod(pod, container, state.namespace, pod_log_dir(), last_id, log_to_pods(), keywords))
|
|
76
|
+
|
|
77
|
+
log2(f'[{last_id}] {last_command}')
|
|
78
|
+
log2()
|
|
79
|
+
tabulize(sorted(results, key=lambda l: l.ordinal), fn=lambda l: l.table_line(), header=LogLine.header, separator='\t')
|
|
80
|
+
|
|
81
|
+
def log_line_for_pod(pod: str, container: str, namespace: str, dir: str, last_id: str, remote: bool, keywords: list[str]):
|
|
82
|
+
logs: list[PodLogFile] = Pods.find_files(pod, container, namespace, f'{dir}/{last_id}*', remote=remote)
|
|
83
|
+
|
|
84
|
+
line = LogLine()
|
|
85
|
+
|
|
86
|
+
procs = []
|
|
87
|
+
if keywords:
|
|
88
|
+
procs = find_pids_for_pod(pod, container, namespace, keywords)
|
|
89
|
+
|
|
90
|
+
for log in logs:
|
|
91
|
+
l = str(log)
|
|
92
|
+
if l.endswith('.log'):
|
|
93
|
+
key = 'out'
|
|
94
|
+
else:
|
|
95
|
+
key = 'err'
|
|
96
|
+
|
|
97
|
+
line.ordinal = log.pod.split('-')[-1]
|
|
98
|
+
line.file = l.replace('.log', '.err')
|
|
99
|
+
|
|
100
|
+
if key == 'out':
|
|
101
|
+
line.out = log.size
|
|
102
|
+
else:
|
|
103
|
+
line.err = log.size
|
|
104
|
+
for proc in procs:
|
|
105
|
+
line.merge(proc)
|
|
106
|
+
|
|
107
|
+
return line
|
|
108
|
+
|
|
109
|
+
def find_pids_for_cluster(state: ReplState, keywords: list[str], match_last_arg = False) -> list['ProcessInfo']:
|
|
110
|
+
container = Devices.of(state).default_container(state)
|
|
111
|
+
|
|
112
|
+
action = 'find-procs'
|
|
113
|
+
msg = 'd`Running|Ran ' + action + ' onto {size} pods'
|
|
114
|
+
pods = Devices.of(state).pod_names(state)
|
|
115
|
+
with Pods.parallelize(pods, len(pods), msg=msg, action=action) as exec:
|
|
116
|
+
r: list[list[ProcessInfo]] = exec.map(lambda pod: find_pids_for_pod(pod, container, state.namespace, keywords, match_last_arg=match_last_arg))
|
|
117
|
+
|
|
118
|
+
return list(itertools.chain.from_iterable(r))
|
|
119
|
+
|
|
120
|
+
def find_pids_for_pod(pod: str, container: str, namespace: str, keywords: list[str], match_last_arg = False) -> list['ProcessInfo']:
|
|
121
|
+
r: PodExecResult = Pods.exec(pod, container, namespace, _find_procs_command(keywords), text_color=Color.gray)
|
|
122
|
+
|
|
123
|
+
return ProcessInfo.from_find_process_results(r, last_arg = keywords[-1] if match_last_arg else None)
|
|
124
|
+
|
|
125
|
+
def _find_procs_command(keywords: list[str]):
|
|
126
|
+
regex_pattern = re.compile(r'[^\w\s]')
|
|
127
|
+
|
|
128
|
+
greps = []
|
|
129
|
+
for a in keywords:
|
|
130
|
+
a = a.strip('"\'').strip(' ')
|
|
131
|
+
|
|
132
|
+
if a and not regex_pattern.search(a):
|
|
133
|
+
greps.append(f'grep -- {a}')
|
|
134
|
+
|
|
135
|
+
awk = "awk '{ print $1, $2, $8, $NF }'"
|
|
136
|
+
|
|
137
|
+
return f"ps -ef | grep -v grep | {' | '.join(greps)} | {awk}"
|
|
138
|
+
|
|
139
|
+
class ProcessInfo:
|
|
140
|
+
header = 'POD\tUSER\tPID\tCMD\tLAST_ARG'
|
|
141
|
+
|
|
142
|
+
def __init__(self, user: str, pid: str, cmd: str, last_arg: str, pod: str = None):
|
|
143
|
+
self.user = user
|
|
144
|
+
self.pid = pid
|
|
145
|
+
self.cmd = cmd
|
|
146
|
+
self.last_arg = last_arg
|
|
147
|
+
self.pod = pod
|
|
148
|
+
|
|
149
|
+
def from_find_process_results(rs: PodExecResult, last_arg: str = None):
|
|
150
|
+
processes: list[ProcessInfo] = []
|
|
151
|
+
|
|
152
|
+
for l in rs.stdout.split('\n'):
|
|
153
|
+
l = l.strip(' \t\r\n')
|
|
154
|
+
if not l:
|
|
155
|
+
continue
|
|
156
|
+
|
|
157
|
+
tokens = l.split(' ')
|
|
158
|
+
if last_arg and tokens[3] != last_arg:
|
|
159
|
+
continue
|
|
160
|
+
|
|
161
|
+
processes.append(ProcessInfo(tokens[0], tokens[1], tokens[2], tokens[3], pod=rs.pod))
|
|
162
|
+
|
|
163
|
+
return processes
|
|
164
|
+
|
|
165
|
+
def table_line(self):
|
|
166
|
+
return '\t'.join([self.pod, self.user, self.pid, self.cmd, self.last_arg])
|
|
167
|
+
|
|
168
|
+
def tabulize(processes: list['ProcessInfo']):
|
|
169
|
+
tabulize(processes, lambda p: p.table_line(), header = ProcessInfo.header, separator='\t')
|
|
170
|
+
|
|
171
|
+
class LogLine(ProcessInfo):
|
|
172
|
+
header='ORDINAL\tPID\tCMD\tLAST_ARG\tOUT_SIZE\tERR_SIZE\tLOG(ERR)_FILES'
|
|
173
|
+
|
|
174
|
+
def __init__(self, ordinal: str = '-', out: str = '-', err: str = '-', file: str = '-', user: str = '-', pid: str = '-', cmd: str = '-', last_arg: str = '-', pod: str = '-'):
|
|
175
|
+
super().__init__(user, pid, cmd, last_arg, pod)
|
|
176
|
+
self.ordinal = ordinal
|
|
177
|
+
self.out = out
|
|
178
|
+
self.err = err
|
|
179
|
+
self.file = file
|
|
180
|
+
|
|
181
|
+
def __repr__(self):
|
|
182
|
+
return f"LogLine({', '.join([self.ordinal, self.pid, self.cmd, self.last_arg, self.out, self.err, self.file])})"
|
|
183
|
+
|
|
184
|
+
def table_line(self):
|
|
185
|
+
return '\t'.join([self.ordinal, self.pid, self.cmd, self.last_arg, self.out, self.err, self.file if self.err not in ['-', '0'] else self.file.replace('.err', '.log')])
|
|
186
|
+
|
|
187
|
+
def merge(self, process: ProcessInfo):
|
|
188
|
+
self.user = process.user
|
|
189
|
+
self.pid = process.pid
|
|
190
|
+
self.cmd = process.cmd
|
|
191
|
+
self.last_arg = process.last_arg
|
|
192
|
+
self.pod = process.pod
|
adam/commands/help.py
CHANGED
|
@@ -28,7 +28,7 @@ class Help(Command):
|
|
|
28
28
|
|
|
29
29
|
lines = []
|
|
30
30
|
lines.append('NAVIGATION')
|
|
31
|
-
lines.append(' a: | c: | l: | p: | x:\
|
|
31
|
+
lines.append(' a: | c: | l: | p: | x:\tswitch to another operational device: App, Cassandra, Audit, Postgres or Export')
|
|
32
32
|
lines.extend(section(ReplCommands.navigation()))
|
|
33
33
|
lines.append('CASSANDRA')
|
|
34
34
|
lines.extend(section(ReplCommands.cassandra_ops()))
|
|
@@ -52,5 +52,5 @@ class Help(Command):
|
|
|
52
52
|
def completion(self, _: ReplState):
|
|
53
53
|
return {Help.COMMAND: None}
|
|
54
54
|
|
|
55
|
-
def help(self,
|
|
55
|
+
def help(self, state: ReplState):
|
|
56
56
|
return None
|
adam/commands/kubectl.py
CHANGED
|
@@ -34,5 +34,5 @@ class Kubectl(Command):
|
|
|
34
34
|
return super().completion(state)
|
|
35
35
|
|
|
36
36
|
|
|
37
|
-
def help(self,
|
|
38
|
-
return
|
|
37
|
+
def help(self, state: ReplState):
|
|
38
|
+
return super().help(state, 'run a kubectl command', args='[kubectl-args]')
|
|
@@ -50,5 +50,5 @@ class MedusaBackup(Command):
|
|
|
50
50
|
def completion(self, state: ReplState):
|
|
51
51
|
return super().completion(state)
|
|
52
52
|
|
|
53
|
-
def help(self,
|
|
54
|
-
return
|
|
53
|
+
def help(self, state: ReplState):
|
|
54
|
+
return super().help(state, 'start a backup job')
|
|
@@ -1,13 +1,12 @@
|
|
|
1
|
-
from collections.abc import Callable
|
|
2
1
|
from datetime import datetime
|
|
3
2
|
from functools import partial
|
|
4
3
|
|
|
5
4
|
from adam.commands import validate_args
|
|
6
5
|
from adam.commands.command import Command, InvalidArgumentsException
|
|
6
|
+
from adam.commands.medusa.utils_medusa import medusa_backup_names
|
|
7
7
|
from adam.utils_k8s.statefulsets import StatefulSets
|
|
8
8
|
from adam.repl_state import ReplState, RequiredState
|
|
9
9
|
from adam.utils_k8s.custom_resources import CustomResources
|
|
10
|
-
from adam.config import Config
|
|
11
10
|
from adam.utils import tabulize, log2, log_exc
|
|
12
11
|
|
|
13
12
|
class MedusaRestore(Command):
|
|
@@ -67,20 +66,7 @@ class MedusaRestore(Command):
|
|
|
67
66
|
return state
|
|
68
67
|
|
|
69
68
|
def completion(self, state: ReplState):
|
|
70
|
-
|
|
71
|
-
ns = state.namespace
|
|
72
|
-
dc: str = StatefulSets.get_datacenter(state.sts, ns)
|
|
73
|
-
if not dc:
|
|
74
|
-
return {}
|
|
75
|
-
|
|
76
|
-
if Config().get('medusa.restore-auto-complete', False):
|
|
77
|
-
leaf = {id: None for id in [f"{x['metadata']['name']}" for x in CustomResources.medusa_show_backupjobs(dc, ns)]}
|
|
78
|
-
|
|
79
|
-
return super().completion(state, leaf)
|
|
80
|
-
else:
|
|
81
|
-
return sc
|
|
82
|
-
|
|
83
|
-
return {}
|
|
69
|
+
return super().completion(state, lambda: {id: None for id in medusa_backup_names(state)}, auto_key='medusa.backups')
|
|
84
70
|
|
|
85
|
-
def help(self,
|
|
86
|
-
return
|
|
71
|
+
def help(self, state: ReplState):
|
|
72
|
+
return super().help(state, 'start a restore job')
|
|
@@ -47,5 +47,5 @@ class MedusaShowBackupJobs(Command):
|
|
|
47
47
|
def completion(self, state: ReplState):
|
|
48
48
|
return super().completion(state)
|
|
49
49
|
|
|
50
|
-
def help(self,
|
|
51
|
-
return
|
|
50
|
+
def help(self, state: ReplState):
|
|
51
|
+
return super().help(state, 'show Medusa backups')
|
|
@@ -43,5 +43,5 @@ class MedusaShowRestoreJobs(Command):
|
|
|
43
43
|
def completion(self, state: ReplState):
|
|
44
44
|
return super().completion(state)
|
|
45
45
|
|
|
46
|
-
def help(self,
|
|
47
|
-
return
|
|
46
|
+
def help(self, state: ReplState):
|
|
47
|
+
return super().help(state, 'show Medusa restores')
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
from adam.config import Config
|
|
2
|
+
from adam.repl_state import ReplState
|
|
3
|
+
from adam.utils_k8s.custom_resources import CustomResources
|
|
4
|
+
from adam.utils_k8s.statefulsets import StatefulSets
|
|
5
|
+
|
|
6
|
+
def medusa_backup_names(state: ReplState, warm=False):
|
|
7
|
+
if warm and (auto := Config().get('medusa.restore-auto-complete', 'off')) in ['off', 'jit', 'lazy']:
|
|
8
|
+
return {}
|
|
9
|
+
|
|
10
|
+
ns = state.namespace
|
|
11
|
+
dc: str = StatefulSets.get_datacenter(state.sts, ns)
|
|
12
|
+
if not dc:
|
|
13
|
+
return {}
|
|
14
|
+
|
|
15
|
+
return {id: None for id in [f"{x['metadata']['name']}" for x in CustomResources.medusa_show_backupjobs(dc, ns)]}
|
|
File without changes
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
import time
|
|
2
|
+
import click
|
|
3
|
+
|
|
4
|
+
from adam.commands import extract_options
|
|
5
|
+
from adam.commands.command import Command
|
|
6
|
+
from adam.commands.command_helpers import ClusterOrPodCommandHelper
|
|
7
|
+
from adam.commands.cql.utils_cql import cassandra
|
|
8
|
+
from adam.commands.devices.devices import Devices
|
|
9
|
+
from adam.commands.nodetool.nodetool_commands import NODETOOL_COMMANDS
|
|
10
|
+
from adam.commands.nodetool.utils_nodetool import abort_nodetool_tasks, find_running_nodetool_tasks
|
|
11
|
+
from adam.config import Config
|
|
12
|
+
from adam.repl_state import ReplState, RequiredState
|
|
13
|
+
from adam.utils import log, log2, tabulize
|
|
14
|
+
|
|
15
|
+
class NodeTool(Command):
|
|
16
|
+
COMMAND = 'nodetool'
|
|
17
|
+
|
|
18
|
+
# the singleton pattern
|
|
19
|
+
def __new__(cls, *args, **kwargs):
|
|
20
|
+
if not hasattr(cls, 'instance'): cls.instance = super(NodeTool, cls).__new__(cls)
|
|
21
|
+
|
|
22
|
+
return cls.instance
|
|
23
|
+
|
|
24
|
+
def __init__(self, successor: Command=None):
|
|
25
|
+
super().__init__(successor)
|
|
26
|
+
|
|
27
|
+
def command(self):
|
|
28
|
+
return NodeTool.COMMAND
|
|
29
|
+
|
|
30
|
+
def required(self):
|
|
31
|
+
return RequiredState.CLUSTER_OR_POD
|
|
32
|
+
|
|
33
|
+
def run(self, cmd: str, state: ReplState):
|
|
34
|
+
if not(args := self.args(cmd)):
|
|
35
|
+
return super().run(cmd, state)
|
|
36
|
+
|
|
37
|
+
with self.validate(args, state) as (args, state):
|
|
38
|
+
with extract_options(args, '--force') as (args, forced):
|
|
39
|
+
with cassandra(state) as pods:
|
|
40
|
+
if subcommand := args[0]:
|
|
41
|
+
if subcommand in ['compact', 'repair']:
|
|
42
|
+
ps = find_running_nodetool_tasks(subcommand, state)
|
|
43
|
+
if ps:
|
|
44
|
+
tabulize(ps, lambda p: '\t'.join(p), header='POD\tCMD\tID/PID\tLAST_ARG\tREAPER_RUN_STATE', separator='\t')
|
|
45
|
+
log2()
|
|
46
|
+
|
|
47
|
+
if forced:
|
|
48
|
+
log2(f"* Found running instances of 'nodetool {subcommand}', aborting existing ones...")
|
|
49
|
+
abort_nodetool_tasks(state, subcommand, ps)
|
|
50
|
+
|
|
51
|
+
wait_duration = Config().get('nodetool.grace-period-after-abort', 10)
|
|
52
|
+
log2(f"* Scheduling new 'nodetool {subcommand}' in {wait_duration} secs...")
|
|
53
|
+
time.sleep(wait_duration)
|
|
54
|
+
else:
|
|
55
|
+
log2(f"* Found running instances of 'nodetool {subcommand}', add --force to abort existing ones.")
|
|
56
|
+
|
|
57
|
+
return state
|
|
58
|
+
|
|
59
|
+
pods.nodetool(' '.join(args), status=(args[0] == 'status'))
|
|
60
|
+
|
|
61
|
+
return state
|
|
62
|
+
|
|
63
|
+
def completion(self, state: ReplState):
|
|
64
|
+
return super().completion(state, {c: {'--force': {'&': None}, '&': None} for c in NODETOOL_COMMANDS}, pods=Devices.of(state).pods(state, '-'))
|
|
65
|
+
|
|
66
|
+
def help(self, state: ReplState):
|
|
67
|
+
return super().help(state, 'run nodetool with arguments', args='<sub-command> [&]')
|
|
68
|
+
|
|
69
|
+
class NodeToolCommandHelper(click.Command):
|
|
70
|
+
def get_help(self, ctx: click.Context):
|
|
71
|
+
log(super().get_help(ctx))
|
|
72
|
+
log()
|
|
73
|
+
log('Sub-Commands:')
|
|
74
|
+
|
|
75
|
+
cmds = ''
|
|
76
|
+
for c in NODETOOL_COMMANDS:
|
|
77
|
+
if cmds:
|
|
78
|
+
cmds += ', '
|
|
79
|
+
cmds += c
|
|
80
|
+
if len(cmds) > Config().get('nodetool.commands_in_line', 40):
|
|
81
|
+
log(' ' + cmds)
|
|
82
|
+
cmds = ''
|
|
83
|
+
|
|
84
|
+
if len(cmds) > 0:
|
|
85
|
+
log(' ' + cmds)
|
|
86
|
+
log()
|
|
87
|
+
ClusterOrPodCommandHelper.cluter_or_pod_help()
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
from adam.commands.devices.devices import Devices
|
|
2
|
+
from adam.commands.export.utils_export import state_with_pod
|
|
3
|
+
from adam.commands.fs.utils_fs import find_pids_for_cluster
|
|
4
|
+
from adam.commands.reaper.utils_reaper import reaper
|
|
5
|
+
from adam.config import Config
|
|
6
|
+
from adam.repl_state import ReplState
|
|
7
|
+
from adam.utils import log2
|
|
8
|
+
|
|
9
|
+
def find_running_nodetool_tasks(subcommand: str, state: ReplState) -> list[list[str]]:
|
|
10
|
+
lines = []
|
|
11
|
+
processes = find_pids_for_cluster(state, [subcommand])
|
|
12
|
+
for p in processes:
|
|
13
|
+
l = [p.pod, p.cmd, p.pid, p.last_arg]
|
|
14
|
+
lines.append(l)
|
|
15
|
+
|
|
16
|
+
if subcommand == 'repair':
|
|
17
|
+
with reaper(state) as http:
|
|
18
|
+
response = http.get('repair_run?state=RUNNING', params={
|
|
19
|
+
'cluster_name': 'all',
|
|
20
|
+
'limit': Config().get('reaper.show-runs-batch', 10)
|
|
21
|
+
})
|
|
22
|
+
|
|
23
|
+
runs = response.json()
|
|
24
|
+
if runs:
|
|
25
|
+
for r in runs:
|
|
26
|
+
l = ['reaper', 'reaper', r['id'], '', r['state']]
|
|
27
|
+
lines.append(l)
|
|
28
|
+
|
|
29
|
+
return lines
|
|
30
|
+
|
|
31
|
+
def abort_nodetool_tasks(state: ReplState, subcommand: str, processes: list[list[str]]):
|
|
32
|
+
for p in processes:
|
|
33
|
+
pod = p[0]
|
|
34
|
+
cmd = p[1]
|
|
35
|
+
id = p[2]
|
|
36
|
+
|
|
37
|
+
if pod == 'reaper':
|
|
38
|
+
with reaper(state) as http:
|
|
39
|
+
http.put(f'repair_run/{id}/state/ABORTED')
|
|
40
|
+
elif cmd == subcommand:
|
|
41
|
+
log2(f'@{pod} bash kill -9 {id}')
|
|
42
|
+
|
|
43
|
+
with state_with_pod(state, pod) as state1:
|
|
44
|
+
Devices.of(state).bash(state, state1, ['kill', '-9', id])
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
from adam.commands.postgres.postgres_databases import PostgresDatabases
|
|
2
|
+
from adam.commands.postgres.utils_postgres import pg_table_names
|
|
3
|
+
from adam.repl_state import ReplState
|
|
4
|
+
from adam.sql.lark_completer import LarkCompleter
|
|
5
|
+
|
|
6
|
+
def completions_p(state: ReplState):
|
|
7
|
+
return {
|
|
8
|
+
'\h': None,
|
|
9
|
+
'\d': None,
|
|
10
|
+
'\dt': None,
|
|
11
|
+
'\du': None
|
|
12
|
+
} | LarkCompleter(expandables={
|
|
13
|
+
'tables': lambda x: pg_table_names(state),
|
|
14
|
+
'columns': ['id'],
|
|
15
|
+
'hosts': ['@' + PostgresDatabases.pod_and_container(state.namespace)[0]],
|
|
16
|
+
}, variant=ReplState.P).completions_for_nesting()
|
|
17
|
+
|
|
18
|
+
def psql0_completions(state: ReplState):
|
|
19
|
+
return {
|
|
20
|
+
'\h': None,
|
|
21
|
+
'\l': None,
|
|
22
|
+
}
|
|
@@ -4,12 +4,12 @@ from adam.commands import extract_trailing_options, validate_args
|
|
|
4
4
|
from adam.commands.command import Command
|
|
5
5
|
from adam.commands.intermediate_command import IntermediateCommand
|
|
6
6
|
from adam.commands.postgres.postgres_databases import pg_path
|
|
7
|
-
from adam.commands.postgres.
|
|
7
|
+
from adam.commands.postgres.completions_p import psql0_completions, completions_p
|
|
8
8
|
from adam.commands.postgres.utils_postgres import pg_table_names, postgres
|
|
9
9
|
from .postgres_ls import PostgresLs
|
|
10
10
|
from .postgres_preview import PostgresPreview
|
|
11
11
|
from adam.repl_state import ReplState
|
|
12
|
-
from adam.utils import log, log2
|
|
12
|
+
from adam.utils import log, log2, log_timing
|
|
13
13
|
|
|
14
14
|
class Postgres(IntermediateCommand):
|
|
15
15
|
COMMAND = 'pg'
|
|
@@ -55,27 +55,20 @@ class Postgres(IntermediateCommand):
|
|
|
55
55
|
|
|
56
56
|
def completion(self, state: ReplState):
|
|
57
57
|
if state.device != state.P:
|
|
58
|
-
# conflicts with cql completions
|
|
59
58
|
return {}
|
|
60
59
|
|
|
61
|
-
leaf = {}
|
|
62
60
|
with pg_path(state) as (host, database):
|
|
63
61
|
if database:
|
|
64
62
|
if pg_table_names(state):
|
|
65
|
-
|
|
63
|
+
with log_timing('psql_completions'):
|
|
64
|
+
return completions_p(state)
|
|
66
65
|
elif host:
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
}
|
|
71
|
-
|
|
72
|
-
if state.pg_path:
|
|
73
|
-
return super().completion(state, leaf) | leaf
|
|
74
|
-
else:
|
|
75
|
-
return {}
|
|
66
|
+
return psql0_completions(state)
|
|
67
|
+
|
|
68
|
+
return {}
|
|
76
69
|
|
|
77
|
-
def help(self,
|
|
78
|
-
return
|
|
70
|
+
def help(self, state: ReplState):
|
|
71
|
+
return super().help(state, 'run queries on Postgres databases', command='[pg] <sql-statements>')
|
|
79
72
|
|
|
80
73
|
class PostgresCommandHelper(click.Command):
|
|
81
74
|
def get_help(self, ctx: click.Context):
|
|
@@ -90,7 +83,4 @@ class PostgresPg(Command):
|
|
|
90
83
|
COMMAND = 'pg'
|
|
91
84
|
|
|
92
85
|
def command(self):
|
|
93
|
-
return PostgresPg.COMMAND
|
|
94
|
-
|
|
95
|
-
def help(self, _: ReplState):
|
|
96
|
-
return f'pg <sql-statements>\t run queries on Postgres databases'
|
|
86
|
+
return PostgresPg.COMMAND
|
|
@@ -144,12 +144,12 @@ class PostgresDatabases:
|
|
|
144
144
|
env_prefix = f'PGPASSWORD="{password}"'
|
|
145
145
|
|
|
146
146
|
r = Pods.exec(pod_name, container_name, state.namespace, cmd, show_out=show_out, backgrounded=backgrounded, env_prefix=env_prefix)
|
|
147
|
-
if r and
|
|
148
|
-
|
|
149
|
-
ReplSession().prompt_session.history.append_string(f'@{r.pod} cat {r.log_file}')
|
|
147
|
+
if r and r.log_file:
|
|
148
|
+
ReplSession().append_history(f':cat {r.log_file}')
|
|
150
149
|
|
|
151
150
|
return r
|
|
152
151
|
|
|
152
|
+
@functools.lru_cache()
|
|
153
153
|
def pod_and_container(namespace: str):
|
|
154
154
|
container_name = Config().get('pg.agent.name', 'ops-pg-agent')
|
|
155
155
|
|