kaqing 2.0.145__py3-none-any.whl → 2.0.189__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of kaqing might be problematic. Click here for more details.
- adam/__init__.py +0 -2
- adam/app_session.py +9 -12
- adam/apps.py +18 -4
- adam/batch.py +4 -4
- adam/checks/check_utils.py +16 -46
- adam/checks/cpu.py +7 -1
- adam/checks/cpu_metrics.py +52 -0
- adam/checks/disk.py +2 -3
- adam/columns/columns.py +3 -1
- adam/columns/cpu.py +3 -1
- adam/columns/cpu_metrics.py +22 -0
- adam/columns/memory.py +3 -4
- adam/commands/__init__.py +24 -0
- adam/commands/alter_tables.py +33 -48
- adam/commands/app/__init__.py +0 -0
- adam/commands/app/app.py +38 -0
- adam/commands/{app_ping.py → app/app_ping.py} +7 -13
- adam/commands/app/show_app_actions.py +49 -0
- adam/commands/{show → app}/show_app_id.py +8 -11
- adam/commands/{show → app}/show_app_queues.py +7 -14
- adam/commands/app/utils_app.py +106 -0
- adam/commands/audit/audit.py +21 -40
- adam/commands/audit/audit_repair_tables.py +14 -19
- adam/commands/audit/audit_run.py +14 -22
- adam/commands/audit/completions_l.py +15 -0
- adam/commands/audit/show_last10.py +4 -19
- adam/commands/audit/show_slow10.py +4 -18
- adam/commands/audit/show_top10.py +4 -16
- adam/commands/audit/utils_show_top10.py +15 -3
- adam/commands/bash/__init__.py +5 -0
- adam/commands/bash/bash.py +7 -104
- adam/commands/bash/utils_bash.py +16 -0
- adam/commands/cat.py +7 -27
- adam/commands/cd.py +7 -11
- adam/commands/check.py +15 -24
- adam/commands/cli_commands.py +8 -4
- adam/commands/clipboard_copy.py +87 -0
- adam/commands/code.py +21 -24
- adam/commands/command.py +207 -42
- adam/commands/commands_utils.py +25 -27
- adam/commands/cql/completions_c.py +28 -0
- adam/commands/cql/cqlsh.py +9 -33
- adam/commands/cql/{cql_utils.py → utils_cql.py} +111 -15
- adam/commands/deploy/code_start.py +7 -10
- adam/commands/deploy/code_stop.py +4 -21
- adam/commands/deploy/code_utils.py +3 -3
- adam/commands/deploy/deploy.py +4 -27
- adam/commands/deploy/deploy_frontend.py +14 -17
- adam/commands/deploy/deploy_pg_agent.py +3 -6
- adam/commands/deploy/deploy_pod.py +64 -68
- adam/commands/deploy/undeploy.py +4 -27
- adam/commands/deploy/undeploy_frontend.py +4 -7
- adam/commands/deploy/undeploy_pg_agent.py +5 -8
- adam/commands/deploy/undeploy_pod.py +9 -12
- adam/commands/devices/device.py +124 -2
- adam/commands/devices/device_app.py +41 -24
- adam/commands/devices/device_auit_log.py +10 -4
- adam/commands/devices/device_cass.py +48 -14
- adam/commands/devices/device_export.py +13 -12
- adam/commands/devices/device_postgres.py +105 -54
- adam/commands/download_file.py +47 -0
- adam/commands/exit.py +1 -4
- adam/commands/export/clean_up_all_export_sessions.py +37 -0
- adam/commands/export/clean_up_export_sessions.py +9 -10
- adam/commands/export/completions_x.py +11 -0
- adam/commands/export/download_export_session.py +40 -0
- adam/commands/export/drop_export_database.py +7 -26
- adam/commands/export/drop_export_databases.py +5 -14
- adam/commands/export/export.py +6 -52
- adam/commands/export/export_databases.py +108 -32
- adam/commands/export/export_select.py +8 -59
- adam/commands/export/export_sessions.py +209 -0
- adam/commands/export/export_use.py +14 -20
- adam/commands/export/export_x_select.py +48 -0
- adam/commands/export/exporter.py +135 -167
- adam/commands/export/import_files.py +44 -0
- adam/commands/export/import_session.py +11 -35
- adam/commands/export/importer.py +19 -5
- adam/commands/export/importer_athena.py +112 -44
- adam/commands/export/importer_sqlite.py +42 -22
- adam/commands/export/show_column_counts.py +13 -31
- adam/commands/export/show_export_databases.py +7 -7
- adam/commands/export/show_export_session.py +8 -20
- adam/commands/export/show_export_sessions.py +6 -16
- adam/commands/export/utils_export.py +64 -11
- adam/commands/find_files.py +51 -0
- adam/commands/find_processes.py +76 -0
- adam/commands/head.py +36 -0
- adam/commands/help.py +2 -2
- adam/commands/intermediate_command.py +52 -0
- adam/commands/issues.py +11 -43
- adam/commands/kubectl.py +3 -6
- adam/commands/login.py +22 -24
- adam/commands/logs.py +3 -6
- adam/commands/ls.py +9 -10
- adam/commands/medusa/medusa.py +4 -22
- adam/commands/medusa/medusa_backup.py +20 -27
- adam/commands/medusa/medusa_restore.py +49 -46
- adam/commands/medusa/medusa_show_backupjobs.py +16 -18
- adam/commands/medusa/medusa_show_restorejobs.py +13 -18
- adam/commands/medusa/utils_medusa.py +15 -0
- adam/commands/nodetool.py +7 -21
- adam/commands/param_get.py +11 -14
- adam/commands/param_set.py +8 -12
- adam/commands/postgres/completions_p.py +22 -0
- adam/commands/postgres/postgres.py +34 -57
- adam/commands/postgres/postgres_databases.py +270 -0
- adam/commands/postgres/postgres_ls.py +4 -8
- adam/commands/postgres/postgres_preview.py +5 -9
- adam/commands/postgres/utils_postgres.py +79 -0
- adam/commands/preview_table.py +8 -45
- adam/commands/pwd.py +13 -16
- adam/commands/reaper/reaper.py +4 -27
- adam/commands/reaper/reaper_forward.py +49 -56
- adam/commands/reaper/reaper_forward_session.py +6 -0
- adam/commands/reaper/reaper_forward_stop.py +10 -16
- adam/commands/reaper/reaper_restart.py +7 -14
- adam/commands/reaper/reaper_run_abort.py +8 -33
- adam/commands/reaper/reaper_runs.py +43 -58
- adam/commands/reaper/reaper_runs_abort.py +29 -49
- adam/commands/reaper/reaper_schedule_activate.py +14 -33
- adam/commands/reaper/reaper_schedule_start.py +9 -33
- adam/commands/reaper/reaper_schedule_stop.py +9 -33
- adam/commands/reaper/reaper_schedules.py +4 -14
- adam/commands/reaper/reaper_status.py +8 -16
- adam/commands/reaper/utils_reaper.py +203 -0
- adam/commands/repair/repair.py +4 -22
- adam/commands/repair/repair_log.py +5 -11
- adam/commands/repair/repair_run.py +27 -34
- adam/commands/repair/repair_scan.py +32 -40
- adam/commands/repair/repair_stop.py +5 -12
- adam/commands/report.py +27 -29
- adam/commands/restart.py +25 -26
- adam/commands/rollout.py +19 -24
- adam/commands/shell.py +12 -4
- adam/commands/show/show.py +11 -27
- adam/commands/show/show_adam.py +3 -3
- adam/commands/show/show_cassandra_repairs.py +37 -0
- adam/commands/show/show_cassandra_status.py +47 -51
- adam/commands/show/show_cassandra_version.py +5 -18
- adam/commands/show/show_cli_commands.py +56 -0
- adam/commands/show/show_host.py +1 -1
- adam/commands/show/show_login.py +20 -27
- adam/commands/show/show_params.py +2 -5
- adam/commands/show/show_processes.py +18 -21
- adam/commands/show/show_storage.py +11 -20
- adam/commands/watch.py +26 -29
- adam/config.py +5 -16
- adam/embedded_params.py +1 -1
- adam/log.py +4 -4
- adam/pod_exec_result.py +3 -3
- adam/repl.py +45 -39
- adam/repl_commands.py +26 -19
- adam/repl_session.py +8 -1
- adam/repl_state.py +85 -36
- adam/sql/lark_completer.py +284 -0
- adam/sql/lark_parser.py +604 -0
- adam/sql/sql_completer.py +4 -6
- adam/sql/sql_state_machine.py +29 -16
- adam/sso/authn_ad.py +6 -8
- adam/sso/authn_okta.py +4 -6
- adam/sso/cred_cache.py +3 -5
- adam/sso/idp.py +9 -12
- adam/utils.py +484 -37
- adam/utils_athena.py +19 -19
- adam/utils_audits.py +12 -12
- adam/utils_issues.py +32 -0
- adam/utils_k8s/app_clusters.py +14 -19
- adam/utils_k8s/app_pods.py +7 -2
- adam/utils_k8s/cassandra_clusters.py +30 -19
- adam/utils_k8s/cassandra_nodes.py +2 -2
- adam/utils_k8s/custom_resources.py +16 -17
- adam/utils_k8s/ingresses.py +2 -2
- adam/utils_k8s/jobs.py +7 -11
- adam/utils_k8s/k8s.py +96 -0
- adam/utils_k8s/kube_context.py +2 -2
- adam/utils_k8s/pods.py +37 -81
- adam/utils_k8s/secrets.py +4 -4
- adam/utils_k8s/service_accounts.py +5 -4
- adam/utils_k8s/services.py +2 -2
- adam/utils_k8s/statefulsets.py +6 -14
- adam/utils_local.py +4 -0
- adam/utils_repl/appendable_completer.py +6 -0
- adam/utils_repl/repl_completer.py +128 -2
- adam/utils_repl/state_machine.py +3 -3
- adam/utils_sqlite.py +78 -42
- adam/version.py +1 -1
- {kaqing-2.0.145.dist-info → kaqing-2.0.189.dist-info}/METADATA +1 -1
- kaqing-2.0.189.dist-info/RECORD +253 -0
- kaqing-2.0.189.dist-info/top_level.txt +2 -0
- teddy/__init__.py +0 -0
- teddy/lark_parser.py +436 -0
- teddy/lark_parser2.py +618 -0
- adam/commands/app.py +0 -67
- adam/commands/cp.py +0 -95
- adam/commands/cql/cql_completions.py +0 -28
- adam/commands/export/clean_up_export_session.py +0 -53
- adam/commands/export/export_select_x.py +0 -54
- adam/commands/postgres/postgres_context.py +0 -248
- adam/commands/postgres/postgres_utils.py +0 -31
- adam/commands/postgres/psql_completions.py +0 -10
- adam/commands/reaper/reaper_session.py +0 -159
- adam/commands/show/show_app_actions.py +0 -56
- adam/commands/show/show_commands.py +0 -61
- adam/commands/show/show_repairs.py +0 -47
- kaqing-2.0.145.dist-info/RECORD +0 -227
- kaqing-2.0.145.dist-info/top_level.txt +0 -1
- {kaqing-2.0.145.dist-info → kaqing-2.0.189.dist-info}/WHEEL +0 -0
- {kaqing-2.0.145.dist-info → kaqing-2.0.189.dist-info}/entry_points.txt +0 -0
adam/commands/cp.py
DELETED
|
@@ -1,95 +0,0 @@
|
|
|
1
|
-
import click
|
|
2
|
-
import pyperclip
|
|
3
|
-
|
|
4
|
-
from adam.commands.command import Command
|
|
5
|
-
from adam.commands.command_helpers import ClusterOrPodCommandHelper
|
|
6
|
-
from adam.commands.cli_commands import CliCommands
|
|
7
|
-
from adam.repl_state import ReplState, RequiredState
|
|
8
|
-
from adam.utils import lines_to_tabular, log, log2
|
|
9
|
-
|
|
10
|
-
class ClipboardCopy(Command):
|
|
11
|
-
COMMAND = 'cp'
|
|
12
|
-
|
|
13
|
-
# the singleton pattern
|
|
14
|
-
def __new__(cls, *args, **kwargs):
|
|
15
|
-
if not hasattr(cls, 'instance'): cls.instance = super(ClipboardCopy, cls).__new__(cls)
|
|
16
|
-
|
|
17
|
-
return cls.instance
|
|
18
|
-
|
|
19
|
-
def __init__(self, successor: Command=None):
|
|
20
|
-
super().__init__(successor)
|
|
21
|
-
|
|
22
|
-
def command(self):
|
|
23
|
-
return ClipboardCopy.COMMAND
|
|
24
|
-
|
|
25
|
-
def required(self):
|
|
26
|
-
return RequiredState.CLUSTER_OR_POD
|
|
27
|
-
|
|
28
|
-
def run(self, cmd: str, state: ReplState):
|
|
29
|
-
if not(args := self.args(cmd)):
|
|
30
|
-
return super().run(cmd, state)
|
|
31
|
-
|
|
32
|
-
state, args = self.apply_state(args, state)
|
|
33
|
-
if not self.validate_state(state):
|
|
34
|
-
return state
|
|
35
|
-
|
|
36
|
-
if len(args) < 1:
|
|
37
|
-
if state.in_repl:
|
|
38
|
-
log2('Key is required.')
|
|
39
|
-
log2()
|
|
40
|
-
log2('Keys:')
|
|
41
|
-
log2(lines_to_tabular([f'{k},{v}' for k, v in CliCommands.values(state, collapse=True).items()], separator=','))
|
|
42
|
-
else:
|
|
43
|
-
log2('* Key is missing.')
|
|
44
|
-
Command.display_help()
|
|
45
|
-
|
|
46
|
-
return 'command-missing'
|
|
47
|
-
|
|
48
|
-
key = args[0]
|
|
49
|
-
if not key in CliCommands.values(state):
|
|
50
|
-
if state.in_repl:
|
|
51
|
-
log2('Key is required.')
|
|
52
|
-
log2()
|
|
53
|
-
log2('Keys:')
|
|
54
|
-
log2(lines_to_tabular([f'{k},{v}' for k, v in CliCommands.values(state, collapse=True).items()], separator=','))
|
|
55
|
-
else:
|
|
56
|
-
log2('* Invalid key')
|
|
57
|
-
Command.display_help()
|
|
58
|
-
|
|
59
|
-
return 'command-invalid'
|
|
60
|
-
|
|
61
|
-
value = CliCommands.values(state)[key]
|
|
62
|
-
pyperclip.copy(value)
|
|
63
|
-
log2('The following line has been copied to clipboard. Use <Ctrl-V> to use it.')
|
|
64
|
-
log2(f' {value}')
|
|
65
|
-
|
|
66
|
-
return 'value-copied'
|
|
67
|
-
|
|
68
|
-
def completion(self, state: ReplState):
|
|
69
|
-
if state.sts:
|
|
70
|
-
return {ClipboardCopy.COMMAND: {key: None for key in CliCommands.values(state).keys()}}
|
|
71
|
-
|
|
72
|
-
return {}
|
|
73
|
-
|
|
74
|
-
def help(self, _: ReplState):
|
|
75
|
-
return f"{ClipboardCopy.COMMAND} <key>\t copy a value to clipboard for conveninence"
|
|
76
|
-
|
|
77
|
-
class CopyCommandHelper(click.Command):
|
|
78
|
-
def lines(self):
|
|
79
|
-
return [
|
|
80
|
-
'node-exec-?: kubectl exec command to the Cassandra pod',
|
|
81
|
-
'reaper-exec: kubectl exec command to the Reaper pod',
|
|
82
|
-
'reaper-forward: kubectl port-forward command to the Reaper pod',
|
|
83
|
-
'reaper-ui: uri to Reaper ui',
|
|
84
|
-
'reaper-username: Reaper user name',
|
|
85
|
-
'reaper-password: Reaper password',
|
|
86
|
-
]
|
|
87
|
-
|
|
88
|
-
def get_help(self, ctx: click.Context):
|
|
89
|
-
log(super().get_help(ctx))
|
|
90
|
-
log()
|
|
91
|
-
log('Keys:')
|
|
92
|
-
|
|
93
|
-
log(lines_to_tabular(self.lines(), separator=':'))
|
|
94
|
-
log()
|
|
95
|
-
ClusterOrPodCommandHelper.cluter_or_pod_help()
|
|
@@ -1,28 +0,0 @@
|
|
|
1
|
-
from adam.commands.cql.cql_utils import cassandra_keyspaces, cassandra_table_names
|
|
2
|
-
from adam.commands.export.exporter import Exporter
|
|
3
|
-
from adam.commands.export.export_databases import ExportDatabases
|
|
4
|
-
from adam.config import Config
|
|
5
|
-
from adam.repl_state import ReplState
|
|
6
|
-
from adam.sql.sql_completer import SqlCompleter, SqlVariant
|
|
7
|
-
|
|
8
|
-
def cql_completions(state: ReplState) -> dict[str, any]:
|
|
9
|
-
ps = Config().get('cql.alter-tables.gc-grace-periods', '3600,86400,864000,7776000').split(',')
|
|
10
|
-
# warm up caches
|
|
11
|
-
cassandra_keyspaces(state)
|
|
12
|
-
cassandra_table_names(state)
|
|
13
|
-
ExportDatabases.database_names()
|
|
14
|
-
|
|
15
|
-
expandables = {
|
|
16
|
-
'keyspaces': lambda: cassandra_keyspaces(state),
|
|
17
|
-
'table-props': lambda: {
|
|
18
|
-
'GC_GRACE_SECONDS': ps
|
|
19
|
-
},
|
|
20
|
-
'export-dbs': lambda: ExportDatabases.database_names(),
|
|
21
|
-
'export-sessions': lambda: Exporter.export_session_names(state.sts, state.pod, state.namespace),
|
|
22
|
-
'export-sessions-incomplete': lambda: Exporter.export_session_names(state.sts, state.pod, state.namespace, export_state='pending_import'),
|
|
23
|
-
}
|
|
24
|
-
|
|
25
|
-
return SqlCompleter(
|
|
26
|
-
lambda: cassandra_table_names(state),
|
|
27
|
-
expandables=expandables,
|
|
28
|
-
variant=SqlVariant.CQL).completions_for_nesting()
|
|
@@ -1,53 +0,0 @@
|
|
|
1
|
-
from adam.commands.command import Command
|
|
2
|
-
from adam.commands.export.exporter import Exporter
|
|
3
|
-
from adam.repl_state import ReplState, RequiredState
|
|
4
|
-
from adam.utils import log, log2
|
|
5
|
-
|
|
6
|
-
class CleanUpExportSession(Command):
|
|
7
|
-
COMMAND = 'clean up export session'
|
|
8
|
-
|
|
9
|
-
# the singleton pattern
|
|
10
|
-
def __new__(cls, *args, **kwargs):
|
|
11
|
-
if not hasattr(cls, 'instance'): cls.instance = super(CleanUpExportSession, cls).__new__(cls)
|
|
12
|
-
|
|
13
|
-
return cls.instance
|
|
14
|
-
|
|
15
|
-
def __init__(self, successor: Command=None):
|
|
16
|
-
super().__init__(successor)
|
|
17
|
-
|
|
18
|
-
def command(self):
|
|
19
|
-
return CleanUpExportSession.COMMAND
|
|
20
|
-
|
|
21
|
-
def required(self):
|
|
22
|
-
return RequiredState.CLUSTER_OR_POD
|
|
23
|
-
|
|
24
|
-
def run(self, cmd: str, state: ReplState):
|
|
25
|
-
if not(args := self.args(cmd)):
|
|
26
|
-
return super().run(cmd, state)
|
|
27
|
-
|
|
28
|
-
state, args = self.apply_state(args, state)
|
|
29
|
-
if not self.validate_state(state):
|
|
30
|
-
return state
|
|
31
|
-
|
|
32
|
-
if not args:
|
|
33
|
-
if state.in_repl:
|
|
34
|
-
log2('Specify export session name.')
|
|
35
|
-
else:
|
|
36
|
-
log2('* Session name is missing.')
|
|
37
|
-
|
|
38
|
-
Command.display_help()
|
|
39
|
-
|
|
40
|
-
return 'command-missing'
|
|
41
|
-
|
|
42
|
-
csv_cnt, log_cnt = Exporter.clean_up_session(state.sts, state.pod, state.namespace, args[0])
|
|
43
|
-
log(f'Removed {csv_cnt} csv and {log_cnt} log files.')
|
|
44
|
-
|
|
45
|
-
Exporter.clear_export_session_cache()
|
|
46
|
-
|
|
47
|
-
return state
|
|
48
|
-
|
|
49
|
-
def completion(self, _: ReplState):
|
|
50
|
-
return {}
|
|
51
|
-
|
|
52
|
-
def help(self, _: ReplState):
|
|
53
|
-
return f'{CleanUpExportSession.COMMAND} <export-session-name>\t clean up export session'
|
|
@@ -1,54 +0,0 @@
|
|
|
1
|
-
from adam.commands.command import Command
|
|
2
|
-
from adam.commands.export.export_databases import ExportDatabases
|
|
3
|
-
from adam.repl_state import ReplState, RequiredState
|
|
4
|
-
from adam.sql.sql_completer import SqlCompleter, SqlVariant
|
|
5
|
-
from adam.utils_athena import Athena
|
|
6
|
-
|
|
7
|
-
# No action body, only for a help entry and auto-completion
|
|
8
|
-
class ExportSelectX(Command):
|
|
9
|
-
COMMAND = 'select_on_x'
|
|
10
|
-
|
|
11
|
-
# the singleton pattern
|
|
12
|
-
def __new__(cls, *args, **kwargs):
|
|
13
|
-
if not hasattr(cls, 'instance'): cls.instance = super(ExportSelectX, cls).__new__(cls)
|
|
14
|
-
|
|
15
|
-
return cls.instance
|
|
16
|
-
|
|
17
|
-
def __init__(self, successor: Command=None):
|
|
18
|
-
super().__init__(successor)
|
|
19
|
-
|
|
20
|
-
def command(self):
|
|
21
|
-
return ExportSelectX.COMMAND
|
|
22
|
-
|
|
23
|
-
def required(self):
|
|
24
|
-
return RequiredState.EXPORT_DB
|
|
25
|
-
|
|
26
|
-
def completion(self, state: ReplState):
|
|
27
|
-
completions = {}
|
|
28
|
-
|
|
29
|
-
if state.device == ReplState.X:
|
|
30
|
-
completions = {'drop': SqlCompleter(
|
|
31
|
-
lambda: ExportDatabases.table_names(state.export_session),
|
|
32
|
-
dml='drop',
|
|
33
|
-
expandables={
|
|
34
|
-
'export-dbs': lambda: ExportDatabases.database_names(),
|
|
35
|
-
'columns':lambda _: Athena.column_names(database=state.export_session, function='export'),
|
|
36
|
-
},
|
|
37
|
-
variant=SqlVariant.ATHENA
|
|
38
|
-
)}
|
|
39
|
-
|
|
40
|
-
if state.export_session:
|
|
41
|
-
completions |= {'select': SqlCompleter(
|
|
42
|
-
lambda: ExportDatabases.table_names(state.export_session),
|
|
43
|
-
dml='select',
|
|
44
|
-
expandables={
|
|
45
|
-
'export-dbs': lambda: ExportDatabases.database_names(),
|
|
46
|
-
'columns':lambda _: Athena.column_names(database=state.export_session, function='export'),
|
|
47
|
-
},
|
|
48
|
-
variant=SqlVariant.ATHENA
|
|
49
|
-
)}
|
|
50
|
-
|
|
51
|
-
return completions
|
|
52
|
-
|
|
53
|
-
def help(self, _: ReplState):
|
|
54
|
-
return f'<sql-select-statements>\t run queries on export database'
|
|
@@ -1,248 +0,0 @@
|
|
|
1
|
-
from datetime import datetime
|
|
2
|
-
import functools
|
|
3
|
-
import re
|
|
4
|
-
import subprocess
|
|
5
|
-
|
|
6
|
-
from adam.config import Config
|
|
7
|
-
from adam.repl_session import ReplSession
|
|
8
|
-
from adam.utils_k8s.kube_context import KubeContext
|
|
9
|
-
from adam.utils_k8s.pods import Pods
|
|
10
|
-
from adam.utils_k8s.secrets import Secrets
|
|
11
|
-
from adam.utils import log2
|
|
12
|
-
|
|
13
|
-
class PostgresContext:
|
|
14
|
-
def apply(namespace: str, path: str, arg: str = None) -> 'PostgresContext':
|
|
15
|
-
context = PostgresContext(namespace, path)
|
|
16
|
-
|
|
17
|
-
if arg:
|
|
18
|
-
if arg == '..':
|
|
19
|
-
if context.db:
|
|
20
|
-
context.db = None
|
|
21
|
-
else:
|
|
22
|
-
context.host = None
|
|
23
|
-
else:
|
|
24
|
-
tks = arg.split('@')
|
|
25
|
-
if not context.host:
|
|
26
|
-
context.host = tks[0]
|
|
27
|
-
else:
|
|
28
|
-
context.db = tks[0]
|
|
29
|
-
|
|
30
|
-
if not namespace and tks[1]:
|
|
31
|
-
context.namespace = tks[1]
|
|
32
|
-
|
|
33
|
-
return context
|
|
34
|
-
|
|
35
|
-
def __init__(self, ns: str, path: str):
|
|
36
|
-
self.namespace = ns
|
|
37
|
-
self.conn_details = None
|
|
38
|
-
self.host = None
|
|
39
|
-
self.db = None
|
|
40
|
-
|
|
41
|
-
if path:
|
|
42
|
-
tks = path.split('/')
|
|
43
|
-
hn = tks[0].split('@')
|
|
44
|
-
self.host = hn[0]
|
|
45
|
-
if len(hn) > 1 and not ns:
|
|
46
|
-
self.namespace = hn[1]
|
|
47
|
-
|
|
48
|
-
if len(tks) > 1:
|
|
49
|
-
self.db = tks[1]
|
|
50
|
-
|
|
51
|
-
def path(self):
|
|
52
|
-
if not self.host:
|
|
53
|
-
return None
|
|
54
|
-
|
|
55
|
-
d = self.host
|
|
56
|
-
if not self.db:
|
|
57
|
-
return d
|
|
58
|
-
|
|
59
|
-
return f'{self.host}/{self.db}'
|
|
60
|
-
|
|
61
|
-
def hosts(ns: str):
|
|
62
|
-
return PostgresContext.hosts_for_namespace(ns)
|
|
63
|
-
|
|
64
|
-
@functools.lru_cache()
|
|
65
|
-
def hosts_for_namespace(ns: str):
|
|
66
|
-
ss = Secrets.list_secrets(ns, name_pattern=Config().get('pg.name-pattern', '^{namespace}.*k8spg.*'))
|
|
67
|
-
|
|
68
|
-
def excludes(name: str):
|
|
69
|
-
exs = Config().get('pg.excludes', '.helm., -admin-secret')
|
|
70
|
-
if exs:
|
|
71
|
-
for ex in exs.split(','):
|
|
72
|
-
if ex.strip(' ') in name:
|
|
73
|
-
return True
|
|
74
|
-
|
|
75
|
-
return False
|
|
76
|
-
|
|
77
|
-
return [s for s in ss if not excludes(s)]
|
|
78
|
-
|
|
79
|
-
def databases(self):
|
|
80
|
-
dbs = []
|
|
81
|
-
# List of databases
|
|
82
|
-
# Name | Owner | Encoding | Collate | Ctype | ICU Locale | Locale Provider | Access privileges
|
|
83
|
-
# ---------------------------------------+----------+----------+-------------+-------------+------------+-----------------+-----------------------
|
|
84
|
-
# postgres | postgres | UTF8 | en_US.UTF-8 | en_US.UTF-8 | | libc |
|
|
85
|
-
# stgawsscpsr_c3_c3 | postgres | UTF8 | C | C | | libc |
|
|
86
|
-
# template1 | postgres | UTF8 | en_US.UTF-8 | en_US.UTF-8 | | libc | =c/postgres +
|
|
87
|
-
# | | | | | | | postgres=CTc/postgres
|
|
88
|
-
# (48 rows)
|
|
89
|
-
if r := self.run_sql('\l', show_out=False):
|
|
90
|
-
s = 0
|
|
91
|
-
for line in r.stdout.split('\n'):
|
|
92
|
-
line: str = line.strip(' \r')
|
|
93
|
-
if s == 0:
|
|
94
|
-
if 'List of databases' in line:
|
|
95
|
-
s = 1
|
|
96
|
-
elif s == 1:
|
|
97
|
-
if 'Name' in line and 'Owner' in line and 'Encoding' in line:
|
|
98
|
-
s = 2
|
|
99
|
-
elif s == 2:
|
|
100
|
-
if line.startswith('---------'):
|
|
101
|
-
s = 3
|
|
102
|
-
elif s == 3:
|
|
103
|
-
groups = re.match(r'^\s*(\S*)\s*\|\s*(\S*)\s*\|.*', line)
|
|
104
|
-
if groups and groups[1] != '|':
|
|
105
|
-
dbs.append({'name': groups[1], 'owner': groups[2]})
|
|
106
|
-
|
|
107
|
-
return dbs
|
|
108
|
-
|
|
109
|
-
def tables(self):
|
|
110
|
-
dbs = []
|
|
111
|
-
# List of relations
|
|
112
|
-
# Schema | Name | Type | Owner
|
|
113
|
-
# ----------+------------------------------------------------------------+-------+---------------
|
|
114
|
-
# postgres | c3_2_admin_aclpriv | table | postgres
|
|
115
|
-
# postgres | c3_2_admin_aclpriv_a | table | postgres
|
|
116
|
-
if r := self.run_sql('\dt', show_out=False):
|
|
117
|
-
s = 0
|
|
118
|
-
for line in r.stdout.split('\n'):
|
|
119
|
-
line: str = line.strip(' \r')
|
|
120
|
-
if s == 0:
|
|
121
|
-
if 'List of relations' in line:
|
|
122
|
-
s = 1
|
|
123
|
-
elif s == 1:
|
|
124
|
-
if 'Schema' in line and 'Name' in line and 'Type' in line:
|
|
125
|
-
s = 2
|
|
126
|
-
elif s == 2:
|
|
127
|
-
if line.startswith('---------'):
|
|
128
|
-
s = 3
|
|
129
|
-
elif s == 3:
|
|
130
|
-
groups = re.match(r'^\s*(\S*)\s*\|\s*(\S*)\s*\|.*', line)
|
|
131
|
-
if groups and groups[1] != '|':
|
|
132
|
-
dbs.append({'schema': groups[1], 'name': groups[2]})
|
|
133
|
-
|
|
134
|
-
return dbs
|
|
135
|
-
|
|
136
|
-
def run_sql(self, sql: str, show_out = True, background = False):
|
|
137
|
-
db = self.db if self.db else PostgresContext.default_db()
|
|
138
|
-
|
|
139
|
-
if KubeContext.in_cluster():
|
|
140
|
-
cmd1 = f'env PGPASSWORD={self.password()} psql -h {self.endpoint()} -p {self.port()} -U {self.username()} {db} --pset pager=off -c'
|
|
141
|
-
log2(f'{cmd1} "{sql}"')
|
|
142
|
-
# remove double quotes from the sql argument
|
|
143
|
-
cmd = cmd1.split(' ') + [sql]
|
|
144
|
-
|
|
145
|
-
r = subprocess.run(cmd, capture_output=not background, text=True)
|
|
146
|
-
if show_out:
|
|
147
|
-
log2(r.stdout)
|
|
148
|
-
log2(r.stderr)
|
|
149
|
-
|
|
150
|
-
return r
|
|
151
|
-
else:
|
|
152
|
-
ns = self.namespace
|
|
153
|
-
pod_name = Config().get('pg.agent.name', 'ops-pg-agent')
|
|
154
|
-
|
|
155
|
-
if Config().get('pg.agent.just-in-time', False):
|
|
156
|
-
if not PostgresContext.deploy_pg_agent(pod_name, ns):
|
|
157
|
-
return
|
|
158
|
-
|
|
159
|
-
real_pod_name = pod_name
|
|
160
|
-
try:
|
|
161
|
-
# try with dedicated pg agent pod name configured
|
|
162
|
-
Pods.get(ns, pod_name)
|
|
163
|
-
except:
|
|
164
|
-
try:
|
|
165
|
-
# try with the ops pod
|
|
166
|
-
pod_name = Config().get('pod.name', 'ops')
|
|
167
|
-
real_pod_name = Pods.get_with_selector(ns, label_selector = Config().get('pod.label-selector', 'run=ops')).metadata.name
|
|
168
|
-
except:
|
|
169
|
-
log2(f"Could not locate {pod_name} pod.")
|
|
170
|
-
return None
|
|
171
|
-
|
|
172
|
-
cmd = f'psql -h {self.endpoint()} -p {self.port()} -U {self.username()} {db} --pset pager=off -c "{sql}"'
|
|
173
|
-
env_prefix = f'PGPASSWORD="{self.password()}"'
|
|
174
|
-
|
|
175
|
-
r = Pods.exec(real_pod_name, pod_name, ns, cmd, show_out=show_out, background=background, env_prefix=env_prefix)
|
|
176
|
-
if r and Config().get('repl.history.push-cat-remote-log-file', True):
|
|
177
|
-
if r.log_file and ReplSession().prompt_session:
|
|
178
|
-
ReplSession().prompt_session.history.append_string(f'@{r.pod} cat {r.log_file}')
|
|
179
|
-
|
|
180
|
-
return r
|
|
181
|
-
|
|
182
|
-
def deploy_pg_agent(pod_name: str, ns: str) -> str:
|
|
183
|
-
image = Config().get('pg.agent.image', 'seanahnsf/kaqing')
|
|
184
|
-
timeout = Config().get('pg.agent.timeout', 3600)
|
|
185
|
-
try:
|
|
186
|
-
Pods.create(ns, pod_name, image, ['sleep', f'{timeout}'], env={'NAMESPACE': ns}, sa_name='c3')
|
|
187
|
-
except Exception as e:
|
|
188
|
-
if e.status == 409:
|
|
189
|
-
if Pods.completed(ns, pod_name):
|
|
190
|
-
try:
|
|
191
|
-
Pods.delete(pod_name, ns)
|
|
192
|
-
Pods.create(ns, pod_name, image, ['sleep', f'{timeout}'], env={'NAMESPACE': ns}, sa_name='c3')
|
|
193
|
-
except Exception as e2:
|
|
194
|
-
log2("Exception when calling BatchV1Api->create_pod: %s\n" % e2)
|
|
195
|
-
|
|
196
|
-
return
|
|
197
|
-
else:
|
|
198
|
-
log2("Exception when calling BatchV1Api->create_pod: %s\n" % e)
|
|
199
|
-
|
|
200
|
-
return
|
|
201
|
-
|
|
202
|
-
Pods.wait_for_running(ns, pod_name)
|
|
203
|
-
|
|
204
|
-
return pod_name
|
|
205
|
-
|
|
206
|
-
def undeploy_pg_agent(pod_name: str, ns: str):
|
|
207
|
-
Pods.delete(pod_name, ns, grace_period_seconds=0)
|
|
208
|
-
|
|
209
|
-
def endpoint(self):
|
|
210
|
-
if not self.conn_details:
|
|
211
|
-
self.conn_details = Secrets.get_data(self.namespace, self.host)
|
|
212
|
-
|
|
213
|
-
endpoint_key = Config().get('pg.secret.endpoint-key', 'postgres-db-endpoint')
|
|
214
|
-
|
|
215
|
-
return self.conn_details[endpoint_key] if endpoint_key in self.conn_details else ''
|
|
216
|
-
|
|
217
|
-
def port(self):
|
|
218
|
-
if not self.conn_details:
|
|
219
|
-
self.conn_details = Secrets.get_data(self.namespace, self.host)
|
|
220
|
-
|
|
221
|
-
port_key = Config().get('pg.secret.port-key', 'postgres-db-port')
|
|
222
|
-
|
|
223
|
-
return self.conn_details[port_key] if port_key in self.conn_details else ''
|
|
224
|
-
|
|
225
|
-
def username(self):
|
|
226
|
-
if not self.conn_details:
|
|
227
|
-
self.conn_details = Secrets.get_data(self.namespace, self.host)
|
|
228
|
-
|
|
229
|
-
username_key = Config().get('pg.secret.username-key', 'postgres-admin-username')
|
|
230
|
-
|
|
231
|
-
return self.conn_details[username_key] if username_key in self.conn_details else ''
|
|
232
|
-
|
|
233
|
-
def password(self):
|
|
234
|
-
if not self.conn_details:
|
|
235
|
-
self.conn_details = Secrets.get_data(self.namespace, self.host)
|
|
236
|
-
|
|
237
|
-
password_key = Config().get('pg.secret.password-key', 'postgres-admin-password')
|
|
238
|
-
|
|
239
|
-
return self.conn_details[password_key] if password_key in self.conn_details else ''
|
|
240
|
-
|
|
241
|
-
def default_db():
|
|
242
|
-
return Config().get('pg.default-db', 'postgres')
|
|
243
|
-
|
|
244
|
-
def default_owner():
|
|
245
|
-
return Config().get('pg.default-owner', 'postgres')
|
|
246
|
-
|
|
247
|
-
def default_schema():
|
|
248
|
-
return Config().get('pg.default-schema', 'postgres')
|
|
@@ -1,31 +0,0 @@
|
|
|
1
|
-
import functools
|
|
2
|
-
|
|
3
|
-
from adam.commands.postgres.postgres_context import PostgresContext
|
|
4
|
-
from adam.config import Config
|
|
5
|
-
|
|
6
|
-
TestPG = [False]
|
|
7
|
-
|
|
8
|
-
@functools.lru_cache()
|
|
9
|
-
def pg_database_names(ns: str, pg_path: str):
|
|
10
|
-
if TestPG[0]:
|
|
11
|
-
return ['azops88_c3ai_c3']
|
|
12
|
-
|
|
13
|
-
Config().wait_log('Inspecting Postgres Databases...')
|
|
14
|
-
|
|
15
|
-
pg = PostgresContext.apply(ns, pg_path)
|
|
16
|
-
return [db['name'] for db in pg.databases() if db['owner'] == PostgresContext.default_owner()]
|
|
17
|
-
|
|
18
|
-
@functools.lru_cache()
|
|
19
|
-
def pg_table_names(ns: str, pg_path: str):
|
|
20
|
-
if TestPG[0]:
|
|
21
|
-
return ['C3_2_XYZ1']
|
|
22
|
-
|
|
23
|
-
Config().wait_log('Inspecting Postgres Database...')
|
|
24
|
-
return [table['name'] for table in pg_tables(ns, pg_path) if table['schema'] == PostgresContext.default_schema()]
|
|
25
|
-
|
|
26
|
-
def pg_tables(ns: str, pg_path: str):
|
|
27
|
-
pg = PostgresContext.apply(ns, pg_path)
|
|
28
|
-
if pg.db:
|
|
29
|
-
return pg.tables()
|
|
30
|
-
|
|
31
|
-
return []
|
|
@@ -1,10 +0,0 @@
|
|
|
1
|
-
from adam.commands.postgres.postgres_utils import pg_table_names
|
|
2
|
-
from adam.sql.sql_completer import SqlCompleter
|
|
3
|
-
|
|
4
|
-
def psql_completions(ns: str, pg_path: str):
|
|
5
|
-
return {
|
|
6
|
-
'\h': None,
|
|
7
|
-
'\d': None,
|
|
8
|
-
'\dt': None,
|
|
9
|
-
'\du': None
|
|
10
|
-
} | SqlCompleter(lambda: pg_table_names(ns, pg_path)).completions_for_nesting()
|
|
@@ -1,159 +0,0 @@
|
|
|
1
|
-
from collections.abc import Callable
|
|
2
|
-
import threading
|
|
3
|
-
from kubernetes import client
|
|
4
|
-
import portforward
|
|
5
|
-
import re
|
|
6
|
-
import requests
|
|
7
|
-
from typing import List, cast
|
|
8
|
-
|
|
9
|
-
from adam.config import Config
|
|
10
|
-
from adam.utils_k8s.kube_context import KubeContext
|
|
11
|
-
from adam.repl_state import ReplState
|
|
12
|
-
from adam.utils import lines_to_tabular, log2
|
|
13
|
-
|
|
14
|
-
class ReaperSession:
|
|
15
|
-
is_forwarding = False
|
|
16
|
-
stopping = threading.Event()
|
|
17
|
-
schedules_ids_by_cluster: dict[str, list[str]] = {}
|
|
18
|
-
|
|
19
|
-
def __init__(self, pod: str, headers: dict[str, str] = None):
|
|
20
|
-
self.pod = pod
|
|
21
|
-
self.headers = headers
|
|
22
|
-
|
|
23
|
-
def login(self, state: ReplState, local_addr: str, remote_addr: str, show_output = True) -> str :
|
|
24
|
-
user, pw = state.user_pass(secret_path='reaper.secret')
|
|
25
|
-
|
|
26
|
-
response = requests.post(f'http://{local_addr}/login', headers={
|
|
27
|
-
'Accept': '*'
|
|
28
|
-
},data={
|
|
29
|
-
'username':user,
|
|
30
|
-
'password':pw})
|
|
31
|
-
if show_output:
|
|
32
|
-
log2(f'POST {remote_addr}/login')
|
|
33
|
-
log2(f' username={user}&password={pw}')
|
|
34
|
-
|
|
35
|
-
if int(response.status_code / 100) != 2:
|
|
36
|
-
if show_output:
|
|
37
|
-
log2("login failed")
|
|
38
|
-
return None
|
|
39
|
-
|
|
40
|
-
return response.headers['Set-Cookie']
|
|
41
|
-
|
|
42
|
-
def port_forwarded(self, state: ReplState, path: str, body: Callable[[str, dict[str, str]], requests.Response], method: str = None, show_output = True):
|
|
43
|
-
local_port = Config().get('reaper.port-forward.local-port', 9001)
|
|
44
|
-
target_port = 8080
|
|
45
|
-
|
|
46
|
-
def f(local_addr: str, remote_addr: str):
|
|
47
|
-
if not self.headers:
|
|
48
|
-
self.headers = self.cookie_header(state, local_addr, remote_addr, show_output=show_output)
|
|
49
|
-
|
|
50
|
-
if show_output and method:
|
|
51
|
-
log2(f'{method} {remote_addr}/{path}')
|
|
52
|
-
response = body(f'http://{local_addr}/{path}', self.headers)
|
|
53
|
-
|
|
54
|
-
if response:
|
|
55
|
-
if int(response.status_code / 100) != 2:
|
|
56
|
-
if show_output:
|
|
57
|
-
log2(response.status_code)
|
|
58
|
-
return response
|
|
59
|
-
|
|
60
|
-
if show_output:
|
|
61
|
-
log2()
|
|
62
|
-
|
|
63
|
-
return response if response else 'no-response'
|
|
64
|
-
|
|
65
|
-
if KubeContext.in_cluster():
|
|
66
|
-
# cs-a526330d23-cs-a526330d23-default-sts-0 ->
|
|
67
|
-
# curl http://cs-a526330d23-cs-a526330d23-reaper-service.stgawsscpsr.svc.cluster.local:8080
|
|
68
|
-
groups = re.match(r'^(.*?-.*?-.*?-.*?-).*', state.sts)
|
|
69
|
-
if groups:
|
|
70
|
-
svc_name = Config().get('reaper.service-name', 'reaper-service')
|
|
71
|
-
svc = f'{groups[1]}{svc_name}.{state.namespace}.svc.cluster.local:{target_port}'
|
|
72
|
-
return f(local_addr=svc, remote_addr=svc)
|
|
73
|
-
else:
|
|
74
|
-
return None
|
|
75
|
-
else:
|
|
76
|
-
with portforward.forward(state.namespace, self.pod, local_port, target_port):
|
|
77
|
-
return f(local_addr=f'localhost:{local_port}', remote_addr=f'{self.pod}:{target_port}')
|
|
78
|
-
|
|
79
|
-
def cookie_header(self, state: ReplState, local_addr, remote_addr, show_output = True):
|
|
80
|
-
return {'Cookie': self.login(state, local_addr, remote_addr, show_output=show_output)}
|
|
81
|
-
|
|
82
|
-
def create(state: ReplState) -> 'ReaperSession':
|
|
83
|
-
pods = ReaperSession.list_reaper_pods(state.sts if state.sts else state.pod, state.namespace)
|
|
84
|
-
if pods:
|
|
85
|
-
return ReaperSession(pods[0].metadata.name)
|
|
86
|
-
else:
|
|
87
|
-
log2('No reaper found.')
|
|
88
|
-
|
|
89
|
-
return None
|
|
90
|
-
|
|
91
|
-
def list_reaper_pods(sts_name: str, namespace: str) -> List[client.V1Pod]:
|
|
92
|
-
v1 = client.CoreV1Api()
|
|
93
|
-
|
|
94
|
-
# k8ssandra.io/reaper: cs-d0767a536f-cs-d0767a536f-reaper
|
|
95
|
-
groups = re.match(Config().get('reaper.pod.cluster-regex', r'(.*?-.*?-.*?-.*?)-.*'), sts_name)
|
|
96
|
-
label_selector = Config().get('reaper.pod.label-selector', 'k8ssandra.io/reaper={cluster}-reaper').replace('{cluster}', groups[1])
|
|
97
|
-
|
|
98
|
-
return cast(List[client.V1Pod], v1.list_namespaced_pod(namespace, label_selector=label_selector).items)
|
|
99
|
-
|
|
100
|
-
def show_schedules(self, state: ReplState, filter: Callable[[list[dict]], dict] = None):
|
|
101
|
-
schedules = self.list_schedules(state, filter=filter)
|
|
102
|
-
# forced refresh of schedule list
|
|
103
|
-
if not filter:
|
|
104
|
-
self.schedules_ids_by_cluster[state.sts] = [schedule['id'] for schedule in schedules]
|
|
105
|
-
self.show_schedules_tabular(schedules)
|
|
106
|
-
|
|
107
|
-
def schedule_ids(self, state: ReplState, show_output = True, filter: Callable[[list[dict]], dict] = None):
|
|
108
|
-
schedules = self.list_schedules(state, show_output=show_output, filter=filter)
|
|
109
|
-
return [schedule['id'] for schedule in schedules]
|
|
110
|
-
|
|
111
|
-
def list_schedules(self, state: ReplState, show_output = True, filter: Callable[[list[dict]], dict] = None) -> list[dict]:
|
|
112
|
-
def body(uri: str, headers: dict[str, str]):
|
|
113
|
-
return requests.get(uri, headers=headers)
|
|
114
|
-
|
|
115
|
-
response = self.port_forwarded(state, 'repair_schedule', body, method='GET', show_output=show_output)
|
|
116
|
-
if not response:
|
|
117
|
-
return
|
|
118
|
-
|
|
119
|
-
res = response.json()
|
|
120
|
-
if filter:
|
|
121
|
-
res = filter(res)
|
|
122
|
-
|
|
123
|
-
return res
|
|
124
|
-
|
|
125
|
-
def show_schedules_tabular(self, schedules: list[dict]):
|
|
126
|
-
log2(lines_to_tabular([f"{schedule['id']} {schedule['state']} {schedule['cluster_name']} {schedule['keyspace_name']}" for schedule in schedules], 'ID STATE CLUSTER KEYSPACE'))
|
|
127
|
-
|
|
128
|
-
def show_schedule(self, state: ReplState, schedule_id: str):
|
|
129
|
-
def filter(schedules: list[dict]):
|
|
130
|
-
return [schedule for schedule in schedules if schedule['id'] == schedule_id]
|
|
131
|
-
|
|
132
|
-
self.show_schedules(state, filter)
|
|
133
|
-
|
|
134
|
-
def reaper_spec(self, state: ReplState) -> dict[str, any]:
|
|
135
|
-
user, pw = state.user_pass(secret_path='reaper.secret')
|
|
136
|
-
local_port = Config().get('reaper.port-forward.local-port', 9001)
|
|
137
|
-
|
|
138
|
-
return {
|
|
139
|
-
'pod': self.pod,
|
|
140
|
-
'exec': f'kubectl exec -it {self.pod} -n {state.namespace} -- bash',
|
|
141
|
-
'forward': f'kubectl port-forward pods/{self.pod} -n {state.namespace} {local_port}:8080',
|
|
142
|
-
'web-uri': f'http://localhost:{local_port}/webui',
|
|
143
|
-
'username': user,
|
|
144
|
-
'password': pw
|
|
145
|
-
}
|
|
146
|
-
|
|
147
|
-
def cached_schedule_ids(state: ReplState) -> list[str]:
|
|
148
|
-
if state.sts in ReaperSession.schedules_ids_by_cluster:
|
|
149
|
-
return ReaperSession.schedules_ids_by_cluster[state.sts]
|
|
150
|
-
|
|
151
|
-
if reaper := ReaperSession.create(state):
|
|
152
|
-
Config().wait_log('Inspecting Cassandra Reaper...')
|
|
153
|
-
|
|
154
|
-
schedules = reaper.schedule_ids(state, show_output = False)
|
|
155
|
-
ReaperSession.schedules_ids_by_cluster[state.sts] = schedules
|
|
156
|
-
|
|
157
|
-
return schedules
|
|
158
|
-
|
|
159
|
-
return []
|