kaqing 2.0.52__py3-none-any.whl → 2.0.184__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of kaqing might be problematic. Click here for more details.
- adam/__init__.py +0 -2
- adam/app_session.py +9 -12
- adam/apps.py +20 -6
- adam/batch.py +15 -19
- adam/checks/check_utils.py +19 -49
- adam/checks/compactionstats.py +1 -1
- adam/checks/cpu.py +9 -3
- adam/checks/cpu_metrics.py +52 -0
- adam/checks/disk.py +3 -4
- adam/checks/gossip.py +1 -1
- adam/checks/memory.py +3 -3
- adam/checks/status.py +1 -1
- adam/columns/columns.py +3 -1
- adam/columns/cpu.py +3 -1
- adam/columns/cpu_metrics.py +22 -0
- adam/columns/memory.py +3 -4
- adam/commands/__init__.py +24 -0
- adam/commands/alter_tables.py +37 -63
- adam/commands/app/app.py +38 -0
- adam/commands/{app_ping.py → app/app_ping.py} +8 -14
- adam/commands/app/show_app_actions.py +49 -0
- adam/commands/{show → app}/show_app_id.py +9 -12
- adam/commands/{show → app}/show_app_queues.py +8 -14
- adam/commands/app/utils_app.py +98 -0
- adam/commands/audit/audit.py +81 -0
- adam/commands/audit/audit_repair_tables.py +72 -0
- adam/commands/audit/audit_run.py +50 -0
- adam/commands/audit/show_last10.py +37 -0
- adam/commands/audit/show_slow10.py +36 -0
- adam/commands/audit/show_top10.py +36 -0
- adam/commands/audit/utils_show_top10.py +71 -0
- adam/commands/bash/__init__.py +5 -0
- adam/commands/bash/bash.py +36 -0
- adam/commands/bash/bash_completer.py +93 -0
- adam/commands/bash/utils_bash.py +16 -0
- adam/commands/cat.py +36 -0
- adam/commands/cd.py +14 -89
- adam/commands/check.py +18 -21
- adam/commands/cli_commands.py +5 -6
- adam/commands/clipboard_copy.py +86 -0
- adam/commands/code.py +57 -0
- adam/commands/command.py +197 -35
- adam/commands/commands_utils.py +15 -31
- adam/commands/cql/cql_completions.py +29 -8
- adam/commands/cql/cqlsh.py +12 -27
- adam/commands/cql/utils_cql.py +297 -0
- adam/commands/deploy/code_start.py +7 -10
- adam/commands/deploy/code_stop.py +4 -21
- adam/commands/deploy/code_utils.py +5 -5
- adam/commands/deploy/deploy.py +4 -21
- adam/commands/deploy/deploy_frontend.py +14 -17
- adam/commands/deploy/deploy_pg_agent.py +3 -6
- adam/commands/deploy/deploy_pod.py +71 -79
- adam/commands/deploy/deploy_utils.py +16 -26
- adam/commands/deploy/undeploy.py +4 -21
- adam/commands/deploy/undeploy_frontend.py +4 -7
- adam/commands/deploy/undeploy_pg_agent.py +6 -8
- adam/commands/deploy/undeploy_pod.py +15 -16
- adam/commands/devices/__init__.py +0 -0
- adam/commands/devices/device.py +123 -0
- adam/commands/devices/device_app.py +163 -0
- adam/commands/devices/device_auit_log.py +49 -0
- adam/commands/devices/device_cass.py +179 -0
- adam/commands/devices/device_export.py +84 -0
- adam/commands/devices/device_postgres.py +150 -0
- adam/commands/devices/devices.py +25 -0
- adam/commands/download_file.py +47 -0
- adam/commands/exit.py +1 -4
- adam/commands/export/__init__.py +0 -0
- adam/commands/export/clean_up_all_export_sessions.py +37 -0
- adam/commands/export/clean_up_export_sessions.py +39 -0
- adam/commands/export/download_export_session.py +39 -0
- adam/commands/export/drop_export_database.py +39 -0
- adam/commands/export/drop_export_databases.py +37 -0
- adam/commands/export/export.py +53 -0
- adam/commands/export/export_databases.py +245 -0
- adam/commands/export/export_select.py +59 -0
- adam/commands/export/export_select_x.py +54 -0
- adam/commands/export/export_sessions.py +209 -0
- adam/commands/export/export_use.py +49 -0
- adam/commands/export/exporter.py +332 -0
- adam/commands/export/import_files.py +44 -0
- adam/commands/export/import_session.py +44 -0
- adam/commands/export/importer.py +81 -0
- adam/commands/export/importer_athena.py +177 -0
- adam/commands/export/importer_sqlite.py +67 -0
- adam/commands/export/show_column_counts.py +45 -0
- adam/commands/export/show_export_databases.py +38 -0
- adam/commands/export/show_export_session.py +39 -0
- adam/commands/export/show_export_sessions.py +37 -0
- adam/commands/export/utils_export.py +343 -0
- adam/commands/find_files.py +51 -0
- adam/commands/find_processes.py +76 -0
- adam/commands/head.py +36 -0
- adam/commands/help.py +14 -9
- adam/commands/intermediate_command.py +49 -0
- adam/commands/issues.py +14 -40
- adam/commands/kubectl.py +38 -0
- adam/commands/login.py +26 -25
- adam/commands/logs.py +5 -7
- adam/commands/ls.py +11 -110
- adam/commands/medusa/medusa.py +4 -22
- adam/commands/medusa/medusa_backup.py +22 -29
- adam/commands/medusa/medusa_restore.py +40 -39
- adam/commands/medusa/medusa_show_backupjobs.py +19 -20
- adam/commands/medusa/medusa_show_restorejobs.py +15 -20
- adam/commands/nodetool.py +11 -15
- adam/commands/param_get.py +11 -14
- adam/commands/param_set.py +8 -12
- adam/commands/postgres/postgres.py +45 -46
- adam/commands/postgres/postgres_databases.py +269 -0
- adam/commands/postgres/postgres_ls.py +4 -8
- adam/commands/postgres/postgres_preview.py +5 -9
- adam/commands/postgres/psql_completions.py +4 -3
- adam/commands/postgres/utils_postgres.py +70 -0
- adam/commands/preview_table.py +10 -61
- adam/commands/pwd.py +14 -43
- adam/commands/reaper/reaper.py +4 -24
- adam/commands/reaper/reaper_forward.py +49 -56
- adam/commands/reaper/reaper_forward_session.py +6 -0
- adam/commands/reaper/reaper_forward_stop.py +10 -16
- adam/commands/reaper/reaper_restart.py +8 -15
- adam/commands/reaper/reaper_run_abort.py +8 -33
- adam/commands/reaper/reaper_runs.py +43 -58
- adam/commands/reaper/reaper_runs_abort.py +29 -49
- adam/commands/reaper/reaper_schedule_activate.py +9 -32
- adam/commands/reaper/reaper_schedule_start.py +9 -32
- adam/commands/reaper/reaper_schedule_stop.py +9 -32
- adam/commands/reaper/reaper_schedules.py +4 -14
- adam/commands/reaper/reaper_status.py +8 -16
- adam/commands/reaper/utils_reaper.py +194 -0
- adam/commands/repair/repair.py +4 -22
- adam/commands/repair/repair_log.py +6 -12
- adam/commands/repair/repair_run.py +29 -36
- adam/commands/repair/repair_scan.py +33 -39
- adam/commands/repair/repair_stop.py +6 -12
- adam/commands/report.py +25 -21
- adam/commands/restart.py +27 -28
- adam/commands/rollout.py +20 -25
- adam/commands/shell.py +12 -4
- adam/commands/show/show.py +11 -23
- adam/commands/show/show_adam.py +3 -3
- adam/commands/show/show_cassandra_repairs.py +35 -0
- adam/commands/show/show_cassandra_status.py +34 -52
- adam/commands/show/show_cassandra_version.py +5 -18
- adam/commands/show/show_commands.py +20 -25
- adam/commands/show/show_host.py +33 -0
- adam/commands/show/show_login.py +23 -27
- adam/commands/show/show_params.py +2 -5
- adam/commands/show/show_processes.py +16 -20
- adam/commands/show/show_storage.py +10 -20
- adam/commands/watch.py +27 -30
- adam/config.py +7 -15
- adam/embedded_params.py +1 -1
- adam/log.py +4 -4
- adam/pod_exec_result.py +13 -5
- adam/repl.py +126 -119
- adam/repl_commands.py +63 -29
- adam/repl_state.py +320 -71
- adam/sql/sql_completer.py +98 -383
- adam/sql/sql_state_machine.py +630 -0
- adam/sql/term_completer.py +14 -4
- adam/sso/authn_ad.py +6 -8
- adam/sso/authn_okta.py +4 -6
- adam/sso/cred_cache.py +4 -6
- adam/sso/idp.py +10 -13
- adam/utils.py +511 -10
- adam/utils_athena.py +145 -0
- adam/utils_audits.py +102 -0
- adam/utils_issues.py +32 -0
- adam/utils_k8s/__init__.py +0 -0
- adam/utils_k8s/app_clusters.py +28 -0
- adam/utils_k8s/app_pods.py +36 -0
- adam/utils_k8s/cassandra_clusters.py +44 -0
- adam/{k8s_utils → utils_k8s}/cassandra_nodes.py +11 -4
- adam/{k8s_utils → utils_k8s}/custom_resources.py +16 -17
- adam/{k8s_utils → utils_k8s}/deployment.py +2 -2
- adam/{k8s_utils → utils_k8s}/ingresses.py +2 -2
- adam/{k8s_utils → utils_k8s}/jobs.py +7 -11
- adam/utils_k8s/k8s.py +87 -0
- adam/{k8s_utils → utils_k8s}/kube_context.py +2 -2
- adam/{k8s_utils → utils_k8s}/pods.py +109 -74
- adam/{k8s_utils → utils_k8s}/secrets.py +7 -3
- adam/{k8s_utils → utils_k8s}/service_accounts.py +5 -4
- adam/{k8s_utils → utils_k8s}/services.py +2 -2
- adam/{k8s_utils → utils_k8s}/statefulsets.py +3 -14
- adam/utils_local.py +4 -0
- adam/utils_net.py +24 -0
- adam/utils_repl/__init__.py +0 -0
- adam/utils_repl/automata_completer.py +48 -0
- adam/utils_repl/repl_completer.py +46 -0
- adam/utils_repl/state_machine.py +173 -0
- adam/utils_sqlite.py +137 -0
- adam/version.py +1 -1
- {kaqing-2.0.52.dist-info → kaqing-2.0.184.dist-info}/METADATA +1 -1
- kaqing-2.0.184.dist-info/RECORD +244 -0
- adam/commands/app.py +0 -67
- adam/commands/bash.py +0 -87
- adam/commands/cp.py +0 -95
- adam/commands/cql/cql_table_completer.py +0 -8
- adam/commands/cql/cql_utils.py +0 -109
- adam/commands/describe/describe.py +0 -46
- adam/commands/describe/describe_keyspace.py +0 -60
- adam/commands/describe/describe_keyspaces.py +0 -50
- adam/commands/describe/describe_table.py +0 -60
- adam/commands/describe/describe_tables.py +0 -50
- adam/commands/devices.py +0 -89
- adam/commands/postgres/postgres_session.py +0 -240
- adam/commands/postgres/postgres_utils.py +0 -31
- adam/commands/postgres/psql_table_completer.py +0 -11
- adam/commands/reaper/reaper_session.py +0 -159
- adam/commands/show/show_app_actions.py +0 -53
- adam/commands/show/show_repairs.py +0 -47
- adam/k8s_utils/cassandra_clusters.py +0 -35
- adam/sql/sql_utils.py +0 -5
- kaqing-2.0.52.dist-info/RECORD +0 -184
- /adam/commands/{describe → app}/__init__.py +0 -0
- /adam/{k8s_utils → commands/audit}/__init__.py +0 -0
- /adam/{k8s_utils → utils_k8s}/config_maps.py +0 -0
- /adam/{k8s_utils → utils_k8s}/volumes.py +0 -0
- {kaqing-2.0.52.dist-info → kaqing-2.0.184.dist-info}/WHEEL +0 -0
- {kaqing-2.0.52.dist-info → kaqing-2.0.184.dist-info}/entry_points.txt +0 -0
- {kaqing-2.0.52.dist-info → kaqing-2.0.184.dist-info}/top_level.txt +0 -0
adam/commands/nodetool.py
CHANGED
|
@@ -2,12 +2,12 @@ import click
|
|
|
2
2
|
|
|
3
3
|
from adam.commands.command import Command
|
|
4
4
|
from adam.commands.command_helpers import ClusterOrPodCommandHelper
|
|
5
|
+
from adam.commands.cql.utils_cql import cassandra
|
|
5
6
|
from adam.commands.nodetool_commands import NODETOOL_COMMANDS
|
|
6
7
|
from adam.config import Config
|
|
7
|
-
from adam.k8s_utils.cassandra_clusters import CassandraClusters
|
|
8
|
-
from adam.k8s_utils.cassandra_nodes import CassandraNodes
|
|
9
8
|
from adam.repl_state import ReplState, RequiredState
|
|
10
9
|
from adam.utils import log
|
|
10
|
+
from adam.utils_k8s.statefulsets import StatefulSets
|
|
11
11
|
|
|
12
12
|
class NodeTool(Command):
|
|
13
13
|
COMMAND = 'nodetool'
|
|
@@ -31,26 +31,22 @@ class NodeTool(Command):
|
|
|
31
31
|
if not(args := self.args(cmd)):
|
|
32
32
|
return super().run(cmd, state)
|
|
33
33
|
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
user, pw = state.user_pass()
|
|
39
|
-
command = f"nodetool -u {user} -pw {pw} {' '.join(args)}"
|
|
34
|
+
with self.validate(args, state) as (args, state):
|
|
35
|
+
with cassandra(state) as pods:
|
|
36
|
+
pods.nodetool(' '.join(args), status=(args[0] == 'status'))
|
|
40
37
|
|
|
41
|
-
|
|
42
|
-
return CassandraNodes.exec(state.pod, state.namespace, command, show_out=True)
|
|
43
|
-
elif state.sts:
|
|
44
|
-
return CassandraClusters.exec(state.sts, state.namespace, command, action='nodetool', show_out=True)
|
|
38
|
+
return state
|
|
45
39
|
|
|
46
40
|
def completion(self, state: ReplState):
|
|
47
|
-
if
|
|
48
|
-
|
|
41
|
+
if super().completion(state):
|
|
42
|
+
d = {c: {'&': None} for c in NODETOOL_COMMANDS}
|
|
43
|
+
return {NodeTool.COMMAND: {'help': None} | d} | \
|
|
44
|
+
{f'@{p}': {NodeTool.COMMAND: d} for p in StatefulSets.pod_names(state.sts, state.namespace)}
|
|
49
45
|
|
|
50
46
|
return {}
|
|
51
47
|
|
|
52
48
|
def help(self, _: ReplState):
|
|
53
|
-
return f'{NodeTool.COMMAND} <sub-command
|
|
49
|
+
return f'{NodeTool.COMMAND} <sub-command> [&]\t run nodetool with arguments'
|
|
54
50
|
|
|
55
51
|
class NodeToolCommandHelper(click.Command):
|
|
56
52
|
def get_help(self, ctx: click.Context):
|
adam/commands/param_get.py
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
|
+
from adam.commands import validate_args
|
|
1
2
|
from adam.commands.command import Command
|
|
2
3
|
from adam.config import Config
|
|
3
4
|
from adam.repl_state import ReplState
|
|
4
|
-
from adam.utils import
|
|
5
|
+
from adam.utils import tabulize, log, log2
|
|
5
6
|
|
|
6
7
|
class GetParam(Command):
|
|
7
8
|
COMMAND = 'get'
|
|
@@ -22,21 +23,17 @@ class GetParam(Command):
|
|
|
22
23
|
if not(args := self.args(cmd)):
|
|
23
24
|
return super().run(cmd, state)
|
|
24
25
|
|
|
25
|
-
|
|
26
|
+
with self.validate(args, state) as (args, state):
|
|
27
|
+
def msg():
|
|
28
|
+
tabulize(Config().keys(), lambda key: f'{key}\t{Config().get(key, None)}', separator='\t')
|
|
26
29
|
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
+
with validate_args(args, state, msg=msg) as key:
|
|
31
|
+
if v := Config().get(key, None):
|
|
32
|
+
log(v)
|
|
33
|
+
else:
|
|
34
|
+
log2(f'{key} is not set.')
|
|
30
35
|
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
key = args[0]
|
|
34
|
-
if v := Config().get(key, None):
|
|
35
|
-
log(v)
|
|
36
|
-
else:
|
|
37
|
-
log2(f'{key} is not set.')
|
|
38
|
-
|
|
39
|
-
return v if v else state
|
|
36
|
+
return v if v else state
|
|
40
37
|
|
|
41
38
|
def completion(self, _: ReplState):
|
|
42
39
|
return {GetParam.COMMAND: {key: None for key in Config().keys()}}
|
adam/commands/param_set.py
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
from adam.commands import validate_args
|
|
1
2
|
from adam.commands.command import Command
|
|
2
3
|
from adam.config import Config
|
|
3
4
|
from adam.repl_state import ReplState
|
|
@@ -22,20 +23,15 @@ class SetParam(Command):
|
|
|
22
23
|
if not(args := self.args(cmd)):
|
|
23
24
|
return super().run(cmd, state)
|
|
24
25
|
|
|
25
|
-
|
|
26
|
+
with self.validate(args, state) as (args, state):
|
|
27
|
+
with validate_args(args, state, exactly=2, msg=lambda: log2('set <key> <value>')):
|
|
28
|
+
key = args[0]
|
|
29
|
+
value = args[1]
|
|
30
|
+
Config().set(key, value)
|
|
26
31
|
|
|
27
|
-
|
|
28
|
-
log2('set <key> <value>')
|
|
32
|
+
log(Config().get(key, None))
|
|
29
33
|
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
key = args[0]
|
|
33
|
-
value = args[1]
|
|
34
|
-
Config().set(key, value)
|
|
35
|
-
|
|
36
|
-
log(Config().get(key, None))
|
|
37
|
-
|
|
38
|
-
return value
|
|
34
|
+
return value
|
|
39
35
|
|
|
40
36
|
def completion(self, _: ReplState):
|
|
41
37
|
return {SetParam.COMMAND: {key: ({'true': None, 'false': None} if Config().get(key, None) in [True, False] else None) for key in Config().keys()}}
|
|
@@ -1,17 +1,18 @@
|
|
|
1
1
|
import click
|
|
2
2
|
|
|
3
|
+
from adam.commands import extract_trailing_options, validate_args
|
|
3
4
|
from adam.commands.command import Command
|
|
5
|
+
from adam.commands.intermediate_command import IntermediateCommand
|
|
6
|
+
from adam.commands.postgres.postgres_databases import pg_path
|
|
4
7
|
from adam.commands.postgres.psql_completions import psql_completions
|
|
5
|
-
from adam.commands.postgres.
|
|
8
|
+
from adam.commands.postgres.utils_postgres import pg_table_names, postgres
|
|
6
9
|
from .postgres_ls import PostgresLs
|
|
7
10
|
from .postgres_preview import PostgresPreview
|
|
8
|
-
from .postgres_session import PostgresSession
|
|
9
11
|
from adam.repl_state import ReplState
|
|
10
12
|
from adam.utils import log, log2
|
|
11
13
|
|
|
12
|
-
class Postgres(
|
|
14
|
+
class Postgres(IntermediateCommand):
|
|
13
15
|
COMMAND = 'pg'
|
|
14
|
-
reaper_login = None
|
|
15
16
|
|
|
16
17
|
# the singleton pattern
|
|
17
18
|
def __new__(cls, *args, **kwargs):
|
|
@@ -29,40 +30,28 @@ class Postgres(Command):
|
|
|
29
30
|
if not(args := self.args(cmd)):
|
|
30
31
|
return super().run(cmd, state)
|
|
31
32
|
|
|
32
|
-
|
|
33
|
+
with self.validate(args, state) as (args, state):
|
|
34
|
+
with extract_trailing_options(args, '&') as (args, backgrounded):
|
|
35
|
+
with validate_args(args, state, name='SQL statement') as sql:
|
|
36
|
+
if not state.pg_path:
|
|
37
|
+
if state.in_repl:
|
|
38
|
+
log2('Enter "use <pg-name>" first.')
|
|
39
|
+
else:
|
|
40
|
+
log2('* pg-name is missing.')
|
|
33
41
|
|
|
34
|
-
|
|
35
|
-
if state.in_repl:
|
|
36
|
-
log2('Please use SQL statement. e.g. pg \l')
|
|
37
|
-
else:
|
|
38
|
-
log2('* Command or SQL statements is missing.')
|
|
39
|
-
Command.display_help()
|
|
42
|
+
return state
|
|
40
43
|
|
|
41
|
-
|
|
44
|
+
if state.in_repl:
|
|
45
|
+
with postgres(state) as pod:
|
|
46
|
+
pod.sql(args, backgrounded=backgrounded)
|
|
47
|
+
elif not self.run_subcommand(cmd, state):
|
|
48
|
+
with postgres(state) as pod:
|
|
49
|
+
pod.sql(args, backgrounded=backgrounded)
|
|
42
50
|
|
|
43
|
-
|
|
44
|
-
self.run_sql(state, args)
|
|
45
|
-
else:
|
|
46
|
-
# head with the Chain of Responsibility pattern
|
|
47
|
-
cmds = Command.chain(Postgres.cmd_list())
|
|
48
|
-
if not cmds.run(cmd, state) :
|
|
49
|
-
self.run_sql(state, args)
|
|
50
|
-
|
|
51
|
-
return state
|
|
52
|
-
|
|
53
|
-
def cmd_list():
|
|
54
|
-
return [PostgresLs(), PostgresPreview()]
|
|
51
|
+
return state
|
|
55
52
|
|
|
56
|
-
def
|
|
57
|
-
|
|
58
|
-
if state.in_repl:
|
|
59
|
-
log2('Enter "use <pg-name>" first.')
|
|
60
|
-
else:
|
|
61
|
-
log2('* pg-name is missing.')
|
|
62
|
-
|
|
63
|
-
return state
|
|
64
|
-
|
|
65
|
-
PostgresSession(state.namespace, state.pg_path).run_sql(' '.join(args))
|
|
53
|
+
def cmd_list(self):
|
|
54
|
+
return [PostgresLs(), PostgresPreview(), PostgresPg()]
|
|
66
55
|
|
|
67
56
|
def completion(self, state: ReplState):
|
|
68
57
|
if state.device != state.P:
|
|
@@ -70,15 +59,15 @@ class Postgres(Command):
|
|
|
70
59
|
return {}
|
|
71
60
|
|
|
72
61
|
leaf = {}
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
62
|
+
with pg_path(state) as (host, database):
|
|
63
|
+
if database:
|
|
64
|
+
if pg_table_names(state):
|
|
65
|
+
leaf = psql_completions(state)
|
|
66
|
+
elif host:
|
|
67
|
+
leaf = {
|
|
68
|
+
'\h': None,
|
|
69
|
+
'\l': None,
|
|
70
|
+
}
|
|
82
71
|
|
|
83
72
|
if state.pg_path:
|
|
84
73
|
return super().completion(state, leaf) | leaf
|
|
@@ -86,12 +75,22 @@ class Postgres(Command):
|
|
|
86
75
|
return {}
|
|
87
76
|
|
|
88
77
|
def help(self, _: ReplState):
|
|
89
|
-
return f'
|
|
78
|
+
return f'<sql-statements> [&]\t run queries on Postgres databases'
|
|
90
79
|
|
|
91
80
|
class PostgresCommandHelper(click.Command):
|
|
92
81
|
def get_help(self, ctx: click.Context):
|
|
93
|
-
|
|
82
|
+
IntermediateCommand.intermediate_help(super().get_help(ctx), Postgres.COMMAND, Postgres().cmd_list(), show_cluster_help=True)
|
|
94
83
|
log('PG-Name: Kubernetes secret for Postgres credentials')
|
|
95
84
|
log(' e.g. stgawsscpsr-c3-c3-k8spg-cs-001')
|
|
96
85
|
log('Database: Postgres database name within a host')
|
|
97
|
-
log(' e.g. stgawsscpsr_c3_c3')
|
|
86
|
+
log(' e.g. stgawsscpsr_c3_c3')
|
|
87
|
+
|
|
88
|
+
# No action body, only for a help entry and auto-completion
|
|
89
|
+
class PostgresPg(Command):
|
|
90
|
+
COMMAND = 'pg'
|
|
91
|
+
|
|
92
|
+
def command(self):
|
|
93
|
+
return PostgresPg.COMMAND
|
|
94
|
+
|
|
95
|
+
def help(self, _: ReplState):
|
|
96
|
+
return f'pg <sql-statements>\t run queries on Postgres databases'
|
|
@@ -0,0 +1,269 @@
|
|
|
1
|
+
import functools
|
|
2
|
+
import re
|
|
3
|
+
import subprocess
|
|
4
|
+
|
|
5
|
+
from adam.config import Config
|
|
6
|
+
from adam.repl_session import ReplSession
|
|
7
|
+
from adam.repl_state import ReplState
|
|
8
|
+
from adam.utils_k8s.kube_context import KubeContext
|
|
9
|
+
from adam.utils_k8s.pods import Pods
|
|
10
|
+
from adam.utils_k8s.secrets import Secrets
|
|
11
|
+
from adam.utils import log2, log_exc
|
|
12
|
+
|
|
13
|
+
class ConnectionDetails:
|
|
14
|
+
def __init__(self, state: ReplState, namespace: str, host: str):
|
|
15
|
+
self.state = state
|
|
16
|
+
self.namespace = namespace
|
|
17
|
+
self.host = host
|
|
18
|
+
|
|
19
|
+
def endpoint(self):
|
|
20
|
+
return PostgresDatabases._connection_property(self.state, 'pg.secret.endpoint-key', 'postgres-db-endpoint', host=self.host)
|
|
21
|
+
|
|
22
|
+
def port(self):
|
|
23
|
+
return PostgresDatabases._connection_property(self.state, 'pg.secret.port-key', 'postgres-db-port', host=self.host)
|
|
24
|
+
|
|
25
|
+
def username(self):
|
|
26
|
+
return PostgresDatabases._connection_property(self.state, 'pg.secret.username-key', 'postgres-admin-username', host=self.host)
|
|
27
|
+
|
|
28
|
+
def password(self):
|
|
29
|
+
return PostgresDatabases._connection_property(self.state, 'pg.secret.password-key', 'postgres-admin-password', host=self.host)
|
|
30
|
+
|
|
31
|
+
class PostgresDatabases:
|
|
32
|
+
def hosts(state: ReplState, namespace: str = None):
|
|
33
|
+
if not namespace:
|
|
34
|
+
namespace = state.namespace
|
|
35
|
+
|
|
36
|
+
return [ConnectionDetails(state, namespace, host) for host in PostgresDatabases.host_names(namespace)]
|
|
37
|
+
|
|
38
|
+
@functools.lru_cache()
|
|
39
|
+
def host_names(namespace: str):
|
|
40
|
+
ss = Secrets.list_secrets(namespace, name_pattern=Config().get('pg.name-pattern', '^{namespace}.*k8spg.*'))
|
|
41
|
+
|
|
42
|
+
def excludes(name: str):
|
|
43
|
+
exs = Config().get('pg.excludes', '.helm., -admin-secret')
|
|
44
|
+
if exs:
|
|
45
|
+
for ex in exs.split(','):
|
|
46
|
+
if ex.strip(' ') in name:
|
|
47
|
+
return True
|
|
48
|
+
|
|
49
|
+
return False
|
|
50
|
+
|
|
51
|
+
return [s for s in ss if not excludes(s)]
|
|
52
|
+
|
|
53
|
+
def databases(state: ReplState, default_owner = False):
|
|
54
|
+
dbs = []
|
|
55
|
+
# List of databases
|
|
56
|
+
# Name | Owner | Encoding | Collate | Ctype | ICU Locale | Locale Provider | Access privileges
|
|
57
|
+
# ---------------------------------------+----------+----------+-------------+-------------+------------+-----------------+-----------------------
|
|
58
|
+
# postgres | postgres | UTF8 | en_US.UTF-8 | en_US.UTF-8 | | libc |
|
|
59
|
+
# stgawsscpsr_c3_c3 | postgres | UTF8 | C | C | | libc |
|
|
60
|
+
# template1 | postgres | UTF8 | en_US.UTF-8 | en_US.UTF-8 | | libc | =c/postgres +
|
|
61
|
+
# | | | | | | | postgres=CTc/postgres
|
|
62
|
+
# (48 rows)
|
|
63
|
+
if r := PostgresDatabases.run_sql(state, '\l', show_out=False):
|
|
64
|
+
s = 0
|
|
65
|
+
for line in r.stdout.split('\n'):
|
|
66
|
+
line: str = line.strip(' \r')
|
|
67
|
+
if s == 0:
|
|
68
|
+
if 'List of databases' in line:
|
|
69
|
+
s = 1
|
|
70
|
+
elif s == 1:
|
|
71
|
+
if 'Name' in line and 'Owner' in line and 'Encoding' in line:
|
|
72
|
+
s = 2
|
|
73
|
+
elif s == 2:
|
|
74
|
+
if line.startswith('---------'):
|
|
75
|
+
s = 3
|
|
76
|
+
elif s == 3:
|
|
77
|
+
groups = re.match(r'^\s*(\S*)\s*\|\s*(\S*)\s*\|.*', line)
|
|
78
|
+
if groups and groups[1] != '|':
|
|
79
|
+
dbs.append({'name': groups[1], 'owner': groups[2]})
|
|
80
|
+
|
|
81
|
+
if default_owner:
|
|
82
|
+
dbs = [db for db in dbs if db['owner'] == PostgresDatabases.default_owner()]
|
|
83
|
+
|
|
84
|
+
return dbs
|
|
85
|
+
|
|
86
|
+
def tables(state: ReplState, default_schema = False):
|
|
87
|
+
dbs = []
|
|
88
|
+
# List of relations
|
|
89
|
+
# Schema | Name | Type | Owner
|
|
90
|
+
# ----------+------------------------------------------------------------+-------+---------------
|
|
91
|
+
# postgres | c3_2_admin_aclpriv | table | postgres
|
|
92
|
+
# postgres | c3_2_admin_aclpriv_a | table | postgres
|
|
93
|
+
if r := PostgresDatabases.run_sql(state, '\dt', show_out=False):
|
|
94
|
+
s = 0
|
|
95
|
+
for line in r.stdout.split('\n'):
|
|
96
|
+
line: str = line.strip(' \r')
|
|
97
|
+
if s == 0:
|
|
98
|
+
if 'List of relations' in line:
|
|
99
|
+
s = 1
|
|
100
|
+
elif s == 1:
|
|
101
|
+
if 'Schema' in line and 'Name' in line and 'Type' in line:
|
|
102
|
+
s = 2
|
|
103
|
+
elif s == 2:
|
|
104
|
+
if line.startswith('---------'):
|
|
105
|
+
s = 3
|
|
106
|
+
elif s == 3:
|
|
107
|
+
groups = re.match(r'^\s*(\S*)\s*\|\s*(\S*)\s*\|.*', line)
|
|
108
|
+
if groups and groups[1] != '|':
|
|
109
|
+
dbs.append({'schema': groups[1], 'name': groups[2]})
|
|
110
|
+
|
|
111
|
+
if default_schema:
|
|
112
|
+
dbs = [db for db in dbs if db["schema"] == PostgresDatabases.default_schema()]
|
|
113
|
+
|
|
114
|
+
return dbs
|
|
115
|
+
|
|
116
|
+
def run_sql(state: ReplState, sql: str, database: str = None, show_out = True, backgrounded = False):
|
|
117
|
+
if not database:
|
|
118
|
+
database = PostgresDatabases.database(state)
|
|
119
|
+
if not database:
|
|
120
|
+
database = PostgresDatabases.default_db()
|
|
121
|
+
|
|
122
|
+
username = PostgresDatabases.username(state)
|
|
123
|
+
password = PostgresDatabases.password(state)
|
|
124
|
+
endpoint = PostgresDatabases.endpoint(state)
|
|
125
|
+
|
|
126
|
+
if KubeContext.in_cluster():
|
|
127
|
+
cmd1 = f'env PGPASSWORD={password} psql -h {endpoint} -p {PostgresDatabases.port()} -U {username} {database} --pset pager=off -c'
|
|
128
|
+
log2(f'{cmd1} "{sql}"')
|
|
129
|
+
# remove double quotes from the sql argument
|
|
130
|
+
cmd = cmd1.split(' ') + [sql]
|
|
131
|
+
|
|
132
|
+
r = subprocess.run(cmd, capture_output=not backgrounded, text=True)
|
|
133
|
+
if show_out:
|
|
134
|
+
log2(r.stdout)
|
|
135
|
+
log2(r.stderr)
|
|
136
|
+
|
|
137
|
+
return r
|
|
138
|
+
else:
|
|
139
|
+
pod_name, container_name = PostgresDatabases.pod_and_container(state.namespace)
|
|
140
|
+
if not pod_name:
|
|
141
|
+
return
|
|
142
|
+
|
|
143
|
+
cmd = f'psql -h {endpoint} -p {PostgresDatabases.port(state)} -U {username} {database} --pset pager=off -c "{sql}"'
|
|
144
|
+
env_prefix = f'PGPASSWORD="{password}"'
|
|
145
|
+
|
|
146
|
+
r = Pods.exec(pod_name, container_name, state.namespace, cmd, show_out=show_out, backgrounded=backgrounded, env_prefix=env_prefix)
|
|
147
|
+
if r and Config().get('repl.history.push-cat-remote-log-file', True):
|
|
148
|
+
if r.log_file and ReplSession().prompt_session:
|
|
149
|
+
ReplSession().prompt_session.history.append_string(f'@{r.pod} cat {r.log_file}')
|
|
150
|
+
|
|
151
|
+
return r
|
|
152
|
+
|
|
153
|
+
def pod_and_container(namespace: str):
|
|
154
|
+
container_name = Config().get('pg.agent.name', 'ops-pg-agent')
|
|
155
|
+
|
|
156
|
+
if Config().get('pg.agent.just-in-time', False):
|
|
157
|
+
if not PostgresDatabases.deploy_pg_agent(container_name, namespace):
|
|
158
|
+
return None
|
|
159
|
+
|
|
160
|
+
pod_name = container_name
|
|
161
|
+
try:
|
|
162
|
+
# try with dedicated pg agent pod name configured
|
|
163
|
+
Pods.get(namespace, container_name)
|
|
164
|
+
except:
|
|
165
|
+
try:
|
|
166
|
+
# try with the ops pod
|
|
167
|
+
container_name = Config().get('pod.name', 'ops')
|
|
168
|
+
pod_name = Pods.get_with_selector(namespace, label_selector = Config().get('pod.label-selector', 'run=ops')).metadata.name
|
|
169
|
+
except:
|
|
170
|
+
log2(f"Could not locate {container_name} pod.")
|
|
171
|
+
return None
|
|
172
|
+
|
|
173
|
+
return pod_name, container_name
|
|
174
|
+
|
|
175
|
+
def deploy_pg_agent(pod_name: str, namespace: str) -> str:
|
|
176
|
+
image = Config().get('pg.agent.image', 'seanahnsf/kaqing')
|
|
177
|
+
timeout = Config().get('pg.agent.timeout', 3600)
|
|
178
|
+
try:
|
|
179
|
+
Pods.create(namespace, pod_name, image, ['sleep', f'{timeout}'], env={'NAMESPACE': namespace}, sa_name='c3')
|
|
180
|
+
except Exception as e:
|
|
181
|
+
if e.status == 409:
|
|
182
|
+
if Pods.completed(namespace, pod_name):
|
|
183
|
+
with log_exc(lambda e2: "Exception when calling BatchV1Api->create_pod: %s\n" % e2):
|
|
184
|
+
Pods.delete(pod_name, namespace)
|
|
185
|
+
Pods.create(namespace, pod_name, image, ['sleep', f'{timeout}'], env={'NAMESPACE': namespace}, sa_name='c3')
|
|
186
|
+
|
|
187
|
+
return
|
|
188
|
+
else:
|
|
189
|
+
log2("Exception when calling BatchV1Api->create_pod: %s\n" % e)
|
|
190
|
+
|
|
191
|
+
return
|
|
192
|
+
|
|
193
|
+
Pods.wait_for_running(namespace, pod_name)
|
|
194
|
+
|
|
195
|
+
return pod_name
|
|
196
|
+
|
|
197
|
+
def undeploy_pg_agent(pod_name: str, namespace: str):
|
|
198
|
+
Pods.delete(pod_name, namespace, grace_period_seconds=0)
|
|
199
|
+
|
|
200
|
+
def endpoint(state: ReplState):
|
|
201
|
+
return PostgresDatabases._connection_property(state, 'pg.secret.endpoint-key', 'postgres-db-endpoint')
|
|
202
|
+
|
|
203
|
+
def port(state: ReplState):
|
|
204
|
+
return PostgresDatabases._connection_property(state, 'pg.secret.port-key', 'postgres-db-port')
|
|
205
|
+
|
|
206
|
+
def username(state: ReplState):
|
|
207
|
+
return PostgresDatabases._connection_property(state, 'pg.secret.username-key', 'postgres-admin-username')
|
|
208
|
+
|
|
209
|
+
def password(state: ReplState):
|
|
210
|
+
return PostgresDatabases._connection_property(state, 'pg.secret.password-key', 'postgres-admin-password')
|
|
211
|
+
|
|
212
|
+
def _connection_property(state: ReplState, config_key: str, default: str, host: str = None, database: str = None):
|
|
213
|
+
with pg_path(state, host=host, database=database) as (host, _):
|
|
214
|
+
if not (conn := PostgresDatabases.conn_details(state.namespace, host)):
|
|
215
|
+
return ''
|
|
216
|
+
|
|
217
|
+
key = Config().get(config_key, default)
|
|
218
|
+
return conn[key] if key in conn else ''
|
|
219
|
+
|
|
220
|
+
def default_db():
|
|
221
|
+
return Config().get('pg.default-db', 'postgres')
|
|
222
|
+
|
|
223
|
+
def default_owner():
|
|
224
|
+
return Config().get('pg.default-owner', 'postgres')
|
|
225
|
+
|
|
226
|
+
def default_schema():
|
|
227
|
+
return Config().get('pg.default-schema', 'postgres')
|
|
228
|
+
|
|
229
|
+
def host(state: ReplState):
|
|
230
|
+
if not state.pg_path:
|
|
231
|
+
return None
|
|
232
|
+
|
|
233
|
+
return state.pg_path.split('/')[0]
|
|
234
|
+
|
|
235
|
+
def database(state: ReplState):
|
|
236
|
+
if not state.pg_path:
|
|
237
|
+
return None
|
|
238
|
+
|
|
239
|
+
tokens = state.pg_path.split('/')
|
|
240
|
+
if len(tokens) > 1:
|
|
241
|
+
return tokens[1]
|
|
242
|
+
|
|
243
|
+
return None
|
|
244
|
+
|
|
245
|
+
@functools.lru_cache()
|
|
246
|
+
def conn_details(namespace: str, host: str):
|
|
247
|
+
return Secrets.get_data(namespace, host)
|
|
248
|
+
|
|
249
|
+
class PostgresPathHandler:
|
|
250
|
+
def __init__(self, state: ReplState, host: str = None, database: str = None):
|
|
251
|
+
self.state = state
|
|
252
|
+
self.host = host
|
|
253
|
+
self.database = database
|
|
254
|
+
|
|
255
|
+
def __enter__(self) -> tuple[str, str]:
|
|
256
|
+
if self.state and self.state.pg_path:
|
|
257
|
+
host_n_db = self.state.pg_path.split('/')
|
|
258
|
+
if not self.host:
|
|
259
|
+
self.host = host_n_db[0]
|
|
260
|
+
if not self.database and len(host_n_db) > 1:
|
|
261
|
+
self.database = host_n_db[1]
|
|
262
|
+
|
|
263
|
+
return self.host, self.database
|
|
264
|
+
|
|
265
|
+
def __exit__(self, exc_type, exc_val, exc_tb):
|
|
266
|
+
return False
|
|
267
|
+
|
|
268
|
+
def pg_path(state: ReplState, host: str = None, database: str = None):
|
|
269
|
+
return PostgresPathHandler(state, host=host, database=database)
|
|
@@ -4,7 +4,6 @@ from adam.repl_state import ReplState, RequiredState
|
|
|
4
4
|
|
|
5
5
|
class PostgresLs(Command):
|
|
6
6
|
COMMAND = 'pg ls'
|
|
7
|
-
reaper_login = None
|
|
8
7
|
|
|
9
8
|
# the singleton pattern
|
|
10
9
|
def __new__(cls, *args, **kwargs):
|
|
@@ -25,15 +24,12 @@ class PostgresLs(Command):
|
|
|
25
24
|
if not(args := self.args(cmd)):
|
|
26
25
|
return super().run(cmd, state)
|
|
27
26
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
return state
|
|
31
|
-
|
|
32
|
-
state.device = ReplState.P
|
|
27
|
+
with self.validate(args, state) as (args, state):
|
|
28
|
+
state.device = ReplState.P
|
|
33
29
|
|
|
34
|
-
|
|
30
|
+
Ls().run('ls', state)
|
|
35
31
|
|
|
36
|
-
|
|
32
|
+
return state
|
|
37
33
|
|
|
38
34
|
def completion(self, state: ReplState):
|
|
39
35
|
if state.sts:
|
|
@@ -4,7 +4,6 @@ from adam.repl_state import ReplState, RequiredState
|
|
|
4
4
|
|
|
5
5
|
class PostgresPreview(Command):
|
|
6
6
|
COMMAND = 'pg preview'
|
|
7
|
-
reaper_login = None
|
|
8
7
|
|
|
9
8
|
# the singleton pattern
|
|
10
9
|
def __new__(cls, *args, **kwargs):
|
|
@@ -19,21 +18,18 @@ class PostgresPreview(Command):
|
|
|
19
18
|
return PostgresPreview.COMMAND
|
|
20
19
|
|
|
21
20
|
def required(self):
|
|
22
|
-
return RequiredState.
|
|
21
|
+
return RequiredState.PG_DATABASE
|
|
23
22
|
|
|
24
23
|
def run(self, cmd: str, state: ReplState):
|
|
25
24
|
if not(args := self.args(cmd)):
|
|
26
25
|
return super().run(cmd, state)
|
|
27
26
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
return state
|
|
31
|
-
|
|
32
|
-
state.device = ReplState.P
|
|
27
|
+
with self.validate(args, state) as (args, state):
|
|
28
|
+
state.device = ReplState.P
|
|
33
29
|
|
|
34
|
-
|
|
30
|
+
PreviewTable().run(f'preview {" ".join(args)}', state)
|
|
35
31
|
|
|
36
|
-
|
|
32
|
+
return state
|
|
37
33
|
|
|
38
34
|
def completion(self, state: ReplState):
|
|
39
35
|
if state.sts:
|
|
@@ -1,10 +1,11 @@
|
|
|
1
|
-
from adam.commands.postgres.
|
|
1
|
+
from adam.commands.postgres.utils_postgres import pg_table_names
|
|
2
|
+
from adam.repl_state import ReplState
|
|
2
3
|
from adam.sql.sql_completer import SqlCompleter
|
|
3
4
|
|
|
4
|
-
def psql_completions(
|
|
5
|
+
def psql_completions(state: ReplState):
|
|
5
6
|
return {
|
|
6
7
|
'\h': None,
|
|
7
8
|
'\d': None,
|
|
8
9
|
'\dt': None,
|
|
9
10
|
'\du': None
|
|
10
|
-
} | SqlCompleter
|
|
11
|
+
} | SqlCompleter(lambda: pg_table_names(state)).completions_for_nesting()
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import functools
|
|
2
|
+
|
|
3
|
+
from adam.commands.postgres.postgres_databases import PostgresDatabases
|
|
4
|
+
from adam.repl_state import ReplState
|
|
5
|
+
from adam.utils import log2, wait_log
|
|
6
|
+
from adam.utils_k8s.pods import Pods
|
|
7
|
+
|
|
8
|
+
TestPG = [False]
|
|
9
|
+
|
|
10
|
+
def pg_database_names(state: ReplState):
|
|
11
|
+
# cache on pg_path
|
|
12
|
+
return _pg_database_names(state, state.pg_path)
|
|
13
|
+
|
|
14
|
+
@functools.lru_cache()
|
|
15
|
+
def _pg_database_names(state: ReplState, pg_path: str):
|
|
16
|
+
if TestPG[0]:
|
|
17
|
+
return ['azops88_c3ai_c3']
|
|
18
|
+
|
|
19
|
+
wait_log('Inspecting Postgres Databases...')
|
|
20
|
+
|
|
21
|
+
return [db['name'] for db in PostgresDatabases.databases(state, default_owner=True)]
|
|
22
|
+
|
|
23
|
+
def pg_table_names(state: ReplState):
|
|
24
|
+
# cache on pg_path
|
|
25
|
+
return _pg_table_names(state, state.pg_path)
|
|
26
|
+
|
|
27
|
+
@functools.lru_cache()
|
|
28
|
+
def _pg_table_names(state: ReplState, pg_path: str):
|
|
29
|
+
if TestPG[0]:
|
|
30
|
+
return ['C3_2_XYZ1']
|
|
31
|
+
|
|
32
|
+
wait_log('Inspecting Postgres Database...')
|
|
33
|
+
return [table['name'] for table in PostgresDatabases.tables(state, default_schema=True)]
|
|
34
|
+
|
|
35
|
+
class PostgresPodService:
|
|
36
|
+
def __init__(self, handler: 'PostgresExecHandler'):
|
|
37
|
+
self.handler = handler
|
|
38
|
+
|
|
39
|
+
def exec(self, command: str, show_out=True):
|
|
40
|
+
state = self.handler.state
|
|
41
|
+
|
|
42
|
+
pod, container = PostgresDatabases.pod_and_container(state.namespace)
|
|
43
|
+
if not pod:
|
|
44
|
+
log2('Cannot locate postgres agent or ops pod.')
|
|
45
|
+
return state
|
|
46
|
+
|
|
47
|
+
return Pods.exec(pod, container, state.namespace, command, show_out=show_out)
|
|
48
|
+
|
|
49
|
+
def sql(self, args: list[str], backgrounded=False):
|
|
50
|
+
state = self.handler.state
|
|
51
|
+
|
|
52
|
+
query = args
|
|
53
|
+
if isinstance(args, list):
|
|
54
|
+
query = ' '.join(args)
|
|
55
|
+
|
|
56
|
+
PostgresDatabases.run_sql(state, query, backgrounded=backgrounded)
|
|
57
|
+
|
|
58
|
+
class PostgresExecHandler:
|
|
59
|
+
def __init__(self, state: ReplState, backgrounded=False):
|
|
60
|
+
self.state = state
|
|
61
|
+
self.backgrounded = backgrounded
|
|
62
|
+
|
|
63
|
+
def __enter__(self):
|
|
64
|
+
return PostgresPodService(self)
|
|
65
|
+
|
|
66
|
+
def __exit__(self, exc_type, exc_val, exc_tb):
|
|
67
|
+
return False
|
|
68
|
+
|
|
69
|
+
def postgres(state: ReplState, backgrounded=False):
|
|
70
|
+
return PostgresExecHandler(state, backgrounded=backgrounded)
|