kaqing 2.0.14__py3-none-any.whl → 2.0.189__py3-none-any.whl

adam/commands/medusa/utils_medusa.py

@@ -0,0 +1,15 @@
+from adam.config import Config
+from adam.repl_state import ReplState
+from adam.utils_k8s.custom_resources import CustomResources
+from adam.utils_k8s.statefulsets import StatefulSets
+
+def medusa_backup_names(state: ReplState, warm=False):
+    if warm and (auto := Config().get('medusa.restore-auto-complete', 'off')) in ['off', 'jit', 'lazy']:
+        return {}
+
+    ns = state.namespace
+    dc: str = StatefulSets.get_datacenter(state.sts, ns)
+    if not dc:
+        return {}
+
+    return {id: None for id in [f"{x['metadata']['name']}" for x in CustomResources.medusa_show_backupjobs(dc, ns)]}

adam/commands/nodetool.py

@@ -2,10 +2,10 @@ import click
 
 from adam.commands.command import Command
 from adam.commands.command_helpers import ClusterOrPodCommandHelper
+from adam.commands.cql.utils_cql import cassandra
+from adam.commands.devices.devices import Devices
 from adam.commands.nodetool_commands import NODETOOL_COMMANDS
 from adam.config import Config
-from adam.k8s_utils.cassandra_clusters import CassandraClusters
-from adam.k8s_utils.cassandra_nodes import CassandraNodes
 from adam.repl_state import ReplState, RequiredState
 from adam.utils import log
 
@@ -31,26 +31,17 @@ class NodeTool(Command):
         if not(args := self.args(cmd)):
             return super().run(cmd, state)
 
-        state, args = self.apply_state(args, state)
-        if not self.validate_state(state):
-            return state
-
-        user, pw = state.user_pass()
-        command = f"nodetool -u {user} -pw {pw} {' '.join(args)}"
+        with self.validate(args, state) as (args, state):
+            with cassandra(state) as pods:
+                pods.nodetool(' '.join(args), status=(args[0] == 'status'))
 
-        if state.pod:
-            return CassandraNodes.exec(state.pod, state.namespace, command, show_out=True)
-        elif state.sts:
-            return CassandraClusters.exec(state.sts, state.namespace, command, action='nodetool', show_out=True)
+            return state
 
     def completion(self, state: ReplState):
-        if state.pod or state.sts:
-            return {NodeTool.COMMAND: {'help': None} | {c: None for c in NODETOOL_COMMANDS}}
-
-        return {}
+        return super().completion(state, {c: {'&': None} for c in NODETOOL_COMMANDS}, pods=Devices.device(state).pods(state, '-'))
 
     def help(self, _: ReplState):
-        return f'{NodeTool.COMMAND} <sub-command>\t run nodetool with arguments'
+        return f'{NodeTool.COMMAND} <sub-command> [&]\t run nodetool with arguments'
 
 class NodeToolCommandHelper(click.Command):
     def get_help(self, ctx: click.Context):

adam/commands/param_get.py

@@ -1,7 +1,8 @@
+from adam.commands import validate_args
 from adam.commands.command import Command
 from adam.config import Config
 from adam.repl_state import ReplState
-from adam.utils import lines_to_tabular, log, log2
+from adam.utils import tabulize, log, log2
 
 class GetParam(Command):
     COMMAND = 'get'
@@ -22,21 +23,17 @@ class GetParam(Command):
         if not(args := self.args(cmd)):
             return super().run(cmd, state)
 
-        state, args = self.apply_state(args, state)
+        with self.validate(args, state) as (args, state):
+            def msg():
+                tabulize(Config().keys(), lambda key: f'{key}\t{Config().get(key, None)}', separator='\t')
 
-        if len(args) < 1:
-            lines = [f'{key}\t{Config().get(key, None)}' for key in Config().keys()]
-            log(lines_to_tabular(lines, separator='\t'))
+            with validate_args(args, state, msg=msg) as key:
+                if v := Config().get(key, None):
+                    log(v)
+                else:
+                    log2(f'{key} is not set.')
 
-            return state
-
-        key = args[0]
-        if v := Config().get(key, None):
-            log(v)
-        else:
-            log2(f'{key} is not set.')
-
-        return v if v else state
+                return v if v else state
 
     def completion(self, _: ReplState):
         return {GetParam.COMMAND: {key: None for key in Config().keys()}}

adam/commands/param_set.py

@@ -1,3 +1,4 @@
+from adam.commands import validate_args
 from adam.commands.command import Command
 from adam.config import Config
 from adam.repl_state import ReplState
@@ -22,23 +23,18 @@ class SetParam(Command):
         if not(args := self.args(cmd)):
             return super().run(cmd, state)
 
-        state, args = self.apply_state(args, state)
+        with self.validate(args, state) as (args, state):
+            with validate_args(args, state, exactly=2, msg=lambda: log2('set <key> <value>')):
+                key = args[0]
+                value = args[1]
+                Config().set(key, value)
 
-        if len(args) < 2:
-            log2('set <key> <value>')
+                log(Config().get(key, None))
 
-            return 'invalid args'
-
-        key = args[0]
-        value = args[1]
-        Config().set(key, value)
-
-        log(Config().get(key, None))
-
-        return value
+                return value
 
     def completion(self, _: ReplState):
-        return {SetParam.COMMAND: {key: None for key in Config().keys()}}
+        return {SetParam.COMMAND: {key: ({'true': None, 'false': None} if Config().get(key, None) in [True, False] else None) for key in Config().keys()}}
 
     def help(self, _: ReplState):
         return f"{SetParam.COMMAND} <key> <value>\t sets a Kaqing parameter to a different value"

adam/commands/postgres/completions_p.py

@@ -0,0 +1,22 @@
+from adam.commands.postgres.postgres_databases import PostgresDatabases
+from adam.commands.postgres.utils_postgres import pg_table_names
+from adam.repl_state import ReplState
+from adam.sql.lark_completer import LarkCompleter
+
+def completions_p(state: ReplState):
+    return {
+        '\h': None,
+        '\d': None,
+        '\dt': None,
+        '\du': None
+    } | LarkCompleter(expandables={
+        'tables': lambda x: pg_table_names(state),
+        'columns': lambda x: ['id'],
+        'hosts': lambda x: ['@' + PostgresDatabases.pod_and_container(state.namespace)[0]],
+    }, variant=ReplState.P).completions_for_nesting()
+
+def psql0_completions(state: ReplState):
+    return {
+        '\h': None,
+        '\l': None,
+    }

adam/commands/postgres/postgres.py

@@ -1,16 +1,18 @@
 import click
 
+from adam.commands import extract_trailing_options, validate_args
 from adam.commands.command import Command
-from adam.commands.command_helpers import ClusterCommandHelper
+from adam.commands.intermediate_command import IntermediateCommand
+from adam.commands.postgres.postgres_databases import pg_path
+from adam.commands.postgres.completions_p import psql0_completions, completions_p
+from adam.commands.postgres.utils_postgres import pg_table_names, postgres
 from .postgres_ls import PostgresLs
 from .postgres_preview import PostgresPreview
-from .postgres_session import PostgresSession
 from adam.repl_state import ReplState
-from adam.utils import lines_to_tabular, log, log2
+from adam.utils import log, log2, log_timing
 
-class Postgres(Command):
+class Postgres(IntermediateCommand):
     COMMAND = 'pg'
-    reaper_login = None
 
     # the singleton pattern
     def __new__(cls, *args, **kwargs):
@@ -28,86 +30,60 @@ class Postgres(Command):
         if not(args := self.args(cmd)):
             return super().run(cmd, state)
 
-        state, args = self.apply_state(args, state)
+        with self.validate(args, state) as (args, state):
+            with extract_trailing_options(args, '&') as (args, backgrounded):
+                with validate_args(args, state, name='SQL statement') as sql:
+                    if not state.pg_path:
+                        if state.in_repl:
+                            log2('Enter "use <pg-name>" first.')
+                        else:
+                            log2('* pg-name is missing.')
 
-        if state.in_repl:
-            if not args:
-                log2('Please use SQL statement. e.g. pg \l')
+                        return state
 
-                return 'command-missing'
-            else:
-                self.run_sql(state, args)
-        else:
-            if not args:
-                log2('* Command or SQL statements is missing.')
-                Command.display_help()
+                    if state.in_repl:
+                        with postgres(state) as pod:
+                            pod.sql(args, backgrounded=backgrounded)
+                    elif not self.run_subcommand(cmd, state):
+                        with postgres(state) as pod:
+                            pod.sql(args, backgrounded=backgrounded)
 
-                return 'command-missing'
-            else:
-                # head with the Chain of Responsibility pattern
-                cmds = Command.chain(Postgres.cmd_list())
-                if not cmds.run(cmd, state) :
-                    if not args:
-                        log2('* Command or SQL statements is missing.')
-                        Command.display_help()
+                    return state
 
-                        return 'command-missing'
-                    else:
-                        self.run_sql(state, args)
-
-        return state
-
-    def cmd_list():
-        return [PostgresLs(), PostgresPreview()]
-
-    def run_sql(self, state: ReplState, args: list[str]):
-        if not state.pg_path:
-            if state.in_repl:
-                log2('Enter "use <pg-name>" first.')
-            else:
-                log2('* pg-name is missing.')
-
-            return state
-
-        PostgresSession(state.namespace, state.pg_path).run_sql(' '.join(args))
+    def cmd_list(self):
+        return [PostgresLs(), PostgresPreview(), PostgresPg()]
 
     def completion(self, state: ReplState):
-        leaf = {}
-        if PostgresSession(state.namespace, state.pg_path).db:
-            leaf = {
-                '\h': None,
-                '\d': None,
-                '\dt': None,
-                '\du': None,
-                'delete': {'from': None},
-                'insert': {'into': None},
-                'select': None,
-                'update': None,
-            }
-        elif state.pg_path:
-            leaf = {
-                '\h': None,
-                '\l': None,
-            }
-
-        if state.pg_path:
-            return super().completion(state, leaf) | leaf
-        else:
+        if state.device != state.P:
             return {}
 
+        with pg_path(state) as (host, database):
+            if database:
+                if pg_table_names(state):
+                    with log_timing('psql_completions'):
+                        return completions_p(state)
+            elif host:
+                return psql0_completions(state)
+
+        return {}
+
     def help(self, _: ReplState):
-        return f'[{Postgres.COMMAND}] <sql-statements>\t run psql with queries'
+        return f'<sql-statements> [&]\t run queries on Postgres databases'
 
 class PostgresCommandHelper(click.Command):
     def get_help(self, ctx: click.Context):
-        log(super().get_help(ctx))
-        log()
-        log('Sub-Commands:')
-
-        log(lines_to_tabular([c.help(ReplState()).replace(f'{Postgres.COMMAND} ', '  ', 1) for c in Postgres.cmd_list()], separator='\t'))
-        log()
-        ClusterCommandHelper.cluster_help()
+        IntermediateCommand.intermediate_help(super().get_help(ctx), Postgres.COMMAND, Postgres().cmd_list(), show_cluster_help=True)
         log('PG-Name:  Kubernetes secret for Postgres credentials')
         log('          e.g. stgawsscpsr-c3-c3-k8spg-cs-001')
         log('Database: Postgres database name within a host')
-        log('          e.g. stgawsscpsr_c3_c3')
+        log('          e.g. stgawsscpsr_c3_c3')
+
+# No action body, only for a help entry and auto-completion
+class PostgresPg(Command):
+    COMMAND = 'pg'
+
+    def command(self):
+        return PostgresPg.COMMAND
+
+    def help(self, _: ReplState):
+        return f'pg <sql-statements>\t run queries on Postgres databases'

adam/commands/postgres/postgres_databases.py

@@ -0,0 +1,270 @@
+import functools
+import re
+import subprocess
+
+from adam.config import Config
+from adam.repl_session import ReplSession
+from adam.repl_state import ReplState
+from adam.utils_k8s.kube_context import KubeContext
+from adam.utils_k8s.pods import Pods
+from adam.utils_k8s.secrets import Secrets
+from adam.utils import log2, log_exc
+
+class ConnectionDetails:
+    def __init__(self, state: ReplState, namespace: str, host: str):
+        self.state = state
+        self.namespace = namespace
+        self.host = host
+
+    def endpoint(self):
+        return PostgresDatabases._connection_property(self.state, 'pg.secret.endpoint-key', 'postgres-db-endpoint', host=self.host)
+
+    def port(self):
+        return PostgresDatabases._connection_property(self.state, 'pg.secret.port-key', 'postgres-db-port', host=self.host)
+
+    def username(self):
+        return PostgresDatabases._connection_property(self.state, 'pg.secret.username-key', 'postgres-admin-username', host=self.host)
+
+    def password(self):
+        return PostgresDatabases._connection_property(self.state, 'pg.secret.password-key', 'postgres-admin-password', host=self.host)
+
+class PostgresDatabases:
+    def hosts(state: ReplState, namespace: str = None):
+        if not namespace:
+            namespace = state.namespace
+
+        return [ConnectionDetails(state, namespace, host) for host in PostgresDatabases.host_names(namespace)]
+
+    @functools.lru_cache()
+    def host_names(namespace: str):
+        ss = Secrets.list_secrets(namespace, name_pattern=Config().get('pg.name-pattern', '^{namespace}.*k8spg.*'))
+
+        def excludes(name: str):
+            exs = Config().get('pg.excludes', '.helm., -admin-secret')
+            if exs:
+                for ex in exs.split(','):
+                    if ex.strip(' ') in name:
+                        return True
+
+            return False
+
+        return [s for s in ss if not excludes(s)]
+
+    def databases(state: ReplState, default_owner = False):
+        dbs = []
+        #  List of databases
+        #                  Name                  |  Owner   | Encoding |   Collate   |    Ctype    | ICU Locale | Locale Provider |   Access privileges
+        # ---------------------------------------+----------+----------+-------------+-------------+------------+-----------------+-----------------------
+        #  postgres                              | postgres | UTF8     | en_US.UTF-8 | en_US.UTF-8 |            | libc            |
+        #  stgawsscpsr_c3_c3                     | postgres | UTF8     | C           | C           |            | libc            |
+        #  template1                             | postgres | UTF8     | en_US.UTF-8 | en_US.UTF-8 |            | libc            | =c/postgres          +
+        #                                        |          |          |             |             |            |                 | postgres=CTc/postgres
+        # (48 rows)
+        if r := PostgresDatabases.run_sql(state, '\l', show_out=False):
+            s = 0
+            for line in r.stdout.split('\n'):
+                line: str = line.strip(' \r')
+                if s == 0:
+                    if 'List of databases' in line:
+                        s = 1
+                elif s == 1:
+                    if 'Name' in line and 'Owner' in line and 'Encoding' in line:
+                        s = 2
+                elif s == 2:
+                    if line.startswith('---------'):
+                        s = 3
+                elif s == 3:
+                    groups = re.match(r'^\s*(\S*)\s*\|\s*(\S*)\s*\|.*', line)
+                    if groups and groups[1] != '|':
+                        dbs.append({'name': groups[1], 'owner': groups[2]})
+
+        if default_owner:
+            dbs = [db for db in dbs if db['owner'] == PostgresDatabases.default_owner()]
+
+        return dbs
+
+    def tables(state: ReplState, default_schema = False):
+        dbs = []
+        #                                            List of relations
+        #   Schema  |                            Name                            | Type  |     Owner
+        # ----------+------------------------------------------------------------+-------+---------------
+        #  postgres | c3_2_admin_aclpriv                                         | table | postgres
+        #  postgres | c3_2_admin_aclpriv_a                                       | table | postgres
+        if r := PostgresDatabases.run_sql(state, '\dt', show_out=False):
+            s = 0
+            for line in r.stdout.split('\n'):
+                line: str = line.strip(' \r')
+                if s == 0:
+                    if 'List of relations' in line:
+                        s = 1
+                elif s == 1:
+                    if 'Schema' in line and 'Name' in line and 'Type' in line:
+                        s = 2
+                elif s == 2:
+                    if line.startswith('---------'):
+                        s = 3
+                elif s == 3:
+                    groups = re.match(r'^\s*(\S*)\s*\|\s*(\S*)\s*\|.*', line)
+                    if groups and groups[1] != '|':
+                        dbs.append({'schema': groups[1], 'name': groups[2]})
+
+        if default_schema:
+            dbs = [db for db in dbs if db["schema"] == PostgresDatabases.default_schema()]
+
+        return dbs
+
+    def run_sql(state: ReplState, sql: str, database: str = None, show_out = True, backgrounded = False):
+        if not database:
+            database = PostgresDatabases.database(state)
+        if not database:
+            database = PostgresDatabases.default_db()
+
+        username = PostgresDatabases.username(state)
+        password = PostgresDatabases.password(state)
+        endpoint = PostgresDatabases.endpoint(state)
+
+        if KubeContext.in_cluster():
+            cmd1 = f'env PGPASSWORD={password} psql -h {endpoint} -p {PostgresDatabases.port()} -U {username} {database} --pset pager=off -c'
+            log2(f'{cmd1} "{sql}"')
+            # remove double quotes from the sql argument
+            cmd = cmd1.split(' ') + [sql]
+
+            r = subprocess.run(cmd, capture_output=not backgrounded, text=True)
+            if show_out:
+                log2(r.stdout)
+                log2(r.stderr)
+
+            return r
+        else:
+            pod_name, container_name = PostgresDatabases.pod_and_container(state.namespace)
+            if not pod_name:
+                return
+
+            cmd = f'psql -h {endpoint} -p {PostgresDatabases.port(state)} -U {username} {database} --pset pager=off -c "{sql}"'
+            env_prefix = f'PGPASSWORD="{password}"'
+
+            r = Pods.exec(pod_name, container_name, state.namespace, cmd, show_out=show_out, backgrounded=backgrounded, env_prefix=env_prefix)
+            if r and Config().get('repl.history.push-cat-remote-log-file', True):
+                if r.log_file and ReplSession().prompt_session:
+                    ReplSession().prompt_session.history.append_string(f'@{r.pod} cat {r.log_file}')
+
+            return r
+
+    @functools.lru_cache()
+    def pod_and_container(namespace: str):
+        container_name = Config().get('pg.agent.name', 'ops-pg-agent')
+
+        if Config().get('pg.agent.just-in-time', False):
+            if not PostgresDatabases.deploy_pg_agent(container_name, namespace):
+                return None
+
+        pod_name = container_name
+        try:
+            # try with dedicated pg agent pod name configured
+            Pods.get(namespace, container_name)
+        except:
+            try:
+                # try with the ops pod
+                container_name = Config().get('pod.name', 'ops')
+                pod_name = Pods.get_with_selector(namespace, label_selector = Config().get('pod.label-selector', 'run=ops')).metadata.name
+            except:
+                log2(f"Could not locate {container_name} pod.")
+                return None
+
+        return pod_name, container_name
+
+    def deploy_pg_agent(pod_name: str, namespace: str) -> str:
+        image = Config().get('pg.agent.image', 'seanahnsf/kaqing')
+        timeout = Config().get('pg.agent.timeout', 3600)
+        try:
+            Pods.create(namespace, pod_name, image, ['sleep', f'{timeout}'], env={'NAMESPACE': namespace}, sa_name='c3')
+        except Exception as e:
+            if e.status == 409:
+                if Pods.completed(namespace, pod_name):
+                    with log_exc(lambda e2: "Exception when calling BatchV1Api->create_pod: %s\n" % e2):
+                        Pods.delete(pod_name, namespace)
+                        Pods.create(namespace, pod_name, image, ['sleep', f'{timeout}'], env={'NAMESPACE': namespace}, sa_name='c3')
+
+                        return
+            else:
+                log2("Exception when calling BatchV1Api->create_pod: %s\n" % e)
+
+                return
+
+        Pods.wait_for_running(namespace, pod_name)
+
+        return pod_name
+
+    def undeploy_pg_agent(pod_name: str, namespace: str):
+        Pods.delete(pod_name, namespace, grace_period_seconds=0)
+
+    def endpoint(state: ReplState):
+        return PostgresDatabases._connection_property(state, 'pg.secret.endpoint-key', 'postgres-db-endpoint')
+
+    def port(state: ReplState):
+        return PostgresDatabases._connection_property(state, 'pg.secret.port-key', 'postgres-db-port')
+
+    def username(state: ReplState):
+        return PostgresDatabases._connection_property(state, 'pg.secret.username-key', 'postgres-admin-username')
+
+    def password(state: ReplState):
+        return PostgresDatabases._connection_property(state, 'pg.secret.password-key', 'postgres-admin-password')
+
+    def _connection_property(state: ReplState, config_key: str, default: str, host: str = None, database: str = None):
+        with pg_path(state, host=host, database=database) as (host, _):
+            if not (conn := PostgresDatabases.conn_details(state.namespace, host)):
+                return ''
+
+            key = Config().get(config_key, default)
+            return conn[key] if key in conn else ''
+
+    def default_db():
+        return Config().get('pg.default-db', 'postgres')
+
+    def default_owner():
+        return Config().get('pg.default-owner', 'postgres')
+
+    def default_schema():
+        return Config().get('pg.default-schema', 'postgres')
+
+    def host(state: ReplState):
+        if not state.pg_path:
+            return None
+
+        return state.pg_path.split('/')[0]
+
+    def database(state: ReplState):
+        if not state.pg_path:
+            return None
+
+        tokens = state.pg_path.split('/')
+        if len(tokens) > 1:
+            return tokens[1]
+
+        return None
+
+    @functools.lru_cache()
+    def conn_details(namespace: str, host: str):
+        return Secrets.get_data(namespace, host)
+
+class PostgresPathHandler:
+    def __init__(self, state: ReplState, host: str = None, database: str = None):
+        self.state = state
+        self.host = host
+        self.database = database
+
+    def __enter__(self) -> tuple[str, str]:
+        if self.state and self.state.pg_path:
+            host_n_db = self.state.pg_path.split('/')
+            if not self.host:
+                self.host = host_n_db[0]
+            if not self.database and len(host_n_db) > 1:
+                self.database = host_n_db[1]
+
+        return self.host, self.database
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        return False
+
+def pg_path(state: ReplState, host: str = None, database: str = None):
+    return PostgresPathHandler(state, host=host, database=database)

adam/commands/postgres/postgres_ls.py

@@ -4,7 +4,6 @@ from adam.repl_state import ReplState, RequiredState
 
 class PostgresLs(Command):
     COMMAND = 'pg ls'
-    reaper_login = None
 
     # the singleton pattern
     def __new__(cls, *args, **kwargs):
@@ -25,15 +24,12 @@ class PostgresLs(Command):
         if not(args := self.args(cmd)):
             return super().run(cmd, state)
 
-        state, args = self.apply_state(args, state)
-        if not self.validate_state(state):
-            return state
-
-        state.device = ReplState.P
+        with self.validate(args, state) as (args, state):
+            state.device = ReplState.P
 
-        Ls().run('ls', state)
+            Ls().run('ls', state)
 
-        return state
+            return state
 
     def completion(self, state: ReplState):
         if state.sts:

adam/commands/postgres/postgres_preview.py

@@ -4,7 +4,6 @@ from adam.repl_state import ReplState, RequiredState
 
 class PostgresPreview(Command):
     COMMAND = 'pg preview'
-    reaper_login = None
 
     # the singleton pattern
     def __new__(cls, *args, **kwargs):
@@ -19,21 +18,18 @@ class PostgresPreview(Command):
         return PostgresPreview.COMMAND
 
     def required(self):
-        return RequiredState.NAMESPACE
+        return RequiredState.PG_DATABASE
 
     def run(self, cmd: str, state: ReplState):
         if not(args := self.args(cmd)):
             return super().run(cmd, state)
 
-        state, args = self.apply_state(args, state)
-        if not self.validate_state(state, RequiredState.PG_DATABASE):
-            return state
-
-        state.device = ReplState.P
+        with self.validate(args, state) as (args, state):
+            state.device = ReplState.P
 
-        PreviewTable().run(f'preview {" ".join(args)}', state)
+            PreviewTable().run(f'preview {" ".join(args)}', state)
 
-        return state
+            return state
 
     def completion(self, state: ReplState):
         if state.sts: