kaqing 2.0.14__py3-none-any.whl → 2.0.145__py3-none-any.whl

adam/commands/ls.py

@@ -1,20 +1,8 @@
 import copy
-import re
 
 from adam.commands.command import Command
-from adam.commands.commands_utils import show_pods, show_rollout
-from adam.commands.cqlsh import Cqlsh
-from adam.commands.postgres.postgres_session import PostgresSession
-from adam.config import Config
-from adam.k8s_utils.custom_resources import CustomResources
-from adam.k8s_utils.ingresses import Ingresses
-from adam.k8s_utils.kube_context import KubeContext
-from adam.k8s_utils.services import Services
-from adam.k8s_utils.statefulsets import StatefulSets
-from adam.pod_exec_result import PodExecResult
+from adam.commands.devices.devices import Devices
 from adam.repl_state import ReplState
-from adam.utils import lines_to_tabular, log, log2
-from adam.apps import Apps
 
 class Ls(Command):
     COMMAND = 'ls'
@@ -43,103 +31,12 @@ class Ls(Command):
                 state = copy.copy(state)
                 state.device = arg.replace(':', '')
 
-        if state.device == ReplState.P:
-            if state.pg_path:
-                pg = PostgresSession(state.namespace, state.pg_path)
-                if pg.db:
-                    self.show_pg_tables(pg)
-                else:
-                    self.show_pg_databases(pg)
-            else:
-                self.show_pg_hosts(state)
-        elif state.device == ReplState.A:
-            if state.app_env:
-                def line(n: str, ns: str):
-                    host = Ingresses.get_host(Config().get('app.login.ingress', '{app_id}-k8singr-appleader-001').replace('{app_id}', f'{ns}-{n}'), ns)
-                    if not host:
-                        return None
-
-                    endpoint = Config().get('app.login.url', 'https://{host}/{env}/{app}').replace('{host}', host).replace('{env}', state.app_env).replace('{app}', 'c3')
-                    if not endpoint:
-                        return None
-
-                    return f"{n.split('-')[1]},{Ingresses.get_host(f'{ns}-{n}-k8singr-appleader-001', ns)},{endpoint}"
-
-                svcs = [l for l in [line(n, ns) for n, ns in Apps.apps(state.app_env)] if l]
-
-                log(lines_to_tabular(svcs, 'APP,HOST,ENDPOINT', separator=','))
-            else:
-                svcs = [n for n, ns in Apps.envs()]
-
-                log(lines_to_tabular(svcs, 'ENV', separator=','))
-        else:
-            if state.pod:
-                r: PodExecResult = Cqlsh().run(f'cql describe tables', state)
-                if r.stderr:
-                    log(r.stderr)
-                log(r.stdout)
-            elif state.sts and state.namespace:
-                show_pods(StatefulSets.pods(state.sts, state.namespace), state.namespace, show_namespace=not KubeContext.in_cluster_namespace())
-                show_rollout(state.sts, state.namespace)
-            else:
-                self.show_statefulsets()
+        Devices.device(state).ls(cmd, state)
 
         return state
 
-    def show_statefulsets(self):
-        ss = StatefulSets.list_sts_names(show_namespace=not KubeContext.in_cluster_namespace())
-        if len(ss) == 0:
-            log2('No cassandra statefulsets found.')
-            return
-
-        app_ids = CustomResources.get_app_ids()
-        list = []
-        for s in ss:
-            cr_name = CustomResources.get_cr_name(s)
-            app_id = 'Unknown'
-            if cr_name in app_ids:
-                app_id = app_ids[cr_name]
-            list.append(f"{s} {app_id}")
-
-        header = 'STATEFULSET_NAME@NAMESPACE APP_ID'
-        if KubeContext.in_cluster_namespace():
-            header = 'STATEFULSET_NAME APP_ID'
-        log(lines_to_tabular(list, header))
-
-    def show_pg_hosts(self, state: ReplState):
-        if state.namespace:
-            def line(pg: PostgresSession):
-                return f'{pg.directory()},{pg.endpoint()}:{pg.port()},{pg.username()},{pg.password()}'
-
-            lines = [line(PostgresSession(state.namespace, pg)) for pg in PostgresSession.hosts(state.namespace)]
-
-            log(lines_to_tabular(lines, 'NAME,ENDPOINT,USERNAME,PASSWORD', separator=','))
-        else:
-            def line(pg: PostgresSession):
-                return f'{pg.directory()},{pg.namespace},{pg.endpoint()}:{pg.port()},{pg.username()},{pg.password()}'
-
-            lines = [line(PostgresSession(state.namespace, pg)) for pg in PostgresSession.hosts(state.namespace)]
-
-            log(lines_to_tabular(lines, 'NAME,NAMESPACE,ENDPOINT,USERNAME,PASSWORD', separator=','))
-
-    def show_pg_databases(self, pg: PostgresSession):
-        lines = [db["name"] for db in pg.databases() if db["owner"] == PostgresSession.default_owner()]
-
-        log(lines_to_tabular(lines, 'DATABASE', separator=','))
-
-    def show_pg_tables(self, pg: PostgresSession):
-        lines = [db["name"] for db in pg.tables() if db["schema"] == PostgresSession.default_schema()]
-
-        log(lines_to_tabular(lines, 'NAME', separator=','))
-
     def completion(self, state: ReplState):
-        if state.pod:
-            return {}
-
-        if not state.sts:
-            return {Ls.COMMAND: {n: None for n in StatefulSets.list_sts_names()}}
-
-        return {Ls.COMMAND: None}
+        return Devices.device(state).ls_completion(Ls.COMMAND, state, default = super().completion(state))
 
     def help(self, _: ReplState):
-        return f'{Ls.COMMAND} [device:]\t list apps, envs, clusters, nodes, pg hosts or pg databases'
+        return f'{Ls.COMMAND} [device:]\t list apps, envs, clusters, nodes, pg hosts/databases or export databases'

adam/commands/medusa/medusa.py

@@ -1,13 +1,11 @@
 import click
 
 from adam.commands.command import Command
-from adam.commands.command_helpers import ClusterCommandHelper
 from .medusa_backup import MedusaBackup
 from .medusa_restore import MedusaRestore
 from .medusa_show_backupjobs import MedusaShowBackupJobs
 from .medusa_show_restorejobs import MedusaShowRestoreJobs
 from adam.repl_state import ReplState, RequiredState
-from adam.utils import lines_to_tabular, log, log2
 
 class Medusa(Command):
     COMMAND = 'medusa'
@@ -31,20 +29,7 @@ class Medusa(Command):
         if not(args := self.args(cmd)):
             return super().run(cmd, state)
 
-        state, args = self.apply_state(args, state)
-        if not self.validate_state(state):
-            return state
-
-        if state.in_repl:
-            log(lines_to_tabular([c.help(ReplState()) for c in Medusa.cmd_list()], separator=':'))
-
-            return 'command-missing'
-        else:
-            # head with the Chain of Responsibility pattern
-            cmds = Command.chain(Medusa.cmd_list())
-            if not cmds.run(cmd, state):
-                log2('* Command is missing.')
-                Command.display_help()
+        return super().intermediate_run(cmd, state, args, Medusa.cmd_list())
 
     def cmd_list():
         return [MedusaBackup(), MedusaRestore(), MedusaShowBackupJobs(), MedusaShowRestoreJobs()]
@@ -55,15 +40,6 @@ class Medusa(Command):
 
         return {}
 
-    def help(self, _: ReplState):
-        return None
-
 class MedusaCommandHelper(click.Command):
     def get_help(self, ctx: click.Context):
-        log(super().get_help(ctx))
-        log()
-        log('Sub-Commands:')
-
-        log(lines_to_tabular([c.help(ReplState()).replace(f'{Medusa.COMMAND} ', '  ', 1) for c in Medusa.cmd_list()], separator=':'))
-        log()
-        ClusterCommandHelper.cluster_help()
+        Command.intermediate_help(super().get_help(ctx), Medusa.COMMAND, Medusa.cmd_list(), show_cluster_help=True)

adam/commands/medusa/medusa_backup.py

@@ -2,9 +2,9 @@ from datetime import datetime
 import re
 
 from adam.commands.command import Command
-from adam.k8s_utils.statefulsets import StatefulSets
+from adam.utils_k8s.statefulsets import StatefulSets
 from adam.repl_state import ReplState, RequiredState
-from adam.k8s_utils.custom_resources import CustomResources
+from adam.utils_k8s.custom_resources import CustomResources
 from adam.utils import log2
 
 

adam/commands/medusa/medusa_restore.py

@@ -1,9 +1,9 @@
 from datetime import datetime
 
 from adam.commands.command import Command
-from adam.k8s_utils.statefulsets import StatefulSets
+from adam.utils_k8s.statefulsets import StatefulSets
 from adam.repl_state import ReplState, RequiredState
-from adam.k8s_utils.custom_resources import CustomResources
+from adam.utils_k8s.custom_resources import CustomResources
 from adam.config import Config
 from adam.utils import lines_to_tabular, log2
 
@@ -59,8 +59,7 @@ class MedusaRestore(Command):
         now_dtformat = datetime.now().strftime("%Y-%m-%d.%H.%M.%S")
         rtname = 'medusa-' + now_dtformat + '-restore-from-' + bkname
         try:
-            print('SEAN doing')
-            # CustomResources.create_medusa_restorejob(rtname, bkname, dc, ns)
+            CustomResources.create_medusa_restorejob(rtname, bkname, dc, ns)
         except Exception as e:
             log2("Exception: MedusaRestore failed: %s\n" % e)
 

adam/commands/medusa/medusa_show_backupjobs.py

@@ -1,7 +1,7 @@
 from adam.commands.command import Command
-from adam.k8s_utils.statefulsets import StatefulSets
+from adam.utils_k8s.statefulsets import StatefulSets
 from adam.repl_state import ReplState, RequiredState
-from adam.k8s_utils.custom_resources import CustomResources
+from adam.utils_k8s.custom_resources import CustomResources
 from adam.utils import lines_to_tabular, log2
 
 
@@ -29,6 +29,7 @@ class MedusaShowBackupJobs(Command):
         state, args = self.apply_state(args, state)
         if not self.validate_state(state):
             return state
+
         ns = state.namespace
         dc = StatefulSets.get_datacenter(state.sts, ns)
         if not dc:
@@ -49,4 +50,4 @@ class MedusaShowBackupJobs(Command):
         return {}
 
     def help(self, _: ReplState):
-        return f'{MedusaShowBackupJobs.COMMAND}\t show backups'
+        return f'{MedusaShowBackupJobs.COMMAND}\t show Medusa backups'

adam/commands/medusa/medusa_show_restorejobs.py

@@ -1,7 +1,7 @@
 from adam.commands.command import Command
-from adam.k8s_utils.statefulsets import StatefulSets
+from adam.utils_k8s.statefulsets import StatefulSets
 from adam.repl_state import ReplState, RequiredState
-from adam.k8s_utils.custom_resources import CustomResources
+from adam.utils_k8s.custom_resources import CustomResources
 from adam.utils import lines_to_tabular, log2
 
 class MedusaShowRestoreJobs(Command):
@@ -49,4 +49,4 @@ class MedusaShowRestoreJobs(Command):
         return {}
 
     def help(self, _: ReplState):
-        return f'{MedusaShowRestoreJobs.COMMAND}\t show restores'
+        return f'{MedusaShowRestoreJobs.COMMAND}\t show Medusa restores'

adam/commands/nodetool.py

@@ -4,10 +4,11 @@ from adam.commands.command import Command
 from adam.commands.command_helpers import ClusterOrPodCommandHelper
 from adam.commands.nodetool_commands import NODETOOL_COMMANDS
 from adam.config import Config
-from adam.k8s_utils.cassandra_clusters import CassandraClusters
-from adam.k8s_utils.cassandra_nodes import CassandraNodes
+from adam.utils_k8s.cassandra_clusters import CassandraClusters
+from adam.utils_k8s.cassandra_nodes import CassandraNodes
 from adam.repl_state import ReplState, RequiredState
 from adam.utils import log
+from adam.utils_k8s.statefulsets import StatefulSets
 
 class NodeTool(Command):
     COMMAND = 'nodetool'
@@ -43,14 +44,18 @@ class NodeTool(Command):
         elif state.sts:
             return CassandraClusters.exec(state.sts, state.namespace, command, action='nodetool', show_out=True)
 
+        return state
+
     def completion(self, state: ReplState):
         if state.pod or state.sts:
-            return {NodeTool.COMMAND: {'help': None} | {c: None for c in NODETOOL_COMMANDS}}
+            d = {c: {'&': None} for c in NODETOOL_COMMANDS}
+            return {NodeTool.COMMAND: {'help': None} | d} | \
+                   {f'@{p}': {NodeTool.COMMAND: d} for p in StatefulSets.pod_names(state.sts, state.namespace)}
 
         return {}
 
     def help(self, _: ReplState):
-        return f'{NodeTool.COMMAND} <sub-command>\t run nodetool with arguments'
+        return f'{NodeTool.COMMAND} <sub-command> [&]\t run nodetool with arguments'
 
 class NodeToolCommandHelper(click.Command):
     def get_help(self, ctx: click.Context):

adam/commands/param_set.py

@@ -38,7 +38,7 @@ class SetParam(Command):
         return value
 
     def completion(self, _: ReplState):
-        return {SetParam.COMMAND: {key: None for key in Config().keys()}}
+        return {SetParam.COMMAND: {key: ({'true': None, 'false': None} if Config().get(key, None) in [True, False] else None) for key in Config().keys()}}
 
     def help(self, _: ReplState):
         return f"{SetParam.COMMAND} <key> <value>\t sets a Kaqing parameter to a different value"

adam/commands/postgres/postgres.py

@@ -1,12 +1,13 @@
 import click
 
 from adam.commands.command import Command
-from adam.commands.command_helpers import ClusterCommandHelper
+from adam.commands.postgres.psql_completions import psql_completions
+from adam.commands.postgres.postgres_utils import pg_table_names
 from .postgres_ls import PostgresLs
 from .postgres_preview import PostgresPreview
-from .postgres_session import PostgresSession
+from .postgres_context import PostgresContext
 from adam.repl_state import ReplState
-from adam.utils import lines_to_tabular, log, log2
+from adam.utils import log, log2
 
 class Postgres(Command):
     COMMAND = 'pg'
@@ -30,35 +31,27 @@ class Postgres(Command):
 
         state, args = self.apply_state(args, state)
 
-        if state.in_repl:
-            if not args:
+        if not args:
+            if state.in_repl:
                 log2('Please use SQL statement. e.g. pg \l')
-
-                return 'command-missing'
             else:
-                self.run_sql(state, args)
-        else:
-            if not args:
                 log2('* Command or SQL statements is missing.')
                 Command.display_help()
 
-                return 'command-missing'
-            else:
-                # head with the Chain of Responsibility pattern
-                cmds = Command.chain(Postgres.cmd_list())
-                if not cmds.run(cmd, state) :
-                    if not args:
-                        log2('* Command or SQL statements is missing.')
-                        Command.display_help()
+            return 'command-missing'
 
-                        return 'command-missing'
-                    else:
-                        self.run_sql(state, args)
+        if state.in_repl:
+            self.run_sql(state, args)
+        else:
+            # head with the Chain of Responsibility pattern
+            cmds = Command.chain(Postgres.cmd_list())
+            if not cmds.run(cmd, state) :
+                self.run_sql(state, args)
 
         return state
 
     def cmd_list():
-        return [PostgresLs(), PostgresPreview()]
+        return [PostgresLs(), PostgresPreview(), PostgresPg()]
 
     def run_sql(self, state: ReplState, args: list[str]):
         if not state.pg_path:
@@ -69,21 +62,23 @@ class Postgres(Command):
 
             return state
 
-        PostgresSession(state.namespace, state.pg_path).run_sql(' '.join(args))
+        background = False
+        if args and args[-1] == '&':
+            args = args[:-1]
+            background = True
+
+        PostgresContext.apply(state.namespace, state.pg_path).run_sql(' '.join(args), background=background)
 
     def completion(self, state: ReplState):
+        if state.device != state.P:
+            # conflicts with cql completions
+            return {}
+
         leaf = {}
-        if PostgresSession(state.namespace, state.pg_path).db:
-            leaf = {
-                '\h': None,
-                '\d': None,
-                '\dt': None,
-                '\du': None,
-                'delete': {'from': None},
-                'insert': {'into': None},
-                'select': None,
-                'update': None,
-            }
+        session = PostgresContext.apply(state.namespace, state.pg_path)
+        if session.db:
+          if pg_table_names(state.namespace, state.pg_path):
+            leaf = psql_completions(state.namespace, state.pg_path)
         elif state.pg_path:
             leaf = {
                 '\h': None,
@@ -96,18 +91,22 @@ class Postgres(Command):
             return {}
 
     def help(self, _: ReplState):
-        return f'[{Postgres.COMMAND}] <sql-statements>\t run psql with queries'
+        return f'<sql-statements> [&]\t run queries on Postgres databases'
 
 class PostgresCommandHelper(click.Command):
     def get_help(self, ctx: click.Context):
-        log(super().get_help(ctx))
-        log()
-        log('Sub-Commands:')
-
-        log(lines_to_tabular([c.help(ReplState()).replace(f'{Postgres.COMMAND} ', '  ', 1) for c in Postgres.cmd_list()], separator='\t'))
-        log()
-        ClusterCommandHelper.cluster_help()
+        Command.intermediate_help(super().get_help(ctx), Postgres.COMMAND, Postgres.cmd_list(), show_cluster_help=True)
         log('PG-Name:  Kubernetes secret for Postgres credentials')
         log('          e.g. stgawsscpsr-c3-c3-k8spg-cs-001')
         log('Database: Postgres database name within a host')
-        log('          e.g. stgawsscpsr_c3_c3')
+        log('          e.g. stgawsscpsr_c3_c3')
+
+# No action body, only for a help entry and auto-completion
+class PostgresPg(Command):
+    COMMAND = 'pg'
+
+    def command(self):
+        return PostgresPg.COMMAND
+
+    def help(self, _: ReplState):
+        return f'pg <sql-statements>\t run queries on Postgres databases'

adam/commands/postgres/{postgres_session.py → postgres_context.py}

@@ -1,14 +1,37 @@
+from datetime import datetime
 import functools
 import re
 import subprocess
 
 from adam.config import Config
-from adam.k8s_utils.kube_context import KubeContext
-from adam.k8s_utils.pods import Pods
-from adam.k8s_utils.secrets import Secrets
+from adam.repl_session import ReplSession
+from adam.utils_k8s.kube_context import KubeContext
+from adam.utils_k8s.pods import Pods
+from adam.utils_k8s.secrets import Secrets
 from adam.utils import log2
 
-class PostgresSession:
+class PostgresContext:
+    def apply(namespace: str, path: str, arg: str = None) -> 'PostgresContext':
+        context = PostgresContext(namespace, path)
+
+        if arg:
+            if arg == '..':
+                if context.db:
+                    context.db = None
+                else:
+                    context.host = None
+            else:
+                tks = arg.split('@')
+                if not context.host:
+                    context.host = tks[0]
+                else:
+                    context.db = tks[0]
+
+                if not namespace and tks[1]:
+                    context.namespace = tks[1]
+
+        return context
+
     def __init__(self, ns: str, path: str):
         self.namespace = ns
         self.conn_details = None
@@ -25,36 +48,7 @@ class PostgresSession:
             if len(tks) > 1:
                 self.db = tks[1]
 
-    # work for databases()
-    def __eq__(self, other: 'PostgresSession'):
-        return self.host == other.host
-
-    def __hash__(self):
-        return hash(self.host)
-
-    def find_namespace(self, arg: str):
-        if arg:
-            tks = arg.split('@')
-            if len(tks) > 1:
-                return tks[1]
-
-        return None
-
-    def directory(self, arg: str = None):
-        if arg:
-            if arg == '..':
-                if self.db:
-                    self.db = None
-                else:
-                    self.host = None
-            else:
-                tks = arg.split('@')
-                arg = tks[0]
-                if not self.host:
-                    self.host = arg
-                else:
-                    self.db = arg
-
+    def path(self):
         if not self.host:
             return None
 
@@ -65,7 +59,7 @@ class PostgresSession:
         return f'{self.host}/{self.db}'
 
     def hosts(ns: str):
-        return PostgresSession.hosts_for_namespace(ns)
+        return PostgresContext.hosts_for_namespace(ns)
 
     @functools.lru_cache()
     def hosts_for_namespace(ns: str):
@@ -82,7 +76,6 @@ class PostgresSession:
 
         return [s for s in ss if not excludes(s)]
 
-    @functools.lru_cache()
     def databases(self):
         dbs = []
         #  List of databases
@@ -140,15 +133,16 @@ class PostgresSession:
 
         return dbs
 
-    def run_sql(self, sql: str, show_out = True):
-        db = self.db if self.db else PostgresSession.default_db()
+    def run_sql(self, sql: str, show_out = True, background = False):
+        db = self.db if self.db else PostgresContext.default_db()
 
         if KubeContext.in_cluster():
             cmd1 = f'env PGPASSWORD={self.password()} psql -h {self.endpoint()} -p {self.port()} -U {self.username()} {db} --pset pager=off -c'
             log2(f'{cmd1} "{sql}"')
             # remove double quotes from the sql argument
             cmd = cmd1.split(' ') + [sql]
-            r = subprocess.run(cmd, capture_output=True, text=True)
+
+            r = subprocess.run(cmd, capture_output=not background, text=True)
             if show_out:
                 log2(r.stdout)
                 log2(r.stderr)
@@ -159,7 +153,7 @@ class PostgresSession:
             pod_name = Config().get('pg.agent.name', 'ops-pg-agent')
 
             if Config().get('pg.agent.just-in-time', False):
-                if not PostgresSession.deploy_pg_agent(pod_name, ns):
+                if not PostgresContext.deploy_pg_agent(pod_name, ns):
                     return
 
             real_pod_name = pod_name
@@ -169,14 +163,21 @@ class PostgresSession:
             except:
                 try:
                     # try with the ops pod
+                    pod_name = Config().get('pod.name', 'ops')
                     real_pod_name = Pods.get_with_selector(ns, label_selector = Config().get('pod.label-selector', 'run=ops')).metadata.name
                 except:
                     log2(f"Could not locate {pod_name} pod.")
                     return None
 
-            cmd = f'PGPASSWORD="{self.password()}" psql -h {self.endpoint()} -p {self.port()} -U {self.username()} {db} --pset pager=off -c "{sql}"'
+            cmd = f'psql -h {self.endpoint()} -p {self.port()} -U {self.username()} {db} --pset pager=off -c "{sql}"'
+            env_prefix = f'PGPASSWORD="{self.password()}"'
+
+            r = Pods.exec(real_pod_name, pod_name, ns, cmd, show_out=show_out, background=background, env_prefix=env_prefix)
+            if r and Config().get('repl.history.push-cat-remote-log-file', True):
+                if r.log_file and ReplSession().prompt_session:
+                    ReplSession().prompt_session.history.append_string(f'@{r.pod} cat {r.log_file}')
 
-            return Pods.exec(real_pod_name, pod_name, ns, cmd, show_out=show_out)
+            return r
 
     def deploy_pg_agent(pod_name: str, ns: str) -> str:
         image = Config().get('pg.agent.image', 'seanahnsf/kaqing')

adam/commands/postgres/postgres_utils.py

@@ -0,0 +1,31 @@
+import functools
+
+from adam.commands.postgres.postgres_context import PostgresContext
+from adam.config import Config
+
+TestPG = [False]
+
+@functools.lru_cache()
+def pg_database_names(ns: str, pg_path: str):
+    if TestPG[0]:
+        return ['azops88_c3ai_c3']
+
+    Config().wait_log('Inspecting Postgres Databases...')
+
+    pg = PostgresContext.apply(ns, pg_path)
+    return [db['name'] for db in pg.databases() if db['owner'] == PostgresContext.default_owner()]
+
+@functools.lru_cache()
+def pg_table_names(ns: str, pg_path: str):
+    if TestPG[0]:
+        return ['C3_2_XYZ1']
+
+    Config().wait_log('Inspecting Postgres Database...')
+    return [table['name'] for table in pg_tables(ns, pg_path) if table['schema'] == PostgresContext.default_schema()]
+
+def pg_tables(ns: str, pg_path: str):
+    pg = PostgresContext.apply(ns, pg_path)
+    if pg.db:
+        return pg.tables()
+
+    return []

adam/commands/postgres/psql_completions.py

@@ -0,0 +1,10 @@
+from adam.commands.postgres.postgres_utils import pg_table_names
+from adam.sql.sql_completer import SqlCompleter
+
+def psql_completions(ns: str, pg_path: str):
+    return {
+        '\h': None,
+        '\d': None,
+        '\dt': None,
+        '\du': None
+    } | SqlCompleter(lambda: pg_table_names(ns, pg_path)).completions_for_nesting()