kaqing 2.0.98__py3-none-any.whl → 2.0.171__py3-none-any.whl

adam/commands/cd.py

@@ -1,12 +1,7 @@
 from adam.commands.command import Command
-from adam.commands.postgres.postgres_utils import pg_database_names
-from adam.commands.postgres.postgres_session import PostgresSession
-from adam.utils_k8s.cassandra_clusters import CassandraClusters
-from adam.utils_k8s.kube_context import KubeContext
-from adam.utils_k8s.statefulsets import StatefulSets
+from adam.commands.devices.device import Device
+from adam.commands.devices.devices import Devices
 from adam.repl_state import ReplState
-from adam.utils import log2
-from adam.apps import Apps
 
 class Cd(Command):
     COMMAND = 'cd'
@@ -23,97 +18,26 @@ class Cd(Command):
     def command(self):
         return Cd.COMMAND
 
+    def required(self):
+        return ReplState.NON_L
+
     def run(self, cmd: str, state: ReplState):
         if not(args := self.args(cmd)):
             return super().run(cmd, state)
 
-        if len(args) < 2:
-            return state
-
-        arg = args[1]
-        for dir in arg.split('/'):
-            if state.device == ReplState.P:
-                if dir == '':
-                    state.pg_path = None
-                else:
-                    session = PostgresSession(state.namespace, state.pg_path)
-                    # patch up state.namespace from pg cd
-                    if not state.namespace and (ns := session.find_namespace(arg)):
-                        state.namespace = ns
-                    state.pg_path = session.directory(arg)
-            elif state.device == ReplState.A:
-                if dir == '':
-                    state.app_env = None
-                    state.app_app = None
-                elif dir == '..':
-                    if state.app_app:
-                        state.app_app = None
-                    else:
-                        state.app_env = None
-                else:
-                    if not state.app_env:
-                        tks = dir.split('@')
-                        if len(tks) > 1:
-                            state.namespace = tks[1]
+        with self.validate(args, state, apply=False) as (args, state):
+            if len(args) < 2:
+                return state
 
-                        state.app_env = dir.split('@')[0]
-                    else:
-                        state.app_app = dir
-            elif state.device == ReplState.L:
-                pass
-            else:
-                if dir == '':
-                    state.sts = None
-                    state.pod = None
-                elif dir == '..':
-                    if state.pod:
-                        state.pod = None
-                    else:
-                        state.sts = None
-                else:
-                    if not state.sts:
-                        ss_and_ns = dir.split('@')
-                        state.sts = ss_and_ns[0]
-                        if len(ss_and_ns) > 1:
-                            state.namespace = ss_and_ns[1]
-                    elif not state.pod:
-                        p, _ = KubeContext.is_pod_name(dir)
-                        if p:
-                            state.pod = p
-                        else:
-                            names = CassandraClusters.pod_names_by_host_id(state.sts, state.namespace);
-                            if dir in names:
-                                state.pod = names[dir]
-                            else:
-                                log2('Not a valid pod name or host id.')
+            device: Device = Devices.device(state)
+            arg = args[1]
+            for dir in arg.split('/'):
+                device.cd(dir, state)
 
-        return state
+            return state
 
     def completion(self, state: ReplState):
-        if state.device == ReplState.P:
-            pg = PostgresSession(state.namespace, state.pg_path) if state.pg_path else None
-            if pg and pg.db:
-                return {Cd.COMMAND: {'..': None}}
-            elif pg and pg.host:
-                return {Cd.COMMAND: {'..': None} | {p: None for p in pg_database_names(state.namespace, pg.directory())}}
-            else:
-                return {Cd.COMMAND: {p: None for p in PostgresSession.hosts(state.namespace)}}
-        elif state.device == ReplState.A:
-            if state.app_app:
-                return {Cd.COMMAND: {'..': None}}
-            elif state.app_env:
-                return {Cd.COMMAND: {'..': None} | {app[0].split('-')[1]: None for app in Apps.apps(state.app_env)}}
-            else:
-                return {Cd.COMMAND: {'..': None} | {env[0]: None for env in Apps.envs()}}
-        elif state.device == ReplState.C:
-            if state.pod:
-                return {Cd.COMMAND: {'..': None}}
-            elif state.sts:
-                return {Cd.COMMAND: {'..': None} | {p: None for p in StatefulSets.pod_names(state.sts, state.namespace)}}
-            else:
-                return {Cd.COMMAND: {p: None for p in StatefulSets.list_sts_names()}}
-
-        return {}
+        return Devices.device(state).cd_completion(Cd.COMMAND, state, default = {})
 
     def help(self, _: ReplState):
-        return f'{Cd.COMMAND} <path> | .. \t move around'
+        return f'{Cd.COMMAND} <path> | .. \t move around on the operational device hierarchy'

adam/commands/check.py

@@ -2,11 +2,13 @@ import click
 
 from adam.checks.check_result import CheckResult
 from adam.checks.check_utils import all_checks, checks_from_csv, run_checks
+from adam.commands import extract_options
 from adam.commands.command import Command
 from adam.commands.command_helpers import ClusterOrPodCommandHelper
 from adam.commands.issues import Issues
 from adam.repl_state import ReplState
 from adam.utils import lines_to_tabular, log
+from adam.utils_issues import IssuesUtils
 
 class Check(Issues):
     COMMAND = 'check'
@@ -23,37 +25,40 @@ class Check(Issues):
     def command(self):
         return Check.COMMAND
 
+    def required(self):
+        return ReplState.NON_L
+
     def run(self, cmd: str, state: ReplState):
         if not(args := self.args(cmd)):
             return super().run(cmd, state)
 
-        state, args = self.apply_state(args, state)
-        args, show = Command.extract_options(args, ['-s', '--show'])
+        with self.validate(args, state) as (args, state):
+            with extract_options(args, ['-s', '--show']) as (args, show_out):
+                if not args:
+                    if state.in_repl:
+                        log(lines_to_tabular([check.help() for check in all_checks()], separator=':'))
+                    else:
+                        log('* Check name is missing.')
+                        Command.display_help()
 
-        if not args:
-            if state.in_repl:
-                log(lines_to_tabular([check.help() for check in all_checks()], separator=':'))
-            else:
-                log('* Check name is missing.')
-                Command.display_help()
-            return 'arg missing'
+                    return 'arg missing'
 
-        checks = checks_from_csv(args[0])
-        if not checks:
-            return 'invalid check name'
+                checks = checks_from_csv(args[0])
+                if not checks:
+                    return 'invalid check name'
 
-        results = run_checks(state.sts, state.namespace, state.pod, checks=checks, show_output=show)
+                results = run_checks(state.sts, state.namespace, state.pod, checks=checks, show_out=show_out)
 
-        issues = CheckResult.collect_issues(results)
-        Issues.show_issues(issues, in_repl=state.in_repl)
+                issues = CheckResult.collect_issues(results)
+                IssuesUtils.show_issues(issues, in_repl=state.in_repl)
 
-        return issues if issues else 'no issues found'
+                return issues if issues else 'no issues found'
 
     def completion(self, _: ReplState):
-        return {Check.COMMAND: {check.name(): None for check in all_checks()}}
+        return {Check.COMMAND: {check.name(): {'-s': None} for check in all_checks()}}
 
     def help(self, _: ReplState):
-        return f'{Check.COMMAND} <check-name>\t run a single check'
+        return f'{Check.COMMAND} <check-name> [-s]\t run a single check'
 
 class CheckCommandHelper(click.Command):
     def get_help(self, ctx: click.Context):

adam/commands/cli_commands.py

@@ -1,7 +1,7 @@
 import functools
 import re
 
-from adam.commands.reaper.reaper_session import ReaperSession
+from adam.commands.reaper.utils_reaper import Reapers
 from adam.config import Config
 from adam.utils_k8s.kube_context import KubeContext
 from adam.utils_k8s.secrets import Secrets
@@ -66,8 +66,7 @@ class CliCommands:
             return key.replace(f'{state.namespace}-', '').replace('-k8spg-cs-001', '')
         d |= {f'pg-{reduce_key(k)}': f'PGPASSWORD={v["postgres-admin-password"]} psql -h {v["postgres-db-endpoint"]} -p {v["postgres-db-port"]} -U {v["postgres-admin-username"]} postgres' for k, v in data.items()}
 
-        if r := ReaperSession.create(state):
-            reaper = r.reaper_spec(state)
+        if reaper := Reapers.reaper_spec(state):
             d |= {
                 'reaper-exec': reaper["exec"],
                 'reaper-forward': reaper["forward"],

adam/commands/code.py

@@ -0,0 +1,57 @@
+import code
+from functools import partial
+
+from adam.commands.command import Command
+from adam.commands.cql.utils_cql import cassandra_table_names, run_cql
+from adam.repl_state import ReplState, RequiredState
+from adam.utils import lines_to_tabular
+from adam.utils_k8s.statefulsets import StatefulSets
+
+class Code(Command):
+    COMMAND = 'python'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(Code, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return Code.COMMAND
+
+    def required(self):
+        return RequiredState.NAMESPACE
+
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        with self.validate(args, state) as (args, state):
+            sts = state.sts
+            pod = state.pod
+            pods = StatefulSets.pod_names(state.sts, state.namespace)
+            tables = cassandra_table_names(state)
+            cql = partial(run_cql, state, show_out=True)
+
+            my_local = globals() | locals()
+            # my_local = globals() | {'StatefulSets': StatefulSets} | locals()
+            lines = [
+                'sts: StatefulSet name',
+                'pod: Pod name',
+                'pods: Pod names in the current StatefulSet',
+                'tables: Cassandra tables names in the current StatefulSet',
+                "cql('select...'): run cql statement"
+            ]
+
+            code.interact(local=my_local, banner=lines_to_tabular(lines, header='Variables', separator=':'))
+
+            return state
+
+    def completion(self, state: ReplState):
+        return super().completion(state)
+
+    def help(self, _: ReplState):
+        return f'{Code.COMMAND} \t invoke interactive Python shell'

adam/commands/command.py

@@ -2,10 +2,10 @@ from abc import abstractmethod
 import copy
 import subprocess
 import sys
+from typing import Union
 
-from adam.commands.command_helpers import ClusterCommandHelper
 from adam.repl_state import ReplState, RequiredState
-from adam.utils import lines_to_tabular, log, log2
+from adam.utils import is_lambda
 
 repl_cmds: list['Command'] = []
 
@@ -28,8 +28,13 @@ class Command:
 
         return None
 
-    def completion(self, _: ReplState, leaf: dict[str, any] = None) -> dict[str, any]:
-        # COMMAND = 'reaper activate schedule'
+    def completion(self, state: ReplState, leaf: dict[str, any] = None) -> dict[str, any]:
+        if not self.validate_state(state, show_err=False):
+            return {}
+
+        if is_lambda(leaf):
+            leaf = leaf()
+
         d = leaf
         for t in reversed(self.command().split(' ')):
             d = {t: d}
@@ -39,8 +44,11 @@ class Command:
     def required(self) -> RequiredState:
         return None
 
-    def validate_state(self, state: ReplState, pg_required: RequiredState = None, app_required: RequiredState = None):
-        return state.validate(self.required(), pg_required=pg_required, app_required=app_required)
+    def validate(self, args: list[str], state: ReplState, apply = True):
+        return ValidateHandler(self, args, state, apply = apply)
+
+    def validate_state(self, state: ReplState, show_err = True):
+        return state.validate(self.required(), show_err=show_err)
 
     def help(self, _: ReplState) -> str:
         return None
@@ -54,6 +62,9 @@ class Command:
         return a
 
     def apply_state(self, args: list[str], state: ReplState, resolve_pg = True, args_to_check = 6) -> tuple[ReplState, list[str]]:
+        """
+        Applies any contextual arguments such as namespace or statefulset to the ReplState and returns any non-contextual arguments.
+        """
         return state.apply_args(args, cmd=self.command_tokens(), resolve_pg=resolve_pg, args_to_check=args_to_check)
 
     def command_tokens(self):
@@ -85,17 +96,30 @@ class Command:
         args.extend(['--help'])
         subprocess.run(args)
 
-    def extract_options(args: list[str], names: list[str]):
-        found: list[str] = []
+    def extract_options(args: list[str], trailing: Union[str, list[str]] = [], options: list[str] = []):
+        found_options: list[str] = []
+        found_trailing = None
+
+        if trailing is None:
+            trailing = []
+        elif isinstance(trailing, str):
+            trailing = [trailing]
+
+        if options is None:
+            options = []
+        elif isinstance(options, str):
+            options = [options]
 
         new_args: list[str] = []
-        for arg in args:
-            if arg in names:
-                found.append(arg)
+        for index, arg in enumerate(args):
+            if index == len(args) - 1 and arg in trailing:
+                found_trailing = arg
+            elif arg in options:
+                found_options.append(arg)
             else:
                 new_args.append(arg)
 
-        return new_args, found
+        return new_args, found_trailing, found_options
 
     def print_chain(cmd: 'Command'):
         print(f'{cmd.command()}', end = '')
@@ -104,31 +128,63 @@ class Command:
             cmd = s
         print()
 
-    def intermediate_run(self, cmd: str, state: ReplState, args: list[str], cmds: list['Command'], separator='\t', display_help=True):
-        state, _ = self.apply_state(args, state)
-
-        if state.in_repl:
-            if display_help:
-                log(lines_to_tabular([c.help(state) for c in cmds], separator=separator))
-
-            return 'command-missing'
-        else:
-            # head with the Chain of Responsibility pattern
-            cmds = Command.chain(cmds)
-            if not cmds.run(cmd, state):
-                if display_help:
-                    log2('* Command is missing.')
-                    Command.display_help()
-                return 'command-missing'
-
-        return state
-
-    def intermediate_help(super_help: str, cmd: str, cmd_list: list['Command'], separator='\t', show_cluster_help=False):
-        log(super_help)
-        log()
-        log('Sub-Commands:')
-
-        log(lines_to_tabular([c.help(ReplState()).replace(f'{cmd} ', '  ', 1) for c in cmd_list], separator=separator))
-        if show_cluster_help:
-            log()
-            ClusterCommandHelper.cluster_help()
+class InvalidState(Exception):
+    def __init__(self, state: ReplState):
+        super().__init__(f'Invalid state')
+
+class ValidateHandler:
+    def __init__(self, cmd: Command, args: list[str], state: ReplState, apply = True):
+        self.cmd = cmd
+        self.args = args
+        self.state = state
+        self.apply = apply
+
+    def __enter__(self) -> tuple[list[str], ReplState]:
+        state = self.state
+        args = self.args
+        if self.apply:
+            state, args = self.cmd.apply_state(args, state)
+
+        if not self.cmd.validate_state(state):
+            raise InvalidState(state)
+
+        return args, state
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        return False
+
+class ExtractOptionsHandler:
+    def __init__(self, args: list[str], options: list[str] = None):
+        self.args = args
+        self.options = options
+
+    def __enter__(self) -> tuple[list[str], list[str]]:
+        args, _, options = Command.extract_options(self.args, options=self.options)
+        return args, options
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        return False
+
+class ExtractTrailingOptionsHandler:
+    def __init__(self, args: list[str], trailing: list[str] = None):
+        self.args = args
+        self.trailing = trailing
+
+    def __enter__(self) -> tuple[list[str], list[str]]:
+        args, trailing, _ = Command.extract_options(self.args, trailing=self.trailing)
+        return args, trailing
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        return False
+
+class ExtractAllOptionsHandler:
+    def __init__(self, args: list[str], trailing: list[str] = None, options: list[str] = None):
+        self.args = args
+        self.trailing = trailing
+        self.options = options
+
+    def __enter__(self) -> tuple[list[str], list[str]]:
+        return Command.extract_options(self.args, options=self.options)
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        return False

adam/commands/commands_utils.py

@@ -1,16 +1,14 @@
-from concurrent.futures import ThreadPoolExecutor
-import time
 from kubernetes import client
 from typing import List
 
 from adam.checks.check_utils import run_checks
 from adam.columns.columns import Columns, collect_checks
-from adam.commands.issues import Issues
+from adam.utils_issues import IssuesUtils
 from adam.utils_k8s.cassandra_nodes import CassandraNodes
 from adam.utils_k8s.pods import Pods
 from adam.utils_k8s.statefulsets import StatefulSets
 from adam.repl_state import ReplState
-from adam.utils import convert_seconds, duration, lines_to_tabular, log, log2
+from adam.utils import duration, lines_to_tabular, log, log2
 
 def show_pods(pods: List[client.V1Pod], ns: str, show_namespace = True, show_host_id = True):
     if len(pods) == 0:
@@ -21,18 +19,10 @@ def show_pods(pods: List[client.V1Pod], ns: str, show_namespace = True, show_hos
     if show_host_id:
         names = [pod.metadata.name for pod in pods]
 
-        def get_host_id_with_pod(pod, ns):
-            return (CassandraNodes.get_host_id(pod, ns), pod)
-
-        def body(executor: ThreadPoolExecutor, pod, ns, show_out):
-            if executor:
-                return executor.submit(get_host_id_with_pod, pod, ns)
-
-            id = CassandraNodes.get_host_id(pod, ns)
-
-            return (id, pod)
-
-        host_ids_by_pod = {pod: id for id, pod in Pods.on_pods(names, ns, body, action='get-host-id', show_out=False)}
+        msg = 'd`Retrieving|Retrived {size} host ids'
+        with Pods.parallelize(names, msg=msg, action = 'get-host-id') as exec:
+            host_pods = exec.map(lambda pod: (CassandraNodes.get_host_id(pod, ns), pod))
+            host_ids_by_pod = {pod: id for id, pod in host_pods}
 
     def line(pod: client.V1Pod):
         pod_cnt = len(pod.status.container_statuses)
@@ -71,10 +61,10 @@ def show_rollout(sts: str, ns: str):
         else:
             log2(f'Cluster has completed rollout {d} ago.')
 
-def show_table(state: ReplState, pods: list[str], cols: str, header: str, show_output=False):
+def show_table(state: ReplState, pods: list[str], cols: str, header: str, show_out=False):
     columns = Columns.create_columns(cols)
 
-    results = run_checks(cluster=state.sts, pod=state.pod, namespace=state.namespace, checks=collect_checks(columns), show_output=show_output)
+    results = run_checks(cluster=state.sts, pod=state.pod, namespace=state.namespace, checks=collect_checks(columns), show_out=show_out)
 
     def line(pod_name: str):
         cells = [c.pod_value(results, pod_name) for c in columns]
@@ -85,4 +75,4 @@ def show_table(state: ReplState, pods: list[str], cols: str, header: str, show_o
 
     log(lines_to_tabular(lines, header, separator=','))
 
-    Issues.show(results, state.in_repl)
+    IssuesUtils.show(results, state.in_repl)

adam/commands/cp.py

@@ -29,47 +29,41 @@ class ClipboardCopy(Command):
         if not(args := self.args(cmd)):
             return super().run(cmd, state)
 
-        state, args = self.apply_state(args, state)
-        if not self.validate_state(state):
-            return state
-
-        if len(args) < 1:
-            if state.in_repl:
-                log2('Key is required.')
-                log2()
-                log2('Keys:')
-                log2(lines_to_tabular([f'{k},{v}' for k, v in CliCommands.values(state, collapse=True).items()], separator=','))
-            else:
-                log2('* Key is missing.')
-                Command.display_help()
-
-            return 'command-missing'
-
-        key = args[0]
-        if not key in CliCommands.values(state):
-            if state.in_repl:
-                log2('Key is required.')
-                log2()
-                log2('Keys:')
-                log2(lines_to_tabular([f'{k},{v}' for k, v in CliCommands.values(state, collapse=True).items()], separator=','))
-            else:
-                log2('* Invalid key')
-                Command.display_help()
-
-            return 'command-invalid'
-
-        value = CliCommands.values(state)[key]
-        pyperclip.copy(value)
-        log2('The following line has been copied to clipboard. Use <Ctrl-V> to use it.')
-        log2(f'  {value}')
-
-        return 'value-copied'
+        with self.validate(args, state) as (args, state):
+            if len(args) < 1:
+                if state.in_repl:
+                    log2('Key is required.')
+                    log2()
+                    log2('Keys:')
+                    log2(lines_to_tabular([f'{k},{v}' for k, v in CliCommands.values(state, collapse=True).items()], separator=','))
+                else:
+                    log2('* Key is missing.')
+                    Command.display_help()
+
+                return 'command-missing'
+
+            key = args[0]
+            if not key in CliCommands.values(state):
+                if state.in_repl:
+                    log2('Key is required.')
+                    log2()
+                    log2('Keys:')
+                    log2(lines_to_tabular([f'{k},{v}' for k, v in CliCommands.values(state, collapse=True).items()], separator=','))
+                else:
+                    log2('* Invalid key')
+                    Command.display_help()
+
+                return 'command-invalid'
+
+            value = CliCommands.values(state)[key]
+            pyperclip.copy(value)
+            log2('The following line has been copied to clipboard. Use <Ctrl-V> to use it.')
+            log2(f'  {value}')
+
+            return 'value-copied'
 
     def completion(self, state: ReplState):
-        if state.sts:
-            return {ClipboardCopy.COMMAND: {key: None for key in CliCommands.values(state).keys()}}
-
-        return {}
+        return super().completion(state, lambda: {key: None for key in CliCommands.values(state).keys()})
 
     def help(self, _: ReplState):
         return f"{ClipboardCopy.COMMAND} <key>\t copy a value to clipboard for conveninence"

adam/commands/cql/cql_completions.py

@@ -1,11 +1,33 @@
-from adam.commands.cql.cql_utils import table_names
+import time
+from adam.commands.cql.utils_cql import cassandra_keyspaces, cassandra_table_names
+from adam.commands.export.exporter import Exporter
+from adam.commands.export.export_databases import ExportDatabases
+from adam.config import Config
 from adam.repl_state import ReplState
-from adam.sql.sql_completer import SqlCompleter
+from adam.sql.sql_completer import SqlCompleter, SqlVariant
+from adam.utils import log_timing
 
 def cql_completions(state: ReplState) -> dict[str, any]:
-    return {
-        'describe': {
-            'keyspaces': None,
-            'table': {t: None for t in table_names(state)},
-            'tables': None},
-    } | SqlCompleter.completions(lambda: table_names(state))
+    ps = Config().get('cql.alter-tables.gc-grace-periods', '3600,86400,864000,7776000').split(',')
+    # warm up caches
+    with log_timing('cassandra_keyspaces'):
+        cassandra_keyspaces(state)
+    with log_timing('cassandra_table_names'):
+        cassandra_table_names(state)
+    with log_timing('ExportDatabases.database_names'):
+        ExportDatabases.database_names()
+
+    expandables = {
+        'keyspaces': lambda: cassandra_keyspaces(state),
+        'table-props': lambda: {
+            'GC_GRACE_SECONDS': ps
+        },
+        'export-dbs': lambda: ExportDatabases.database_names(),
+        'export-sessions': lambda: Exporter.export_session_names(state.sts, state.pod, state.namespace),
+        'export-sessions-incomplete': lambda: Exporter.export_session_names(state.sts, state.pod, state.namespace, export_state='pending_import'),
+    }
+
+    return SqlCompleter(
+        lambda: cassandra_table_names(state),
+        expandables=expandables,
+        variant=SqlVariant.CQL).completions_for_nesting()