kaqing 2.0.110__py3-none-any.whl → 2.0.214__py3-none-any.whl

adam/commands/command.py

@@ -1,11 +1,14 @@
 from abc import abstractmethod
+from collections.abc import Callable
 import copy
 import subprocess
 import sys
+from typing import Union
 
-from adam.commands.command_helpers import ClusterCommandHelper
+from adam.config import Config
 from adam.repl_state import ReplState, RequiredState
-from adam.utils import lines_to_tabular, log, log2
+from adam.sql.lark_completer import LarkCompleter
+from adam.utils import log2
 
 repl_cmds: list['Command'] = []
 
@@ -28,10 +31,39 @@ class Command:
 
         return None
 
-    def completion(self, state: ReplState, leaf: dict[str, any] = None) -> dict[str, any]:
-        if not self.validate_state(state, show_err=False):
+    def completion(self, state: ReplState, leaf: dict[str, any] = None, pods: tuple[list[str], str] = None, auto: str = 'on', auto_key: str = None) -> dict[str, any]:
+        # pods is a tuple of list of pod names and the current pod repl is on
+        if not pods:
+            return self._completion(state, leaf, auto=auto, auto_key=auto_key)
+
+        c = {}
+
+        pod_names = pods[0]
+        pod = pods[1]
+
+        if pod:
+            c |= self._completion(state, leaf, auto=auto, auto_key=auto_key)
+
+        c |= {f'@{p}': self._completion(state.with_pod(p), leaf, to_validate=False, auto=auto, auto_key=auto_key) for p in pod_names if p != pod}
+
+        return c
+
+    def _completion(self, state: ReplState, leaf: dict[str, any] = None, to_validate = True, auto: str = 'on', auto_key: str = None) -> dict[str, any]:
+        if to_validate and not self.validate_state(state, show_err=False):
             return {}
 
+        if callable(leaf):
+            if auto_key:
+                auto_key = f'auto-complete.{auto_key}'
+                auto = Config().get(auto_key, 'off')
+
+            if auto == 'on':
+                leaf = leaf()
+            elif auto in ['lazy', 'jit']:
+                leaf = LarkCompleter.from_lambda(auto_key, leaf, auto=auto)
+            else:
+                leaf = None
+
         d = leaf
         for t in reversed(self.command().split(' ')):
             d = {t: d}
@@ -41,7 +73,10 @@ class Command:
     def required(self) -> RequiredState:
         return None
 
-    def validate_state(self, state: ReplState, show_err=True):
+    def validate(self, args: list[str], state: ReplState, apply = True):
+        return ValidateStateHandler(self, args, state, apply = apply)
+
+    def validate_state(self, state: ReplState, show_err = True):
         return state.validate(self.required(), show_err=show_err)
 
     def help(self, _: ReplState) -> str:
@@ -56,6 +91,9 @@ class Command:
         return a
 
     def apply_state(self, args: list[str], state: ReplState, resolve_pg = True, args_to_check = 6) -> tuple[ReplState, list[str]]:
+        """
+        Applies any contextual arguments such as namespace or statefulset to the ReplState and returns any non-contextual arguments.
+        """
         return state.apply_args(args, cmd=self.command_tokens(), resolve_pg=resolve_pg, args_to_check=args_to_check)
 
     def command_tokens(self):
@@ -87,17 +125,52 @@ class Command:
         args.extend(['--help'])
         subprocess.run(args)
 
-    def extract_options(args: list[str], names: list[str]):
-        found: list[str] = []
+    def extract_options(args: list[str], trailing: Union[str, list[str]] = [], sequence: list[str] = [], options: list[str] = []):
+        found_options: list[str] = []
+        found_trailing = None
+        found_sequence = None
+
+        if trailing is None:
+            trailing = []
+        elif isinstance(trailing, str):
+            trailing = [trailing]
+
+        if options is None:
+            options = []
+        elif isinstance(options, str):
+            options = [options]
+
+        if args and trailing:
+            while args and args[-1] in trailing:
+                found_trailing = True
+                args = args[:-1]
+
+        if args and sequence:
+            args, found_sequence = Command.extract_option_sequence(args, sequence)
 
         new_args: list[str] = []
-        for arg in args:
-            if arg in names:
-                found.append(arg)
-            else:
-                new_args.append(arg)
+        if args:
+            for arg in args:
+                if arg in options:
+                    found_options.append(arg)
+                else:
+                    new_args.append(arg)
+
+        return new_args, found_trailing, found_sequence, found_options
+
+    def extract_option_sequence(args, sequence):
+        new_args = args
 
-        return new_args, found
+        len_sub = len(sequence)
+        len_main = len(args)
+        for i in range(len_main - len_sub + 1):
+            if args[i:i+len_sub] == sequence:
+                new_args = copy.copy(args)
+                del new_args[i:i+len_sub]
+
+                return new_args, True
+
+        return new_args, False
 
     def print_chain(cmd: 'Command'):
         print(f'{cmd.command()}', end = '')
@@ -106,31 +179,129 @@ class Command:
             cmd = s
         print()
 
-    def intermediate_run(self, cmd: str, state: ReplState, args: list[str], cmds: list['Command'], separator='\t', display_help=True):
-        state, _ = self.apply_state(args, state)
-
-        if state.in_repl:
-            if display_help:
-                log(lines_to_tabular([c.help(state) for c in cmds], separator=separator))
-
-            return 'command-missing'
+class InvalidStateException(Exception):
+    def __init__(self, state: ReplState):
+        super().__init__(f'Invalid state')
+
+class ValidateStateHandler:
+    def __init__(self, cmd: Command, args: list[str], state: ReplState, apply = True):
+        self.cmd = cmd
+        self.args = args
+        self.state = state
+        self.apply = apply
+
+    def __enter__(self) -> tuple[list[str], ReplState]:
+        state = self.state
+        args = self.args
+        if self.apply:
+            state, args = self.cmd.apply_state(args, state)
         else:
-            # head with the Chain of Responsibility pattern
-            cmds = Command.chain(cmds)
-            if not cmds.run(cmd, state):
-                if display_help:
-                    log2('* Command is missing.')
-                    Command.display_help()
-                return 'command-missing'
-
-        return state
-
-    def intermediate_help(super_help: str, cmd: str, cmd_list: list['Command'], separator='\t', show_cluster_help=False):
-        log(super_help)
-        log()
-        log('Sub-Commands:')
-
-        log(lines_to_tabular([c.help(ReplState()).replace(f'{cmd} ', '  ', 1) for c in cmd_list], separator=separator))
-        if show_cluster_help:
-            log()
-            ClusterCommandHelper.cluster_help()
+            args = args[len(self.cmd.command_tokens()):]
+
+        if not self.cmd.validate_state(state):
+            raise InvalidStateException(state)
+
+        return args, state
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        return False
+
+class InvalidArgumentsException(Exception):
+    def __init__(self):
+        super().__init__(f'Invalid arguments')
+
+class ValidateArgCountHandler:
+    def __init__(self, args: list[str], state: ReplState, at_least: int = 1, exactly: int = 1,
+                 name: str = None, msg: Callable[[], None] = None, default: Union[str, list[str]] = None,
+                 separator: str = None):
+        self.args = args
+        self.state = state
+        self.at_least = at_least
+        self.exactly = exactly
+        self.name = name
+        self.msg = msg
+        self.default = default
+        self.separator = separator
+
+    def __enter__(self) -> Union[tuple[list[str], ReplState], tuple[str, ReplState]]:
+        if self.exactly > 0 and len(self.args) != self.exactly or len(self.args) < self.at_least:
+            if self.default:
+                v = self.default
+                if isinstance(v, list):
+                    v = ' '.join(v)
+
+                return v
+
+            if self.state.in_repl:
+                if self.msg:
+                    self.msg()
+                elif self.name:
+                    log2(f'{self.name} is required.')
+            elif self.name:
+                log2(f'* {self.name} is missing.')
+
+                Command.display_help()
+
+            raise InvalidArgumentsException()
+
+        if self.separator == ' ':
+            return self.args
+
+        # join and re-split with separator
+        args_to_return = ' '.join(self.args)
+        if self.separator:
+            args_to_return = [arg.strip(' ') for arg in args_to_return.split(self.separator)]
+
+        return args_to_return
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        return False
+
+class ExtractOptionsHandler:
+    def __init__(self, args: list[str], options: list[str] = None):
+        self.args = args
+        self.options = options
+
+    def __enter__(self) -> tuple[list[str], list[str]]:
+        args, _, _, options = Command.extract_options(self.args, options=self.options)
+        return args, options
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        return False
+
+class ExtractTrailingOptionsHandler:
+    def __init__(self, args: list[str], trailing: list[str] = None):
+        self.args = args
+        self.trailing = trailing
+
+    def __enter__(self) -> tuple[list[str], list[str]]:
+        args, trailing, _, _ = Command.extract_options(self.args, trailing=self.trailing)
+        return args, trailing
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        return False
+
+class ExtractAllOptionsHandler:
+    def __init__(self, args: list[str], trailing: list[str] = None, sequence: list[str] = None, options: list[str] = None):
+        self.args = args
+        self.trailing = trailing
+        self.sequence = sequence
+        self.options = options
+
+    def __enter__(self) -> tuple[list[str], list[str]]:
+        return Command.extract_options(self.args, trailing=self.trailing, sequence=self.sequence, options=self.options)
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        return False
+
+class ExtractSequenceOptionsHandler:
+    def __init__(self, args: list[str], sequence: list[str]):
+        self.args = args
+        self.sequence = sequence
+
+    def __enter__(self) -> tuple[list[str], list[str]]:
+        args, _, sequence, _ = Command.extract_options(self.args, sequence=self.sequence)
+        return args, sequence
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        return False

adam/commands/commands_utils.py

@@ -1,15 +1,14 @@
-from concurrent.futures import ThreadPoolExecutor
 from kubernetes import client
 from typing import List
 
 from adam.checks.check_utils import run_checks
 from adam.columns.columns import Columns, collect_checks
-from adam.commands.issues import Issues
+from adam.utils_issues import IssuesUtils
 from adam.utils_k8s.cassandra_nodes import CassandraNodes
 from adam.utils_k8s.pods import Pods
 from adam.utils_k8s.statefulsets import StatefulSets
 from adam.repl_state import ReplState
-from adam.utils import duration, lines_to_tabular, log, log2
+from adam.utils import SORT, duration, kaqing_log_file, tabulize, log2
 
 def show_pods(pods: List[client.V1Pod], ns: str, show_namespace = True, show_host_id = True):
     if len(pods) == 0:
@@ -20,18 +19,10 @@ def show_pods(pods: List[client.V1Pod], ns: str, show_namespace = True, show_hos
     if show_host_id:
         names = [pod.metadata.name for pod in pods]
 
-        def get_host_id_with_pod(pod, ns):
-            return (CassandraNodes.get_host_id(pod, ns), pod)
-
-        def body(executor: ThreadPoolExecutor, pod, ns, show_out):
-            if executor:
-                return executor.submit(get_host_id_with_pod, pod, ns)
-
-            id = CassandraNodes.get_host_id(pod, ns)
-
-            return (id, pod)
-
-        host_ids_by_pod = {pod: id for id, pod in Pods.on_pods(names, ns, body, action='get-host-id', show_out=False)}
+        msg = 'd`Retrieving|Retrived {size} host ids'
+        with Pods.parallelize(names, msg=msg, action = 'get-host-id') as exec:
+            host_pods = exec.map(lambda pod: (CassandraNodes.get_host_id(pod, ns), pod))
+            host_ids_by_pod = {pod: id for id, pod in host_pods}
 
     def line(pod: client.V1Pod):
         pod_cnt = len(pod.status.container_statuses)
@@ -57,9 +48,7 @@ def show_pods(pods: List[client.V1Pod], ns: str, show_namespace = True, show_hos
             line += f"@{ns}"
         return line + f" {ready}/{pod_cnt} {status}"
 
-    pod_names = [line(pod) for pod in pods]
-
-    log(lines_to_tabular(pod_names, 'HOST_ID POD_NAME READY POD_STATUS' if show_host_id else 'POD_NAME READY POD_STATUS'))
+    tabulize(pods, line, header='HOST_ID POD_NAME READY POD_STATUS' if show_host_id else 'POD_NAME READY POD_STATUS')
 
 def show_rollout(sts: str, ns: str):
     restarted, rollingout = StatefulSets.restarted_at(sts, ns)
@@ -70,18 +59,22 @@ def show_rollout(sts: str, ns: str):
         else:
             log2(f'Cluster has completed rollout {d} ago.')
 
-def show_table(state: ReplState, pods: list[str], cols: str, header: str, show_output=False):
+def show_table(state: ReplState, pods: list[str], cols: str, header: str, show_out=False, backgrounded = False):
     columns = Columns.create_columns(cols)
 
-    results = run_checks(cluster=state.sts, pod=state.pod, namespace=state.namespace, checks=collect_checks(columns), show_output=show_output)
+    results = run_checks(cluster=state.sts, pod=state.pod, namespace=state.namespace, checks=collect_checks(columns), show_out=show_out)
+
+    r = tabulize(pods, lambda p: ','.join([c.pod_value(results, p) for c in columns]), header=header, separator=',', sorted=SORT, to = 0 if backgrounded else 1)
+
+    if backgrounded:
+        r = write_to_kaqing_log_file(r)
 
-    def line(pod_name: str):
-        cells = [c.pod_value(results, pod_name) for c in columns]
-        return ','.join(cells)
+    IssuesUtils.show(results, state.in_repl)
 
-    lines = [line(pod) for pod in pods]
-    lines.sort()
+    return r
 
-    log(lines_to_tabular(lines, header, separator=','))
+def write_to_kaqing_log_file(r: str):
+    with kaqing_log_file() as f:
+        f.write(r)
 
-    Issues.show(results, state.in_repl)
+        return f.name

adam/commands/{param_get.py → config/param_get.py}

@@ -1,7 +1,8 @@
+from adam.commands import validate_args
 from adam.commands.command import Command
 from adam.config import Config
 from adam.repl_state import ReplState
-from adam.utils import lines_to_tabular, log, log2
+from adam.utils import tabulize, log, log2
 
 class GetParam(Command):
     COMMAND = 'get'
@@ -22,21 +23,17 @@ class GetParam(Command):
         if not(args := self.args(cmd)):
             return super().run(cmd, state)
 
-        state, args = self.apply_state(args, state)
+        with self.validate(args, state) as (args, state):
+            def msg():
+                tabulize(Config().keys(), lambda key: f'{key}\t{Config().get(key, None)}', separator='\t')
 
-        if len(args) < 1:
-            lines = [f'{key}\t{Config().get(key, None)}' for key in Config().keys()]
-            log(lines_to_tabular(lines, separator='\t'))
+            with validate_args(args, state, msg=msg) as key:
+                if v := Config().get(key, None):
+                    log(v)
+                else:
+                    log2(f'{key} is not set.')
 
-            return state
-
-        key = args[0]
-        if v := Config().get(key, None):
-            log(v)
-        else:
-            log2(f'{key} is not set.')
-
-        return v if v else state
+                return v if v else state
 
     def completion(self, _: ReplState):
         return {GetParam.COMMAND: {key: None for key in Config().keys()}}

adam/commands/{param_set.py → config/param_set.py}

@@ -1,3 +1,4 @@
+from adam.commands import validate_args
 from adam.commands.command import Command
 from adam.config import Config
 from adam.repl_state import ReplState
@@ -22,20 +23,15 @@ class SetParam(Command):
         if not(args := self.args(cmd)):
             return super().run(cmd, state)
 
-        state, args = self.apply_state(args, state)
+        with self.validate(args, state) as (args, state):
+            with validate_args(args, state, exactly=2, msg=lambda: log2('set <key> <value>')):
+                key = args[0]
+                value = args[1]
+                Config().set(key, value)
 
-        if len(args) < 2:
-            log2('set <key> <value>')
+                log(Config().get(key, None))
 
-            return 'invalid args'
-
-        key = args[0]
-        value = args[1]
-        Config().set(key, value)
-
-        log(Config().get(key, None))
-
-        return value
+                return value
 
     def completion(self, _: ReplState):
         return {SetParam.COMMAND: {key: ({'true': None, 'false': None} if Config().get(key, None) in [True, False] else None) for key in Config().keys()}}

adam/commands/{show → config}/show_params.py

@@ -1,7 +1,7 @@
 from adam.commands.command import Command
 from adam.config import Config
 from adam.repl_state import ReplState
-from adam.utils import lines_to_tabular, log
+from adam.utils import tabulize
 
 class ShowParams(Command):
     COMMAND = 'show params'
@@ -22,10 +22,7 @@ class ShowParams(Command):
         if not self.args(cmd):
             return super().run(cmd, state)
 
-        lines = [f'{key}\t{Config().get(key, None)}' for key in Config().keys()]
-        log(lines_to_tabular(lines, separator='\t'))
-
-        return lines
+        return tabulize(Config().keys(), lambda k: f'{k}\t{Config().get(k, None)}', separator='\t')
 
     def completion(self, state: ReplState):
         return super().completion(state)

adam/commands/cql/alter_tables.py

@@ -0,0 +1,66 @@
+from adam.commands import extract_options, validate_args
+from adam.commands.command import Command
+from adam.commands.cql.utils_cql import cassandra, cassandra_tables as get_tables
+from adam.config import Config
+from adam.repl_state import ReplState, RequiredState
+from adam.utils import log2, log_exc
+
+class AlterTables(Command):
+    COMMAND = 'alter tables with'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(AlterTables, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def required(self):
+        return RequiredState.CLUSTER
+
+    def command(self):
+        return AlterTables.COMMAND
+
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        with self.validate(args, state) as (args, state):
+            with extract_options(args, '--include-reaper') as (args, include_reaper):
+                with validate_args(args, state, name='gc grace in seconds') as arg_str:
+                    excludes = [e.strip(' \r\n') for e in Config().get(
+                        'cql.alter-tables.excludes',
+                        'system_auth,system_traces,reaper_db,system_distributed,system_views,system,system_schema,system_virtual_schema').split(',')]
+                    batching = Config().get('cql.alter-tables.batching', True)
+                    tables = get_tables(state, on_any=True)
+                    for k, v in tables.items():
+                        if k not in excludes or k == 'reaper_db' and include_reaper:
+                            if batching:
+                                # alter table <table_name> with GC_GRACE_SECONDS = <timeout>;
+                                cql = ';\n'.join([f'alter table {k}.{t} with {arg_str}' for t in v])
+                                with log_exc(True):
+                                    with cassandra(state) as pods:
+                                        pods.cql(cql, show_out=Config().is_debug(), show_query=not Config().is_debug(), on_any=True)
+                                    continue
+                            else:
+                                for t in v:
+                                    with log_exc(True):
+                                        # alter table <table_name> with GC_GRACE_SECONDS = <timeout>;
+                                        cql = f'alter table {k}.{t} with {arg_str}'
+                                        with cassandra(state) as pods:
+                                            pods.cql(show_out=Config().is_debug(), show_query=not Config().is_debug(), on_any=True)
+                                        continue
+
+                            log2(f'{len(v)} tables altered in {k}.')
+
+                    # do not continue to cql route
+                    return state
+
+    def completion(self, _: ReplState) -> dict[str, any]:
+        # auto completion is taken care of by lark completer
+        return {}
+
+    def help(self, _: ReplState) -> str:
+        return f'{AlterTables.COMMAND} <param = value> [--include-reaper] \t alter schema on all tables'

adam/commands/cql/completions_c.py

@@ -0,0 +1,29 @@
+from adam.commands.cql.utils_cql import cassandra_keyspaces, cassandra_table_names
+from adam.commands.export.export_sessions import ExportSessions
+from adam.commands.export.export_databases import ExportDatabases
+from adam.config import Config
+from adam.repl_state import ReplState
+from adam.sql.lark_completer import LarkCompleter
+from adam.utils import log_timing
+from adam.utils_k8s.statefulsets import StatefulSets
+
+def completions_c(state: ReplState) -> dict[str, any]:
+    ps = Config().get('cql.alter-tables.gc-grace-periods', '3600,86400,864000,7776000').split(',')
+
+    with log_timing('lark.completions'):
+        return LarkCompleter(
+            expandables = {
+                'tables': lambda x: cassandra_table_names(state),
+                'keyspaces': lambda x: cassandra_keyspaces(state.with_no_pod()),
+                'table-props': {
+                    'GC_GRACE_SECONDS': ps
+                },
+                'table-props-value': lambda x: {None: None, 'GC_GRACE_SECONDS': ps}[x],
+                'export-database-types': ['athena', 'sqlite', 'csv'],
+                'export-databases': lambda x: ExportDatabases.database_names(),
+                'export-sessions': lambda x: ExportSessions.export_session_names(state.sts, state.pod, state.namespace),
+                'export-sessions-incomplete': lambda x: ExportSessions.export_session_names(state.sts, state.pod, state.namespace, export_state='pending_import'),
+                'hosts': lambda x: [f'@{p}' for p in StatefulSets.pod_names(state.sts, state.namespace)],
+            },
+            variant=ReplState.C
+        ).completions_for_nesting()

adam/commands/cql/cqlsh.py

@@ -1,11 +1,12 @@
 import click
 
+from adam.commands import extract_trailing_options
 from adam.commands.command import Command
 from adam.commands.command_helpers import ClusterOrPodCommandHelper
-from adam.commands.cql.cql_completions import cql_completions
-from .cql_utils import run_cql, cassandra_table_names
+from adam.commands.cql.completions_c import completions_c
+from adam.commands.cql.utils_cql import cassandra
 from adam.repl_state import ReplState, RequiredState
-from adam.utils import log, log2
+from adam.utils import log
 
 class Cqlsh(Command):
     COMMAND = 'cql'
@@ -29,45 +30,22 @@ class Cqlsh(Command):
         if not(args := self.args(cmd)):
             return super().run(cmd, state)
 
-        state, args = self.apply_state(args, state)
-        if not self.validate_state(state):
-            return state
-
-        background = False
-        opts = []
-        cqls = []
-        for index, arg in enumerate(args):
-            if arg.startswith('--'):
-                opts.append(arg)
-            elif index == len(args) -1 and arg == '&':
-                background = True
-            elif arg != '-e':
-                cqls.append(arg)
-        if not cqls:
-            if state.in_repl:
-                log2('Please enter cql statement. e.g. select host_id from system.local')
-            else:
-                log2('* CQL statement is missing.')
-                log2()
-                Command.display_help()
-
-            return 'no-cql'
-
-        cql = ' '.join(cqls)
-        return run_cql(state, cql, opts, show_out=True, background=background)
+        with self.validate(args, state) as (args, state):
+            with extract_trailing_options(args, '&') as (args, backgrounded):
+                with cassandra(state) as pods:
+                    pods.cql(args, backgrounded=backgrounded)
 
     def completion(self, state: ReplState) -> dict[str, any]:
         if state.device != state.C:
-            # conflicts with psql completions
             return {}
 
         if state.sts or state.pod:
-            return {Cqlsh.COMMAND: None} | cql_completions(state)
+            return completions_c(state)
 
         return {}
 
     def help(self, _: ReplState) -> str:
-        return f'[{Cqlsh.COMMAND}] <cql-statements>\t run cqlsh with queries'
+        return f'<cql-statements> [&]\t run cqlsh with queries'
 
 class CqlCommandHelper(click.Command):
     def get_help(self, ctx: click.Context):