kaqing 2.0.172__py3-none-any.whl → 2.0.186__py3-none-any.whl

adam/commands/audit/show_slow10.py

@@ -1,9 +1,6 @@
-from adam.commands.audit.utils_show_top10 import extract_limit_and_duration
+from adam.commands.audit.utils_show_top10 import run_configured_query
 from adam.commands.command import Command
 from adam.repl_state import ReplState
-from adam.utils import log2
-from adam.utils_athena import Athena
-from adam.utils_audits import Audits
 
 class ShowSlow10(Command):
     COMMAND = 'show slow'
@@ -28,15 +25,7 @@ class ShowSlow10(Command):
             return super().run(cmd, state)
 
         with self.validate(args, state) as (args, state):
-            limit, date_condition = extract_limit_and_duration(args)
-
-            query = '\n    '.join([
-                "SELECT * FROM audit",
-                f"WHERE drive <> 'z' and ({date_condition})",
-                f"ORDER BY CAST(duration AS REAL) DESC LIMIT {limit};"])
-            log2(query)
-            log2()
-            Athena.run_query(query)
+            run_configured_query('audit.queries.slow10', args)
 
             return state
 

adam/commands/audit/show_top10.py

@@ -1,8 +1,6 @@
-from adam.commands.audit.utils_show_top10 import extract_limit_and_duration
+from adam.commands.audit.utils_show_top10 import run_configured_query
 from adam.commands.command import Command
 from adam.repl_state import ReplState
-from adam.utils import log2
-from adam.utils_athena import Athena
 
 class ShowTop10(Command):
     COMMAND = 'show top'
@@ -27,14 +25,7 @@ class ShowTop10(Command):
             return super().run(cmd, state)
 
         with self.validate(args, state) as (args, state):
-            limit, date_condition = extract_limit_and_duration(args)
-            query = '\n    '.join([
-                "SELECT min(c) AS cluster, line, COUNT(*) AS cnt, avg(CAST(duration AS REAL)) AS duration",
-                f"FROM audit WHERE drive <> 'z' and ({date_condition})",
-                f"GROUP BY line ORDER BY cnt DESC LIMIT {limit};"])
-            log2(query)
-            log2()
-            Athena.run_query(query)
+            run_configured_query('audit.queries.top10', args)
 
             return state
 

adam/commands/audit/utils_show_top10.py

@@ -1,10 +1,23 @@
 from datetime import datetime, timedelta
 
-from adam.utils import log_exc
+from adam.config import Config
+from adam.utils import log2, log_exc
+from adam.utils_athena import Athena
 from adam.utils_audits import Audits
 from adam.utils_repl.automata_completer import AutomataCompleter
 from adam.utils_repl.state_machine import StateMachine
 
+def run_configured_query(config_key: str, args: list[str]):
+    limit, date_condition = extract_limit_and_duration(args)
+    if query := Config().get(config_key, None):
+        query = '\n    '.join(query.split('\n'))
+        query = query.replace('{date_condition}', date_condition)
+        query = query.replace('{limit}', str(limit))
+
+        log2(query)
+        log2()
+        Athena.run_query(query)
+
 def extract_limit_and_duration(args: list[str]) -> tuple[int, datetime]:
     limit = 10
     _from = datetime.now() - timedelta(days=30)

adam/commands/bash/bash.py

@@ -30,7 +30,7 @@ class Bash(Command):
                 return exec(args)
 
     def completion(self, state: ReplState):
-        return Devices.device(state).bash_completion(Bash.COMMAND, state, default = {})
+        return super().completion(state, {c : {'&': None} for c in ['ls', 'cat', 'head']}, pods=Devices.device(state).pods(state, '-'))
 
     def help(self, _: ReplState):
         return f'{Bash.COMMAND} [pod-name] [bash-commands] [&]\t run bash on the Cassandra nodes'

adam/commands/cat.py

@@ -1,9 +1,7 @@
+from adam.commands import validate_args
 from adam.commands.command import Command
 from adam.commands.devices.devices import Devices
 from adam.repl_state import ReplState, RequiredState
-from adam.utils import log2
-from adam.utils_k8s.app_pods import AppPods
-from adam.utils_k8s.statefulsets import StatefulSets
 
 class Cat(Command):
     COMMAND = 'cat'
@@ -28,23 +26,11 @@ class Cat(Command):
             return super().run(cmd, state)
 
         with self.validate(args, state) as (args, state):
-            if len(args) < 1:
-                if state.in_repl:
-                    log2('File is required.')
-                    log2()
-                else:
-                    log2('* File is missing.')
-                    Command.display_help()
-
-                return 'command-missing'
-
-            return Devices.device(state).cat(cmd, state)
+            with validate_args(args, state, name='file'):
+                return Devices.device(state).bash(state, state, cmd.split(' '))
 
     def completion(self, state: ReplState):
-        if super().completion(state):
-            return Devices.device(state).cat_completion(Cat.COMMAND, state, default={})
-
-        return {}
+        return super().completion(state, lambda: {f: None for f in Devices.device(state).files(state)}, pods=Devices.device(state).pods(state, '-'), auto='jit')
 
     def help(self, _: ReplState):
-        return f'{Cat.COMMAND} file [&]\t run cat command on the Cassandra nodes'
+        return f'{Cat.COMMAND} file [&]\t run cat command on the pod'

adam/commands/cd.py

@@ -1,3 +1,4 @@
+from adam.commands import validate_args
 from adam.commands.command import Command
 from adam.commands.devices.device import Device
 from adam.commands.devices.devices import Devices
@@ -26,15 +27,12 @@ class Cd(Command):
             return super().run(cmd, state)
 
         with self.validate(args, state, apply=False) as (args, state):
-            if len(args) < 2:
-                return state
-
-            device: Device = Devices.device(state)
-            arg = args[1]
-            for dir in arg.split('/'):
-                device.cd(dir, state)
+            with validate_args(args, state, name='directory') as arg_str:
+                device: Device = Devices.device(state)
+                for dir in arg_str.split('/'):
+                    device.cd(dir, state)
 
-            return state
+                return state
 
     def completion(self, state: ReplState):
         return Devices.device(state).cd_completion(Cd.COMMAND, state, default = {})

adam/commands/check.py

@@ -2,12 +2,12 @@ import click
 
 from adam.checks.check_result import CheckResult
 from adam.checks.check_utils import all_checks, checks_from_csv, run_checks
-from adam.commands import extract_options
+from adam.commands import extract_options, validate_args
 from adam.commands.command import Command
 from adam.commands.command_helpers import ClusterOrPodCommandHelper
 from adam.commands.issues import Issues
 from adam.repl_state import ReplState
-from adam.utils import lines_to_tabular, log
+from adam.utils import tabulize, log
 from adam.utils_issues import IssuesUtils
 
 class Check(Issues):
@@ -34,25 +34,17 @@ class Check(Issues):
 
         with self.validate(args, state) as (args, state):
             with extract_options(args, ['-s', '--show']) as (args, show_out):
-                if not args:
-                    if state.in_repl:
-                        log(lines_to_tabular([check.help() for check in all_checks()], separator=':'))
-                    else:
-                        log('* Check name is missing.')
-                        Command.display_help()
+                with validate_args(args, state, name='check name', msg=lambda: tabulize([check.help() for check in all_checks()], separator=':')) as arg:
+                    checks = checks_from_csv(args[0])
+                    if not checks:
+                        return 'invalid check name'
 
-                    return 'arg missing'
+                    results = run_checks(state.sts, state.namespace, state.pod, checks=checks, show_out=show_out)
 
-                checks = checks_from_csv(args[0])
-                if not checks:
-                    return 'invalid check name'
+                    issues = CheckResult.collect_issues(results)
+                    IssuesUtils.show_issues(issues, in_repl=state.in_repl)
 
-                results = run_checks(state.sts, state.namespace, state.pod, checks=checks, show_out=show_out)
-
-                issues = CheckResult.collect_issues(results)
-                IssuesUtils.show_issues(issues, in_repl=state.in_repl)
-
-                return issues if issues else 'no issues found'
+                    return issues if issues else 'no issues found'
 
     def completion(self, _: ReplState):
         return {Check.COMMAND: {check.name(): {'-s': None} for check in all_checks()}}

adam/commands/cli_commands.py

@@ -3,14 +3,19 @@ import re
 
 from adam.commands.reaper.utils_reaper import Reapers
 from adam.config import Config
+from adam.utils import log_timing
 from adam.utils_k8s.kube_context import KubeContext
 from adam.utils_k8s.secrets import Secrets
 from adam.utils_k8s.statefulsets import StatefulSets
 from adam.repl_state import ReplState
 
 class CliCommands:
-    @functools.lru_cache()
     def values(state: ReplState, collapse = False):
+        with log_timing('CliCommands.values'):
+            return CliCommands._values(state, collapse)
+
+    @functools.lru_cache()
+    def _values(state: ReplState, collapse = False):
         # node-exec-?, nodetool-?, cql-?, reaper-exec, reaper-forward, reaper-ui, reaper-usernae, reaper-password
         d = {}
 

adam/commands/{cp.py → clipboard_copy.py}

@@ -1,14 +1,17 @@
+from functools import partial
 import click
 import pyperclip
 
-from adam.commands.command import Command
+from adam.commands import validate_args
+from adam.commands.command import Command, InvalidArgumentsException
 from adam.commands.command_helpers import ClusterOrPodCommandHelper
 from adam.commands.cli_commands import CliCommands
+from adam.config import Config
 from adam.repl_state import ReplState, RequiredState
-from adam.utils import lines_to_tabular, log, log2
+from adam.utils import tabulize, log, log2
 
 class ClipboardCopy(Command):
-    COMMAND = 'cp'
+    COMMAND = 'cli cp'
 
     # the singleton pattern
     def __new__(cls, *args, **kwargs):
@@ -30,40 +33,37 @@ class ClipboardCopy(Command):
             return super().run(cmd, state)
 
         with self.validate(args, state) as (args, state):
-            if len(args) < 1:
-                if state.in_repl:
+            def display_keys(missing = False):
+                if missing:
                     log2('Key is required.')
-                    log2()
-                    log2('Keys:')
-                    log2(lines_to_tabular([f'{k},{v}' for k, v in CliCommands.values(state, collapse=True).items()], separator=','))
                 else:
-                    log2('* Key is missing.')
-                    Command.display_help()
-
-                return 'command-missing'
-
-            key = args[0]
-            if not key in CliCommands.values(state):
-                if state.in_repl:
-                    log2('Key is required.')
-                    log2()
-                    log2('Keys:')
-                    log2(lines_to_tabular([f'{k},{v}' for k, v in CliCommands.values(state, collapse=True).items()], separator=','))
-                else:
-                    log2('* Invalid key')
-                    Command.display_help()
-
-                return 'command-invalid'
-
-            value = CliCommands.values(state)[key]
-            pyperclip.copy(value)
-            log2('The following line has been copied to clipboard. Use <Ctrl-V> to use it.')
-            log2(f'  {value}')
-
-            return 'value-copied'
+                    log2('Key is invalid.')
+                log2()
+                tabulize(CliCommands.values(state, collapse=True).items(),
+                         lambda a: f'{a[0]},{a[1]}',
+                         header='KEY,VALUE',
+                         separator=',',
+                         to=2)
+
+            with validate_args(args, state, name='key', msg=partial(display_keys, True)) as key:
+                if not key in CliCommands.values(state):
+                    if state.in_repl:
+                        display_keys(False)
+                    else:
+                        log2('* Invalid key')
+                        Command.display_help()
+
+                    raise InvalidArgumentsException()
+
+                value = CliCommands.values(state)[key]
+                pyperclip.copy(value)
+                log2('The following line has been copied to clipboard. Use <Ctrl-V> to use it.')
+                log2(f'  {value}')
+
+                return state
 
     def completion(self, state: ReplState):
-        return super().completion(state, lambda: {key: None for key in CliCommands.values(state).keys()})
+        return super().completion(state, lambda: {key: None for key in CliCommands.values(state).keys()}, auto_key='cli.cp-auto-complete')
 
     def help(self, _: ReplState):
         return f"{ClipboardCopy.COMMAND} <key>\t copy a value to clipboard for conveninence"
@@ -82,8 +82,6 @@ class CopyCommandHelper(click.Command):
     def get_help(self, ctx: click.Context):
         log(super().get_help(ctx))
         log()
-        log('Keys:')
-
-        log(lines_to_tabular(self.lines(), separator=':'))
+        tabulize(self.lines(), header='KEY:DESC', separator=':')
         log()
         ClusterOrPodCommandHelper.cluter_or_pod_help()

adam/commands/code.py

@@ -4,7 +4,7 @@ from functools import partial
 from adam.commands.command import Command
 from adam.commands.cql.utils_cql import cassandra_table_names, run_cql
 from adam.repl_state import ReplState, RequiredState
-from adam.utils import lines_to_tabular
+from adam.utils import tabulize
 from adam.utils_k8s.statefulsets import StatefulSets
 
 class Code(Command):
@@ -46,7 +46,7 @@ class Code(Command):
                 "cql('select...'): run cql statement"
             ]
 
-            code.interact(local=my_local, banner=lines_to_tabular(lines, header='Variables', separator=':'))
+            code.interact(local=my_local, banner=tabulize(lines, header='Variables', separator=':'))
 
             return state
 

adam/commands/command.py

@@ -1,11 +1,14 @@
 from abc import abstractmethod
+from collections.abc import Callable
 import copy
 import subprocess
 import sys
 from typing import Union
 
+from adam.config import Config
 from adam.repl_state import ReplState, RequiredState
-from adam.utils import is_lambda
+from adam.sql.lark_completer import LarkCompleter
+from adam.utils import log2
 
 repl_cmds: list['Command'] = []
 
@@ -28,12 +31,38 @@ class Command:
 
         return None
 
-    def completion(self, state: ReplState, leaf: dict[str, any] = None) -> dict[str, any]:
-        if not self.validate_state(state, show_err=False):
+    def completion(self, state: ReplState, leaf: dict[str, any] = None, pods: tuple[list[str], str] = None, auto: str = 'on', auto_key: str = None) -> dict[str, any]:
+        # pods is a tuple of list of pod names and the current pod repl is on
+        if not pods:
+            return self._completion(state, leaf, auto=auto, auto_key=auto_key)
+
+        c = {}
+
+        pod_names = pods[0]
+        pod = pods[1]
+
+        if pod:
+            c |= self._completion(state, leaf, auto=auto, auto_key=auto_key)
+
+        c |= {f'@{p}': self._completion(state, leaf, to_validate=False, auto=auto, auto_key=auto_key) for p in pod_names if p != pod}
+
+        return c
+
+    def _completion(self, state: ReplState, leaf: dict[str, any] = None, to_validate = True, auto: str = 'on', auto_key: str = None) -> dict[str, any]:
+        if to_validate and not self.validate_state(state, show_err=False):
             return {}
 
-        if is_lambda(leaf):
-            leaf = leaf()
+        if callable(leaf):
+            if auto_key:
+                auto_key = f'auto-complete.{auto_key}'
+                auto = Config().get(auto_key, 'off')
+
+            if auto == 'on':
+                leaf = leaf()
+            elif auto in ['lazy', 'jit']:
+                leaf = LarkCompleter.from_lambda(auto_key, leaf, auto=auto)
+            else:
+                leaf = None
 
         d = leaf
         for t in reversed(self.command().split(' ')):
@@ -45,7 +74,7 @@ class Command:
         return None
 
     def validate(self, args: list[str], state: ReplState, apply = True):
-        return ValidateHandler(self, args, state, apply = apply)
+        return ValidateStateHandler(self, args, state, apply = apply)
 
     def validate_state(self, state: ReplState, show_err = True):
         return state.validate(self.required(), show_err=show_err)
@@ -112,7 +141,7 @@ class Command:
             options = [options]
 
         if args and trailing:
-            while args[-1] in trailing:
+            while args and args[-1] in trailing:
                 found_trailing = True
                 args = args[:-1]
 
@@ -130,6 +159,8 @@ class Command:
         return new_args, found_trailing, found_sequence, found_options
 
     def extract_option_sequence(args, sequence):
+        new_args = args
+
         len_sub = len(sequence)
         len_main = len(args)
         for i in range(len_main - len_sub + 1):
@@ -148,11 +179,11 @@ class Command:
             cmd = s
         print()
 
-class InvalidState(Exception):
+class InvalidStateException(Exception):
     def __init__(self, state: ReplState):
         super().__init__(f'Invalid state')
 
-class ValidateHandler:
+class ValidateStateHandler:
     def __init__(self, cmd: Command, args: list[str], state: ReplState, apply = True):
         self.cmd = cmd
         self.args = args
@@ -164,15 +195,68 @@ class ValidateHandler:
         args = self.args
         if self.apply:
             state, args = self.cmd.apply_state(args, state)
+        else:
+            args = args[len(self.cmd.command_tokens()):]
 
         if not self.cmd.validate_state(state):
-            raise InvalidState(state)
+            raise InvalidStateException(state)
 
         return args, state
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         return False
 
+class InvalidArgumentsException(Exception):
+    def __init__(self):
+        super().__init__(f'Invalid arguments')
+
+class ValidateArgCountHandler:
+    def __init__(self, args: list[str], state: ReplState, at_least: int = 1, exactly: int = 1,
+                 name: str = None, msg: Callable[[], None] = None, default: Union[str, list[str]] = None,
+                 separator: str = None):
+        self.args = args
+        self.state = state
+        self.at_least = at_least
+        self.exactly = exactly
+        self.name = name
+        self.msg = msg
+        self.default = default
+        self.separator = separator
+
+    def __enter__(self) -> Union[tuple[list[str], ReplState], tuple[str, ReplState]]:
+        if self.exactly > 0 and len(self.args) != self.exactly or len(self.args) < self.at_least:
+            if self.default:
+                v = self.default
+                if isinstance(v, list):
+                    v = ' '.join(v)
+
+                return v
+
+            if self.state.in_repl:
+                if self.msg:
+                    self.msg()
+                elif self.name:
+                    log2(f'{self.name} is required.')
+            elif self.name:
+                log2(f'* {self.name} is missing.')
+
+                Command.display_help()
+
+            raise InvalidArgumentsException()
+
+        if self.separator == ' ':
+            return self.args
+
+        # join and re-split with separator
+        args_to_return = ' '.join(self.args)
+        if self.separator:
+            args_to_return = [arg.strip(' ') for arg in args_to_return.split(self.separator)]
+
+        return args_to_return
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        return False
+
 class ExtractOptionsHandler:
     def __init__(self, args: list[str], options: list[str] = None):
         self.args = args

adam/commands/commands_utils.py

@@ -1,14 +1,17 @@
+from datetime import datetime
 from kubernetes import client
 from typing import List
 
 from adam.checks.check_utils import run_checks
 from adam.columns.columns import Columns, collect_checks
+from adam.config import Config
+from adam.repl_session import ReplSession
 from adam.utils_issues import IssuesUtils
 from adam.utils_k8s.cassandra_nodes import CassandraNodes
 from adam.utils_k8s.pods import Pods
 from adam.utils_k8s.statefulsets import StatefulSets
 from adam.repl_state import ReplState
-from adam.utils import duration, lines_to_tabular, log, log2
+from adam.utils import SORT, duration, tabulize, log, log2
 
 def show_pods(pods: List[client.V1Pod], ns: str, show_namespace = True, show_host_id = True):
     if len(pods) == 0:
@@ -48,9 +51,7 @@ def show_pods(pods: List[client.V1Pod], ns: str, show_namespace = True, show_hos
             line += f"@{ns}"
         return line + f" {ready}/{pod_cnt} {status}"
 
-    pod_names = [line(pod) for pod in pods]
-
-    log(lines_to_tabular(pod_names, 'HOST_ID POD_NAME READY POD_STATUS' if show_host_id else 'POD_NAME READY POD_STATUS'))
+    tabulize(pods, line, header='HOST_ID POD_NAME READY POD_STATUS' if show_host_id else 'POD_NAME READY POD_STATUS')
 
 def show_rollout(sts: str, ns: str):
     restarted, rollingout = StatefulSets.restarted_at(sts, ns)
@@ -61,18 +62,24 @@ def show_rollout(sts: str, ns: str):
         else:
             log2(f'Cluster has completed rollout {d} ago.')
 
-def show_table(state: ReplState, pods: list[str], cols: str, header: str, show_out=False):
+def show_table(state: ReplState, pods: list[str], cols: str, header: str, show_out=False, backgrounded = False):
     columns = Columns.create_columns(cols)
 
     results = run_checks(cluster=state.sts, pod=state.pod, namespace=state.namespace, checks=collect_checks(columns), show_out=show_out)
 
-    def line(pod_name: str):
-        cells = [c.pod_value(results, pod_name) for c in columns]
-        return ','.join(cells)
+    r = tabulize(pods, lambda p: ','.join([c.pod_value(results, p) for c in columns]), header=header, separator=',', sorted=SORT, to = 0 if backgrounded else 1)
+
+    if backgrounded:
+        log_prefix = Config().get('log-prefix', '/tmp/qing')
+        log_file = f'{log_prefix}-{datetime.now().strftime("%d%H%M%S")}.log'
+
+        with open(log_file, 'w') as f:
+            f.write(r)
+
+        ReplSession().append_history(f':sh cat {log_file}')
 
-    lines = [line(pod) for pod in pods]
-    lines.sort()
+        r = log_file
 
-    log(lines_to_tabular(lines, header, separator=','))
+    IssuesUtils.show(results, state.in_repl)
 
-    IssuesUtils.show(results, state.in_repl)
+    return r

adam/commands/cql/completions_c.py

@@ -0,0 +1,28 @@
+from adam.commands.cql.utils_cql import cassandra_keyspaces, cassandra_table_names
+from adam.commands.export.export_sessions import ExportSessions
+from adam.commands.export.export_databases import ExportDatabases
+from adam.config import Config
+from adam.repl_state import ReplState
+from adam.sql.lark_completer import LarkCompleter
+from adam.utils import log_timing
+from adam.utils_k8s.statefulsets import StatefulSets
+
+def completions_c(state: ReplState) -> dict[str, any]:
+    ps = Config().get('cql.alter-tables.gc-grace-periods', '3600,86400,864000,7776000').split(',')
+
+    with log_timing('lark.completions'):
+        return LarkCompleter(
+            expandables = {
+                'tables': lambda x: cassandra_table_names(state),
+                'keyspaces': lambda x: cassandra_keyspaces(state.with_no_pod()),
+                'table-props': lambda x: {
+                    'GC_GRACE_SECONDS': ps
+                },
+                'export-database-types': lambda x: ['athena', 'sqlite', 'csv'],
+                'export-databases': lambda x: ExportDatabases.database_names(),
+                'export-sessions': lambda x: ExportSessions.export_session_names(state.sts, state.pod, state.namespace),
+                'export-sessions-incomplete': lambda x: ExportSessions.export_session_names(state.sts, state.pod, state.namespace, export_state='pending_import'),
+                'hosts': lambda x: [f'@{p}' for p in StatefulSets.pod_names(state.sts, state.namespace)],
+            },
+            variant=ReplState.C
+        ).completions_for_nesting()

adam/commands/cql/cqlsh.py

@@ -3,9 +3,8 @@ import click
 from adam.commands import extract_trailing_options
 from adam.commands.command import Command
 from adam.commands.command_helpers import ClusterOrPodCommandHelper
-from adam.commands.cql.cql_completions import cql_completions
+from adam.commands.cql.completions_c import completions_c
 from adam.commands.cql.utils_cql import cassandra
-from adam.utils_k8s.statefulsets import StatefulSets
 from adam.repl_state import ReplState, RequiredState
 from adam.utils import log
 
@@ -34,17 +33,14 @@ class Cqlsh(Command):
         with self.validate(args, state) as (args, state):
             with extract_trailing_options(args, '&') as (args, backgrounded):
                 with cassandra(state) as pods:
-                    pods.cql(args, background=backgrounded)
+                    pods.cql(args, backgrounded=backgrounded)
 
     def completion(self, state: ReplState) -> dict[str, any]:
         if state.device != state.C:
-            # conflicts with psql completions
             return {}
 
         if state.sts or state.pod:
-            c = cql_completions(state)
-            return c | \
-                   {f'@{p}': c for p in StatefulSets.pod_names(state.sts, state.namespace)}
+            return completions_c(state)
 
         return {}