kaqing 1.98.15__py3-none-any.whl → 2.0.145__py3-none-any.whl

adam/commands/audit/show_slow10.py

@@ -0,0 +1,50 @@
+from adam.commands.audit.utils_show_top10 import extract_limit_and_duration
+from adam.commands.command import Command
+from adam.repl_state import ReplState
+from adam.utils import log2
+from adam.utils_athena import Athena
+from adam.utils_audits import Audits
+
+class ShowSlow10(Command):
+    COMMAND = 'show slow'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(ShowSlow10, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return ShowSlow10.COMMAND
+
+    def required(self):
+        return ReplState.L
+
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        state, args = self.apply_state(args, state)
+        if not self.validate_state(state):
+            return state
+
+        limit, date_condition = extract_limit_and_duration(args)
+
+        query = '\n    '.join([
+            "SELECT * FROM audit",
+            f"WHERE drive <> 'z' and ({date_condition})",
+            f"ORDER BY CAST(duration AS REAL) DESC LIMIT {limit};"])
+        log2(query)
+        log2()
+        Athena.run_query(query)
+
+        return state
+
+    def completion(self, _: ReplState):
+        return {}
+
+    def help(self, _: ReplState):
+        return f'{ShowSlow10.COMMAND} [limit]\t show slow <limit> audit lines'

adam/commands/audit/show_top10.py

@@ -0,0 +1,48 @@
+from adam.commands.audit.utils_show_top10 import extract_limit_and_duration
+from adam.commands.command import Command
+from adam.repl_state import ReplState
+from adam.utils import log2
+from adam.utils_athena import Athena
+
+class ShowTop10(Command):
+    COMMAND = 'show top'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(ShowTop10, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return ShowTop10.COMMAND
+
+    def required(self):
+        return ReplState.L
+
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        state, args = self.apply_state(args, state)
+        if not self.validate_state(state):
+            return state
+
+        limit, date_condition = extract_limit_and_duration(args)
+        query = '\n    '.join([
+            "SELECT min(c) AS cluster, line, COUNT(*) AS cnt, avg(CAST(duration AS REAL)) AS duration",
+            f"FROM audit WHERE drive <> 'z' and ({date_condition})",
+            f"GROUP BY line ORDER BY cnt DESC LIMIT {limit};"])
+        log2(query)
+        log2()
+        Athena.run_query(query)
+
+        return state
+
+    def completion(self, _: ReplState):
+        return {}
+
+    def help(self, _: ReplState):
+        return f'{ShowTop10.COMMAND} [limit]\t show top <limit> audit lines'

adam/commands/audit/utils_show_top10.py

@@ -0,0 +1,59 @@
+from datetime import datetime, timedelta
+
+from adam.utils_audits import Audits
+from adam.utils_repl.automata_completer import AutomataCompleter
+from adam.utils_repl.state_machine import StateMachine
+
+def extract_limit_and_duration(args: list[str]) -> tuple[int, datetime]:
+    limit = 10
+    _from = datetime.now() - timedelta(days=30)
+    if args:
+        try:
+            limit = int(args[0])
+        except:
+            pass
+
+        if len(args) > 2 and args[1] == 'over':
+            if args[2] == 'day':
+                _from = datetime.now() - timedelta(days=1)
+
+    return (limit, Audits.date_from(_from))
+
+def limit_and_duration_completion():
+    return {'10': {'over': {
+        'day': None,
+        'month': None
+    }}}
+
+SHOW_TOP10_SPEC = [
+    '                                > show           > show',
+    'show                            > last|slow|top  > show_top                            ^ last,slow,top',
+    'show_top                        > word           > show_top_n                          ^ 10',
+    'show_top_n                      > over           > show_top_n_over                     ^ over',
+    'show_top_n_over                 > day|month      > show_top_n_over$                    ^ day,month',
+]
+
+SHOW_TOP10_KEYWORDS = [
+    'show',
+    'top',
+    'last',
+    'slow',
+    'over',
+    'day',
+    'month'
+]
+
+class ShowTop10StateMachine(StateMachine[str]):
+    def spec(self) -> str:
+        return SHOW_TOP10_SPEC
+
+    def keywords(self) -> list[str]:
+        return SHOW_TOP10_KEYWORDS
+
+def show_top10_completions_for_nesting():
+    return {
+        'show': {
+            'last': AutomataCompleter(ShowTop10StateMachine(), first_term='show last'),
+            'slow': AutomataCompleter(ShowTop10StateMachine(), first_term='show slow'),
+            'top': AutomataCompleter(ShowTop10StateMachine(), first_term='show top'),
+    }}

adam/commands/bash/bash.py

@@ -0,0 +1,133 @@
+from typing import Callable
+
+from adam.commands.bash.bash_completer import BashCompleter
+from adam.commands.command import Command
+from adam.utils_k8s.app_clusters import AppClusters
+from adam.utils_k8s.app_pods import AppPods
+from adam.utils_k8s.cassandra_clusters import CassandraClusters
+from adam.utils_k8s.cassandra_nodes import CassandraNodes
+from adam.pod_exec_result import PodExecResult
+from adam.repl_state import BashSession, ReplState, RequiredState
+from adam.utils_k8s.pods import Pods
+from adam.utils_k8s.statefulsets import StatefulSets
+from adam.utils_k8s.cassandra_nodes import CassandraNodes
+
+class Bash(Command):
+    COMMAND = 'bash'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(Bash, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return Bash.COMMAND
+
+    def required(self):
+        return [RequiredState.CLUSTER_OR_POD, RequiredState.APP_APP, ReplState.P]
+
+    def run(self, cmd: str, s0: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, s0)
+
+        state, args = self.apply_state(args, s0, args_to_check=2)
+        if not self.validate_state(state):
+            return state
+
+        def _run(state_changed: bool, cli: Callable[[str], None]):
+            if state.in_repl:
+                if state_changed:
+                    r = self.exec_with_dir(state, args)
+                else:
+                    r = self.exec_with_dir(s0, args)
+
+                if not r:
+                    state.exit_bash()
+
+                    return 'inconsistent pwd'
+
+                return r
+            else:
+                cli(' '.join(args))
+
+                return state
+
+        if state.device == ReplState.A:
+            def cli(command: str):
+                if state.app_pod:
+                    AppPods.exec(state.app_pod, state.namespace, command, show_out=True)
+                elif state.app_app:
+                    AppClusters.exec(AppPods.pod_names(state.namespace, state.app_env, state.app_app), state.namespace, command, action='bash', show_out=True)
+
+            return _run(s0.app_env != state.app_env or s0.app_app != state.app_app or s0.app_pod != state.app_pod, cli)
+        elif state.device == ReplState.P:
+            def cli(command: str):
+                Pods.exec(state.app_pod, 'ops', state.namespace, command, show_out=True)
+
+            return _run(s0.pg_path != state.pg_path, cli)
+
+        def cli(command: str):
+            if state.pod:
+                CassandraNodes.exec(state.pod, state.namespace, command, show_out=True)
+            elif state.sts:
+                CassandraClusters.exec(state.sts, state.namespace, command, action='bash', show_out=True)
+
+        return _run(s0.sts != state.sts or s0.pod != state.pod, cli)
+
+    def exec_with_dir(self, state: ReplState, args: list[str]) -> list[PodExecResult]:
+        session_just_created = False
+        if not args:
+            session_just_created = True
+            session = BashSession(state.device)
+            state.enter_bash(session)
+
+        if state.bash_session:
+            if args != ['pwd']:
+                if args:
+                    args.append('&&')
+                args.extend(['pwd', '>', f'/tmp/.qing-{state.bash_session.session_id}'])
+
+            if not session_just_created:
+                if pwd := state.bash_session.pwd(state):
+                    args = ['cd', pwd, '&&'] + args
+
+        command = ' '.join(args)
+
+        rs = []
+
+        if state.device == ReplState.A:
+            if state.app_pod:
+                rs = [AppPods.exec(state.app_pod, state.namespace, command,
+                                        show_out=not session_just_created, shell='bash')]
+            elif state.app_app:
+                rs = AppClusters.exec(AppPods.pod_names(state.namespace, state.app_env, state.app_app), state.namespace, command, action='bash',
+                                            show_out=not session_just_created, shell='bash')
+        elif state.device == ReplState.P:
+            rs = [Pods.exec(state.app_pod, 'ops', state.namespace, command,
+                                    show_out=not session_just_created, shell='bash')]
+        else:
+            if state.pod:
+                rs = [CassandraNodes.exec(state.pod, state.namespace, command,
+                                        show_out=not session_just_created, shell='bash')]
+            elif state.sts:
+                rs = CassandraClusters.exec(state.sts, state.namespace, command, action='bash',
+                                            show_out=not session_just_created, shell='bash')
+
+        return rs
+
+    def completion(self, state: ReplState):
+        if state.device == ReplState.A and state.app_app:
+            return { Bash.COMMAND: BashCompleter(lambda: []) } | \
+                   {f'@{p}': {Bash.COMMAND: BashCompleter(lambda: [])} for p in AppPods.pod_names(state.namespace, state.app_env, state.app_app)}
+        elif state.sts:
+            return { Bash.COMMAND: BashCompleter(lambda: []) } | \
+                   {f'@{p}': {Bash.COMMAND: BashCompleter(lambda: [])} for p in StatefulSets.pod_names(state.sts, state.namespace)}
+
+        return {}
+
+    def help(self, _: ReplState):
+        return f'{Bash.COMMAND} [pod-name] [bash-commands] [&]\t run bash on the Cassandra nodes'

adam/commands/bash/bash_completer.py

@@ -0,0 +1,93 @@
+from typing import Callable
+from prompt_toolkit.completion import WordCompleter
+
+from adam.utils_repl.automata_completer import AutomataCompleter
+from adam.utils_repl.state_machine import State, StateMachine
+
+BASH_SPEC = [
+    # <command> ::= <simple_command> | <pipeline> | <conditional_command>
+    # <simple_command> ::= <word> <argument>* <redirection>*
+    # <pipeline> ::= <command> '|' <command>
+    # <conditional_command> ::= <command> '&&' <command> | <command> '||' <command>
+    # <word> ::= <letter> <letter_or_digit>*
+    # <argument> ::= <word>
+    # <redirection> ::= '>' <filename> | '<' <filename>
+    # <filename> ::= <word>
+    # <letter> ::= 'a' | 'b' | ... | 'z' | 'A' | 'B' | ... | 'Z'
+    # <digit> ::= '0' | '1' | ... | '9'
+    # <letter_or_digit> ::= <letter> | <digit>
+
+    '                                > word           > cmd                                ^ hosts',
+    'cmd                             > word           > cmd                                ^ |,>,2>,<,&,&&,||',
+    '-                               > pipe           > cmd_pipe',
+    '-                               > _rdr0_         > cmd_rdr0',
+    '-                               > _rdr1_         > cmd_rdr1',
+    '-                               > _rdr2_         > cmd_rdr2',
+    '-                               > &              > cmd_bg                             ^ |,>,2>,<,&,&&,||',
+    '-                               > &&|_or_        > nocmd',
+    'cmd_a                           > word           > cmd',
+    'cmd_pipe                        > word           > cmd',
+    'cmd_rdr0                        > word           > cmd_rdr0_f',
+    'cmd_rdr1                        > word           > cmd_rdr1_f',
+    'cmd_rdr2                        > word           > cmd_rdr2_f',
+    'cmd_rdr1_f                      > pipe           > cmd_pipe                           ^ |,2>,<,&,&&,||',
+    '-                               > _rdr2_         > cmd_rdr2',
+    '-                               > _rdr0_         > cmd_rdr0',
+    'cmd_rdr2_f                      > pipe           > cmd_pipe                           ^ |,<,&,&&,||',
+    '-                               > _rdr0_         > cmd_rdr0',
+    '-                               > &              > cmd_bg                             ^ |,>,2>,<,&,&&,||',
+    '-                               > &&|_or_        > nocmd',
+    'cmd_rdr0_f                      > pipe           > cmd_pipe                           ^ |,&,&&,||',
+    '-                               > &              > cmd_bg                             ^ |,>,2>,<,&,&&,||',
+    '-                               > &&|_or_        > cmd',
+    'cmd_bg                          > &&|_or_        > nocmd                              ^ &&,||',
+    'nocmd                           > word           > cmd',
+]
+
+BASH_KEYWORDS = [
+    '&',
+    '&&',
+    '|',
+    '||',
+    '>',
+    '2>',
+    '>>',
+    '<',
+    'hosts'
+]
+
+class BashStateMachine(StateMachine[str]):
+    def spec(self) -> str:
+        return BASH_SPEC
+
+    def keywords(self) -> list[str]:
+        return BASH_KEYWORDS
+
+class BashCompleter(AutomataCompleter[str]):
+    def __init__(self,
+                 hosts: Callable[[], list[str]],
+                 debug = False):
+        super().__init__(BashStateMachine(debug=debug), '', debug=debug)
+
+        self.hosts = hosts
+        self.debug = debug
+
+    def suggestions_completer(self, state: State, suggestions: str) -> list[str]:
+        if not suggestions:
+            return None
+
+        terms = []
+        for suggestion in suggestions.split(','):
+            terms.extend(self._terms(state, suggestion))
+
+        return WordCompleter(terms)
+
+    def _terms(self, _: State, word: str) -> list[str]:
+        terms = []
+
+        if word == 'hosts':
+            terms.extend(self.hosts())
+        else:
+            terms.append(word)
+
+        return terms

adam/commands/cat.py

@@ -0,0 +1,56 @@
+from adam.commands.bash.bash import Bash
+from adam.commands.command import Command
+from adam.repl_state import ReplState, RequiredState
+from adam.utils import log2
+from adam.utils_k8s.app_pods import AppPods
+from adam.utils_k8s.statefulsets import StatefulSets
+
+class Cat(Command):
+    COMMAND = 'cat'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(Cat, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return Cat.COMMAND
+
+    def required(self):
+        return [RequiredState.CLUSTER_OR_POD, RequiredState.APP_APP, ReplState.P]
+
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        state, args = self.apply_state(args, state)
+
+        if len(args) < 1:
+            if state.in_repl:
+                log2('File is required.')
+                log2()
+            else:
+                log2('* File is missing.')
+                Command.display_help()
+
+            return 'command-missing'
+
+
+        return Bash().run('bash ' + cmd, state)
+
+    def completion(self, state: ReplState):
+        if state.device == ReplState.A and state.app_app:
+            return super().completion(state) | \
+                {f'@{p}': {Cat.COMMAND: None} for p in AppPods.pod_names(state.namespace, state.app_env, state.app_app)}
+        elif state.device == ReplState.C and state.sts:
+            return super().completion(state) | \
+                {f'@{p}': {Cat.COMMAND: None} for p in StatefulSets.pod_names(state.sts, state.namespace)}
+
+        return {}
+
+    def help(self, _: ReplState):
+        return f'{Cat.COMMAND} file [&]\t run cat command on the Cassandra nodes'

adam/commands/cd.py

@@ -1,11 +1,7 @@
 from adam.commands.command import Command
-from adam.commands.postgres.postgres_session import PostgresSession
-from adam.k8s_utils.cassandra_clusters import CassandraClusters
-from adam.k8s_utils.kube_context import KubeContext
-from adam.k8s_utils.statefulsets import StatefulSets
+from adam.commands.devices.device import Device
+from adam.commands.devices.devices import Devices
 from adam.repl_state import ReplState
-from adam.utils import log2
-from adam.apps import Apps
 
 class Cd(Command):
     COMMAND = 'cd'
@@ -22,94 +18,28 @@ class Cd(Command):
     def command(self):
         return Cd.COMMAND
 
+    def required(self):
+        return ReplState.NON_L
+
     def run(self, cmd: str, state: ReplState):
         if not(args := self.args(cmd)):
             return super().run(cmd, state)
 
+        if not self.validate_state(state):
+            return state
+
         if len(args) < 2:
             return state
 
+        device: Device = Devices.device(state)
         arg = args[1]
         for dir in arg.split('/'):
-            if state.device == ReplState.P:
-                if dir == '':
-                    state.pg_path = None
-                else:
-                    session = PostgresSession(state.namespace, state.pg_path)
-                    # patch up state.namespace from pg cd
-                    if not state.namespace and (ns := session.find_namespace(arg)):
-                        state.namespace = ns
-                    state.pg_path = session.directory(arg)
-            elif state.device == ReplState.A:
-                if dir == '':
-                    state.app_env = None
-                    state.app_app = None
-                elif dir == '..':
-                    if state.app_app:
-                        state.app_app = None
-                    else:
-                        state.app_env = None
-                else:
-                    if not state.app_env:
-                        tks = dir.split('@')
-                        if len(tks) > 1:
-                            state.namespace = tks[1]
-
-                        state.app_env = dir.split('@')[0]
-                    else:
-                        state.app_app = dir
-            else:
-                if dir == '':
-                    state.sts = None
-                    state.pod = None
-                elif dir == '..':
-                    if state.pod:
-                        state.pod = None
-                    else:
-                        state.sts = None
-                else:
-                    if not state.sts:
-                        ss_and_ns = dir.split('@')
-                        state.sts = ss_and_ns[0]
-                        state.namespace = ss_and_ns[1]
-                    elif not state.pod:
-                        p, _ = KubeContext.is_pod_name(dir)
-                        if p:
-                            state.pod = p
-                        else:
-                            names = CassandraClusters.pod_names_by_host_id(state.sts, state.namespace);
-                            if dir in names:
-                                state.pod = names[dir]
-                            else:
-                                log2('Not a valid pod name or host id.')
+            device.cd(dir, state)
 
         return state
 
     def completion(self, state: ReplState):
-        if state.device == ReplState.P:
-            pg = PostgresSession(state.namespace, state.pg_path) if state.pg_path else None
-            if pg and pg.db:
-                return {Cd.COMMAND: {'..': None}}
-            elif pg and pg.host:
-                return {Cd.COMMAND: {'..': None} | {p['name']: None for p in pg.databases()}}
-            else:
-                return {Cd.COMMAND: {p: None for p in PostgresSession.hosts(state.namespace)}}
-        elif state.device == ReplState.A:
-            if state.app_app:
-                return {Cd.COMMAND: {'..': None}}
-            elif state.app_env:
-                return {Cd.COMMAND: {'..': None} | {app[0].split('-')[1]: None for app in Apps.apps(state.app_env)}}
-            else:
-                return {Cd.COMMAND: {'..': None} | {env[0]: None for env in Apps.envs()}}
-        else:
-            if state.pod:
-                return {Cd.COMMAND: {'..': None}}
-            elif state.sts:
-                return {Cd.COMMAND: {'..': None} | {p: None for p in StatefulSets.pod_names(state.sts, state.namespace)}}
-            else:
-                return {Cd.COMMAND: {p: None for p in StatefulSets.list_sts_names()}}
-
-        return {}
+        return Devices.device(state).cd_completion(Cd.COMMAND, state, default = {})
 
     def help(self, _: ReplState):
-        return f'{Cd.COMMAND} <path> | .. \t move around'
+        return f'{Cd.COMMAND} <path> | .. \t move around on the operational device hierarchy'

adam/commands/check.py

@@ -23,11 +23,17 @@ class Check(Issues):
     def command(self):
         return Check.COMMAND
 
+    def required(self):
+        return ReplState.NON_L
+
     def run(self, cmd: str, state: ReplState):
         if not(args := self.args(cmd)):
             return super().run(cmd, state)
 
         state, args = self.apply_state(args, state)
+        if not self.validate_state(state):
+            return state
+
         args, show = Command.extract_options(args, ['-s', '--show'])
 
         if not args:

adam/commands/cli_commands.py

@@ -3,9 +3,9 @@ import re
 
 from adam.commands.reaper.reaper_session import ReaperSession
 from adam.config import Config
-from adam.k8s_utils.kube_context import KubeContext
-from adam.k8s_utils.secrets import Secrets
-from adam.k8s_utils.statefulsets import StatefulSets
+from adam.utils_k8s.kube_context import KubeContext
+from adam.utils_k8s.secrets import Secrets
+from adam.utils_k8s.statefulsets import StatefulSets
 from adam.repl_state import ReplState
 
 class CliCommands:

adam/commands/code.py

@@ -0,0 +1,60 @@
+import code
+from functools import partial
+
+from adam.commands.command import Command
+from adam.commands.cql.cql_utils import cassandra_table_names, run_cql
+from adam.repl_state import ReplState, RequiredState
+from adam.utils import lines_to_tabular
+from adam.utils_k8s.statefulsets import StatefulSets
+
+class Code(Command):
+    COMMAND = 'python'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(Code, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return Code.COMMAND
+
+    def required(self):
+        return RequiredState.NAMESPACE
+
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        state, args = self.apply_state(args, state)
+        if not self.validate_state(state):
+            return state
+
+        sts = state.sts
+        pod = state.pod
+        pods = StatefulSets.pod_names(state.sts, state.namespace)
+        tables = cassandra_table_names(state)
+        cql = partial(run_cql, state, show_out=True)
+
+        my_local = globals() | locals()
+        # my_local = globals() | {'StatefulSets': StatefulSets} | locals()
+        lines = [
+            'sts: StatefulSet name',
+            'pod: Pod name',
+            'pods: Pod names in the current StatefulSet',
+            'tables: Cassandra tables names in the current StatefulSet',
+            "cql('select...'): run cql statement"
+        ]
+
+        code.interact(local=my_local, banner=lines_to_tabular(lines, header='Variables', separator=':'))
+
+        return state
+
+    def completion(self, state: ReplState):
+        return super().completion(state)
+
+    def help(self, _: ReplState):
+        return f'{Code.COMMAND} \t invoke interactive Python shell'