kaqing 1.77.0__py3-none-any.whl → 2.0.171__py3-none-any.whl

adam/commands/audit/audit_repair_tables.py

@@ -0,0 +1,74 @@
+import time
+
+from adam.commands.command import Command
+from adam.config import Config
+from adam.repl_state import ReplState
+from adam.utils import log, log2
+from adam.utils_athena import Athena
+from adam.utils_audits import AuditMeta, Audits
+
+class AuditRepairTables(Command):
+    COMMAND = 'audit repair'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(AuditRepairTables, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+        self.auto_repaired = False
+
+    def command(self):
+        return AuditRepairTables.COMMAND
+
+    def required(self):
+        return ReplState.L
+
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        with self.validate(args, state) as (args, state):
+            tables = Config().get('audit.athena.repair-partition-tables', 'audit').split(',')
+            if args:
+                tables = args
+
+            meta = Audits.get_meta()
+            self.repair(tables, meta)
+
+            return state
+
+    def completion(self, state: ReplState):
+        # trigger auto repair if on l: drive
+        if state.device == ReplState.L:
+            if not self.auto_repaired:
+                if hours := Config().get('audit.athena.auto-repair.elapsed_hours', 12):
+                    with Audits.offload() as exec:
+                        exec.submit(lambda: self.auto_repair(hours))
+
+            return super().completion(state)
+
+        return {}
+
+    def auto_repair(self, hours: int):
+        self.auto_repaired = True
+
+        meta: AuditMeta = Audits.get_meta()
+        if meta.partitions_last_checked + hours * 60 * 60 < time.time():
+            tables = Config().get('audit.athena.repair-partition-tables', 'audit').split(',')
+            self.repair(tables, meta, show_sql=True)
+            log2(f'Audit tables have been auto-repaired.')
+
+    def repair(self, tables: list[str], meta: AuditMeta, show_sql = False):
+        with Audits.offload() as exec:
+            for table in tables:
+                if show_sql:
+                    log(f'MSCK REPAIR TABLE {table}')
+
+                exec.submit(Athena.query, f'MSCK REPAIR TABLE {table}', None,)
+            exec.submit(Audits.put_meta, Audits.PARTITIONS_ADDED, meta,)
+
+    def help(self, _: ReplState):
+        return f"{AuditRepairTables.COMMAND}\t run MSCK REPAIR command for new partition discovery"

adam/commands/audit/audit_run.py

@@ -0,0 +1,50 @@
+from adam.commands.command import Command
+from adam.config import Config
+from adam.repl_state import ReplState
+from adam.utils import log2
+from adam.utils_athena import Athena
+from adam.utils_audits import AuditMeta, Audits
+
+class AuditRun(Command):
+    COMMAND = 'audit run'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(AuditRun, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+        self.auto_repaired = False
+
+    def command(self):
+        return AuditRun.COMMAND
+
+    def required(self):
+        return ReplState.L
+
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        with self.validate(args, state) as (args, state):
+            meta: AuditMeta = Audits.get_meta()
+            clusters = Audits.find_new_clusters(meta.cluster_last_checked)
+            Audits.put_meta(Audits.ADD_CLUSTERS, meta, clusters=clusters)
+            if clusters:
+                log2(f'Added {len(clusters)} new clusters.')
+                tables = Config().get('audit.athena.repair-cluster-tables', 'cluster').split(',')
+                with Audits.offload() as exec:
+                    for table in tables:
+                        exec.submit(Athena.query, f'MSCK REPAIR TABLE {table}', None,)
+            else:
+                log2(f'No new clusters were found.')
+
+            return state
+
+    def completion(self, state: ReplState):
+        return super().completion(state)
+
+    def help(self, _: ReplState):
+        return f"{AuditRun.COMMAND}\t run"

adam/commands/audit/show_last10.py

@@ -0,0 +1,48 @@
+from adam.commands.audit.utils_show_top10 import extract_limit_and_duration
+from adam.commands.command import Command
+from adam.repl_state import ReplState
+from adam.utils import log2
+from adam.utils_athena import Athena
+from adam.utils_audits import Audits
+
+class ShowLast10(Command):
+    COMMAND = 'show last'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(ShowLast10, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return ShowLast10.COMMAND
+
+    def required(self):
+        return ReplState.L
+
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        with self.validate(args, state) as (args, state):
+            limit, date_condition = extract_limit_and_duration(args)
+
+            query = '\n    '.join([
+                "SELECT * FROM audit",
+                f"WHERE drive <> 'z' and ({date_condition})",
+                f"ORDER BY ts DESC LIMIT {limit};"])
+            log2(query)
+            log2()
+            Athena.run_query(query)
+
+            return state
+
+    def completion(self, _: ReplState):
+
+        return {}
+
+    def help(self, _: ReplState):
+        return f'{ShowLast10.COMMAND} [limit]\t show last <limit> audit lines'

adam/commands/audit/show_slow10.py

@@ -0,0 +1,47 @@
+from adam.commands.audit.utils_show_top10 import extract_limit_and_duration
+from adam.commands.command import Command
+from adam.repl_state import ReplState
+from adam.utils import log2
+from adam.utils_athena import Athena
+from adam.utils_audits import Audits
+
+class ShowSlow10(Command):
+    COMMAND = 'show slow'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(ShowSlow10, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return ShowSlow10.COMMAND
+
+    def required(self):
+        return ReplState.L
+
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        with self.validate(args, state) as (args, state):
+            limit, date_condition = extract_limit_and_duration(args)
+
+            query = '\n    '.join([
+                "SELECT * FROM audit",
+                f"WHERE drive <> 'z' and ({date_condition})",
+                f"ORDER BY CAST(duration AS REAL) DESC LIMIT {limit};"])
+            log2(query)
+            log2()
+            Athena.run_query(query)
+
+            return state
+
+    def completion(self, _: ReplState):
+        return {}
+
+    def help(self, _: ReplState):
+        return f'{ShowSlow10.COMMAND} [limit]\t show slow <limit> audit lines'

adam/commands/audit/show_top10.py

@@ -0,0 +1,45 @@
+from adam.commands.audit.utils_show_top10 import extract_limit_and_duration
+from adam.commands.command import Command
+from adam.repl_state import ReplState
+from adam.utils import log2
+from adam.utils_athena import Athena
+
+class ShowTop10(Command):
+    COMMAND = 'show top'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(ShowTop10, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return ShowTop10.COMMAND
+
+    def required(self):
+        return ReplState.L
+
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        with self.validate(args, state) as (args, state):
+            limit, date_condition = extract_limit_and_duration(args)
+            query = '\n    '.join([
+                "SELECT min(c) AS cluster, line, COUNT(*) AS cnt, avg(CAST(duration AS REAL)) AS duration",
+                f"FROM audit WHERE drive <> 'z' and ({date_condition})",
+                f"GROUP BY line ORDER BY cnt DESC LIMIT {limit};"])
+            log2(query)
+            log2()
+            Athena.run_query(query)
+
+            return state
+
+    def completion(self, _: ReplState):
+        return {}
+
+    def help(self, _: ReplState):
+        return f'{ShowTop10.COMMAND} [limit]\t show top <limit> audit lines'

adam/commands/audit/utils_show_top10.py

@@ -0,0 +1,59 @@
+from datetime import datetime, timedelta
+
+from adam.utils_audits import Audits
+from adam.utils_repl.automata_completer import AutomataCompleter
+from adam.utils_repl.state_machine import StateMachine
+
+def extract_limit_and_duration(args: list[str]) -> tuple[int, datetime]:
+    limit = 10
+    _from = datetime.now() - timedelta(days=30)
+    if args:
+        try:
+            limit = int(args[0])
+        except:
+            pass
+
+        if len(args) > 2 and args[1] == 'over':
+            if args[2] == 'day':
+                _from = datetime.now() - timedelta(days=1)
+
+    return (limit, Audits.date_from(_from))
+
+def limit_and_duration_completion():
+    return {'10': {'over': {
+        'day': None,
+        'month': None
+    }}}
+
+SHOW_TOP10_SPEC = [
+    '                                > show           > show',
+    'show                            > last|slow|top  > show_top                            ^ last,slow,top',
+    'show_top                        > word           > show_top_n                          ^ 10',
+    'show_top_n                      > over           > show_top_n_over                     ^ over',
+    'show_top_n_over                 > day|month      > show_top_n_over$                    ^ day,month',
+]
+
+SHOW_TOP10_KEYWORDS = [
+    'show',
+    'top',
+    'last',
+    'slow',
+    'over',
+    'day',
+    'month'
+]
+
+class ShowTop10StateMachine(StateMachine[str]):
+    def spec(self) -> str:
+        return SHOW_TOP10_SPEC
+
+    def keywords(self) -> list[str]:
+        return SHOW_TOP10_KEYWORDS
+
+def show_top10_completions_for_nesting():
+    return {
+        'show': {
+            'last': AutomataCompleter(ShowTop10StateMachine(), first_term='show last'),
+            'slow': AutomataCompleter(ShowTop10StateMachine(), first_term='show slow'),
+            'top': AutomataCompleter(ShowTop10StateMachine(), first_term='show top'),
+    }}

adam/commands/bash/__init__.py

@@ -0,0 +1,5 @@
+from adam.commands.bash.utils_bash import BashHandler
+from adam.repl_state import ReplState
+
+def bash(s0: ReplState, s1: ReplState):
+    return BashHandler(s0, s1)

adam/commands/bash/bash.py

@@ -0,0 +1,36 @@
+from adam.commands.bash import bash
+from adam.commands.command import Command
+from adam.commands.devices.devices import Devices
+from adam.repl_state import ReplState, RequiredState
+
+class Bash(Command):
+    COMMAND = 'bash'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(Bash, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return Bash.COMMAND
+
+    def required(self):
+        return [RequiredState.CLUSTER_OR_POD, RequiredState.APP_APP, ReplState.P]
+
+    def run(self, cmd: str, s0: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, s0)
+
+        with self.validate(args, s0) as (args, s1):
+            with bash(s0, s1) as exec:
+                return exec(args)
+
+    def completion(self, state: ReplState):
+        return Devices.device(state).bash_completion(Bash.COMMAND, state, default = {})
+
+    def help(self, _: ReplState):
+        return f'{Bash.COMMAND} [pod-name] [bash-commands] [&]\t run bash on the Cassandra nodes'

adam/commands/bash/bash_completer.py

@@ -0,0 +1,93 @@
+from typing import Callable
+from prompt_toolkit.completion import WordCompleter
+
+from adam.utils_repl.automata_completer import AutomataCompleter
+from adam.utils_repl.state_machine import State, StateMachine
+
+BASH_SPEC = [
+    # <command> ::= <simple_command> | <pipeline> | <conditional_command>
+    # <simple_command> ::= <word> <argument>* <redirection>*
+    # <pipeline> ::= <command> '|' <command>
+    # <conditional_command> ::= <command> '&&' <command> | <command> '||' <command>
+    # <word> ::= <letter> <letter_or_digit>*
+    # <argument> ::= <word>
+    # <redirection> ::= '>' <filename> | '<' <filename>
+    # <filename> ::= <word>
+    # <letter> ::= 'a' | 'b' | ... | 'z' | 'A' | 'B' | ... | 'Z'
+    # <digit> ::= '0' | '1' | ... | '9'
+    # <letter_or_digit> ::= <letter> | <digit>
+
+    '                                > word           > cmd                                ^ hosts',
+    'cmd                             > word           > cmd                                ^ |,>,2>,<,&,&&,||',
+    '-                               > pipe           > cmd_pipe',
+    '-                               > _rdr0_         > cmd_rdr0',
+    '-                               > _rdr1_         > cmd_rdr1',
+    '-                               > _rdr2_         > cmd_rdr2',
+    '-                               > &              > cmd_bg                             ^ |,>,2>,<,&,&&,||',
+    '-                               > &&|_or_        > nocmd',
+    'cmd_a                           > word           > cmd',
+    'cmd_pipe                        > word           > cmd',
+    'cmd_rdr0                        > word           > cmd_rdr0_f',
+    'cmd_rdr1                        > word           > cmd_rdr1_f',
+    'cmd_rdr2                        > word           > cmd_rdr2_f',
+    'cmd_rdr1_f                      > pipe           > cmd_pipe                           ^ |,2>,<,&,&&,||',
+    '-                               > _rdr2_         > cmd_rdr2',
+    '-                               > _rdr0_         > cmd_rdr0',
+    'cmd_rdr2_f                      > pipe           > cmd_pipe                           ^ |,<,&,&&,||',
+    '-                               > _rdr0_         > cmd_rdr0',
+    '-                               > &              > cmd_bg                             ^ |,>,2>,<,&,&&,||',
+    '-                               > &&|_or_        > nocmd',
+    'cmd_rdr0_f                      > pipe           > cmd_pipe                           ^ |,&,&&,||',
+    '-                               > &              > cmd_bg                             ^ |,>,2>,<,&,&&,||',
+    '-                               > &&|_or_        > cmd',
+    'cmd_bg                          > &&|_or_        > nocmd                              ^ &&,||',
+    'nocmd                           > word           > cmd',
+]
+
+BASH_KEYWORDS = [
+    '&',
+    '&&',
+    '|',
+    '||',
+    '>',
+    '2>',
+    '>>',
+    '<',
+    'hosts'
+]
+
+class BashStateMachine(StateMachine[str]):
+    def spec(self) -> str:
+        return BASH_SPEC
+
+    def keywords(self) -> list[str]:
+        return BASH_KEYWORDS
+
+class BashCompleter(AutomataCompleter[str]):
+    def __init__(self,
+                 hosts: Callable[[], list[str]],
+                 debug = False):
+        super().__init__(BashStateMachine(debug=debug), '', debug=debug)
+
+        self.hosts = hosts
+        self.debug = debug
+
+    def suggestions_completer(self, state: State, suggestions: str) -> list[str]:
+        if not suggestions:
+            return None
+
+        terms = []
+        for suggestion in suggestions.split(','):
+            terms.extend(self._terms(state, suggestion))
+
+        return WordCompleter(terms)
+
+    def _terms(self, _: State, word: str) -> list[str]:
+        terms = []
+
+        if word == 'hosts':
+            terms.extend(self.hosts())
+        else:
+            terms.append(word)
+
+        return terms

adam/commands/bash/utils_bash.py

@@ -0,0 +1,16 @@
+from adam.commands.devices.devices import Devices
+from adam.repl_state import ReplState
+
+class BashHandler:
+    def __init__(self, s0: ReplState, s1: ReplState):
+        self.s0 = s0
+        self.s1 = s1
+
+    def __enter__(self):
+        return self.exec
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        return False
+
+    def exec(self, args: list[str]):
+        return Devices.device(self.s1).bash(self.s0, self.s1, args)

adam/commands/cat.py

@@ -0,0 +1,50 @@
+from adam.commands.command import Command
+from adam.commands.devices.devices import Devices
+from adam.repl_state import ReplState, RequiredState
+from adam.utils import log2
+from adam.utils_k8s.app_pods import AppPods
+from adam.utils_k8s.statefulsets import StatefulSets
+
+class Cat(Command):
+    COMMAND = 'cat'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(Cat, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return Cat.COMMAND
+
+    def required(self):
+        return [RequiredState.CLUSTER_OR_POD, RequiredState.APP_APP, ReplState.P]
+
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        with self.validate(args, state) as (args, state):
+            if len(args) < 1:
+                if state.in_repl:
+                    log2('File is required.')
+                    log2()
+                else:
+                    log2('* File is missing.')
+                    Command.display_help()
+
+                return 'command-missing'
+
+            return Devices.device(state).cat(cmd, state)
+
+    def completion(self, state: ReplState):
+        if super().completion(state):
+            return Devices.device(state).cat_completion(Cat.COMMAND, state, default={})
+
+        return {}
+
+    def help(self, _: ReplState):
+        return f'{Cat.COMMAND} file [&]\t run cat command on the Cassandra nodes'

adam/commands/cd.py

@@ -0,0 +1,43 @@
+from adam.commands.command import Command
+from adam.commands.devices.device import Device
+from adam.commands.devices.devices import Devices
+from adam.repl_state import ReplState
+
+class Cd(Command):
+    COMMAND = 'cd'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(Cd, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return Cd.COMMAND
+
+    def required(self):
+        return ReplState.NON_L
+
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        with self.validate(args, state, apply=False) as (args, state):
+            if len(args) < 2:
+                return state
+
+            device: Device = Devices.device(state)
+            arg = args[1]
+            for dir in arg.split('/'):
+                device.cd(dir, state)
+
+            return state
+
+    def completion(self, state: ReplState):
+        return Devices.device(state).cd_completion(Cd.COMMAND, state, default = {})
+
+    def help(self, _: ReplState):
+        return f'{Cd.COMMAND} <path> | .. \t move around on the operational device hierarchy'

adam/commands/check.py

@@ -0,0 +1,73 @@
+import click
+
+from adam.checks.check_result import CheckResult
+from adam.checks.check_utils import all_checks, checks_from_csv, run_checks
+from adam.commands import extract_options
+from adam.commands.command import Command
+from adam.commands.command_helpers import ClusterOrPodCommandHelper
+from adam.commands.issues import Issues
+from adam.repl_state import ReplState
+from adam.utils import lines_to_tabular, log
+from adam.utils_issues import IssuesUtils
+
+class Check(Issues):
+    COMMAND = 'check'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(Check, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return Check.COMMAND
+
+    def required(self):
+        return ReplState.NON_L
+
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        with self.validate(args, state) as (args, state):
+            with extract_options(args, ['-s', '--show']) as (args, show_out):
+                if not args:
+                    if state.in_repl:
+                        log(lines_to_tabular([check.help() for check in all_checks()], separator=':'))
+                    else:
+                        log('* Check name is missing.')
+                        Command.display_help()
+
+                    return 'arg missing'
+
+                checks = checks_from_csv(args[0])
+                if not checks:
+                    return 'invalid check name'
+
+                results = run_checks(state.sts, state.namespace, state.pod, checks=checks, show_out=show_out)
+
+                issues = CheckResult.collect_issues(results)
+                IssuesUtils.show_issues(issues, in_repl=state.in_repl)
+
+                return issues if issues else 'no issues found'
+
+    def completion(self, _: ReplState):
+        return {Check.COMMAND: {check.name(): {'-s': None} for check in all_checks()}}
+
+    def help(self, _: ReplState):
+        return f'{Check.COMMAND} <check-name> [-s]\t run a single check'
+
+class CheckCommandHelper(click.Command):
+    def get_help(self, ctx: click.Context):
+        log(super().get_help(ctx))
+        log()
+        log('Check-names:')
+
+        for check in all_checks():
+            log(f'  {check.name()}')
+        log()
+
+        ClusterOrPodCommandHelper.cluter_or_pod_help()

{walker → adam}/commands/cli_commands.py

@@ -1,12 +1,12 @@
 import functools
 import re
 
-from walker.commands.reaper.reaper_session import ReaperSession
-from walker.config import Config
-from walker.k8s_utils.kube_context import KubeContext
-from walker.k8s_utils.secrets import Secrets
-from walker.k8s_utils.statefulsets import StatefulSets
-from walker.repl_state import ReplState
+from adam.commands.reaper.utils_reaper import Reapers
+from adam.config import Config
+from adam.utils_k8s.kube_context import KubeContext
+from adam.utils_k8s.secrets import Secrets
+from adam.utils_k8s.statefulsets import StatefulSets
+from adam.repl_state import ReplState
 
 class CliCommands:
     @functools.lru_cache()
@@ -66,8 +66,7 @@ class CliCommands:
             return key.replace(f'{state.namespace}-', '').replace('-k8spg-cs-001', '')
         d |= {f'pg-{reduce_key(k)}': f'PGPASSWORD={v["postgres-admin-password"]} psql -h {v["postgres-db-endpoint"]} -p {v["postgres-db-port"]} -U {v["postgres-admin-username"]} postgres' for k, v in data.items()}
 
-        if r := ReaperSession.create(state):
-            reaper = r.reaper_spec(state)
+        if reaper := Reapers.reaper_spec(state):
             d |= {
                 'reaper-exec': reaper["exec"],
                 'reaper-forward': reaper["forward"],