kaqing 2.0.105__tar.gz → 2.0.111__tar.gz

{kaqing-2.0.105 → kaqing-2.0.111}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: kaqing
-Version: 2.0.105
+Version: 2.0.111
 Summary: UNKNOWN
 Home-page: UNKNOWN
 License: UNKNOWN

{kaqing-2.0.105 → kaqing-2.0.111}/adam/commands/app.py

@@ -22,14 +22,14 @@ class App(Command):
         return App.COMMAND
 
     def required(self):
-        return RequiredState.CLUSTER_OR_POD
+        return RequiredState.APP_APP
 
     def run(self, cmd: str, state: ReplState):
         if not(args := self.args(cmd)):
             return super().run(cmd, state)
 
         state, args = self.apply_state(args, state)
-        if not self.validate_state(state, app_required=RequiredState.APP_APP):
+        if not self.validate_state(state):
             return state
 
         args, forced = Command.extract_options(args, '--force')

{kaqing-2.0.105 → kaqing-2.0.111}/adam/commands/app_ping.py

@@ -18,14 +18,14 @@ class AppPing(Command):
         return AppPing.COMMAND
 
     def required(self):
-        return RequiredState.CLUSTER_OR_POD
+        return RequiredState.APP_APP
 
     def run(self, cmd: str, state: ReplState):
         if not(args := self.args(cmd)):
             return super().run(cmd, state)
 
         state, args = self.apply_state(args, state)
-        if not self.validate_state(state, app_required=RequiredState.APP_APP):
+        if not self.validate_state(state):
             return state
 
         _, forced = Command.extract_options(args, '--force')

{kaqing-2.0.105 → kaqing-2.0.111}/adam/commands/audit/audit.py

@@ -2,6 +2,10 @@ import click
 
 from adam.commands.audit.audit_repair_tables import AuditRepairTables
 from adam.commands.audit.audit_run import AuditRun
+from adam.commands.audit.show_last10 import ShowLast10
+from adam.commands.audit.show_slow10 import ShowSlow10
+from adam.commands.audit.show_top10 import ShowTop10
+from adam.commands.audit.utils_show_top10 import show_top10_completions_for_nesting
 from adam.commands.command import Command
 from adam.config import Config
 from adam.repl_state import ReplState
@@ -25,11 +29,16 @@ class Audit(Command):
     def command(self):
         return Audit.COMMAND
 
+    def required(self):
+        return ReplState.L
+
     def run(self, cmd: str, state: ReplState):
         if not(args := self.args(cmd)):
             return super().run(cmd, state)
 
         state, args = self.apply_state(args, state)
+        if not self.validate_state(state):
+            return state
 
         r = None
         if len(args) > 0:
@@ -52,10 +61,11 @@ class Audit(Command):
                 Config().wait_log(f'Inspecting audit database schema...')
                 self.schema_read = True
                 # warm up the caches first time when l: drive is accessed
+                Audits.audit_table_names()
                 Audits.audit_column_names()
                 Audits.audit_column_names(partition_cols_only=True)
 
-            return super().completion(state) | SqlCompleter(
+            return super().completion(state) | show_top10_completions_for_nesting() | SqlCompleter(
                 lambda: Audits.audit_table_names(),
                 columns=lambda table: Audits.audit_column_names(),
                 partition_columns=lambda table: Audits.audit_column_names(partition_cols_only=True),
@@ -65,7 +75,7 @@ class Audit(Command):
         return {}
 
     def cmd_list():
-        return [AuditRepairTables(), AuditRun()]
+        return [AuditRepairTables(), AuditRun(), ShowLast10(), ShowSlow10(), ShowTop10()]
 
     def help(self, _: ReplState):
         return f'[{Audit.COMMAND}] [<sql-statements>]\t run SQL queries on Authena audit database'

{kaqing-2.0.105 → kaqing-2.0.111}/adam/commands/audit/audit_repair_tables.py

@@ -23,11 +23,16 @@ class AuditRepairTables(Command):
     def command(self):
         return AuditRepairTables.COMMAND
 
+    def required(self):
+        return ReplState.L
+
     def run(self, cmd: str, state: ReplState):
         if not(args := self.args(cmd)):
             return super().run(cmd, state)
 
         state, args = self.apply_state(args, state)
+        if not self.validate_state(state):
+            return state
 
         tables = Config().get('audit.athena.repair-partition-tables', 'audit').split(',')
         if args:

{kaqing-2.0.105 → kaqing-2.0.111}/adam/commands/audit/audit_run.py

@@ -1,4 +1,5 @@
 import concurrent
+
 from adam.commands.command import Command
 from adam.config import Config
 from adam.repl_state import ReplState
@@ -21,11 +22,16 @@ class AuditRun(Command):
     def command(self):
         return AuditRun.COMMAND
 
+    def required(self):
+        return ReplState.L
+
     def run(self, cmd: str, state: ReplState):
         if not(args := self.args(cmd)):
             return super().run(cmd, state)
 
         state, args = self.apply_state(args, state)
+        if not self.validate_state(state):
+            return state
 
         meta: AuditMeta = Audits.get_meta()
         clusters = Audits.find_new_clusters(meta.cluster_last_checked)
@@ -35,7 +41,7 @@ class AuditRun(Command):
             tables = Config().get('audit.athena.repair-cluster-tables', 'cluster').split(',')
             with concurrent.futures.ThreadPoolExecutor(max_workers=Config().get('audit.workers', 3)) as executor:
                 for table in tables:
-                    Audits.run_audit_query(f'MSCK REPAIR TABLE {table}')
+                    executor.submit(Audits.run_audit_query, f'MSCK REPAIR TABLE {table}', None,)
         else:
             log2(f'No new clusters were found.')
 

kaqing-2.0.111/adam/commands/audit/show_last10.py

@@ -0,0 +1,50 @@
+from adam.commands.audit.utils_show_top10 import extract_limit_and_duration
+from adam.commands.command import Command
+from adam.repl_state import ReplState
+from adam.utils import log2
+from adam.utils_audits import Audits
+
+class ShowLast10(Command):
+    COMMAND = 'show last'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(ShowLast10, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return ShowLast10.COMMAND
+
+    def required(self):
+        return ReplState.L
+
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        state, args = self.apply_state(args, state)
+        if not self.validate_state(state):
+            return state
+
+        limit, date_condition = extract_limit_and_duration(args)
+
+        query = '\n    '.join([
+            "SELECT * FROM audit",
+            f"WHERE drive <> 'z' and ({date_condition})",
+            f"ORDER BY ts DESC LIMIT {limit};"])
+        log2(query)
+        log2()
+        Audits.run_audit_query(query)
+
+        return state
+
+    def completion(self, _: ReplState):
+
+        return {}
+
+    def help(self, _: ReplState):
+        return f'{ShowLast10.COMMAND} [limit]\t show last <limit> audit lines'

kaqing-2.0.111/adam/commands/audit/show_slow10.py

@@ -0,0 +1,49 @@
+from adam.commands.audit.utils_show_top10 import extract_limit_and_duration
+from adam.commands.command import Command
+from adam.repl_state import ReplState
+from adam.utils import log2
+from adam.utils_audits import Audits
+
+class ShowSlow10(Command):
+    COMMAND = 'show slow'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(ShowSlow10, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return ShowSlow10.COMMAND
+
+    def required(self):
+        return ReplState.L
+
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        state, args = self.apply_state(args, state)
+        if not self.validate_state(state):
+            return state
+
+        limit, date_condition = extract_limit_and_duration(args)
+
+        query = '\n    '.join([
+            "SELECT * FROM audit",
+            f"WHERE drive <> 'z' and ({date_condition})",
+            f"ORDER BY CAST(duration AS REAL) DESC LIMIT {limit};"])
+        log2(query)
+        log2()
+        Audits.run_audit_query(query)
+
+        return state
+
+    def completion(self, _: ReplState):
+        return {}
+
+    def help(self, _: ReplState):
+        return f'{ShowSlow10.COMMAND} [limit]\t show slow <limit> audit lines'

kaqing-2.0.111/adam/commands/audit/show_top10.py

@@ -0,0 +1,48 @@
+from adam.commands.audit.utils_show_top10 import extract_limit_and_duration
+from adam.commands.command import Command
+from adam.repl_state import ReplState
+from adam.utils import log2
+from adam.utils_audits import Audits
+
+class ShowTop10(Command):
+    COMMAND = 'show top'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(ShowTop10, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return ShowTop10.COMMAND
+
+    def required(self):
+        return ReplState.L
+
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        state, args = self.apply_state(args, state)
+        if not self.validate_state(state):
+            return state
+
+        limit, date_condition = extract_limit_and_duration(args)
+        query = '\n    '.join([
+            "SELECT min(c) AS cluster, line, COUNT(*) AS cnt, avg(CAST(duration AS REAL)) AS duration",
+            f"FROM audit WHERE drive <> 'z' and ({date_condition})",
+            f"GROUP BY line ORDER BY cnt DESC LIMIT {limit};"])
+        log2(query)
+        log2()
+        Audits.run_audit_query(query)
+
+        return state
+
+    def completion(self, _: ReplState):
+        return {}
+
+    def help(self, _: ReplState):
+        return f'{ShowTop10.COMMAND} [limit]\t show top <limit> audit lines'

kaqing-2.0.111/adam/commands/audit/utils_show_top10.py

@@ -0,0 +1,59 @@
+from datetime import datetime, timedelta
+
+from adam.utils_audits import Audits
+from adam.utils_repl.automata_completer import AutomataCompleter
+from adam.utils_repl.state_machine import StateMachine
+
+def extract_limit_and_duration(args: list[str]) -> tuple[int, datetime]:
+    limit = 10
+    _from = datetime.now() - timedelta(days=30)
+    if args:
+        try:
+            limit = int(args[0])
+        except:
+            pass
+
+        if len(args) > 2 and args[1] == 'over':
+            if args[2] == 'day':
+                _from = datetime.now() - timedelta(days=1)
+
+    return (limit, Audits.date_from(_from))
+
+def limit_and_duration_completion():
+    return {'10': {'over': {
+        'day': None,
+        'month': None
+    }}}
+
+SHOW_TOP10_SPEC = [
+    '                                > show           > show',
+    'show                            > last|slow|top  > show_top                            ^ last,slow,top',
+    'show_top                        > word           > show_top_n                          ^ 10',
+    'show_top_n                      > over           > show_top_n_over                     ^ over',
+    'show_top_n_over                 > day|month      > show_top_n_over$                    ^ day,month',
+]
+
+SHOW_TOP10_KEYWORDS = [
+    'show',
+    'top',
+    'last',
+    'slow',
+    'over',
+    'day',
+    'month'
+]
+
+class ShowTop10StateMachine(StateMachine[str]):
+    def spec(self) -> str:
+        return SHOW_TOP10_SPEC
+
+    def keywords(self) -> list[str]:
+        return SHOW_TOP10_KEYWORDS
+
+def show_top10_completions_for_nesting():
+    return {
+        'show': {
+            'last': AutomataCompleter(ShowTop10StateMachine(), first_term='show last'),
+            'slow': AutomataCompleter(ShowTop10StateMachine(), first_term='show slow'),
+            'top': AutomataCompleter(ShowTop10StateMachine(), first_term='show top'),
+    }}

{kaqing-2.0.105 → kaqing-2.0.111}/adam/commands/cd.py

@@ -23,10 +23,16 @@ class Cd(Command):
     def command(self):
         return Cd.COMMAND
 
+    def required(self):
+        return ReplState.NON_L
+
     def run(self, cmd: str, state: ReplState):
         if not(args := self.args(cmd)):
             return super().run(cmd, state)
 
+        if not self.validate_state(state):
+            return state
+
         if len(args) < 2:
             return state
 

{kaqing-2.0.105 → kaqing-2.0.111}/adam/commands/check.py

@@ -23,11 +23,17 @@ class Check(Issues):
     def command(self):
         return Check.COMMAND
 
+    def required(self):
+        return ReplState.NON_L
+
     def run(self, cmd: str, state: ReplState):
         if not(args := self.args(cmd)):
             return super().run(cmd, state)
 
         state, args = self.apply_state(args, state)
+        if not self.validate_state(state):
+            return state
+
         args, show = Command.extract_options(args, ['-s', '--show'])
 
         if not args:

{kaqing-2.0.105 → kaqing-2.0.111}/adam/commands/command.py

@@ -28,8 +28,10 @@ class Command:
 
         return None
 
-    def completion(self, _: ReplState, leaf: dict[str, any] = None) -> dict[str, any]:
-        # COMMAND = 'reaper activate schedule'
+    def completion(self, state: ReplState, leaf: dict[str, any] = None) -> dict[str, any]:
+        if not self.validate_state(state, show_err=False):
+            return {}
+
         d = leaf
         for t in reversed(self.command().split(' ')):
             d = {t: d}
@@ -39,8 +41,8 @@ class Command:
     def required(self) -> RequiredState:
         return None
 
-    def validate_state(self, state: ReplState, pg_required: RequiredState = None, app_required: RequiredState = None):
-        return state.validate(self.required(), pg_required=pg_required, app_required=app_required)
+    def validate_state(self, state: ReplState, show_err=True):
+        return state.validate(self.required(), show_err=show_err)
 
     def help(self, _: ReplState) -> str:
         return None

{kaqing-2.0.105 → kaqing-2.0.111}/adam/commands/cql/cql_utils.py

@@ -35,7 +35,7 @@ def tables(state: ReplState, on_any=False) -> dict[str, list[str]]:
 
     return parse_cql_desc_tables(r.stdout if state.pod else r[0].stdout)
 
-def run_cql(state: ReplState, cql: str, opts: list = [], show_out = False, use_single_quotes = False, on_any = False, background=False):
+def run_cql(state: ReplState, cql: str, opts: list = [], show_out = False, use_single_quotes = False, on_any = False, background=False) -> list[PodExecResult]:
     user, pw = Secrets.get_user_pass(state.sts if state.sts else state.pod, state.namespace, secret_path='cql.secret')
     if use_single_quotes:
         command = f"cqlsh -u {user} -p {pw} {' '.join(opts)} -e '{cql}'"

{kaqing-2.0.105 → kaqing-2.0.111}/adam/commands/deploy/deploy.py

@@ -4,7 +4,7 @@ from adam.commands.command import Command
 from adam.commands.deploy.deploy_pg_agent import DeployPgAgent
 from adam.commands.deploy.deploy_pod import DeployPod
 from .deploy_frontend import DeployFrontend
-from adam.repl_state import ReplState
+from adam.repl_state import ReplState, RequiredState
 
 class Deploy(Command):
     COMMAND = 'deploy'
@@ -22,10 +22,16 @@ class Deploy(Command):
     def command(self):
         return Deploy.COMMAND
 
+    def required(self):
+        return RequiredState.NAMESPACE
+
     def run(self, cmd: str, state: ReplState):
         if not(args := self.args(cmd)):
             return super().run(cmd, state)
 
+        if not self.validate_state(state):
+            return state
+
         return super().intermediate_run(cmd, state, args, Deploy.cmd_list())
 
     def cmd_list():

{kaqing-2.0.105 → kaqing-2.0.111}/adam/commands/deploy/undeploy.py

@@ -4,7 +4,7 @@ from adam.commands.command import Command
 from adam.commands.deploy.undeploy_frontend import UndeployFrontend
 from adam.commands.deploy.undeploy_pg_agent import UndeployPgAgent
 from adam.commands.deploy.undeploy_pod import UndeployPod
-from adam.repl_state import ReplState
+from adam.repl_state import ReplState, RequiredState
 
 class Undeploy(Command):
     COMMAND = 'undeploy'
@@ -22,10 +22,16 @@ class Undeploy(Command):
     def command(self):
         return Undeploy.COMMAND
 
+    def required(self):
+        return RequiredState.NAMESPACE
+
     def run(self, cmd: str, state: ReplState):
         if not(args := self.args(cmd)):
             return super().run(cmd, state)
 
+        if not self.validate_state(state):
+            return state
+
         return super().intermediate_run(cmd, state, args, Undeploy.cmd_list())
 
     def cmd_list():

{kaqing-2.0.105 → kaqing-2.0.111}/adam/commands/help.py

@@ -30,14 +30,16 @@ class Help(Command):
         lines.append('NAVIGATION')
         lines.append('  a: | c: | l: | p:\t switch to another operational device: App, Cassandra, Audit or Postgres')
         lines.extend(section(ReplCommands.navigation()))
-        lines.append('CHECK CASSANDRA')
-        lines.extend(section(ReplCommands.cassandra_check()))
-        lines.append('CASSANDRA OPERATIONS')
+        lines.append('CASSANDRA')
         lines.extend(section(ReplCommands.cassandra_ops()))
+        lines.append('POSTGRES')
+        lines.extend(section(ReplCommands.postgres_ops()))
+        lines.append('APP')
+        lines.extend(section(ReplCommands.app_ops()))
+        lines.append('AUDIT')
+        lines.extend(section(ReplCommands.audit_ops()))
         lines.append('TOOLS')
         lines.extend(section(ReplCommands.tools()))
-        lines.append('APP')
-        lines.extend(section(ReplCommands.app()))
         lines.append('')
         lines.extend(section(ReplCommands.exit()))
 

{kaqing-2.0.105 → kaqing-2.0.111}/adam/commands/issues.py

@@ -21,11 +21,17 @@ class Issues(Command):
     def command(self):
         return Issues.COMMAND
 
+    def required(self):
+        return ReplState.NON_L
+
     def run(self, cmd: str, state: ReplState):
         if not(args := self.args(cmd)):
             return super().run(cmd, state)
 
         state, args = self.apply_state(args, state)
+        if not self.validate_state(state):
+            return state
+
         args, show = Command.extract_options(args, ['-s', '--show'])
 
         results = run_checks(state.sts, state.namespace, state.pod, show_output=show)

{kaqing-2.0.105 → kaqing-2.0.111}/adam/commands/login.py

@@ -8,7 +8,7 @@ from adam.config import Config
 from adam.sso.idp import Idp
 from adam.sso.idp_login import IdpLogin
 from adam.commands.command import Command
-from adam.repl_state import ReplState
+from adam.repl_state import ReplState, RequiredState
 from adam.utils import log, log2
 
 class Login(Command):
@@ -26,6 +26,9 @@ class Login(Command):
     def command(self):
         return Login.COMMAND
 
+    def required(self):
+        return ReplState.NON_L
+
     def run(self, cmd: str, state: ReplState):
         def custom_handler(signum, frame):
             AppSession.ctrl_c_entered = True
@@ -59,8 +62,8 @@ class Login(Command):
 
         return state
 
-    def completion(self, _: ReplState):
-        return {}
+    def completion(self, state: ReplState):
+        return super().completion(state)
 
     def help(self, _: ReplState):
         return f'{Login.COMMAND}\t SSO login'

{kaqing-2.0.105 → kaqing-2.0.111}/adam/commands/logs.py

@@ -33,6 +33,7 @@ class Logs(Command):
         return CassandraNodes.exec(state.pod, state.namespace, f'cat {path}')
 
     def completion(self, _: ReplState):
+        # available only on cli
         return {}
 
     def help(self, _: ReplState):

{kaqing-2.0.105 → kaqing-2.0.111}/adam/commands/ls.py

@@ -134,13 +134,13 @@ class Ls(Command):
         log(lines_to_tabular(Audits.audit_table_names(), 'NAME', separator=','))
 
     def completion(self, state: ReplState):
-        if state.pod:
-            return {}
-
-        if state.device == ReplState.C and not state.sts:
-            return {Ls.COMMAND: {n: None for n in StatefulSets.list_sts_names()}}
+        if state.device == ReplState.C:
+            if state.pod:
+                return super().completion(state)
+            elif not state.sts:
+                return {Ls.COMMAND: {n: None for n in StatefulSets.list_sts_names()}}
 
-        return {Ls.COMMAND: None}
+        return super().completion(state)
 
     def help(self, _: ReplState):
         return f'{Ls.COMMAND} [device:]\t list apps, envs, clusters, nodes, pg hosts or pg databases'

{kaqing-2.0.105 → kaqing-2.0.111}/adam/commands/medusa/medusa_show_backupjobs.py

@@ -29,6 +29,7 @@ class MedusaShowBackupJobs(Command):
         state, args = self.apply_state(args, state)
         if not self.validate_state(state):
             return state
+
         ns = state.namespace
         dc = StatefulSets.get_datacenter(state.sts, ns)
         if not dc:

{kaqing-2.0.105 → kaqing-2.0.111}/adam/commands/preview_table.py

@@ -22,19 +22,15 @@ class PreviewTable(Command):
         return PreviewTable.COMMAND
 
     def required(self):
-        return RequiredState.CLUSTER_OR_POD
+        return [RequiredState.CLUSTER_OR_POD, RequiredState.PG_DATABASE, ReplState.L]
 
     def run(self, cmd: str, state: ReplState):
         if not(args := self.args(cmd)):
             return super().run(cmd, state)
 
         state, args = self.apply_state(args, state)
-        if state.device == ReplState.P:
-            if not self.validate_state(state, RequiredState.PG_DATABASE):
-                return state
-        elif state.device != ReplState.L:
-            if not self.validate_state(state):
-                return state
+        if not self.validate_state(state):
+            return state
 
         if not args:
             def show_tables():

{kaqing-2.0.105 → kaqing-2.0.111}/adam/commands/reaper/reaper.py

@@ -37,6 +37,9 @@ class Reaper(Command):
         if not(args := self.args(cmd)):
             return super().run(cmd, state)
 
+        if not self.validate_state(state):
+            return state
+
         return super().intermediate_run(cmd, state, args, Reaper.cmd_list())
 
     def cmd_list():

{kaqing-2.0.105 → kaqing-2.0.111}/adam/commands/repair/repair.py

@@ -28,6 +28,8 @@ class Repair(Command):
     def run(self, cmd: str, state: ReplState):
         if not(args := self.args(cmd)):
             return super().run(cmd, state)
+        if not self.validate_state(state):
+            return state
 
         return super().intermediate_run(cmd, state, args, Repair.cmd_list())
 
@@ -35,9 +37,7 @@ class Repair(Command):
         return [RepairRun(), RepairScan(), RepairStop(), RepairLog()]
 
     def completion(self, state: ReplState):
-        if state.sts:
-            return super().completion(state)
-        return {}
+        return super().completion(state)
 
 class RepairCommandHelper(click.Command):
     def get_help(self, ctx: click.Context):

{kaqing-2.0.105 → kaqing-2.0.111}/adam/commands/report.py

@@ -22,12 +22,18 @@ class Report(Command):
     def command(self):
         return Report.COMMAND
 
+    def required(self):
+        return ReplState.NON_L
+
     def run(self, cmd: str, state: ReplState):
         if not(args := self.args(cmd)):
             return super().run(cmd, state)
 
         output: dict[str, any] = {}
         state, args = self.apply_state(args, state)
+        if not self.validate_state(state):
+            return state
+
         if state.in_repl:
             args, show = Command.extract_options(args, ['-s', '--show'])