kaqing 2.0.174__py3-none-any.whl → 2.0.188__py3-none-any.whl

adam/app_session.py

@@ -8,7 +8,7 @@ from adam.log import Log
 from adam.sso.idp import Idp
 from adam.sso.idp_login import IdpLogin
 from adam.config import Config
-from adam.utils import debug, debug_trace, json_to_csv, lines_to_tabular, log, log2, log_exc
+from adam.utils import debug, debug_trace, json_to_csv, tabulize, log2, log_exc
 from adam.apps import Apps
 
 class AppLogin:
@@ -63,7 +63,7 @@ class AppSession:
                             js = r.json()
                             with log_exc(js):
                                 header, lines = json_to_csv(js, delimiter='\t')
-                                log(lines_to_tabular(lines, header=header, separator='\t'))
+                                tabulize(lines, header=header, separator='\t')
                         except:
                             if urlparse(r.url).hostname != urlparse(uri).hostname and not retried:
                                 app_login = app_session.login(idp_uri=app_login.idp_uri, forced=forced, use_token_from_env=False, use_cached_creds=False)

adam/apps.py

@@ -12,7 +12,8 @@ from adam.utils_k8s.services import Services
 from adam.utils import copy_config_file
 
 class AppAction:
-    def __init__(self, name: str, payload: str = None, args: dict[str, str] = None, help: str = None):
+    def __init__(self, typ: str, name: str, payload: str = None, args: dict[str, str] = None, help: str = None):
+        self.typ = typ
         self.name = name
         self.payload = payload
         self.args = args
@@ -33,6 +34,19 @@ class AppAction:
 
         return args
 
+    def __str__(self):
+        line = None
+
+        args = ','.join(self.arguments())
+        if args:
+            line = f'{self.typ}.{self.name},{args}'
+        else:
+            line = f'{self.typ}.{self.name},'
+        if self.help:
+            line = f'{line},{self.help}'
+
+        return line
+
 class AppType:
     all_types: list['AppType'] = []
 
@@ -62,12 +76,12 @@ class AppType:
                         aa: AppAction = None
 
                         if isinstance(v, str):
-                            aa = AppAction(k, help=v)
+                            aa = AppAction(typ, k, help=v)
                         elif isinstance(v, dict):
                             args = {k: v1 for k, v1 in v.items()}
-                            aa = AppAction(k, payload=v['payload'], args=args, help=v['help'])
+                            aa = AppAction(typ, k, payload=v['payload'], args=args, help=v['help'])
                         else:
-                            aa = AppAction(k)
+                            aa = AppAction(typ, k)
 
                         app_actions.append(aa)
 

adam/batch.py

@@ -3,7 +3,7 @@ import click
 from adam.commands.audit.audit import Audit, AuditCommandHelper
 from adam.commands.bash.bash import Bash
 from adam.commands.check import Check, CheckCommandHelper
-from adam.commands.cp import ClipboardCopy, CopyCommandHelper
+from adam.commands.clipboard_copy import ClipboardCopy, CopyCommandHelper
 from adam.commands.command import Command
 from adam.commands.command_helpers import ClusterCommandHelper, ClusterOrPodCommandHelper, PodCommandHelper
 from adam.commands.cql.cqlsh import CqlCommandHelper, Cqlsh

adam/checks/check_utils.py

@@ -47,7 +47,9 @@ def run_checks(cluster: str = None, namespace: str = None, pod: str = None, chec
                 if not pod or pod == pod_name:
                     sts_ns_pods.append((sts, ns, pod_name))
 
-    with parallelize(sts_ns_pods, Config().action_workers('issues', 30), msg='d`Running|Ran checks on {size} pods') as exec:
+    with parallelize(sts_ns_pods,
+                     Config().action_workers('issues', 30),
+                     msg='d`Running|Ran checks on {size} pods') as exec:
         return exec.map(lambda sts_ns_pod: run_checks_on_pod(checks, sts_ns_pod[0], sts_ns_pod[1], sts_ns_pod[2], show_out))
 
 def run_checks_on_pod(checks: list[Check], cluster: str = None, namespace: str = None, pod: str = None, show_out=True):

adam/commands/__init__.py

@@ -18,5 +18,7 @@ def extract_all_options(args: list[str], trailing = None, sequence = None, optio
 def extract_sequence(args: list[str], sequence: list[str]):
     return ExtractSequenceOptionsHandler(args, sequence = sequence)
 
-def validate_args(args: list[str], state: ReplState, count_at_least: int = 1, name: str = None, msg: Callable[[], None] = None):
-    return ValidateArgCountHandler(args, state, count_at_least=count_at_least, name=name, msg=msg)
+def validate_args(args: list[str], state: ReplState, at_least: int = 1, exactly: int = -1,
+                  name: str = None, msg: Callable[[], None] = None, default: str = None,
+                  separator=None):
+    return ValidateArgCountHandler(args, state, at_least=at_least, exactly=exactly, name=name, msg=msg, default=default, separator=separator)

adam/commands/app/app.py

@@ -0,0 +1,38 @@
+from adam.commands import app, extract_options
+from adam.commands.command import Command
+from adam.repl_state import ReplState, RequiredState
+
+class App(Command):
+    COMMAND = 'app'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(App, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return App.COMMAND
+
+    def required(self):
+        return RequiredState.APP_APP
+
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        with self.validate(args, state) as (args, state):
+            with extract_options(args, '--force') as (args, forced):
+                with app(state) as http:
+                    http.post(args, forced=forced)
+
+                return state
+
+    def completion(self, _: ReplState):
+        return {}
+
+    def help(self, _: ReplState):
+        return f"<AppType>.<AppAction> <args> [--force]\t post app action; check with 'show app actions' command"

adam/commands/app/app_ping.py

@@ -0,0 +1,38 @@
+from adam.commands import app, extract_options
+from adam.commands.command import Command
+from adam.repl_state import ReplState, RequiredState
+
+class AppPing(Command):
+    COMMAND = 'app ping'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(AppPing, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return AppPing.COMMAND
+
+    def required(self):
+        return RequiredState.APP_APP
+
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        with self.validate(args, state) as (args, state):
+            with extract_options(args, '--force') as (args, forced):
+                with app(state) as http:
+                    http.post(['Echo.echoStatic'], forced=forced)
+
+                    return state
+
+    def completion(self, state: ReplState):
+        return super().completion(state, {'--force': None})
+
+    def help(self, _: ReplState):
+        return f"{AppPing.COMMAND} [--force]\t ping app server with Echo.echoStatic()"

adam/commands/app/show_app_actions.py

@@ -0,0 +1,49 @@
+from adam.app_session import AppSession
+from adam.apps import Apps
+from adam.commands.command import Command
+from adam.config import Config
+from adam.repl_state import ReplState
+from adam.utils import tabulize, log
+
+class ShowAppActions(Command):
+    COMMAND = 'show app actions'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(ShowAppActions, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return ShowAppActions.COMMAND
+
+    def required(self):
+        return ReplState.A
+
+    def run(self, cmd: str, state: ReplState):
+        if not (args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        with self.validate(args, state) as (args, state):
+            actions = []
+            for typ in Apps().app_types():
+                actions.extend(typ.actions)
+
+            lines = tabulize(actions, lambda a: str(a), header='ACTION,ARGS,DESCRIPTION', separator=',')
+            log()
+
+            app_session: AppSession = AppSession.create(state.app_env or 'c3', state.app_app or 'c3')
+            endpoint = Config().get('app.console-endpoint', 'https://{host}/{env}/{app}/static/console/index.html')
+            endpoint = endpoint.replace('{host}', app_session.host).replace('{env}', app_session.env).replace('{app}', state.app_app or 'c3')
+            tabulize([f'CONSOLE:,{endpoint}'], separator=',')
+
+            return lines
+
+    def completion(self, state: ReplState):
+        return super().completion(state)
+
+    def help(self, _: ReplState):
+        return f"{ShowAppActions.COMMAND}\t show app actions"

adam/commands/app/show_app_id.py

@@ -0,0 +1,44 @@
+from adam.commands.command import Command
+from adam.utils_k8s.custom_resources import CustomResources
+from adam.repl_state import ReplState, RequiredState
+from adam.utils import log
+
+class ShowAppId(Command):
+    COMMAND = 'show app id'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(ShowAppId, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return ShowAppId.COMMAND
+
+    def required(self):
+        return RequiredState.CLUSTER_OR_POD
+
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        with self.validate(args, state) as (args, state):
+            c3_app_id = 'Unknown'
+
+            apps = CustomResources.get_app_ids()
+            cr_name = CustomResources.get_cr_name(state.sts if state.sts else state.pod, namespace=state.namespace)
+            if cr_name in apps:
+                c3_app_id = (apps[cr_name])
+
+            log(c3_app_id)
+
+            return c3_app_id
+
+    def completion(self, state: ReplState):
+        return super().completion(state)
+
+    def help(self, _: ReplState):
+        return f'{ShowAppId.COMMAND}\t show app id for the Cassandra cluster'

adam/commands/app/show_app_queues.py

@@ -0,0 +1,38 @@
+from adam.commands import app, extract_options
+from adam.commands.command import Command
+from adam.repl_state import ReplState, RequiredState
+
+class ShowAppQueues(Command):
+    COMMAND = 'show app queues'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(ShowAppQueues, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return ShowAppQueues.COMMAND
+
+    def required(self):
+        return RequiredState.APP_APP
+
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        with self.validate(args, state) as (_, state):
+            with extract_options(args, '--force') as (args, forced):
+                with app(state) as http:
+                    http.post(['InvalidationQueue.countAll'], forced=forced)
+
+                return state
+
+    def completion(self, state: ReplState):
+        return super().completion(state, {'--force': None})
+
+    def help(self, _: ReplState):
+        return f"{ShowAppQueues.COMMAND} [--force]\t show invalidation queue counts"

adam/commands/app/utils_app.py

@@ -0,0 +1,106 @@
+import json
+from typing import Union
+
+from adam.app_session import AppSession
+from adam.apps import Apps
+from adam.pod_exec_result import PodExecResult
+from adam.repl_state import ReplState
+from adam.utils import log2
+from adam.utils_k8s.app_clusters import AppClusters
+from adam.utils_k8s.app_pods import AppPods
+
+def app_cd_dirs(state: ReplState):
+    if state.app_app:
+        return ['..'] + AppPods.pod_names(state.namespace, state.app_env, state.app_app)
+    elif state.app_env:
+        return ['..'] + [app[0].split('-')[1] for app in Apps.apps(state.app_env)]
+    else:
+        return [env[0] for env in Apps.envs()]
+
+class AppRestHandler:
+    def __init__(self, state: ReplState, forced = False):
+        self.state = state
+        self.forced = forced
+
+    def __enter__(self):
+        return self.post
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        return False
+
+    def post(self, args: list[str]) -> Union[ReplState, str]:
+        if not args:
+            return 'arg missing'
+
+        t_f = args[0].split('.')
+        if len(t_f) < 2:
+            return 'arg missing'
+
+        state = self.state
+
+        payload, valid = Apps().payload(t_f[0], t_f[1], args[1:] if len(args) > 1 else [])
+        if not valid:
+            log2('Missing one or more action arguments.')
+            return state
+
+        if payload:
+            try:
+                payload = json.loads(payload)
+            except json.decoder.JSONDecodeError as e:
+                log2(f'Invalid json argument: {e}')
+                return state
+
+        AppSession.run(state.app_env, state.app_app, state.namespace, t_f[0], t_f[1], payload=payload, forced=self.forced)
+
+        return state
+
+class AppPodService:
+    def __init__(self, handler: 'AppHandler'):
+        self.handler = handler
+
+    def exec(self, command: str, show_out = True) -> Union[PodExecResult, list[PodExecResult]]:
+        state = self.handler.state
+
+        if state.app_pod:
+            return AppPods.exec(state.app_pod, state.namespace, command, show_out=show_out, shell='bash')
+        elif state.app_app:
+            pods = AppPods.pod_names(state.namespace, state.app_env, state.app_app)
+            return AppClusters.exec(pods, state.namespace, command, action='bash', show_out=show_out, shell='bash')
+
+        return []
+
+    def post(self, args: list[str], forced=False) -> Union[ReplState, str]:
+        state = self.handler.state
+
+        if not args:
+            return 'arg missing'
+
+        t_f = args[0].split('.')
+        if len(t_f) < 2:
+            return 'arg missing'
+
+        payload, valid = Apps().payload(t_f[0], t_f[1], args[1:] if len(args) > 1 else [])
+        if not valid:
+            log2('Missing one or more action arguments.')
+            return state
+
+        if payload:
+            try:
+                payload = json.loads(payload)
+            except json.decoder.JSONDecodeError as e:
+                log2(f'Invalid json argument: {e}')
+                return state
+
+        AppSession.run(state.app_env, state.app_app, state.namespace, t_f[0], t_f[1], payload=payload, forced=forced)
+
+        return state
+
+class AppHandler:
+    def __init__(self, state: ReplState):
+        self.state = state
+
+    def __enter__(self):
+        return AppPodService(self)
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        return False

adam/commands/audit/audit.py

@@ -1,5 +1,7 @@
 import click
 
+from adam.commands import validate_args
+from adam.commands.audit.completions_l import completions_l
 from adam.commands.audit.audit_repair_tables import AuditRepairTables
 from adam.commands.audit.audit_run import AuditRun
 from adam.commands.audit.show_last10 import ShowLast10
@@ -8,9 +10,8 @@ from adam.commands.audit.show_top10 import ShowTop10
 from adam.commands.audit.utils_show_top10 import show_top10_completions_for_nesting
 from adam.commands.command import Command
 from adam.commands.intermediate_command import IntermediateCommand
-from adam.config import Config
 from adam.repl_state import ReplState
-from adam.sql.sql_completer import SqlCompleter, SqlVariant
+from adam.sql.lark_completer import LarkCompleter
 from adam.utils import log2, wait_log
 from adam.utils_athena import Athena
 
@@ -43,36 +44,17 @@ class Audit(IntermediateCommand):
                 r = self.intermediate_run(cmd, state, args, self.cmd_list(), display_help=False)
 
             if not r or isinstance(r, str) and r == 'command-missing':
-                sql = 'select * from audit order by ts desc limit 10'
-                if args:
-                    sql = ' '.join(args)
-                else:
+                with validate_args(args, state, default='select * from audit order by ts desc limit 10') as sql:
                     log2(sql)
-
-                Athena.run_query(sql)
+                    Athena.run_query(sql)
 
             return state
 
     def completion(self, state: ReplState):
-        if state.device == ReplState.L:
-            if not self.schema_read:
-                wait_log(f'Inspecting audit database schema...')
-                self.schema_read = True
-                # warm up the caches first time when l: drive is accessed
-                Athena.table_names()
-                Athena.column_names()
-                Athena.column_names(partition_cols_only=True)
-
-            return super().completion(state) | show_top10_completions_for_nesting() | SqlCompleter(
-                lambda: Athena.table_names(),
-                expandables={
-                    'columns': lambda table: Athena.column_names(),
-                    'partition_columns': lambda table: Athena.column_names(partition_cols_only=True)
-                },
-                variant=SqlVariant.ATHENA
-            ).completions_for_nesting()
-
-        return {}
+        if state.device != ReplState.L:
+            return {}
+
+        return completions_l()
 
     def cmd_list(self):
         return [AuditRepairTables(), AuditRun(), ShowLast10(), ShowSlow10(), ShowTop10()]

adam/commands/audit/audit_repair_tables.py

@@ -1,5 +1,6 @@
 import time
 
+from adam.commands import validate_args
 from adam.commands.command import Command
 from adam.config import Config
 from adam.repl_state import ReplState
@@ -31,12 +32,9 @@ class AuditRepairTables(Command):
             return super().run(cmd, state)
 
         with self.validate(args, state) as (args, state):
-            tables = Config().get('audit.athena.repair-partition-tables', 'audit').split(',')
-            if args:
-                tables = args
-
-            meta = Audits.get_meta()
-            self.repair(tables, meta)
+            with validate_args(args, state, default=Config().get('audit.athena.repair-partition-tables', 'audit').split(',')) as tables:
+                meta = Audits.get_meta()
+                self.repair(tables, meta)
 
             return state
 
@@ -48,7 +46,7 @@ class AuditRepairTables(Command):
                     with Audits.offload() as exec:
                         exec.submit(lambda: self.auto_repair(hours))
 
-            return super().completion(state)
+            # return super().completion(state)
 
         return {}
 

adam/commands/audit/audit_run.py

@@ -44,7 +44,7 @@ class AuditRun(Command):
             return state
 
     def completion(self, state: ReplState):
-        return super().completion(state)
+        return {}
 
     def help(self, _: ReplState):
         return f"{AuditRun.COMMAND}\t run"

adam/commands/audit/completions_l.py

@@ -0,0 +1,15 @@
+from adam.repl_state import ReplState
+from adam.sql.lark_completer import LarkCompleter
+from adam.utils_athena import Athena
+
+def completions_l():
+    return LarkCompleter(
+        expandables={
+            'tables': lambda x: Athena.table_names(),
+            'columns': lambda table: Athena.column_names(),
+            'partition-columns': lambda table: Athena.column_names(partition_cols_only=True),
+            'topn-counts': lambda x: ['10'],
+            'topn-types': lambda x: ['last', 'slow', 'top'],
+            'topn-windows': lambda x: ['day', 'month'],
+        }, variant=ReplState.L
+    ).completions_for_nesting()

adam/commands/audit/show_last10.py

@@ -1,9 +1,6 @@
-from adam.commands.audit.utils_show_top10 import extract_limit_and_duration
+from adam.commands.audit.utils_show_top10 import run_configured_query
 from adam.commands.command import Command
 from adam.repl_state import ReplState
-from adam.utils import log2
-from adam.utils_athena import Athena
-from adam.utils_audits import Audits
 
 class ShowLast10(Command):
     COMMAND = 'show last'
@@ -28,20 +25,11 @@ class ShowLast10(Command):
             return super().run(cmd, state)
 
         with self.validate(args, state) as (args, state):
-            limit, date_condition = extract_limit_and_duration(args)
-
-            query = '\n    '.join([
-                "SELECT * FROM audit",
-                f"WHERE drive <> 'z' and ({date_condition})",
-                f"ORDER BY ts DESC LIMIT {limit};"])
-            log2(query)
-            log2()
-            Athena.run_query(query)
+            run_configured_query('audit.queries.last10', args)
 
             return state
 
     def completion(self, _: ReplState):
-
         return {}
 
     def help(self, _: ReplState):

adam/commands/audit/show_slow10.py

@@ -1,9 +1,6 @@
-from adam.commands.audit.utils_show_top10 import extract_limit_and_duration
+from adam.commands.audit.utils_show_top10 import run_configured_query
 from adam.commands.command import Command
 from adam.repl_state import ReplState
-from adam.utils import log2
-from adam.utils_athena import Athena
-from adam.utils_audits import Audits
 
 class ShowSlow10(Command):
     COMMAND = 'show slow'
@@ -28,15 +25,7 @@ class ShowSlow10(Command):
             return super().run(cmd, state)
 
         with self.validate(args, state) as (args, state):
-            limit, date_condition = extract_limit_and_duration(args)
-
-            query = '\n    '.join([
-                "SELECT * FROM audit",
-                f"WHERE drive <> 'z' and ({date_condition})",
-                f"ORDER BY CAST(duration AS REAL) DESC LIMIT {limit};"])
-            log2(query)
-            log2()
-            Athena.run_query(query)
+            run_configured_query('audit.queries.slow10', args)
 
             return state
 

adam/commands/audit/show_top10.py

@@ -1,8 +1,6 @@
-from adam.commands.audit.utils_show_top10 import extract_limit_and_duration
+from adam.commands.audit.utils_show_top10 import run_configured_query
 from adam.commands.command import Command
 from adam.repl_state import ReplState
-from adam.utils import log2
-from adam.utils_athena import Athena
 
 class ShowTop10(Command):
     COMMAND = 'show top'
@@ -27,14 +25,7 @@ class ShowTop10(Command):
             return super().run(cmd, state)
 
         with self.validate(args, state) as (args, state):
-            limit, date_condition = extract_limit_and_duration(args)
-            query = '\n    '.join([
-                "SELECT min(c) AS cluster, line, COUNT(*) AS cnt, avg(CAST(duration AS REAL)) AS duration",
-                f"FROM audit WHERE drive <> 'z' and ({date_condition})",
-                f"GROUP BY line ORDER BY cnt DESC LIMIT {limit};"])
-            log2(query)
-            log2()
-            Athena.run_query(query)
+            run_configured_query('audit.queries.top10', args)
 
             return state
 

adam/commands/audit/utils_show_top10.py

@@ -1,10 +1,23 @@
 from datetime import datetime, timedelta
 
-from adam.utils import log_exc
+from adam.config import Config
+from adam.utils import log2, log_exc
+from adam.utils_athena import Athena
 from adam.utils_audits import Audits
 from adam.utils_repl.automata_completer import AutomataCompleter
 from adam.utils_repl.state_machine import StateMachine
 
+def run_configured_query(config_key: str, args: list[str]):
+    limit, date_condition = extract_limit_and_duration(args)
+    if query := Config().get(config_key, None):
+        query = '\n    '.join(query.split('\n'))
+        query = query.replace('{date_condition}', date_condition)
+        query = query.replace('{limit}', str(limit))
+
+        log2(query)
+        log2()
+        Athena.run_query(query)
+
 def extract_limit_and_duration(args: list[str]) -> tuple[int, datetime]:
     limit = 10
     _from = datetime.now() - timedelta(days=30)

adam/commands/bash/bash.py

@@ -30,7 +30,7 @@ class Bash(Command):
                 return exec(args)
 
     def completion(self, state: ReplState):
-        return Devices.device(state).bash_completion(Bash.COMMAND, state, default = {})
+        return super().completion(state, {c : {'&': None} for c in ['ls', 'cat', 'head']}, pods=Devices.device(state).pods(state, '-'))
 
     def help(self, _: ReplState):
         return f'{Bash.COMMAND} [pod-name] [bash-commands] [&]\t run bash on the Cassandra nodes'