kaqing 2.0.52__py3-none-any.whl → 2.0.184__py3-none-any.whl

adam/commands/alter_tables.py

@@ -1,9 +1,9 @@
+from adam.commands import extract_options, validate_args
 from adam.commands.command import Command
-from adam.commands.cql.cql_utils import tables as get_tables, run_cql
+from adam.commands.cql.utils_cql import cassandra, cassandra_tables as get_tables
 from adam.config import Config
-from adam.pod_exec_result import PodExecResult
 from adam.repl_state import ReplState, RequiredState
-from adam.utils import log2
+from adam.utils import log2, log_exc
 
 class AlterTables(Command):
     COMMAND = 'alter tables with'
@@ -27,66 +27,40 @@ class AlterTables(Command):
         if not(args := self.args(cmd)):
             return super().run(cmd, state)
 
-        state, args = self.apply_state(args, state)
-        if not self.validate_state(state):
-            return state
-
-        if not args:
-            if state.in_repl:
-                log2('Please enter gc grace in seconds. e.g. alter gc-grace-seconds 3600')
-            else:
-                log2('* gc grace second is missing.')
-                log2()
-                Command.display_help()
-
-            return 'missing-arg'
-
-        args, include_reaper = Command.extract_options(args, '--include-reaper')
-        arg_str = ' '.join(args)
-
-        # r: list[PodExecResult] = run_cql(state, 'describe tables', show_out=False, on_any=True)
-        # if not r:
-        #     log2('No pod is available')
-        #     return 'no-pod'
-
-        excludes = [e.strip(' \r\n') for e in Config().get(
-            'cql.alter-tables.excludes',
-            'system_auth,system_traces,reaper_db,system_distributed,system_views,system,system_schema,system_virtual_schema').split(',')]
-        batching = Config().get('cql.alter-tables.batching', True)
-        tables = get_tables(state, on_any=True)
-        for k, v in tables.items():
-            if k not in excludes or k == 'reaper_db' and include_reaper:
-                if batching:
-                    # alter table <table_name> with GC_GRACE_SECONDS = <timeout>;
-                    cql = ';\n'.join([f'alter table {k}.{t} with {arg_str}' for t in v])
-                    try:
-                        run_cql(state, cql, [], show_out=Config().is_debug(), on_any=True)
-                    except Exception as e:
-                        log2(e)
-                        continue
-                else:
-                    for t in v:
-                        try:
-                            # alter table <table_name> with GC_GRACE_SECONDS = <timeout>;
-                            cql = f'alter table {k}.{t} with {arg_str}'
-                            run_cql(state, cql, [], show_out=Config().is_debug(), on_any=True)
-                        except Exception as e:
-                            log2(e)
-                            continue
-
-                log2(f'{len(v)} tables altered in {k}.')
-
-        # do not continue to cql route
-        return state
-
-    def completion(self, state: ReplState) -> dict[str, any]:
-        if state.sts:
-            ps = Config().get('cql.alter-tables.gc-grace-periods', '3600,86400,864000,7776000').split(',')
-            return super().completion(state, {
-                'GC_GRACE_SECONDS': {'=': {p.strip(' \r\n'): {'--include-reaper': None} for p in ps}},
-            })
-
+        with self.validate(args, state) as (args, state):
+            with extract_options(args, '--include-reaper') as (args, include_reaper):
+                with validate_args(args, state, name='gc grace in seconds') as arg_str:
+                    excludes = [e.strip(' \r\n') for e in Config().get(
+                        'cql.alter-tables.excludes',
+                        'system_auth,system_traces,reaper_db,system_distributed,system_views,system,system_schema,system_virtual_schema').split(',')]
+                    batching = Config().get('cql.alter-tables.batching', True)
+                    tables = get_tables(state, on_any=True)
+                    for k, v in tables.items():
+                        if k not in excludes or k == 'reaper_db' and include_reaper:
+                            if batching:
+                                # alter table <table_name> with GC_GRACE_SECONDS = <timeout>;
+                                cql = ';\n'.join([f'alter table {k}.{t} with {arg_str}' for t in v])
+                                with log_exc(True):
+                                    with cassandra(state) as pods:
+                                        pods.cql(cql, show_out=Config().is_debug(), show_query=not Config().is_debug(), on_any=True)
+                                    continue
+                            else:
+                                for t in v:
+                                    with log_exc(True):
+                                        # alter table <table_name> with GC_GRACE_SECONDS = <timeout>;
+                                        cql = f'alter table {k}.{t} with {arg_str}'
+                                        with cassandra(state) as pods:
+                                            pods.cql(show_out=Config().is_debug(), show_query=not Config().is_debug(), on_any=True)
+                                        continue
+
+                            log2(f'{len(v)} tables altered in {k}.')
+
+                    # do not continue to cql route
+                    return state
+
+    def completion(self, _: ReplState) -> dict[str, any]:
+        # auto completion is taken care of by sql completer
         return {}
 
     def help(self, _: ReplState) -> str:
-        return f'{AlterTables.COMMAND} <param = value> [--include-reaper] \t alter on all tables'
+        return f'{AlterTables.COMMAND} <param = value> [--include-reaper] \t alter schema on all tables'

adam/commands/app/app.py

@@ -0,0 +1,38 @@
+from adam.commands import app, extract_options
+from adam.commands.command import Command
+from adam.repl_state import ReplState, RequiredState
+
+class App(Command):
+    COMMAND = 'app'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(App, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return App.COMMAND
+
+    def required(self):
+        return RequiredState.APP_APP
+
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        with self.validate(args, state) as (args, state):
+            with extract_options(args, '--force') as (args, forced):
+                with app(state) as http:
+                    http.post(args, forced=forced)
+
+                return state
+
+    def completion(self, state: ReplState):
+        return super().completion(state, {'--force': None})
+
+    def help(self, _: ReplState):
+        return f"<AppType>.<AppAction> <args> [--force]\t post app action; check with 'show app actions' command"

adam/commands/{app_ping.py → app/app_ping.py}

@@ -1,6 +1,6 @@
+from adam.commands import app, extract_options
 from adam.commands.command import Command
 from adam.repl_state import ReplState, RequiredState
-from adam.app_session import AppSession
 
 class AppPing(Command):
     COMMAND = 'app ping'
@@ -18,27 +18,21 @@ class AppPing(Command):
         return AppPing.COMMAND
 
     def required(self):
-        return RequiredState.CLUSTER_OR_POD
+        return RequiredState.APP_APP
 
     def run(self, cmd: str, state: ReplState):
         if not(args := self.args(cmd)):
             return super().run(cmd, state)
 
-        state, args = self.apply_state(args, state)
-        if not self.validate_state(state, app_required=RequiredState.APP_APP):
-            return state
+        with self.validate(args, state) as (args, state):
+            with extract_options(args, '--force') as (args, forced):
+                with app(state) as http:
+                    http.post(['Echo.echoStatic'], forced=forced)
 
-        _, forced = Command.extract_options(args, '--force')
-
-        AppSession.run(state.app_env, state.app_app, state.namespace, 'Echo', 'echoStatic', forced= forced)
-
-        return state
+                    return state
 
     def completion(self, state: ReplState):
-        if state.app_app:
-            return super().completion(state, {'--force': None})
-
-        return {}
+        return super().completion(state, {'--force': None})
 
     def help(self, _: ReplState):
         return f"{AppPing.COMMAND} [--force]\t ping app server with Echo.echoStatic()"

adam/commands/app/show_app_actions.py

@@ -0,0 +1,49 @@
+from adam.app_session import AppSession
+from adam.apps import AppAction, Apps
+from adam.commands.command import Command
+from adam.config import Config
+from adam.repl_state import ReplState
+from adam.utils import tabulize, log
+
+class ShowAppActions(Command):
+    COMMAND = 'show app actions'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(ShowAppActions, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return ShowAppActions.COMMAND
+
+    def required(self):
+        return ReplState.A
+
+    def run(self, cmd: str, state: ReplState):
+        if not (args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        with self.validate(args, state) as (args, state):
+            actions = []
+            for typ in Apps().app_types():
+                actions.extend(typ.actions)
+
+            lines = tabulize(actions, lambda a: str(a), header='ACTION,ARGS,DESCRIPTION', separator=',')
+            log()
+
+            app_session: AppSession = AppSession.create(state.app_env or 'c3', state.app_app or 'c3')
+            endpoint = Config().get('app.console-endpoint', 'https://{host}/{env}/{app}/static/console/index.html')
+            endpoint = endpoint.replace('{host}', app_session.host).replace('{env}', app_session.env).replace('{app}', state.app_app or 'c3')
+            tabulize([f'CONSOLE:,{endpoint}'], separator=',')
+
+            return lines
+
+    def completion(self, state: ReplState):
+        return super().completion(state)
+
+    def help(self, _: ReplState):
+        return f"{ShowAppActions.COMMAND}\t show app actions"

adam/commands/{show → app}/show_app_id.py

@@ -1,5 +1,5 @@
 from adam.commands.command import Command
-from adam.k8s_utils.custom_resources import CustomResources
+from adam.utils_k8s.custom_resources import CustomResources
 from adam.repl_state import ReplState, RequiredState
 from adam.utils import log
 
@@ -25,20 +25,17 @@ class ShowAppId(Command):
         if not(args := self.args(cmd)):
             return super().run(cmd, state)
 
-        state, _ = state.apply_args(args)
-        if not self.validate_state(state):
-            return state
+        with self.validate(args, state) as (args, state):
+            c3_app_id = 'Unknown'
 
-        c3_app_id = 'Unknown'
+            apps = CustomResources.get_app_ids()
+            cr_name = CustomResources.get_cr_name(state.sts if state.sts else state.pod, namespace=state.namespace)
+            if cr_name in apps:
+                c3_app_id = (apps[cr_name])
 
-        apps = CustomResources.get_app_ids()
-        cr_name = CustomResources.get_cr_name(state.sts if state.sts else state.pod, namespace=state.namespace)
-        if cr_name in apps:
-            c3_app_id = (apps[cr_name])
+            log(c3_app_id)
 
-        log(c3_app_id)
-
-        return c3_app_id
+            return c3_app_id
 
     def completion(self, state: ReplState):
         return super().completion(state)

adam/commands/{show → app}/show_app_queues.py

@@ -1,6 +1,6 @@
+from adam.commands import app, extract_options
 from adam.commands.command import Command
 from adam.repl_state import ReplState, RequiredState
-from adam.app_session import AppSession
 
 class ShowAppQueues(Command):
     COMMAND = 'show app queues'
@@ -18,27 +18,21 @@ class ShowAppQueues(Command):
         return ShowAppQueues.COMMAND
 
     def required(self):
-        return RequiredState.CLUSTER_OR_POD
+        return RequiredState.APP_APP
 
     def run(self, cmd: str, state: ReplState):
         if not(args := self.args(cmd)):
             return super().run(cmd, state)
 
-        state, args = self.apply_state(args, state)
-        if not self.validate_state(state, app_required=RequiredState.APP_APP):
-            return state
+        with self.validate(args, state) as (_, state):
+            with extract_options(args, '--force') as (args, forced):
+                with app(state) as http:
+                    http.post(['InvalidationQueue.countAll'], forced=forced)
 
-        _, forced = Command.extract_options(args, '--force')
-
-        AppSession.run(state.app_env, state.app_app, state.namespace, 'InvalidationQueue', 'countAll', forced= forced)
-
-        return state
+                return state
 
     def completion(self, state: ReplState):
-        if state.app_app:
-            return super().completion(state, {'--force': None})
-
-        return {}
+        return super().completion(state, {'--force': None})
 
     def help(self, _: ReplState):
         return f"{ShowAppQueues.COMMAND} [--force]\t show invalidation queue counts"

adam/commands/app/utils_app.py

@@ -0,0 +1,98 @@
+import json
+from typing import Union
+
+from adam.app_session import AppSession
+from adam.apps import Apps
+from adam.pod_exec_result import PodExecResult
+from adam.repl_state import ReplState
+from adam.utils import log2
+from adam.utils_k8s.app_clusters import AppClusters
+from adam.utils_k8s.app_pods import AppPods
+
+class AppRestHandler:
+    def __init__(self, state: ReplState, forced = False):
+        self.state = state
+        self.forced = forced
+
+    def __enter__(self):
+        return self.post
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        return False
+
+    def post(self, args: list[str]) -> Union[ReplState, str]:
+        if not args:
+            return 'arg missing'
+
+        t_f = args[0].split('.')
+        if len(t_f) < 2:
+            return 'arg missing'
+
+        state = self.state
+
+        payload, valid = Apps().payload(t_f[0], t_f[1], args[1:] if len(args) > 1 else [])
+        if not valid:
+            log2('Missing one or more action arguments.')
+            return state
+
+        if payload:
+            try:
+                payload = json.loads(payload)
+            except json.decoder.JSONDecodeError as e:
+                log2(f'Invalid json argument: {e}')
+                return state
+
+        AppSession.run(state.app_env, state.app_app, state.namespace, t_f[0], t_f[1], payload=payload, forced=self.forced)
+
+        return state
+
+class AppPodService:
+    def __init__(self, handler: 'AppHandler'):
+        self.handler = handler
+
+    def exec(self, command: str, show_out = True) -> Union[PodExecResult, list[PodExecResult]]:
+        state = self.handler.state
+
+        if state.app_pod:
+            return AppPods.exec(state.app_pod, state.namespace, command, show_out=show_out, shell='bash')
+        elif state.app_app:
+            pods = AppPods.pod_names(state.namespace, state.app_env, state.app_app)
+            return AppClusters.exec(pods, state.namespace, command, action='bash', show_out=show_out, shell='bash')
+
+        return []
+
+    def post(self, args: list[str], forced=False) -> Union[ReplState, str]:
+        state = self.handler.state
+
+        if not args:
+            return 'arg missing'
+
+        t_f = args[0].split('.')
+        if len(t_f) < 2:
+            return 'arg missing'
+
+        payload, valid = Apps().payload(t_f[0], t_f[1], args[1:] if len(args) > 1 else [])
+        if not valid:
+            log2('Missing one or more action arguments.')
+            return state
+
+        if payload:
+            try:
+                payload = json.loads(payload)
+            except json.decoder.JSONDecodeError as e:
+                log2(f'Invalid json argument: {e}')
+                return state
+
+        AppSession.run(state.app_env, state.app_app, state.namespace, t_f[0], t_f[1], payload=payload, forced=forced)
+
+        return state
+
+class AppHandler:
+    def __init__(self, state: ReplState):
+        self.state = state
+
+    def __enter__(self):
+        return AppPodService(self)
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        return False

adam/commands/audit/audit.py

@@ -0,0 +1,81 @@
+import click
+
+from adam.commands import validate_args
+from adam.commands.audit.audit_repair_tables import AuditRepairTables
+from adam.commands.audit.audit_run import AuditRun
+from adam.commands.audit.show_last10 import ShowLast10
+from adam.commands.audit.show_slow10 import ShowSlow10
+from adam.commands.audit.show_top10 import ShowTop10
+from adam.commands.audit.utils_show_top10 import show_top10_completions_for_nesting
+from adam.commands.command import Command
+from adam.commands.intermediate_command import IntermediateCommand
+from adam.repl_state import ReplState
+from adam.sql.sql_completer import SqlCompleter, SqlVariant
+from adam.utils import log2, wait_log
+from adam.utils_athena import Athena
+
+class Audit(IntermediateCommand):
+    COMMAND = 'audit'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(Audit, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+        self.schema_read = False
+
+    def command(self):
+        return Audit.COMMAND
+
+    def required(self):
+        return ReplState.L
+
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        with self.validate(args, state) as (args, state):
+            r = None
+            if len(args) > 0:
+                r = self.intermediate_run(cmd, state, args, self.cmd_list(), display_help=False)
+
+            if not r or isinstance(r, str) and r == 'command-missing':
+                with validate_args(args, state, default='select * from audit order by ts desc limit 10') as sql:
+                    log2(sql)
+                    Athena.run_query(sql)
+
+            return state
+
+    def completion(self, state: ReplState):
+        if state.device == ReplState.L:
+            if not self.schema_read:
+                wait_log(f'Inspecting audit database schema...')
+                self.schema_read = True
+                # warm up the caches first time when l: drive is accessed
+                Athena.table_names()
+                Athena.column_names()
+                Athena.column_names(partition_cols_only=True)
+
+            return super().completion(state) | show_top10_completions_for_nesting() | SqlCompleter(
+                lambda: Athena.table_names(),
+                expandables={
+                    'columns': lambda table: Athena.column_names(),
+                    'partition_columns': lambda table: Athena.column_names(partition_cols_only=True)
+                },
+                variant=SqlVariant.ATHENA
+            ).completions_for_nesting()
+
+        return {}
+
+    def cmd_list(self):
+        return [AuditRepairTables(), AuditRun(), ShowLast10(), ShowSlow10(), ShowTop10()]
+
+    def help(self, _: ReplState):
+        return f'[{Audit.COMMAND}] [<sql-statements>]\t run SQL queries on Authena audit database'
+
+class AuditCommandHelper(click.Command):
+    def get_help(self, ctx: click.Context):
+        IntermediateCommand.intermediate_help(super().get_help(ctx), Audit.COMMAND, Audit().cmd_list(), show_cluster_help=False)

adam/commands/audit/audit_repair_tables.py

@@ -0,0 +1,72 @@
+import time
+
+from adam.commands import validate_args
+from adam.commands.command import Command
+from adam.config import Config
+from adam.repl_state import ReplState
+from adam.utils import log, log2
+from adam.utils_athena import Athena
+from adam.utils_audits import AuditMeta, Audits
+
+class AuditRepairTables(Command):
+    COMMAND = 'audit repair'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(AuditRepairTables, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+        self.auto_repaired = False
+
+    def command(self):
+        return AuditRepairTables.COMMAND
+
+    def required(self):
+        return ReplState.L
+
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        with self.validate(args, state) as (args, state):
+            with validate_args(args, state, default=Config().get('audit.athena.repair-partition-tables', 'audit').split(',')) as tables:
+                meta = Audits.get_meta()
+                self.repair(tables, meta)
+
+            return state
+
+    def completion(self, state: ReplState):
+        # trigger auto repair if on l: drive
+        if state.device == ReplState.L:
+            if not self.auto_repaired:
+                if hours := Config().get('audit.athena.auto-repair.elapsed_hours', 12):
+                    with Audits.offload() as exec:
+                        exec.submit(lambda: self.auto_repair(hours))
+
+            return super().completion(state)
+
+        return {}
+
+    def auto_repair(self, hours: int):
+        self.auto_repaired = True
+
+        meta: AuditMeta = Audits.get_meta()
+        if meta.partitions_last_checked + hours * 60 * 60 < time.time():
+            tables = Config().get('audit.athena.repair-partition-tables', 'audit').split(',')
+            self.repair(tables, meta, show_sql=True)
+            log2(f'Audit tables have been auto-repaired.')
+
+    def repair(self, tables: list[str], meta: AuditMeta, show_sql = False):
+        with Audits.offload() as exec:
+            for table in tables:
+                if show_sql:
+                    log(f'MSCK REPAIR TABLE {table}')
+
+                exec.submit(Athena.query, f'MSCK REPAIR TABLE {table}', None,)
+            exec.submit(Audits.put_meta, Audits.PARTITIONS_ADDED, meta,)
+
+    def help(self, _: ReplState):
+        return f"{AuditRepairTables.COMMAND}\t run MSCK REPAIR command for new partition discovery"

adam/commands/audit/audit_run.py

@@ -0,0 +1,50 @@
+from adam.commands.command import Command
+from adam.config import Config
+from adam.repl_state import ReplState
+from adam.utils import log2
+from adam.utils_athena import Athena
+from adam.utils_audits import AuditMeta, Audits
+
+class AuditRun(Command):
+    COMMAND = 'audit run'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(AuditRun, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+        self.auto_repaired = False
+
+    def command(self):
+        return AuditRun.COMMAND
+
+    def required(self):
+        return ReplState.L
+
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        with self.validate(args, state) as (args, state):
+            meta: AuditMeta = Audits.get_meta()
+            clusters = Audits.find_new_clusters(meta.cluster_last_checked)
+            Audits.put_meta(Audits.ADD_CLUSTERS, meta, clusters=clusters)
+            if clusters:
+                log2(f'Added {len(clusters)} new clusters.')
+                tables = Config().get('audit.athena.repair-cluster-tables', 'cluster').split(',')
+                with Audits.offload() as exec:
+                    for table in tables:
+                        exec.submit(Athena.query, f'MSCK REPAIR TABLE {table}', None,)
+            else:
+                log2(f'No new clusters were found.')
+
+            return state
+
+    def completion(self, state: ReplState):
+        return super().completion(state)
+
+    def help(self, _: ReplState):
+        return f"{AuditRun.COMMAND}\t run"

adam/commands/audit/show_last10.py

@@ -0,0 +1,37 @@
+from adam.commands.audit.utils_show_top10 import run_configured_query
+from adam.commands.command import Command
+from adam.repl_state import ReplState
+
+class ShowLast10(Command):
+    COMMAND = 'show last'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(ShowLast10, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return ShowLast10.COMMAND
+
+    def required(self):
+        return ReplState.L
+
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        with self.validate(args, state) as (args, state):
+            run_configured_query('audit.queries.last10', args)
+
+            return state
+
+    def completion(self, _: ReplState):
+
+        return {}
+
+    def help(self, _: ReplState):
+        return f'{ShowLast10.COMMAND} [limit]\t show last <limit> audit lines'