kaqing 2.0.188__py3-none-any.whl → 2.0.211__py3-none-any.whl

adam/batch.py

@@ -11,16 +11,16 @@ from adam.commands.deploy.deploy import Deploy, DeployCommandHelper
 from adam.commands.deploy.undeploy import Undeploy, UndeployCommandHelper
 from adam.commands.issues import Issues
 from adam.commands.login import Login
-from adam.commands.logs import Logs
-from adam.commands.ls import Ls
+from adam.commands.download_cassandra_log import DownloadCassandraLog
+from adam.commands.fs.ls import Ls
 from adam.commands.medusa.medusa import Medusa
 from adam.commands.nodetool import NodeTool, NodeToolCommandHelper
 from adam.commands.postgres.postgres import Postgres, PostgresCommandHelper
 from adam.commands.preview_table import PreviewTable
 from adam.commands.reaper.reaper import Reaper, ReaperCommandHelper
 from adam.commands.repair.repair import Repair, RepairCommandHelper
-from adam.commands.report import Report
-from adam.commands.restart import Restart
+from adam.commands.generate_report import GenerateReport
+from adam.commands.restart_nodes import RestartNodes
 from adam.commands.rollout import RollOut
 from adam.commands.show.show import Show, ShowCommandHelper
 from adam.commands.watch import Watch
@@ -128,7 +128,7 @@ def login(kubeconfig: str, config: str, param: list[str], namespace: str, pod: s
 @click.option('--pod', '-p', required=False, metavar='pod', help='Kubernetes pod name')
 @click.argument('extra_args', nargs=-1, metavar='<pod>', type=click.UNPROCESSED)
 def logs(kubeconfig: str, config: str, param: list[str], namespace: str, pod: str, extra_args):
-    run_command(Logs(), kubeconfig, config, param, None, namespace, pod, extra_args)
+    run_command(DownloadCassandraLog(), kubeconfig, config, param, None, namespace, pod, extra_args)
 
 
 @cli.command(context_settings=dict(ignore_unknown_options=True, allow_extra_args=True), cls=ClusterCommandHelper, help='List statefulset or pods.')
@@ -221,7 +221,7 @@ def repair(kubeconfig: str, config: str, param: list[str], cluster: str, namespa
 @click.option('--show', '-s', is_flag=True, help='show output from Cassandra nodes')
 @click.argument('extra_args', nargs=-1, metavar='[cluster|pod]', type=click.UNPROCESSED)
 def report(kubeconfig: str, config: str, param: list[str], cluster: str, namespace: str, pod: str, show: bool, extra_args):
-    run_command(Report(), kubeconfig, config, param, cluster, namespace, pod, ('-s',) + extra_args if show else extra_args)
+    run_command(GenerateReport(), kubeconfig, config, param, cluster, namespace, pod, ('-s',) + extra_args if show else extra_args)
 
 
 @cli.command(context_settings=dict(ignore_unknown_options=True, allow_extra_args=True), cls=ClusterOrPodCommandHelper, help='Restart Cassandra Cluster or Node')
@@ -234,7 +234,7 @@ def report(kubeconfig: str, config: str, param: list[str], cluster: str, namespa
 @click.option('--force', is_flag=True, help='need for restarting the whole cluster')
 @click.argument('extra_args', nargs=-1, metavar='<cluster|pod>', type=click.UNPROCESSED)
 def restart(kubeconfig: str, config: str, param: list[str], cluster: str, namespace: str, pod: str, force: bool, extra_args):
-    run_command(Restart(), kubeconfig, config, param, cluster, namespace, pod, ('--force',) + extra_args if force else extra_args)
+    run_command(RestartNodes(), kubeconfig, config, param, cluster, namespace, pod, ('--force',) + extra_args if force else extra_args)
 
 
 @cli.command(context_settings=dict(ignore_unknown_options=True, allow_extra_args=True), cls=ClusterOrPodCommandHelper, help='Rolling restart Cassandra Cluster.')

adam/commands/app/utils_app.py

@@ -3,7 +3,7 @@ from typing import Union
 
 from adam.app_session import AppSession
 from adam.apps import Apps
-from adam.pod_exec_result import PodExecResult
+from adam.utils_k8s.pod_exec_result import PodExecResult
 from adam.repl_state import ReplState
 from adam.utils import log2
 from adam.utils_k8s.app_clusters import AppClusters

adam/commands/bash/bash.py

@@ -30,7 +30,7 @@ class Bash(Command):
                 return exec(args)
 
     def completion(self, state: ReplState):
-        return super().completion(state, {c : {'&': None} for c in ['ls', 'cat', 'head']}, pods=Devices.device(state).pods(state, '-'))
+        return super().completion(state, {c : {'&': None} for c in ['ls', 'cat', 'head']}, pods=Devices.of(state).pods(state, '-'))
 
     def help(self, _: ReplState):
         return f'{Bash.COMMAND} [pod-name] [bash-commands] [&]\t run bash on the Cassandra nodes'

adam/commands/bash/utils_bash.py

@@ -13,4 +13,4 @@ class BashHandler:
         return False
 
     def exec(self, args: list[str]):
-        return Devices.device(self.s1).bash(self.s0, self.s1, args)
+        return Devices.of(self.s1).bash(self.s0, self.s1, args)

adam/commands/command.py

@@ -44,7 +44,7 @@ class Command:
         if pod:
             c |= self._completion(state, leaf, auto=auto, auto_key=auto_key)
 
-        c |= {f'@{p}': self._completion(state, leaf, to_validate=False, auto=auto, auto_key=auto_key) for p in pod_names if p != pod}
+        c |= {f'@{p}': self._completion(state.with_pod(p), leaf, to_validate=False, auto=auto, auto_key=auto_key) for p in pod_names if p != pod}
 
         return c
 

adam/commands/commands_utils.py

@@ -1,17 +1,14 @@
-from datetime import datetime
 from kubernetes import client
 from typing import List
 
 from adam.checks.check_utils import run_checks
 from adam.columns.columns import Columns, collect_checks
-from adam.config import Config
-from adam.repl_session import ReplSession
 from adam.utils_issues import IssuesUtils
 from adam.utils_k8s.cassandra_nodes import CassandraNodes
 from adam.utils_k8s.pods import Pods
 from adam.utils_k8s.statefulsets import StatefulSets
 from adam.repl_state import ReplState
-from adam.utils import SORT, duration, tabulize, log, log2
+from adam.utils import SORT, duration, kaqing_log_file, tabulize, log2
 
 def show_pods(pods: List[client.V1Pod], ns: str, show_namespace = True, show_host_id = True):
     if len(pods) == 0:
@@ -70,16 +67,14 @@ def show_table(state: ReplState, pods: list[str], cols: str, header: str, show_o
     r = tabulize(pods, lambda p: ','.join([c.pod_value(results, p) for c in columns]), header=header, separator=',', sorted=SORT, to = 0 if backgrounded else 1)
 
     if backgrounded:
-        log_prefix = Config().get('log-prefix', '/tmp/qing')
-        log_file = f'{log_prefix}-{datetime.now().strftime("%d%H%M%S")}.log'
+        r = write_to_kaqing_log_file(r)
 
-        with open(log_file, 'w') as f:
-            f.write(r)
-
-        ReplSession().append_history(f':sh cat {log_file}')
+    IssuesUtils.show(results, state.in_repl)
 
-        r = log_file
+    return r
 
-    IssuesUtils.show(results, state.in_repl)
+def write_to_kaqing_log_file(r: str):
+    with kaqing_log_file() as f:
+        f.write(r)
 
-    return r
+        return f.name

adam/commands/{alter_tables.py → cql/alter_tables.py}

@@ -59,7 +59,7 @@ class AlterTables(Command):
                     return state
 
     def completion(self, _: ReplState) -> dict[str, any]:
-        # auto completion is taken care of by sql completer
+        # auto completion is taken care of by lark completer
         return {}
 
     def help(self, _: ReplState) -> str:

adam/commands/cql/completions_c.py

@@ -18,6 +18,7 @@ def completions_c(state: ReplState) -> dict[str, any]:
                 'table-props': lambda x: {
                     'GC_GRACE_SECONDS': ps
                 },
+                'table-props-value': lambda x: {None: None, 'GC_GRACE_SECONDS': ps}[x],
                 'export-database-types': lambda x: ['athena', 'sqlite', 'csv'],
                 'export-databases': lambda x: ExportDatabases.database_names(),
                 'export-sessions': lambda x: ExportSessions.export_session_names(state.sts, state.pod, state.namespace),

adam/commands/cql/utils_cql.py

@@ -7,9 +7,9 @@ from adam.commands.commands_utils import show_table
 from adam.utils_k8s.cassandra_clusters import CassandraClusters
 from adam.utils_k8s.cassandra_nodes import CassandraNodes
 from adam.utils_k8s.secrets import Secrets
-from adam.pod_exec_result import PodExecResult
+from adam.utils_k8s.pod_exec_result import PodExecResult
 from adam.repl_state import ReplState
-from adam.utils import log2, log_timing, wait_log
+from adam.utils import ing, log2, log_timing, wait_log
 from adam.utils_k8s.statefulsets import StatefulSets
 
 def cd_dirs(state: ReplState) -> list[str]:
@@ -55,7 +55,7 @@ def table_spec(state: ReplState, table: str, on_any=False) -> 'TableSpec':
 
     return parse_cql_desc_table(r.stdout if state.pod else r[0].stdout)
 
-def run_cql(state: ReplState, cql: str, opts: list = [], show_out = False, show_query = False, use_single_quotes = False, on_any = False, backgrounded=False, log_file=None) -> list[PodExecResult]:
+def run_cql(state: ReplState, cql: str, opts: list = [], show_out = False, show_query = False, use_single_quotes = False, on_any = False, backgrounded=False, log_file=None, history=True) -> list[PodExecResult]:
     if show_query:
         log2(cql)
 
@@ -69,7 +69,7 @@ def run_cql(state: ReplState, cql: str, opts: list = [], show_out = False, show_
 
     with log_timing(cql):
         with cassandra(state) as pods:
-            return pods.exec(command, action='cql', show_out=show_out, on_any=on_any, backgrounded=backgrounded, log_file=log_file)
+            return pods.exec(command, action='cql', show_out=show_out, on_any=on_any, backgrounded=backgrounded, log_file=log_file, history=history)
 
 def parse_cql_desc_tables(out: str):
     # Keyspace data_endpoint_auth
@@ -227,16 +227,16 @@ class CassandraPodService:
     def __init__(self, handler: 'CassandraExecHandler'):
         self.handler = handler
 
-    def exec(self, command: str, action='bash', show_out = True, on_any = False, throw_err = False, shell = '/bin/sh', backgrounded = False, log_file = None) -> Union[PodExecResult, list[PodExecResult]]:
+    def exec(self, command: str, action='bash', show_out = True, on_any = False, throw_err = False, shell = '/bin/sh', backgrounded = False, log_file = None, history=True) -> Union[PodExecResult, list[PodExecResult]]:
         state = self.handler.state
         pod = self.handler.pod
 
         if pod:
             return CassandraNodes.exec(pod, state.namespace, command,
-                                    show_out=show_out, throw_err=throw_err, shell=shell, backgrounded=backgrounded, log_file=log_file)
+                                    show_out=show_out, throw_err=throw_err, shell=shell, backgrounded=backgrounded, log_file=log_file, history=history)
         elif state.sts:
             return CassandraClusters.exec(state.sts, state.namespace, command, action=action,
-                                        show_out=show_out, on_any=on_any, shell=shell, backgrounded=backgrounded, log_file=log_file)
+                                        show_out=show_out, on_any=on_any, shell=shell, backgrounded=backgrounded, log_file=log_file, history=history)
 
         return []
 
@@ -267,14 +267,15 @@ class CassandraPodService:
 
         return run_cql(state, query, opts=opts, show_out=show_out, show_query=show_query, use_single_quotes=use_single_quotes, on_any=on_any, backgrounded=backgrounded, log_file=log_file)
 
-    def display_table(self, cols: str, header: str, show_out = True, backgrounded = False):
+    def display_table(self, cols: str, header: str, show_out = True, backgrounded = False, msg: str = None):
         state = self.handler.state
 
-        if state.pod:
-            return show_table(state, [state.pod], cols, header, show_out=show_out, backgrounded = backgrounded)
-        elif state.sts:
-            pod_names = [pod.metadata.name for pod in StatefulSets.pods(state.sts, state.namespace)]
-            return show_table(state, pod_names, cols, header, show_out=show_out, backgrounded = backgrounded)
+        with ing(msg=msg, condition=backgrounded and msg):
+            if state.pod:
+                return show_table(state, [state.pod], cols, header, show_out=show_out, backgrounded = backgrounded)
+            elif state.sts:
+                pod_names = [pod.metadata.name for pod in StatefulSets.pods(state.sts, state.namespace)]
+                return show_table(state, pod_names, cols, header, show_out=show_out, backgrounded = backgrounded)
 
     def nodetool(self, args: str, status = False, show_out = True, backgrounded = False) -> Union[PodExecResult, list[PodExecResult]]:
         state = self.handler.state

adam/commands/debug/debug.py

@@ -0,0 +1,22 @@
+from adam.commands.command import Command
+from adam.commands.debug.debug_completes import DebugCompletes
+from adam.commands.debug.debug_timings import DebugTimings
+from adam.commands.intermediate_command import IntermediateCommand
+
+class Debug(IntermediateCommand):
+    COMMAND = 'debug'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(Debug, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return Debug.COMMAND
+
+    def cmd_list(self):
+        return [DebugTimings(), DebugCompletes()]

adam/commands/debug/debug_completes.py

@@ -0,0 +1,35 @@
+from adam.commands import validate_args
+from adam.commands.command import Command
+from adam.config import Config
+from adam.repl_state import ReplState
+
+class DebugCompletes(Command):
+    COMMAND = 'debug completes'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(DebugCompletes, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return DebugCompletes.COMMAND
+
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        with self.validate(args, state) as (args, state):
+            with validate_args(args, state, name='on, off or file') as args:
+                Config().set('debugs.complete', args)
+
+                return state
+
+    def completion(self, state: ReplState):
+        return super().completion(state, {f: None for f in ['on', 'off', 'file']})
+
+    def help(self, _: ReplState):
+        return f'{DebugCompletes.COMMAND} on|off|file\t turn auto complete debug on or off'

adam/commands/debug/debug_timings.py

@@ -0,0 +1,35 @@
+from adam.commands import validate_args
+from adam.commands.command import Command
+from adam.config import Config
+from adam.repl_state import ReplState
+
+class DebugTimings(Command):
+    COMMAND = 'debug timings'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(DebugTimings, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return DebugTimings.COMMAND
+
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        with self.validate(args, state) as (args, state):
+            with validate_args(args, state, name='on, off or file') as args:
+                Config().set('debugs.timings', args)
+
+                return state
+
+    def completion(self, state: ReplState):
+        return super().completion(state, {f: None for f in ['on', 'off', 'file']})
+
+    def help(self, _: ReplState):
+        return f'{DebugTimings.COMMAND} on|off|file\t turn timing debug on or off'

adam/commands/devices/device.py

@@ -2,7 +2,7 @@ from abc import abstractmethod
 
 from adam.commands.command import Command
 from adam.config import Config
-from adam.pod_exec_result import PodExecResult
+from adam.utils_k8s.pod_exec_result import PodExecResult
 from adam.repl_state import BashSession, ReplState
 from adam.utils import log2
 from adam.utils_k8s.pods import Pods

adam/commands/devices/devices.py

@@ -7,7 +7,7 @@ from adam.commands.devices.device_postgres import DevicePostgres
 from adam.repl_state import ReplState
 
 class Devices:
-    def device(state: ReplState) -> Device:
+    def of(state: ReplState) -> Device:
         if state.device == ReplState.A:
             return DeviceApp()
         elif state.device == ReplState.C:

adam/commands/download_cassandra_log.py

@@ -0,0 +1,45 @@
+from adam.commands.command import Command
+from adam.commands.devices.devices import Devices
+from adam.config import Config
+from adam.utils_k8s.pod_exec_result import PodExecResult
+from adam.utils import log2
+from adam.utils_k8s.cassandra_nodes import CassandraNodes
+from adam.repl_state import ReplState, RequiredState
+from adam.utils_k8s.pods import Pods
+
+class DownloadCassandraLog(Command):
+    COMMAND = 'download cassandra log'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(DownloadCassandraLog, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return DownloadCassandraLog.COMMAND
+
+    def required(self):
+        return RequiredState.POD
+
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        with self.validate(args, state) as (args, state):
+            path = Config().get('logs.path', '/c3/cassandra/logs/system.log')
+            r: PodExecResult = CassandraNodes.exec(state.pod, state.namespace, f'cat {path}', backgrounded=True, history=False)
+
+            to_file = Pods.download_file(state.pod, 'cassandra', state.namespace, path)
+            log2(f'Downloaded to {to_file}.')
+
+            return r
+
+    def completion(self, state: ReplState):
+        return super().completion(state, pods=Devices.of(state).pods(state, '-'), auto='jit')
+
+    def help(self, _: ReplState):
+        return f'{DownloadCassandraLog.COMMAND}\t download cassandra system log'

adam/commands/export/export_databases.py

@@ -1,10 +1,12 @@
 from collections.abc import Callable
 from datetime import datetime
+from functools import partial
 import os
 import boto3
 
 from adam.commands.export.export_sessions import ExportSessions
 from adam.commands.export.importer import Importer
+from adam.commands.export.utils_export import export_log_dir
 from adam.config import Config
 from adam.repl_session import ReplSession
 from adam.repl_state import ReplState
@@ -31,7 +33,7 @@ class ExportDatabases:
 
         if Config().get('repl.history.push-cat-log-file', True):
             if log_file and ReplSession().prompt_session:
-                ReplSession().prompt_session.history.append_string(f':sh cat {log_file}')
+                ReplSession().prompt_session.history.append_string(f':cat {log_file}')
 
         return cnt
 
@@ -192,20 +194,23 @@ class ExportDatabaseService:
     def __init__(self, handler: 'ExportDatabaseHandler'):
         self.handler = handler
 
-    def sql(self, query: str, database: str = None, backgrounded = False):
+    def sql(self, query: str, database: str = None, backgrounded = False, export_log: str = None):
         if not database:
             database = self.handler.state.export_session
 
-        def output(out: str):
-            log_prefix = Config().get('export.log-prefix', '/tmp/qing')
-            log_file = f'{log_prefix}-{datetime.now().strftime("%d%H%M%S")}-export.log'
+        def output(export_log: str, out: str):
+            # serving for 1. export log or 2. job log for the sampling select 10 rows afterward
+            flag = 'at'
+            if not export_log:
+                export_log = f'{export_log_dir()}/{datetime.now().strftime("%d%H%M%S")}-export.log'
+                flag = 'w'
 
-            with open(log_file, 'w') as f:
+            with open(export_log, flag) as f:
                 f.write(out)
 
-            return log_file
+            return export_log
 
-        ExportDatabases.run_query(query, database, output = output if backgrounded else None, show_query = not backgrounded)
+        ExportDatabases.run_query(query, database, output = partial(output, export_log) if backgrounded else None, show_query = not backgrounded)
 
     def drop(self, database: str):
         state = self.handler.state

adam/commands/export/export_sessions.py

@@ -3,14 +3,14 @@ import os
 import re
 
 from adam.commands.export.importer import Importer
-from adam.commands.export.utils_export import ExportTableStatus, csv_dir, find_files
+from adam.commands.export.utils_export import ExportTableStatus, csv_dir, export_log_dir, find_files, os_system_exec
 from adam.config import Config
 from adam.repl_state import ReplState
 from adam.utils import log2, tabulize, log, parallelize
 from adam.utils_k8s.cassandra_nodes import CassandraNodes
-from adam.utils_k8s.pods import Pods, log_prefix
+from adam.utils_k8s.pods import Pods
 from adam.utils_k8s.statefulsets import StatefulSets
-from adam.utils_local import local_tmp_dir
+from adam.utils_local import local_downloads_dir
 
 class ExportSessions:
     def clear_export_session_cache():
@@ -36,13 +36,13 @@ class ExportSessions:
 
         prefix = Importer.prefix_from_importer(importer)
 
-        log_files: list[str] = find_files(pod, namespace, f'{log_prefix()}-{prefix}*_*.log*')
+        log_files: list[str] = find_files(pod, namespace, f'{export_log_dir()}/{prefix}*_*.log*')
 
         if not log_files:
             return {}
 
         for log_file in log_files[:limit]:
-            m = re.match(f'{log_prefix()}-(.*?)_.*\.log?(.*)', log_file)
+            m = re.match(f'{export_log_dir()}/([ces].*?)_.*\.log?(.*)', log_file)
             if m:
                 s = m.group(1)
                 state = m.group(2) # '', '.pending_import', '.done'
@@ -66,7 +66,8 @@ class ExportSessions:
             pod = StatefulSets.pod_names(sts, namespace)[0]
 
         CassandraNodes.exec(pod, namespace, f'rm -rf {csv_dir()}/*', show_out=Config().is_debug(), shell='bash')
-        CassandraNodes.exec(pod, namespace, f'rm -rf {log_prefix()}-*.log*', show_out=Config().is_debug(), shell='bash')
+        cmd = f'rm -rf {export_log_dir()}/*.log*'
+        os_system_exec(cmd, show_out=Config().is_debug())
 
         return True
 
@@ -102,17 +103,18 @@ class ExportSessions:
         csv_cnt = 0
         log_cnt = 0
 
-        log_files: list[str] = find_files(pod, namespace, f'{log_prefix()}-{session}_*.log*')
+        log_files: list[str] = find_files(pod, namespace, f'{export_log_dir()}/{session}_*.log*')
 
         for log_file in log_files:
-            m = re.match(f'{log_prefix()}-{session}_(.*?)\.(.*?)\.log.*', log_file)
+            m = re.match(f'{export_log_dir()}/{session}_(.*?)\.(.*?)\.log.*', log_file)
             if m:
                 table = m.group(2)
 
                 CassandraNodes.exec(pod, namespace, f'rm -rf {csv_dir()}/{session}_{table}', show_out=not multi_tables, shell='bash')
                 csv_cnt += 1
 
-                CassandraNodes.exec(pod, namespace, f'rm -rf {log_file}', show_out=not multi_tables, shell='bash')
+                cmd = f'rm -rf {log_file}'
+                os_system_exec(cmd, show_out=not multi_tables)
                 log_cnt += 1
 
         return csv_cnt, log_cnt
@@ -142,8 +144,7 @@ class ExportSessions:
         def download_csv(table):
             from_path: str = table.csv_file
 
-            to_path = from_path.replace(csv_dir(), local_tmp_dir())
-            os.makedirs(os.path.dirname(to_path), exist_ok=True)
+            to_path = from_path.replace(csv_dir(), local_downloads_dir())
             Pods.download_file(pod, 'cassandra', namespace, from_path, to_path)
 
             log2(f'[{session}] Downloaded to {to_path}.')