kaqing 2.0.214__py3-none-any.whl → 2.0.227__py3-none-any.whl

adam/commands/export/show_export_session.py

@@ -1,6 +1,6 @@
 from adam.commands import validate_args
 from adam.commands.command import Command
-from adam.commands.export.export_sessions import ExportSessions, export_session
+from adam.commands.export.export_sessions import export_session
 from adam.repl_state import ReplState, RequiredState
 
 class ShowExportSession(Command):
@@ -32,8 +32,8 @@ class ShowExportSession(Command):
 
             return state
 
-    def completion(self, state: ReplState):
+    def completion(self, _: ReplState):
         return {}
 
-    def help(self, _: ReplState):
-        return f'{ShowExportSession.COMMAND} <export-session-name>\t show export session'
+    def help(self, state: ReplState):
+        return super().help(state, 'show export session', args='<export-session-name>')

adam/commands/export/show_export_sessions.py

@@ -30,8 +30,8 @@ class ShowExportSessions(Command):
 
             return state
 
-    def completion(self, state: ReplState):
+    def completion(self, _: ReplState):
         return {}
 
-    def help(self, _: ReplState):
-        return f'{ShowExportSessions.COMMAND}\t list export sessions'
+    def help(self, state: ReplState):
+        return super().help(state, 'list export sessions')

adam/commands/export/utils_export.py

@@ -3,9 +3,9 @@ import os
 import re
 
 from adam.config import Config
-from adam.utils import ExecResult, creating_dir, log2
+from adam.utils import ExecResult, creating_dir, log2, log_to_pods
 from adam.repl_state import ReplState
-from adam.utils_k8s.cassandra_nodes import CassandraNodes
+from adam.utils_k8s.pods import Pods
 from adam.utils_k8s.statefulsets import StatefulSets
 from adam.utils_local import local_exec
 
@@ -197,7 +197,7 @@ class ExportTableStatus:
         statuses: list[ExportTableStatus] = []
 
         status_in_whole = 'done'
-        log_files: list[str] = find_files(pod, namespace, f'{export_log_dir()}/{export_session}_*.log*')
+        log_files: list[str] = Pods.find_files(pod, 'cassandra', namespace, f'{table_log_dir(pod, namespace)}/{export_session}_*.log*', remote=log_to_pods())
 
         for log_file in log_files:
             status: ExportTableStatus = ExportTableStatus.from_log_file(pod, namespace, export_session, log_file)
@@ -212,7 +212,7 @@ class ExportTableStatus:
         def get_csv_files_n_table(target_table: str):
             db = f'{copy_session}_{target_table}'
             csv_file = f'{csv_dir()}/{db}/*.csv'
-            csv_files: list[str] = find_files(pod, namespace, csv_file, remote=True)
+            csv_files: list[str] = Pods.find_files(pod, 'cassandra', namespace, csv_file, remote=log_to_pods())
             if csv_files:
                 table = target_table
                 m = re.match(f'{csv_dir()}/{db}/(.*).csv', csv_files[0])
@@ -222,7 +222,7 @@ class ExportTableStatus:
 
             return csv_files, target_table
 
-        m = re.match(f'{export_log_dir()}/{copy_session}_(.*?)\.(.*?)\.log(.*)', log_file)
+        m = re.match(f'{table_log_dir(pod, namespace)}/{copy_session}_(.*?)\.(.*?)\.log(.*)', log_file)
         if m:
             keyspace = m.group(1)
             target_table = m.group(2)
@@ -235,7 +235,10 @@ class ExportTableStatus:
 
             # 4 rows exported to 1 files in 0 day, 0 hour, 0 minute, and 1.335 seconds.
             pattern = 'rows exported to'
-            r: ExecResult = local_exec(['grep', pattern, log_file], show_out=Config().is_debug())
+            if log_to_pods():
+                r: ExecResult = Pods.exec(pod, 'cassandra', namespace, f"grep '{pattern}' {log_file}", show_out=Config().is_debug())
+            else:
+                r: ExecResult = local_exec(["grep", pattern, log_file], show_out=Config().is_debug())
 
             if r.exit_code() == 0:
                 csv_files, table = get_csv_files_n_table(target_table)
@@ -248,32 +251,6 @@ class ExportTableStatus:
 
         return ExportTableStatus(None, None, 'unknown')
 
-def find_files(pod: str, namespace: str, pattern: str, mmin: int = 0, remote = False):
-    stdout = ''
-    if not remote:
-        # find . -maxdepth 1 -type f -name '*'
-        dir = os.path.dirname(pattern)
-        base = os.path.basename(pattern)
-        cmd = ['find', dir, '-name', base]
-        if mmin:
-                cmd += ['-mmin',  f'-{mmin}']
-
-        stdout = local_exec(cmd, show_out=Config().is_debug()).stdout
-    else:
-        cmd = f'find {pattern}'
-        if mmin:
-                cmd = f'{cmd} -mmin -{mmin}'
-
-        stdout = CassandraNodes.exec(pod, namespace, cmd, show_out=Config().is_debug(), shell='bash').stdout
-
-    log_files = []
-    for line in stdout.split('\n'):
-        line = line.strip(' \r')
-        if line:
-            log_files.append(line)
-
-    return log_files
-
 class GeneratorStream(io.RawIOBase):
     def __init__(self, generator):
         self._generator = generator
@@ -354,6 +331,12 @@ class PodPushHandler:
 def state_with_pod(state: ReplState, pod: str = None):
     return PodPushHandler(state, pod=pod)
 
+def fs_exec(pod: str, namespace: str, cmd: str, show_out = False):
+    if log_to_pods():
+        Pods.exec(pod, 'cassandra', namespace, cmd, show_out = show_out)
+    else:
+        os_system_exec(cmd, show_out=show_out)
+
 def os_system_exec(cmd: str, show_out = False):
     if show_out: log2(cmd)
 
@@ -362,5 +345,14 @@ def os_system_exec(cmd: str, show_out = False):
 def csv_dir():
     return Config().get('export.csv_dir', '/c3/cassandra/tmp')
 
+def table_log_dir(pod: str, namespace: str):
+    if log_to_pods():
+        return remote_export_log_dir(pod, namespace)
+    else:
+        return export_log_dir()
+
 def export_log_dir():
-    return creating_dir(Config().get('export.log-dir', '/tmp/qing-db/q/export/logs'))
+    return creating_dir(Config().get('export.log-dir', '/tmp/qing-db/q/export/logs'))
+
+def remote_export_log_dir(pod: str, namespace: str):
+    return Pods.creating_dir(pod, 'cassandra', namespace, Config().get('export.remote.log-dir', '/tmp/q/export/logs'))

adam/commands/fs/cat.py

@@ -32,5 +32,5 @@ class Cat(Command):
     def completion(self, state: ReplState):
         return super().completion(state, lambda: {f: None for f in Devices.of(state).files(state)}, pods=Devices.of(state).pods(state, '-'), auto='jit')
 
-    def help(self, _: ReplState):
-        return f'{Cat.COMMAND} file [&]\t run cat command on the pod'
+    def help(self, state: ReplState):
+        return super().help(state, 'run cat command on the pod', args='<file>')

adam/commands/fs/cat_local.py

@@ -38,5 +38,5 @@ class CatLocal(Command):
     def completion(self, state: ReplState):
         return super().completion(state, lambda: {n: None for n in find_local_files(file_type='f', max_depth=1)}, auto='jit')
 
-    def help(self, _: ReplState):
-        return f'{CatLocal.COMMAND} file\t run cat command on local system'
+    def help(self, state: ReplState):
+        return super().help(state, 'run cat command on local file system', args='<file>')

adam/commands/fs/cd.py

@@ -37,5 +37,5 @@ class Cd(Command):
     def completion(self, state: ReplState):
         return Devices.of(state).cd_completion(Cd.COMMAND, state, default = {})
 
-    def help(self, _: ReplState):
-        return f'{Cd.COMMAND} <path> | .. \t move around on the operational device hierarchy'
+    def help(self, state: ReplState):
+        return super().help(state, 'move around on the operational device hierarchy', args='<path> | ..')

adam/commands/fs/download_file.py

@@ -43,5 +43,5 @@ class DownloadFile(Command):
     def completion(self, state: ReplState):
         return super().completion(state, lambda: {f: None for f in Devices.of(state).files(state)}, pods=Devices.of(state).pods(state, '-'), auto='jit')
 
-    def help(self, _: ReplState):
-        return f'{DownloadFile.COMMAND} from-file [to-file]\t download file from pod'
+    def help(self, state: ReplState):
+        return super().help(state, 'download file from pod', args='<from-file> [to-file]')

adam/commands/fs/find_files.py

@@ -35,7 +35,7 @@ class FindLocalFiles(Command):
                 new_args = [f"'{arg}'" if '*' in arg else arg for arg in args]
                 cmd = 'find ' + ' '.join(new_args)
 
-            log2(cmd)
+            log2(cmd, text_color='gray')
             os.system(cmd)
 
             return state
@@ -47,5 +47,5 @@ class FindLocalFiles(Command):
             '*': None
         })
 
-    def help(self, _: ReplState):
-        return f'{FindLocalFiles.COMMAND} [linux-find-arguments]\t find files from local machine'
+    def help(self, state: ReplState):
+        return super().help(state, 'find files from local machine', args='[linux-find-arguments]')

adam/commands/fs/find_processes.py

@@ -2,8 +2,10 @@ from adam.commands import extract_options, validate_args
 from adam.commands.command import Command
 from adam.commands.devices.devices import Devices
 from adam.commands.export.utils_export import state_with_pod
+from adam.commands.fs.utils_fs import ProcessInfo, find_pids_for_cluster
 from adam.repl_state import ReplState, RequiredState
 from adam.utils import log2, tabulize
+from adam.utils_k8s.pod_exec_result import PodExecResult
 
 class FindProcesses(Command):
     COMMAND = 'find processes'
@@ -29,26 +31,16 @@ class FindProcesses(Command):
 
         with self.validate(args, state) as (args, state):
             with extract_options(args, '-kill') as (args, kill):
-                with validate_args(args, state, name='words to look for'):
-                    arg = ' | '.join([f'grep {a}' for a in args])
-                    awk = "awk '{ print $1, $2, $8, $NF }'"
-                    rs = Devices.of(state).bash(state, state, f"ps -ef | grep -v grep | {arg} | {awk}".split(' '))
-
-                    lines: list[list[str]] = []
-                    for r in rs:
-                        for l in r.stdout.split('\n'):
-                            l = l.strip(' \t\r\n')
-                            if not l:
-                                continue
-
-                            tokens = [r.pod] + l.split(' ')
-                            lines.append(tokens)
+                with validate_args(args, state, name='words to look for', separator=' ') as keywords:
+                    action = 'find-files'
+                    msg = 'd`Running|Ran ' + action + ' command onto {size} pods'
+                    processes = find_pids_for_cluster(state, keywords)
 
                     pids = []
-                    for l in lines:
-                        pids.append(f'{l[2]}@{l[0]}')
+                    for p in processes:
+                        pids.append(f'{p.pid}@{p.pod}')
 
-                    tabulize(lines, lambda l: '\t'.join(l), header = 'POD\tUSER\tPID\tCMD\tLAST_ARG', separator='\t')
+                    ProcessInfo.tabulize(processes)
                     log2()
                     log2(f'PIDS with {",".join(args)}: {",".join(pids)}')
 
@@ -67,10 +59,10 @@ class FindProcesses(Command):
                             with state_with_pod(state, pod) as state1:
                                 Devices.of(state).bash(state, state1, ['kill', '-9', pid])
 
-                    return rs
+                    return state
 
     def completion(self, state: ReplState):
         return super().completion(state)
 
-    def help(self, _: ReplState):
-        return f'{FindProcesses.COMMAND} word... [-kill]\t find processes with words'
+    def help(self, state: ReplState):
+        return super().help(state, 'find processes with words  --kill kill matching processes', args='word... [-kill]')

adam/commands/fs/head.py

@@ -26,11 +26,11 @@ class Head(Command):
             return super().run(cmd, state)
 
         with self.validate(args, state) as (args, state):
-            with validate_args(args, state, name='file'):
-                return Devices.of(state).bash(state, state, cmd.split(' '))
+            with validate_args(args, state, name='file') as args:
+                return Devices.of(state).bash(state, state, ['head', '-n', '10', args])
 
     def completion(self, state: ReplState):
         return super().completion(state, lambda: {f: None for f in Devices.of(state).files(state)}, pods=Devices.of(state).pods(state, '-'), auto='jit')
 
-    def help(self, _: ReplState):
-        return f'{Head.COMMAND} file [&]\t run head command on the pod'
+    def help(self, state: ReplState):
+        return super().help(state, 'run head command on the pod', args='<file>')

adam/commands/fs/head_local.py

@@ -0,0 +1,46 @@
+import os
+
+from adam.commands import validate_args
+from adam.commands.command import Command
+from adam.repl_state import ReplState, RequiredState
+from adam.utils import log2
+from adam.utils_local import find_local_files
+
+class HeadLocal(Command):
+    COMMAND = ':head'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(HeadLocal, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return HeadLocal.COMMAND
+
+    def required(self):
+        return [RequiredState.CLUSTER_OR_POD, RequiredState.APP_APP, ReplState.P]
+
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        with self.validate(args, state) as (args, state):
+            with validate_args(args, state, name='file') as args:
+                cmd = f'head -n 10 {args}'
+                log2(cmd)
+                log2()
+
+                os.system(cmd)
+                log2()
+
+                return state
+
+    def completion(self, state: ReplState):
+        return super().completion(state, lambda: {n: None for n in find_local_files(file_type='f', max_depth=1)}, auto='jit')
+
+    def help(self, state: ReplState):
+        return super().help(state, 'run head command on local system', args='<file>')

adam/commands/fs/ls.py

@@ -37,5 +37,5 @@ class Ls(Command):
     def completion(self, state: ReplState):
         return super().completion(state, {'&': None}, pods=Devices.of(state).pods(state, '-'))
 
-    def help(self, _: ReplState):
-        return f'{Ls.COMMAND} [device:]\t list apps, envs, clusters, nodes, pg hosts/databases or export databases'
+    def help(self, state: ReplState):
+        return super().help(state, 'list apps, envs, clusters, nodes, pg hosts/databases or export databases', args='[device:]')

adam/commands/fs/ls_local.py

@@ -36,5 +36,5 @@ class LsLocal(Command):
     def completion(self, state: ReplState):
         return super().completion(state)
 
-    def help(self, _: ReplState):
-        return f'{LsLocal.COMMAND} [dir]\t list files on local system'
+    def help(self, state: ReplState):
+        return super().help(state, 'list files on local system', args='[dir]')

adam/commands/fs/pwd.py

@@ -41,5 +41,5 @@ class Pwd(Command):
     def completion(self, state: ReplState):
         return super().completion(state)
 
-    def help(self, _: ReplState):
-        return f'{Pwd.COMMAND}\t print current working directories'
+    def help(self, state: ReplState):
+        return super().help(state, 'print current working directories')

adam/commands/fs/rm.py

@@ -1,5 +1,5 @@
 from adam.commands.fs.rm_downloads import RmDownloads
-from adam.commands.fs.rm_logs import RmLogs
+from adam.commands.fs.rm_logs_local import RmLogsLocal
 from adam.commands.intermediate_command import IntermediateCommand
 
 class RmLocal(IntermediateCommand):
@@ -15,4 +15,4 @@ class RmLocal(IntermediateCommand):
         return RmLocal.COMMAND
 
     def cmd_list(self):
-        return [RmDownloads(), RmLogs()]
+        return [RmDownloads(), RmLogsLocal()]

adam/commands/fs/rm_downloads.py

@@ -35,5 +35,5 @@ class RmDownloads(Command):
     def completion(self, state: ReplState):
         return super().completion(state)
 
-    def help(self, _: ReplState):
-        return f'{RmDownloads.COMMAND}\t remove all downloads files under {local_downloads_dir()}'
+    def help(self, state: ReplState):
+        return super().help(state, f'remove all downloads files under {local_downloads_dir()}')

adam/commands/fs/rm_logs.py

@@ -1,11 +1,14 @@
 import os
 
 from adam.commands.command import Command
+from adam.commands.devices.devices import Devices
+from adam.config import Config
 from adam.repl_state import ReplState
 from adam.utils import log2, log_dir
+from adam.utils_k8s.pods import Pods
 
 class RmLogs(Command):
-    COMMAND = ':rm logs'
+    COMMAND = 'rm logs'
 
     # the singleton pattern
     def __new__(cls, *args, **kwargs):
@@ -24,15 +27,18 @@ class RmLogs(Command):
             return super().run(cmd, state)
 
         with self.validate(args, state) as (args, state):
-            cmd = f'rm -rf {log_dir()}/*'
-            log2(cmd)
-            os.system(cmd)
-            log2()
+            cmd = f'rm -rf {self.log_dir()}/*'
+            action = 'rm-logs'
+            msg = 'd`Running|Ran ' + action + ' onto {size} pods'
+            pods = Devices.of(state).pod_names(state)
+            container = Devices.of(state).default_container(state)
+            with Pods.parallelize(pods, len(pods), msg=msg, action=action) as exec:
+                exec.map(lambda pod: Pods.exec(pod, container, state.namespace, cmd, show_out=True, text_color='gray'))
 
             return state
 
     def completion(self, state: ReplState):
         return super().completion(state)
 
-    def help(self, _: ReplState):
-        return f'{RmLogs.COMMAND}\t remove all qing log files under {log_dir()}'
+    def help(self, state: ReplState):
+        return super().help(state, f'remove all qing log files under {self.log_dir()}')

adam/commands/fs/rm_logs_local.py

@@ -0,0 +1,38 @@
+import os
+
+from adam.commands.command import Command
+from adam.repl_state import ReplState
+from adam.utils import log2, log_dir
+
+class RmLogsLocal(Command):
+    COMMAND = ':rm logs'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(RmLogsLocal, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return RmLogsLocal.COMMAND
+
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        with self.validate(args, state) as (args, state):
+            cmd = f'rm -rf {log_dir()}/*'
+            log2(cmd, text_color='gray')
+            os.system(cmd)
+            log2()
+
+            return state
+
+    def completion(self, state: ReplState):
+        return super().completion(state)
+
+    def help(self, state: ReplState):
+        return super().help(state, f'remove all qing log files under {log_dir()}')

adam/commands/fs/shell.py

@@ -37,5 +37,5 @@ class Shell(Command):
     def completion(self, state: ReplState):
         return super().completion(state)
 
-    def help(self, _: ReplState):
-        return f'{Shell.COMMAND}\t drop down to shell'
+    def help(self, state: ReplState):
+        return super().help(state, 'drop down to shell')

adam/commands/fs/show_adam.py

@@ -44,5 +44,5 @@ class ShowAdam(Command):
     def completion(self, state: ReplState):
         return super().completion(state)
 
-    def help(self, _: ReplState):
-        return f'{ShowAdam.COMMAND}\t show kaqing version'
+    def help(self, state: ReplState):
+        return super().help(state, 'show kaqing version')

adam/commands/fs/show_host.py

@@ -29,5 +29,5 @@ class ShowHost(Command):
     def completion(self, state: ReplState):
         return super().completion(state)
 
-    def help(self, _: ReplState):
-        return f'{ShowHost.COMMAND}\t show host'
+    def help(self, state: ReplState):
+        return super().help(state, 'show host')

adam/commands/fs/show_last_results.py

@@ -0,0 +1,39 @@
+from adam.commands.fs.utils_fs import show_last_pod_results, show_last_results
+from adam.commands.command import Command
+from adam.repl_state import ReplState
+from adam.utils import log_to_pods
+
+class ShowLastResults(Command):
+    COMMAND = 'show last results'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(ShowLastResults, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return ShowLastResults.COMMAND
+
+    def aliases(self):
+        return [':?']
+
+    def run(self, cmd: str, state: ReplState):
+        if not self.args(cmd):
+            return super().run(cmd, state)
+
+        if log_to_pods():
+            show_last_pod_results(state)
+        else:
+            show_last_results(state)
+
+        return state
+
+    def completion(self, state: ReplState):
+        return super().completion(state)
+
+    def help(self, state: ReplState):
+        return super().help(state, 'show results of last command')

adam/commands/fs/tail.py

@@ -0,0 +1,36 @@
+from adam.commands import validate_args
+from adam.commands.command import Command
+from adam.commands.devices.devices import Devices
+from adam.repl_state import ReplState, RequiredState
+
+class Tail(Command):
+    COMMAND = 'tail'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(Tail, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return Tail.COMMAND
+
+    def required(self):
+        return [RequiredState.CLUSTER_OR_POD, RequiredState.APP_APP, ReplState.P]
+
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        with self.validate(args, state) as (args, state):
+            with validate_args(args, state, name='file') as args:
+                return Devices.of(state).bash(state, state, ['tail', '-n', '10', args])
+
+    def completion(self, state: ReplState):
+        return super().completion(state, lambda: {f: None for f in Devices.of(state).files(state)}, pods=Devices.of(state).pods(state, '-'), auto='jit')
+
+    def help(self, state: ReplState):
+        return super().help(state, 'run tail command on the pod', args='<file>')

adam/commands/fs/tail_local.py

@@ -0,0 +1,46 @@
+import os
+
+from adam.commands import validate_args
+from adam.commands.command import Command
+from adam.repl_state import ReplState, RequiredState
+from adam.utils import log2
+from adam.utils_local import find_local_files
+
+class TailLocal(Command):
+    COMMAND = ':tail'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(TailLocal, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return TailLocal.COMMAND
+
+    def required(self):
+        return [RequiredState.CLUSTER_OR_POD, RequiredState.APP_APP, ReplState.P]
+
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        with self.validate(args, state) as (args, state):
+            with validate_args(args, state, name='file') as args:
+                cmd = f'tail -n 10 {args}'
+                log2(cmd)
+                log2()
+
+                os.system(cmd)
+                log2()
+
+                return state
+
+    def completion(self, state: ReplState):
+        return super().completion(state, lambda: {n: None for n in find_local_files(file_type='f', max_depth=1)}, auto='jit')
+
+    def help(self, state: ReplState):
+        return super().help(state, 'run tail command on local file system', args='<file>')