kaqing 2.0.184__py3-none-any.whl → 2.0.214__py3-none-any.whl

adam/commands/export/utils_export.py

@@ -1,12 +1,13 @@
 import io
+import os
 import re
 
 from adam.config import Config
-from adam.pod_exec_result import PodExecResult
+from adam.utils import ExecResult, creating_dir, log2
 from adam.repl_state import ReplState
 from adam.utils_k8s.cassandra_nodes import CassandraNodes
-from adam.utils_k8s.pods import log_prefix
 from adam.utils_k8s.statefulsets import StatefulSets
+from adam.utils_local import local_exec
 
 class ImportSpec:
     def __init__(self, table_name: str, session: str = None, files: list[str] = None, importer: str = None):
@@ -196,7 +197,7 @@ class ExportTableStatus:
         statuses: list[ExportTableStatus] = []
 
         status_in_whole = 'done'
-        log_files: list[str] = find_files(pod, namespace, f'{log_prefix()}-{export_session}_*.log*')
+        log_files: list[str] = find_files(pod, namespace, f'{export_log_dir()}/{export_session}_*.log*')
 
         for log_file in log_files:
             status: ExportTableStatus = ExportTableStatus.from_log_file(pod, namespace, export_session, log_file)
@@ -211,7 +212,7 @@ class ExportTableStatus:
         def get_csv_files_n_table(target_table: str):
             db = f'{copy_session}_{target_table}'
             csv_file = f'{csv_dir()}/{db}/*.csv'
-            csv_files: list[str] = find_files(pod, namespace, csv_file)
+            csv_files: list[str] = find_files(pod, namespace, csv_file, remote=True)
             if csv_files:
                 table = target_table
                 m = re.match(f'{csv_dir()}/{db}/(.*).csv', csv_files[0])
@@ -221,7 +222,7 @@ class ExportTableStatus:
 
             return csv_files, target_table
 
-        m = re.match(f'{log_prefix()}-{copy_session}_(.*?)\.(.*?)\.log(.*)', log_file)
+        m = re.match(f'{export_log_dir()}/{copy_session}_(.*?)\.(.*?)\.log(.*)', log_file)
         if m:
             keyspace = m.group(1)
             target_table = m.group(2)
@@ -234,7 +235,8 @@ class ExportTableStatus:
 
             # 4 rows exported to 1 files in 0 day, 0 hour, 0 minute, and 1.335 seconds.
             pattern = 'rows exported to'
-            r: PodExecResult = CassandraNodes.exec(pod, namespace, f"grep '{pattern}' {log_file}", show_out=Config().is_debug(), shell='bash')
+            r: ExecResult = local_exec(['grep', pattern, log_file], show_out=Config().is_debug())
+
             if r.exit_code() == 0:
                 csv_files, table = get_csv_files_n_table(target_table)
                 if csv_files:
@@ -246,17 +248,26 @@ class ExportTableStatus:
 
         return ExportTableStatus(None, None, 'unknown')
 
-def csv_dir():
-    return Config().get('export.csv_dir', '/c3/cassandra/tmp')
-
-def find_files(pod: str, namespace: str, pattern: str, mmin: int = 0):
-    if mmin:
-        r = CassandraNodes.exec(pod, namespace, f'find {pattern} -mmin -{mmin}', show_out=Config().is_debug(), shell='bash')
+def find_files(pod: str, namespace: str, pattern: str, mmin: int = 0, remote = False):
+    stdout = ''
+    if not remote:
+        # find . -maxdepth 1 -type f -name '*'
+        dir = os.path.dirname(pattern)
+        base = os.path.basename(pattern)
+        cmd = ['find', dir, '-name', base]
+        if mmin:
+                cmd += ['-mmin',  f'-{mmin}']
+
+        stdout = local_exec(cmd, show_out=Config().is_debug()).stdout
     else:
-        r = CassandraNodes.exec(pod, namespace, f'find {pattern}', show_out=Config().is_debug(), shell='bash')
+        cmd = f'find {pattern}'
+        if mmin:
+                cmd = f'{cmd} -mmin -{mmin}'
+
+        stdout = CassandraNodes.exec(pod, namespace, cmd, show_out=Config().is_debug(), shell='bash').stdout
 
     log_files = []
-    for line in r.stdout.split('\n'):
+    for line in stdout.split('\n'):
         line = line.strip(' \r')
         if line:
             log_files.append(line)
@@ -325,6 +336,7 @@ class PodPushHandler:
         state = self.state
 
         if not state.pod:
+            self.pushed = True
             state.push()
 
             if not self.pod:
@@ -340,4 +352,15 @@ class PodPushHandler:
         return False
 
 def state_with_pod(state: ReplState, pod: str = None):
-    return PodPushHandler(state, pod=pod)
+    return PodPushHandler(state, pod=pod)
+
+def os_system_exec(cmd: str, show_out = False):
+    if show_out: log2(cmd)
+
+    os.system(cmd)
+
+def csv_dir():
+    return Config().get('export.csv_dir', '/c3/cassandra/tmp')
+
+def export_log_dir():
+    return creating_dir(Config().get('export.log-dir', '/tmp/qing-db/q/export/logs'))

adam/commands/{cat.py → fs/cat.py}

@@ -27,10 +27,10 @@ class Cat(Command):
 
         with self.validate(args, state) as (args, state):
             with validate_args(args, state, name='file'):
-                return Devices.device(state).bash(state, state, cmd.split(' '))
+                return Devices.of(state).bash(state, state, cmd.split(' '))
 
     def completion(self, state: ReplState):
-        return super().completion(state, pods=Devices.device(state).pods(state))
+        return super().completion(state, lambda: {f: None for f in Devices.of(state).files(state)}, pods=Devices.of(state).pods(state, '-'), auto='jit')
 
     def help(self, _: ReplState):
         return f'{Cat.COMMAND} file [&]\t run cat command on the pod'

adam/commands/fs/cat_local.py

@@ -0,0 +1,42 @@
+import os
+
+from adam.commands import validate_args
+from adam.commands.command import Command
+from adam.repl_state import ReplState, RequiredState
+from adam.utils import log2
+from adam.utils_local import find_local_files
+
+class CatLocal(Command):
+    COMMAND = ':cat'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(CatLocal, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return CatLocal.COMMAND
+
+    def required(self):
+        return [RequiredState.CLUSTER_OR_POD, RequiredState.APP_APP, ReplState.P]
+
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        with self.validate(args, state) as (args, state):
+            with validate_args(args, state, name='file') as args:
+                os.system(f'cat {args}')
+                log2()
+
+                return state
+
+    def completion(self, state: ReplState):
+        return super().completion(state, lambda: {n: None for n in find_local_files(file_type='f', max_depth=1)}, auto='jit')
+
+    def help(self, _: ReplState):
+        return f'{CatLocal.COMMAND} file\t run cat command on local system'

adam/commands/{cd.py → fs/cd.py}

@@ -28,14 +28,14 @@ class Cd(Command):
 
         with self.validate(args, state, apply=False) as (args, state):
             with validate_args(args, state, name='directory') as arg_str:
-                device: Device = Devices.device(state)
+                device: Device = Devices.of(state)
                 for dir in arg_str.split('/'):
                     device.cd(dir, state)
 
                 return state
 
     def completion(self, state: ReplState):
-        return Devices.device(state).cd_completion(Cd.COMMAND, state, default = {})
+        return Devices.of(state).cd_completion(Cd.COMMAND, state, default = {})
 
     def help(self, _: ReplState):
         return f'{Cd.COMMAND} <path> | .. \t move around on the operational device hierarchy'

adam/commands/{download_file.py → fs/download_file.py}

@@ -1,6 +1,8 @@
 from adam.commands import validate_args
 from adam.commands.command import Command
 from adam.commands.devices.devices import Devices
+from adam.config import Config
+from adam.utils_k8s.pod_exec_result import PodExecResult
 from adam.repl_state import ReplState, RequiredState
 from adam.utils import log2
 from adam.utils_k8s.pods import Pods
@@ -29,8 +31,8 @@ class DownloadFile(Command):
 
         with self.validate(args, state) as (args, state):
             with validate_args(args, state, name='file'):
-                to_file = Pods.download_file(state.pod,
-                                             Devices.device(state).default_container(state),
+                to_file = Pods.download_file(Devices.of(state).pod(state),
+                                             Devices.of(state).default_container(state),
                                              state.namespace,
                                              args[0],
                                              args[1] if len(args) > 1 else None)
@@ -39,9 +41,7 @@ class DownloadFile(Command):
                 return state
 
     def completion(self, state: ReplState):
-        return super().completion(state, pods=Devices.device(state).pods(state))
-
-        return {}
+        return super().completion(state, lambda: {f: None for f in Devices.of(state).files(state)}, pods=Devices.of(state).pods(state, '-'), auto='jit')
 
     def help(self, _: ReplState):
         return f'{DownloadFile.COMMAND} from-file [to-file]\t download file from pod'

adam/commands/{find_files.py → fs/find_files.py}

@@ -3,10 +3,10 @@ import os
 from adam.commands.command import Command
 from adam.repl_state import ReplState
 from adam.utils import log2
-from adam.utils_local import local_tmp_dir
+from adam.utils_local import local_qing_dir
 
 class FindLocalFiles(Command):
-    COMMAND = 'find local'
+    COMMAND = ':find file'
 
     # the singleton pattern
     def __new__(cls, *args, **kwargs):
@@ -28,9 +28,9 @@ class FindLocalFiles(Command):
             cmd = 'find'
 
             if not args:
-                cmd = f'find {local_tmp_dir()}'
+                cmd = f'find {local_qing_dir()}'
             elif len(args) == 1:
-                cmd = f"find {local_tmp_dir()} -name '{args[0]}'"
+                cmd = f"find {local_qing_dir()} -name '{args[0]}'"
             else:
                 new_args = [f"'{arg}'" if '*' in arg else arg for arg in args]
                 cmd = 'find ' + ' '.join(new_args)

adam/commands/{find_processes.py → fs/find_processes.py}

@@ -32,7 +32,7 @@ class FindProcesses(Command):
                 with validate_args(args, state, name='words to look for'):
                     arg = ' | '.join([f'grep {a}' for a in args])
                     awk = "awk '{ print $1, $2, $8, $NF }'"
-                    rs = Devices.device(state).bash(state, state, f"ps -ef | grep -v grep | {arg} | {awk}".split(' '))
+                    rs = Devices.of(state).bash(state, state, f"ps -ef | grep -v grep | {arg} | {awk}".split(' '))
 
                     lines: list[list[str]] = []
                     for r in rs:
@@ -62,10 +62,10 @@ class FindProcesses(Command):
 
                             pod = pid_n_pod[1]
 
-                            log2(f'@{pod} kill -9 {pid}')
+                            log2(f'@{pod} bash kill -9 {pid}')
 
                             with state_with_pod(state, pod) as state1:
-                                Devices.device(state).bash(state, state1, ['kill', '-9', pid])
+                                Devices.of(state).bash(state, state1, ['kill', '-9', pid])
 
                     return rs
 

adam/commands/{head.py → fs/head.py}

@@ -27,10 +27,10 @@ class Head(Command):
 
         with self.validate(args, state) as (args, state):
             with validate_args(args, state, name='file'):
-                return Devices.device(state).bash(state, state, cmd.split(' '))
+                return Devices.of(state).bash(state, state, cmd.split(' '))
 
     def completion(self, state: ReplState):
-        return super().completion(state, pods=Devices.device(state).pods(state))
+        return super().completion(state, lambda: {f: None for f in Devices.of(state).files(state)}, pods=Devices.of(state).pods(state, '-'), auto='jit')
 
     def help(self, _: ReplState):
         return f'{Head.COMMAND} file [&]\t run head command on the pod'

adam/commands/{ls.py → fs/ls.py}

@@ -30,12 +30,12 @@ class Ls(Command):
                     state = copy.copy(state)
                     state.device = arg.replace(':', '')
 
-            Devices.device(state).ls(cmd, state)
+            Devices.of(state).ls(cmd, state)
 
             return state
 
     def completion(self, state: ReplState):
-        return Devices.device(state).ls_completion(Ls.COMMAND, state, default = super().completion(state))
+        return super().completion(state, {'&': None}, pods=Devices.of(state).pods(state, '-'))
 
     def help(self, _: ReplState):
         return f'{Ls.COMMAND} [device:]\t list apps, envs, clusters, nodes, pg hosts/databases or export databases'

adam/commands/fs/ls_local.py

@@ -0,0 +1,40 @@
+import os
+
+from adam.commands.command import Command
+from adam.repl_state import ReplState
+from adam.utils import log2
+from adam.utils_local import local_qing_dir
+
+class LsLocal(Command):
+    COMMAND = ':ls'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(LsLocal, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return LsLocal.COMMAND
+
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        with self.validate(args, state) as (args, state):
+            if args:
+                os.system(f'ls {args}')
+            else:
+                os.system(f'ls {local_qing_dir()}')
+            log2()
+
+            return state
+
+    def completion(self, state: ReplState):
+        return super().completion(state)
+
+    def help(self, _: ReplState):
+        return f'{LsLocal.COMMAND} [dir]\t list files on local system'

adam/commands/fs/rm.py

@@ -0,0 +1,18 @@
+from adam.commands.fs.rm_downloads import RmDownloads
+from adam.commands.fs.rm_logs import RmLogs
+from adam.commands.intermediate_command import IntermediateCommand
+
+class RmLocal(IntermediateCommand):
+    COMMAND = ':rm'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(RmLocal, cls).__new__(cls)
+
+        return cls.instance
+
+    def command(self):
+        return RmLocal.COMMAND
+
+    def cmd_list(self):
+        return [RmDownloads(), RmLogs()]

adam/commands/fs/rm_downloads.py

@@ -0,0 +1,39 @@
+import os
+
+from adam.commands.command import Command
+from adam.repl_state import ReplState
+from adam.utils import log2
+from adam.utils_local import local_downloads_dir
+
+class RmDownloads(Command):
+    COMMAND = ':rm downloads'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(RmDownloads, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return RmDownloads.COMMAND
+
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        with self.validate(args, state) as (args, state):
+            cmd = f'rm -rf {local_downloads_dir()}/*'
+            log2(cmd)
+            os.system(cmd)
+            log2()
+
+            return state
+
+    def completion(self, state: ReplState):
+        return super().completion(state)
+
+    def help(self, _: ReplState):
+        return f'{RmDownloads.COMMAND}\t remove all downloads files under {local_downloads_dir()}'

adam/commands/fs/rm_logs.py

@@ -0,0 +1,38 @@
+import os
+
+from adam.commands.command import Command
+from adam.repl_state import ReplState
+from adam.utils import log2, log_dir
+
+class RmLogs(Command):
+    COMMAND = ':rm logs'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(RmLogs, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return RmLogs.COMMAND
+
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        with self.validate(args, state) as (args, state):
+            cmd = f'rm -rf {log_dir()}/*'
+            log2(cmd)
+            os.system(cmd)
+            log2()
+
+            return state
+
+    def completion(self, state: ReplState):
+        return super().completion(state)
+
+    def help(self, _: ReplState):
+        return f'{RmLogs.COMMAND}\t remove all qing log files under {log_dir()}'

adam/commands/{show → fs}/show_adam.py

@@ -1,7 +1,7 @@
 import sys
 import os
 
-from adam.utils import tabulize, log2
+from adam.utils import tabulize
 
 current_dir = os.path.dirname(os.path.abspath(__file__))
 

adam/commands/intermediate_command.py

@@ -12,6 +12,9 @@ class IntermediateCommand(Command):
 
         return self.intermediate_run(cmd, state, args, self.cmd_list())
 
+    def completion(self, state: ReplState):
+        return {}
+
     @abstractmethod
     def cmd_list(self):
         pass

adam/commands/medusa/medusa_restore.py

@@ -1,13 +1,12 @@
-from collections.abc import Callable
 from datetime import datetime
 from functools import partial
 
 from adam.commands import validate_args
 from adam.commands.command import Command, InvalidArgumentsException
+from adam.commands.medusa.utils_medusa import medusa_backup_names
 from adam.utils_k8s.statefulsets import StatefulSets
 from adam.repl_state import ReplState, RequiredState
 from adam.utils_k8s.custom_resources import CustomResources
-from adam.config import Config
 from adam.utils import tabulize, log2, log_exc
 
 class MedusaRestore(Command):
@@ -67,20 +66,7 @@ class MedusaRestore(Command):
                 return state
 
     def completion(self, state: ReplState):
-        if sc := super().completion(state):
-            ns = state.namespace
-            dc: str = StatefulSets.get_datacenter(state.sts, ns)
-            if not dc:
-                return {}
-
-            if Config().get('medusa.restore-auto-complete', False):
-                leaf = {id: None for id in [f"{x['metadata']['name']}" for x in CustomResources.medusa_show_backupjobs(dc, ns)]}
-
-                return super().completion(state, leaf)
-            else:
-                return sc
-
-        return {}
+        return super().completion(state, lambda: {id: None for id in medusa_backup_names(state)}, auto_key='medusa.backups')
 
     def help(self, _: ReplState):
         return f'{MedusaRestore.COMMAND}\t start a restore job'

adam/commands/medusa/utils_medusa.py

@@ -0,0 +1,15 @@
+from adam.config import Config
+from adam.repl_state import ReplState
+from adam.utils_k8s.custom_resources import CustomResources
+from adam.utils_k8s.statefulsets import StatefulSets
+
+def medusa_backup_names(state: ReplState, warm=False):
+    if warm and (auto := Config().get('medusa.restore-auto-complete', 'off')) in ['off', 'jit', 'lazy']:
+        return {}
+
+    ns = state.namespace
+    dc: str = StatefulSets.get_datacenter(state.sts, ns)
+    if not dc:
+        return {}
+
+    return {id: None for id in [f"{x['metadata']['name']}" for x in CustomResources.medusa_show_backupjobs(dc, ns)]}

adam/commands/{nodetool.py → nodetool/nodetool.py}

@@ -3,11 +3,11 @@ import click
 from adam.commands.command import Command
 from adam.commands.command_helpers import ClusterOrPodCommandHelper
 from adam.commands.cql.utils_cql import cassandra
-from adam.commands.nodetool_commands import NODETOOL_COMMANDS
+from adam.commands.devices.devices import Devices
+from adam.commands.nodetool.nodetool_commands import NODETOOL_COMMANDS
 from adam.config import Config
 from adam.repl_state import ReplState, RequiredState
 from adam.utils import log
-from adam.utils_k8s.statefulsets import StatefulSets
 
 class NodeTool(Command):
     COMMAND = 'nodetool'
@@ -38,12 +38,7 @@ class NodeTool(Command):
             return state
 
     def completion(self, state: ReplState):
-        if super().completion(state):
-            d = {c: {'&': None} for c in NODETOOL_COMMANDS}
-            return {NodeTool.COMMAND: {'help': None} | d} | \
-                   {f'@{p}': {NodeTool.COMMAND: d} for p in StatefulSets.pod_names(state.sts, state.namespace)}
-
-        return {}
+        return super().completion(state, {c: {'&': None} for c in NODETOOL_COMMANDS}, pods=Devices.of(state).pods(state, '-'))
 
     def help(self, _: ReplState):
         return f'{NodeTool.COMMAND} <sub-command> [&]\t run nodetool with arguments'

adam/commands/postgres/completions_p.py

@@ -0,0 +1,22 @@
+from adam.commands.postgres.postgres_databases import PostgresDatabases
+from adam.commands.postgres.utils_postgres import pg_table_names
+from adam.repl_state import ReplState
+from adam.sql.lark_completer import LarkCompleter
+
+def completions_p(state: ReplState):
+    return {
+        '\h': None,
+        '\d': None,
+        '\dt': None,
+        '\du': None
+    } | LarkCompleter(expandables={
+        'tables': lambda x: pg_table_names(state),
+        'columns': ['id'],
+        'hosts': ['@' + PostgresDatabases.pod_and_container(state.namespace)[0]],
+    }, variant=ReplState.P).completions_for_nesting()
+
+def psql0_completions(state: ReplState):
+    return {
+        '\h': None,
+        '\l': None,
+    }

adam/commands/postgres/postgres.py

@@ -4,12 +4,12 @@ from adam.commands import extract_trailing_options, validate_args
 from adam.commands.command import Command
 from adam.commands.intermediate_command import IntermediateCommand
 from adam.commands.postgres.postgres_databases import pg_path
-from adam.commands.postgres.psql_completions import psql_completions
+from adam.commands.postgres.completions_p import psql0_completions, completions_p
 from adam.commands.postgres.utils_postgres import pg_table_names, postgres
 from .postgres_ls import PostgresLs
 from .postgres_preview import PostgresPreview
 from adam.repl_state import ReplState
-from adam.utils import log, log2
+from adam.utils import log, log2, log_timing
 
 class Postgres(IntermediateCommand):
     COMMAND = 'pg'
@@ -55,24 +55,17 @@ class Postgres(IntermediateCommand):
 
     def completion(self, state: ReplState):
         if state.device != state.P:
-            # conflicts with cql completions
             return {}
 
-        leaf = {}
         with pg_path(state) as (host, database):
             if database:
                 if pg_table_names(state):
-                    leaf = psql_completions(state)
+                    with log_timing('psql_completions'):
+                        return completions_p(state)
             elif host:
-                leaf = {
-                    '\h': None,
-                    '\l': None,
-                }
-
-        if state.pg_path:
-            return super().completion(state, leaf) | leaf
-        else:
-            return {}
+                return psql0_completions(state)
+
+        return {}
 
     def help(self, _: ReplState):
         return f'<sql-statements> [&]\t run queries on Postgres databases'

adam/commands/postgres/postgres_databases.py

@@ -144,12 +144,12 @@ class PostgresDatabases:
             env_prefix = f'PGPASSWORD="{password}"'
 
             r = Pods.exec(pod_name, container_name, state.namespace, cmd, show_out=show_out, backgrounded=backgrounded, env_prefix=env_prefix)
-            if r and Config().get('repl.history.push-cat-remote-log-file', True):
-                if r.log_file and ReplSession().prompt_session:
-                    ReplSession().prompt_session.history.append_string(f'@{r.pod} cat {r.log_file}')
+            if r and r.log_file:
+                ReplSession().append_history(f':cat {r.log_file}')
 
             return r
 
+    @functools.lru_cache()
     def pod_and_container(namespace: str):
         container_name = Config().get('pg.agent.name', 'ops-pg-agent')
 

adam/commands/postgres/postgres_ls.py

@@ -1,5 +1,5 @@
 from adam.commands.command import Command
-from adam.commands.ls import Ls
+from adam.commands.fs.ls import Ls
 from adam.repl_state import ReplState, RequiredState
 
 class PostgresLs(Command):