kaqing 2.0.188__py3-none-any.whl → 2.0.211__py3-none-any.whl

adam/commands/export/exporter.py

@@ -1,5 +1,7 @@
 from datetime import datetime
+import os
 import time
+import traceback
 
 from adam.commands.command import InvalidArgumentsException
 from adam.commands.cql.utils_cql import cassandra_table_names, run_cql, table_spec
@@ -8,13 +10,12 @@ from adam.commands.export.export_sessions import ExportSessions
 from adam.commands.export.importer import Importer
 from adam.commands.export.importer_athena import AthenaImporter
 from adam.commands.export.importer_sqlite import SqliteImporter
-from adam.commands.export.utils_export import ExportSpec, ExportTableStatus, ExportTableSpec, ImportSpec, csv_dir, find_files, state_with_pod
+from adam.commands.export.utils_export import ExportSpec, ExportTableStatus, ExportTableSpec, ImportSpec, csv_dir, export_log_dir, find_files, os_system_exec, state_with_pod
 from adam.config import Config
-from adam.pod_exec_result import PodExecResult
+from adam.repl_session import ReplSession
 from adam.repl_state import ReplState
-from adam.utils import debug, log, parallelize, log2, ing, log_exc
+from adam.utils import debug, kaqing_log_file_name, log, offload, parallelize, log2, ing, log_exc
 from adam.utils_k8s.cassandra_nodes import CassandraNodes
-from adam.utils_k8s.pods import log_prefix
 
 class Exporter:
     def export_tables(args: list[str], state: ReplState, export_only: bool = False, max_workers = 0) -> tuple[list[str], ExportSpec]:
@@ -133,11 +134,9 @@ class Exporter:
 
             prefix = Importer.prefix_from_importer(spec.importer)
             if spec.session:
-                spec.session = f'{prefix}{spec.session[1:]}'
+                state.export_session = f'{prefix}{spec.session[1:]}'
             else:
-                spec.session = f'{prefix}{datetime.now().strftime("%Y%m%d%H%M%S")[3:]}'
-
-            state.export_session = spec.session
+                state.export_session = f'{prefix}{datetime.now().strftime("%Y%m%d%H%M%S")[3:]}'
 
         return spec
 
@@ -154,89 +153,160 @@ class Exporter:
         if export_state == 'init':
             CassandraNodes.exec(state.pod, state.namespace, f'rm -rf {csv_dir()}/{spec.session}_*', show_out=Config().is_debug(), shell='bash')
 
-        action = f'[{spec.session}] Exporting|Exported'
+        job_log = kaqing_log_file_name()
+
+        action = f'[{spec.session}] Triggering export of'
         if export_state == 'init':
             action = f'[{spec.session}] Preparing|Prepared'
         elif export_state == 'import':
             action = f'[{spec.session}] Importing|Imported'
 
-        with parallelize(spec.tables, max_workers, msg=action + ' {size} Cassandra tables') as exec:
-            return exec.map(lambda table: Exporter.export_table(table, state, spec.session, spec.importer, export_only, len(spec.tables) > 1, consistency=spec.consistency, export_state=export_state)), spec
-
-    def export_table(spec: ExportTableSpec, state: ReplState, session: str, importer: str, export_only = False, multi_tables = True, consistency: str = None, export_state=None):
+        msg = action + ' {size} Cassandra tables'
+
+        if export_state != 'init':
+            log2(f'[{spec.session}] Logging to {job_log}...')
+            ReplSession().append_history(f':cat {job_log}')
+
+        pod = state.pod
+        with parallelize(spec.tables, max_workers, msg=msg, collect=export_state == 'init', name='exporter') as exec:
+            return exec.map(lambda table: Exporter.export_table(table,
+                                                                state.with_pod(pod),
+                                                                spec.session,
+                                                                spec.importer,
+                                                                export_only,
+                                                                len(spec.tables) > 1,
+                                                                consistency=spec.consistency,
+                                                                export_state=export_state,
+                                                                job_log=None if export_state == 'init' else job_log)), spec
+
+    def export_table(spec: ExportTableSpec,
+                     state: ReplState,
+                     session: str,
+                     importer: str,
+                     export_only = False,
+                     multi_tables = True,
+                     consistency: str = None,
+                     export_state=None,
+                     job_log: str = None):
         s: str = None
 
         table, target_table, columns = Exporter.resove_table_n_columns(spec, state, include_ks_in_target=False, importer=importer)
 
-        log_file = f'{log_prefix()}-{session}_{spec.keyspace}.{target_table}.log'
+        log_file = f'{export_log_dir()}/{session}_{spec.keyspace}.{target_table}.log'
         create_db = not state.export_session
 
         if export_state == 'init':
             Exporter.create_table_log(spec, state, session, table, target_table)
             return 'table_log_created'
         else:
-            if export_state == 'pending_export':
-                Exporter.export_to_csv(spec, state, session, table, target_table, columns, multi_tables=multi_tables, consistency=consistency)
+            try:
+                if export_state == 'pending_export':
+                    Exporter.export_to_csv(spec, state, session, table, target_table, columns, multi_tables=multi_tables, consistency=consistency, job_log=job_log)
 
-            log_files: list[str] = find_files(state.pod, state.namespace, f'{log_file}*')
-            if not log_files:
-                return s
+                log_files: list[str] = find_files(state.pod, state.namespace, f'{log_file}*')
+                if not log_files:
+                    return s
 
-            log_file = log_files[0]
+                log_file = log_files[0]
 
-            status: ExportTableStatus = ExportTableStatus.from_log_file(state.pod, state.namespace, session, log_file)
-            while status.status != 'done':
-                if status.status == 'export_in_pregress':
-                    debug('Exporting to CSV is still in progess, sleeping for 1 sec...')
-                    time.sleep(1)
-                elif status.status == 'exported':
-                    log_file = Exporter.rename_to_pending_import(spec, state, session, target_table)
-                    if importer == 'csv' or export_only:
-                        return 'pending_import'
-                elif status.status == 'pending_import':
-                    log_file, session = Exporter.import_from_csv(spec, state, session, importer, table, target_table, columns, multi_tables=multi_tables, create_db=create_db)
+                status: ExportTableStatus = ExportTableStatus.from_log_file(state.pod, state.namespace, session, log_file)
 
-                status = ExportTableStatus.from_log_file(state.pod, state.namespace, session, log_file)
+                with offload(name='exporter') as exec:
+                    ctx = ExportTableContext(spec, state, session, importer, export_only, multi_tables, table, target_table, columns, create_db, log_file, status, job_log)
+                    exec.submit(lambda: Exporter.export_loop(ctx))
+                # Exporter.export_loop(ExportTableContext(spec, state, session, importer, export_only, multi_tables, table, target_table, columns, create_db, log_file, status))
+            except:
+                traceback.print_exc()
 
             return status.status
 
+    def export_loop(ctx: 'ExportTableContext'):
+        try:
+            while ctx.status.status != 'done':
+                if ctx.status.status == 'export_in_pregress':
+                    debug('Exporting to CSV is still in progess, sleeping for 1 sec...')
+                    time.sleep(1)
+                elif ctx.status.status == 'exported':
+                    ctx.log_file = Exporter.rename_to_pending_import(ctx.spec, ctx.state, ctx.session, ctx.target_table)
+                    ExportSessions.clear_export_session_cache()
+                    if ctx.importer == 'csv' or ctx.export_only:
+                        return 'pending_import'
+                elif ctx.status.status == 'pending_import':
+                    ctx.log_file, ctx.session = Exporter.import_from_csv(ctx.spec,
+                                                                         ctx.state,
+                                                                         ctx.session,
+                                                                         ctx.importer,
+                                                                         ctx.table,
+                                                                         ctx.target_table,
+                                                                         ctx.columns,
+                                                                         multi_tables=ctx.multi_tables,
+                                                                         create_db=ctx.create_db,
+                                                                         job_log=ctx.f)
+
+                ctx.status = ExportTableStatus.from_log_file(ctx.state.pod, ctx.state.namespace, ctx.session, ctx.log_file)
+
+            return ctx.status.status
+        except:
+            traceback.print_exc()
+
     def create_table_log(spec: ExportTableSpec, state: ReplState, session: str, table: str, target_table: str):
-        log_file = f'{log_prefix()}-{session}_{spec.keyspace}.{target_table}.log'
+        log_file = f'{export_log_dir()}/{session}_{spec.keyspace}.{target_table}.log'
+        dir = os.path.dirname(log_file)
 
-        CassandraNodes.exec(state.pod, state.namespace, f'rm -f {log_file}* && touch {log_file}', show_out=Config().is_debug(), shell='bash')
+        cmd = f'rm -f {log_file}* && mkdir -p {dir} && touch {log_file}'
+        os_system_exec(cmd, show_out=Config().is_debug())
 
         return table
 
-    def export_to_csv(spec: ExportTableSpec, state: ReplState, session: str, table: str, target_table: str, columns: str, multi_tables = True, consistency: str = None):
+    def export_to_csv(spec: ExportTableSpec,
+                      state: ReplState,
+                      session: str,
+                      table: str,
+                      target_table: str,
+                      columns: str,
+                      multi_tables = True,
+                      consistency: str = None,
+                      job_log: str = None):
         db = f'{session}_{target_table}'
 
         CassandraNodes.exec(state.pod, state.namespace, f'mkdir -p {csv_dir()}/{db}', show_out=Config().is_debug(), shell='bash')
         csv_file = f'{csv_dir()}/{db}/{table}.csv'
-        log_file = f'{log_prefix()}-{session}_{spec.keyspace}.{target_table}.log'
+        table_log_file = f'{export_log_dir()}/{session}_{spec.keyspace}.{target_table}.log'
 
         suppress_ing_log = Config().is_debug() or multi_tables
         queries = []
         if consistency:
             queries.append(f'CONSISTENCY {consistency}')
         queries.append(f"COPY {spec.keyspace}.{table}({columns}) TO '{csv_file}' WITH HEADER = TRUE")
-        r: PodExecResult = ing(
-            f'[{session}] Dumping table {spec.keyspace}.{table}{f" with consistency {consistency}" if consistency else ""}',
-            lambda: run_cql(state, ';'.join(queries), show_out=Config().is_debug(), backgrounded=True, log_file=log_file),
-            suppress_log=suppress_ing_log)
 
-        return log_file
+        with ing(f'[{session}] Triggering dump of table {spec.keyspace}.{table}{f" with consistency {consistency}" if consistency else ""}',
+                 suppress_log=suppress_ing_log,
+                 job_log = job_log):
+            run_cql(state, ';'.join(queries), show_out=Config().is_debug(), backgrounded=True, log_file=table_log_file, history=False)
+
+        return table_log_file
 
     def rename_to_pending_import(spec: ExportTableSpec, state: ReplState, session: str, target_table: str):
-        log_file = f'{log_prefix()}-{session}_{spec.keyspace}.{target_table}.log'
+        log_file = f'{export_log_dir()}/{session}_{spec.keyspace}.{target_table}.log'
         to = f'{log_file}.pending_import'
 
-        CassandraNodes.exec(state.pod, state.namespace, f'mv {log_file} {to}', show_out=Config().is_debug(), shell='bash')
+        cmd =f'mv {log_file} {to}'
+        os_system_exec(cmd, show_out=Config().is_debug())
 
         return to
 
-    def import_from_csv(spec: ExportTableSpec, state: ReplState, session: str, importer: str, table: str, target_table: str, columns: str, multi_tables = True, create_db = False):
+    def import_from_csv(spec: ExportTableSpec,
+                        state: ReplState,
+                        session: str,
+                        importer: str,
+                        table: str,
+                        target_table: str,
+                        columns: str,
+                        multi_tables = True,
+                        create_db = False,
+                        job_log: str = None):
         im = AthenaImporter() if importer == 'athena' else SqliteImporter()
-        return im.import_from_csv(state, session if session else state.export_session, spec.keyspace, table, target_table, columns, multi_tables, create_db)
+        return im.import_from_csv(state, session if session else state.export_session, spec.keyspace, table, target_table, columns, multi_tables, create_db, job_log=job_log)
 
     def resove_table_n_columns(spec: ExportTableSpec, state: ReplState, include_ks_in_target = False, importer = 'sqlite'):
         table = spec.table
@@ -262,6 +332,22 @@ class Exporter:
 
         return table, target_table, columns
 
+class ExportTableContext:
+    def __init__(self, spec: ExportTableSpec, state: ReplState, session: str, importer: str, export_only = False, multi_tables = True, table: str = None, target_table: str = None, columns: str = None, create_db = False, log_file: str = None, status: ExportTableStatus = None, f: str = None):
+        self.spec = spec
+        self.state = state
+        self.session = session
+        self.importer = importer
+        self.export_only = export_only
+        self.multi_tables = multi_tables
+        self.table = table
+        self.target_table = target_table
+        self.columns = columns
+        self.create_db = create_db
+        self.log_file = log_file
+        self.status = status
+        self.f = f
+
 class ExportService:
     def __init__(self, handler: 'ExporterHandler'):
         self.handler = handler
@@ -279,16 +365,17 @@ class ExportService:
 
                 ExportSessions.clear_export_session_cache()
 
-                if spec.importer == 'csv' or export_only:
-                    ExportSessions.show_session(state.sts, state.pod, state.namespace, spec.session)
-                else:
-                    log()
-                    with export_db(state) as dbs:
-                        dbs.show_database()
+                # if spec.importer == 'csv' or export_only:
+                #     ExportSessions.show_session(state.sts, state.pod, state.namespace, spec.session)
+                # else:
+                #     log()
+                #     with export_db(state) as dbs:
+                #         dbs.show_database()
         finally:
+            pass
             # if exporting to csv, do not bind the new session id to repl state
-            if spec and spec.importer == 'csv':
-                state.export_session = export_session
+            # if spec and spec.importer == 'csv':
+            #     state.export_session = export_session
 
         return state
 

adam/commands/export/import_session.py

@@ -34,10 +34,6 @@ class ImportSession(Command):
                         return exporter.import_session(spec)
 
     def completion(self, state: ReplState):
-        # warm up cache
-        # ExportSessions.export_session_names(state.sts, state.pod, state.namespace)
-        # ExportSessions.export_session_names(state.sts, state.pod, state.namespace, export_state='pending_import')
-
         return {}
 
     def help(self, _: ReplState):

adam/commands/export/importer.py

@@ -1,11 +1,9 @@
 from abc import abstractmethod
 
-from adam.commands.export.utils_export import csv_dir
+from adam.commands.export.utils_export import csv_dir, export_log_dir, os_system_exec
 from adam.config import Config
 from adam.repl_state import ReplState
 from adam.utils import ing
-from adam.utils_k8s.cassandra_nodes import CassandraNodes
-from adam.utils_k8s.pods import log_prefix
 
 class Importer:
     @abstractmethod
@@ -13,7 +11,7 @@ class Importer:
         pass
 
     @abstractmethod
-    def import_from_csv(self, state: ReplState, from_session: str, keyspace: str, table: str, target_table: str, columns: str, multi_tables = True, create_db = False):
+    def import_from_csv(self, state: ReplState, from_session: str, keyspace: str, table: str, target_table: str, columns: str, multi_tables = True, create_db = False, log_file: str = None):
         pass
 
     @abstractmethod
@@ -27,11 +25,12 @@ class Importer:
         pod = state.pod
         namespace = state.namespace
         to_session = state.export_session
-        log_file = f'{log_prefix()}-{from_session}_{keyspace}.{target_table}.log.pending_import'
+        log_file = f'{export_log_dir()}/{from_session}_{keyspace}.{target_table}.log.pending_import'
 
-        to = f'{log_prefix()}-{to_session}_{keyspace}.{target_table}.log.done'
+        to = f'{export_log_dir()}/{to_session}_{keyspace}.{target_table}.log.done'
 
-        CassandraNodes.exec(pod, namespace, f'mv {log_file} {to}', show_out=Config().is_debug(), shell='bash')
+        cmd = f'mv {log_file} {to}'
+        os_system_exec(cmd, show_out=Config().is_debug())
 
         return to, to_session
 
@@ -41,12 +40,13 @@ class Importer:
 
         return session
 
-    def remove_csv(self, state: ReplState, from_session: str, table: str, target_table: str, multi_tables = True):
+    def remove_csv(self, state: ReplState, from_session: str, table: str, target_table: str, multi_tables = True, job_log: str = None):
         pod = state.pod
         namespace = state.namespace
 
-        with ing(f'[{from_session}] Cleaning up temporary files', suppress_log=multi_tables):
-            CassandraNodes.exec(pod, namespace, f'rm -rf {self.csv_file(from_session, table, target_table)}', show_out=Config().is_debug(), shell='bash')
+        with ing(f'[{from_session}] Cleaning up temporary files', suppress_log=multi_tables, job_log=job_log):
+            cmd = f'rm -rf {self.csv_file(from_session, table, target_table)}'
+            os_system_exec(cmd, show_out=Config().is_debug())
 
     def db(self, session: str, keyspace: str):
         return f'{session}_{keyspace}'
@@ -78,4 +78,4 @@ class Importer:
         elif session.startswith('e'):
             importer = 'athena'
 
-        return importer
+        return importer

adam/commands/export/importer_athena.py

@@ -19,9 +19,16 @@ class AthenaImporter(Importer):
     def prefix(self):
         return 'e'
 
-    def import_from_csv(self, state: ReplState, from_session: str,
-                        keyspace: str, table: str, target_table: str, columns: str,
-                        multi_tables = True, create_db = False):
+    def import_from_csv(self,
+                        state: ReplState,
+                        from_session: str,
+                        keyspace: str,
+                        table: str,
+                        target_table: str,
+                        columns: str,
+                        multi_tables = True,
+                        create_db = False,
+                        job_log: str = None):
         csv_file = self.csv_file(from_session, table, target_table)
         pod = state.pod
         namespace = state.namespace
@@ -47,12 +54,14 @@ class AthenaImporter(Importer):
             return to, to_session
         finally:
             if succeeded:
-                self.remove_csv(state, from_session, table, target_table, multi_tables)
+                self.remove_csv(state, from_session, table, target_table, multi_tables, job_log=job_log)
                 Athena.clear_cache()
 
-                if not multi_tables:
+                if multi_tables:
+                    log2(f'[{to_session}] {keyspace}.{target_table} OK', file=job_log)
+                else:
                     with export_db(state) as dbs:
-                        dbs.sql(f'select * from {database}.{target_table} limit 10')
+                        dbs.sql(f'select * from {keyspace}.{target_table} limit 10', backgrounded=True, export_log=job_log)
 
     def import_from_local_csv(self, state: ReplState,
                         keyspace: str, table: str, csv_file: str, multi_tables = True, create_db = False):

adam/commands/export/importer_sqlite.py

@@ -1,9 +1,10 @@
+from typing import TextIO
 import pandas
 
 from adam.commands.export.export_databases import export_db
 from adam.commands.export.importer import Importer
 from adam.repl_state import ReplState
-from adam.utils import GeneratorStream, bytes_generator_from_file, ing
+from adam.utils import GeneratorStream, bytes_generator_from_file, ing, log2
 from adam.utils_k8s.pods import Pods
 from adam.utils_sqlite import SQLite, sqlite
 
@@ -11,9 +12,17 @@ class SqliteImporter(Importer):
     def prefix(self):
         return 's'
 
-    def import_from_csv(self, state: ReplState, from_session: str,
-                        keyspace: str, table: str, target_table: str, columns: str,
-                        multi_tables = True, create_db = False):
+    def import_from_csv(self,
+                        state: ReplState,
+                        from_session: str,
+                        keyspace: str,
+                        table: str,
+                        target_table: str,
+                        columns: str,
+                        multi_tables = True,
+                        create_db = False,
+                        job_log: str = None):
+
         csv_file = self.csv_file(from_session, table, target_table)
         pod = state.pod
         namespace = state.namespace
@@ -21,7 +30,7 @@ class SqliteImporter(Importer):
 
         succeeded = False
         try:
-            with ing(f'[{to_session}] Uploading to Sqlite', suppress_log=multi_tables):
+            with ing(f'[{to_session}] Uploading to Sqlite', suppress_log=multi_tables, job_log=job_log):
                 # create a connection to single keyspace
                 with sqlite(to_session, keyspace) as conn:
                     bytes = Pods.read_file(pod, 'cassandra', namespace, csv_file)
@@ -35,12 +44,14 @@ class SqliteImporter(Importer):
             return to, to_session
         finally:
             if succeeded:
-                self.remove_csv(state, from_session, table, target_table, multi_tables)
+                self.remove_csv(state, from_session, table, target_table, multi_tables, job_log=job_log)
                 SQLite.clear_cache()
 
-                if not multi_tables:
+                if multi_tables:
+                    log2(f'[{to_session}] {keyspace}.{target_table} OK', file=job_log)
+                else:
                     with export_db(state) as dbs:
-                        dbs.sql(f'select * from {keyspace}.{target_table} limit 10')
+                        dbs.sql(f'select * from {keyspace}.{target_table} limit 10', backgrounded=True, export_log=job_log)
 
     def import_from_local_csv(self, state: ReplState,
                         keyspace: str, table: str, csv_file: str, multi_tables = True, create_db = False):

adam/commands/export/utils_export.py

@@ -1,12 +1,13 @@
 import io
+import os
 import re
 
 from adam.config import Config
-from adam.pod_exec_result import PodExecResult
+from adam.utils import ExecResult, creating_dir, log2
 from adam.repl_state import ReplState
 from adam.utils_k8s.cassandra_nodes import CassandraNodes
-from adam.utils_k8s.pods import log_prefix
 from adam.utils_k8s.statefulsets import StatefulSets
+from adam.utils_local import local_exec
 
 class ImportSpec:
     def __init__(self, table_name: str, session: str = None, files: list[str] = None, importer: str = None):
@@ -196,7 +197,7 @@ class ExportTableStatus:
         statuses: list[ExportTableStatus] = []
 
         status_in_whole = 'done'
-        log_files: list[str] = find_files(pod, namespace, f'{log_prefix()}-{export_session}_*.log*')
+        log_files: list[str] = find_files(pod, namespace, f'{export_log_dir()}/{export_session}_*.log*')
 
         for log_file in log_files:
             status: ExportTableStatus = ExportTableStatus.from_log_file(pod, namespace, export_session, log_file)
@@ -211,7 +212,7 @@ class ExportTableStatus:
         def get_csv_files_n_table(target_table: str):
             db = f'{copy_session}_{target_table}'
             csv_file = f'{csv_dir()}/{db}/*.csv'
-            csv_files: list[str] = find_files(pod, namespace, csv_file)
+            csv_files: list[str] = find_files(pod, namespace, csv_file, remote=True)
             if csv_files:
                 table = target_table
                 m = re.match(f'{csv_dir()}/{db}/(.*).csv', csv_files[0])
@@ -221,7 +222,7 @@ class ExportTableStatus:
 
             return csv_files, target_table
 
-        m = re.match(f'{log_prefix()}-{copy_session}_(.*?)\.(.*?)\.log(.*)', log_file)
+        m = re.match(f'{export_log_dir()}/{copy_session}_(.*?)\.(.*?)\.log(.*)', log_file)
         if m:
             keyspace = m.group(1)
             target_table = m.group(2)
@@ -234,7 +235,8 @@ class ExportTableStatus:
 
             # 4 rows exported to 1 files in 0 day, 0 hour, 0 minute, and 1.335 seconds.
             pattern = 'rows exported to'
-            r: PodExecResult = CassandraNodes.exec(pod, namespace, f"grep '{pattern}' {log_file}", show_out=Config().is_debug(), shell='bash')
+            r: ExecResult = local_exec(['grep', pattern, log_file], show_out=Config().is_debug())
+
             if r.exit_code() == 0:
                 csv_files, table = get_csv_files_n_table(target_table)
                 if csv_files:
@@ -246,17 +248,26 @@ class ExportTableStatus:
 
         return ExportTableStatus(None, None, 'unknown')
 
-def csv_dir():
-    return Config().get('export.csv_dir', '/c3/cassandra/tmp')
-
-def find_files(pod: str, namespace: str, pattern: str, mmin: int = 0):
-    if mmin:
-        r = CassandraNodes.exec(pod, namespace, f'find {pattern} -mmin -{mmin}', show_out=Config().is_debug(), shell='bash')
+def find_files(pod: str, namespace: str, pattern: str, mmin: int = 0, remote = False):
+    stdout = ''
+    if not remote:
+        # find . -maxdepth 1 -type f -name '*'
+        dir = os.path.dirname(pattern)
+        base = os.path.basename(pattern)
+        cmd = ['find', dir, '-name', base]
+        if mmin:
+                cmd += ['-mmin',  f'-{mmin}']
+
+        stdout = local_exec(cmd, show_out=Config().is_debug()).stdout
     else:
-        r = CassandraNodes.exec(pod, namespace, f'find {pattern}', show_out=Config().is_debug(), shell='bash')
+        cmd = f'find {pattern}'
+        if mmin:
+                cmd = f'{cmd} -mmin -{mmin}'
+
+        stdout = CassandraNodes.exec(pod, namespace, cmd, show_out=Config().is_debug(), shell='bash').stdout
 
     log_files = []
-    for line in r.stdout.split('\n'):
+    for line in stdout.split('\n'):
         line = line.strip(' \r')
         if line:
             log_files.append(line)
@@ -341,4 +352,15 @@ class PodPushHandler:
         return False
 
 def state_with_pod(state: ReplState, pod: str = None):
-    return PodPushHandler(state, pod=pod)
+    return PodPushHandler(state, pod=pod)
+
+def os_system_exec(cmd: str, show_out = False):
+    if show_out: log2(cmd)
+
+    os.system(cmd)
+
+def csv_dir():
+    return Config().get('export.csv_dir', '/c3/cassandra/tmp')
+
+def export_log_dir():
+    return creating_dir(Config().get('export.log-dir', '/tmp/qing-db/q/export/logs'))

adam/commands/generate_report.py

@@ -0,0 +1,52 @@
+import click
+import json
+
+from adam.checks.check_result import CheckResult
+from adam.checks.check_utils import run_checks
+from adam.commands import extract_options
+from adam.commands.command import Command
+from adam.commands.commands_utils import kaqing_log_file
+from adam.repl_state import ReplState
+from adam.utils import log2
+
+class GenerateReport(Command):
+    COMMAND = 'generate report'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(GenerateReport, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return GenerateReport.COMMAND
+
+    def required(self):
+        return ReplState.NON_L
+
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        with self.validate(args, state) as (args, state):
+            with extract_options(args, ['-s', '--show']) as (args, show_out):
+                results = run_checks(state.sts, state.namespace, state.pod, show_out=show_out)
+                output = CheckResult.report(results)
+
+                if state.in_repl:
+                    with kaqing_log_file() as json_file:
+                        json.dump(output, json_file, indent=2)
+                        log2(f'Report stored in {json_file.name}.')
+                else:
+                    click.echo(json.dumps(output, indent=2))
+
+                return output
+
+    def completion(self, state: ReplState):
+        return super().completion(state, {'-s': None})
+
+    def help(self, _: ReplState):
+        return f"{GenerateReport.COMMAND} [-s]\t generate report"

adam/commands/medusa/medusa_restore.py

@@ -1,4 +1,3 @@
-from collections.abc import Callable
 from datetime import datetime
 from functools import partial
 
@@ -8,7 +7,6 @@ from adam.commands.medusa.utils_medusa import medusa_backup_names
 from adam.utils_k8s.statefulsets import StatefulSets
 from adam.repl_state import ReplState, RequiredState
 from adam.utils_k8s.custom_resources import CustomResources
-from adam.config import Config
 from adam.utils import tabulize, log2, log_exc
 
 class MedusaRestore(Command):
@@ -69,20 +67,6 @@ class MedusaRestore(Command):
 
     def completion(self, state: ReplState):
         return super().completion(state, lambda: {id: None for id in medusa_backup_names(state)}, auto_key='medusa.backups')
-        # if sc := super().completion(state):
-        #     ns = state.namespace
-        #     dc: str = StatefulSets.get_datacenter(state.sts, ns)
-        #     if not dc:
-        #         return {}
-
-        #     if Config().get('medusa.restore-auto-complete', False):
-        #         leaf = {id: None for id in [f"{x['metadata']['name']}" for x in CustomResources.medusa_show_backupjobs(dc, ns)]}
-
-        #         return super().completion(state, leaf)
-        #     else:
-        #         return sc
-
-        return {}
 
     def help(self, _: ReplState):
         return f'{MedusaRestore.COMMAND}\t start a restore job'

adam/commands/nodetool.py

@@ -38,7 +38,7 @@ class NodeTool(Command):
             return state
 
     def completion(self, state: ReplState):
-        return super().completion(state, {c: {'&': None} for c in NODETOOL_COMMANDS}, pods=Devices.device(state).pods(state, '-'))
+        return super().completion(state, {c: {'&': None} for c in NODETOOL_COMMANDS}, pods=Devices.of(state).pods(state, '-'))
 
     def help(self, _: ReplState):
         return f'{NodeTool.COMMAND} <sub-command> [&]\t run nodetool with arguments'