kaqing 2.0.200__py3-none-any.whl → 2.0.211__py3-none-any.whl

adam/batch.py

@@ -12,7 +12,7 @@ from adam.commands.deploy.undeploy import Undeploy, UndeployCommandHelper
 from adam.commands.issues import Issues
 from adam.commands.login import Login
 from adam.commands.download_cassandra_log import DownloadCassandraLog
-from adam.commands.ls import Ls
+from adam.commands.fs.ls import Ls
 from adam.commands.medusa.medusa import Medusa
 from adam.commands.nodetool import NodeTool, NodeToolCommandHelper
 from adam.commands.postgres.postgres import Postgres, PostgresCommandHelper

adam/commands/app/utils_app.py

@@ -3,7 +3,7 @@ from typing import Union
 
 from adam.app_session import AppSession
 from adam.apps import Apps
-from adam.pod_exec_result import PodExecResult
+from adam.utils_k8s.pod_exec_result import PodExecResult
 from adam.repl_state import ReplState
 from adam.utils import log2
 from adam.utils_k8s.app_clusters import AppClusters

adam/commands/cql/completions_c.py

@@ -18,7 +18,7 @@ def completions_c(state: ReplState) -> dict[str, any]:
                 'table-props': lambda x: {
                     'GC_GRACE_SECONDS': ps
                 },
-                'table-props-value': lambda x: {'GC_GRACE_SECONDS': ps}[x],
+                'table-props-value': lambda x: {None: None, 'GC_GRACE_SECONDS': ps}[x],
                 'export-database-types': lambda x: ['athena', 'sqlite', 'csv'],
                 'export-databases': lambda x: ExportDatabases.database_names(),
                 'export-sessions': lambda x: ExportSessions.export_session_names(state.sts, state.pod, state.namespace),

adam/commands/cql/utils_cql.py

@@ -7,9 +7,9 @@ from adam.commands.commands_utils import show_table
 from adam.utils_k8s.cassandra_clusters import CassandraClusters
 from adam.utils_k8s.cassandra_nodes import CassandraNodes
 from adam.utils_k8s.secrets import Secrets
-from adam.pod_exec_result import PodExecResult
+from adam.utils_k8s.pod_exec_result import PodExecResult
 from adam.repl_state import ReplState
-from adam.utils import log2, log_timing, wait_log
+from adam.utils import ing, log2, log_timing, wait_log
 from adam.utils_k8s.statefulsets import StatefulSets
 
 def cd_dirs(state: ReplState) -> list[str]:
@@ -55,7 +55,7 @@ def table_spec(state: ReplState, table: str, on_any=False) -> 'TableSpec':
 
     return parse_cql_desc_table(r.stdout if state.pod else r[0].stdout)
 
-def run_cql(state: ReplState, cql: str, opts: list = [], show_out = False, show_query = False, use_single_quotes = False, on_any = False, backgrounded=False, log_file=None, via_sh=True) -> list[PodExecResult]:
+def run_cql(state: ReplState, cql: str, opts: list = [], show_out = False, show_query = False, use_single_quotes = False, on_any = False, backgrounded=False, log_file=None, history=True) -> list[PodExecResult]:
     if show_query:
         log2(cql)
 
@@ -69,7 +69,7 @@ def run_cql(state: ReplState, cql: str, opts: list = [], show_out = False, show_
 
     with log_timing(cql):
         with cassandra(state) as pods:
-            return pods.exec(command, action='cql', show_out=show_out, on_any=on_any, backgrounded=backgrounded, log_file=log_file, via_sh=via_sh)
+            return pods.exec(command, action='cql', show_out=show_out, on_any=on_any, backgrounded=backgrounded, log_file=log_file, history=history)
 
 def parse_cql_desc_tables(out: str):
     # Keyspace data_endpoint_auth
@@ -227,16 +227,16 @@ class CassandraPodService:
     def __init__(self, handler: 'CassandraExecHandler'):
         self.handler = handler
 
-    def exec(self, command: str, action='bash', show_out = True, on_any = False, throw_err = False, shell = '/bin/sh', backgrounded = False, log_file = None, via_sh = True) -> Union[PodExecResult, list[PodExecResult]]:
+    def exec(self, command: str, action='bash', show_out = True, on_any = False, throw_err = False, shell = '/bin/sh', backgrounded = False, log_file = None, history=True) -> Union[PodExecResult, list[PodExecResult]]:
         state = self.handler.state
         pod = self.handler.pod
 
         if pod:
             return CassandraNodes.exec(pod, state.namespace, command,
-                                    show_out=show_out, throw_err=throw_err, shell=shell, backgrounded=backgrounded, log_file=log_file, via_sh=via_sh)
+                                    show_out=show_out, throw_err=throw_err, shell=shell, backgrounded=backgrounded, log_file=log_file, history=history)
         elif state.sts:
             return CassandraClusters.exec(state.sts, state.namespace, command, action=action,
-                                        show_out=show_out, on_any=on_any, shell=shell, backgrounded=backgrounded, log_file=log_file, via_sh=via_sh)
+                                        show_out=show_out, on_any=on_any, shell=shell, backgrounded=backgrounded, log_file=log_file, history=history)
 
         return []
 
@@ -267,14 +267,15 @@ class CassandraPodService:
 
         return run_cql(state, query, opts=opts, show_out=show_out, show_query=show_query, use_single_quotes=use_single_quotes, on_any=on_any, backgrounded=backgrounded, log_file=log_file)
 
-    def display_table(self, cols: str, header: str, show_out = True, backgrounded = False):
+    def display_table(self, cols: str, header: str, show_out = True, backgrounded = False, msg: str = None):
         state = self.handler.state
 
-        if state.pod:
-            return show_table(state, [state.pod], cols, header, show_out=show_out, backgrounded = backgrounded)
-        elif state.sts:
-            pod_names = [pod.metadata.name for pod in StatefulSets.pods(state.sts, state.namespace)]
-            return show_table(state, pod_names, cols, header, show_out=show_out, backgrounded = backgrounded)
+        with ing(msg=msg, condition=backgrounded and msg):
+            if state.pod:
+                return show_table(state, [state.pod], cols, header, show_out=show_out, backgrounded = backgrounded)
+            elif state.sts:
+                pod_names = [pod.metadata.name for pod in StatefulSets.pods(state.sts, state.namespace)]
+                return show_table(state, pod_names, cols, header, show_out=show_out, backgrounded = backgrounded)
 
     def nodetool(self, args: str, status = False, show_out = True, backgrounded = False) -> Union[PodExecResult, list[PodExecResult]]:
         state = self.handler.state

adam/commands/devices/device.py

@@ -2,7 +2,7 @@ from abc import abstractmethod
 
 from adam.commands.command import Command
 from adam.config import Config
-from adam.pod_exec_result import PodExecResult
+from adam.utils_k8s.pod_exec_result import PodExecResult
 from adam.repl_state import BashSession, ReplState
 from adam.utils import log2
 from adam.utils_k8s.pods import Pods

adam/commands/download_cassandra_log.py

@@ -1,7 +1,7 @@
 from adam.commands.command import Command
 from adam.commands.devices.devices import Devices
 from adam.config import Config
-from adam.pod_exec_result import PodExecResult
+from adam.utils_k8s.pod_exec_result import PodExecResult
 from adam.utils import log2
 from adam.utils_k8s.cassandra_nodes import CassandraNodes
 from adam.repl_state import ReplState, RequiredState
@@ -31,7 +31,7 @@ class DownloadCassandraLog(Command):
 
         with self.validate(args, state) as (args, state):
             path = Config().get('logs.path', '/c3/cassandra/logs/system.log')
-            r: PodExecResult = CassandraNodes.exec(state.pod, state.namespace, f'cat {path}', backgrounded=True, no_history=True)
+            r: PodExecResult = CassandraNodes.exec(state.pod, state.namespace, f'cat {path}', backgrounded=True, history=False)
 
             to_file = Pods.download_file(state.pod, 'cassandra', state.namespace, path)
             log2(f'Downloaded to {to_file}.')

adam/commands/export/export_databases.py

@@ -1,10 +1,12 @@
 from collections.abc import Callable
 from datetime import datetime
+from functools import partial
 import os
 import boto3
 
 from adam.commands.export.export_sessions import ExportSessions
 from adam.commands.export.importer import Importer
+from adam.commands.export.utils_export import export_log_dir
 from adam.config import Config
 from adam.repl_session import ReplSession
 from adam.repl_state import ReplState
@@ -31,7 +33,7 @@ class ExportDatabases:
 
         if Config().get('repl.history.push-cat-log-file', True):
             if log_file and ReplSession().prompt_session:
-                ReplSession().prompt_session.history.append_string(f':sh cat {log_file}')
+                ReplSession().prompt_session.history.append_string(f':cat {log_file}')
 
         return cnt
 
@@ -192,20 +194,23 @@ class ExportDatabaseService:
     def __init__(self, handler: 'ExportDatabaseHandler'):
         self.handler = handler
 
-    def sql(self, query: str, database: str = None, backgrounded = False):
+    def sql(self, query: str, database: str = None, backgrounded = False, export_log: str = None):
         if not database:
             database = self.handler.state.export_session
 
-        def output(out: str):
-            log_prefix = Config().get('export.log-prefix', '/tmp/qing')
-            log_file = f'{log_prefix}-{datetime.now().strftime("%d%H%M%S")}-export.log'
+        def output(export_log: str, out: str):
+            # serving for 1. export log or 2. job log for the sampling select 10 rows afterward
+            flag = 'at'
+            if not export_log:
+                export_log = f'{export_log_dir()}/{datetime.now().strftime("%d%H%M%S")}-export.log'
+                flag = 'w'
 
-            with open(log_file, 'w') as f:
+            with open(export_log, flag) as f:
                 f.write(out)
 
-            return log_file
+            return export_log
 
-        ExportDatabases.run_query(query, database, output = output if backgrounded else None, show_query = not backgrounded)
+        ExportDatabases.run_query(query, database, output = partial(output, export_log) if backgrounded else None, show_query = not backgrounded)
 
     def drop(self, database: str):
         state = self.handler.state

adam/commands/export/export_sessions.py

@@ -3,14 +3,14 @@ import os
 import re
 
 from adam.commands.export.importer import Importer
-from adam.commands.export.utils_export import ExportTableStatus, csv_dir, find_files
+from adam.commands.export.utils_export import ExportTableStatus, csv_dir, export_log_dir, find_files, os_system_exec
 from adam.config import Config
 from adam.repl_state import ReplState
 from adam.utils import log2, tabulize, log, parallelize
 from adam.utils_k8s.cassandra_nodes import CassandraNodes
-from adam.utils_k8s.pods import Pods, log_prefix
+from adam.utils_k8s.pods import Pods
 from adam.utils_k8s.statefulsets import StatefulSets
-from adam.utils_local import local_tmp_dir
+from adam.utils_local import local_downloads_dir
 
 class ExportSessions:
     def clear_export_session_cache():
@@ -36,13 +36,13 @@ class ExportSessions:
 
         prefix = Importer.prefix_from_importer(importer)
 
-        log_files: list[str] = find_files(pod, namespace, f'{log_prefix()}-{prefix}*_*.log*')
+        log_files: list[str] = find_files(pod, namespace, f'{export_log_dir()}/{prefix}*_*.log*')
 
         if not log_files:
             return {}
 
         for log_file in log_files[:limit]:
-            m = re.match(f'{log_prefix()}-([ces].*?)_.*\.log?(.*)', log_file)
+            m = re.match(f'{export_log_dir()}/([ces].*?)_.*\.log?(.*)', log_file)
             if m:
                 s = m.group(1)
                 state = m.group(2) # '', '.pending_import', '.done'
@@ -66,7 +66,8 @@ class ExportSessions:
             pod = StatefulSets.pod_names(sts, namespace)[0]
 
         CassandraNodes.exec(pod, namespace, f'rm -rf {csv_dir()}/*', show_out=Config().is_debug(), shell='bash')
-        CassandraNodes.exec(pod, namespace, f'rm -rf {log_prefix()}-*.log*', show_out=Config().is_debug(), shell='bash')
+        cmd = f'rm -rf {export_log_dir()}/*.log*'
+        os_system_exec(cmd, show_out=Config().is_debug())
 
         return True
 
@@ -102,17 +103,18 @@ class ExportSessions:
         csv_cnt = 0
         log_cnt = 0
 
-        log_files: list[str] = find_files(pod, namespace, f'{log_prefix()}-{session}_*.log*')
+        log_files: list[str] = find_files(pod, namespace, f'{export_log_dir()}/{session}_*.log*')
 
         for log_file in log_files:
-            m = re.match(f'{log_prefix()}-{session}_(.*?)\.(.*?)\.log.*', log_file)
+            m = re.match(f'{export_log_dir()}/{session}_(.*?)\.(.*?)\.log.*', log_file)
             if m:
                 table = m.group(2)
 
                 CassandraNodes.exec(pod, namespace, f'rm -rf {csv_dir()}/{session}_{table}', show_out=not multi_tables, shell='bash')
                 csv_cnt += 1
 
-                CassandraNodes.exec(pod, namespace, f'rm -rf {log_file}', show_out=not multi_tables, shell='bash')
+                cmd = f'rm -rf {log_file}'
+                os_system_exec(cmd, show_out=not multi_tables)
                 log_cnt += 1
 
         return csv_cnt, log_cnt
@@ -142,8 +144,7 @@ class ExportSessions:
         def download_csv(table):
             from_path: str = table.csv_file
 
-            to_path = from_path.replace(csv_dir(), local_tmp_dir())
-            os.makedirs(os.path.dirname(to_path), exist_ok=True)
+            to_path = from_path.replace(csv_dir(), local_downloads_dir())
             Pods.download_file(pod, 'cassandra', namespace, from_path, to_path)
 
             log2(f'[{session}] Downloaded to {to_path}.')

adam/commands/export/exporter.py

@@ -1,5 +1,7 @@
 from datetime import datetime
+import os
 import time
+import traceback
 
 from adam.commands.command import InvalidArgumentsException
 from adam.commands.cql.utils_cql import cassandra_table_names, run_cql, table_spec
@@ -8,13 +10,12 @@ from adam.commands.export.export_sessions import ExportSessions
 from adam.commands.export.importer import Importer
 from adam.commands.export.importer_athena import AthenaImporter
 from adam.commands.export.importer_sqlite import SqliteImporter
-from adam.commands.export.utils_export import ExportSpec, ExportTableStatus, ExportTableSpec, ImportSpec, csv_dir, find_files, state_with_pod
+from adam.commands.export.utils_export import ExportSpec, ExportTableStatus, ExportTableSpec, ImportSpec, csv_dir, export_log_dir, find_files, os_system_exec, state_with_pod
 from adam.config import Config
-from adam.pod_exec_result import PodExecResult
+from adam.repl_session import ReplSession
 from adam.repl_state import ReplState
-from adam.utils import debug, log, parallelize, log2, ing, log_exc
+from adam.utils import debug, kaqing_log_file_name, log, offload, parallelize, log2, ing, log_exc
 from adam.utils_k8s.cassandra_nodes import CassandraNodes
-from adam.utils_k8s.pods import log_prefix
 
 class Exporter:
     def export_tables(args: list[str], state: ReplState, export_only: bool = False, max_workers = 0) -> tuple[list[str], ExportSpec]:
@@ -133,11 +134,9 @@ class Exporter:
 
             prefix = Importer.prefix_from_importer(spec.importer)
             if spec.session:
-                spec.session = f'{prefix}{spec.session[1:]}'
+                state.export_session = f'{prefix}{spec.session[1:]}'
             else:
-                spec.session = f'{prefix}{datetime.now().strftime("%Y%m%d%H%M%S")[3:]}'
-
-            state.export_session = spec.session
+                state.export_session = f'{prefix}{datetime.now().strftime("%Y%m%d%H%M%S")[3:]}'
 
         return spec
 
@@ -154,89 +153,160 @@ class Exporter:
         if export_state == 'init':
             CassandraNodes.exec(state.pod, state.namespace, f'rm -rf {csv_dir()}/{spec.session}_*', show_out=Config().is_debug(), shell='bash')
 
-        action = f'[{spec.session}] Exporting|Exported'
+        job_log = kaqing_log_file_name()
+
+        action = f'[{spec.session}] Triggering export of'
         if export_state == 'init':
             action = f'[{spec.session}] Preparing|Prepared'
         elif export_state == 'import':
             action = f'[{spec.session}] Importing|Imported'
 
-        with parallelize(spec.tables, max_workers, msg=action + ' {size} Cassandra tables') as exec:
-            return exec.map(lambda table: Exporter.export_table(table, state, spec.session, spec.importer, export_only, len(spec.tables) > 1, consistency=spec.consistency, export_state=export_state)), spec
-
-    def export_table(spec: ExportTableSpec, state: ReplState, session: str, importer: str, export_only = False, multi_tables = True, consistency: str = None, export_state=None):
+        msg = action + ' {size} Cassandra tables'
+
+        if export_state != 'init':
+            log2(f'[{spec.session}] Logging to {job_log}...')
+            ReplSession().append_history(f':cat {job_log}')
+
+        pod = state.pod
+        with parallelize(spec.tables, max_workers, msg=msg, collect=export_state == 'init', name='exporter') as exec:
+            return exec.map(lambda table: Exporter.export_table(table,
+                                                                state.with_pod(pod),
+                                                                spec.session,
+                                                                spec.importer,
+                                                                export_only,
+                                                                len(spec.tables) > 1,
+                                                                consistency=spec.consistency,
+                                                                export_state=export_state,
+                                                                job_log=None if export_state == 'init' else job_log)), spec
+
+    def export_table(spec: ExportTableSpec,
+                     state: ReplState,
+                     session: str,
+                     importer: str,
+                     export_only = False,
+                     multi_tables = True,
+                     consistency: str = None,
+                     export_state=None,
+                     job_log: str = None):
         s: str = None
 
         table, target_table, columns = Exporter.resove_table_n_columns(spec, state, include_ks_in_target=False, importer=importer)
 
-        log_file = f'{log_prefix()}-{session}_{spec.keyspace}.{target_table}.log'
+        log_file = f'{export_log_dir()}/{session}_{spec.keyspace}.{target_table}.log'
         create_db = not state.export_session
 
         if export_state == 'init':
             Exporter.create_table_log(spec, state, session, table, target_table)
             return 'table_log_created'
         else:
-            if export_state == 'pending_export':
-                Exporter.export_to_csv(spec, state, session, table, target_table, columns, multi_tables=multi_tables, consistency=consistency)
+            try:
+                if export_state == 'pending_export':
+                    Exporter.export_to_csv(spec, state, session, table, target_table, columns, multi_tables=multi_tables, consistency=consistency, job_log=job_log)
 
-            log_files: list[str] = find_files(state.pod, state.namespace, f'{log_file}*')
-            if not log_files:
-                return s
+                log_files: list[str] = find_files(state.pod, state.namespace, f'{log_file}*')
+                if not log_files:
+                    return s
 
-            log_file = log_files[0]
+                log_file = log_files[0]
 
-            status: ExportTableStatus = ExportTableStatus.from_log_file(state.pod, state.namespace, session, log_file)
-            while status.status != 'done':
-                if status.status == 'export_in_pregress':
-                    debug('Exporting to CSV is still in progess, sleeping for 1 sec...')
-                    time.sleep(1)
-                elif status.status == 'exported':
-                    log_file = Exporter.rename_to_pending_import(spec, state, session, target_table)
-                    if importer == 'csv' or export_only:
-                        return 'pending_import'
-                elif status.status == 'pending_import':
-                    log_file, session = Exporter.import_from_csv(spec, state, session, importer, table, target_table, columns, multi_tables=multi_tables, create_db=create_db)
+                status: ExportTableStatus = ExportTableStatus.from_log_file(state.pod, state.namespace, session, log_file)
 
-                status = ExportTableStatus.from_log_file(state.pod, state.namespace, session, log_file)
+                with offload(name='exporter') as exec:
+                    ctx = ExportTableContext(spec, state, session, importer, export_only, multi_tables, table, target_table, columns, create_db, log_file, status, job_log)
+                    exec.submit(lambda: Exporter.export_loop(ctx))
+                # Exporter.export_loop(ExportTableContext(spec, state, session, importer, export_only, multi_tables, table, target_table, columns, create_db, log_file, status))
+            except:
+                traceback.print_exc()
 
             return status.status
 
+    def export_loop(ctx: 'ExportTableContext'):
+        try:
+            while ctx.status.status != 'done':
+                if ctx.status.status == 'export_in_pregress':
+                    debug('Exporting to CSV is still in progess, sleeping for 1 sec...')
+                    time.sleep(1)
+                elif ctx.status.status == 'exported':
+                    ctx.log_file = Exporter.rename_to_pending_import(ctx.spec, ctx.state, ctx.session, ctx.target_table)
+                    ExportSessions.clear_export_session_cache()
+                    if ctx.importer == 'csv' or ctx.export_only:
+                        return 'pending_import'
+                elif ctx.status.status == 'pending_import':
+                    ctx.log_file, ctx.session = Exporter.import_from_csv(ctx.spec,
+                                                                         ctx.state,
+                                                                         ctx.session,
+                                                                         ctx.importer,
+                                                                         ctx.table,
+                                                                         ctx.target_table,
+                                                                         ctx.columns,
+                                                                         multi_tables=ctx.multi_tables,
+                                                                         create_db=ctx.create_db,
+                                                                         job_log=ctx.f)
+
+                ctx.status = ExportTableStatus.from_log_file(ctx.state.pod, ctx.state.namespace, ctx.session, ctx.log_file)
+
+            return ctx.status.status
+        except:
+            traceback.print_exc()
+
     def create_table_log(spec: ExportTableSpec, state: ReplState, session: str, table: str, target_table: str):
-        log_file = f'{log_prefix()}-{session}_{spec.keyspace}.{target_table}.log'
+        log_file = f'{export_log_dir()}/{session}_{spec.keyspace}.{target_table}.log'
+        dir = os.path.dirname(log_file)
 
-        CassandraNodes.exec(state.pod, state.namespace, f'rm -f {log_file}* && touch {log_file}', show_out=Config().is_debug(), shell='bash')
+        cmd = f'rm -f {log_file}* && mkdir -p {dir} && touch {log_file}'
+        os_system_exec(cmd, show_out=Config().is_debug())
 
         return table
 
-    def export_to_csv(spec: ExportTableSpec, state: ReplState, session: str, table: str, target_table: str, columns: str, multi_tables = True, consistency: str = None):
+    def export_to_csv(spec: ExportTableSpec,
+                      state: ReplState,
+                      session: str,
+                      table: str,
+                      target_table: str,
+                      columns: str,
+                      multi_tables = True,
+                      consistency: str = None,
+                      job_log: str = None):
         db = f'{session}_{target_table}'
 
         CassandraNodes.exec(state.pod, state.namespace, f'mkdir -p {csv_dir()}/{db}', show_out=Config().is_debug(), shell='bash')
         csv_file = f'{csv_dir()}/{db}/{table}.csv'
-        log_file = f'{log_prefix()}-{session}_{spec.keyspace}.{target_table}.log'
+        table_log_file = f'{export_log_dir()}/{session}_{spec.keyspace}.{target_table}.log'
 
         suppress_ing_log = Config().is_debug() or multi_tables
         queries = []
         if consistency:
             queries.append(f'CONSISTENCY {consistency}')
         queries.append(f"COPY {spec.keyspace}.{table}({columns}) TO '{csv_file}' WITH HEADER = TRUE")
-        r: PodExecResult = ing(
-            f'[{session}] Dumping table {spec.keyspace}.{table}{f" with consistency {consistency}" if consistency else ""}',
-            lambda: run_cql(state, ';'.join(queries), show_out=Config().is_debug(), backgrounded=True, log_file=log_file, via_sh=False),
-            suppress_log=suppress_ing_log)
 
-        return log_file
+        with ing(f'[{session}] Triggering dump of table {spec.keyspace}.{table}{f" with consistency {consistency}" if consistency else ""}',
+                 suppress_log=suppress_ing_log,
+                 job_log = job_log):
+            run_cql(state, ';'.join(queries), show_out=Config().is_debug(), backgrounded=True, log_file=table_log_file, history=False)
+
+        return table_log_file
 
     def rename_to_pending_import(spec: ExportTableSpec, state: ReplState, session: str, target_table: str):
-        log_file = f'{log_prefix()}-{session}_{spec.keyspace}.{target_table}.log'
+        log_file = f'{export_log_dir()}/{session}_{spec.keyspace}.{target_table}.log'
         to = f'{log_file}.pending_import'
 
-        CassandraNodes.exec(state.pod, state.namespace, f'mv {log_file} {to}', show_out=Config().is_debug(), shell='bash')
+        cmd =f'mv {log_file} {to}'
+        os_system_exec(cmd, show_out=Config().is_debug())
 
         return to
 
-    def import_from_csv(spec: ExportTableSpec, state: ReplState, session: str, importer: str, table: str, target_table: str, columns: str, multi_tables = True, create_db = False):
+    def import_from_csv(spec: ExportTableSpec,
+                        state: ReplState,
+                        session: str,
+                        importer: str,
+                        table: str,
+                        target_table: str,
+                        columns: str,
+                        multi_tables = True,
+                        create_db = False,
+                        job_log: str = None):
         im = AthenaImporter() if importer == 'athena' else SqliteImporter()
-        return im.import_from_csv(state, session if session else state.export_session, spec.keyspace, table, target_table, columns, multi_tables, create_db)
+        return im.import_from_csv(state, session if session else state.export_session, spec.keyspace, table, target_table, columns, multi_tables, create_db, job_log=job_log)
 
     def resove_table_n_columns(spec: ExportTableSpec, state: ReplState, include_ks_in_target = False, importer = 'sqlite'):
         table = spec.table
@@ -262,6 +332,22 @@ class Exporter:
 
         return table, target_table, columns
 
+class ExportTableContext:
+    def __init__(self, spec: ExportTableSpec, state: ReplState, session: str, importer: str, export_only = False, multi_tables = True, table: str = None, target_table: str = None, columns: str = None, create_db = False, log_file: str = None, status: ExportTableStatus = None, f: str = None):
+        self.spec = spec
+        self.state = state
+        self.session = session
+        self.importer = importer
+        self.export_only = export_only
+        self.multi_tables = multi_tables
+        self.table = table
+        self.target_table = target_table
+        self.columns = columns
+        self.create_db = create_db
+        self.log_file = log_file
+        self.status = status
+        self.f = f
+
 class ExportService:
     def __init__(self, handler: 'ExporterHandler'):
         self.handler = handler
@@ -279,16 +365,17 @@ class ExportService:
 
                 ExportSessions.clear_export_session_cache()
 
-                if spec.importer == 'csv' or export_only:
-                    ExportSessions.show_session(state.sts, state.pod, state.namespace, spec.session)
-                else:
-                    log()
-                    with export_db(state) as dbs:
-                        dbs.show_database()
+                # if spec.importer == 'csv' or export_only:
+                #     ExportSessions.show_session(state.sts, state.pod, state.namespace, spec.session)
+                # else:
+                #     log()
+                #     with export_db(state) as dbs:
+                #         dbs.show_database()
         finally:
+            pass
             # if exporting to csv, do not bind the new session id to repl state
-            if spec and spec.importer == 'csv':
-                state.export_session = export_session
+            # if spec and spec.importer == 'csv':
+            #     state.export_session = export_session
 
         return state
 

adam/commands/export/import_session.py

@@ -34,10 +34,6 @@ class ImportSession(Command):
                         return exporter.import_session(spec)
 
     def completion(self, state: ReplState):
-        # warm up cache
-        # ExportSessions.export_session_names(state.sts, state.pod, state.namespace)
-        # ExportSessions.export_session_names(state.sts, state.pod, state.namespace, export_state='pending_import')
-
         return {}
 
     def help(self, _: ReplState):

adam/commands/export/importer.py

@@ -1,11 +1,9 @@
 from abc import abstractmethod
 
-from adam.commands.export.utils_export import csv_dir
+from adam.commands.export.utils_export import csv_dir, export_log_dir, os_system_exec
 from adam.config import Config
 from adam.repl_state import ReplState
 from adam.utils import ing
-from adam.utils_k8s.cassandra_nodes import CassandraNodes
-from adam.utils_k8s.pods import log_prefix
 
 class Importer:
     @abstractmethod
@@ -13,7 +11,7 @@ class Importer:
         pass
 
     @abstractmethod
-    def import_from_csv(self, state: ReplState, from_session: str, keyspace: str, table: str, target_table: str, columns: str, multi_tables = True, create_db = False):
+    def import_from_csv(self, state: ReplState, from_session: str, keyspace: str, table: str, target_table: str, columns: str, multi_tables = True, create_db = False, log_file: str = None):
         pass
 
     @abstractmethod
@@ -27,11 +25,12 @@ class Importer:
         pod = state.pod
         namespace = state.namespace
         to_session = state.export_session
-        log_file = f'{log_prefix()}-{from_session}_{keyspace}.{target_table}.log.pending_import'
+        log_file = f'{export_log_dir()}/{from_session}_{keyspace}.{target_table}.log.pending_import'
 
-        to = f'{log_prefix()}-{to_session}_{keyspace}.{target_table}.log.done'
+        to = f'{export_log_dir()}/{to_session}_{keyspace}.{target_table}.log.done'
 
-        CassandraNodes.exec(pod, namespace, f'mv {log_file} {to}', show_out=Config().is_debug(), shell='bash')
+        cmd = f'mv {log_file} {to}'
+        os_system_exec(cmd, show_out=Config().is_debug())
 
         return to, to_session
 
@@ -41,12 +40,13 @@ class Importer:
 
         return session
 
-    def remove_csv(self, state: ReplState, from_session: str, table: str, target_table: str, multi_tables = True):
+    def remove_csv(self, state: ReplState, from_session: str, table: str, target_table: str, multi_tables = True, job_log: str = None):
         pod = state.pod
         namespace = state.namespace
 
-        with ing(f'[{from_session}] Cleaning up temporary files', suppress_log=multi_tables):
-            CassandraNodes.exec(pod, namespace, f'rm -rf {self.csv_file(from_session, table, target_table)}', show_out=Config().is_debug(), shell='bash')
+        with ing(f'[{from_session}] Cleaning up temporary files', suppress_log=multi_tables, job_log=job_log):
+            cmd = f'rm -rf {self.csv_file(from_session, table, target_table)}'
+            os_system_exec(cmd, show_out=Config().is_debug())
 
     def db(self, session: str, keyspace: str):
         return f'{session}_{keyspace}'
@@ -78,4 +78,4 @@ class Importer:
         elif session.startswith('e'):
             importer = 'athena'
 
-        return importer
+        return importer