kaqing 2.0.193__tar.gz → 2.0.209__tar.gz

{kaqing-2.0.193 → kaqing-2.0.209}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: kaqing
-Version: 2.0.193
+Version: 2.0.209
 Summary: UNKNOWN
 Home-page: UNKNOWN
 License: UNKNOWN

{kaqing-2.0.193 → kaqing-2.0.209}/adam/batch.py

@@ -12,7 +12,7 @@ from adam.commands.deploy.undeploy import Undeploy, UndeployCommandHelper
 from adam.commands.issues import Issues
 from adam.commands.login import Login
 from adam.commands.download_cassandra_log import DownloadCassandraLog
-from adam.commands.ls import Ls
+from adam.commands.fs.ls import Ls
 from adam.commands.medusa.medusa import Medusa
 from adam.commands.nodetool import NodeTool, NodeToolCommandHelper
 from adam.commands.postgres.postgres import Postgres, PostgresCommandHelper

{kaqing-2.0.193 → kaqing-2.0.209}/adam/commands/app/utils_app.py

@@ -3,7 +3,7 @@ from typing import Union
 
 from adam.app_session import AppSession
 from adam.apps import Apps
-from adam.pod_exec_result import PodExecResult
+from adam.utils_k8s.pod_exec_result import PodExecResult
 from adam.repl_state import ReplState
 from adam.utils import log2
 from adam.utils_k8s.app_clusters import AppClusters

{kaqing-2.0.193 → kaqing-2.0.209}/adam/commands/commands_utils.py

@@ -1,17 +1,14 @@
-from datetime import datetime
 from kubernetes import client
 from typing import List
 
 from adam.checks.check_utils import run_checks
 from adam.columns.columns import Columns, collect_checks
-from adam.config import Config
-from adam.repl_session import ReplSession
 from adam.utils_issues import IssuesUtils
 from adam.utils_k8s.cassandra_nodes import CassandraNodes
 from adam.utils_k8s.pods import Pods
 from adam.utils_k8s.statefulsets import StatefulSets
 from adam.repl_state import ReplState
-from adam.utils import SORT, duration, tabulize, log2
+from adam.utils import SORT, duration, kaqing_log_file, tabulize, log2
 
 def show_pods(pods: List[client.V1Pod], ns: str, show_namespace = True, show_host_id = True):
     if len(pods) == 0:
@@ -80,28 +77,4 @@ def write_to_kaqing_log_file(r: str):
     with kaqing_log_file() as f:
         f.write(r)
 
-        return f.name
-
-def _kaqing_log_file(suffix = 'log'):
-    log_prefix = Config().get('log-prefix', '/tmp/qing')
-    return f'{log_prefix}-{datetime.now().strftime("%d%H%M%S")}.{suffix}'
-
-class LogFileHandler:
-    def __init__(self, suffix = 'log'):
-        self.suffix = suffix
-
-    def __enter__(self):
-        self.f = open(_kaqing_log_file(), 'w')
-        self.f.__enter__()
-
-        return self.f
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        self.f.__exit__(exc_type, exc_val, exc_tb)
-
-        ReplSession().append_history(f':sh cat {self.f.name}')
-
-        return False
-
-def kaqing_log_file(suffix = 'log'):
-    return LogFileHandler(suffix = suffix)
+        return f.name

{kaqing-2.0.193/adam/commands → kaqing-2.0.209/adam/commands/cql}/alter_tables.py

@@ -59,7 +59,7 @@ class AlterTables(Command):
                     return state
 
     def completion(self, _: ReplState) -> dict[str, any]:
-        # auto completion is taken care of by sql completer
+        # auto completion is taken care of by lark completer
         return {}
 
     def help(self, _: ReplState) -> str:

{kaqing-2.0.193 → kaqing-2.0.209}/adam/commands/cql/completions_c.py

@@ -18,7 +18,7 @@ def completions_c(state: ReplState) -> dict[str, any]:
                 'table-props': lambda x: {
                     'GC_GRACE_SECONDS': ps
                 },
-                'table-props-value': lambda x: {'GC_GRACE_SECONDS': ps}[x],
+                'table-props-value': lambda x: {None: None, 'GC_GRACE_SECONDS': ps}[x],
                 'export-database-types': lambda x: ['athena', 'sqlite', 'csv'],
                 'export-databases': lambda x: ExportDatabases.database_names(),
                 'export-sessions': lambda x: ExportSessions.export_session_names(state.sts, state.pod, state.namespace),

{kaqing-2.0.193 → kaqing-2.0.209}/adam/commands/cql/utils_cql.py

@@ -7,7 +7,7 @@ from adam.commands.commands_utils import show_table
 from adam.utils_k8s.cassandra_clusters import CassandraClusters
 from adam.utils_k8s.cassandra_nodes import CassandraNodes
 from adam.utils_k8s.secrets import Secrets
-from adam.pod_exec_result import PodExecResult
+from adam.utils_k8s.pod_exec_result import PodExecResult
 from adam.repl_state import ReplState
 from adam.utils import log2, log_timing, wait_log
 from adam.utils_k8s.statefulsets import StatefulSets
@@ -55,7 +55,7 @@ def table_spec(state: ReplState, table: str, on_any=False) -> 'TableSpec':
 
     return parse_cql_desc_table(r.stdout if state.pod else r[0].stdout)
 
-def run_cql(state: ReplState, cql: str, opts: list = [], show_out = False, show_query = False, use_single_quotes = False, on_any = False, backgrounded=False, log_file=None) -> list[PodExecResult]:
+def run_cql(state: ReplState, cql: str, opts: list = [], show_out = False, show_query = False, use_single_quotes = False, on_any = False, backgrounded=False, log_file=None, history=True) -> list[PodExecResult]:
     if show_query:
         log2(cql)
 
@@ -69,7 +69,7 @@ def run_cql(state: ReplState, cql: str, opts: list = [], show_out = False, show_
 
     with log_timing(cql):
         with cassandra(state) as pods:
-            return pods.exec(command, action='cql', show_out=show_out, on_any=on_any, backgrounded=backgrounded, log_file=log_file)
+            return pods.exec(command, action='cql', show_out=show_out, on_any=on_any, backgrounded=backgrounded, log_file=log_file, history=history)
 
 def parse_cql_desc_tables(out: str):
     # Keyspace data_endpoint_auth
@@ -227,16 +227,16 @@ class CassandraPodService:
     def __init__(self, handler: 'CassandraExecHandler'):
         self.handler = handler
 
-    def exec(self, command: str, action='bash', show_out = True, on_any = False, throw_err = False, shell = '/bin/sh', backgrounded = False, log_file = None) -> Union[PodExecResult, list[PodExecResult]]:
+    def exec(self, command: str, action='bash', show_out = True, on_any = False, throw_err = False, shell = '/bin/sh', backgrounded = False, log_file = None, history=True) -> Union[PodExecResult, list[PodExecResult]]:
         state = self.handler.state
         pod = self.handler.pod
 
         if pod:
             return CassandraNodes.exec(pod, state.namespace, command,
-                                    show_out=show_out, throw_err=throw_err, shell=shell, backgrounded=backgrounded, log_file=log_file)
+                                    show_out=show_out, throw_err=throw_err, shell=shell, backgrounded=backgrounded, log_file=log_file, history=history)
         elif state.sts:
             return CassandraClusters.exec(state.sts, state.namespace, command, action=action,
-                                        show_out=show_out, on_any=on_any, shell=shell, backgrounded=backgrounded, log_file=log_file)
+                                        show_out=show_out, on_any=on_any, shell=shell, backgrounded=backgrounded, log_file=log_file, history=history)
 
         return []
 

kaqing-2.0.209/adam/commands/debug/debug.py

@@ -0,0 +1,22 @@
+from adam.commands.command import Command
+from adam.commands.debug.debug_completes import DebugCompletes
+from adam.commands.debug.debug_timings import DebugTimings
+from adam.commands.intermediate_command import IntermediateCommand
+
+class Debug(IntermediateCommand):
+    COMMAND = 'debug'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(Debug, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return Debug.COMMAND
+
+    def cmd_list(self):
+        return [DebugTimings(), DebugCompletes()]

kaqing-2.0.209/adam/commands/debug/debug_completes.py

@@ -0,0 +1,35 @@
+from adam.commands import validate_args
+from adam.commands.command import Command
+from adam.config import Config
+from adam.repl_state import ReplState
+
+class DebugCompletes(Command):
+    COMMAND = 'debug completes'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(DebugCompletes, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return DebugCompletes.COMMAND
+
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        with self.validate(args, state) as (args, state):
+            with validate_args(args, state, name='on, off or file') as args:
+                Config().set('debugs.complete', args)
+
+                return state
+
+    def completion(self, state: ReplState):
+        return super().completion(state, {f: None for f in ['on', 'off', 'file']})
+
+    def help(self, _: ReplState):
+        return f'{DebugCompletes.COMMAND} on|off|file\t turn auto complete debug on or off'

kaqing-2.0.209/adam/commands/debug/debug_timings.py

@@ -0,0 +1,35 @@
+from adam.commands import validate_args
+from adam.commands.command import Command
+from adam.config import Config
+from adam.repl_state import ReplState
+
+class DebugTimings(Command):
+    COMMAND = 'debug timings'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(DebugTimings, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return DebugTimings.COMMAND
+
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        with self.validate(args, state) as (args, state):
+            with validate_args(args, state, name='on, off or file') as args:
+                Config().set('debugs.timings', args)
+
+                return state
+
+    def completion(self, state: ReplState):
+        return super().completion(state, {f: None for f in ['on', 'off', 'file']})
+
+    def help(self, _: ReplState):
+        return f'{DebugTimings.COMMAND} on|off|file\t turn timing debug on or off'

{kaqing-2.0.193 → kaqing-2.0.209}/adam/commands/devices/device.py

@@ -2,7 +2,7 @@ from abc import abstractmethod
 
 from adam.commands.command import Command
 from adam.config import Config
-from adam.pod_exec_result import PodExecResult
+from adam.utils_k8s.pod_exec_result import PodExecResult
 from adam.repl_state import BashSession, ReplState
 from adam.utils import log2
 from adam.utils_k8s.pods import Pods

{kaqing-2.0.193 → kaqing-2.0.209}/adam/commands/download_cassandra_log.py

@@ -1,7 +1,7 @@
 from adam.commands.command import Command
 from adam.commands.devices.devices import Devices
 from adam.config import Config
-from adam.pod_exec_result import PodExecResult
+from adam.utils_k8s.pod_exec_result import PodExecResult
 from adam.utils import log2
 from adam.utils_k8s.cassandra_nodes import CassandraNodes
 from adam.repl_state import ReplState, RequiredState
@@ -31,7 +31,7 @@ class DownloadCassandraLog(Command):
 
         with self.validate(args, state) as (args, state):
             path = Config().get('logs.path', '/c3/cassandra/logs/system.log')
-            r: PodExecResult = CassandraNodes.exec(state.pod, state.namespace, f'cat {path}', backgrounded=True, no_history=True)
+            r: PodExecResult = CassandraNodes.exec(state.pod, state.namespace, f'cat {path}', backgrounded=True, history=False)
 
             to_file = Pods.download_file(state.pod, 'cassandra', state.namespace, path)
             log2(f'Downloaded to {to_file}.')

{kaqing-2.0.193 → kaqing-2.0.209}/adam/commands/export/export_databases.py

@@ -5,6 +5,7 @@ import boto3
 
 from adam.commands.export.export_sessions import ExportSessions
 from adam.commands.export.importer import Importer
+from adam.commands.export.utils_export import export_log_dir
 from adam.config import Config
 from adam.repl_session import ReplSession
 from adam.repl_state import ReplState
@@ -197,8 +198,7 @@ class ExportDatabaseService:
             database = self.handler.state.export_session
 
         def output(out: str):
-            log_prefix = Config().get('export.log-prefix', '/tmp/qing')
-            log_file = f'{log_prefix}-{datetime.now().strftime("%d%H%M%S")}-export.log'
+            log_file = f'{export_log_dir()}/{datetime.now().strftime("%d%H%M%S")}-export.log'
 
             with open(log_file, 'w') as f:
                 f.write(out)

{kaqing-2.0.193 → kaqing-2.0.209}/adam/commands/export/export_sessions.py

@@ -3,14 +3,14 @@ import os
 import re
 
 from adam.commands.export.importer import Importer
-from adam.commands.export.utils_export import ExportTableStatus, csv_dir, find_files
+from adam.commands.export.utils_export import ExportTableStatus, csv_dir, export_log_dir, find_files, os_system_exec
 from adam.config import Config
 from adam.repl_state import ReplState
 from adam.utils import log2, tabulize, log, parallelize
 from adam.utils_k8s.cassandra_nodes import CassandraNodes
-from adam.utils_k8s.pods import Pods, log_prefix
+from adam.utils_k8s.pods import Pods
 from adam.utils_k8s.statefulsets import StatefulSets
-from adam.utils_local import local_tmp_dir
+from adam.utils_local import local_downloads_dir
 
 class ExportSessions:
     def clear_export_session_cache():
@@ -36,13 +36,13 @@ class ExportSessions:
 
         prefix = Importer.prefix_from_importer(importer)
 
-        log_files: list[str] = find_files(pod, namespace, f'{log_prefix()}-{prefix}*_*.log*')
+        log_files: list[str] = find_files(pod, namespace, f'{export_log_dir()}/{prefix}*_*.log*')
 
         if not log_files:
             return {}
 
         for log_file in log_files[:limit]:
-            m = re.match(f'{log_prefix()}-([ces].*?)_.*\.log?(.*)', log_file)
+            m = re.match(f'{export_log_dir()}/([ces].*?)_.*\.log?(.*)', log_file)
             if m:
                 s = m.group(1)
                 state = m.group(2) # '', '.pending_import', '.done'
@@ -66,7 +66,8 @@ class ExportSessions:
             pod = StatefulSets.pod_names(sts, namespace)[0]
 
         CassandraNodes.exec(pod, namespace, f'rm -rf {csv_dir()}/*', show_out=Config().is_debug(), shell='bash')
-        CassandraNodes.exec(pod, namespace, f'rm -rf {log_prefix()}-*.log*', show_out=Config().is_debug(), shell='bash')
+        cmd = f'rm -rf {export_log_dir()}/*.log*'
+        os_system_exec(cmd, show_out=Config().is_debug())
 
         return True
 
@@ -102,17 +103,18 @@ class ExportSessions:
         csv_cnt = 0
         log_cnt = 0
 
-        log_files: list[str] = find_files(pod, namespace, f'{log_prefix()}-{session}_*.log*')
+        log_files: list[str] = find_files(pod, namespace, f'{export_log_dir()}/{session}_*.log*')
 
         for log_file in log_files:
-            m = re.match(f'{log_prefix()}-{session}_(.*?)\.(.*?)\.log.*', log_file)
+            m = re.match(f'{export_log_dir()}/{session}_(.*?)\.(.*?)\.log.*', log_file)
             if m:
                 table = m.group(2)
 
                 CassandraNodes.exec(pod, namespace, f'rm -rf {csv_dir()}/{session}_{table}', show_out=not multi_tables, shell='bash')
                 csv_cnt += 1
 
-                CassandraNodes.exec(pod, namespace, f'rm -rf {log_file}', show_out=not multi_tables, shell='bash')
+                cmd = f'rm -rf {log_file}'
+                os_system_exec(cmd, show_out=not multi_tables)
                 log_cnt += 1
 
         return csv_cnt, log_cnt
@@ -142,8 +144,7 @@ class ExportSessions:
         def download_csv(table):
             from_path: str = table.csv_file
 
-            to_path = from_path.replace(csv_dir(), local_tmp_dir())
-            os.makedirs(os.path.dirname(to_path), exist_ok=True)
+            to_path = from_path.replace(csv_dir(), local_downloads_dir())
             Pods.download_file(pod, 'cassandra', namespace, from_path, to_path)
 
             log2(f'[{session}] Downloaded to {to_path}.')

{kaqing-2.0.193 → kaqing-2.0.209}/adam/commands/export/exporter.py

@@ -1,5 +1,7 @@
 from datetime import datetime
+import os
 import time
+import traceback
 
 from adam.commands.command import InvalidArgumentsException
 from adam.commands.cql.utils_cql import cassandra_table_names, run_cql, table_spec
@@ -8,13 +10,12 @@ from adam.commands.export.export_sessions import ExportSessions
 from adam.commands.export.importer import Importer
 from adam.commands.export.importer_athena import AthenaImporter
 from adam.commands.export.importer_sqlite import SqliteImporter
-from adam.commands.export.utils_export import ExportSpec, ExportTableStatus, ExportTableSpec, ImportSpec, csv_dir, find_files, state_with_pod
+from adam.commands.export.utils_export import ExportSpec, ExportTableStatus, ExportTableSpec, ImportSpec, csv_dir, export_log_dir, find_files, os_system_exec, state_with_pod
 from adam.config import Config
-from adam.pod_exec_result import PodExecResult
+from adam.repl_session import ReplSession
 from adam.repl_state import ReplState
-from adam.utils import debug, log, parallelize, log2, ing, log_exc
+from adam.utils import debug, kaqing_log_file_name, log, offload, parallelize, log2, ing, log_exc
 from adam.utils_k8s.cassandra_nodes import CassandraNodes
-from adam.utils_k8s.pods import log_prefix
 
 class Exporter:
     def export_tables(args: list[str], state: ReplState, export_only: bool = False, max_workers = 0) -> tuple[list[str], ExportSpec]:
@@ -133,11 +134,9 @@ class Exporter:
 
             prefix = Importer.prefix_from_importer(spec.importer)
             if spec.session:
-                spec.session = f'{prefix}{spec.session[1:]}'
+                state.export_session = f'{prefix}{spec.session[1:]}'
             else:
-                spec.session = f'{prefix}{datetime.now().strftime("%Y%m%d%H%M%S")[3:]}'
-
-            state.export_session = spec.session
+                state.export_session = f'{prefix}{datetime.now().strftime("%Y%m%d%H%M%S")[3:]}'
 
         return spec
 
@@ -154,89 +153,160 @@ class Exporter:
         if export_state == 'init':
             CassandraNodes.exec(state.pod, state.namespace, f'rm -rf {csv_dir()}/{spec.session}_*', show_out=Config().is_debug(), shell='bash')
 
-        action = f'[{spec.session}] Exporting|Exported'
+        job_log = kaqing_log_file_name()
+
+        action = f'[{spec.session}] Triggering export of'
         if export_state == 'init':
             action = f'[{spec.session}] Preparing|Prepared'
         elif export_state == 'import':
             action = f'[{spec.session}] Importing|Imported'
 
-        with parallelize(spec.tables, max_workers, msg=action + ' {size} Cassandra tables') as exec:
-            return exec.map(lambda table: Exporter.export_table(table, state, spec.session, spec.importer, export_only, len(spec.tables) > 1, consistency=spec.consistency, export_state=export_state)), spec
-
-    def export_table(spec: ExportTableSpec, state: ReplState, session: str, importer: str, export_only = False, multi_tables = True, consistency: str = None, export_state=None):
+        msg = action + ' {size} Cassandra tables'
+
+        if export_state != 'init':
+            log2(f'[{spec.session}] Logging to {job_log}...')
+            ReplSession().append_history(f':sh cat {job_log}')
+
+        pod = state.pod
+        with parallelize(spec.tables, max_workers, msg=msg, collect=export_state == 'init', name='exporter') as exec:
+            return exec.map(lambda table: Exporter.export_table(table,
+                                                                state.with_pod(pod),
+                                                                spec.session,
+                                                                spec.importer,
+                                                                export_only,
+                                                                len(spec.tables) > 1,
+                                                                consistency=spec.consistency,
+                                                                export_state=export_state,
+                                                                job_log=None if export_state == 'init' else job_log)), spec
+
+    def export_table(spec: ExportTableSpec,
+                     state: ReplState,
+                     session: str,
+                     importer: str,
+                     export_only = False,
+                     multi_tables = True,
+                     consistency: str = None,
+                     export_state=None,
+                     job_log: str = None):
         s: str = None
 
         table, target_table, columns = Exporter.resove_table_n_columns(spec, state, include_ks_in_target=False, importer=importer)
 
-        log_file = f'{log_prefix()}-{session}_{spec.keyspace}.{target_table}.log'
+        log_file = f'{export_log_dir()}/{session}_{spec.keyspace}.{target_table}.log'
         create_db = not state.export_session
 
         if export_state == 'init':
             Exporter.create_table_log(spec, state, session, table, target_table)
             return 'table_log_created'
         else:
-            if export_state == 'pending_export':
-                Exporter.export_to_csv(spec, state, session, table, target_table, columns, multi_tables=multi_tables, consistency=consistency)
+            try:
+                if export_state == 'pending_export':
+                    Exporter.export_to_csv(spec, state, session, table, target_table, columns, multi_tables=multi_tables, consistency=consistency, job_log=job_log)
 
-            log_files: list[str] = find_files(state.pod, state.namespace, f'{log_file}*')
-            if not log_files:
-                return s
+                log_files: list[str] = find_files(state.pod, state.namespace, f'{log_file}*')
+                if not log_files:
+                    return s
 
-            log_file = log_files[0]
+                log_file = log_files[0]
 
-            status: ExportTableStatus = ExportTableStatus.from_log_file(state.pod, state.namespace, session, log_file)
-            while status.status != 'done':
-                if status.status == 'export_in_pregress':
-                    debug('Exporting to CSV is still in progess, sleeping for 1 sec...')
-                    time.sleep(1)
-                elif status.status == 'exported':
-                    log_file = Exporter.rename_to_pending_import(spec, state, session, target_table)
-                    if importer == 'csv' or export_only:
-                        return 'pending_import'
-                elif status.status == 'pending_import':
-                    log_file, session = Exporter.import_from_csv(spec, state, session, importer, table, target_table, columns, multi_tables=multi_tables, create_db=create_db)
+                status: ExportTableStatus = ExportTableStatus.from_log_file(state.pod, state.namespace, session, log_file)
 
-                status = ExportTableStatus.from_log_file(state.pod, state.namespace, session, log_file)
+                with offload(name='exporter') as exec:
+                    ctx = ExportTableContext(spec, state, session, importer, export_only, multi_tables, table, target_table, columns, create_db, log_file, status, job_log)
+                    exec.submit(lambda: Exporter.export_loop(ctx))
+                # Exporter.export_loop(ExportTableContext(spec, state, session, importer, export_only, multi_tables, table, target_table, columns, create_db, log_file, status))
+            except:
+                traceback.print_exc()
 
             return status.status
 
+    def export_loop(ctx: 'ExportTableContext'):
+        try:
+            while ctx.status.status != 'done':
+                if ctx.status.status == 'export_in_pregress':
+                    debug('Exporting to CSV is still in progess, sleeping for 1 sec...')
+                    time.sleep(1)
+                elif ctx.status.status == 'exported':
+                    ctx.log_file = Exporter.rename_to_pending_import(ctx.spec, ctx.state, ctx.session, ctx.target_table)
+                    ExportSessions.clear_export_session_cache()
+                    if ctx.importer == 'csv' or ctx.export_only:
+                        return 'pending_import'
+                elif ctx.status.status == 'pending_import':
+                    ctx.log_file, ctx.session = Exporter.import_from_csv(ctx.spec,
+                                                                         ctx.state,
+                                                                         ctx.session,
+                                                                         ctx.importer,
+                                                                         ctx.table,
+                                                                         ctx.target_table,
+                                                                         ctx.columns,
+                                                                         multi_tables=ctx.multi_tables,
+                                                                         create_db=ctx.create_db,
+                                                                         job_log=ctx.f)
+
+                ctx.status = ExportTableStatus.from_log_file(ctx.state.pod, ctx.state.namespace, ctx.session, ctx.log_file)
+
+            return ctx.status.status
+        except:
+            traceback.print_exc()
+
     def create_table_log(spec: ExportTableSpec, state: ReplState, session: str, table: str, target_table: str):
-        log_file = f'{log_prefix()}-{session}_{spec.keyspace}.{target_table}.log'
+        log_file = f'{export_log_dir()}/{session}_{spec.keyspace}.{target_table}.log'
+        dir = os.path.dirname(log_file)
 
-        CassandraNodes.exec(state.pod, state.namespace, f'rm -f {log_file}* && touch {log_file}', show_out=Config().is_debug(), shell='bash')
+        cmd = f'rm -f {log_file}* && mkdir -p {dir} && touch {log_file}'
+        os_system_exec(cmd, show_out=Config().is_debug())
 
         return table
 
-    def export_to_csv(spec: ExportTableSpec, state: ReplState, session: str, table: str, target_table: str, columns: str, multi_tables = True, consistency: str = None):
+    def export_to_csv(spec: ExportTableSpec,
+                      state: ReplState,
+                      session: str,
+                      table: str,
+                      target_table: str,
+                      columns: str,
+                      multi_tables = True,
+                      consistency: str = None,
+                      job_log: str = None):
         db = f'{session}_{target_table}'
 
         CassandraNodes.exec(state.pod, state.namespace, f'mkdir -p {csv_dir()}/{db}', show_out=Config().is_debug(), shell='bash')
         csv_file = f'{csv_dir()}/{db}/{table}.csv'
-        log_file = f'{log_prefix()}-{session}_{spec.keyspace}.{target_table}.log'
+        table_log_file = f'{export_log_dir()}/{session}_{spec.keyspace}.{target_table}.log'
 
         suppress_ing_log = Config().is_debug() or multi_tables
         queries = []
         if consistency:
             queries.append(f'CONSISTENCY {consistency}')
         queries.append(f"COPY {spec.keyspace}.{table}({columns}) TO '{csv_file}' WITH HEADER = TRUE")
-        r: PodExecResult = ing(
-            f'[{session}] Dumping table {spec.keyspace}.{table}{f" with consistency {consistency}" if consistency else ""}',
-            lambda: run_cql(state, ';'.join(queries), show_out=Config().is_debug(), backgrounded=True, log_file=log_file),
-            suppress_log=suppress_ing_log)
 
-        return log_file
+        with ing(f'[{session}] Triggering dump of table {spec.keyspace}.{table}{f" with consistency {consistency}" if consistency else ""}',
+                 suppress_log=suppress_ing_log,
+                 job_log = job_log):
+            run_cql(state, ';'.join(queries), show_out=Config().is_debug(), backgrounded=True, log_file=table_log_file, history=False)
+
+        return table_log_file
 
     def rename_to_pending_import(spec: ExportTableSpec, state: ReplState, session: str, target_table: str):
-        log_file = f'{log_prefix()}-{session}_{spec.keyspace}.{target_table}.log'
+        log_file = f'{export_log_dir()}/{session}_{spec.keyspace}.{target_table}.log'
         to = f'{log_file}.pending_import'
 
-        CassandraNodes.exec(state.pod, state.namespace, f'mv {log_file} {to}', show_out=Config().is_debug(), shell='bash')
+        cmd =f'mv {log_file} {to}'
+        os_system_exec(cmd, show_out=Config().is_debug())
 
         return to
 
-    def import_from_csv(spec: ExportTableSpec, state: ReplState, session: str, importer: str, table: str, target_table: str, columns: str, multi_tables = True, create_db = False):
+    def import_from_csv(spec: ExportTableSpec,
+                        state: ReplState,
+                        session: str,
+                        importer: str,
+                        table: str,
+                        target_table: str,
+                        columns: str,
+                        multi_tables = True,
+                        create_db = False,
+                        job_log: str = None):
         im = AthenaImporter() if importer == 'athena' else SqliteImporter()
-        return im.import_from_csv(state, session if session else state.export_session, spec.keyspace, table, target_table, columns, multi_tables, create_db)
+        return im.import_from_csv(state, session if session else state.export_session, spec.keyspace, table, target_table, columns, multi_tables, create_db, job_log=job_log)
 
     def resove_table_n_columns(spec: ExportTableSpec, state: ReplState, include_ks_in_target = False, importer = 'sqlite'):
         table = spec.table
@@ -262,6 +332,22 @@ class Exporter:
 
         return table, target_table, columns
 
+class ExportTableContext:
+    def __init__(self, spec: ExportTableSpec, state: ReplState, session: str, importer: str, export_only = False, multi_tables = True, table: str = None, target_table: str = None, columns: str = None, create_db = False, log_file: str = None, status: ExportTableStatus = None, f: str = None):
+        self.spec = spec
+        self.state = state
+        self.session = session
+        self.importer = importer
+        self.export_only = export_only
+        self.multi_tables = multi_tables
+        self.table = table
+        self.target_table = target_table
+        self.columns = columns
+        self.create_db = create_db
+        self.log_file = log_file
+        self.status = status
+        self.f = f
+
 class ExportService:
     def __init__(self, handler: 'ExporterHandler'):
         self.handler = handler
@@ -279,16 +365,17 @@ class ExportService:
 
                 ExportSessions.clear_export_session_cache()
 
-                if spec.importer == 'csv' or export_only:
-                    ExportSessions.show_session(state.sts, state.pod, state.namespace, spec.session)
-                else:
-                    log()
-                    with export_db(state) as dbs:
-                        dbs.show_database()
+                # if spec.importer == 'csv' or export_only:
+                #     ExportSessions.show_session(state.sts, state.pod, state.namespace, spec.session)
+                # else:
+                #     log()
+                #     with export_db(state) as dbs:
+                #         dbs.show_database()
         finally:
+            pass
             # if exporting to csv, do not bind the new session id to repl state
-            if spec and spec.importer == 'csv':
-                state.export_session = export_session
+            # if spec and spec.importer == 'csv':
+            #     state.export_session = export_session
 
         return state