kaqing 2.0.214__py3-none-any.whl → 2.0.227__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of kaqing might be problematic. Click here for more details.
- adam/checks/compactionstats.py +2 -1
- adam/checks/cpu.py +2 -1
- adam/checks/disk.py +6 -5
- adam/checks/gossip.py +2 -1
- adam/checks/memory.py +2 -1
- adam/checks/status.py +2 -1
- adam/commands/app/app.py +2 -2
- adam/commands/app/app_ping.py +2 -2
- adam/commands/app/login.py +2 -2
- adam/commands/app/show_app_actions.py +2 -2
- adam/commands/app/show_app_id.py +2 -2
- adam/commands/app/show_app_queues.py +2 -2
- adam/commands/app/show_login.py +2 -2
- adam/commands/audit/audit.py +3 -5
- adam/commands/audit/audit_repair_tables.py +2 -2
- adam/commands/audit/audit_run.py +2 -2
- adam/commands/audit/show_last10.py +2 -2
- adam/commands/audit/show_slow10.py +2 -2
- adam/commands/audit/show_top10.py +2 -2
- adam/commands/bash/bash.py +2 -2
- adam/commands/cassandra/download_cassandra_log.py +2 -2
- adam/commands/cassandra/restart_cluster.py +2 -2
- adam/commands/cassandra/restart_node.py +2 -2
- adam/commands/cassandra/restart_nodes.py +2 -2
- adam/commands/cassandra/rollout.py +2 -2
- adam/commands/cassandra/show_cassandra_repairs.py +2 -2
- adam/commands/cassandra/show_cassandra_status.py +2 -2
- adam/commands/cassandra/show_cassandra_version.py +2 -2
- adam/commands/cassandra/show_processes.py +6 -6
- adam/commands/cassandra/show_storage.py +2 -2
- adam/commands/cassandra/watch.py +2 -2
- adam/commands/cli/clipboard_copy.py +2 -2
- adam/commands/cli/show_cli_commands.py +3 -3
- adam/commands/code.py +2 -2
- adam/commands/command.py +32 -5
- adam/commands/config/param_get.py +2 -2
- adam/commands/config/param_set.py +2 -2
- adam/commands/config/show_params.py +2 -2
- adam/commands/cql/alter_tables.py +2 -2
- adam/commands/cql/cqlsh.py +2 -2
- adam/commands/cql/utils_cql.py +13 -3
- adam/commands/debug/debug_completes.py +2 -2
- adam/commands/debug/debug_timings.py +2 -2
- adam/commands/debug/show_offloaded_completes.py +2 -2
- adam/commands/deploy/code_start.py +2 -2
- adam/commands/deploy/code_stop.py +2 -2
- adam/commands/deploy/deploy_frontend.py +2 -2
- adam/commands/deploy/deploy_pg_agent.py +2 -2
- adam/commands/deploy/deploy_pod.py +2 -2
- adam/commands/deploy/undeploy_frontend.py +2 -2
- adam/commands/deploy/undeploy_pg_agent.py +2 -2
- adam/commands/deploy/undeploy_pod.py +2 -2
- adam/commands/devices/device.py +7 -7
- adam/commands/devices/device_app.py +6 -6
- adam/commands/devices/device_auit_log.py +2 -2
- adam/commands/devices/device_cass.py +6 -6
- adam/commands/devices/device_export.py +2 -2
- adam/commands/devices/device_postgres.py +6 -6
- adam/commands/diag/check.py +2 -2
- adam/commands/diag/generate_report.py +2 -2
- adam/commands/diag/issues.py +3 -2
- adam/commands/exit.py +2 -2
- adam/commands/export/clean_up_all_export_sessions.py +2 -2
- adam/commands/export/clean_up_export_sessions.py +2 -2
- adam/commands/export/download_export_session.py +4 -5
- adam/commands/export/drop_export_database.py +2 -2
- adam/commands/export/drop_export_databases.py +2 -2
- adam/commands/export/export.py +3 -3
- adam/commands/export/export_databases.py +3 -0
- adam/commands/export/export_select.py +2 -2
- adam/commands/export/export_sessions.py +10 -9
- adam/commands/export/export_use.py +3 -3
- adam/commands/export/export_x_select.py +2 -2
- adam/commands/export/exporter.py +11 -11
- adam/commands/export/import_files.py +3 -7
- adam/commands/export/import_session.py +2 -2
- adam/commands/export/importer.py +6 -7
- adam/commands/export/show_column_counts.py +2 -3
- adam/commands/export/show_export_databases.py +3 -4
- adam/commands/export/show_export_session.py +4 -4
- adam/commands/export/show_export_sessions.py +3 -3
- adam/commands/export/utils_export.py +25 -33
- adam/commands/fs/cat.py +2 -2
- adam/commands/fs/cat_local.py +2 -2
- adam/commands/fs/cd.py +2 -2
- adam/commands/fs/download_file.py +2 -2
- adam/commands/fs/find_files.py +3 -3
- adam/commands/fs/find_processes.py +12 -20
- adam/commands/fs/head.py +4 -4
- adam/commands/fs/head_local.py +46 -0
- adam/commands/fs/ls.py +2 -2
- adam/commands/fs/ls_local.py +2 -2
- adam/commands/fs/pwd.py +2 -2
- adam/commands/fs/rm.py +2 -2
- adam/commands/fs/rm_downloads.py +2 -2
- adam/commands/fs/rm_logs.py +13 -7
- adam/commands/fs/rm_logs_local.py +38 -0
- adam/commands/fs/shell.py +2 -2
- adam/commands/fs/show_adam.py +2 -2
- adam/commands/fs/show_host.py +2 -2
- adam/commands/fs/show_last_results.py +39 -0
- adam/commands/fs/tail.py +36 -0
- adam/commands/fs/tail_local.py +46 -0
- adam/commands/fs/utils_fs.py +192 -0
- adam/commands/help.py +2 -2
- adam/commands/kubectl.py +2 -2
- adam/commands/medusa/medusa_backup.py +2 -2
- adam/commands/medusa/medusa_restore.py +2 -2
- adam/commands/medusa/medusa_show_backupjobs.py +2 -2
- adam/commands/medusa/medusa_show_restorejobs.py +2 -2
- adam/commands/nodetool/nodetool.py +30 -7
- adam/commands/nodetool/utils_nodetool.py +44 -0
- adam/commands/postgres/postgres.py +3 -6
- adam/commands/postgres/postgres_ls.py +2 -2
- adam/commands/postgres/postgres_preview.py +2 -2
- adam/commands/preview_table.py +2 -3
- adam/commands/reaper/reaper_forward.py +2 -2
- adam/commands/reaper/reaper_forward_stop.py +2 -2
- adam/commands/reaper/reaper_restart.py +2 -2
- adam/commands/reaper/reaper_run_abort.py +2 -2
- adam/commands/reaper/reaper_runs.py +14 -12
- adam/commands/reaper/reaper_runs_abort.py +2 -2
- adam/commands/reaper/reaper_schedule_activate.py +2 -2
- adam/commands/reaper/reaper_schedule_start.py +2 -2
- adam/commands/reaper/reaper_schedule_stop.py +2 -2
- adam/commands/reaper/reaper_schedules.py +2 -2
- adam/commands/reaper/reaper_status.py +2 -2
- adam/commands/reaper/utils_reaper.py +31 -5
- adam/commands/repair/repair_log.py +2 -2
- adam/commands/repair/repair_run.py +2 -2
- adam/commands/repair/repair_scan.py +2 -2
- adam/commands/repair/repair_stop.py +2 -2
- adam/embedded_params.py +1 -1
- adam/repl.py +2 -1
- adam/repl_commands.py +25 -10
- adam/repl_session.py +10 -3
- adam/sql/qingl.lark +58 -59
- adam/utils.py +48 -8
- adam/utils_async_job.py +73 -0
- adam/utils_k8s/cassandra_clusters.py +15 -7
- adam/utils_k8s/cassandra_nodes.py +5 -4
- adam/utils_k8s/pods.py +152 -51
- adam/version.py +1 -1
- {kaqing-2.0.214.dist-info → kaqing-2.0.227.dist-info}/METADATA +1 -1
- kaqing-2.0.227.dist-info/RECORD +280 -0
- kaqing-2.0.214.dist-info/RECORD +0 -272
- {kaqing-2.0.214.dist-info → kaqing-2.0.227.dist-info}/WHEEL +0 -0
- {kaqing-2.0.214.dist-info → kaqing-2.0.227.dist-info}/entry_points.txt +0 -0
- {kaqing-2.0.214.dist-info → kaqing-2.0.227.dist-info}/top_level.txt +0 -0
|
@@ -45,5 +45,5 @@ class DeployFrontend(Command):
|
|
|
45
45
|
def completion(self, state: ReplState):
|
|
46
46
|
return super().completion(state)
|
|
47
47
|
|
|
48
|
-
def help(self,
|
|
49
|
-
return
|
|
48
|
+
def help(self, state: ReplState):
|
|
49
|
+
return super().help(state, 'deploy Web frontend')
|
|
@@ -31,5 +31,5 @@ class DeployPgAgent(Command):
|
|
|
31
31
|
def completion(self, state: ReplState):
|
|
32
32
|
return super().completion(state)
|
|
33
33
|
|
|
34
|
-
def help(self,
|
|
35
|
-
return
|
|
34
|
+
def help(self, state: ReplState):
|
|
35
|
+
return super().help(state, 'deploy Postgres agent')
|
|
@@ -102,5 +102,5 @@ class DeployPod(Command):
|
|
|
102
102
|
def completion(self, state: ReplState):
|
|
103
103
|
return super().completion(state, {'--force': None})
|
|
104
104
|
|
|
105
|
-
def help(self,
|
|
106
|
-
return
|
|
105
|
+
def help(self, state: ReplState):
|
|
106
|
+
return super().help(state, 'deploy Ops pod --force to undeploy first', args='[--force]')
|
|
@@ -34,5 +34,5 @@ class UndeployFrontend(Command):
|
|
|
34
34
|
def completion(self, state: ReplState):
|
|
35
35
|
return super().completion(state)
|
|
36
36
|
|
|
37
|
-
def help(self,
|
|
38
|
-
return
|
|
37
|
+
def help(self, state: ReplState):
|
|
38
|
+
return super().help(state, 'undeploy Web frontend')
|
|
@@ -35,5 +35,5 @@ class UndeployPgAgent(Command):
|
|
|
35
35
|
def completion(self, state: ReplState):
|
|
36
36
|
return super().completion(state)
|
|
37
37
|
|
|
38
|
-
def help(self,
|
|
39
|
-
return
|
|
38
|
+
def help(self, state: ReplState):
|
|
39
|
+
return super().help(state, 'undeploy Postgres agent')
|
|
@@ -44,5 +44,5 @@ class UndeployPod(Command):
|
|
|
44
44
|
def completion(self, state: ReplState):
|
|
45
45
|
return super().completion(state)
|
|
46
46
|
|
|
47
|
-
def help(self,
|
|
48
|
-
return
|
|
47
|
+
def help(self, state: ReplState):
|
|
48
|
+
return super().help(state, 'undeploy Ops pod')
|
adam/commands/devices/device.py
CHANGED
|
@@ -83,12 +83,12 @@ class Device:
|
|
|
83
83
|
def show_table_preview(self, state: ReplState, table: str, rows: int):
|
|
84
84
|
pass
|
|
85
85
|
|
|
86
|
-
def bash(self, s0: ReplState, s1: ReplState, args: list[str]):
|
|
86
|
+
def bash(self, s0: ReplState, s1: ReplState, args: list[str], text_color: str = None):
|
|
87
87
|
if s1.in_repl:
|
|
88
88
|
if self.bash_target_changed(s0, s1):
|
|
89
|
-
r = self._exec_with_dir(s1, args)
|
|
89
|
+
r = self._exec_with_dir(s1, args, text_color=text_color)
|
|
90
90
|
else:
|
|
91
|
-
r = self._exec_with_dir(s0, args)
|
|
91
|
+
r = self._exec_with_dir(s0, args, text_color=text_color)
|
|
92
92
|
|
|
93
93
|
if not r:
|
|
94
94
|
s1.exit_bash()
|
|
@@ -101,7 +101,7 @@ class Device:
|
|
|
101
101
|
|
|
102
102
|
return s1
|
|
103
103
|
|
|
104
|
-
def _exec_with_dir(self, state: ReplState, args: list[str]) -> list[PodExecResult]:
|
|
104
|
+
def _exec_with_dir(self, state: ReplState, args: list[str], text_color: str = None) -> list[PodExecResult]:
|
|
105
105
|
session_just_created = False
|
|
106
106
|
if not args:
|
|
107
107
|
session_just_created = True
|
|
@@ -118,18 +118,18 @@ class Device:
|
|
|
118
118
|
if pwd := state.bash_session.pwd(state):
|
|
119
119
|
args = ['cd', pwd, '&&'] + args
|
|
120
120
|
|
|
121
|
-
return self.exec_with_dir(' '.join(args), session_just_created, state)
|
|
121
|
+
return self.exec_with_dir(' '.join(args), session_just_created, state, text_color=text_color)
|
|
122
122
|
|
|
123
123
|
@abstractmethod
|
|
124
124
|
def bash_target_changed(self, s0: ReplState, s1: ReplState):
|
|
125
125
|
pass
|
|
126
126
|
|
|
127
127
|
@abstractmethod
|
|
128
|
-
def exec_no_dir(self, command: str, state: ReplState):
|
|
128
|
+
def exec_no_dir(self, command: str, state: ReplState, text_color: str = None):
|
|
129
129
|
pass
|
|
130
130
|
|
|
131
131
|
@abstractmethod
|
|
132
|
-
def exec_with_dir(self, command: str, session_just_created: bool, state: ReplState):
|
|
132
|
+
def exec_with_dir(self, command: str, session_just_created: bool, state: ReplState, text_color: str = None):
|
|
133
133
|
pass
|
|
134
134
|
|
|
135
135
|
def bash_completion(self, cmd: str, state: ReplState, default: dict = {}):
|
|
@@ -35,8 +35,8 @@ class DeviceApp(Command, Device):
|
|
|
35
35
|
def completion(self, state: ReplState):
|
|
36
36
|
return super().completion(state)
|
|
37
37
|
|
|
38
|
-
def help(self,
|
|
39
|
-
return
|
|
38
|
+
def help(self, state: ReplState):
|
|
39
|
+
return super().help(state, 'move to App Operations device')
|
|
40
40
|
|
|
41
41
|
def pod(self, state: ReplState) -> str:
|
|
42
42
|
return state.app_pod
|
|
@@ -150,13 +150,13 @@ class DeviceApp(Command, Device):
|
|
|
150
150
|
def bash_target_changed(self, s0: ReplState, s1: ReplState):
|
|
151
151
|
return s0.app_env != s1.app_env or s0.app_app != s1.app_app or s0.app_pod != s1.app_pod
|
|
152
152
|
|
|
153
|
-
def exec_no_dir(self, command: str, state: ReplState):
|
|
153
|
+
def exec_no_dir(self, command: str, state: ReplState, text_color: str = None):
|
|
154
154
|
with app(state) as pods:
|
|
155
|
-
return pods.exec(command)
|
|
155
|
+
return pods.exec(command, text_color=text_color)
|
|
156
156
|
|
|
157
|
-
def exec_with_dir(self, command: str, session_just_created: bool, state: ReplState):
|
|
157
|
+
def exec_with_dir(self, command: str, session_just_created: bool, state: ReplState, text_color: str = None):
|
|
158
158
|
with app(state) as pods:
|
|
159
|
-
return pods.exec(command, not session_just_created)
|
|
159
|
+
return pods.exec(command, not session_just_created, text_color=text_color)
|
|
160
160
|
|
|
161
161
|
def bash_completion(self, cmd: str, state: ReplState, default: dict = {}):
|
|
162
162
|
return {cmd: BashCompleter(lambda: [])} | \
|
|
@@ -30,8 +30,8 @@ class DeviceAuditLog(Command, Device):
|
|
|
30
30
|
def completion(self, state: ReplState):
|
|
31
31
|
return super().completion(state)
|
|
32
32
|
|
|
33
|
-
def help(self,
|
|
34
|
-
return
|
|
33
|
+
def help(self, state: ReplState):
|
|
34
|
+
return super().help(state, 'move to Audit Log Operations device')
|
|
35
35
|
|
|
36
36
|
def ls(self, cmd: str, _: ReplState):
|
|
37
37
|
tabulize(Athena.table_names(), header='NAME', separator=',')
|
|
@@ -43,8 +43,8 @@ class DeviceCass(Command, Device):
|
|
|
43
43
|
def completion(self, state: ReplState):
|
|
44
44
|
return super().completion(state)
|
|
45
45
|
|
|
46
|
-
def help(self,
|
|
47
|
-
return
|
|
46
|
+
def help(self, state: ReplState):
|
|
47
|
+
return super().help(state, 'move to Cassandra Operations device')
|
|
48
48
|
|
|
49
49
|
def default_container(self, _: ReplState) -> str:
|
|
50
50
|
return 'cassandra'
|
|
@@ -161,13 +161,13 @@ class DeviceCass(Command, Device):
|
|
|
161
161
|
def bash_target_changed(self, s0: ReplState, s1: ReplState):
|
|
162
162
|
return s0.sts != s1.sts or s0.pod != s1.pod
|
|
163
163
|
|
|
164
|
-
def exec_no_dir(self, command: str, state: ReplState):
|
|
164
|
+
def exec_no_dir(self, command: str, state: ReplState, text_color: str = None):
|
|
165
165
|
with cassandra(state) as pods:
|
|
166
|
-
return pods.exec(command, action='bash', show_out=True, shell='bash')
|
|
166
|
+
return pods.exec(command, action='bash', show_out=True, shell='bash', text_color=text_color)
|
|
167
167
|
|
|
168
|
-
def exec_with_dir(self, command: str, session_just_created: bool, state: ReplState):
|
|
168
|
+
def exec_with_dir(self, command: str, session_just_created: bool, state: ReplState, text_color: str = None):
|
|
169
169
|
with cassandra(state) as pods:
|
|
170
|
-
return pods.exec(command, action='bash', show_out=not session_just_created, shell='bash')
|
|
170
|
+
return pods.exec(command, action='bash', show_out=not session_just_created, shell='bash', text_color=text_color)
|
|
171
171
|
|
|
172
172
|
def bash_completion(self, cmd: str, state: ReplState, default: dict = {}):
|
|
173
173
|
completions = {cmd: BashCompleter(lambda: [])}
|
|
@@ -31,8 +31,8 @@ class DeviceExport(Command, Device):
|
|
|
31
31
|
def completion(self, state: ReplState):
|
|
32
32
|
return super().completion(state)
|
|
33
33
|
|
|
34
|
-
def help(self,
|
|
35
|
-
return
|
|
34
|
+
def help(self, state: ReplState):
|
|
35
|
+
return super().help(state, 'Export Database Operations device')
|
|
36
36
|
|
|
37
37
|
def ls(self, cmd: str, state: ReplState):
|
|
38
38
|
if state.export_session:
|
|
@@ -32,8 +32,8 @@ class DevicePostgres(Command, Device):
|
|
|
32
32
|
def completion(self, state: ReplState):
|
|
33
33
|
return super().completion(state)
|
|
34
34
|
|
|
35
|
-
def help(self,
|
|
36
|
-
return
|
|
35
|
+
def help(self, state: ReplState):
|
|
36
|
+
return super().help(state, 'move to Postgres Operations device')
|
|
37
37
|
|
|
38
38
|
def pod(self, state: ReplState) -> str:
|
|
39
39
|
pod, _ = PostgresDatabases.pod_and_container(state.namespace)
|
|
@@ -148,13 +148,13 @@ class DevicePostgres(Command, Device):
|
|
|
148
148
|
def bash_target_changed(self, s0: ReplState, s1: ReplState):
|
|
149
149
|
return s0.pg_path != s1.pg_path
|
|
150
150
|
|
|
151
|
-
def exec_no_dir(self, command: str, state: ReplState):
|
|
151
|
+
def exec_no_dir(self, command: str, state: ReplState, text_color: str = None):
|
|
152
152
|
with postgres(state) as pod:
|
|
153
|
-
return pod.exec(command, show_out=True)
|
|
153
|
+
return pod.exec(command, show_out=True, text_color=text_color)
|
|
154
154
|
|
|
155
|
-
def exec_with_dir(self, command: str, session_just_created: bool, state: ReplState):
|
|
155
|
+
def exec_with_dir(self, command: str, session_just_created: bool, state: ReplState, text_color: str = None):
|
|
156
156
|
with postgres(state) as pod:
|
|
157
|
-
return pod.exec(command, show_out=not session_just_created)
|
|
157
|
+
return pod.exec(command, show_out=not session_just_created, text_color=text_color)
|
|
158
158
|
|
|
159
159
|
def bash_completion(self, cmd: str, state: ReplState, default: dict = {}):
|
|
160
160
|
return {cmd: BashCompleter(lambda: [])}
|
adam/commands/diag/check.py
CHANGED
|
@@ -49,8 +49,8 @@ class Check(Issues):
|
|
|
49
49
|
def completion(self, _: ReplState):
|
|
50
50
|
return {Check.COMMAND: {check.name(): {'-s': None} for check in all_checks()}}
|
|
51
51
|
|
|
52
|
-
def help(self,
|
|
53
|
-
return
|
|
52
|
+
def help(self, state: ReplState):
|
|
53
|
+
return super().help(state, 'run a single check -s show processing details', args='<check-name> [-s]')
|
|
54
54
|
|
|
55
55
|
class CheckCommandHelper(click.Command):
|
|
56
56
|
def get_help(self, ctx: click.Context):
|
|
@@ -48,5 +48,5 @@ class GenerateReport(Command):
|
|
|
48
48
|
def completion(self, state: ReplState):
|
|
49
49
|
return super().completion(state, {'-s': None})
|
|
50
50
|
|
|
51
|
-
def help(self,
|
|
52
|
-
return
|
|
51
|
+
def help(self, state: ReplState):
|
|
52
|
+
return super().help(state, 'generate report -s show processing details', args='[-s]')
|
adam/commands/diag/issues.py
CHANGED
|
@@ -39,5 +39,6 @@ class Issues(Command):
|
|
|
39
39
|
def completion(self, state: ReplState):
|
|
40
40
|
return super().completion(state, {'-s': None})
|
|
41
41
|
|
|
42
|
-
def help(self,
|
|
43
|
-
|
|
42
|
+
def help(self, state: ReplState, desc: str = None, args: str = None):
|
|
43
|
+
args1 = args if args else '[-s]'
|
|
44
|
+
return super().help(state, 'find all issues -s show processing details', args=args1)
|
adam/commands/exit.py
CHANGED
|
@@ -25,5 +25,5 @@ class Exit(Command):
|
|
|
25
25
|
def completion(self, state: ReplState):
|
|
26
26
|
return {Exit.COMMAND: None}
|
|
27
27
|
|
|
28
|
-
def help(self,
|
|
29
|
-
return
|
|
28
|
+
def help(self, state: ReplState):
|
|
29
|
+
return super().help(state, 'exit kaqing <Ctrl-D>')
|
|
@@ -33,5 +33,5 @@ class CleanUpAllExportSessions(Command):
|
|
|
33
33
|
def completion(self, _: ReplState):
|
|
34
34
|
return {}
|
|
35
35
|
|
|
36
|
-
def help(self,
|
|
37
|
-
return
|
|
36
|
+
def help(self, state: ReplState):
|
|
37
|
+
return super().help(state, 'clean up all export sessions')
|
|
@@ -35,5 +35,5 @@ class CleanUpExportSessions(Command):
|
|
|
35
35
|
def completion(self, _: ReplState):
|
|
36
36
|
return {}
|
|
37
37
|
|
|
38
|
-
def help(self,
|
|
39
|
-
return
|
|
38
|
+
def help(self, state: ReplState):
|
|
39
|
+
return super().help(state, 'clean up export sessions', args='<export-session-name>,...')
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
from adam.commands import validate_args
|
|
2
2
|
from adam.commands.command import Command
|
|
3
|
-
from adam.commands.export.export_sessions import
|
|
3
|
+
from adam.commands.export.export_sessions import export_session
|
|
4
4
|
from adam.repl_state import ReplState, RequiredState
|
|
5
5
|
|
|
6
6
|
class DownloadExportSession(Command):
|
|
@@ -32,9 +32,8 @@ class DownloadExportSession(Command):
|
|
|
32
32
|
|
|
33
33
|
return state
|
|
34
34
|
|
|
35
|
-
def completion(self,
|
|
35
|
+
def completion(self, _: ReplState):
|
|
36
36
|
return {}
|
|
37
|
-
# return super().completion(state, {session: None for session in ExportSessions.export_session_names(state.sts, state.pod, state.namespace, export_state='pending_import')})
|
|
38
37
|
|
|
39
|
-
def help(self,
|
|
40
|
-
return
|
|
38
|
+
def help(self, state: ReplState):
|
|
39
|
+
return super().help(state, 'download csv files in export session', args='<export-session-name>')
|
|
@@ -35,5 +35,5 @@ class DropExportDatabase(Command):
|
|
|
35
35
|
def completion(self, _: ReplState):
|
|
36
36
|
return {}
|
|
37
37
|
|
|
38
|
-
def help(self,
|
|
39
|
-
return
|
|
38
|
+
def help(self, state: ReplState):
|
|
39
|
+
return super().help(state, 'drop export database', args='<export-database-name>')
|
|
@@ -33,5 +33,5 @@ class DropExportDatabases(Command):
|
|
|
33
33
|
def completion(self, _: ReplState):
|
|
34
34
|
return {}
|
|
35
35
|
|
|
36
|
-
def help(self,
|
|
37
|
-
return
|
|
36
|
+
def help(self, state: ReplState):
|
|
37
|
+
return super().help(state, 'drop all export databases')
|
adam/commands/export/export.py
CHANGED
|
@@ -30,8 +30,8 @@ class ExportTables(Command):
|
|
|
30
30
|
with export(state) as exporter:
|
|
31
31
|
return exporter.export(args, export_only=export_only)
|
|
32
32
|
|
|
33
|
-
def completion(self,
|
|
33
|
+
def completion(self, _: ReplState):
|
|
34
34
|
return {}
|
|
35
35
|
|
|
36
|
-
def help(self,
|
|
37
|
-
return
|
|
36
|
+
def help(self, state: ReplState):
|
|
37
|
+
return super().help(state, 'export tables to Sqlite, Athena or CSV file', args='TABLE...')
|
|
@@ -30,5 +30,5 @@ class ExportSelect(Command):
|
|
|
30
30
|
|
|
31
31
|
return {}
|
|
32
32
|
|
|
33
|
-
def help(self,
|
|
34
|
-
return
|
|
33
|
+
def help(self, state: ReplState):
|
|
34
|
+
return super().help(state, 'run queries on export database', command='<sql-select-statements>')
|
|
@@ -3,10 +3,10 @@ import os
|
|
|
3
3
|
import re
|
|
4
4
|
|
|
5
5
|
from adam.commands.export.importer import Importer
|
|
6
|
-
from adam.commands.export.utils_export import ExportTableStatus, csv_dir,
|
|
6
|
+
from adam.commands.export.utils_export import ExportTableStatus, csv_dir, fs_exec, table_log_dir
|
|
7
7
|
from adam.config import Config
|
|
8
8
|
from adam.repl_state import ReplState
|
|
9
|
-
from adam.utils import log2, tabulize, log, parallelize
|
|
9
|
+
from adam.utils import log2, log_to_pods, tabulize, log, parallelize
|
|
10
10
|
from adam.utils_k8s.cassandra_nodes import CassandraNodes
|
|
11
11
|
from adam.utils_k8s.pods import Pods
|
|
12
12
|
from adam.utils_k8s.statefulsets import StatefulSets
|
|
@@ -36,13 +36,13 @@ class ExportSessions:
|
|
|
36
36
|
|
|
37
37
|
prefix = Importer.prefix_from_importer(importer)
|
|
38
38
|
|
|
39
|
-
log_files: list[str] = find_files(pod, namespace, f'{
|
|
39
|
+
log_files: list[str] = Pods.find_files(pod, 'cassandra', namespace, f'{table_log_dir(pod, namespace)}/{prefix}*_*.log*', remote=log_to_pods())
|
|
40
40
|
|
|
41
41
|
if not log_files:
|
|
42
42
|
return {}
|
|
43
43
|
|
|
44
44
|
for log_file in log_files[:limit]:
|
|
45
|
-
m = re.match(f'{
|
|
45
|
+
m = re.match(f'{table_log_dir(pod, namespace)}/([ces].*?)_.*\.log?(.*)', log_file)
|
|
46
46
|
if m:
|
|
47
47
|
s = m.group(1)
|
|
48
48
|
state = m.group(2) # '', '.pending_import', '.done'
|
|
@@ -66,8 +66,8 @@ class ExportSessions:
|
|
|
66
66
|
pod = StatefulSets.pod_names(sts, namespace)[0]
|
|
67
67
|
|
|
68
68
|
CassandraNodes.exec(pod, namespace, f'rm -rf {csv_dir()}/*', show_out=Config().is_debug(), shell='bash')
|
|
69
|
-
cmd = f'rm -rf {
|
|
70
|
-
|
|
69
|
+
cmd = f'rm -rf {table_log_dir(pod, namespace)}/*.log*'
|
|
70
|
+
fs_exec(pod, namespace, cmd, show_out=Config().is_debug())
|
|
71
71
|
|
|
72
72
|
return True
|
|
73
73
|
|
|
@@ -103,10 +103,10 @@ class ExportSessions:
|
|
|
103
103
|
csv_cnt = 0
|
|
104
104
|
log_cnt = 0
|
|
105
105
|
|
|
106
|
-
log_files: list[str] = find_files(pod, namespace, f'{
|
|
106
|
+
log_files: list[str] = Pods.find_files(pod, 'cassandra', namespace, f'{table_log_dir(pod, namespace)}/{session}_*.log*', remote=log_to_pods())
|
|
107
107
|
|
|
108
108
|
for log_file in log_files:
|
|
109
|
-
m = re.match(f'{
|
|
109
|
+
m = re.match(f'{table_log_dir(pod, namespace)}/{session}_(.*?)\.(.*?)\.log.*', log_file)
|
|
110
110
|
if m:
|
|
111
111
|
table = m.group(2)
|
|
112
112
|
|
|
@@ -114,7 +114,7 @@ class ExportSessions:
|
|
|
114
114
|
csv_cnt += 1
|
|
115
115
|
|
|
116
116
|
cmd = f'rm -rf {log_file}'
|
|
117
|
-
|
|
117
|
+
fs_exec(pod, namespace, cmd, show_out=not multi_tables)
|
|
118
118
|
log_cnt += 1
|
|
119
119
|
|
|
120
120
|
return csv_cnt, log_cnt
|
|
@@ -145,6 +145,7 @@ class ExportSessions:
|
|
|
145
145
|
from_path: str = table.csv_file
|
|
146
146
|
|
|
147
147
|
to_path = from_path.replace(csv_dir(), local_downloads_dir())
|
|
148
|
+
os.makedirs(os.path.dirname(to_path), exist_ok=True)
|
|
148
149
|
Pods.download_file(pod, 'cassandra', namespace, from_path, to_path)
|
|
149
150
|
|
|
150
151
|
log2(f'[{session}] Downloaded to {to_path}.')
|
|
@@ -42,8 +42,8 @@ class ExportUse(Command):
|
|
|
42
42
|
|
|
43
43
|
return state
|
|
44
44
|
|
|
45
|
-
def completion(self,
|
|
45
|
+
def completion(self, _: ReplState):
|
|
46
46
|
return {}
|
|
47
47
|
|
|
48
|
-
def help(self,
|
|
49
|
-
return
|
|
48
|
+
def help(self, state: ReplState):
|
|
49
|
+
return super().help(state, 'use export database', args='<export-database-name>')
|
|
@@ -44,5 +44,5 @@ class ExportXSelect(Command):
|
|
|
44
44
|
# add only xelect completions to c: drive from lark
|
|
45
45
|
return {ExportXSelect.COMMAND: completions_x(state)[ExportXSelect.COMMAND]}
|
|
46
46
|
|
|
47
|
-
def help(self,
|
|
48
|
-
return
|
|
47
|
+
def help(self, state: ReplState):
|
|
48
|
+
return super().help(state, 'run queries on export database', command='xelect...')
|
adam/commands/export/exporter.py
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
from datetime import datetime
|
|
2
|
-
import os
|
|
3
2
|
import time
|
|
4
3
|
import traceback
|
|
5
4
|
|
|
@@ -10,12 +9,13 @@ from adam.commands.export.export_sessions import ExportSessions
|
|
|
10
9
|
from adam.commands.export.importer import Importer
|
|
11
10
|
from adam.commands.export.importer_athena import AthenaImporter
|
|
12
11
|
from adam.commands.export.importer_sqlite import SqliteImporter
|
|
13
|
-
from adam.commands.export.utils_export import ExportSpec, ExportTableStatus, ExportTableSpec, ImportSpec, csv_dir,
|
|
12
|
+
from adam.commands.export.utils_export import ExportSpec, ExportTableStatus, ExportTableSpec, ImportSpec, csv_dir, fs_exec, state_with_pod, table_log_dir
|
|
14
13
|
from adam.config import Config
|
|
15
14
|
from adam.repl_session import ReplSession
|
|
16
15
|
from adam.repl_state import ReplState
|
|
17
|
-
from adam.utils import debug, kaqing_log_file_name, log, offload, parallelize, log2, ing, log_exc
|
|
16
|
+
from adam.utils import debug, kaqing_log_file_name, log, log_to_pods, offload, parallelize, log2, ing, log_exc
|
|
18
17
|
from adam.utils_k8s.cassandra_nodes import CassandraNodes
|
|
18
|
+
from adam.utils_k8s.pods import Pods
|
|
19
19
|
|
|
20
20
|
class Exporter:
|
|
21
21
|
def export_tables(args: list[str], state: ReplState, export_only: bool = False, max_workers = 0) -> tuple[list[str], ExportSpec]:
|
|
@@ -192,7 +192,7 @@ class Exporter:
|
|
|
192
192
|
|
|
193
193
|
table, target_table, columns = Exporter.resove_table_n_columns(spec, state, include_ks_in_target=False, importer=importer)
|
|
194
194
|
|
|
195
|
-
log_file = f'{
|
|
195
|
+
log_file = f'{table_log_dir(state.pod, state.namespace)}/{session}_{spec.keyspace}.{target_table}.log'
|
|
196
196
|
create_db = not state.export_session
|
|
197
197
|
|
|
198
198
|
if export_state == 'init':
|
|
@@ -203,7 +203,7 @@ class Exporter:
|
|
|
203
203
|
if export_state == 'pending_export':
|
|
204
204
|
Exporter.export_to_csv(spec, state, session, table, target_table, columns, multi_tables=multi_tables, consistency=consistency, job_log=job_log)
|
|
205
205
|
|
|
206
|
-
log_files: list[str] = find_files(state.pod, state.namespace, f'{log_file}*')
|
|
206
|
+
log_files: list[str] = Pods.find_files(state.pod, 'cassandra', state.namespace, f'{log_file}*', remote=log_to_pods())
|
|
207
207
|
if not log_files:
|
|
208
208
|
return s
|
|
209
209
|
|
|
@@ -250,11 +250,11 @@ class Exporter:
|
|
|
250
250
|
traceback.print_exc()
|
|
251
251
|
|
|
252
252
|
def create_table_log(spec: ExportTableSpec, state: ReplState, session: str, table: str, target_table: str):
|
|
253
|
-
|
|
254
|
-
|
|
253
|
+
dir = table_log_dir(state.pod, state.namespace)
|
|
254
|
+
log_file = f'{dir}/{session}_{spec.keyspace}.{target_table}.log'
|
|
255
255
|
|
|
256
256
|
cmd = f'rm -f {log_file}* && mkdir -p {dir} && touch {log_file}'
|
|
257
|
-
|
|
257
|
+
fs_exec(state.pod, state.namespace, cmd, show_out=Config().is_debug())
|
|
258
258
|
|
|
259
259
|
return table
|
|
260
260
|
|
|
@@ -271,7 +271,7 @@ class Exporter:
|
|
|
271
271
|
|
|
272
272
|
CassandraNodes.exec(state.pod, state.namespace, f'mkdir -p {csv_dir()}/{db}', show_out=Config().is_debug(), shell='bash')
|
|
273
273
|
csv_file = f'{csv_dir()}/{db}/{table}.csv'
|
|
274
|
-
table_log_file = f'{
|
|
274
|
+
table_log_file = f'{table_log_dir(state.pod, state.namespace)}/{session}_{spec.keyspace}.{target_table}.log'
|
|
275
275
|
|
|
276
276
|
suppress_ing_log = Config().is_debug() or multi_tables
|
|
277
277
|
queries = []
|
|
@@ -287,11 +287,11 @@ class Exporter:
|
|
|
287
287
|
return table_log_file
|
|
288
288
|
|
|
289
289
|
def rename_to_pending_import(spec: ExportTableSpec, state: ReplState, session: str, target_table: str):
|
|
290
|
-
log_file = f'{
|
|
290
|
+
log_file = f'{table_log_dir(state.pod, state.namespace)}/{session}_{spec.keyspace}.{target_table}.log'
|
|
291
291
|
to = f'{log_file}.pending_import'
|
|
292
292
|
|
|
293
293
|
cmd =f'mv {log_file} {to}'
|
|
294
|
-
|
|
294
|
+
fs_exec(state.pod, state.namespace, cmd, show_out=Config().is_debug())
|
|
295
295
|
|
|
296
296
|
return to
|
|
297
297
|
|
|
@@ -33,12 +33,8 @@ class ImportCSVFiles(Command):
|
|
|
33
33
|
with export(state) as exporter:
|
|
34
34
|
return exporter.import_files(spec)
|
|
35
35
|
|
|
36
|
-
def completion(self,
|
|
37
|
-
# warm up cache
|
|
38
|
-
# ExportSessions.export_session_names(state.sts, state.pod, state.namespace)
|
|
39
|
-
# ExportSessions.export_session_names(state.sts, state.pod, state.namespace, export_state='pending_import')
|
|
40
|
-
|
|
36
|
+
def completion(self, _: ReplState):
|
|
41
37
|
return {}
|
|
42
38
|
|
|
43
|
-
def help(self,
|
|
44
|
-
return
|
|
39
|
+
def help(self, state: ReplState):
|
|
40
|
+
return super().help(state, 'import files in session to SQLite(or Athena)', args='<file-names>,...')
|
|
@@ -36,5 +36,5 @@ class ImportSession(Command):
|
|
|
36
36
|
def completion(self, state: ReplState):
|
|
37
37
|
return {}
|
|
38
38
|
|
|
39
|
-
def help(self,
|
|
40
|
-
return
|
|
39
|
+
def help(self, state: ReplState):
|
|
40
|
+
return super().help(state, 'import tables in session to SQLite(or Athena)', args='<export-session-name>')
|
adam/commands/export/importer.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
from abc import abstractmethod
|
|
2
2
|
|
|
3
|
-
from adam.commands.export.utils_export import csv_dir,
|
|
3
|
+
from adam.commands.export.utils_export import csv_dir, fs_exec, table_log_dir
|
|
4
4
|
from adam.config import Config
|
|
5
5
|
from adam.repl_state import ReplState
|
|
6
6
|
from adam.utils import ing
|
|
@@ -22,15 +22,14 @@ class Importer:
|
|
|
22
22
|
pass
|
|
23
23
|
|
|
24
24
|
def move_to_done(self, state: ReplState, from_session: str, keyspace: str, target_table: str):
|
|
25
|
-
|
|
26
|
-
namespace = state.namespace
|
|
25
|
+
dir = table_log_dir(state.pod, state.namespace)
|
|
27
26
|
to_session = state.export_session
|
|
28
|
-
log_file = f'{
|
|
27
|
+
log_file = f'{dir}/{from_session}_{keyspace}.{target_table}.log.pending_import'
|
|
29
28
|
|
|
30
|
-
to = f'{
|
|
29
|
+
to = f'{dir}/{to_session}_{keyspace}.{target_table}.log.done'
|
|
31
30
|
|
|
32
31
|
cmd = f'mv {log_file} {to}'
|
|
33
|
-
|
|
32
|
+
fs_exec(state.pod, state.namespace, cmd, show_out=Config().is_debug())
|
|
34
33
|
|
|
35
34
|
return to, to_session
|
|
36
35
|
|
|
@@ -46,7 +45,7 @@ class Importer:
|
|
|
46
45
|
|
|
47
46
|
with ing(f'[{from_session}] Cleaning up temporary files', suppress_log=multi_tables, job_log=job_log):
|
|
48
47
|
cmd = f'rm -rf {self.csv_file(from_session, table, target_table)}'
|
|
49
|
-
|
|
48
|
+
fs_exec(state.pod, state.namespace, cmd, show_out=Config().is_debug())
|
|
50
49
|
|
|
51
50
|
def db(self, session: str, keyspace: str):
|
|
52
51
|
return f'{session}_{keyspace}'
|
|
@@ -39,7 +39,6 @@ class ShowColumnCounts(Command):
|
|
|
39
39
|
|
|
40
40
|
def completion(self, state: ReplState):
|
|
41
41
|
return super().completion(state, lambda: {t: None for t in ExportDatabases.table_names(state.export_session)}, auto_key='x.tables')
|
|
42
|
-
# return {}
|
|
43
42
|
|
|
44
|
-
def help(self,
|
|
45
|
-
return
|
|
43
|
+
def help(self, state: ReplState):
|
|
44
|
+
return super().help(state, 'show column count per id', args='<export-table-name>')
|
|
@@ -31,9 +31,8 @@ class ShowExportDatabases(Command):
|
|
|
31
31
|
|
|
32
32
|
return state
|
|
33
33
|
|
|
34
|
-
def completion(self,
|
|
34
|
+
def completion(self, _: ReplState):
|
|
35
35
|
return {}
|
|
36
|
-
# return DeviceExport().ls_completion(ShowExportDatabases.COMMAND, state, default = super().completion(state))
|
|
37
36
|
|
|
38
|
-
def help(self,
|
|
39
|
-
return
|
|
37
|
+
def help(self, state: ReplState):
|
|
38
|
+
return super().help(state, 'list export databases')
|