kaqing 2.0.145__py3-none-any.whl → 2.0.172__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of kaqing might be problematic. Click here for more details.
- adam/__init__.py +0 -2
- adam/app_session.py +8 -11
- adam/batch.py +3 -3
- adam/checks/check_utils.py +14 -46
- adam/checks/cpu.py +7 -1
- adam/checks/cpu_metrics.py +52 -0
- adam/checks/disk.py +2 -3
- adam/columns/columns.py +3 -1
- adam/columns/cpu.py +3 -1
- adam/columns/cpu_metrics.py +22 -0
- adam/columns/memory.py +3 -4
- adam/commands/__init__.py +18 -0
- adam/commands/alter_tables.py +43 -47
- adam/commands/audit/audit.py +22 -23
- adam/commands/audit/audit_repair_tables.py +14 -17
- adam/commands/audit/audit_run.py +15 -23
- adam/commands/audit/show_last10.py +10 -13
- adam/commands/audit/show_slow10.py +10 -13
- adam/commands/audit/show_top10.py +10 -13
- adam/commands/audit/utils_show_top10.py +2 -3
- adam/commands/bash/__init__.py +5 -0
- adam/commands/bash/bash.py +7 -104
- adam/commands/bash/utils_bash.py +16 -0
- adam/commands/cat.py +13 -19
- adam/commands/cd.py +8 -10
- adam/commands/check.py +20 -21
- adam/commands/cli_commands.py +2 -3
- adam/commands/code.py +20 -23
- adam/commands/command.py +120 -39
- adam/commands/commands_utils.py +8 -17
- adam/commands/cp.py +33 -39
- adam/commands/cql/cql_completions.py +9 -4
- adam/commands/cql/cqlsh.py +10 -30
- adam/commands/cql/{cql_utils.py → utils_cql.py} +149 -15
- adam/commands/deploy/code_start.py +7 -10
- adam/commands/deploy/code_stop.py +4 -21
- adam/commands/deploy/code_utils.py +3 -3
- adam/commands/deploy/deploy.py +4 -27
- adam/commands/deploy/deploy_frontend.py +14 -17
- adam/commands/deploy/deploy_pg_agent.py +2 -5
- adam/commands/deploy/deploy_pod.py +64 -68
- adam/commands/deploy/undeploy.py +4 -27
- adam/commands/deploy/undeploy_frontend.py +4 -7
- adam/commands/deploy/undeploy_pg_agent.py +4 -7
- adam/commands/deploy/undeploy_pod.py +9 -12
- adam/commands/devices/device.py +93 -2
- adam/commands/devices/device_app.py +37 -10
- adam/commands/devices/device_auit_log.py +8 -2
- adam/commands/devices/device_cass.py +47 -7
- adam/commands/devices/device_export.py +2 -2
- adam/commands/devices/device_postgres.py +41 -6
- adam/commands/exit.py +1 -4
- adam/commands/export/clean_up_all_export_sessions.py +37 -0
- adam/commands/export/clean_up_export_sessions.py +18 -7
- adam/commands/export/drop_export_database.py +15 -18
- adam/commands/export/drop_export_databases.py +6 -9
- adam/commands/export/export.py +8 -38
- adam/commands/export/export_databases.py +16 -12
- adam/commands/export/export_handlers.py +71 -0
- adam/commands/export/export_select.py +33 -24
- adam/commands/export/export_use.py +12 -15
- adam/commands/export/exporter.py +37 -48
- adam/commands/export/import_session.py +4 -32
- adam/commands/export/importer_athena.py +4 -7
- adam/commands/export/importer_sqlite.py +19 -27
- adam/commands/export/show_column_counts.py +13 -22
- adam/commands/export/show_export_databases.py +3 -6
- adam/commands/export/show_export_session.py +10 -13
- adam/commands/export/show_export_sessions.py +8 -11
- adam/commands/export/utils_export.py +24 -1
- adam/commands/intermediate_command.py +49 -0
- adam/commands/issues.py +11 -43
- adam/commands/kubectl.py +3 -6
- adam/commands/login.py +22 -24
- adam/commands/logs.py +3 -6
- adam/commands/ls.py +8 -9
- adam/commands/medusa/medusa.py +4 -22
- adam/commands/medusa/medusa_backup.py +20 -24
- adam/commands/medusa/medusa_restore.py +29 -33
- adam/commands/medusa/medusa_show_backupjobs.py +14 -18
- adam/commands/medusa/medusa_show_restorejobs.py +11 -18
- adam/commands/nodetool.py +6 -15
- adam/commands/param_get.py +11 -12
- adam/commands/param_set.py +9 -10
- adam/commands/postgres/postgres.py +29 -37
- adam/commands/postgres/postgres_context.py +47 -23
- adam/commands/postgres/postgres_ls.py +4 -8
- adam/commands/postgres/postgres_preview.py +5 -9
- adam/commands/postgres/psql_completions.py +1 -1
- adam/commands/postgres/utils_postgres.py +66 -0
- adam/commands/preview_table.py +5 -44
- adam/commands/pwd.py +13 -16
- adam/commands/reaper/reaper.py +4 -27
- adam/commands/reaper/reaper_forward.py +48 -55
- adam/commands/reaper/reaper_forward_session.py +6 -0
- adam/commands/reaper/reaper_forward_stop.py +10 -16
- adam/commands/reaper/reaper_restart.py +7 -14
- adam/commands/reaper/reaper_run_abort.py +11 -30
- adam/commands/reaper/reaper_runs.py +42 -57
- adam/commands/reaper/reaper_runs_abort.py +29 -49
- adam/commands/reaper/reaper_schedule_activate.py +11 -30
- adam/commands/reaper/reaper_schedule_start.py +10 -29
- adam/commands/reaper/reaper_schedule_stop.py +10 -29
- adam/commands/reaper/reaper_schedules.py +4 -14
- adam/commands/reaper/reaper_status.py +8 -16
- adam/commands/reaper/utils_reaper.py +196 -0
- adam/commands/repair/repair.py +4 -22
- adam/commands/repair/repair_log.py +5 -11
- adam/commands/repair/repair_run.py +27 -34
- adam/commands/repair/repair_scan.py +32 -38
- adam/commands/repair/repair_stop.py +5 -11
- adam/commands/report.py +27 -29
- adam/commands/restart.py +25 -26
- adam/commands/rollout.py +19 -24
- adam/commands/shell.py +10 -4
- adam/commands/show/show.py +10 -26
- adam/commands/show/show_cassandra_repairs.py +35 -0
- adam/commands/show/show_cassandra_status.py +32 -43
- adam/commands/show/show_cassandra_version.py +5 -18
- adam/commands/show/show_commands.py +19 -24
- adam/commands/show/show_host.py +1 -1
- adam/commands/show/show_login.py +20 -27
- adam/commands/show/show_processes.py +15 -19
- adam/commands/show/show_storage.py +10 -20
- adam/commands/watch.py +26 -29
- adam/config.py +4 -16
- adam/embedded_params.py +1 -1
- adam/log.py +4 -4
- adam/pod_exec_result.py +3 -3
- adam/repl.py +29 -32
- adam/repl_commands.py +11 -11
- adam/repl_state.py +52 -26
- adam/sql/sql_completer.py +4 -6
- adam/sql/sql_state_machine.py +21 -14
- adam/sso/authn_ad.py +6 -8
- adam/sso/authn_okta.py +4 -6
- adam/sso/cred_cache.py +3 -5
- adam/sso/idp.py +9 -12
- adam/utils.py +393 -33
- adam/utils_athena.py +14 -13
- adam/utils_audits.py +12 -12
- adam/utils_issues.py +32 -0
- adam/utils_k8s/app_clusters.py +13 -18
- adam/utils_k8s/app_pods.py +2 -0
- adam/utils_k8s/cassandra_clusters.py +21 -18
- adam/utils_k8s/custom_resources.py +16 -17
- adam/utils_k8s/ingresses.py +2 -2
- adam/utils_k8s/jobs.py +7 -11
- adam/utils_k8s/k8s.py +87 -0
- adam/utils_k8s/pods.py +14 -76
- adam/utils_k8s/secrets.py +4 -4
- adam/utils_k8s/service_accounts.py +5 -4
- adam/utils_k8s/services.py +2 -2
- adam/utils_k8s/statefulsets.py +1 -12
- adam/utils_repl/state_machine.py +3 -3
- adam/utils_sqlite.py +78 -42
- adam/version.py +1 -1
- {kaqing-2.0.145.dist-info → kaqing-2.0.172.dist-info}/METADATA +1 -1
- kaqing-2.0.172.dist-info/RECORD +230 -0
- adam/commands/app.py +0 -67
- adam/commands/app_ping.py +0 -44
- adam/commands/export/clean_up_export_session.py +0 -53
- adam/commands/postgres/postgres_utils.py +0 -31
- adam/commands/reaper/reaper_session.py +0 -159
- adam/commands/show/show_app_actions.py +0 -56
- adam/commands/show/show_app_id.py +0 -47
- adam/commands/show/show_app_queues.py +0 -45
- adam/commands/show/show_repairs.py +0 -47
- kaqing-2.0.145.dist-info/RECORD +0 -227
- {kaqing-2.0.145.dist-info → kaqing-2.0.172.dist-info}/WHEEL +0 -0
- {kaqing-2.0.145.dist-info → kaqing-2.0.172.dist-info}/entry_points.txt +0 -0
- {kaqing-2.0.145.dist-info → kaqing-2.0.172.dist-info}/top_level.txt +0 -0
|
@@ -27,26 +27,23 @@ class ExportUse(Command):
|
|
|
27
27
|
if not(args := self.args(cmd)):
|
|
28
28
|
return super().run(cmd, state)
|
|
29
29
|
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
if not args:
|
|
35
|
-
state.export_session = None
|
|
30
|
+
with self.validate(args, state) as (args, state):
|
|
31
|
+
if not args:
|
|
32
|
+
state.export_session = None
|
|
36
33
|
|
|
37
|
-
|
|
34
|
+
log2('Export database is unset.')
|
|
38
35
|
|
|
39
|
-
|
|
36
|
+
return state
|
|
40
37
|
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
38
|
+
state.export_session = args[0]
|
|
39
|
+
if state.export_session.startswith('e'):
|
|
40
|
+
Athena.clear_cache()
|
|
41
|
+
else:
|
|
42
|
+
SQLite.clear_cache()
|
|
46
43
|
|
|
47
|
-
|
|
44
|
+
ExportDatabases.display_export_db(state.export_session)
|
|
48
45
|
|
|
49
|
-
|
|
46
|
+
return state
|
|
50
47
|
|
|
51
48
|
def completion(self, state: ReplState):
|
|
52
49
|
return super().completion(state, {n: None for n in ExportDatabases.database_names()})
|
adam/commands/export/exporter.py
CHANGED
|
@@ -1,11 +1,9 @@
|
|
|
1
|
-
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
2
1
|
from datetime import datetime
|
|
3
2
|
import functools
|
|
4
3
|
import re
|
|
5
4
|
import time
|
|
6
|
-
import traceback
|
|
7
5
|
|
|
8
|
-
from adam.commands.cql.
|
|
6
|
+
from adam.commands.cql.utils_cql import cassandra_table_names, run_cql, table_spec
|
|
9
7
|
from adam.commands.export.export_databases import ExportDatabases
|
|
10
8
|
from adam.commands.export.importer import Importer
|
|
11
9
|
from adam.commands.export.importer_athena import AthenaImporter
|
|
@@ -14,7 +12,7 @@ from adam.commands.export.utils_export import ExportSpec, ExportTableStatus, Exp
|
|
|
14
12
|
from adam.config import Config
|
|
15
13
|
from adam.pod_exec_result import PodExecResult
|
|
16
14
|
from adam.repl_state import ReplState
|
|
17
|
-
from adam.utils import
|
|
15
|
+
from adam.utils import debug, parallelize, log2, ing, log_exc
|
|
18
16
|
from adam.utils_k8s.cassandra_nodes import CassandraNodes
|
|
19
17
|
from adam.utils_k8s.pods import log_prefix
|
|
20
18
|
from adam.utils_k8s.statefulsets import StatefulSets
|
|
@@ -25,7 +23,7 @@ class Exporter:
|
|
|
25
23
|
log2('export-only for testing')
|
|
26
24
|
|
|
27
25
|
spec: ExportSpec = None
|
|
28
|
-
|
|
26
|
+
with log_exc(True):
|
|
29
27
|
spec = Exporter.export_spec(' '.join(args), state)
|
|
30
28
|
|
|
31
29
|
statuses, spec = Exporter._export_tables(spec, state, max_workers=max_workers, export_state='init')
|
|
@@ -33,8 +31,6 @@ class Exporter:
|
|
|
33
31
|
return statuses, spec
|
|
34
32
|
|
|
35
33
|
return Exporter._export_tables(spec, state, export_only, max_workers, 'pending_export')
|
|
36
|
-
except Exception as e:
|
|
37
|
-
log2(e)
|
|
38
34
|
|
|
39
35
|
return [], None
|
|
40
36
|
|
|
@@ -53,12 +49,19 @@ class Exporter:
|
|
|
53
49
|
raise Exception(f"You're currently using {importer_from_session} export database. You cannot export tables with {spec.importer} type database.")
|
|
54
50
|
else:
|
|
55
51
|
spec.importer = Importer.importer_from_session(session)
|
|
52
|
+
|
|
53
|
+
if spec.importer == 'athena' and not AthenaImporter.ping():
|
|
54
|
+
raise Exception('Credentials for Athena is not present.')
|
|
56
55
|
else:
|
|
57
56
|
if not spec.importer:
|
|
58
57
|
spec.importer = Config().get('export.default-importer', 'sqlite')
|
|
59
58
|
|
|
60
59
|
prefix = Importer.prefix_from_importer(spec.importer)
|
|
61
60
|
session = f'{prefix}{datetime.now().strftime("%Y%m%d%H%M%S")[3:]}'
|
|
61
|
+
|
|
62
|
+
if spec.importer == 'athena' and not AthenaImporter.ping():
|
|
63
|
+
raise Exception('Credentials for Athena is not present.')
|
|
64
|
+
|
|
62
65
|
if spec.importer != 'csv':
|
|
63
66
|
state.export_session = session
|
|
64
67
|
|
|
@@ -68,7 +71,7 @@ class Exporter:
|
|
|
68
71
|
|
|
69
72
|
def import_session(args: list[str], state: ReplState, max_workers = 0) -> tuple[list[str], ExportSpec]:
|
|
70
73
|
import_spec: ImportSpec = None
|
|
71
|
-
|
|
74
|
+
with log_exc(True):
|
|
72
75
|
import_spec = Exporter.import_spec(' '.join(args), state)
|
|
73
76
|
tables, status_in_whole = ExportTableStatus.from_session(state.sts, state.pod, state.namespace, import_spec.session)
|
|
74
77
|
if status_in_whole == 'done':
|
|
@@ -77,12 +80,7 @@ class Exporter:
|
|
|
77
80
|
|
|
78
81
|
spec = ExportSpec(None, None, importer=import_spec.importer, tables=[ExportTableSpec.from_status(table) for table in tables], session=import_spec.session)
|
|
79
82
|
|
|
80
|
-
return Exporter._export_tables(spec, state, max_workers=max_workers)
|
|
81
|
-
except Exception as e:
|
|
82
|
-
if Config().is_debug():
|
|
83
|
-
traceback.print_exception(e)
|
|
84
|
-
else:
|
|
85
|
-
log2(e)
|
|
83
|
+
return Exporter._export_tables(spec, state, max_workers=max_workers, export_state = 'import')
|
|
86
84
|
|
|
87
85
|
return [], None
|
|
88
86
|
|
|
@@ -99,16 +97,19 @@ class Exporter:
|
|
|
99
97
|
spec.importer = Importer.importer_from_session(state.export_session)
|
|
100
98
|
if not spec.importer:
|
|
101
99
|
spec.importer = Config().get('export.default-importer', 'sqlite')
|
|
100
|
+
|
|
101
|
+
if spec.importer == 'athena' and not AthenaImporter.ping():
|
|
102
|
+
raise Exception('Credentials for Athena is not present.')
|
|
102
103
|
else:
|
|
103
|
-
if spec.importer:
|
|
104
|
-
if not AthenaImporter.ping():
|
|
105
|
-
raise Exception('Credentials for Athena are not present.')
|
|
106
|
-
else:
|
|
104
|
+
if not spec.importer:
|
|
107
105
|
spec.importer = Importer.importer_from_session(spec.session)
|
|
108
106
|
|
|
109
107
|
if spec.importer == 'csv':
|
|
110
108
|
spec.importer = Config().get('export.default-importer', 'sqlite')
|
|
111
109
|
|
|
110
|
+
if spec.importer == 'athena' and not AthenaImporter.ping():
|
|
111
|
+
raise Exception('Credentials for Athena is not present.')
|
|
112
|
+
|
|
112
113
|
prefix = Importer.prefix_from_importer(spec.importer)
|
|
113
114
|
session = f'{prefix}{spec.session[1:]}'
|
|
114
115
|
state.export_session = session
|
|
@@ -128,20 +129,14 @@ class Exporter:
|
|
|
128
129
|
if export_state == 'init':
|
|
129
130
|
CassandraNodes.exec(state.pod, state.namespace, f'rm -rf {csv_dir()}/{spec.session}_*', show_out=Config().is_debug(), shell='bash')
|
|
130
131
|
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
return [future.result() for future in as_completed(futures)], spec
|
|
141
|
-
finally:
|
|
142
|
-
log2(f"{len(spec.tables)} parallel table export elapsed time: {elapsed_time(start_time)} with {max_workers} workers")
|
|
143
|
-
else:
|
|
144
|
-
return [Exporter.export_table(table, state, spec.session, spec.importer, export_only, multi_tables=len(spec.tables) > 1, consistency=spec.consistency, export_state=export_state) for table in spec.tables], spec
|
|
132
|
+
action = f'[{spec.session}] Exporting|Exported'
|
|
133
|
+
if export_state == 'init':
|
|
134
|
+
action = f'[{spec.session}] Preparing|Prepared'
|
|
135
|
+
elif export_state == 'import':
|
|
136
|
+
action = f'[{spec.session}] Importing|Imported'
|
|
137
|
+
|
|
138
|
+
with parallelize(spec.tables, max_workers, msg=action + ' {size} Cassandra tables') as exec:
|
|
139
|
+
return exec.map(lambda table: Exporter.export_table(table, state, spec.session, spec.importer, export_only, len(spec.tables) > 1, consistency=spec.consistency, export_state=export_state)), spec
|
|
145
140
|
|
|
146
141
|
def export_table(spec: ExportTableSpec, state: ReplState, session: str, importer: str, export_only = False, multi_tables = True, consistency: str = None, export_state=None):
|
|
147
142
|
s: str = None
|
|
@@ -167,8 +162,7 @@ class Exporter:
|
|
|
167
162
|
status: ExportTableStatus = ExportTableStatus.from_log_file(state.pod, state.namespace, session, log_file)
|
|
168
163
|
while status.status != 'done':
|
|
169
164
|
if status.status == 'export_in_pregress':
|
|
170
|
-
|
|
171
|
-
log2('Exporting to CSV is still in progess, sleeping for 1 sec...')
|
|
165
|
+
debug('Exporting to CSV is still in progess, sleeping for 1 sec...')
|
|
172
166
|
time.sleep(1)
|
|
173
167
|
elif status.status == 'exported':
|
|
174
168
|
log_file = Exporter.rename_to_pending_import(spec, state, session, target_table)
|
|
@@ -283,20 +277,15 @@ class Exporter:
|
|
|
283
277
|
if not max_workers:
|
|
284
278
|
max_workers = Config().action_workers('export', 8)
|
|
285
279
|
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
return [future.result() for future in as_completed(futures)]
|
|
296
|
-
finally:
|
|
297
|
-
log2(f"{len(sessions)} parallel session clean ups elapsed time: {elapsed_time(start_time)} with {max_workers} workers")
|
|
298
|
-
else:
|
|
299
|
-
return [Exporter.clean_up_session(sts, pod, namespace, session) for session in sessions]
|
|
280
|
+
with parallelize(sessions, max_workers, msg='Cleaning|Cleaned up {size} export sessions') as exec:
|
|
281
|
+
cnt_tuples = exec.map(lambda session: Exporter.clean_up_session(sts, pod, namespace, session, True))
|
|
282
|
+
csv_cnt = 0
|
|
283
|
+
log_cnt = 0
|
|
284
|
+
for (csv, log) in cnt_tuples:
|
|
285
|
+
csv_cnt += csv
|
|
286
|
+
log_cnt += log
|
|
287
|
+
|
|
288
|
+
return csv_cnt, log_cnt
|
|
300
289
|
|
|
301
290
|
def clean_up_session(sts: str, pod: str, namespace: str, session: str, multi_tables = True):
|
|
302
291
|
if not sts or not namespace:
|
|
@@ -1,9 +1,7 @@
|
|
|
1
1
|
from adam.commands.command import Command
|
|
2
|
-
from adam.commands.export.
|
|
2
|
+
from adam.commands.export.export_handlers import export
|
|
3
3
|
from adam.commands.export.exporter import Exporter
|
|
4
4
|
from adam.repl_state import ReplState, RequiredState
|
|
5
|
-
from adam.utils import log, log2
|
|
6
|
-
from adam.utils_k8s.statefulsets import StatefulSets
|
|
7
5
|
|
|
8
6
|
class ImportSession(Command):
|
|
9
7
|
COMMAND = 'import session'
|
|
@@ -27,35 +25,9 @@ class ImportSession(Command):
|
|
|
27
25
|
if not(args := self.args(cmd)):
|
|
28
26
|
return super().run(cmd, state)
|
|
29
27
|
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
if not args:
|
|
35
|
-
if state.in_repl:
|
|
36
|
-
log2('Specify export session name.')
|
|
37
|
-
else:
|
|
38
|
-
log2('* Export session name is missing.')
|
|
39
|
-
|
|
40
|
-
Command.display_help()
|
|
41
|
-
|
|
42
|
-
return 'command-missing'
|
|
43
|
-
|
|
44
|
-
if not state.pod:
|
|
45
|
-
state.push()
|
|
46
|
-
state.pod = StatefulSets.pod_names(state.sts, state.namespace)[0]
|
|
47
|
-
|
|
48
|
-
try:
|
|
49
|
-
tables, _ = Exporter.import_session(args, state)
|
|
50
|
-
if tables:
|
|
51
|
-
Exporter.clear_export_session_cache()
|
|
52
|
-
|
|
53
|
-
log()
|
|
54
|
-
ExportDatabases.display_export_db(state.export_session)
|
|
55
|
-
finally:
|
|
56
|
-
state.pop()
|
|
57
|
-
|
|
58
|
-
return state
|
|
28
|
+
with self.validate(args, state) as (args, state):
|
|
29
|
+
with export(state) as exporter:
|
|
30
|
+
return exporter.import_sesion(args)
|
|
59
31
|
|
|
60
32
|
def completion(self, state: ReplState):
|
|
61
33
|
# warm up cache
|
|
@@ -3,7 +3,7 @@ import boto3
|
|
|
3
3
|
from adam.commands.export.importer import Importer
|
|
4
4
|
from adam.commands.export.utils_export import GeneratorStream
|
|
5
5
|
from adam.config import Config
|
|
6
|
-
from adam.utils import log2, ing
|
|
6
|
+
from adam.utils import debug, log2, ing
|
|
7
7
|
from adam.utils_athena import Athena
|
|
8
8
|
from adam.utils_k8s.pods import Pods
|
|
9
9
|
|
|
@@ -38,13 +38,11 @@ class AthenaImporter(Importer):
|
|
|
38
38
|
msg = f"[{to_session}] Creating table {target_table}"
|
|
39
39
|
with ing(msg, suppress_log=multi_tables):
|
|
40
40
|
query = f'CREATE DATABASE IF NOT EXISTS {db};'
|
|
41
|
-
|
|
42
|
-
log2(query)
|
|
41
|
+
debug(query)
|
|
43
42
|
Athena.query(query, 'default')
|
|
44
43
|
|
|
45
44
|
query = f'DROP TABLE IF EXISTS {target_table};'
|
|
46
|
-
|
|
47
|
-
log2(query)
|
|
45
|
+
debug(query)
|
|
48
46
|
Athena.query(query, db)
|
|
49
47
|
|
|
50
48
|
athena_columns = ', '.join([f'{c} string' for c in columns.split(',')])
|
|
@@ -56,8 +54,7 @@ class AthenaImporter(Importer):
|
|
|
56
54
|
' "quoteChar" = "\\"")\n' + \
|
|
57
55
|
f"LOCATION 's3://{bucket}/export/{db}/{keyspace}/{target_table}'\n" + \
|
|
58
56
|
'TBLPROPERTIES ("skip.header.line.count"="1");'
|
|
59
|
-
|
|
60
|
-
log2(query)
|
|
57
|
+
debug(query)
|
|
61
58
|
try:
|
|
62
59
|
Athena.query(query, db)
|
|
63
60
|
except Exception as e:
|
|
@@ -1,12 +1,10 @@
|
|
|
1
|
-
import os
|
|
2
|
-
import sqlite3
|
|
3
1
|
import pandas
|
|
4
2
|
|
|
5
3
|
from adam.commands.export.importer import Importer
|
|
6
4
|
from adam.commands.export.utils_export import GeneratorStream
|
|
7
5
|
from adam.utils import log2, ing
|
|
8
6
|
from adam.utils_k8s.pods import Pods
|
|
9
|
-
from adam.utils_sqlite import SQLite
|
|
7
|
+
from adam.utils_sqlite import SQLite, sqlite
|
|
10
8
|
|
|
11
9
|
class SqliteImporter(Importer):
|
|
12
10
|
def prefix(self):
|
|
@@ -14,34 +12,28 @@ class SqliteImporter(Importer):
|
|
|
14
12
|
|
|
15
13
|
def import_from_csv(self, pod: str, namespace: str, to_session: str, from_session: str, keyspace: str, table: str, target_table: str, columns: str, multi_tables = True, create_db = False):
|
|
16
14
|
csv_file = self.csv_file(from_session, table, target_table)
|
|
17
|
-
db = self.db(to_session, keyspace)
|
|
18
15
|
|
|
19
16
|
succeeded = False
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
17
|
+
with sqlite(to_session, keyspace) as conn:
|
|
18
|
+
try:
|
|
19
|
+
with ing(f'[{to_session}] Uploading to Sqlite', suppress_log=multi_tables):
|
|
20
|
+
bytes = Pods.read_file(pod, 'cassandra', namespace, csv_file)
|
|
21
|
+
df = pandas.read_csv(GeneratorStream(bytes))
|
|
24
22
|
|
|
25
|
-
|
|
26
|
-
bytes = Pods.read_file(pod, 'cassandra', namespace, csv_file)
|
|
27
|
-
df = pandas.read_csv(GeneratorStream(bytes))
|
|
23
|
+
df.to_sql(target_table, conn, index=False, if_exists='replace')
|
|
28
24
|
|
|
29
|
-
|
|
25
|
+
to, _ = self.move_to_done(pod, namespace, to_session, from_session, keyspace, target_table)
|
|
30
26
|
|
|
31
|
-
|
|
27
|
+
succeeded = True
|
|
32
28
|
|
|
33
|
-
|
|
29
|
+
return to, to_session
|
|
30
|
+
finally:
|
|
31
|
+
if succeeded:
|
|
32
|
+
self.remove_csv(pod, namespace, from_session, table, target_table, multi_tables)
|
|
33
|
+
SQLite.clear_cache()
|
|
34
34
|
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
if not multi_tables:
|
|
42
|
-
query = f'select * from {target_table} limit 10'
|
|
43
|
-
log2(query)
|
|
44
|
-
SQLite.run_query(query, conn_passed=conn)
|
|
45
|
-
|
|
46
|
-
if conn:
|
|
47
|
-
conn.close()
|
|
35
|
+
if not multi_tables:
|
|
36
|
+
with sqlite(to_session) as conn:
|
|
37
|
+
query = f'select * from {keyspace}.{target_table} limit 10'
|
|
38
|
+
log2(query)
|
|
39
|
+
SQLite.run_query_with_conn(conn, query)
|
|
@@ -3,8 +3,6 @@ from adam.commands.export.export_databases import ExportDatabases
|
|
|
3
3
|
from adam.config import Config
|
|
4
4
|
from adam.repl_state import ReplState, RequiredState
|
|
5
5
|
from adam.utils import log2
|
|
6
|
-
from adam.utils_athena import Athena
|
|
7
|
-
from adam.utils_sqlite import SQLite
|
|
8
6
|
|
|
9
7
|
class ShowColumnCounts(Command):
|
|
10
8
|
COMMAND = 'show column counts on'
|
|
@@ -28,30 +26,23 @@ class ShowColumnCounts(Command):
|
|
|
28
26
|
if not(args := self.args(cmd)):
|
|
29
27
|
return super().run(cmd, state)
|
|
30
28
|
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
log2('Use a SQL statement.')
|
|
38
|
-
else:
|
|
39
|
-
log2('* SQL statement is missing.')
|
|
29
|
+
with self.validate(args, state) as (args, state):
|
|
30
|
+
if not args:
|
|
31
|
+
if state.in_repl:
|
|
32
|
+
log2('Use a SQL statement.')
|
|
33
|
+
else:
|
|
34
|
+
log2('* SQL statement is missing.')
|
|
40
35
|
|
|
41
|
-
|
|
36
|
+
Command.display_help()
|
|
42
37
|
|
|
43
|
-
|
|
38
|
+
return 'command-missing'
|
|
44
39
|
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
40
|
+
table = args[0]
|
|
41
|
+
query = Config().get(f'export.column_counts_query', 'select id, count(id) as columns from {table} group by id')
|
|
42
|
+
query = query.replace('{table}', table)
|
|
43
|
+
ExportDatabases.run_query(query, state.export_session)
|
|
48
44
|
|
|
49
|
-
|
|
50
|
-
query = Config().get(f'{copy_or_export}.column_counts_query', 'select id, count(id) as columns from {table} group by id')
|
|
51
|
-
query = query.replace('{table}', table)
|
|
52
|
-
ExportDatabases.run_query(query, state.export_session)
|
|
53
|
-
|
|
54
|
-
return state
|
|
45
|
+
return state
|
|
55
46
|
|
|
56
47
|
def completion(self, state: ReplState):
|
|
57
48
|
if not state.export_session:
|
|
@@ -24,13 +24,10 @@ class ShowExportDatabases(Command):
|
|
|
24
24
|
if not(args := self.args(cmd)):
|
|
25
25
|
return super().run(cmd, state)
|
|
26
26
|
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
return state
|
|
30
|
-
|
|
31
|
-
DeviceExport().show_export_databases()
|
|
27
|
+
with self.validate(args, state) as (args, state):
|
|
28
|
+
DeviceExport().show_export_databases()
|
|
32
29
|
|
|
33
|
-
|
|
30
|
+
return state
|
|
34
31
|
|
|
35
32
|
def completion(self, state: ReplState):
|
|
36
33
|
return DeviceExport().ls_completion(ShowExportDatabases.COMMAND, state, default = super().completion(state))
|
|
@@ -26,23 +26,20 @@ class ShowExportSession(Command):
|
|
|
26
26
|
if not(args := self.args(cmd)):
|
|
27
27
|
return super().run(cmd, state)
|
|
28
28
|
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
log2('Specify export database name.')
|
|
36
|
-
else:
|
|
37
|
-
log2('* Database name is missing.')
|
|
29
|
+
with self.validate(args, state) as (args, state):
|
|
30
|
+
if not args:
|
|
31
|
+
if state.in_repl:
|
|
32
|
+
log2('Specify export database name.')
|
|
33
|
+
else:
|
|
34
|
+
log2('* Database name is missing.')
|
|
38
35
|
|
|
39
|
-
|
|
36
|
+
Command.display_help()
|
|
40
37
|
|
|
41
|
-
|
|
38
|
+
return 'command-missing'
|
|
42
39
|
|
|
43
|
-
|
|
40
|
+
ExportDatabases.disply_export_session(state.sts, state.pod, state.namespace, args[0])
|
|
44
41
|
|
|
45
|
-
|
|
42
|
+
return state
|
|
46
43
|
|
|
47
44
|
def completion(self, state: ReplState):
|
|
48
45
|
return super().completion(state, {session: None for session in Exporter.export_session_names(state.sts, state.pod, state.namespace)})
|
|
@@ -26,19 +26,16 @@ class ShowExportSessions(Command):
|
|
|
26
26
|
if not(args := self.args(cmd)):
|
|
27
27
|
return super().run(cmd, state)
|
|
28
28
|
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
pod = state.pod
|
|
34
|
-
if not pod:
|
|
35
|
-
pod = StatefulSets.pod_names(state.sts, state.namespace)[0]
|
|
29
|
+
with self.validate(args, state) as (args, state):
|
|
30
|
+
pod = state.pod
|
|
31
|
+
if not pod:
|
|
32
|
+
pod = StatefulSets.pod_names(state.sts, state.namespace)[0]
|
|
36
33
|
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
34
|
+
sessions: dict[str, str] = Exporter.find_export_sessions(pod, state.namespace)
|
|
35
|
+
log(lines_to_tabular([f'{session}\t{export_state}' for session, export_state in sorted(sessions.items(), reverse=True)],
|
|
36
|
+
header='EXPORT_SESSION\tSTATUS', separator='\t'))
|
|
40
37
|
|
|
41
|
-
|
|
38
|
+
return state
|
|
42
39
|
|
|
43
40
|
def completion(self, state: ReplState):
|
|
44
41
|
return super().completion(state)
|
|
@@ -3,8 +3,10 @@ import re
|
|
|
3
3
|
|
|
4
4
|
from adam.config import Config
|
|
5
5
|
from adam.pod_exec_result import PodExecResult
|
|
6
|
+
from adam.repl_state import ReplState
|
|
6
7
|
from adam.utils_k8s.cassandra_nodes import CassandraNodes
|
|
7
8
|
from adam.utils_k8s.pods import log_prefix
|
|
9
|
+
from adam.utils_k8s.statefulsets import StatefulSets
|
|
8
10
|
|
|
9
11
|
class ImportSpec:
|
|
10
12
|
def __init__(self, session: str, importer: str):
|
|
@@ -288,4 +290,25 @@ class GeneratorStream(io.RawIOBase):
|
|
|
288
290
|
|
|
289
291
|
data = self._buffer[:size]
|
|
290
292
|
self._buffer = self._buffer[size:]
|
|
291
|
-
return data
|
|
293
|
+
return data
|
|
294
|
+
|
|
295
|
+
class PodHandler:
|
|
296
|
+
def __init__(self, state: ReplState):
|
|
297
|
+
self.state = state
|
|
298
|
+
|
|
299
|
+
def __enter__(self):
|
|
300
|
+
state = self.state
|
|
301
|
+
|
|
302
|
+
if not state.pod:
|
|
303
|
+
state.push()
|
|
304
|
+
state.pod = StatefulSets.pod_names(state.sts, state.namespace)[0]
|
|
305
|
+
|
|
306
|
+
return state
|
|
307
|
+
|
|
308
|
+
def __exit__(self, exc_type, exc_val, exc_tb):
|
|
309
|
+
self.state.pop()
|
|
310
|
+
|
|
311
|
+
return False
|
|
312
|
+
|
|
313
|
+
def state_with_pod(state: ReplState):
|
|
314
|
+
return PodHandler(state)
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
from abc import abstractmethod
|
|
2
|
+
|
|
3
|
+
from adam.commands.command import Command
|
|
4
|
+
from adam.commands.command_helpers import ClusterCommandHelper
|
|
5
|
+
from adam.repl_state import ReplState
|
|
6
|
+
from adam.utils import lines_to_tabular, log, log2
|
|
7
|
+
|
|
8
|
+
class IntermediateCommand(Command):
|
|
9
|
+
def run(self, cmd: str, state: ReplState):
|
|
10
|
+
if not(args := self.args(cmd)):
|
|
11
|
+
return super().run(cmd, state)
|
|
12
|
+
|
|
13
|
+
return self.intermediate_run(cmd, state, args, self.cmd_list())
|
|
14
|
+
|
|
15
|
+
@abstractmethod
|
|
16
|
+
def cmd_list(self):
|
|
17
|
+
pass
|
|
18
|
+
|
|
19
|
+
def intermediate_run(self, cmd: str, state: ReplState, args: list[str], cmds: list['Command'], separator='\t', display_help=True):
|
|
20
|
+
state, _ = self.apply_state(args, state)
|
|
21
|
+
|
|
22
|
+
if state.in_repl:
|
|
23
|
+
if display_help:
|
|
24
|
+
log(lines_to_tabular([c.help(state) for c in cmds], separator=separator))
|
|
25
|
+
|
|
26
|
+
return 'command-missing'
|
|
27
|
+
else:
|
|
28
|
+
# head with the Chain of Responsibility pattern
|
|
29
|
+
if not self.run_subcommand(cmd, state):
|
|
30
|
+
if display_help:
|
|
31
|
+
log2('* Command is missing.')
|
|
32
|
+
Command.display_help()
|
|
33
|
+
return 'command-missing'
|
|
34
|
+
|
|
35
|
+
return state
|
|
36
|
+
|
|
37
|
+
def run_subcommand(self, cmd: str, state: ReplState):
|
|
38
|
+
cmds = Command.chain(self.cmd_list())
|
|
39
|
+
return cmds.run(cmd, state)
|
|
40
|
+
|
|
41
|
+
def intermediate_help(super_help: str, cmd: str, cmd_list: list['Command'], separator='\t', show_cluster_help=False):
|
|
42
|
+
log(super_help)
|
|
43
|
+
log()
|
|
44
|
+
log('Sub-Commands:')
|
|
45
|
+
|
|
46
|
+
log(lines_to_tabular([c.help(ReplState()).replace(f'{cmd} ', ' ', 1) for c in cmd_list], separator=separator))
|
|
47
|
+
if show_cluster_help:
|
|
48
|
+
log()
|
|
49
|
+
ClusterCommandHelper.cluster_help()
|
adam/commands/issues.py
CHANGED
|
@@ -1,10 +1,9 @@
|
|
|
1
1
|
from adam.checks.check_result import CheckResult
|
|
2
2
|
from adam.checks.check_utils import run_checks
|
|
3
|
-
from adam.
|
|
3
|
+
from adam.commands import extract_options
|
|
4
4
|
from adam.commands.command import Command
|
|
5
|
-
from adam.repl_session import ReplSession
|
|
6
5
|
from adam.repl_state import ReplState
|
|
7
|
-
from adam.
|
|
6
|
+
from adam.utils_issues import IssuesUtils
|
|
8
7
|
|
|
9
8
|
class Issues(Command):
|
|
10
9
|
COMMAND = 'issues'
|
|
@@ -28,48 +27,17 @@ class Issues(Command):
|
|
|
28
27
|
if not(args := self.args(cmd)):
|
|
29
28
|
return super().run(cmd, state)
|
|
30
29
|
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
30
|
+
with self.validate(args, state) as (args, state):
|
|
31
|
+
with extract_options(args, ['-s', '--show']) as (args, show_out):
|
|
32
|
+
results = run_checks(state.sts, state.namespace, state.pod, show_out=show_out)
|
|
34
33
|
|
|
35
|
-
|
|
34
|
+
issues = CheckResult.collect_issues(results)
|
|
35
|
+
IssuesUtils.show_issues(issues, in_repl=state.in_repl)
|
|
36
36
|
|
|
37
|
-
|
|
37
|
+
return issues if issues else 'issues'
|
|
38
38
|
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
return issues if issues else 'issues'
|
|
43
|
-
|
|
44
|
-
def show(check_results: list[CheckResult], in_repl = False):
|
|
45
|
-
Issues.show_issues(CheckResult.collect_issues(check_results), in_repl=in_repl)
|
|
46
|
-
|
|
47
|
-
def show_issues(issues: list[Issue], in_repl = False):
|
|
48
|
-
if not issues:
|
|
49
|
-
log2('No issues found.')
|
|
50
|
-
else:
|
|
51
|
-
suggested = 0
|
|
52
|
-
log2(f'* {len(issues)} issues found.')
|
|
53
|
-
lines = []
|
|
54
|
-
for i, issue in enumerate(issues, start=1):
|
|
55
|
-
lines.append(f"{i}||{issue.category}||{issue.desc}")
|
|
56
|
-
lines.append(f"||statefulset||{issue.statefulset}@{issue.namespace}")
|
|
57
|
-
lines.append(f"||pod||{issue.pod}@{issue.namespace}")
|
|
58
|
-
if issue.details:
|
|
59
|
-
lines.append(f"||details||{issue.details}")
|
|
60
|
-
|
|
61
|
-
if issue.suggestion:
|
|
62
|
-
lines.append(f'||suggestion||{issue.suggestion}')
|
|
63
|
-
if in_repl:
|
|
64
|
-
ReplSession().prompt_session.history.append_string(issue.suggestion)
|
|
65
|
-
suggested += 1
|
|
66
|
-
log(lines_to_tabular(lines, separator='||'))
|
|
67
|
-
if suggested:
|
|
68
|
-
log2()
|
|
69
|
-
log2(f'* {suggested} suggested commands are added to history. Press <Up> arrow to access them.')
|
|
70
|
-
|
|
71
|
-
def completion(self, _: ReplState):
|
|
72
|
-
return {Issues.COMMAND: None}
|
|
39
|
+
def completion(self, state: ReplState):
|
|
40
|
+
return super().completion(state, {'-s': None})
|
|
73
41
|
|
|
74
42
|
def help(self, _: ReplState):
|
|
75
|
-
return f'{Issues.COMMAND}\t find all issues'
|
|
43
|
+
return f'{Issues.COMMAND} [-s]\t find all issues'
|