kaqing 2.0.115__py3-none-any.whl → 2.0.172__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of kaqing might be problematic. Click here for more details.
- adam/__init__.py +0 -2
- adam/app_session.py +8 -11
- adam/batch.py +3 -3
- adam/checks/check_utils.py +14 -46
- adam/checks/cpu.py +7 -1
- adam/checks/cpu_metrics.py +52 -0
- adam/checks/disk.py +2 -3
- adam/columns/columns.py +3 -1
- adam/columns/cpu.py +3 -1
- adam/columns/cpu_metrics.py +22 -0
- adam/columns/memory.py +3 -4
- adam/commands/__init__.py +18 -0
- adam/commands/alter_tables.py +43 -47
- adam/commands/audit/audit.py +24 -25
- adam/commands/audit/audit_repair_tables.py +14 -17
- adam/commands/audit/audit_run.py +15 -23
- adam/commands/audit/show_last10.py +10 -13
- adam/commands/audit/show_slow10.py +10 -13
- adam/commands/audit/show_top10.py +10 -14
- adam/commands/audit/utils_show_top10.py +2 -3
- adam/commands/bash/__init__.py +5 -0
- adam/commands/bash/bash.py +8 -96
- adam/commands/bash/utils_bash.py +16 -0
- adam/commands/cat.py +14 -19
- adam/commands/cd.py +12 -100
- adam/commands/check.py +20 -21
- adam/commands/cli_commands.py +2 -3
- adam/commands/code.py +20 -23
- adam/commands/command.py +123 -39
- adam/commands/commands_utils.py +8 -17
- adam/commands/cp.py +33 -39
- adam/commands/cql/cql_completions.py +28 -10
- adam/commands/cql/cqlsh.py +10 -30
- adam/commands/cql/utils_cql.py +343 -0
- adam/commands/deploy/code_start.py +7 -10
- adam/commands/deploy/code_stop.py +4 -21
- adam/commands/deploy/code_utils.py +3 -3
- adam/commands/deploy/deploy.py +4 -27
- adam/commands/deploy/deploy_frontend.py +14 -17
- adam/commands/deploy/deploy_pg_agent.py +2 -5
- adam/commands/deploy/deploy_pod.py +65 -73
- adam/commands/deploy/deploy_utils.py +14 -24
- adam/commands/deploy/undeploy.py +4 -27
- adam/commands/deploy/undeploy_frontend.py +4 -7
- adam/commands/deploy/undeploy_pg_agent.py +5 -7
- adam/commands/deploy/undeploy_pod.py +11 -12
- adam/commands/devices/__init__.py +0 -0
- adam/commands/devices/device.py +118 -0
- adam/commands/devices/device_app.py +173 -0
- adam/commands/devices/device_auit_log.py +49 -0
- adam/commands/devices/device_cass.py +185 -0
- adam/commands/devices/device_export.py +86 -0
- adam/commands/devices/device_postgres.py +144 -0
- adam/commands/devices/devices.py +25 -0
- adam/commands/exit.py +1 -4
- adam/commands/export/clean_up_all_export_sessions.py +37 -0
- adam/commands/export/clean_up_export_sessions.py +51 -0
- adam/commands/export/drop_export_database.py +55 -0
- adam/commands/export/drop_export_databases.py +43 -0
- adam/commands/export/export.py +19 -26
- adam/commands/export/export_databases.py +174 -0
- adam/commands/export/export_handlers.py +71 -0
- adam/commands/export/export_select.py +48 -22
- adam/commands/export/export_select_x.py +54 -0
- adam/commands/export/export_use.py +19 -23
- adam/commands/export/exporter.py +353 -0
- adam/commands/export/import_session.py +40 -0
- adam/commands/export/importer.py +67 -0
- adam/commands/export/importer_athena.py +77 -0
- adam/commands/export/importer_sqlite.py +39 -0
- adam/commands/export/show_column_counts.py +54 -0
- adam/commands/export/show_export_databases.py +36 -0
- adam/commands/export/show_export_session.py +48 -0
- adam/commands/export/show_export_sessions.py +44 -0
- adam/commands/export/utils_export.py +223 -162
- adam/commands/help.py +1 -1
- adam/commands/intermediate_command.py +49 -0
- adam/commands/issues.py +11 -43
- adam/commands/kubectl.py +3 -6
- adam/commands/login.py +22 -24
- adam/commands/logs.py +3 -6
- adam/commands/ls.py +11 -128
- adam/commands/medusa/medusa.py +4 -22
- adam/commands/medusa/medusa_backup.py +20 -24
- adam/commands/medusa/medusa_restore.py +29 -33
- adam/commands/medusa/medusa_show_backupjobs.py +14 -18
- adam/commands/medusa/medusa_show_restorejobs.py +11 -18
- adam/commands/nodetool.py +6 -15
- adam/commands/param_get.py +11 -12
- adam/commands/param_set.py +9 -10
- adam/commands/postgres/postgres.py +41 -34
- adam/commands/postgres/postgres_context.py +57 -24
- adam/commands/postgres/postgres_ls.py +4 -8
- adam/commands/postgres/postgres_preview.py +5 -9
- adam/commands/postgres/psql_completions.py +1 -1
- adam/commands/postgres/utils_postgres.py +66 -0
- adam/commands/preview_table.py +5 -44
- adam/commands/pwd.py +14 -47
- adam/commands/reaper/reaper.py +4 -27
- adam/commands/reaper/reaper_forward.py +48 -55
- adam/commands/reaper/reaper_forward_session.py +6 -0
- adam/commands/reaper/reaper_forward_stop.py +10 -16
- adam/commands/reaper/reaper_restart.py +7 -14
- adam/commands/reaper/reaper_run_abort.py +11 -30
- adam/commands/reaper/reaper_runs.py +42 -57
- adam/commands/reaper/reaper_runs_abort.py +29 -49
- adam/commands/reaper/reaper_schedule_activate.py +11 -30
- adam/commands/reaper/reaper_schedule_start.py +10 -29
- adam/commands/reaper/reaper_schedule_stop.py +10 -29
- adam/commands/reaper/reaper_schedules.py +4 -14
- adam/commands/reaper/reaper_status.py +8 -16
- adam/commands/reaper/utils_reaper.py +196 -0
- adam/commands/repair/repair.py +4 -22
- adam/commands/repair/repair_log.py +5 -11
- adam/commands/repair/repair_run.py +27 -34
- adam/commands/repair/repair_scan.py +32 -38
- adam/commands/repair/repair_stop.py +5 -11
- adam/commands/report.py +27 -29
- adam/commands/restart.py +25 -26
- adam/commands/rollout.py +19 -24
- adam/commands/shell.py +10 -4
- adam/commands/show/show.py +10 -25
- adam/commands/show/show_cassandra_repairs.py +35 -0
- adam/commands/show/show_cassandra_status.py +32 -43
- adam/commands/show/show_cassandra_version.py +5 -18
- adam/commands/show/show_commands.py +19 -24
- adam/commands/show/show_host.py +1 -1
- adam/commands/show/show_login.py +20 -27
- adam/commands/show/show_processes.py +15 -19
- adam/commands/show/show_storage.py +10 -20
- adam/commands/watch.py +26 -29
- adam/config.py +5 -14
- adam/embedded_params.py +1 -1
- adam/log.py +4 -4
- adam/pod_exec_result.py +3 -3
- adam/repl.py +40 -103
- adam/repl_commands.py +32 -16
- adam/repl_state.py +57 -28
- adam/sql/sql_completer.py +44 -28
- adam/sql/sql_state_machine.py +89 -28
- adam/sso/authn_ad.py +6 -8
- adam/sso/authn_okta.py +4 -6
- adam/sso/cred_cache.py +3 -5
- adam/sso/idp.py +9 -12
- adam/utils.py +435 -6
- adam/utils_athena.py +57 -37
- adam/utils_audits.py +12 -14
- adam/utils_issues.py +32 -0
- adam/utils_k8s/app_clusters.py +13 -18
- adam/utils_k8s/app_pods.py +2 -0
- adam/utils_k8s/cassandra_clusters.py +22 -19
- adam/utils_k8s/cassandra_nodes.py +2 -2
- adam/utils_k8s/custom_resources.py +16 -17
- adam/utils_k8s/ingresses.py +2 -2
- adam/utils_k8s/jobs.py +7 -11
- adam/utils_k8s/k8s.py +87 -0
- adam/utils_k8s/pods.py +40 -77
- adam/utils_k8s/secrets.py +4 -4
- adam/utils_k8s/service_accounts.py +5 -4
- adam/utils_k8s/services.py +2 -2
- adam/utils_k8s/statefulsets.py +1 -12
- adam/utils_net.py +4 -4
- adam/utils_repl/__init__.py +0 -0
- adam/utils_repl/automata_completer.py +48 -0
- adam/utils_repl/repl_completer.py +46 -0
- adam/utils_repl/state_machine.py +173 -0
- adam/utils_sqlite.py +137 -0
- adam/version.py +1 -1
- {kaqing-2.0.115.dist-info → kaqing-2.0.172.dist-info}/METADATA +1 -1
- kaqing-2.0.172.dist-info/RECORD +230 -0
- adam/commands/app.py +0 -67
- adam/commands/app_ping.py +0 -44
- adam/commands/cql/cql_utils.py +0 -204
- adam/commands/devices.py +0 -147
- adam/commands/export/export_on_x.py +0 -76
- adam/commands/export/export_rmdbs.py +0 -65
- adam/commands/postgres/postgres_utils.py +0 -31
- adam/commands/reaper/reaper_session.py +0 -159
- adam/commands/show/show_app_actions.py +0 -56
- adam/commands/show/show_app_id.py +0 -47
- adam/commands/show/show_app_queues.py +0 -45
- adam/commands/show/show_repairs.py +0 -47
- adam/utils_export.py +0 -42
- kaqing-2.0.115.dist-info/RECORD +0 -203
- {kaqing-2.0.115.dist-info → kaqing-2.0.172.dist-info}/WHEEL +0 -0
- {kaqing-2.0.115.dist-info → kaqing-2.0.172.dist-info}/entry_points.txt +0 -0
- {kaqing-2.0.115.dist-info → kaqing-2.0.172.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
from adam.commands.command import Command
|
|
2
|
+
from adam.commands.export.export_databases import ExportDatabases
|
|
3
|
+
from adam.commands.export.exporter import Exporter
|
|
4
|
+
from adam.repl_state import ReplState, RequiredState
|
|
5
|
+
from adam.utils import log2
|
|
6
|
+
|
|
7
|
+
class ShowExportSession(Command):
|
|
8
|
+
COMMAND = 'show export session'
|
|
9
|
+
|
|
10
|
+
# the singleton pattern
|
|
11
|
+
def __new__(cls, *args, **kwargs):
|
|
12
|
+
if not hasattr(cls, 'instance'): cls.instance = super(ShowExportSession, cls).__new__(cls)
|
|
13
|
+
|
|
14
|
+
return cls.instance
|
|
15
|
+
|
|
16
|
+
def __init__(self, successor: Command=None):
|
|
17
|
+
super().__init__(successor)
|
|
18
|
+
|
|
19
|
+
def command(self):
|
|
20
|
+
return ShowExportSession.COMMAND
|
|
21
|
+
|
|
22
|
+
def required(self):
|
|
23
|
+
return RequiredState.CLUSTER_OR_POD
|
|
24
|
+
|
|
25
|
+
def run(self, cmd: str, state: ReplState):
|
|
26
|
+
if not(args := self.args(cmd)):
|
|
27
|
+
return super().run(cmd, state)
|
|
28
|
+
|
|
29
|
+
with self.validate(args, state) as (args, state):
|
|
30
|
+
if not args:
|
|
31
|
+
if state.in_repl:
|
|
32
|
+
log2('Specify export database name.')
|
|
33
|
+
else:
|
|
34
|
+
log2('* Database name is missing.')
|
|
35
|
+
|
|
36
|
+
Command.display_help()
|
|
37
|
+
|
|
38
|
+
return 'command-missing'
|
|
39
|
+
|
|
40
|
+
ExportDatabases.disply_export_session(state.sts, state.pod, state.namespace, args[0])
|
|
41
|
+
|
|
42
|
+
return state
|
|
43
|
+
|
|
44
|
+
def completion(self, state: ReplState):
|
|
45
|
+
return super().completion(state, {session: None for session in Exporter.export_session_names(state.sts, state.pod, state.namespace)})
|
|
46
|
+
|
|
47
|
+
def help(self, _: ReplState):
|
|
48
|
+
return f'{ShowExportSession.COMMAND} <export-session-name>\t show export session'
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
from adam.commands.command import Command
|
|
2
|
+
from adam.commands.export.exporter import Exporter
|
|
3
|
+
from adam.repl_state import ReplState, RequiredState
|
|
4
|
+
from adam.utils import lines_to_tabular, log
|
|
5
|
+
from adam.utils_k8s.statefulsets import StatefulSets
|
|
6
|
+
|
|
7
|
+
class ShowExportSessions(Command):
|
|
8
|
+
COMMAND = 'show export sessions'
|
|
9
|
+
|
|
10
|
+
# the singleton pattern
|
|
11
|
+
def __new__(cls, *args, **kwargs):
|
|
12
|
+
if not hasattr(cls, 'instance'): cls.instance = super(ShowExportSessions, cls).__new__(cls)
|
|
13
|
+
|
|
14
|
+
return cls.instance
|
|
15
|
+
|
|
16
|
+
def __init__(self, successor: Command=None):
|
|
17
|
+
super().__init__(successor)
|
|
18
|
+
|
|
19
|
+
def command(self):
|
|
20
|
+
return ShowExportSessions.COMMAND
|
|
21
|
+
|
|
22
|
+
def required(self):
|
|
23
|
+
return RequiredState.CLUSTER_OR_POD
|
|
24
|
+
|
|
25
|
+
def run(self, cmd: str, state: ReplState):
|
|
26
|
+
if not(args := self.args(cmd)):
|
|
27
|
+
return super().run(cmd, state)
|
|
28
|
+
|
|
29
|
+
with self.validate(args, state) as (args, state):
|
|
30
|
+
pod = state.pod
|
|
31
|
+
if not pod:
|
|
32
|
+
pod = StatefulSets.pod_names(state.sts, state.namespace)[0]
|
|
33
|
+
|
|
34
|
+
sessions: dict[str, str] = Exporter.find_export_sessions(pod, state.namespace)
|
|
35
|
+
log(lines_to_tabular([f'{session}\t{export_state}' for session, export_state in sorted(sessions.items(), reverse=True)],
|
|
36
|
+
header='EXPORT_SESSION\tSTATUS', separator='\t'))
|
|
37
|
+
|
|
38
|
+
return state
|
|
39
|
+
|
|
40
|
+
def completion(self, state: ReplState):
|
|
41
|
+
return super().completion(state)
|
|
42
|
+
|
|
43
|
+
def help(self, _: ReplState):
|
|
44
|
+
return f'{ShowExportSessions.COMMAND}\t list export sessions'
|
|
@@ -1,54 +1,122 @@
|
|
|
1
|
-
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
2
|
-
from datetime import datetime
|
|
3
1
|
import io
|
|
4
2
|
import re
|
|
5
|
-
import time
|
|
6
|
-
from typing import Callable
|
|
7
|
-
import boto3
|
|
8
3
|
|
|
9
|
-
from adam.commands.cql.cql_utils import cassandra_table_names, run_cql, table_spec
|
|
10
4
|
from adam.config import Config
|
|
5
|
+
from adam.pod_exec_result import PodExecResult
|
|
11
6
|
from adam.repl_state import ReplState
|
|
12
|
-
from adam.utils import elapsed_time, log2
|
|
13
|
-
from adam.utils_athena import Athena
|
|
14
7
|
from adam.utils_k8s.cassandra_nodes import CassandraNodes
|
|
15
|
-
from adam.utils_k8s.pods import
|
|
8
|
+
from adam.utils_k8s.pods import log_prefix
|
|
9
|
+
from adam.utils_k8s.statefulsets import StatefulSets
|
|
16
10
|
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
11
|
+
class ImportSpec:
|
|
12
|
+
def __init__(self, session: str, importer: str):
|
|
13
|
+
self.session = session
|
|
14
|
+
self.importer = importer
|
|
20
15
|
|
|
21
|
-
|
|
22
|
-
|
|
16
|
+
def parse_specs(specs_str: str):
|
|
17
|
+
session: str = None
|
|
18
|
+
importer: str = None
|
|
23
19
|
|
|
24
|
-
|
|
25
|
-
|
|
20
|
+
if specs_str:
|
|
21
|
+
importer, session = ImportSpec._extract_importer(specs_str.strip(' '))
|
|
26
22
|
|
|
27
|
-
|
|
28
|
-
max_workers = Config().action_workers('export', 8)
|
|
23
|
+
return ImportSpec(session, importer)
|
|
29
24
|
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
25
|
+
def _extract_importer(spec_str: str) -> tuple[str, str]:
|
|
26
|
+
importer = None
|
|
27
|
+
rest = spec_str
|
|
28
|
+
|
|
29
|
+
p = re.compile(r"(.*?)to\s+(.*)", re.IGNORECASE)
|
|
30
|
+
match = p.match(spec_str)
|
|
31
|
+
if match:
|
|
32
|
+
rest = match.group(1).strip(' ')
|
|
33
|
+
importer = match.group(2).strip(' ')
|
|
34
|
+
|
|
35
|
+
return importer, rest
|
|
36
|
+
|
|
37
|
+
class ExportSpec(ImportSpec):
|
|
38
|
+
def __init__(self, keyspace: str, consistency: str, importer: str, tables: list['ExportTableSpec'], session: str = None):
|
|
39
|
+
super().__init__(None, importer)
|
|
40
|
+
|
|
41
|
+
self.keyspace = keyspace
|
|
42
|
+
self.consistency = consistency
|
|
43
|
+
self.tables = tables
|
|
44
|
+
self.session = session
|
|
45
|
+
|
|
46
|
+
def __str__(self):
|
|
47
|
+
return f'keyspace: {self.keyspace}, consistency: {self.consistency}, importer: {self.importer}, tables: {",".join([t.table for t in self.tables])}, session: {self.session}'
|
|
48
|
+
|
|
49
|
+
def __eq__(self, other):
|
|
50
|
+
if not isinstance(other, ExportSpec):
|
|
51
|
+
return NotImplemented
|
|
52
|
+
|
|
53
|
+
return self.keyspace == other.keyspace and self.tables == other.tables and self.consistency == other.consistency and self.importer == other.importer and self.session == other.session
|
|
54
|
+
|
|
55
|
+
def parse_specs(specs_str: str):
|
|
56
|
+
keyspace: str = None
|
|
57
|
+
consistency: str = None
|
|
58
|
+
importer: str = None
|
|
59
|
+
specs: list[ExportTableSpec] = None
|
|
60
|
+
|
|
61
|
+
if specs_str:
|
|
62
|
+
importer, specs_str = ExportSpec._extract_importer(specs_str.strip(' '))
|
|
63
|
+
keyspace, specs_str = ExportSpec._extract_keyspace(specs_str)
|
|
64
|
+
consistency, specs = ExportSpec._extract_consisteny(specs_str)
|
|
65
|
+
|
|
66
|
+
return ExportSpec(keyspace, consistency, importer, specs)
|
|
67
|
+
|
|
68
|
+
def _extract_keyspace(spec_str: str) -> tuple[str, str]:
|
|
69
|
+
keyspace = None
|
|
70
|
+
rest = spec_str
|
|
71
|
+
|
|
72
|
+
p = re.compile(r"\s*\*\s+in\s+(\S+)(.*)", re.IGNORECASE)
|
|
73
|
+
match = p.match(spec_str)
|
|
74
|
+
if match:
|
|
75
|
+
keyspace = match.group(1).strip(' ')
|
|
76
|
+
rest = match.group(2).strip(' ')
|
|
77
|
+
elif spec_str.startswith('*'):
|
|
78
|
+
keyspace = '*'
|
|
79
|
+
rest = spec_str[1:].strip(' ')
|
|
80
|
+
|
|
81
|
+
return keyspace, rest
|
|
82
|
+
|
|
83
|
+
def _extract_consisteny(spec_str: str) -> tuple[str, list['ExportTableSpec']]:
|
|
84
|
+
consistency = None
|
|
85
|
+
|
|
86
|
+
p = re.compile(r"(.*?)with\s+consistency\s+(.*)", re.IGNORECASE)
|
|
87
|
+
match = p.match(spec_str)
|
|
88
|
+
if match:
|
|
89
|
+
spec_str = match.group(1).strip(' ')
|
|
90
|
+
consistency = match.group(2)
|
|
91
|
+
|
|
92
|
+
if spec_str:
|
|
93
|
+
p = r",\s*(?![^()]*\))"
|
|
94
|
+
specs = re.split(p, spec_str)
|
|
95
|
+
|
|
96
|
+
return consistency, [ExportTableSpec.parse(spec) for spec in specs]
|
|
97
|
+
|
|
98
|
+
return consistency, None
|
|
44
99
|
|
|
45
|
-
class
|
|
46
|
-
def __init__(self, table: str, columns: str = None, target_table: str = None):
|
|
100
|
+
class ExportTableSpec:
|
|
101
|
+
def __init__(self, keyspace: str, table: str, columns: str = None, target_table: str = None):
|
|
102
|
+
self.keyspace = keyspace
|
|
47
103
|
self.table = table
|
|
48
104
|
self.columns = columns
|
|
49
105
|
self.target_table = target_table
|
|
50
106
|
|
|
51
|
-
def
|
|
107
|
+
def __str__(self):
|
|
108
|
+
return f'{self.keyspace}.{self.table}({self.columns}) AS {self.target_table}'
|
|
109
|
+
|
|
110
|
+
def __eq__(self, other):
|
|
111
|
+
if not isinstance(other, ExportTableSpec):
|
|
112
|
+
return NotImplemented
|
|
113
|
+
|
|
114
|
+
return self.keyspace == other.keyspace and self.table == other.table and self.columns == other.columns and self.target_table == other.target_table
|
|
115
|
+
|
|
116
|
+
def from_status(status: 'ExportTableStatus'):
|
|
117
|
+
return ExportTableSpec(status.keyspace, status.table, target_table=status.target_table)
|
|
118
|
+
|
|
119
|
+
def parse(spec_str: str) -> 'ExportTableSpec':
|
|
52
120
|
target = None
|
|
53
121
|
|
|
54
122
|
p = re.compile(r"(.*?)\s+as\s+(.*)", re.IGNORECASE)
|
|
@@ -57,148 +125,120 @@ class ExportSpec:
|
|
|
57
125
|
spec_str = match.group(1)
|
|
58
126
|
target = match.group(2)
|
|
59
127
|
|
|
128
|
+
keyspace = None
|
|
60
129
|
table = spec_str
|
|
61
130
|
columns = None
|
|
62
131
|
|
|
63
|
-
p = re.compile('(.*?)\((.*)\)')
|
|
132
|
+
p = re.compile('(.*?)\.(.*?)\((.*)\)')
|
|
64
133
|
match = p.match(spec_str)
|
|
65
134
|
if match:
|
|
66
|
-
|
|
67
|
-
|
|
135
|
+
keyspace = match.group(1)
|
|
136
|
+
table = match.group(2)
|
|
137
|
+
columns = match.group(3)
|
|
138
|
+
else:
|
|
139
|
+
p = re.compile('(.*?)\.(.*)')
|
|
140
|
+
match = p.match(spec_str)
|
|
141
|
+
if match:
|
|
142
|
+
keyspace = match.group(1)
|
|
143
|
+
table = match.group(2)
|
|
68
144
|
|
|
69
|
-
return
|
|
145
|
+
return ExportTableSpec(keyspace, table, columns, target)
|
|
70
146
|
|
|
71
147
|
def __eq__(self, other):
|
|
72
|
-
if isinstance(other,
|
|
73
|
-
return self.table == other.table and self.columns == other.columns and self.target_table == other.target_table
|
|
148
|
+
if isinstance(other, ExportTableSpec):
|
|
149
|
+
return self.keyspace == other.keyspace and self.table == other.table and self.columns == other.columns and self.target_table == other.target_table
|
|
74
150
|
|
|
75
151
|
return False
|
|
76
152
|
|
|
77
153
|
def __str__(self):
|
|
78
|
-
return f'{self.table}
|
|
154
|
+
return f'{self.keyspace}.{self.table}({self.columns}) as {self.target_table}'
|
|
79
155
|
|
|
80
|
-
|
|
81
|
-
|
|
156
|
+
class ExportTableStatus:
|
|
157
|
+
def __init__(self, keyspace: str, target_table: str, status: str, table: str = None):
|
|
158
|
+
self.keyspace = keyspace
|
|
159
|
+
self.target_table = target_table
|
|
160
|
+
self.status = status
|
|
161
|
+
self.table = table
|
|
82
162
|
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
if match:
|
|
86
|
-
spec_str = match.group(1).strip(' ')
|
|
87
|
-
consistency = match.group(2)
|
|
163
|
+
def __str__(self):
|
|
164
|
+
return f'{self.keyspace}.{self.table} as {self.target_table} = {self.status}'
|
|
88
165
|
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
166
|
+
def __eq__(self, other):
|
|
167
|
+
if isinstance(other, ExportTableStatus):
|
|
168
|
+
return self.keyspace == other.keyspace and self.table == other.table and self.status == other.status and self.target_table == other.target_table
|
|
92
169
|
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
' "quoteChar" = "\\"")\n' + \
|
|
166
|
-
f"LOCATION 's3://c3.ops--qing/export/{session}/{athena_table}'\n" + \
|
|
167
|
-
'TBLPROPERTIES ("skip.header.line.count"="1");'
|
|
168
|
-
if Config().is_debug():
|
|
169
|
-
log2(query)
|
|
170
|
-
try:
|
|
171
|
-
Athena.query(query, db)
|
|
172
|
-
except Exception as e:
|
|
173
|
-
log2(f'*** Failed query:\n{query}')
|
|
174
|
-
raise e
|
|
175
|
-
|
|
176
|
-
ing(f"Creating database {db}" if create_db else f"Creating table {athena_table}", create_schema, suppress_log=suppress_ing_log)
|
|
177
|
-
|
|
178
|
-
succeeded = True
|
|
179
|
-
except Exception as e:
|
|
180
|
-
log2(e)
|
|
181
|
-
finally:
|
|
182
|
-
ing('Cleaning up temporary files',
|
|
183
|
-
lambda: CassandraNodes.exec(state.pod, state.namespace, f'rm -rf {csv_file}', show_out=False, shell='bash'),
|
|
184
|
-
suppress_log=suppress_ing_log)
|
|
185
|
-
|
|
186
|
-
if succeeded:
|
|
187
|
-
Athena.clear_cache()
|
|
188
|
-
|
|
189
|
-
if not suppress_ing_log:
|
|
190
|
-
query = f'select * from {athena_table} limit 10'
|
|
191
|
-
log2(query)
|
|
192
|
-
Athena.run_query(query, db)
|
|
193
|
-
|
|
194
|
-
return table
|
|
195
|
-
|
|
196
|
-
def ing(msg: str, body: Callable[[], None], suppress_log=False):
|
|
197
|
-
if not suppress_log:
|
|
198
|
-
log2(f'{msg}...', nl=False)
|
|
199
|
-
body()
|
|
200
|
-
if not suppress_log:
|
|
201
|
-
log2(' OK')
|
|
170
|
+
return False
|
|
171
|
+
|
|
172
|
+
def from_session(sts: str, pod: str, namespace: str, export_session: str):
|
|
173
|
+
statuses: list[ExportTableStatus] = []
|
|
174
|
+
|
|
175
|
+
status_in_whole = 'done'
|
|
176
|
+
log_files: list[str] = find_files(pod, namespace, f'{log_prefix()}-{export_session}_*.log*')
|
|
177
|
+
|
|
178
|
+
for log_file in log_files:
|
|
179
|
+
status: ExportTableStatus = ExportTableStatus.from_log_file(pod, namespace, export_session, log_file)
|
|
180
|
+
statuses.append(status)
|
|
181
|
+
|
|
182
|
+
if status.status != 'done':
|
|
183
|
+
status_in_whole = status.status
|
|
184
|
+
|
|
185
|
+
return statuses, status_in_whole
|
|
186
|
+
|
|
187
|
+
def from_log_file(pod: str, namespace: str, copy_session: str, log_file: str):
|
|
188
|
+
def get_csv_files_n_table(target_table: str):
|
|
189
|
+
db = f'{copy_session}_{target_table}'
|
|
190
|
+
csv_file = f'{csv_dir()}/{db}/*.csv'
|
|
191
|
+
csv_files: list[str] = find_files(pod, namespace, csv_file)
|
|
192
|
+
if csv_files:
|
|
193
|
+
table = target_table
|
|
194
|
+
m = re.match(f'{csv_dir()}/{db}/(.*).csv', csv_files[0])
|
|
195
|
+
if m:
|
|
196
|
+
table = m.group(1)
|
|
197
|
+
return csv_files, table
|
|
198
|
+
|
|
199
|
+
return csv_files, target_table
|
|
200
|
+
|
|
201
|
+
m = re.match(f'{log_prefix()}-{copy_session}_(.*?)\.(.*?)\.log(.*)', log_file)
|
|
202
|
+
if m:
|
|
203
|
+
keyspace = m.group(1)
|
|
204
|
+
target_table = m.group(2)
|
|
205
|
+
state = m.group(3)
|
|
206
|
+
if state == '.pending_import':
|
|
207
|
+
_, table = get_csv_files_n_table(target_table)
|
|
208
|
+
return ExportTableStatus(keyspace, target_table, 'pending_import', table)
|
|
209
|
+
elif state == '.done':
|
|
210
|
+
return ExportTableStatus(keyspace, target_table, 'done', target_table)
|
|
211
|
+
|
|
212
|
+
# 4 rows exported to 1 files in 0 day, 0 hour, 0 minute, and 1.335 seconds.
|
|
213
|
+
pattern = 'rows exported to'
|
|
214
|
+
r: PodExecResult = CassandraNodes.exec(pod, namespace, f"grep '{pattern}' {log_file}", show_out=Config().is_debug(), shell='bash')
|
|
215
|
+
if r.exit_code() == 0:
|
|
216
|
+
csv_files, table = get_csv_files_n_table(target_table)
|
|
217
|
+
if csv_files:
|
|
218
|
+
return ExportTableStatus(keyspace, target_table, 'exported', table)
|
|
219
|
+
else:
|
|
220
|
+
return ExportTableStatus(keyspace, target_table, 'imported', target_table)
|
|
221
|
+
else:
|
|
222
|
+
return ExportTableStatus(keyspace, target_table, 'export_in_pregress')
|
|
223
|
+
|
|
224
|
+
return ExportTableStatus(None, None, 'unknown')
|
|
225
|
+
|
|
226
|
+
def csv_dir():
|
|
227
|
+
return Config().get('export.csv_dir', '/c3/cassandra/tmp')
|
|
228
|
+
|
|
229
|
+
def find_files(pod: str, namespace: str, pattern: str, mmin: int = 0):
|
|
230
|
+
if mmin:
|
|
231
|
+
r = CassandraNodes.exec(pod, namespace, f'find {pattern} -mmin -{mmin}', show_out=Config().is_debug(), shell='bash')
|
|
232
|
+
else:
|
|
233
|
+
r = CassandraNodes.exec(pod, namespace, f'find {pattern}', show_out=Config().is_debug(), shell='bash')
|
|
234
|
+
|
|
235
|
+
log_files = []
|
|
236
|
+
for line in r.stdout.split('\n'):
|
|
237
|
+
line = line.strip(' \r')
|
|
238
|
+
if line:
|
|
239
|
+
log_files.append(line)
|
|
240
|
+
|
|
241
|
+
return log_files
|
|
202
242
|
|
|
203
243
|
class GeneratorStream(io.RawIOBase):
|
|
204
244
|
def __init__(self, generator):
|
|
@@ -250,4 +290,25 @@ class GeneratorStream(io.RawIOBase):
|
|
|
250
290
|
|
|
251
291
|
data = self._buffer[:size]
|
|
252
292
|
self._buffer = self._buffer[size:]
|
|
253
|
-
return data
|
|
293
|
+
return data
|
|
294
|
+
|
|
295
|
+
class PodHandler:
|
|
296
|
+
def __init__(self, state: ReplState):
|
|
297
|
+
self.state = state
|
|
298
|
+
|
|
299
|
+
def __enter__(self):
|
|
300
|
+
state = self.state
|
|
301
|
+
|
|
302
|
+
if not state.pod:
|
|
303
|
+
state.push()
|
|
304
|
+
state.pod = StatefulSets.pod_names(state.sts, state.namespace)[0]
|
|
305
|
+
|
|
306
|
+
return state
|
|
307
|
+
|
|
308
|
+
def __exit__(self, exc_type, exc_val, exc_tb):
|
|
309
|
+
self.state.pop()
|
|
310
|
+
|
|
311
|
+
return False
|
|
312
|
+
|
|
313
|
+
def state_with_pod(state: ReplState):
|
|
314
|
+
return PodHandler(state)
|
adam/commands/help.py
CHANGED
|
@@ -28,7 +28,7 @@ class Help(Command):
|
|
|
28
28
|
|
|
29
29
|
lines = []
|
|
30
30
|
lines.append('NAVIGATION')
|
|
31
|
-
lines.append(' a: | c: | l: | p:\t switch to another operational device: App, Cassandra, Audit or
|
|
31
|
+
lines.append(' a: | c: | l: | p: | x:\t switch to another operational device: App, Cassandra, Audit, Postgres or Export')
|
|
32
32
|
lines.extend(section(ReplCommands.navigation()))
|
|
33
33
|
lines.append('CASSANDRA')
|
|
34
34
|
lines.extend(section(ReplCommands.cassandra_ops()))
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
from abc import abstractmethod
|
|
2
|
+
|
|
3
|
+
from adam.commands.command import Command
|
|
4
|
+
from adam.commands.command_helpers import ClusterCommandHelper
|
|
5
|
+
from adam.repl_state import ReplState
|
|
6
|
+
from adam.utils import lines_to_tabular, log, log2
|
|
7
|
+
|
|
8
|
+
class IntermediateCommand(Command):
|
|
9
|
+
def run(self, cmd: str, state: ReplState):
|
|
10
|
+
if not(args := self.args(cmd)):
|
|
11
|
+
return super().run(cmd, state)
|
|
12
|
+
|
|
13
|
+
return self.intermediate_run(cmd, state, args, self.cmd_list())
|
|
14
|
+
|
|
15
|
+
@abstractmethod
|
|
16
|
+
def cmd_list(self):
|
|
17
|
+
pass
|
|
18
|
+
|
|
19
|
+
def intermediate_run(self, cmd: str, state: ReplState, args: list[str], cmds: list['Command'], separator='\t', display_help=True):
|
|
20
|
+
state, _ = self.apply_state(args, state)
|
|
21
|
+
|
|
22
|
+
if state.in_repl:
|
|
23
|
+
if display_help:
|
|
24
|
+
log(lines_to_tabular([c.help(state) for c in cmds], separator=separator))
|
|
25
|
+
|
|
26
|
+
return 'command-missing'
|
|
27
|
+
else:
|
|
28
|
+
# head with the Chain of Responsibility pattern
|
|
29
|
+
if not self.run_subcommand(cmd, state):
|
|
30
|
+
if display_help:
|
|
31
|
+
log2('* Command is missing.')
|
|
32
|
+
Command.display_help()
|
|
33
|
+
return 'command-missing'
|
|
34
|
+
|
|
35
|
+
return state
|
|
36
|
+
|
|
37
|
+
def run_subcommand(self, cmd: str, state: ReplState):
|
|
38
|
+
cmds = Command.chain(self.cmd_list())
|
|
39
|
+
return cmds.run(cmd, state)
|
|
40
|
+
|
|
41
|
+
def intermediate_help(super_help: str, cmd: str, cmd_list: list['Command'], separator='\t', show_cluster_help=False):
|
|
42
|
+
log(super_help)
|
|
43
|
+
log()
|
|
44
|
+
log('Sub-Commands:')
|
|
45
|
+
|
|
46
|
+
log(lines_to_tabular([c.help(ReplState()).replace(f'{cmd} ', ' ', 1) for c in cmd_list], separator=separator))
|
|
47
|
+
if show_cluster_help:
|
|
48
|
+
log()
|
|
49
|
+
ClusterCommandHelper.cluster_help()
|
adam/commands/issues.py
CHANGED
|
@@ -1,10 +1,9 @@
|
|
|
1
1
|
from adam.checks.check_result import CheckResult
|
|
2
2
|
from adam.checks.check_utils import run_checks
|
|
3
|
-
from adam.
|
|
3
|
+
from adam.commands import extract_options
|
|
4
4
|
from adam.commands.command import Command
|
|
5
|
-
from adam.repl_session import ReplSession
|
|
6
5
|
from adam.repl_state import ReplState
|
|
7
|
-
from adam.
|
|
6
|
+
from adam.utils_issues import IssuesUtils
|
|
8
7
|
|
|
9
8
|
class Issues(Command):
|
|
10
9
|
COMMAND = 'issues'
|
|
@@ -28,48 +27,17 @@ class Issues(Command):
|
|
|
28
27
|
if not(args := self.args(cmd)):
|
|
29
28
|
return super().run(cmd, state)
|
|
30
29
|
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
30
|
+
with self.validate(args, state) as (args, state):
|
|
31
|
+
with extract_options(args, ['-s', '--show']) as (args, show_out):
|
|
32
|
+
results = run_checks(state.sts, state.namespace, state.pod, show_out=show_out)
|
|
34
33
|
|
|
35
|
-
|
|
34
|
+
issues = CheckResult.collect_issues(results)
|
|
35
|
+
IssuesUtils.show_issues(issues, in_repl=state.in_repl)
|
|
36
36
|
|
|
37
|
-
|
|
37
|
+
return issues if issues else 'issues'
|
|
38
38
|
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
return issues if issues else 'issues'
|
|
43
|
-
|
|
44
|
-
def show(check_results: list[CheckResult], in_repl = False):
|
|
45
|
-
Issues.show_issues(CheckResult.collect_issues(check_results), in_repl=in_repl)
|
|
46
|
-
|
|
47
|
-
def show_issues(issues: list[Issue], in_repl = False):
|
|
48
|
-
if not issues:
|
|
49
|
-
log2('No issues found.')
|
|
50
|
-
else:
|
|
51
|
-
suggested = 0
|
|
52
|
-
log2(f'* {len(issues)} issues found.')
|
|
53
|
-
lines = []
|
|
54
|
-
for i, issue in enumerate(issues, start=1):
|
|
55
|
-
lines.append(f"{i}||{issue.category}||{issue.desc}")
|
|
56
|
-
lines.append(f"||statefulset||{issue.statefulset}@{issue.namespace}")
|
|
57
|
-
lines.append(f"||pod||{issue.pod}@{issue.namespace}")
|
|
58
|
-
if issue.details:
|
|
59
|
-
lines.append(f"||details||{issue.details}")
|
|
60
|
-
|
|
61
|
-
if issue.suggestion:
|
|
62
|
-
lines.append(f'||suggestion||{issue.suggestion}')
|
|
63
|
-
if in_repl:
|
|
64
|
-
ReplSession().prompt_session.history.append_string(issue.suggestion)
|
|
65
|
-
suggested += 1
|
|
66
|
-
log(lines_to_tabular(lines, separator='||'))
|
|
67
|
-
if suggested:
|
|
68
|
-
log2()
|
|
69
|
-
log2(f'* {suggested} suggested commands are added to history. Press <Up> arrow to access them.')
|
|
70
|
-
|
|
71
|
-
def completion(self, _: ReplState):
|
|
72
|
-
return {Issues.COMMAND: None}
|
|
39
|
+
def completion(self, state: ReplState):
|
|
40
|
+
return super().completion(state, {'-s': None})
|
|
73
41
|
|
|
74
42
|
def help(self, _: ReplState):
|
|
75
|
-
return f'{Issues.COMMAND}\t find all issues'
|
|
43
|
+
return f'{Issues.COMMAND} [-s]\t find all issues'
|