kaqing 2.0.145__py3-none-any.whl → 2.0.174__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of kaqing might be problematic. Click here for more details.
- adam/__init__.py +0 -2
- adam/app_session.py +8 -11
- adam/batch.py +3 -3
- adam/checks/check_utils.py +14 -46
- adam/checks/cpu.py +7 -1
- adam/checks/cpu_metrics.py +52 -0
- adam/checks/disk.py +2 -3
- adam/columns/columns.py +3 -1
- adam/columns/cpu.py +3 -1
- adam/columns/cpu_metrics.py +22 -0
- adam/columns/memory.py +3 -4
- adam/commands/__init__.py +22 -0
- adam/commands/alter_tables.py +33 -48
- adam/commands/audit/audit.py +22 -23
- adam/commands/audit/audit_repair_tables.py +14 -17
- adam/commands/audit/audit_run.py +15 -23
- adam/commands/audit/show_last10.py +10 -13
- adam/commands/audit/show_slow10.py +10 -13
- adam/commands/audit/show_top10.py +10 -13
- adam/commands/audit/utils_show_top10.py +2 -3
- adam/commands/bash/__init__.py +5 -0
- adam/commands/bash/bash.py +7 -104
- adam/commands/bash/utils_bash.py +16 -0
- adam/commands/cat.py +7 -23
- adam/commands/cd.py +7 -11
- adam/commands/check.py +14 -23
- adam/commands/cli_commands.py +2 -3
- adam/commands/code.py +20 -23
- adam/commands/command.py +152 -37
- adam/commands/commands_utils.py +8 -17
- adam/commands/cp.py +18 -32
- adam/commands/cql/cql_completions.py +11 -7
- adam/commands/cql/cqlsh.py +10 -30
- adam/commands/cql/{cql_utils.py → utils_cql.py} +147 -15
- adam/commands/deploy/code_start.py +7 -10
- adam/commands/deploy/code_stop.py +4 -21
- adam/commands/deploy/code_utils.py +3 -3
- adam/commands/deploy/deploy.py +4 -27
- adam/commands/deploy/deploy_frontend.py +14 -17
- adam/commands/deploy/deploy_pg_agent.py +2 -5
- adam/commands/deploy/deploy_pod.py +64 -68
- adam/commands/deploy/undeploy.py +4 -27
- adam/commands/deploy/undeploy_frontend.py +4 -7
- adam/commands/deploy/undeploy_pg_agent.py +4 -7
- adam/commands/deploy/undeploy_pod.py +9 -12
- adam/commands/devices/device.py +93 -2
- adam/commands/devices/device_app.py +37 -10
- adam/commands/devices/device_auit_log.py +8 -2
- adam/commands/devices/device_cass.py +47 -7
- adam/commands/devices/device_export.py +9 -11
- adam/commands/devices/device_postgres.py +41 -6
- adam/commands/exit.py +1 -4
- adam/commands/export/clean_up_all_export_sessions.py +37 -0
- adam/commands/export/clean_up_export_sessions.py +12 -8
- adam/commands/export/drop_export_database.py +7 -26
- adam/commands/export/drop_export_databases.py +5 -14
- adam/commands/export/export.py +8 -38
- adam/commands/export/export_databases.py +86 -27
- adam/commands/export/export_select.py +25 -27
- adam/commands/export/export_select_x.py +3 -3
- adam/commands/export/export_sessions.py +124 -0
- adam/commands/export/export_use.py +8 -17
- adam/commands/export/exporter.py +88 -158
- adam/commands/export/import_session.py +7 -35
- adam/commands/export/importer.py +12 -5
- adam/commands/export/importer_athena.py +21 -20
- adam/commands/export/importer_sqlite.py +16 -21
- adam/commands/export/show_column_counts.py +7 -25
- adam/commands/export/show_export_databases.py +4 -6
- adam/commands/export/show_export_session.py +7 -18
- adam/commands/export/show_export_sessions.py +9 -12
- adam/commands/export/utils_export.py +26 -1
- adam/commands/intermediate_command.py +49 -0
- adam/commands/issues.py +11 -43
- adam/commands/kubectl.py +3 -6
- adam/commands/login.py +22 -24
- adam/commands/logs.py +3 -6
- adam/commands/ls.py +8 -9
- adam/commands/medusa/medusa.py +4 -22
- adam/commands/medusa/medusa_backup.py +20 -25
- adam/commands/medusa/medusa_restore.py +34 -36
- adam/commands/medusa/medusa_show_backupjobs.py +14 -18
- adam/commands/medusa/medusa_show_restorejobs.py +11 -18
- adam/commands/nodetool.py +6 -15
- adam/commands/param_get.py +11 -13
- adam/commands/param_set.py +8 -12
- adam/commands/postgres/postgres.py +22 -38
- adam/commands/postgres/postgres_context.py +47 -23
- adam/commands/postgres/postgres_ls.py +4 -8
- adam/commands/postgres/postgres_preview.py +5 -9
- adam/commands/postgres/psql_completions.py +1 -1
- adam/commands/postgres/utils_postgres.py +70 -0
- adam/commands/preview_table.py +6 -45
- adam/commands/pwd.py +13 -16
- adam/commands/reaper/reaper.py +4 -27
- adam/commands/reaper/reaper_forward.py +48 -55
- adam/commands/reaper/reaper_forward_session.py +6 -0
- adam/commands/reaper/reaper_forward_stop.py +10 -16
- adam/commands/reaper/reaper_restart.py +7 -14
- adam/commands/reaper/reaper_run_abort.py +8 -33
- adam/commands/reaper/reaper_runs.py +42 -57
- adam/commands/reaper/reaper_runs_abort.py +29 -49
- adam/commands/reaper/reaper_schedule_activate.py +9 -32
- adam/commands/reaper/reaper_schedule_start.py +9 -32
- adam/commands/reaper/reaper_schedule_stop.py +9 -32
- adam/commands/reaper/reaper_schedules.py +4 -14
- adam/commands/reaper/reaper_status.py +8 -16
- adam/commands/reaper/utils_reaper.py +196 -0
- adam/commands/repair/repair.py +4 -22
- adam/commands/repair/repair_log.py +5 -11
- adam/commands/repair/repair_run.py +27 -34
- adam/commands/repair/repair_scan.py +32 -38
- adam/commands/repair/repair_stop.py +5 -11
- adam/commands/report.py +27 -29
- adam/commands/restart.py +25 -26
- adam/commands/rollout.py +19 -24
- adam/commands/shell.py +10 -4
- adam/commands/show/show.py +10 -26
- adam/commands/show/show_cassandra_repairs.py +35 -0
- adam/commands/show/show_cassandra_status.py +32 -43
- adam/commands/show/show_cassandra_version.py +5 -18
- adam/commands/show/show_commands.py +19 -24
- adam/commands/show/show_host.py +1 -1
- adam/commands/show/show_login.py +20 -27
- adam/commands/show/show_processes.py +15 -19
- adam/commands/show/show_storage.py +10 -20
- adam/commands/watch.py +26 -29
- adam/config.py +4 -16
- adam/embedded_params.py +1 -1
- adam/log.py +4 -4
- adam/pod_exec_result.py +3 -3
- adam/repl.py +31 -32
- adam/repl_commands.py +11 -11
- adam/repl_state.py +52 -26
- adam/sql/sql_completer.py +4 -6
- adam/sql/sql_state_machine.py +21 -14
- adam/sso/authn_ad.py +6 -8
- adam/sso/authn_okta.py +4 -6
- adam/sso/cred_cache.py +3 -5
- adam/sso/idp.py +9 -12
- adam/utils.py +393 -33
- adam/utils_athena.py +14 -13
- adam/utils_audits.py +12 -12
- adam/utils_issues.py +32 -0
- adam/utils_k8s/app_clusters.py +13 -18
- adam/utils_k8s/app_pods.py +2 -0
- adam/utils_k8s/cassandra_clusters.py +21 -18
- adam/utils_k8s/custom_resources.py +16 -17
- adam/utils_k8s/ingresses.py +2 -2
- adam/utils_k8s/jobs.py +7 -11
- adam/utils_k8s/k8s.py +87 -0
- adam/utils_k8s/pods.py +14 -76
- adam/utils_k8s/secrets.py +4 -4
- adam/utils_k8s/service_accounts.py +5 -4
- adam/utils_k8s/services.py +2 -2
- adam/utils_k8s/statefulsets.py +1 -12
- adam/utils_repl/state_machine.py +3 -3
- adam/utils_sqlite.py +78 -42
- adam/version.py +1 -1
- {kaqing-2.0.145.dist-info → kaqing-2.0.174.dist-info}/METADATA +1 -1
- kaqing-2.0.174.dist-info/RECORD +230 -0
- adam/commands/app.py +0 -67
- adam/commands/app_ping.py +0 -44
- adam/commands/export/clean_up_export_session.py +0 -53
- adam/commands/postgres/postgres_utils.py +0 -31
- adam/commands/reaper/reaper_session.py +0 -159
- adam/commands/show/show_app_actions.py +0 -56
- adam/commands/show/show_app_id.py +0 -47
- adam/commands/show/show_app_queues.py +0 -45
- adam/commands/show/show_repairs.py +0 -47
- kaqing-2.0.145.dist-info/RECORD +0 -227
- {kaqing-2.0.145.dist-info → kaqing-2.0.174.dist-info}/WHEEL +0 -0
- {kaqing-2.0.145.dist-info → kaqing-2.0.174.dist-info}/entry_points.txt +0 -0
- {kaqing-2.0.145.dist-info → kaqing-2.0.174.dist-info}/top_level.txt +0 -0
adam/commands/export/importer.py
CHANGED
|
@@ -2,6 +2,7 @@ from abc import abstractmethod
|
|
|
2
2
|
|
|
3
3
|
from adam.commands.export.utils_export import csv_dir
|
|
4
4
|
from adam.config import Config
|
|
5
|
+
from adam.repl_state import ReplState
|
|
5
6
|
from adam.utils import ing
|
|
6
7
|
from adam.utils_k8s.cassandra_nodes import CassandraNodes
|
|
7
8
|
from adam.utils_k8s.pods import log_prefix
|
|
@@ -12,10 +13,13 @@ class Importer:
|
|
|
12
13
|
pass
|
|
13
14
|
|
|
14
15
|
@abstractmethod
|
|
15
|
-
def import_from_csv(self,
|
|
16
|
+
def import_from_csv(self, state: ReplState, from_session: str, keyspace: str, table: str, target_table: str, columns: str, multi_tables = True, create_db = False):
|
|
16
17
|
pass
|
|
17
18
|
|
|
18
|
-
def move_to_done(self,
|
|
19
|
+
def move_to_done(self, state: ReplState, from_session: str, keyspace: str, target_table: str):
|
|
20
|
+
pod = state.pod
|
|
21
|
+
namespace = state.namespace
|
|
22
|
+
to_session = state.export_session
|
|
19
23
|
log_file = f'{log_prefix()}-{from_session}_{keyspace}.{target_table}.log.pending_import'
|
|
20
24
|
|
|
21
25
|
to = f'{log_prefix()}-{to_session}_{keyspace}.{target_table}.log.done'
|
|
@@ -30,9 +34,12 @@ class Importer:
|
|
|
30
34
|
|
|
31
35
|
return session
|
|
32
36
|
|
|
33
|
-
def remove_csv(self,
|
|
34
|
-
|
|
35
|
-
|
|
37
|
+
def remove_csv(self, state: ReplState, from_session: str, table: str, target_table: str, multi_tables = True):
|
|
38
|
+
pod = state.pod
|
|
39
|
+
namespace = state.namespace
|
|
40
|
+
|
|
41
|
+
with ing(f'[{from_session}] Cleaning up temporary files', suppress_log=multi_tables):
|
|
42
|
+
CassandraNodes.exec(pod, namespace, f'rm -rf {self.csv_file(from_session, table, target_table)}', show_out=Config().is_debug(), shell='bash')
|
|
36
43
|
|
|
37
44
|
def db(self, session: str, keyspace: str):
|
|
38
45
|
return f'{session}_{keyspace}'
|
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
import boto3
|
|
2
2
|
|
|
3
|
+
from adam.commands.export.export_databases import export_db
|
|
3
4
|
from adam.commands.export.importer import Importer
|
|
4
5
|
from adam.commands.export.utils_export import GeneratorStream
|
|
5
6
|
from adam.config import Config
|
|
6
|
-
from adam.
|
|
7
|
+
from adam.repl_state import ReplState
|
|
8
|
+
from adam.utils import debug, log2, ing
|
|
7
9
|
from adam.utils_athena import Athena
|
|
8
10
|
from adam.utils_k8s.pods import Pods
|
|
9
11
|
|
|
@@ -17,9 +19,12 @@ class AthenaImporter(Importer):
|
|
|
17
19
|
def prefix(self):
|
|
18
20
|
return 'e'
|
|
19
21
|
|
|
20
|
-
def import_from_csv(self,
|
|
22
|
+
def import_from_csv(self, state: ReplState, from_session: str, keyspace: str, table: str, target_table: str, columns: str, multi_tables = True, create_db = False):
|
|
21
23
|
csv_file = self.csv_file(from_session, table, target_table)
|
|
22
|
-
|
|
24
|
+
pod = state.pod
|
|
25
|
+
namespace = state.namespace
|
|
26
|
+
to_session = state.export_session
|
|
27
|
+
database = self.db(to_session, keyspace)
|
|
23
28
|
|
|
24
29
|
succeeded = False
|
|
25
30
|
try:
|
|
@@ -29,23 +34,21 @@ class AthenaImporter(Importer):
|
|
|
29
34
|
bytes = Pods.read_file(pod, 'cassandra', namespace, csv_file)
|
|
30
35
|
|
|
31
36
|
s3 = boto3.client('s3')
|
|
32
|
-
s3.upload_fileobj(GeneratorStream(bytes), bucket, f'export/{
|
|
37
|
+
s3.upload_fileobj(GeneratorStream(bytes), bucket, f'export/{database}/{keyspace}/{target_table}/{table}.csv')
|
|
33
38
|
|
|
34
39
|
msg: str = None
|
|
35
40
|
if create_db:
|
|
36
|
-
msg = f"[{to_session}] Creating database {
|
|
41
|
+
msg = f"[{to_session}] Creating database {database}"
|
|
37
42
|
else:
|
|
38
43
|
msg = f"[{to_session}] Creating table {target_table}"
|
|
39
44
|
with ing(msg, suppress_log=multi_tables):
|
|
40
|
-
query = f'CREATE DATABASE IF NOT EXISTS {
|
|
41
|
-
|
|
42
|
-
log2(query)
|
|
45
|
+
query = f'CREATE DATABASE IF NOT EXISTS {database};'
|
|
46
|
+
debug(query)
|
|
43
47
|
Athena.query(query, 'default')
|
|
44
48
|
|
|
45
49
|
query = f'DROP TABLE IF EXISTS {target_table};'
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
Athena.query(query, db)
|
|
50
|
+
debug(query)
|
|
51
|
+
Athena.query(query, database)
|
|
49
52
|
|
|
50
53
|
athena_columns = ', '.join([f'{c} string' for c in columns.split(',')])
|
|
51
54
|
query = f'CREATE EXTERNAL TABLE IF NOT EXISTS {target_table}(\n' + \
|
|
@@ -54,27 +57,25 @@ class AthenaImporter(Importer):
|
|
|
54
57
|
'WITH SERDEPROPERTIES (\n' + \
|
|
55
58
|
' "separatorChar" = ",",\n' + \
|
|
56
59
|
' "quoteChar" = "\\"")\n' + \
|
|
57
|
-
f"LOCATION 's3://{bucket}/export/{
|
|
60
|
+
f"LOCATION 's3://{bucket}/export/{database}/{keyspace}/{target_table}'\n" + \
|
|
58
61
|
'TBLPROPERTIES ("skip.header.line.count"="1");'
|
|
59
|
-
|
|
60
|
-
log2(query)
|
|
62
|
+
debug(query)
|
|
61
63
|
try:
|
|
62
|
-
Athena.query(query,
|
|
64
|
+
Athena.query(query, database)
|
|
63
65
|
except Exception as e:
|
|
64
66
|
log2(f'*** Failed query:\n{query}')
|
|
65
67
|
raise e
|
|
66
68
|
|
|
67
|
-
to, _ = self.move_to_done(
|
|
69
|
+
to, _ = self.move_to_done(state, from_session, keyspace, target_table)
|
|
68
70
|
|
|
69
71
|
succeeded = True
|
|
70
72
|
|
|
71
73
|
return to, to_session
|
|
72
74
|
finally:
|
|
73
75
|
if succeeded:
|
|
74
|
-
self.remove_csv(
|
|
76
|
+
self.remove_csv(state, from_session, table, target_table, multi_tables)
|
|
75
77
|
Athena.clear_cache()
|
|
76
78
|
|
|
77
79
|
if not multi_tables:
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
Athena.run_query(query, db)
|
|
80
|
+
with export_db(state) as dbms:
|
|
81
|
+
dbms.sql(f'select * from {database}.{target_table} limit 10')
|
|
@@ -1,47 +1,42 @@
|
|
|
1
|
-
import os
|
|
2
|
-
import sqlite3
|
|
3
1
|
import pandas
|
|
4
2
|
|
|
3
|
+
from adam.commands.export.export_databases import export_db
|
|
5
4
|
from adam.commands.export.importer import Importer
|
|
6
5
|
from adam.commands.export.utils_export import GeneratorStream
|
|
6
|
+
from adam.repl_state import ReplState
|
|
7
7
|
from adam.utils import log2, ing
|
|
8
8
|
from adam.utils_k8s.pods import Pods
|
|
9
|
-
from adam.utils_sqlite import SQLite
|
|
9
|
+
from adam.utils_sqlite import SQLite, sqlite
|
|
10
10
|
|
|
11
11
|
class SqliteImporter(Importer):
|
|
12
12
|
def prefix(self):
|
|
13
13
|
return 's'
|
|
14
14
|
|
|
15
|
-
def import_from_csv(self,
|
|
15
|
+
def import_from_csv(self, state: ReplState, from_session: str, keyspace: str, table: str, target_table: str, columns: str, multi_tables = True, create_db = False):
|
|
16
16
|
csv_file = self.csv_file(from_session, table, target_table)
|
|
17
|
-
|
|
17
|
+
pod = state.pod
|
|
18
|
+
namespace = state.namespace
|
|
19
|
+
to_session = state.export_session
|
|
18
20
|
|
|
19
21
|
succeeded = False
|
|
20
|
-
conn = None
|
|
21
22
|
try:
|
|
22
|
-
os.makedirs(SQLite.local_db_dir(), exist_ok=True)
|
|
23
|
-
conn = sqlite3.connect(f'{SQLite.local_db_dir()}/{db}.db')
|
|
24
|
-
|
|
25
23
|
with ing(f'[{to_session}] Uploading to Sqlite', suppress_log=multi_tables):
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
24
|
+
# create a connection to single keyspace
|
|
25
|
+
with sqlite(to_session, keyspace) as conn:
|
|
26
|
+
bytes = Pods.read_file(pod, 'cassandra', namespace, csv_file)
|
|
27
|
+
df = pandas.read_csv(GeneratorStream(bytes))
|
|
28
|
+
df.to_sql(target_table, conn, index=False, if_exists='replace')
|
|
30
29
|
|
|
31
|
-
to, _ = self.move_to_done(
|
|
30
|
+
to, _ = self.move_to_done(state, from_session, keyspace, target_table)
|
|
32
31
|
|
|
33
32
|
succeeded = True
|
|
34
33
|
|
|
35
34
|
return to, to_session
|
|
36
35
|
finally:
|
|
37
36
|
if succeeded:
|
|
38
|
-
self.remove_csv(
|
|
37
|
+
self.remove_csv(state, from_session, table, target_table, multi_tables)
|
|
39
38
|
SQLite.clear_cache()
|
|
40
39
|
|
|
41
40
|
if not multi_tables:
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
SQLite.run_query(query, conn_passed=conn)
|
|
45
|
-
|
|
46
|
-
if conn:
|
|
47
|
-
conn.close()
|
|
41
|
+
with export_db(state) as dbms:
|
|
42
|
+
dbms.sql(f'select * from {keyspace}.{target_table} limit 10')
|
|
@@ -1,10 +1,9 @@
|
|
|
1
|
+
from adam.commands import validate_args
|
|
1
2
|
from adam.commands.command import Command
|
|
2
3
|
from adam.commands.export.export_databases import ExportDatabases
|
|
3
4
|
from adam.config import Config
|
|
4
5
|
from adam.repl_state import ReplState, RequiredState
|
|
5
6
|
from adam.utils import log2
|
|
6
|
-
from adam.utils_athena import Athena
|
|
7
|
-
from adam.utils_sqlite import SQLite
|
|
8
7
|
|
|
9
8
|
class ShowColumnCounts(Command):
|
|
10
9
|
COMMAND = 'show column counts on'
|
|
@@ -28,30 +27,13 @@ class ShowColumnCounts(Command):
|
|
|
28
27
|
if not(args := self.args(cmd)):
|
|
29
28
|
return super().run(cmd, state)
|
|
30
29
|
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
if state.in_repl:
|
|
37
|
-
log2('Use a SQL statement.')
|
|
38
|
-
else:
|
|
39
|
-
log2('* SQL statement is missing.')
|
|
40
|
-
|
|
41
|
-
Command.display_help()
|
|
42
|
-
|
|
43
|
-
return 'command-missing'
|
|
30
|
+
with self.validate(args, state) as (args, state):
|
|
31
|
+
with validate_args(args, state, name='SQL statement') as table:
|
|
32
|
+
query = Config().get(f'export.column_counts_query', 'select id, count(id) as columns from {table} group by id')
|
|
33
|
+
query = query.replace('{table}', table)
|
|
34
|
+
ExportDatabases.run_query(query, state.export_session)
|
|
44
35
|
|
|
45
|
-
|
|
46
|
-
if state.export_session.startswith('e'):
|
|
47
|
-
copy_or_export = 'export'
|
|
48
|
-
|
|
49
|
-
table = args[0]
|
|
50
|
-
query = Config().get(f'{copy_or_export}.column_counts_query', 'select id, count(id) as columns from {table} group by id')
|
|
51
|
-
query = query.replace('{table}', table)
|
|
52
|
-
ExportDatabases.run_query(query, state.export_session)
|
|
53
|
-
|
|
54
|
-
return state
|
|
36
|
+
return state
|
|
55
37
|
|
|
56
38
|
def completion(self, state: ReplState):
|
|
57
39
|
if not state.export_session:
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
from adam.commands.command import Command
|
|
2
2
|
from adam.commands.devices.device_export import DeviceExport
|
|
3
|
+
from adam.commands.export.export_databases import ExportDatabases
|
|
3
4
|
from adam.repl_state import ReplState
|
|
4
5
|
|
|
5
6
|
class ShowExportDatabases(Command):
|
|
@@ -24,13 +25,10 @@ class ShowExportDatabases(Command):
|
|
|
24
25
|
if not(args := self.args(cmd)):
|
|
25
26
|
return super().run(cmd, state)
|
|
26
27
|
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
return state
|
|
30
|
-
|
|
31
|
-
DeviceExport().show_export_databases()
|
|
28
|
+
with self.validate(args, state) as (args, state):
|
|
29
|
+
ExportDatabases.show_databases()
|
|
32
30
|
|
|
33
|
-
|
|
31
|
+
return state
|
|
34
32
|
|
|
35
33
|
def completion(self, state: ReplState):
|
|
36
34
|
return DeviceExport().ls_completion(ShowExportDatabases.COMMAND, state, default = super().completion(state))
|
|
@@ -1,6 +1,7 @@
|
|
|
1
|
+
from adam.commands import validate_args
|
|
1
2
|
from adam.commands.command import Command
|
|
2
3
|
from adam.commands.export.export_databases import ExportDatabases
|
|
3
|
-
from adam.commands.export.
|
|
4
|
+
from adam.commands.export.export_sessions import ExportSessions
|
|
4
5
|
from adam.repl_state import ReplState, RequiredState
|
|
5
6
|
from adam.utils import log2
|
|
6
7
|
|
|
@@ -26,26 +27,14 @@ class ShowExportSession(Command):
|
|
|
26
27
|
if not(args := self.args(cmd)):
|
|
27
28
|
return super().run(cmd, state)
|
|
28
29
|
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
if not args:
|
|
34
|
-
if state.in_repl:
|
|
35
|
-
log2('Specify export database name.')
|
|
36
|
-
else:
|
|
37
|
-
log2('* Database name is missing.')
|
|
38
|
-
|
|
39
|
-
Command.display_help()
|
|
30
|
+
with self.validate(args, state) as (args, state):
|
|
31
|
+
with validate_args(args, state, name='database name'):
|
|
32
|
+
ExportSessions.disply_export_session(state.sts, state.pod, state.namespace, args[0])
|
|
40
33
|
|
|
41
|
-
return
|
|
42
|
-
|
|
43
|
-
ExportDatabases.disply_export_session(state.sts, state.pod, state.namespace, args[0])
|
|
44
|
-
|
|
45
|
-
return state
|
|
34
|
+
return state
|
|
46
35
|
|
|
47
36
|
def completion(self, state: ReplState):
|
|
48
|
-
return super().completion(state, {session: None for session in
|
|
37
|
+
return super().completion(state, {session: None for session in ExportSessions.export_session_names(state.sts, state.pod, state.namespace)})
|
|
49
38
|
|
|
50
39
|
def help(self, _: ReplState):
|
|
51
40
|
return f'{ShowExportSession.COMMAND} <export-session-name>\t show export session'
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
from adam.commands.command import Command
|
|
2
|
-
from adam.commands.export.
|
|
2
|
+
from adam.commands.export.export_sessions import ExportSessions
|
|
3
3
|
from adam.repl_state import ReplState, RequiredState
|
|
4
4
|
from adam.utils import lines_to_tabular, log
|
|
5
5
|
from adam.utils_k8s.statefulsets import StatefulSets
|
|
@@ -26,19 +26,16 @@ class ShowExportSessions(Command):
|
|
|
26
26
|
if not(args := self.args(cmd)):
|
|
27
27
|
return super().run(cmd, state)
|
|
28
28
|
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
pod = state.pod
|
|
34
|
-
if not pod:
|
|
35
|
-
pod = StatefulSets.pod_names(state.sts, state.namespace)[0]
|
|
29
|
+
with self.validate(args, state) as (args, state):
|
|
30
|
+
pod = state.pod
|
|
31
|
+
if not pod:
|
|
32
|
+
pod = StatefulSets.pod_names(state.sts, state.namespace)[0]
|
|
36
33
|
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
34
|
+
sessions: dict[str, str] = ExportSessions.find_export_sessions(pod, state.namespace)
|
|
35
|
+
log(lines_to_tabular([f'{session}\t{export_state}' for session, export_state in sorted(sessions.items(), reverse=True)],
|
|
36
|
+
header='EXPORT_SESSION\tSTATUS', separator='\t'))
|
|
40
37
|
|
|
41
|
-
|
|
38
|
+
return state
|
|
42
39
|
|
|
43
40
|
def completion(self, state: ReplState):
|
|
44
41
|
return super().completion(state)
|
|
@@ -3,8 +3,10 @@ import re
|
|
|
3
3
|
|
|
4
4
|
from adam.config import Config
|
|
5
5
|
from adam.pod_exec_result import PodExecResult
|
|
6
|
+
from adam.repl_state import ReplState
|
|
6
7
|
from adam.utils_k8s.cassandra_nodes import CassandraNodes
|
|
7
8
|
from adam.utils_k8s.pods import log_prefix
|
|
9
|
+
from adam.utils_k8s.statefulsets import StatefulSets
|
|
8
10
|
|
|
9
11
|
class ImportSpec:
|
|
10
12
|
def __init__(self, session: str, importer: str):
|
|
@@ -288,4 +290,27 @@ class GeneratorStream(io.RawIOBase):
|
|
|
288
290
|
|
|
289
291
|
data = self._buffer[:size]
|
|
290
292
|
self._buffer = self._buffer[size:]
|
|
291
|
-
return data
|
|
293
|
+
return data
|
|
294
|
+
|
|
295
|
+
class PodPushHandler:
|
|
296
|
+
def __init__(self, state: ReplState):
|
|
297
|
+
self.state = state
|
|
298
|
+
self.pushed = False
|
|
299
|
+
|
|
300
|
+
def __enter__(self):
|
|
301
|
+
state = self.state
|
|
302
|
+
|
|
303
|
+
if not state.pod:
|
|
304
|
+
state.push()
|
|
305
|
+
state.pod = StatefulSets.pod_names(state.sts, state.namespace)[0]
|
|
306
|
+
|
|
307
|
+
return state
|
|
308
|
+
|
|
309
|
+
def __exit__(self, exc_type, exc_val, exc_tb):
|
|
310
|
+
if self.pushed:
|
|
311
|
+
self.state.pop()
|
|
312
|
+
|
|
313
|
+
return False
|
|
314
|
+
|
|
315
|
+
def state_with_pod(state: ReplState):
|
|
316
|
+
return PodPushHandler(state)
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
from abc import abstractmethod
|
|
2
|
+
|
|
3
|
+
from adam.commands.command import Command
|
|
4
|
+
from adam.commands.command_helpers import ClusterCommandHelper
|
|
5
|
+
from adam.repl_state import ReplState
|
|
6
|
+
from adam.utils import lines_to_tabular, log, log2
|
|
7
|
+
|
|
8
|
+
class IntermediateCommand(Command):
|
|
9
|
+
def run(self, cmd: str, state: ReplState):
|
|
10
|
+
if not(args := self.args(cmd)):
|
|
11
|
+
return super().run(cmd, state)
|
|
12
|
+
|
|
13
|
+
return self.intermediate_run(cmd, state, args, self.cmd_list())
|
|
14
|
+
|
|
15
|
+
@abstractmethod
|
|
16
|
+
def cmd_list(self):
|
|
17
|
+
pass
|
|
18
|
+
|
|
19
|
+
def intermediate_run(self, cmd: str, state: ReplState, args: list[str], cmds: list['Command'], separator='\t', display_help=True):
|
|
20
|
+
state, _ = self.apply_state(args, state)
|
|
21
|
+
|
|
22
|
+
if state.in_repl:
|
|
23
|
+
if display_help:
|
|
24
|
+
log(lines_to_tabular([c.help(state) for c in cmds], separator=separator))
|
|
25
|
+
|
|
26
|
+
return 'command-missing'
|
|
27
|
+
else:
|
|
28
|
+
# head with the Chain of Responsibility pattern
|
|
29
|
+
if not self.run_subcommand(cmd, state):
|
|
30
|
+
if display_help:
|
|
31
|
+
log2('* Command is missing.')
|
|
32
|
+
Command.display_help()
|
|
33
|
+
return 'command-missing'
|
|
34
|
+
|
|
35
|
+
return state
|
|
36
|
+
|
|
37
|
+
def run_subcommand(self, cmd: str, state: ReplState):
|
|
38
|
+
cmds = Command.chain(self.cmd_list())
|
|
39
|
+
return cmds.run(cmd, state)
|
|
40
|
+
|
|
41
|
+
def intermediate_help(super_help: str, cmd: str, cmd_list: list['Command'], separator='\t', show_cluster_help=False):
|
|
42
|
+
log(super_help)
|
|
43
|
+
log()
|
|
44
|
+
log('Sub-Commands:')
|
|
45
|
+
|
|
46
|
+
log(lines_to_tabular([c.help(ReplState()).replace(f'{cmd} ', ' ', 1) for c in cmd_list], separator=separator))
|
|
47
|
+
if show_cluster_help:
|
|
48
|
+
log()
|
|
49
|
+
ClusterCommandHelper.cluster_help()
|
adam/commands/issues.py
CHANGED
|
@@ -1,10 +1,9 @@
|
|
|
1
1
|
from adam.checks.check_result import CheckResult
|
|
2
2
|
from adam.checks.check_utils import run_checks
|
|
3
|
-
from adam.
|
|
3
|
+
from adam.commands import extract_options
|
|
4
4
|
from adam.commands.command import Command
|
|
5
|
-
from adam.repl_session import ReplSession
|
|
6
5
|
from adam.repl_state import ReplState
|
|
7
|
-
from adam.
|
|
6
|
+
from adam.utils_issues import IssuesUtils
|
|
8
7
|
|
|
9
8
|
class Issues(Command):
|
|
10
9
|
COMMAND = 'issues'
|
|
@@ -28,48 +27,17 @@ class Issues(Command):
|
|
|
28
27
|
if not(args := self.args(cmd)):
|
|
29
28
|
return super().run(cmd, state)
|
|
30
29
|
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
30
|
+
with self.validate(args, state) as (args, state):
|
|
31
|
+
with extract_options(args, ['-s', '--show']) as (args, show_out):
|
|
32
|
+
results = run_checks(state.sts, state.namespace, state.pod, show_out=show_out)
|
|
34
33
|
|
|
35
|
-
|
|
34
|
+
issues = CheckResult.collect_issues(results)
|
|
35
|
+
IssuesUtils.show_issues(issues, in_repl=state.in_repl)
|
|
36
36
|
|
|
37
|
-
|
|
37
|
+
return issues if issues else 'issues'
|
|
38
38
|
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
return issues if issues else 'issues'
|
|
43
|
-
|
|
44
|
-
def show(check_results: list[CheckResult], in_repl = False):
|
|
45
|
-
Issues.show_issues(CheckResult.collect_issues(check_results), in_repl=in_repl)
|
|
46
|
-
|
|
47
|
-
def show_issues(issues: list[Issue], in_repl = False):
|
|
48
|
-
if not issues:
|
|
49
|
-
log2('No issues found.')
|
|
50
|
-
else:
|
|
51
|
-
suggested = 0
|
|
52
|
-
log2(f'* {len(issues)} issues found.')
|
|
53
|
-
lines = []
|
|
54
|
-
for i, issue in enumerate(issues, start=1):
|
|
55
|
-
lines.append(f"{i}||{issue.category}||{issue.desc}")
|
|
56
|
-
lines.append(f"||statefulset||{issue.statefulset}@{issue.namespace}")
|
|
57
|
-
lines.append(f"||pod||{issue.pod}@{issue.namespace}")
|
|
58
|
-
if issue.details:
|
|
59
|
-
lines.append(f"||details||{issue.details}")
|
|
60
|
-
|
|
61
|
-
if issue.suggestion:
|
|
62
|
-
lines.append(f'||suggestion||{issue.suggestion}')
|
|
63
|
-
if in_repl:
|
|
64
|
-
ReplSession().prompt_session.history.append_string(issue.suggestion)
|
|
65
|
-
suggested += 1
|
|
66
|
-
log(lines_to_tabular(lines, separator='||'))
|
|
67
|
-
if suggested:
|
|
68
|
-
log2()
|
|
69
|
-
log2(f'* {suggested} suggested commands are added to history. Press <Up> arrow to access them.')
|
|
70
|
-
|
|
71
|
-
def completion(self, _: ReplState):
|
|
72
|
-
return {Issues.COMMAND: None}
|
|
39
|
+
def completion(self, state: ReplState):
|
|
40
|
+
return super().completion(state, {'-s': None})
|
|
73
41
|
|
|
74
42
|
def help(self, _: ReplState):
|
|
75
|
-
return f'{Issues.COMMAND}\t find all issues'
|
|
43
|
+
return f'{Issues.COMMAND} [-s]\t find all issues'
|
adam/commands/kubectl.py
CHANGED
|
@@ -25,13 +25,10 @@ class Kubectl(Command):
|
|
|
25
25
|
if not(args := self.args(cmd)):
|
|
26
26
|
return super().run(cmd, state)
|
|
27
27
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
return state
|
|
31
|
-
|
|
32
|
-
subprocess.run(["kubectl"] + args)
|
|
28
|
+
with self.validate(args, state) as (args, state):
|
|
29
|
+
subprocess.run(["kubectl"] + args)
|
|
33
30
|
|
|
34
|
-
|
|
31
|
+
return state
|
|
35
32
|
|
|
36
33
|
def completion(self, state: ReplState):
|
|
37
34
|
return super().completion(state)
|
adam/commands/login.py
CHANGED
|
@@ -1,15 +1,15 @@
|
|
|
1
1
|
import os
|
|
2
2
|
import signal
|
|
3
|
-
import traceback
|
|
4
3
|
|
|
5
4
|
from adam.app_session import AppSession
|
|
6
5
|
from adam.apps import Apps
|
|
6
|
+
from adam.commands import extract_options
|
|
7
7
|
from adam.config import Config
|
|
8
8
|
from adam.sso.idp import Idp
|
|
9
9
|
from adam.sso.idp_login import IdpLogin
|
|
10
10
|
from adam.commands.command import Command
|
|
11
|
-
from adam.repl_state import ReplState
|
|
12
|
-
from adam.utils import log, log2
|
|
11
|
+
from adam.repl_state import ReplState
|
|
12
|
+
from adam.utils import log, log2, log_exc
|
|
13
13
|
|
|
14
14
|
class Login(Command):
|
|
15
15
|
COMMAND = 'login'
|
|
@@ -30,37 +30,35 @@ class Login(Command):
|
|
|
30
30
|
return ReplState.NON_L
|
|
31
31
|
|
|
32
32
|
def run(self, cmd: str, state: ReplState):
|
|
33
|
+
if not(args := self.args(cmd)):
|
|
34
|
+
return super().run(cmd, state)
|
|
35
|
+
|
|
33
36
|
def custom_handler(signum, frame):
|
|
34
37
|
AppSession.ctrl_c_entered = True
|
|
35
38
|
|
|
36
39
|
signal.signal(signal.SIGINT, custom_handler)
|
|
37
40
|
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
args, debug = Command.extract_options(args, ['d'])
|
|
43
|
-
if debug:
|
|
44
|
-
Config().set('debug', True)
|
|
41
|
+
with self.validate(args, state) as (args, state):
|
|
42
|
+
with extract_options(args, 'd') as (args, debug):
|
|
43
|
+
if debug:
|
|
44
|
+
Config().set('debug', True)
|
|
45
45
|
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
46
|
+
username: str = os.getenv('USERNAME')
|
|
47
|
+
if len(args) > 0:
|
|
48
|
+
username = args[0]
|
|
49
49
|
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
50
|
+
login: IdpLogin = None
|
|
51
|
+
with log_exc(True):
|
|
52
|
+
if not(host := Apps.app_host('c3', 'c3', state.namespace)):
|
|
53
|
+
log2('Cannot locate ingress for app.')
|
|
54
|
+
return state
|
|
55
55
|
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
except:
|
|
59
|
-
log2(traceback.format_exc())
|
|
56
|
+
if not (login := Idp.login(host, username=username, use_token_from_env=False)):
|
|
57
|
+
log2('Invalid username/password. Please try again.')
|
|
60
58
|
|
|
61
|
-
|
|
59
|
+
log(f'IDP_TOKEN={login.ser() if login else ""}')
|
|
62
60
|
|
|
63
|
-
|
|
61
|
+
return state
|
|
64
62
|
|
|
65
63
|
def completion(self, state: ReplState):
|
|
66
64
|
return super().completion(state)
|
adam/commands/logs.py
CHANGED
|
@@ -25,12 +25,9 @@ class Logs(Command):
|
|
|
25
25
|
if not(args := self.args(cmd)):
|
|
26
26
|
return super().run(cmd, state)
|
|
27
27
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
return state
|
|
31
|
-
|
|
32
|
-
path = Config().get('logs.path', '/c3/cassandra/logs/system.log')
|
|
33
|
-
return CassandraNodes.exec(state.pod, state.namespace, f'cat {path}')
|
|
28
|
+
with self.validate(args, state) as (args, state):
|
|
29
|
+
path = Config().get('logs.path', '/c3/cassandra/logs/system.log')
|
|
30
|
+
return CassandraNodes.exec(state.pod, state.namespace, f'cat {path}')
|
|
34
31
|
|
|
35
32
|
def completion(self, _: ReplState):
|
|
36
33
|
# available only on cli
|
adam/commands/ls.py
CHANGED
|
@@ -23,17 +23,16 @@ class Ls(Command):
|
|
|
23
23
|
if not(args := self.args(cmd)):
|
|
24
24
|
return super().run(cmd, state)
|
|
25
25
|
|
|
26
|
-
|
|
26
|
+
with self.validate(args, state) as (args, state):
|
|
27
|
+
if len(args) > 0:
|
|
28
|
+
arg = args[0]
|
|
29
|
+
if arg in ['p:', 'c:'] and arg != f'{state.device}:':
|
|
30
|
+
state = copy.copy(state)
|
|
31
|
+
state.device = arg.replace(':', '')
|
|
27
32
|
|
|
28
|
-
|
|
29
|
-
arg = args[0]
|
|
30
|
-
if arg in ['p:', 'c:'] and arg != f'{state.device}:':
|
|
31
|
-
state = copy.copy(state)
|
|
32
|
-
state.device = arg.replace(':', '')
|
|
33
|
+
Devices.device(state).ls(cmd, state)
|
|
33
34
|
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
return state
|
|
35
|
+
return state
|
|
37
36
|
|
|
38
37
|
def completion(self, state: ReplState):
|
|
39
38
|
return Devices.device(state).ls_completion(Ls.COMMAND, state, default = super().completion(state))
|