kaqing 2.0.14__py3-none-any.whl → 2.0.145__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of kaqing might be problematic. Click here for more details.
- adam/apps.py +2 -2
- adam/batch.py +13 -3
- adam/checks/check_utils.py +4 -4
- adam/checks/compactionstats.py +1 -1
- adam/checks/cpu.py +2 -2
- adam/checks/disk.py +1 -1
- adam/checks/gossip.py +1 -1
- adam/checks/memory.py +3 -3
- adam/checks/status.py +1 -1
- adam/commands/alter_tables.py +81 -0
- adam/commands/app.py +3 -3
- adam/commands/app_ping.py +2 -2
- adam/commands/audit/audit.py +86 -0
- adam/commands/audit/audit_repair_tables.py +77 -0
- adam/commands/audit/audit_run.py +58 -0
- adam/commands/audit/show_last10.py +51 -0
- adam/commands/audit/show_slow10.py +50 -0
- adam/commands/audit/show_top10.py +48 -0
- adam/commands/audit/utils_show_top10.py +59 -0
- adam/commands/bash/__init__.py +0 -0
- adam/commands/bash/bash.py +133 -0
- adam/commands/bash/bash_completer.py +93 -0
- adam/commands/cat.py +56 -0
- adam/commands/cd.py +12 -82
- adam/commands/check.py +6 -0
- adam/commands/cli_commands.py +3 -3
- adam/commands/code.py +60 -0
- adam/commands/command.py +48 -12
- adam/commands/commands_utils.py +4 -5
- adam/commands/cql/__init__.py +0 -0
- adam/commands/cql/cql_completions.py +28 -0
- adam/commands/cql/cql_utils.py +209 -0
- adam/commands/{cqlsh.py → cql/cqlsh.py} +15 -10
- adam/commands/deploy/code_utils.py +2 -2
- adam/commands/deploy/deploy.py +8 -21
- adam/commands/deploy/deploy_frontend.py +1 -1
- adam/commands/deploy/deploy_pg_agent.py +3 -3
- adam/commands/deploy/deploy_pod.py +28 -27
- adam/commands/deploy/deploy_utils.py +16 -26
- adam/commands/deploy/undeploy.py +8 -21
- adam/commands/deploy/undeploy_frontend.py +1 -1
- adam/commands/deploy/undeploy_pg_agent.py +5 -3
- adam/commands/deploy/undeploy_pod.py +12 -10
- adam/commands/devices/__init__.py +0 -0
- adam/commands/devices/device.py +27 -0
- adam/commands/devices/device_app.py +146 -0
- adam/commands/devices/device_auit_log.py +43 -0
- adam/commands/devices/device_cass.py +145 -0
- adam/commands/devices/device_export.py +86 -0
- adam/commands/devices/device_postgres.py +109 -0
- adam/commands/devices/devices.py +25 -0
- adam/commands/export/__init__.py +0 -0
- adam/commands/export/clean_up_export_session.py +53 -0
- adam/commands/export/clean_up_export_sessions.py +40 -0
- adam/commands/export/drop_export_database.py +58 -0
- adam/commands/export/drop_export_databases.py +46 -0
- adam/commands/export/export.py +83 -0
- adam/commands/export/export_databases.py +170 -0
- adam/commands/export/export_select.py +85 -0
- adam/commands/export/export_select_x.py +54 -0
- adam/commands/export/export_use.py +55 -0
- adam/commands/export/exporter.py +364 -0
- adam/commands/export/import_session.py +68 -0
- adam/commands/export/importer.py +67 -0
- adam/commands/export/importer_athena.py +80 -0
- adam/commands/export/importer_sqlite.py +47 -0
- adam/commands/export/show_column_counts.py +63 -0
- adam/commands/export/show_export_databases.py +39 -0
- adam/commands/export/show_export_session.py +51 -0
- adam/commands/export/show_export_sessions.py +47 -0
- adam/commands/export/utils_export.py +291 -0
- adam/commands/help.py +12 -7
- adam/commands/issues.py +6 -0
- adam/commands/kubectl.py +41 -0
- adam/commands/login.py +7 -4
- adam/commands/logs.py +2 -1
- adam/commands/ls.py +4 -107
- adam/commands/medusa/medusa.py +2 -26
- adam/commands/medusa/medusa_backup.py +2 -2
- adam/commands/medusa/medusa_restore.py +3 -4
- adam/commands/medusa/medusa_show_backupjobs.py +4 -3
- adam/commands/medusa/medusa_show_restorejobs.py +3 -3
- adam/commands/nodetool.py +9 -4
- adam/commands/param_set.py +1 -1
- adam/commands/postgres/postgres.py +42 -43
- adam/commands/postgres/{postgres_session.py → postgres_context.py} +43 -42
- adam/commands/postgres/postgres_utils.py +31 -0
- adam/commands/postgres/psql_completions.py +10 -0
- adam/commands/preview_table.py +18 -40
- adam/commands/pwd.py +2 -28
- adam/commands/reaper/reaper.py +4 -24
- adam/commands/reaper/reaper_restart.py +1 -1
- adam/commands/reaper/reaper_session.py +2 -2
- adam/commands/repair/repair.py +3 -27
- adam/commands/repair/repair_log.py +1 -1
- adam/commands/repair/repair_run.py +2 -2
- adam/commands/repair/repair_scan.py +1 -1
- adam/commands/repair/repair_stop.py +1 -1
- adam/commands/report.py +6 -0
- adam/commands/restart.py +2 -2
- adam/commands/rollout.py +1 -1
- adam/commands/show/show.py +11 -26
- adam/commands/show/show_app_actions.py +3 -0
- adam/commands/show/show_app_id.py +1 -1
- adam/commands/show/show_app_queues.py +3 -2
- adam/commands/show/show_cassandra_status.py +3 -3
- adam/commands/show/show_cassandra_version.py +3 -3
- adam/commands/show/show_host.py +33 -0
- adam/commands/show/show_login.py +3 -0
- adam/commands/show/show_processes.py +1 -1
- adam/commands/show/show_repairs.py +2 -2
- adam/commands/show/show_storage.py +1 -1
- adam/commands/watch.py +1 -1
- adam/config.py +16 -3
- adam/embedded_params.py +1 -1
- adam/pod_exec_result.py +10 -2
- adam/repl.py +127 -117
- adam/repl_commands.py +51 -16
- adam/repl_state.py +276 -55
- adam/sql/__init__.py +0 -0
- adam/sql/sql_completer.py +120 -0
- adam/sql/sql_state_machine.py +617 -0
- adam/sql/term_completer.py +76 -0
- adam/sso/authn_ad.py +1 -1
- adam/sso/cred_cache.py +1 -1
- adam/sso/idp.py +1 -1
- adam/utils.py +83 -2
- adam/utils_athena.py +145 -0
- adam/utils_audits.py +102 -0
- adam/utils_k8s/__init__.py +0 -0
- adam/utils_k8s/app_clusters.py +33 -0
- adam/utils_k8s/app_pods.py +31 -0
- adam/{k8s_utils → utils_k8s}/cassandra_clusters.py +6 -21
- adam/{k8s_utils → utils_k8s}/cassandra_nodes.py +12 -5
- adam/{k8s_utils → utils_k8s}/deployment.py +2 -2
- adam/{k8s_utils → utils_k8s}/kube_context.py +1 -1
- adam/{k8s_utils → utils_k8s}/pods.py +119 -26
- adam/{k8s_utils → utils_k8s}/secrets.py +4 -0
- adam/{k8s_utils → utils_k8s}/statefulsets.py +5 -4
- adam/utils_net.py +24 -0
- adam/utils_repl/__init__.py +0 -0
- adam/utils_repl/automata_completer.py +48 -0
- adam/utils_repl/repl_completer.py +46 -0
- adam/utils_repl/state_machine.py +173 -0
- adam/utils_sqlite.py +101 -0
- adam/version.py +1 -1
- {kaqing-2.0.14.dist-info → kaqing-2.0.145.dist-info}/METADATA +1 -1
- kaqing-2.0.145.dist-info/RECORD +227 -0
- adam/commands/bash.py +0 -87
- adam/commands/cql_utils.py +0 -53
- adam/commands/devices.py +0 -89
- kaqing-2.0.14.dist-info/RECORD +0 -167
- /adam/{k8s_utils → commands/audit}/__init__.py +0 -0
- /adam/{k8s_utils → utils_k8s}/config_maps.py +0 -0
- /adam/{k8s_utils → utils_k8s}/custom_resources.py +0 -0
- /adam/{k8s_utils → utils_k8s}/ingresses.py +0 -0
- /adam/{k8s_utils → utils_k8s}/jobs.py +0 -0
- /adam/{k8s_utils → utils_k8s}/service_accounts.py +0 -0
- /adam/{k8s_utils → utils_k8s}/services.py +0 -0
- /adam/{k8s_utils → utils_k8s}/volumes.py +0 -0
- {kaqing-2.0.14.dist-info → kaqing-2.0.145.dist-info}/WHEEL +0 -0
- {kaqing-2.0.14.dist-info → kaqing-2.0.145.dist-info}/entry_points.txt +0 -0
- {kaqing-2.0.14.dist-info → kaqing-2.0.145.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
from adam.commands.cql.cql_utils import cassandra_keyspaces, cassandra_table_names
|
|
2
|
+
from adam.commands.export.exporter import Exporter
|
|
3
|
+
from adam.commands.export.export_databases import ExportDatabases
|
|
4
|
+
from adam.config import Config
|
|
5
|
+
from adam.repl_state import ReplState
|
|
6
|
+
from adam.sql.sql_completer import SqlCompleter, SqlVariant
|
|
7
|
+
|
|
8
|
+
def cql_completions(state: ReplState) -> dict[str, any]:
|
|
9
|
+
ps = Config().get('cql.alter-tables.gc-grace-periods', '3600,86400,864000,7776000').split(',')
|
|
10
|
+
# warm up caches
|
|
11
|
+
cassandra_keyspaces(state)
|
|
12
|
+
cassandra_table_names(state)
|
|
13
|
+
ExportDatabases.database_names()
|
|
14
|
+
|
|
15
|
+
expandables = {
|
|
16
|
+
'keyspaces': lambda: cassandra_keyspaces(state),
|
|
17
|
+
'table-props': lambda: {
|
|
18
|
+
'GC_GRACE_SECONDS': ps
|
|
19
|
+
},
|
|
20
|
+
'export-dbs': lambda: ExportDatabases.database_names(),
|
|
21
|
+
'export-sessions': lambda: Exporter.export_session_names(state.sts, state.pod, state.namespace),
|
|
22
|
+
'export-sessions-incomplete': lambda: Exporter.export_session_names(state.sts, state.pod, state.namespace, export_state='pending_import'),
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
return SqlCompleter(
|
|
26
|
+
lambda: cassandra_table_names(state),
|
|
27
|
+
expandables=expandables,
|
|
28
|
+
variant=SqlVariant.CQL).completions_for_nesting()
|
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
import functools
|
|
2
|
+
import re
|
|
3
|
+
|
|
4
|
+
from adam.config import Config
|
|
5
|
+
from adam.utils_k8s.cassandra_clusters import CassandraClusters
|
|
6
|
+
from adam.utils_k8s.cassandra_nodes import CassandraNodes
|
|
7
|
+
from adam.utils_k8s.secrets import Secrets
|
|
8
|
+
from adam.pod_exec_result import PodExecResult
|
|
9
|
+
from adam.repl_state import ReplState
|
|
10
|
+
from adam.utils import log2
|
|
11
|
+
|
|
12
|
+
@functools.lru_cache()
|
|
13
|
+
def cassandra_keyspaces(state: ReplState, on_any=True):
|
|
14
|
+
if state.pod:
|
|
15
|
+
Config().wait_log(f'Inspecting Cassandra Keyspaces on {state.pod}...')
|
|
16
|
+
else:
|
|
17
|
+
Config().wait_log(f'Inspecting Cassandra Keyspaces...')
|
|
18
|
+
|
|
19
|
+
r: list[PodExecResult] = run_cql(state, 'describe keyspaces', show_out=False, on_any=on_any)
|
|
20
|
+
if not r:
|
|
21
|
+
log2('No pod is available')
|
|
22
|
+
return []
|
|
23
|
+
|
|
24
|
+
return parse_cql_desc_keyspaces(r.stdout if state.pod else r[0].stdout)
|
|
25
|
+
|
|
26
|
+
def cassandra_table_names(state: ReplState, keyspace = None):
|
|
27
|
+
return [f'{k}.{t}' for k, ts in cassandra_tables(state, on_any=True).items() for t in ts if not keyspace or keyspace == '*' or k == keyspace]
|
|
28
|
+
|
|
29
|
+
@functools.lru_cache()
|
|
30
|
+
def cassandra_tables(state: ReplState, on_any=False) -> dict[str, list[str]]:
|
|
31
|
+
r: list[PodExecResult] = run_cql(state, 'describe tables', show_out=False, on_any=on_any)
|
|
32
|
+
if not r:
|
|
33
|
+
log2('No pod is available')
|
|
34
|
+
return {}
|
|
35
|
+
|
|
36
|
+
return parse_cql_desc_tables(r.stdout if state.pod else r[0].stdout)
|
|
37
|
+
|
|
38
|
+
@functools.lru_cache()
|
|
39
|
+
def table_spec(state: ReplState, table: str, on_any=False) -> 'TableSpec':
|
|
40
|
+
r: list[PodExecResult] = run_cql(state, f'describe table {table}', show_out=False, on_any=on_any)
|
|
41
|
+
if not r:
|
|
42
|
+
log2('No pod is available')
|
|
43
|
+
return None
|
|
44
|
+
|
|
45
|
+
return parse_cql_desc_table(r.stdout if state.pod else r[0].stdout)
|
|
46
|
+
|
|
47
|
+
def run_cql(state: ReplState, cql: str, opts: list = [], show_out = False, use_single_quotes = False, on_any = False, background=False, log_file=None) -> list[PodExecResult]:
|
|
48
|
+
user, pw = Secrets.get_user_pass(state.sts if state.sts else state.pod, state.namespace, secret_path='cql.secret')
|
|
49
|
+
if use_single_quotes:
|
|
50
|
+
command = f"cqlsh -u {user} -p {pw} {' '.join(opts)} -e '{cql}'"
|
|
51
|
+
else:
|
|
52
|
+
command = f'cqlsh -u {user} -p {pw} {" ".join(opts)} -e "{cql}"'
|
|
53
|
+
|
|
54
|
+
if state.pod:
|
|
55
|
+
return CassandraNodes.exec(state.pod, state.namespace, command, show_out=show_out, background=background, log_file=log_file)
|
|
56
|
+
else:
|
|
57
|
+
return CassandraClusters.exec(state.sts, state.namespace, command, show_out=show_out, action='cql', on_any=on_any, background=background, log_file=log_file)
|
|
58
|
+
|
|
59
|
+
def parse_cql_desc_tables(out: str):
|
|
60
|
+
# Keyspace data_endpoint_auth
|
|
61
|
+
# ---------------------------
|
|
62
|
+
# "token"
|
|
63
|
+
|
|
64
|
+
# Keyspace reaper_db
|
|
65
|
+
# ------------------
|
|
66
|
+
# repair_run schema_migration
|
|
67
|
+
# repair_run_by_cluster schema_migration_leader
|
|
68
|
+
|
|
69
|
+
# Keyspace system
|
|
70
|
+
tables_by_keyspace: dict[str, list[str]] = {}
|
|
71
|
+
keyspace = None
|
|
72
|
+
state = 's0'
|
|
73
|
+
for line in out.split('\n'):
|
|
74
|
+
if state == 's0':
|
|
75
|
+
groups = re.match(r'^Keyspace (.*)$', line)
|
|
76
|
+
if groups:
|
|
77
|
+
keyspace = groups[1].strip(' \r')
|
|
78
|
+
state = 's1'
|
|
79
|
+
elif state == 's1':
|
|
80
|
+
if line.startswith('---'):
|
|
81
|
+
state = 's2'
|
|
82
|
+
elif state == 's2':
|
|
83
|
+
if not line.strip(' \r'):
|
|
84
|
+
state = 's0'
|
|
85
|
+
else:
|
|
86
|
+
for table in line.split(' '):
|
|
87
|
+
if t := table.strip(' \r'):
|
|
88
|
+
if not keyspace in tables_by_keyspace:
|
|
89
|
+
tables_by_keyspace[keyspace] = []
|
|
90
|
+
tables_by_keyspace[keyspace].append(t)
|
|
91
|
+
|
|
92
|
+
return tables_by_keyspace
|
|
93
|
+
|
|
94
|
+
def parse_cql_desc_keyspaces(out: str) -> list[str]:
|
|
95
|
+
#
|
|
96
|
+
# Warning: Cannot create directory at `/home/cassandra/.cassandra`. Command history will not be saved. Please check what was the environment property CQL_HISTORY set to.
|
|
97
|
+
#
|
|
98
|
+
#
|
|
99
|
+
# Warning: Using a password on the command line interface can be insecure.
|
|
100
|
+
# Recommendation: use the credentials file to securely provide the password.
|
|
101
|
+
#
|
|
102
|
+
#
|
|
103
|
+
# azops88_db system_auth system_traces
|
|
104
|
+
# reaper_db system_distributed system_views
|
|
105
|
+
# system system_schema system_virtual_schema
|
|
106
|
+
#
|
|
107
|
+
kses = []
|
|
108
|
+
for line in out.split('\n'):
|
|
109
|
+
line = line.strip(' \r')
|
|
110
|
+
if not line:
|
|
111
|
+
continue
|
|
112
|
+
if line.startswith('Warning:'):
|
|
113
|
+
continue
|
|
114
|
+
if line.startswith('Recommendation:'):
|
|
115
|
+
continue
|
|
116
|
+
|
|
117
|
+
for ks in line.split(' '):
|
|
118
|
+
if s := ks.strip(' \r\t'):
|
|
119
|
+
kses.append(s)
|
|
120
|
+
|
|
121
|
+
return kses
|
|
122
|
+
|
|
123
|
+
class ColumnSpec:
|
|
124
|
+
def __init__(self, name: str, type: str, key_index = -1):
|
|
125
|
+
self.name = name
|
|
126
|
+
self.type = type
|
|
127
|
+
self.key_index = key_index
|
|
128
|
+
|
|
129
|
+
def __eq__(self, other):
|
|
130
|
+
if not isinstance(other, ColumnSpec):
|
|
131
|
+
return NotImplemented
|
|
132
|
+
|
|
133
|
+
return self.name == other.name and self.type == other.type and self.key_index == other.key_index
|
|
134
|
+
|
|
135
|
+
class TableSpec:
|
|
136
|
+
def __init__(self, columns: list[ColumnSpec]):
|
|
137
|
+
self.columns = columns
|
|
138
|
+
|
|
139
|
+
def row_key(self):
|
|
140
|
+
for c in self.columns:
|
|
141
|
+
if c.key_index == 0:
|
|
142
|
+
return c.name
|
|
143
|
+
|
|
144
|
+
def keys(self):
|
|
145
|
+
return [c.name for c in self.columns if c.key_index > -1]
|
|
146
|
+
|
|
147
|
+
def parse_cql_desc_table(out: str) -> TableSpec:
|
|
148
|
+
# CREATE TABLE azops88_db.analyticscontainer_dfeevalhistory (
|
|
149
|
+
# id text,
|
|
150
|
+
# columnname text,
|
|
151
|
+
# version bigint static,
|
|
152
|
+
# contentb blob,
|
|
153
|
+
# contentbool boolean,
|
|
154
|
+
# contentn double,
|
|
155
|
+
# contents text,
|
|
156
|
+
# PRIMARY KEY (id, columnname)
|
|
157
|
+
# ) WITH CLUSTERING ORDER BY (columnname ASC)
|
|
158
|
+
# AND additional_write_policy = '99p'
|
|
159
|
+
# AND bloom_filter_fp_chance = 0.1
|
|
160
|
+
# AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
|
|
161
|
+
# AND cdc = false
|
|
162
|
+
# AND comment = ''
|
|
163
|
+
# AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
|
|
164
|
+
# AND compression = {'chunk_length_in_kb': '16', 'class': 'org.apache.cassandra.io.compress.SnappyCompressor'}
|
|
165
|
+
# AND memtable = 'default'
|
|
166
|
+
# AND crc_check_chance = 1.0
|
|
167
|
+
# AND default_time_to_live = 0
|
|
168
|
+
# AND extensions = {}
|
|
169
|
+
# AND gc_grace_seconds = 3600
|
|
170
|
+
# AND max_index_interval = 2048
|
|
171
|
+
# AND memtable_flush_period_in_ms = 0
|
|
172
|
+
# AND min_index_interval = 128
|
|
173
|
+
# AND read_repair = 'BLOCKING'
|
|
174
|
+
# AND speculative_retry = '99p';
|
|
175
|
+
pkeys = {}
|
|
176
|
+
columns: list[ColumnSpec] = []
|
|
177
|
+
|
|
178
|
+
state = 's0'
|
|
179
|
+
for line in out.split('\n'):
|
|
180
|
+
if state == 's0':
|
|
181
|
+
if line.startswith('CREATE TABLE'):
|
|
182
|
+
state = 's1'
|
|
183
|
+
elif state == 's1':
|
|
184
|
+
if line.startswith(')'):
|
|
185
|
+
state = 's2'
|
|
186
|
+
continue
|
|
187
|
+
|
|
188
|
+
groups = re.match(r'^\s*PRIMARY KEY\s*\((.*)\).*$', line)
|
|
189
|
+
if groups:
|
|
190
|
+
pkeys = {n.strip(' '): i for i, n in enumerate(groups[1].strip(' \r').split(','))}
|
|
191
|
+
continue
|
|
192
|
+
|
|
193
|
+
# single key column - name text PRIMARY KEY,
|
|
194
|
+
groups = re.match(r'^\s*(\S*?)\s*(\S*?)\s*PRIMARY KEY,.*$', line)
|
|
195
|
+
if groups:
|
|
196
|
+
columns.append(ColumnSpec(groups[1], groups[2]))
|
|
197
|
+
pkeys[groups[1]] = 0
|
|
198
|
+
else:
|
|
199
|
+
groups = re.match(r'^\s*(\S*?)\s*(\S*?),.*$', line)
|
|
200
|
+
if groups:
|
|
201
|
+
columns.append(ColumnSpec(groups[1], groups[2]))
|
|
202
|
+
elif state == 's2':
|
|
203
|
+
pass
|
|
204
|
+
|
|
205
|
+
for column in columns:
|
|
206
|
+
if column.name in pkeys.keys():
|
|
207
|
+
column.key_index = pkeys[column.name]
|
|
208
|
+
|
|
209
|
+
return TableSpec(columns)
|
|
@@ -2,7 +2,9 @@ import click
|
|
|
2
2
|
|
|
3
3
|
from adam.commands.command import Command
|
|
4
4
|
from adam.commands.command_helpers import ClusterOrPodCommandHelper
|
|
5
|
-
from adam.commands.
|
|
5
|
+
from adam.commands.cql.cql_completions import cql_completions
|
|
6
|
+
from adam.utils_k8s.statefulsets import StatefulSets
|
|
7
|
+
from .cql_utils import run_cql
|
|
6
8
|
from adam.repl_state import ReplState, RequiredState
|
|
7
9
|
from adam.utils import log, log2
|
|
8
10
|
|
|
@@ -32,11 +34,14 @@ class Cqlsh(Command):
|
|
|
32
34
|
if not self.validate_state(state):
|
|
33
35
|
return state
|
|
34
36
|
|
|
37
|
+
background = False
|
|
35
38
|
opts = []
|
|
36
39
|
cqls = []
|
|
37
|
-
for arg in args:
|
|
40
|
+
for index, arg in enumerate(args):
|
|
38
41
|
if arg.startswith('--'):
|
|
39
42
|
opts.append(arg)
|
|
43
|
+
elif index == len(args) -1 and arg == '&':
|
|
44
|
+
background = True
|
|
40
45
|
elif arg != '-e':
|
|
41
46
|
cqls.append(arg)
|
|
42
47
|
if not cqls:
|
|
@@ -50,21 +55,21 @@ class Cqlsh(Command):
|
|
|
50
55
|
return 'no-cql'
|
|
51
56
|
|
|
52
57
|
cql = ' '.join(cqls)
|
|
53
|
-
return run_cql(state, cql, opts, show_out=True)
|
|
58
|
+
return run_cql(state, cql, opts, show_out=True, background=background)
|
|
54
59
|
|
|
55
60
|
def completion(self, state: ReplState) -> dict[str, any]:
|
|
61
|
+
if state.device != state.C:
|
|
62
|
+
# conflicts with psql completions
|
|
63
|
+
return {}
|
|
64
|
+
|
|
56
65
|
if state.sts or state.pod:
|
|
57
|
-
return
|
|
58
|
-
|
|
59
|
-
'insert': {'into': None},
|
|
60
|
-
'select': None,
|
|
61
|
-
'update': None,
|
|
62
|
-
}
|
|
66
|
+
return cql_completions(state) | \
|
|
67
|
+
{f'@{p}': cql_completions(state) for p in StatefulSets.pod_names(state.sts, state.namespace)}
|
|
63
68
|
|
|
64
69
|
return {}
|
|
65
70
|
|
|
66
71
|
def help(self, _: ReplState) -> str:
|
|
67
|
-
return f'
|
|
72
|
+
return f'<cql-statements> [&]\t run cqlsh with queries'
|
|
68
73
|
|
|
69
74
|
class CqlCommandHelper(click.Command):
|
|
70
75
|
def get_help(self, ctx: click.Context):
|
|
@@ -5,8 +5,8 @@ import subprocess
|
|
|
5
5
|
|
|
6
6
|
from adam.apps import Apps
|
|
7
7
|
from adam.config import Config
|
|
8
|
-
from adam.
|
|
9
|
-
from adam.
|
|
8
|
+
from adam.utils_k8s.ingresses import Ingresses
|
|
9
|
+
from adam.utils_k8s.services import Services
|
|
10
10
|
from adam.utils import log2, random_alphanumeric
|
|
11
11
|
|
|
12
12
|
def start_user_code(namespace: str):
|
adam/commands/deploy/deploy.py
CHANGED
|
@@ -4,8 +4,7 @@ from adam.commands.command import Command
|
|
|
4
4
|
from adam.commands.deploy.deploy_pg_agent import DeployPgAgent
|
|
5
5
|
from adam.commands.deploy.deploy_pod import DeployPod
|
|
6
6
|
from .deploy_frontend import DeployFrontend
|
|
7
|
-
from adam.repl_state import ReplState
|
|
8
|
-
from adam.utils import lines_to_tabular, log, log2
|
|
7
|
+
from adam.repl_state import ReplState, RequiredState
|
|
9
8
|
|
|
10
9
|
class Deploy(Command):
|
|
11
10
|
COMMAND = 'deploy'
|
|
@@ -23,22 +22,17 @@ class Deploy(Command):
|
|
|
23
22
|
def command(self):
|
|
24
23
|
return Deploy.COMMAND
|
|
25
24
|
|
|
25
|
+
def required(self):
|
|
26
|
+
return RequiredState.NAMESPACE
|
|
27
|
+
|
|
26
28
|
def run(self, cmd: str, state: ReplState):
|
|
27
29
|
if not(args := self.args(cmd)):
|
|
28
30
|
return super().run(cmd, state)
|
|
29
31
|
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
if state.in_repl:
|
|
33
|
-
log(lines_to_tabular([c.help(ReplState()) for c in Deploy.cmd_list()], separator=':'))
|
|
32
|
+
if not self.validate_state(state):
|
|
33
|
+
return state
|
|
34
34
|
|
|
35
|
-
|
|
36
|
-
else:
|
|
37
|
-
# head with the Chain of Responsibility pattern
|
|
38
|
-
cmds = Command.chain(Deploy.cmd_list())
|
|
39
|
-
if not cmds.run(cmd, state):
|
|
40
|
-
log2('* Command is missing.')
|
|
41
|
-
Command.display_help()
|
|
35
|
+
return super().intermediate_run(cmd, state, args, Deploy.cmd_list())
|
|
42
36
|
|
|
43
37
|
def cmd_list():
|
|
44
38
|
return [DeployFrontend(), DeployPod(), DeployPgAgent()]
|
|
@@ -49,13 +43,6 @@ class Deploy(Command):
|
|
|
49
43
|
|
|
50
44
|
return {}
|
|
51
45
|
|
|
52
|
-
def help(self, _: ReplState):
|
|
53
|
-
return None
|
|
54
|
-
|
|
55
46
|
class DeployCommandHelper(click.Command):
|
|
56
47
|
def get_help(self, ctx: click.Context):
|
|
57
|
-
|
|
58
|
-
log()
|
|
59
|
-
log('Sub-Commands:')
|
|
60
|
-
|
|
61
|
-
log(lines_to_tabular([c.help(ReplState()).replace(f'{Deploy.COMMAND} ', ' ', 1) for c in Deploy.cmd_list()], separator=':'))
|
|
48
|
+
Command.intermediate_help(super().get_help(ctx), Deploy.COMMAND, Deploy.cmd_list())
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
from adam.commands.command import Command
|
|
2
|
-
from adam.commands.postgres.
|
|
2
|
+
from adam.commands.postgres.postgres_context import PostgresContext
|
|
3
3
|
from adam.config import Config
|
|
4
4
|
from adam.repl_state import ReplState, RequiredState
|
|
5
5
|
|
|
@@ -29,10 +29,10 @@ class DeployPgAgent(Command):
|
|
|
29
29
|
if not self.validate_state(state):
|
|
30
30
|
return state
|
|
31
31
|
|
|
32
|
-
|
|
32
|
+
PostgresContext.deploy_pg_agent(Config().get('pg.agent.name', 'ops-pg-agent'), state.namespace)
|
|
33
33
|
|
|
34
34
|
def completion(self, state: ReplState):
|
|
35
35
|
return super().completion(state)
|
|
36
36
|
|
|
37
37
|
def help(self, _: ReplState):
|
|
38
|
-
return f'{DeployPgAgent.COMMAND}\t deploy
|
|
38
|
+
return f'{DeployPgAgent.COMMAND}\t deploy Postgres agent'
|
|
@@ -1,17 +1,17 @@
|
|
|
1
1
|
from kubernetes import client
|
|
2
|
-
import yaml
|
|
3
2
|
|
|
4
3
|
from adam.commands.command import Command
|
|
5
|
-
from adam.commands.deploy.deploy_utils import
|
|
4
|
+
from adam.commands.deploy.deploy_utils import deploy_frontend, gen_labels
|
|
5
|
+
from adam.commands.deploy.undeploy_pod import UndeployPod
|
|
6
6
|
from adam.config import Config
|
|
7
|
-
from adam.
|
|
8
|
-
from adam.
|
|
9
|
-
from adam.
|
|
10
|
-
from adam.
|
|
11
|
-
from adam.
|
|
12
|
-
from adam.
|
|
7
|
+
from adam.utils_k8s.config_maps import ConfigMaps
|
|
8
|
+
from adam.utils_k8s.deployment import Deployments
|
|
9
|
+
from adam.utils_k8s.kube_context import KubeContext
|
|
10
|
+
from adam.utils_k8s.pods import Pods
|
|
11
|
+
from adam.utils_k8s.service_accounts import ServiceAccounts
|
|
12
|
+
from adam.utils_k8s.volumes import ConfigMapMount
|
|
13
13
|
from adam.repl_state import ReplState, RequiredState
|
|
14
|
-
from adam.utils import log2
|
|
14
|
+
from adam.utils import ing, log2
|
|
15
15
|
|
|
16
16
|
class DeployPod(Command):
|
|
17
17
|
COMMAND = 'deploy pod'
|
|
@@ -39,6 +39,10 @@ class DeployPod(Command):
|
|
|
39
39
|
if not self.validate_state(state):
|
|
40
40
|
return state
|
|
41
41
|
|
|
42
|
+
args, forced = Command.extract_options(args, '--force')
|
|
43
|
+
if forced:
|
|
44
|
+
UndeployPod().run(UndeployPod.COMMAND, state)
|
|
45
|
+
|
|
42
46
|
if KubeContext.in_cluster():
|
|
43
47
|
log2('This is doable only from outside of the Kubernetes cluster.')
|
|
44
48
|
return state
|
|
@@ -49,10 +53,8 @@ class DeployPod(Command):
|
|
|
49
53
|
label_selector = Config().get('pod.label-selector', 'run=ops')
|
|
50
54
|
labels = gen_labels(label_selector)
|
|
51
55
|
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
sa_proto, labels=labels,
|
|
55
|
-
add_cluster_roles=additional_cluster_roles))
|
|
56
|
+
with ing('Creating service account'):
|
|
57
|
+
ServiceAccounts.replicate(sa_name, state.namespace, sa_proto, labels=labels, add_cluster_roles=additional_cluster_roles)
|
|
56
58
|
|
|
57
59
|
settings_filename = 'settings.yaml'
|
|
58
60
|
settings_path = f'/kaqing/{settings_filename}'
|
|
@@ -75,10 +77,8 @@ class DeployPod(Command):
|
|
|
75
77
|
map_data = {
|
|
76
78
|
settings_filename : settings_data
|
|
77
79
|
}
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
map_data,
|
|
81
|
-
labels=labels))
|
|
80
|
+
with ing('Creating config map'):
|
|
81
|
+
ConfigMaps.create(cm_name, state.namespace, map_data, labels=labels)
|
|
82
82
|
|
|
83
83
|
pod_name = Config().get('pod.name', 'ops')
|
|
84
84
|
image = Config().get('pod.image', 'seanahnsf/kaqing')
|
|
@@ -87,14 +87,15 @@ class DeployPod(Command):
|
|
|
87
87
|
add=["SYS_PTRACE"]
|
|
88
88
|
)
|
|
89
89
|
)
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
90
|
+
with ing('Creating deployment'):
|
|
91
|
+
Deployments.create(state.namespace,
|
|
92
|
+
pod_name,
|
|
93
|
+
image,
|
|
94
|
+
env={'NAMESPACE': state.namespace},
|
|
95
|
+
container_security_context=security_context,
|
|
96
|
+
labels=labels,
|
|
97
|
+
sa_name=sa_name,
|
|
98
|
+
config_map_mount=ConfigMapMount(cm_name, settings_filename, settings_path))
|
|
98
99
|
|
|
99
100
|
uri = deploy_frontend(pod_name, state.namespace, label_selector)
|
|
100
101
|
|
|
@@ -103,7 +104,7 @@ class DeployPod(Command):
|
|
|
103
104
|
return state
|
|
104
105
|
|
|
105
106
|
def completion(self, state: ReplState):
|
|
106
|
-
return super().completion(state)
|
|
107
|
+
return super().completion(state, {'--force': None})
|
|
107
108
|
|
|
108
109
|
def help(self, _: ReplState):
|
|
109
|
-
return f'{DeployPod.COMMAND}\t deploy pod'
|
|
110
|
+
return f'{DeployPod.COMMAND} [--force]\t deploy Ops pod, --force to undeploy first'
|
|
@@ -1,39 +1,29 @@
|
|
|
1
|
-
from typing import Callable
|
|
2
1
|
from adam.app_session import AppSession
|
|
3
|
-
from adam.
|
|
4
|
-
from adam.
|
|
5
|
-
from adam.
|
|
2
|
+
from adam.utils import ing
|
|
3
|
+
from adam.utils_k8s.ingresses import Ingresses
|
|
4
|
+
from adam.utils_k8s.services import Services
|
|
6
5
|
|
|
7
6
|
def deploy_frontend(name: str, namespace: str, label_selector: str):
|
|
8
7
|
app_session: AppSession = AppSession.create('c3', 'c3', namespace)
|
|
9
8
|
port = 7678
|
|
10
9
|
labels = gen_labels(label_selector)
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
10
|
+
with ing('Creating service'):
|
|
11
|
+
Services.create_service(name, namespace, port, labels, labels=labels)
|
|
12
|
+
with ing('Creating ingress'):
|
|
13
|
+
Ingresses.create_ingress(name, namespace, app_session.host, '/c3/c3/ops($|/)', port, annotations={
|
|
14
|
+
'kubernetes.io/ingress.class': 'nginx',
|
|
15
|
+
'nginx.ingress.kubernetes.io/use-regex': 'true',
|
|
16
|
+
'nginx.ingress.kubernetes.io/rewrite-target': '/'
|
|
17
|
+
}, labels=labels)
|
|
17
18
|
|
|
18
19
|
return f'https://{app_session.host}/c3/c3/ops'
|
|
19
20
|
|
|
20
21
|
def undeploy_frontend(namespace: str, label_selector: str):
|
|
21
|
-
|
|
22
|
-
|
|
22
|
+
with ing('Deleting ingress'):
|
|
23
|
+
Ingresses.delete_ingresses(namespace, label_selector=label_selector)
|
|
24
|
+
with ing('Deleting service'):
|
|
25
|
+
Services.delete_services(namespace, label_selector=label_selector)
|
|
23
26
|
|
|
24
27
|
def gen_labels(label_selector: str):
|
|
25
28
|
kv = label_selector.split('=')
|
|
26
|
-
return {kv[0]: kv[1]}
|
|
27
|
-
|
|
28
|
-
def creating(name: str, body: Callable[[], None]):
|
|
29
|
-
log2(f'Creating {name}...', nl=False)
|
|
30
|
-
body()
|
|
31
|
-
log2(' OK')
|
|
32
|
-
|
|
33
|
-
def deleting(name: str, body: Callable[[], None]):
|
|
34
|
-
try:
|
|
35
|
-
log2(f'Deleting {name}...', nl=False)
|
|
36
|
-
body()
|
|
37
|
-
log2(' OK')
|
|
38
|
-
except Exception as e:
|
|
39
|
-
log2(e)
|
|
29
|
+
return {kv[0]: kv[1]}
|
adam/commands/deploy/undeploy.py
CHANGED
|
@@ -4,8 +4,7 @@ from adam.commands.command import Command
|
|
|
4
4
|
from adam.commands.deploy.undeploy_frontend import UndeployFrontend
|
|
5
5
|
from adam.commands.deploy.undeploy_pg_agent import UndeployPgAgent
|
|
6
6
|
from adam.commands.deploy.undeploy_pod import UndeployPod
|
|
7
|
-
from adam.repl_state import ReplState
|
|
8
|
-
from adam.utils import lines_to_tabular, log, log2
|
|
7
|
+
from adam.repl_state import ReplState, RequiredState
|
|
9
8
|
|
|
10
9
|
class Undeploy(Command):
|
|
11
10
|
COMMAND = 'undeploy'
|
|
@@ -23,22 +22,17 @@ class Undeploy(Command):
|
|
|
23
22
|
def command(self):
|
|
24
23
|
return Undeploy.COMMAND
|
|
25
24
|
|
|
25
|
+
def required(self):
|
|
26
|
+
return RequiredState.NAMESPACE
|
|
27
|
+
|
|
26
28
|
def run(self, cmd: str, state: ReplState):
|
|
27
29
|
if not(args := self.args(cmd)):
|
|
28
30
|
return super().run(cmd, state)
|
|
29
31
|
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
if state.in_repl:
|
|
33
|
-
log(lines_to_tabular([c.help(ReplState()) for c in Undeploy.cmd_list()], separator=':'))
|
|
32
|
+
if not self.validate_state(state):
|
|
33
|
+
return state
|
|
34
34
|
|
|
35
|
-
|
|
36
|
-
else:
|
|
37
|
-
# head with the Chain of Responsibility pattern
|
|
38
|
-
cmds = Command.chain(Undeploy.cmd_list())
|
|
39
|
-
if not cmds.run(cmd, state):
|
|
40
|
-
log2('* Command is missing.')
|
|
41
|
-
Command.display_help()
|
|
35
|
+
return super().intermediate_run(cmd, state, args, Undeploy.cmd_list())
|
|
42
36
|
|
|
43
37
|
def cmd_list():
|
|
44
38
|
return [UndeployFrontend(), UndeployPod(), UndeployPgAgent()]
|
|
@@ -49,13 +43,6 @@ class Undeploy(Command):
|
|
|
49
43
|
|
|
50
44
|
return {}
|
|
51
45
|
|
|
52
|
-
def help(self, _: ReplState):
|
|
53
|
-
return None
|
|
54
|
-
|
|
55
46
|
class UndeployCommandHelper(click.Command):
|
|
56
47
|
def get_help(self, ctx: click.Context):
|
|
57
|
-
|
|
58
|
-
log()
|
|
59
|
-
log('Sub-Commands:')
|
|
60
|
-
|
|
61
|
-
log(lines_to_tabular([c.help(ReplState()).replace(f'{Undeploy.COMMAND} ', ' ', 1) for c in Undeploy.cmd_list()], separator=':'))
|
|
48
|
+
Command.intermediate_help(super().get_help(ctx), Undeploy.COMMAND, Undeploy.cmd_list())
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
from adam.commands.command import Command
|
|
2
|
-
from adam.commands.postgres.
|
|
2
|
+
from adam.commands.postgres.postgres_context import PostgresContext
|
|
3
3
|
from adam.config import Config
|
|
4
4
|
from adam.repl_state import ReplState, RequiredState
|
|
5
|
+
from adam.utils import ing
|
|
5
6
|
|
|
6
7
|
class UndeployPgAgent(Command):
|
|
7
8
|
COMMAND = 'undeploy pg-agent'
|
|
@@ -29,7 +30,8 @@ class UndeployPgAgent(Command):
|
|
|
29
30
|
if not self.validate_state(state):
|
|
30
31
|
return state
|
|
31
32
|
|
|
32
|
-
|
|
33
|
+
with ing('Deleting pod'):
|
|
34
|
+
PostgresContext.undeploy_pg_agent(Config().get('pg.agent.name', 'ops-pg-agent'), state.namespace)
|
|
33
35
|
|
|
34
36
|
return state
|
|
35
37
|
|
|
@@ -37,4 +39,4 @@ class UndeployPgAgent(Command):
|
|
|
37
39
|
return super().completion(state)
|
|
38
40
|
|
|
39
41
|
def help(self, _: ReplState):
|
|
40
|
-
return f'{UndeployPgAgent.COMMAND}\t undeploy
|
|
42
|
+
return f'{UndeployPgAgent.COMMAND}\t undeploy Postgres agent'
|
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
from adam.commands.command import Command
|
|
2
|
-
from adam.commands.deploy.deploy_utils import undeploy_frontend
|
|
2
|
+
from adam.commands.deploy.deploy_utils import undeploy_frontend
|
|
3
3
|
from adam.config import Config
|
|
4
|
-
from adam.
|
|
5
|
-
from adam.
|
|
6
|
-
from adam.
|
|
7
|
-
from adam.
|
|
4
|
+
from adam.utils import ing
|
|
5
|
+
from adam.utils_k8s.config_maps import ConfigMaps
|
|
6
|
+
from adam.utils_k8s.deployment import Deployments
|
|
7
|
+
from adam.utils_k8s.pods import Pods
|
|
8
|
+
from adam.utils_k8s.service_accounts import ServiceAccounts
|
|
8
9
|
from adam.repl_state import ReplState, RequiredState
|
|
9
10
|
|
|
10
11
|
class UndeployPod(Command):
|
|
@@ -34,10 +35,11 @@ class UndeployPod(Command):
|
|
|
34
35
|
return state
|
|
35
36
|
|
|
36
37
|
label_selector = Config().get('pod.label-selector', 'run=ops')
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
38
|
+
with ing('Deleting service account'):
|
|
39
|
+
ServiceAccounts.delete(state.namespace, label_selector=label_selector)
|
|
40
|
+
with ing('Deleting config map'): ConfigMaps.delete_with_selector(state.namespace, label_selector)
|
|
41
|
+
with ing('Deleting deployment'): Deployments.delete_with_selector(state.namespace, label_selector, grace_period_seconds=0)
|
|
42
|
+
with ing('Deleting pod'): Pods.delete_with_selector(state.namespace, label_selector, grace_period_seconds=0)
|
|
41
43
|
undeploy_frontend(state.namespace, label_selector)
|
|
42
44
|
|
|
43
45
|
return state
|
|
@@ -46,4 +48,4 @@ class UndeployPod(Command):
|
|
|
46
48
|
return super().completion(state)
|
|
47
49
|
|
|
48
50
|
def help(self, _: ReplState):
|
|
49
|
-
return f'{UndeployPod.COMMAND}\t undeploy pod'
|
|
51
|
+
return f'{UndeployPod.COMMAND}\t undeploy Ops pod'
|