kaqing 1.98.15__py3-none-any.whl → 2.0.145__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of kaqing might be problematic. Click here for more details.
- adam/app_session.py +1 -1
- adam/apps.py +2 -2
- adam/batch.py +30 -31
- adam/checks/check_utils.py +4 -4
- adam/checks/compactionstats.py +1 -1
- adam/checks/cpu.py +2 -2
- adam/checks/disk.py +1 -1
- adam/checks/gossip.py +1 -1
- adam/checks/memory.py +3 -3
- adam/checks/status.py +1 -1
- adam/commands/alter_tables.py +81 -0
- adam/commands/app.py +3 -3
- adam/commands/app_ping.py +2 -2
- adam/commands/audit/audit.py +86 -0
- adam/commands/audit/audit_repair_tables.py +77 -0
- adam/commands/audit/audit_run.py +58 -0
- adam/commands/audit/show_last10.py +51 -0
- adam/commands/audit/show_slow10.py +50 -0
- adam/commands/audit/show_top10.py +48 -0
- adam/commands/audit/utils_show_top10.py +59 -0
- adam/commands/bash/bash.py +133 -0
- adam/commands/bash/bash_completer.py +93 -0
- adam/commands/cat.py +56 -0
- adam/commands/cd.py +12 -82
- adam/commands/check.py +6 -0
- adam/commands/cli_commands.py +3 -3
- adam/commands/code.py +60 -0
- adam/commands/command.py +48 -12
- adam/commands/commands_utils.py +4 -5
- adam/commands/cql/cql_completions.py +28 -0
- adam/commands/cql/cql_utils.py +209 -0
- adam/commands/{cqlsh.py → cql/cqlsh.py} +15 -10
- adam/commands/deploy/__init__.py +0 -0
- adam/commands/{frontend → deploy}/code_start.py +1 -1
- adam/commands/{frontend → deploy}/code_stop.py +1 -1
- adam/commands/{frontend → deploy}/code_utils.py +2 -2
- adam/commands/deploy/deploy.py +48 -0
- adam/commands/deploy/deploy_frontend.py +52 -0
- adam/commands/deploy/deploy_pg_agent.py +38 -0
- adam/commands/deploy/deploy_pod.py +110 -0
- adam/commands/deploy/deploy_utils.py +29 -0
- adam/commands/deploy/undeploy.py +48 -0
- adam/commands/deploy/undeploy_frontend.py +41 -0
- adam/commands/deploy/undeploy_pg_agent.py +42 -0
- adam/commands/deploy/undeploy_pod.py +51 -0
- adam/commands/devices/__init__.py +0 -0
- adam/commands/devices/device.py +27 -0
- adam/commands/devices/device_app.py +146 -0
- adam/commands/devices/device_auit_log.py +43 -0
- adam/commands/devices/device_cass.py +145 -0
- adam/commands/devices/device_export.py +86 -0
- adam/commands/devices/device_postgres.py +109 -0
- adam/commands/devices/devices.py +25 -0
- adam/commands/export/__init__.py +0 -0
- adam/commands/export/clean_up_export_session.py +53 -0
- adam/commands/{frontend/teardown_frontend.py → export/clean_up_export_sessions.py} +9 -11
- adam/commands/export/drop_export_database.py +58 -0
- adam/commands/export/drop_export_databases.py +46 -0
- adam/commands/export/export.py +83 -0
- adam/commands/export/export_databases.py +170 -0
- adam/commands/export/export_select.py +85 -0
- adam/commands/export/export_select_x.py +54 -0
- adam/commands/export/export_use.py +55 -0
- adam/commands/export/exporter.py +364 -0
- adam/commands/export/import_session.py +68 -0
- adam/commands/export/importer.py +67 -0
- adam/commands/export/importer_athena.py +80 -0
- adam/commands/export/importer_sqlite.py +47 -0
- adam/commands/export/show_column_counts.py +63 -0
- adam/commands/export/show_export_databases.py +39 -0
- adam/commands/export/show_export_session.py +51 -0
- adam/commands/export/show_export_sessions.py +47 -0
- adam/commands/export/utils_export.py +291 -0
- adam/commands/help.py +12 -7
- adam/commands/issues.py +6 -0
- adam/commands/kubectl.py +41 -0
- adam/commands/login.py +9 -5
- adam/commands/logs.py +2 -1
- adam/commands/ls.py +4 -107
- adam/commands/medusa/medusa.py +2 -26
- adam/commands/medusa/medusa_backup.py +2 -2
- adam/commands/medusa/medusa_restore.py +3 -4
- adam/commands/medusa/medusa_show_backupjobs.py +4 -3
- adam/commands/medusa/medusa_show_restorejobs.py +3 -3
- adam/commands/nodetool.py +9 -4
- adam/commands/param_set.py +1 -1
- adam/commands/postgres/postgres.py +42 -43
- adam/commands/postgres/postgres_context.py +248 -0
- adam/commands/postgres/postgres_preview.py +0 -1
- adam/commands/postgres/postgres_utils.py +31 -0
- adam/commands/postgres/psql_completions.py +10 -0
- adam/commands/preview_table.py +18 -40
- adam/commands/pwd.py +2 -28
- adam/commands/reaper/reaper.py +4 -24
- adam/commands/reaper/reaper_restart.py +1 -1
- adam/commands/reaper/reaper_session.py +2 -2
- adam/commands/repair/repair.py +3 -27
- adam/commands/repair/repair_log.py +1 -1
- adam/commands/repair/repair_run.py +2 -2
- adam/commands/repair/repair_scan.py +2 -7
- adam/commands/repair/repair_stop.py +1 -1
- adam/commands/report.py +6 -0
- adam/commands/restart.py +2 -2
- adam/commands/rollout.py +1 -1
- adam/commands/shell.py +33 -0
- adam/commands/show/show.py +11 -26
- adam/commands/show/show_app_actions.py +3 -0
- adam/commands/show/show_app_id.py +1 -1
- adam/commands/show/show_app_queues.py +3 -2
- adam/commands/show/show_cassandra_status.py +3 -3
- adam/commands/show/show_cassandra_version.py +3 -3
- adam/commands/show/show_commands.py +4 -1
- adam/commands/show/show_host.py +33 -0
- adam/commands/show/show_login.py +3 -0
- adam/commands/show/show_processes.py +1 -1
- adam/commands/show/show_repairs.py +2 -2
- adam/commands/show/show_storage.py +1 -1
- adam/commands/watch.py +1 -1
- adam/config.py +16 -3
- adam/embedded_params.py +1 -1
- adam/pod_exec_result.py +10 -2
- adam/repl.py +132 -117
- adam/repl_commands.py +62 -18
- adam/repl_state.py +276 -55
- adam/sql/__init__.py +0 -0
- adam/sql/sql_completer.py +120 -0
- adam/sql/sql_state_machine.py +617 -0
- adam/sql/term_completer.py +76 -0
- adam/sso/authenticator.py +1 -1
- adam/sso/authn_ad.py +36 -56
- adam/sso/authn_okta.py +6 -32
- adam/sso/cred_cache.py +1 -1
- adam/sso/idp.py +74 -9
- adam/sso/idp_login.py +2 -2
- adam/sso/idp_session.py +10 -7
- adam/utils.py +85 -4
- adam/utils_athena.py +145 -0
- adam/utils_audits.py +102 -0
- adam/utils_k8s/__init__.py +0 -0
- adam/utils_k8s/app_clusters.py +33 -0
- adam/utils_k8s/app_pods.py +31 -0
- adam/{k8s_utils → utils_k8s}/cassandra_clusters.py +6 -21
- adam/{k8s_utils → utils_k8s}/cassandra_nodes.py +12 -5
- adam/utils_k8s/config_maps.py +34 -0
- adam/utils_k8s/deployment.py +56 -0
- adam/{k8s_utils → utils_k8s}/jobs.py +1 -1
- adam/{k8s_utils → utils_k8s}/kube_context.py +1 -1
- adam/utils_k8s/pods.py +342 -0
- adam/{k8s_utils → utils_k8s}/secrets.py +4 -0
- adam/utils_k8s/service_accounts.py +169 -0
- adam/{k8s_utils → utils_k8s}/statefulsets.py +5 -4
- adam/{k8s_utils → utils_k8s}/volumes.py +9 -0
- adam/utils_net.py +24 -0
- adam/utils_repl/__init__.py +0 -0
- adam/utils_repl/automata_completer.py +48 -0
- adam/utils_repl/repl_completer.py +46 -0
- adam/utils_repl/state_machine.py +173 -0
- adam/utils_sqlite.py +101 -0
- adam/version.py +1 -1
- {kaqing-1.98.15.dist-info → kaqing-2.0.145.dist-info}/METADATA +1 -1
- kaqing-2.0.145.dist-info/RECORD +227 -0
- adam/commands/bash.py +0 -87
- adam/commands/cql_utils.py +0 -53
- adam/commands/devices.py +0 -89
- adam/commands/frontend/setup.py +0 -60
- adam/commands/frontend/setup_frontend.py +0 -58
- adam/commands/frontend/teardown.py +0 -61
- adam/commands/postgres/postgres_session.py +0 -225
- adam/commands/user_entry.py +0 -77
- adam/k8s_utils/pods.py +0 -211
- kaqing-1.98.15.dist-info/RECORD +0 -160
- /adam/commands/{frontend → audit}/__init__.py +0 -0
- /adam/{k8s_utils → commands/bash}/__init__.py +0 -0
- /adam/{medusa_show_restorejobs.py → commands/cql/__init__.py} +0 -0
- /adam/{k8s_utils → utils_k8s}/custom_resources.py +0 -0
- /adam/{k8s_utils → utils_k8s}/ingresses.py +0 -0
- /adam/{k8s_utils → utils_k8s}/services.py +0 -0
- {kaqing-1.98.15.dist-info → kaqing-2.0.145.dist-info}/WHEEL +0 -0
- {kaqing-1.98.15.dist-info → kaqing-2.0.145.dist-info}/entry_points.txt +0 -0
- {kaqing-1.98.15.dist-info → kaqing-2.0.145.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,248 @@
|
|
|
1
|
+
from datetime import datetime
|
|
2
|
+
import functools
|
|
3
|
+
import re
|
|
4
|
+
import subprocess
|
|
5
|
+
|
|
6
|
+
from adam.config import Config
|
|
7
|
+
from adam.repl_session import ReplSession
|
|
8
|
+
from adam.utils_k8s.kube_context import KubeContext
|
|
9
|
+
from adam.utils_k8s.pods import Pods
|
|
10
|
+
from adam.utils_k8s.secrets import Secrets
|
|
11
|
+
from adam.utils import log2
|
|
12
|
+
|
|
13
|
+
class PostgresContext:
|
|
14
|
+
def apply(namespace: str, path: str, arg: str = None) -> 'PostgresContext':
|
|
15
|
+
context = PostgresContext(namespace, path)
|
|
16
|
+
|
|
17
|
+
if arg:
|
|
18
|
+
if arg == '..':
|
|
19
|
+
if context.db:
|
|
20
|
+
context.db = None
|
|
21
|
+
else:
|
|
22
|
+
context.host = None
|
|
23
|
+
else:
|
|
24
|
+
tks = arg.split('@')
|
|
25
|
+
if not context.host:
|
|
26
|
+
context.host = tks[0]
|
|
27
|
+
else:
|
|
28
|
+
context.db = tks[0]
|
|
29
|
+
|
|
30
|
+
if not namespace and tks[1]:
|
|
31
|
+
context.namespace = tks[1]
|
|
32
|
+
|
|
33
|
+
return context
|
|
34
|
+
|
|
35
|
+
def __init__(self, ns: str, path: str):
|
|
36
|
+
self.namespace = ns
|
|
37
|
+
self.conn_details = None
|
|
38
|
+
self.host = None
|
|
39
|
+
self.db = None
|
|
40
|
+
|
|
41
|
+
if path:
|
|
42
|
+
tks = path.split('/')
|
|
43
|
+
hn = tks[0].split('@')
|
|
44
|
+
self.host = hn[0]
|
|
45
|
+
if len(hn) > 1 and not ns:
|
|
46
|
+
self.namespace = hn[1]
|
|
47
|
+
|
|
48
|
+
if len(tks) > 1:
|
|
49
|
+
self.db = tks[1]
|
|
50
|
+
|
|
51
|
+
def path(self):
|
|
52
|
+
if not self.host:
|
|
53
|
+
return None
|
|
54
|
+
|
|
55
|
+
d = self.host
|
|
56
|
+
if not self.db:
|
|
57
|
+
return d
|
|
58
|
+
|
|
59
|
+
return f'{self.host}/{self.db}'
|
|
60
|
+
|
|
61
|
+
def hosts(ns: str):
|
|
62
|
+
return PostgresContext.hosts_for_namespace(ns)
|
|
63
|
+
|
|
64
|
+
@functools.lru_cache()
|
|
65
|
+
def hosts_for_namespace(ns: str):
|
|
66
|
+
ss = Secrets.list_secrets(ns, name_pattern=Config().get('pg.name-pattern', '^{namespace}.*k8spg.*'))
|
|
67
|
+
|
|
68
|
+
def excludes(name: str):
|
|
69
|
+
exs = Config().get('pg.excludes', '.helm., -admin-secret')
|
|
70
|
+
if exs:
|
|
71
|
+
for ex in exs.split(','):
|
|
72
|
+
if ex.strip(' ') in name:
|
|
73
|
+
return True
|
|
74
|
+
|
|
75
|
+
return False
|
|
76
|
+
|
|
77
|
+
return [s for s in ss if not excludes(s)]
|
|
78
|
+
|
|
79
|
+
def databases(self):
|
|
80
|
+
dbs = []
|
|
81
|
+
# List of databases
|
|
82
|
+
# Name | Owner | Encoding | Collate | Ctype | ICU Locale | Locale Provider | Access privileges
|
|
83
|
+
# ---------------------------------------+----------+----------+-------------+-------------+------------+-----------------+-----------------------
|
|
84
|
+
# postgres | postgres | UTF8 | en_US.UTF-8 | en_US.UTF-8 | | libc |
|
|
85
|
+
# stgawsscpsr_c3_c3 | postgres | UTF8 | C | C | | libc |
|
|
86
|
+
# template1 | postgres | UTF8 | en_US.UTF-8 | en_US.UTF-8 | | libc | =c/postgres +
|
|
87
|
+
# | | | | | | | postgres=CTc/postgres
|
|
88
|
+
# (48 rows)
|
|
89
|
+
if r := self.run_sql('\l', show_out=False):
|
|
90
|
+
s = 0
|
|
91
|
+
for line in r.stdout.split('\n'):
|
|
92
|
+
line: str = line.strip(' \r')
|
|
93
|
+
if s == 0:
|
|
94
|
+
if 'List of databases' in line:
|
|
95
|
+
s = 1
|
|
96
|
+
elif s == 1:
|
|
97
|
+
if 'Name' in line and 'Owner' in line and 'Encoding' in line:
|
|
98
|
+
s = 2
|
|
99
|
+
elif s == 2:
|
|
100
|
+
if line.startswith('---------'):
|
|
101
|
+
s = 3
|
|
102
|
+
elif s == 3:
|
|
103
|
+
groups = re.match(r'^\s*(\S*)\s*\|\s*(\S*)\s*\|.*', line)
|
|
104
|
+
if groups and groups[1] != '|':
|
|
105
|
+
dbs.append({'name': groups[1], 'owner': groups[2]})
|
|
106
|
+
|
|
107
|
+
return dbs
|
|
108
|
+
|
|
109
|
+
def tables(self):
|
|
110
|
+
dbs = []
|
|
111
|
+
# List of relations
|
|
112
|
+
# Schema | Name | Type | Owner
|
|
113
|
+
# ----------+------------------------------------------------------------+-------+---------------
|
|
114
|
+
# postgres | c3_2_admin_aclpriv | table | postgres
|
|
115
|
+
# postgres | c3_2_admin_aclpriv_a | table | postgres
|
|
116
|
+
if r := self.run_sql('\dt', show_out=False):
|
|
117
|
+
s = 0
|
|
118
|
+
for line in r.stdout.split('\n'):
|
|
119
|
+
line: str = line.strip(' \r')
|
|
120
|
+
if s == 0:
|
|
121
|
+
if 'List of relations' in line:
|
|
122
|
+
s = 1
|
|
123
|
+
elif s == 1:
|
|
124
|
+
if 'Schema' in line and 'Name' in line and 'Type' in line:
|
|
125
|
+
s = 2
|
|
126
|
+
elif s == 2:
|
|
127
|
+
if line.startswith('---------'):
|
|
128
|
+
s = 3
|
|
129
|
+
elif s == 3:
|
|
130
|
+
groups = re.match(r'^\s*(\S*)\s*\|\s*(\S*)\s*\|.*', line)
|
|
131
|
+
if groups and groups[1] != '|':
|
|
132
|
+
dbs.append({'schema': groups[1], 'name': groups[2]})
|
|
133
|
+
|
|
134
|
+
return dbs
|
|
135
|
+
|
|
136
|
+
def run_sql(self, sql: str, show_out = True, background = False):
|
|
137
|
+
db = self.db if self.db else PostgresContext.default_db()
|
|
138
|
+
|
|
139
|
+
if KubeContext.in_cluster():
|
|
140
|
+
cmd1 = f'env PGPASSWORD={self.password()} psql -h {self.endpoint()} -p {self.port()} -U {self.username()} {db} --pset pager=off -c'
|
|
141
|
+
log2(f'{cmd1} "{sql}"')
|
|
142
|
+
# remove double quotes from the sql argument
|
|
143
|
+
cmd = cmd1.split(' ') + [sql]
|
|
144
|
+
|
|
145
|
+
r = subprocess.run(cmd, capture_output=not background, text=True)
|
|
146
|
+
if show_out:
|
|
147
|
+
log2(r.stdout)
|
|
148
|
+
log2(r.stderr)
|
|
149
|
+
|
|
150
|
+
return r
|
|
151
|
+
else:
|
|
152
|
+
ns = self.namespace
|
|
153
|
+
pod_name = Config().get('pg.agent.name', 'ops-pg-agent')
|
|
154
|
+
|
|
155
|
+
if Config().get('pg.agent.just-in-time', False):
|
|
156
|
+
if not PostgresContext.deploy_pg_agent(pod_name, ns):
|
|
157
|
+
return
|
|
158
|
+
|
|
159
|
+
real_pod_name = pod_name
|
|
160
|
+
try:
|
|
161
|
+
# try with dedicated pg agent pod name configured
|
|
162
|
+
Pods.get(ns, pod_name)
|
|
163
|
+
except:
|
|
164
|
+
try:
|
|
165
|
+
# try with the ops pod
|
|
166
|
+
pod_name = Config().get('pod.name', 'ops')
|
|
167
|
+
real_pod_name = Pods.get_with_selector(ns, label_selector = Config().get('pod.label-selector', 'run=ops')).metadata.name
|
|
168
|
+
except:
|
|
169
|
+
log2(f"Could not locate {pod_name} pod.")
|
|
170
|
+
return None
|
|
171
|
+
|
|
172
|
+
cmd = f'psql -h {self.endpoint()} -p {self.port()} -U {self.username()} {db} --pset pager=off -c "{sql}"'
|
|
173
|
+
env_prefix = f'PGPASSWORD="{self.password()}"'
|
|
174
|
+
|
|
175
|
+
r = Pods.exec(real_pod_name, pod_name, ns, cmd, show_out=show_out, background=background, env_prefix=env_prefix)
|
|
176
|
+
if r and Config().get('repl.history.push-cat-remote-log-file', True):
|
|
177
|
+
if r.log_file and ReplSession().prompt_session:
|
|
178
|
+
ReplSession().prompt_session.history.append_string(f'@{r.pod} cat {r.log_file}')
|
|
179
|
+
|
|
180
|
+
return r
|
|
181
|
+
|
|
182
|
+
def deploy_pg_agent(pod_name: str, ns: str) -> str:
|
|
183
|
+
image = Config().get('pg.agent.image', 'seanahnsf/kaqing')
|
|
184
|
+
timeout = Config().get('pg.agent.timeout', 3600)
|
|
185
|
+
try:
|
|
186
|
+
Pods.create(ns, pod_name, image, ['sleep', f'{timeout}'], env={'NAMESPACE': ns}, sa_name='c3')
|
|
187
|
+
except Exception as e:
|
|
188
|
+
if e.status == 409:
|
|
189
|
+
if Pods.completed(ns, pod_name):
|
|
190
|
+
try:
|
|
191
|
+
Pods.delete(pod_name, ns)
|
|
192
|
+
Pods.create(ns, pod_name, image, ['sleep', f'{timeout}'], env={'NAMESPACE': ns}, sa_name='c3')
|
|
193
|
+
except Exception as e2:
|
|
194
|
+
log2("Exception when calling BatchV1Api->create_pod: %s\n" % e2)
|
|
195
|
+
|
|
196
|
+
return
|
|
197
|
+
else:
|
|
198
|
+
log2("Exception when calling BatchV1Api->create_pod: %s\n" % e)
|
|
199
|
+
|
|
200
|
+
return
|
|
201
|
+
|
|
202
|
+
Pods.wait_for_running(ns, pod_name)
|
|
203
|
+
|
|
204
|
+
return pod_name
|
|
205
|
+
|
|
206
|
+
def undeploy_pg_agent(pod_name: str, ns: str):
|
|
207
|
+
Pods.delete(pod_name, ns, grace_period_seconds=0)
|
|
208
|
+
|
|
209
|
+
def endpoint(self):
|
|
210
|
+
if not self.conn_details:
|
|
211
|
+
self.conn_details = Secrets.get_data(self.namespace, self.host)
|
|
212
|
+
|
|
213
|
+
endpoint_key = Config().get('pg.secret.endpoint-key', 'postgres-db-endpoint')
|
|
214
|
+
|
|
215
|
+
return self.conn_details[endpoint_key] if endpoint_key in self.conn_details else ''
|
|
216
|
+
|
|
217
|
+
def port(self):
|
|
218
|
+
if not self.conn_details:
|
|
219
|
+
self.conn_details = Secrets.get_data(self.namespace, self.host)
|
|
220
|
+
|
|
221
|
+
port_key = Config().get('pg.secret.port-key', 'postgres-db-port')
|
|
222
|
+
|
|
223
|
+
return self.conn_details[port_key] if port_key in self.conn_details else ''
|
|
224
|
+
|
|
225
|
+
def username(self):
|
|
226
|
+
if not self.conn_details:
|
|
227
|
+
self.conn_details = Secrets.get_data(self.namespace, self.host)
|
|
228
|
+
|
|
229
|
+
username_key = Config().get('pg.secret.username-key', 'postgres-admin-username')
|
|
230
|
+
|
|
231
|
+
return self.conn_details[username_key] if username_key in self.conn_details else ''
|
|
232
|
+
|
|
233
|
+
def password(self):
|
|
234
|
+
if not self.conn_details:
|
|
235
|
+
self.conn_details = Secrets.get_data(self.namespace, self.host)
|
|
236
|
+
|
|
237
|
+
password_key = Config().get('pg.secret.password-key', 'postgres-admin-password')
|
|
238
|
+
|
|
239
|
+
return self.conn_details[password_key] if password_key in self.conn_details else ''
|
|
240
|
+
|
|
241
|
+
def default_db():
|
|
242
|
+
return Config().get('pg.default-db', 'postgres')
|
|
243
|
+
|
|
244
|
+
def default_owner():
|
|
245
|
+
return Config().get('pg.default-owner', 'postgres')
|
|
246
|
+
|
|
247
|
+
def default_schema():
|
|
248
|
+
return Config().get('pg.default-schema', 'postgres')
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import functools
|
|
2
|
+
|
|
3
|
+
from adam.commands.postgres.postgres_context import PostgresContext
|
|
4
|
+
from adam.config import Config
|
|
5
|
+
|
|
6
|
+
TestPG = [False]
|
|
7
|
+
|
|
8
|
+
@functools.lru_cache()
|
|
9
|
+
def pg_database_names(ns: str, pg_path: str):
|
|
10
|
+
if TestPG[0]:
|
|
11
|
+
return ['azops88_c3ai_c3']
|
|
12
|
+
|
|
13
|
+
Config().wait_log('Inspecting Postgres Databases...')
|
|
14
|
+
|
|
15
|
+
pg = PostgresContext.apply(ns, pg_path)
|
|
16
|
+
return [db['name'] for db in pg.databases() if db['owner'] == PostgresContext.default_owner()]
|
|
17
|
+
|
|
18
|
+
@functools.lru_cache()
|
|
19
|
+
def pg_table_names(ns: str, pg_path: str):
|
|
20
|
+
if TestPG[0]:
|
|
21
|
+
return ['C3_2_XYZ1']
|
|
22
|
+
|
|
23
|
+
Config().wait_log('Inspecting Postgres Database...')
|
|
24
|
+
return [table['name'] for table in pg_tables(ns, pg_path) if table['schema'] == PostgresContext.default_schema()]
|
|
25
|
+
|
|
26
|
+
def pg_tables(ns: str, pg_path: str):
|
|
27
|
+
pg = PostgresContext.apply(ns, pg_path)
|
|
28
|
+
if pg.db:
|
|
29
|
+
return pg.tables()
|
|
30
|
+
|
|
31
|
+
return []
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
from adam.commands.postgres.postgres_utils import pg_table_names
|
|
2
|
+
from adam.sql.sql_completer import SqlCompleter
|
|
3
|
+
|
|
4
|
+
def psql_completions(ns: str, pg_path: str):
|
|
5
|
+
return {
|
|
6
|
+
'\h': None,
|
|
7
|
+
'\d': None,
|
|
8
|
+
'\dt': None,
|
|
9
|
+
'\du': None
|
|
10
|
+
} | SqlCompleter(lambda: pg_table_names(ns, pg_path)).completions_for_nesting()
|
adam/commands/preview_table.py
CHANGED
|
@@ -1,12 +1,11 @@
|
|
|
1
|
-
import functools
|
|
2
|
-
|
|
3
1
|
from adam.commands.command import Command
|
|
4
|
-
from adam.commands.cql_utils import
|
|
5
|
-
from adam.commands.postgres.
|
|
2
|
+
from adam.commands.cql.cql_utils import cassandra_table_names, run_cql
|
|
3
|
+
from adam.commands.postgres.postgres_context import PostgresContext
|
|
6
4
|
from adam.config import Config
|
|
7
|
-
from adam.pod_exec_result import PodExecResult
|
|
8
5
|
from adam.repl_state import ReplState, RequiredState
|
|
9
6
|
from adam.utils import lines_to_tabular, log, log2
|
|
7
|
+
from adam.utils_athena import Athena
|
|
8
|
+
from adam.utils_audits import Audits
|
|
10
9
|
|
|
11
10
|
class PreviewTable(Command):
|
|
12
11
|
COMMAND = 'preview'
|
|
@@ -24,28 +23,26 @@ class PreviewTable(Command):
|
|
|
24
23
|
return PreviewTable.COMMAND
|
|
25
24
|
|
|
26
25
|
def required(self):
|
|
27
|
-
return RequiredState.CLUSTER_OR_POD
|
|
26
|
+
return [RequiredState.CLUSTER_OR_POD, RequiredState.PG_DATABASE, ReplState.L]
|
|
28
27
|
|
|
29
28
|
def run(self, cmd: str, state: ReplState):
|
|
30
29
|
if not(args := self.args(cmd)):
|
|
31
30
|
return super().run(cmd, state)
|
|
32
31
|
|
|
33
32
|
state, args = self.apply_state(args, state)
|
|
34
|
-
if
|
|
35
|
-
|
|
36
|
-
return state
|
|
37
|
-
else:
|
|
38
|
-
if not self.validate_state(state):
|
|
39
|
-
return state
|
|
33
|
+
if not self.validate_state(state):
|
|
34
|
+
return state
|
|
40
35
|
|
|
41
36
|
if not args:
|
|
42
37
|
def show_tables():
|
|
43
38
|
if state.device == ReplState.P:
|
|
44
|
-
pg =
|
|
45
|
-
lines = [db["name"] for db in pg.tables() if db["schema"] ==
|
|
39
|
+
pg = PostgresContext.apply(state.namespace, state.pg_path)
|
|
40
|
+
lines = [db["name"] for db in pg.tables() if db["schema"] == PostgresContext.default_schema()]
|
|
46
41
|
log(lines_to_tabular(lines, separator=','))
|
|
42
|
+
elif state.device == ReplState.L:
|
|
43
|
+
log(lines_to_tabular(Athena.table_names(), separator=','))
|
|
47
44
|
else:
|
|
48
|
-
|
|
45
|
+
log(lines_to_tabular(cassandra_table_names(state), separator=','))
|
|
49
46
|
|
|
50
47
|
if state.in_repl:
|
|
51
48
|
log2('Table is required.')
|
|
@@ -64,35 +61,16 @@ class PreviewTable(Command):
|
|
|
64
61
|
|
|
65
62
|
rows = Config().get('preview.rows', 10)
|
|
66
63
|
if state.device == ReplState.P:
|
|
67
|
-
|
|
64
|
+
PostgresContext.apply(state.namespace, state.pg_path).run_sql(f'select * from {table} limit {rows}')
|
|
65
|
+
elif state.device == ReplState.L:
|
|
66
|
+
Athena.run_query(f'select * from {table} limit {rows}')
|
|
68
67
|
else:
|
|
69
|
-
run_cql(state, f'select * from {table} limit {rows}', show_out=True, use_single_quotes=True)
|
|
68
|
+
run_cql(state, f'select * from {table} limit {rows}', show_out=True, use_single_quotes=True, on_any=True)
|
|
70
69
|
|
|
71
70
|
return state
|
|
72
71
|
|
|
73
|
-
def completion(self,
|
|
74
|
-
if state.device == ReplState.P:
|
|
75
|
-
if tables := PreviewTable.pg_tables(state.namespace, state.pg_path):
|
|
76
|
-
return {PreviewTable.COMMAND: {db["name"]: None for db in tables if db["schema"] == PostgresSession.default_schema()}}
|
|
77
|
-
else:
|
|
78
|
-
if state.pod:
|
|
79
|
-
tables = PreviewTable.cql_tables(state)
|
|
80
|
-
return {PreviewTable.COMMAND: {f'{k}.{t}': None for k, ts in tables.items() for t in ts}}
|
|
81
|
-
|
|
72
|
+
def completion(self, _: ReplState):
|
|
82
73
|
return {}
|
|
83
74
|
|
|
84
75
|
def help(self, _: ReplState):
|
|
85
|
-
return f'{PreviewTable.COMMAND} TABLE\t preview table'
|
|
86
|
-
|
|
87
|
-
@functools.lru_cache()
|
|
88
|
-
def cql_tables(state: ReplState):
|
|
89
|
-
r: PodExecResult = run_cql(state, 'describe tables', show_out=False)
|
|
90
|
-
return parse_cql_desc_tables(r.stdout)
|
|
91
|
-
|
|
92
|
-
@functools.lru_cache()
|
|
93
|
-
def pg_tables(ns: str, pg_path: str):
|
|
94
|
-
pg = PostgresSession(ns, pg_path)
|
|
95
|
-
if pg.db:
|
|
96
|
-
return pg.tables()
|
|
97
|
-
|
|
98
|
-
return None
|
|
76
|
+
return f'{PreviewTable.COMMAND} TABLE\t preview table'
|
adam/commands/pwd.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
from adam.app_session import AppSession
|
|
2
2
|
from adam.commands.command import Command
|
|
3
|
-
from adam.commands.
|
|
3
|
+
from adam.commands.devices.devices import Devices
|
|
4
4
|
from adam.repl_state import ReplState
|
|
5
5
|
from adam.utils import lines_to_tabular, log
|
|
6
6
|
|
|
@@ -25,29 +25,6 @@ class Pwd(Command):
|
|
|
25
25
|
|
|
26
26
|
state, _ = self.apply_state(args, state)
|
|
27
27
|
|
|
28
|
-
def device_line(state: ReplState, device: str):
|
|
29
|
-
words = []
|
|
30
|
-
|
|
31
|
-
if device == ReplState.P:
|
|
32
|
-
pg = PostgresSession(state.namespace, state.pg_path)
|
|
33
|
-
|
|
34
|
-
if pg.host:
|
|
35
|
-
words.append(f'host/{pg.host}')
|
|
36
|
-
if pg.db:
|
|
37
|
-
words.append(f'database/{pg.db}')
|
|
38
|
-
elif device == ReplState.A:
|
|
39
|
-
if state.app_env:
|
|
40
|
-
words.append(f'env/{state.app_env}')
|
|
41
|
-
if state.app_app:
|
|
42
|
-
words.append(f'app/{state.app_app}')
|
|
43
|
-
else:
|
|
44
|
-
if state.sts:
|
|
45
|
-
words.append(f'sts/{state.sts}')
|
|
46
|
-
if state.pod:
|
|
47
|
-
words.append(f'pod/{state.pod}')
|
|
48
|
-
|
|
49
|
-
return '\t'.join([f'{device}:>'] + (words if words else ['/']))
|
|
50
|
-
|
|
51
28
|
host = "unknown"
|
|
52
29
|
try:
|
|
53
30
|
app_session: AppSession = AppSession.create('c3', 'c3')
|
|
@@ -55,10 +32,7 @@ class Pwd(Command):
|
|
|
55
32
|
except:
|
|
56
33
|
pass
|
|
57
34
|
|
|
58
|
-
log(lines_to_tabular([
|
|
59
|
-
device_line(state, ReplState.A),
|
|
60
|
-
device_line(state, ReplState.C),
|
|
61
|
-
device_line(state, ReplState.P),
|
|
35
|
+
log(lines_to_tabular([device.pwd(state) for device in Devices.all()] + [
|
|
62
36
|
f'',
|
|
63
37
|
f'HOST\t{host}',
|
|
64
38
|
f'NAMESPACE\t{state.namespace if state.namespace else "/"}',
|
adam/commands/reaper/reaper.py
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import click
|
|
2
2
|
|
|
3
3
|
from adam.commands.command import Command
|
|
4
|
-
from adam.commands.command_helpers import ClusterCommandHelper
|
|
5
4
|
from .reaper_forward import ReaperForward
|
|
6
5
|
from .reaper_forward_stop import ReaperForwardStop
|
|
7
6
|
from .reaper_restart import ReaperRestart
|
|
@@ -14,7 +13,6 @@ from .reaper_schedule_stop import ReaperScheduleStop
|
|
|
14
13
|
from .reaper_schedules import ReaperSchedules
|
|
15
14
|
from .reaper_status import ReaperStatus
|
|
16
15
|
from adam.repl_state import ReplState, RequiredState
|
|
17
|
-
from adam.utils import lines_to_tabular, log, log2
|
|
18
16
|
|
|
19
17
|
class Reaper(Command):
|
|
20
18
|
COMMAND = 'reaper'
|
|
@@ -39,24 +37,15 @@ class Reaper(Command):
|
|
|
39
37
|
if not(args := self.args(cmd)):
|
|
40
38
|
return super().run(cmd, state)
|
|
41
39
|
|
|
42
|
-
state, args = self.apply_state(args, state)
|
|
43
40
|
if not self.validate_state(state):
|
|
44
41
|
return state
|
|
45
42
|
|
|
46
|
-
|
|
47
|
-
log(lines_to_tabular([c.help(ReplState()) for c in Reaper.cmd_list()], separator='\t'))
|
|
48
|
-
|
|
49
|
-
return 'command-missing'
|
|
50
|
-
else:
|
|
51
|
-
# head with the Chain of Responsibility pattern
|
|
52
|
-
cmds = Command.chain(Reaper.cmd_list())
|
|
53
|
-
if not cmds.run(cmd, state):
|
|
54
|
-
log2('* Command is missing.')
|
|
55
|
-
Command.display_help()
|
|
43
|
+
return super().intermediate_run(cmd, state, args, Reaper.cmd_list())
|
|
56
44
|
|
|
57
45
|
def cmd_list():
|
|
58
46
|
return [ReaperSchedules(), ReaperScheduleStop(), ReaperScheduleActivate(), ReaperScheduleStart(),
|
|
59
|
-
ReaperForwardStop(), ReaperForward(), ReaperRunAbort(), ReaperRunsAbort(), ReaperRestart(),
|
|
47
|
+
ReaperForwardStop(), ReaperForward(), ReaperRunAbort(), ReaperRunsAbort(), ReaperRestart(),
|
|
48
|
+
ReaperRuns(), ReaperStatus()]
|
|
60
49
|
|
|
61
50
|
def completion(self, state: ReplState):
|
|
62
51
|
if state.sts:
|
|
@@ -64,15 +53,6 @@ class Reaper(Command):
|
|
|
64
53
|
|
|
65
54
|
return {}
|
|
66
55
|
|
|
67
|
-
def help(self, _: ReplState):
|
|
68
|
-
return None
|
|
69
|
-
|
|
70
56
|
class ReaperCommandHelper(click.Command):
|
|
71
57
|
def get_help(self, ctx: click.Context):
|
|
72
|
-
|
|
73
|
-
log()
|
|
74
|
-
log('Sub-Commands:')
|
|
75
|
-
|
|
76
|
-
log(lines_to_tabular([c.help(ReplState()).replace(f'{Reaper.COMMAND} ', ' ', 1) for c in Reaper.cmd_list()], separator='\t'))
|
|
77
|
-
log()
|
|
78
|
-
ClusterCommandHelper.cluster_help()
|
|
58
|
+
Command.intermediate_help(super().get_help(ctx), Reaper.COMMAND, Reaper.cmd_list(), show_cluster_help=True)
|
|
@@ -7,7 +7,7 @@ import requests
|
|
|
7
7
|
from typing import List, cast
|
|
8
8
|
|
|
9
9
|
from adam.config import Config
|
|
10
|
-
from adam.
|
|
10
|
+
from adam.utils_k8s.kube_context import KubeContext
|
|
11
11
|
from adam.repl_state import ReplState
|
|
12
12
|
from adam.utils import lines_to_tabular, log2
|
|
13
13
|
|
|
@@ -149,7 +149,7 @@ class ReaperSession:
|
|
|
149
149
|
return ReaperSession.schedules_ids_by_cluster[state.sts]
|
|
150
150
|
|
|
151
151
|
if reaper := ReaperSession.create(state):
|
|
152
|
-
|
|
152
|
+
Config().wait_log('Inspecting Cassandra Reaper...')
|
|
153
153
|
|
|
154
154
|
schedules = reaper.schedule_ids(state, show_output = False)
|
|
155
155
|
ReaperSession.schedules_ids_by_cluster[state.sts] = schedules
|
adam/commands/repair/repair.py
CHANGED
|
@@ -1,13 +1,11 @@
|
|
|
1
1
|
import click
|
|
2
2
|
|
|
3
3
|
from adam.commands.command import Command
|
|
4
|
-
from adam.commands.command_helpers import ClusterCommandHelper
|
|
5
4
|
from .repair_run import RepairRun
|
|
6
5
|
from .repair_scan import RepairScan
|
|
7
6
|
from .repair_stop import RepairStop
|
|
8
7
|
from .repair_log import RepairLog
|
|
9
8
|
from adam.repl_state import ReplState, RequiredState
|
|
10
|
-
from adam.utils import lines_to_tabular, log, log2
|
|
11
9
|
|
|
12
10
|
class Repair(Command):
|
|
13
11
|
COMMAND = 'repair'
|
|
@@ -30,39 +28,17 @@ class Repair(Command):
|
|
|
30
28
|
def run(self, cmd: str, state: ReplState):
|
|
31
29
|
if not(args := self.args(cmd)):
|
|
32
30
|
return super().run(cmd, state)
|
|
33
|
-
|
|
34
|
-
state, args = self.apply_state(args, state)
|
|
35
31
|
if not self.validate_state(state):
|
|
36
32
|
return state
|
|
37
33
|
|
|
38
|
-
|
|
39
|
-
log(lines_to_tabular([c.help(ReplState()) for c in Repair.cmd_list()], separator='\t'))
|
|
40
|
-
|
|
41
|
-
return 'command-missing'
|
|
42
|
-
else:
|
|
43
|
-
# head with the Chain of Responsibility pattern
|
|
44
|
-
cmds = Command.chain(Repair.cmd_list())
|
|
45
|
-
if not cmds.run(cmd, state):
|
|
46
|
-
log2('* Command is missing.')
|
|
47
|
-
Command.display_help()
|
|
34
|
+
return super().intermediate_run(cmd, state, args, Repair.cmd_list())
|
|
48
35
|
|
|
49
36
|
def cmd_list():
|
|
50
37
|
return [RepairRun(), RepairScan(), RepairStop(), RepairLog()]
|
|
51
38
|
|
|
52
39
|
def completion(self, state: ReplState):
|
|
53
|
-
|
|
54
|
-
return super().completion(state)
|
|
55
|
-
return {}
|
|
56
|
-
|
|
57
|
-
def help(self, _: ReplState):
|
|
58
|
-
return None
|
|
40
|
+
return super().completion(state)
|
|
59
41
|
|
|
60
42
|
class RepairCommandHelper(click.Command):
|
|
61
43
|
def get_help(self, ctx: click.Context):
|
|
62
|
-
|
|
63
|
-
log()
|
|
64
|
-
log('Sub-Commands:')
|
|
65
|
-
|
|
66
|
-
log(lines_to_tabular([c.help(ReplState()).replace(f'{Repair.COMMAND} ', ' ', 1) for c in Repair.cmd_list()], separator='\t'))
|
|
67
|
-
log()
|
|
68
|
-
ClusterCommandHelper.cluster_help()
|
|
44
|
+
Command.intermediate_help(super().get_help(ctx), Repair.COMMAND, Repair.cmd_list(), show_cluster_help=True)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
from adam.commands.command import Command
|
|
2
|
-
from adam.
|
|
3
|
-
from adam.
|
|
2
|
+
from adam.utils_k8s.jobs import Jobs
|
|
3
|
+
from adam.utils_k8s.volumes import Volumes
|
|
4
4
|
from adam.repl_state import ReplState, RequiredState
|
|
5
5
|
from adam.config import Config
|
|
6
6
|
from adam.commands.reaper.reaper_session import ReaperSession
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import time
|
|
2
2
|
|
|
3
3
|
from adam.commands.command import Command
|
|
4
|
-
from adam.
|
|
4
|
+
from adam.utils_k8s.pods import Pods
|
|
5
5
|
from adam.repl_state import ReplState, RequiredState
|
|
6
6
|
from adam.utils import log2
|
|
7
7
|
from adam.config import Config
|
|
@@ -55,12 +55,7 @@ class RepairScan(Command):
|
|
|
55
55
|
else:
|
|
56
56
|
log2("Exception when calling BatchV1Apii->create_namespaced_job: %s\n" % e)
|
|
57
57
|
|
|
58
|
-
|
|
59
|
-
while Pods.get(ns, pod_name).status.phase != 'Running':
|
|
60
|
-
if not msged:
|
|
61
|
-
log2("Waiting for the scanner pod to start up...")
|
|
62
|
-
msged = True
|
|
63
|
-
time.sleep(5)
|
|
58
|
+
Pods.wait_for_running(ns, pod_name, 'Waiting for the scanner pod to start up...')
|
|
64
59
|
|
|
65
60
|
try:
|
|
66
61
|
Pods.exec(pod_name, pod_name, ns, f"find {log_path} -type f -mtime -{n} -print0 | xargs -0 grep failed")
|
adam/commands/report.py
CHANGED
|
@@ -22,12 +22,18 @@ class Report(Command):
|
|
|
22
22
|
def command(self):
|
|
23
23
|
return Report.COMMAND
|
|
24
24
|
|
|
25
|
+
def required(self):
|
|
26
|
+
return ReplState.NON_L
|
|
27
|
+
|
|
25
28
|
def run(self, cmd: str, state: ReplState):
|
|
26
29
|
if not(args := self.args(cmd)):
|
|
27
30
|
return super().run(cmd, state)
|
|
28
31
|
|
|
29
32
|
output: dict[str, any] = {}
|
|
30
33
|
state, args = self.apply_state(args, state)
|
|
34
|
+
if not self.validate_state(state):
|
|
35
|
+
return state
|
|
36
|
+
|
|
31
37
|
if state.in_repl:
|
|
32
38
|
args, show = Command.extract_options(args, ['-s', '--show'])
|
|
33
39
|
|
adam/commands/restart.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
from adam.commands.command import Command
|
|
2
|
-
from adam.
|
|
3
|
-
from adam.
|
|
2
|
+
from adam.utils_k8s.pods import Pods
|
|
3
|
+
from adam.utils_k8s.statefulsets import StatefulSets
|
|
4
4
|
from adam.repl_state import ReplState, RequiredState
|
|
5
5
|
from adam.utils import log2
|
|
6
6
|
|
adam/commands/rollout.py
CHANGED
|
@@ -4,7 +4,7 @@ from kubernetes.client.rest import ApiException
|
|
|
4
4
|
|
|
5
5
|
from adam.commands.command import Command
|
|
6
6
|
from adam.commands.watch import Watch
|
|
7
|
-
from adam.
|
|
7
|
+
from adam.utils_k8s.statefulsets import StatefulSets
|
|
8
8
|
from adam.config import Config
|
|
9
9
|
from adam.repl_state import ReplState, RequiredState
|
|
10
10
|
from adam.utils import duration, log2
|