kaqing 2.0.50__py3-none-any.whl → 2.0.110__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of kaqing might be problematic. Click here for more details.
- adam/apps.py +2 -2
- adam/batch.py +11 -15
- adam/checks/check_utils.py +4 -4
- adam/checks/compactionstats.py +1 -1
- adam/checks/cpu.py +2 -2
- adam/checks/disk.py +1 -1
- adam/checks/gossip.py +1 -1
- adam/checks/memory.py +3 -3
- adam/checks/status.py +1 -1
- adam/commands/alter_tables.py +3 -14
- adam/commands/app.py +2 -2
- adam/commands/app_ping.py +2 -2
- adam/commands/audit/audit.py +85 -0
- adam/commands/audit/audit_repair_tables.py +76 -0
- adam/commands/audit/audit_run.py +57 -0
- adam/commands/audit/show_last10.py +50 -0
- adam/commands/audit/show_slow10.py +49 -0
- adam/commands/audit/show_top10.py +48 -0
- adam/commands/audit/utils_show_top10.py +59 -0
- adam/commands/bash.py +76 -13
- adam/commands/cd.py +22 -13
- adam/commands/check.py +6 -0
- adam/commands/cli_commands.py +3 -3
- adam/commands/command.py +15 -11
- adam/commands/commands_utils.py +4 -5
- adam/commands/cql/cql_completions.py +7 -5
- adam/commands/cql/cql_utils.py +13 -10
- adam/commands/cql/cqlsh.py +6 -3
- adam/commands/deploy/code_utils.py +2 -2
- adam/commands/deploy/deploy.py +7 -1
- adam/commands/deploy/deploy_pg_agent.py +2 -2
- adam/commands/deploy/deploy_pod.py +6 -6
- adam/commands/deploy/deploy_utils.py +2 -2
- adam/commands/deploy/undeploy.py +7 -1
- adam/commands/deploy/undeploy_pg_agent.py +2 -2
- adam/commands/deploy/undeploy_pod.py +4 -4
- adam/commands/devices.py +29 -0
- adam/commands/help.py +10 -7
- adam/commands/issues.py +6 -0
- adam/commands/login.py +6 -3
- adam/commands/logs.py +2 -1
- adam/commands/ls.py +30 -24
- adam/commands/medusa/medusa_backup.py +2 -2
- adam/commands/medusa/medusa_restore.py +2 -2
- adam/commands/medusa/medusa_show_backupjobs.py +3 -2
- adam/commands/medusa/medusa_show_restorejobs.py +2 -2
- adam/commands/nodetool.py +5 -3
- adam/commands/postgres/postgres.py +3 -3
- adam/commands/postgres/{postgres_session.py → postgres_context.py} +29 -30
- adam/commands/postgres/postgres_utils.py +5 -5
- adam/commands/postgres/psql_completions.py +2 -3
- adam/commands/preview_table.py +17 -32
- adam/commands/pwd.py +5 -2
- adam/commands/reaper/reaper.py +3 -0
- adam/commands/reaper/reaper_restart.py +1 -1
- adam/commands/reaper/reaper_session.py +1 -1
- adam/commands/repair/repair.py +3 -3
- adam/commands/repair/repair_log.py +1 -1
- adam/commands/repair/repair_run.py +2 -2
- adam/commands/repair/repair_scan.py +1 -1
- adam/commands/repair/repair_stop.py +1 -1
- adam/commands/report.py +6 -0
- adam/commands/restart.py +2 -2
- adam/commands/rollout.py +1 -1
- adam/commands/show/show.py +5 -2
- adam/commands/show/show_app_actions.py +3 -0
- adam/commands/show/show_app_id.py +1 -1
- adam/commands/show/show_app_queues.py +3 -2
- adam/commands/show/show_cassandra_status.py +3 -3
- adam/commands/show/show_cassandra_version.py +3 -3
- adam/commands/show/show_host.py +33 -0
- adam/commands/show/show_login.py +3 -0
- adam/commands/show/show_processes.py +1 -1
- adam/commands/show/show_repairs.py +2 -2
- adam/commands/show/show_storage.py +1 -1
- adam/commands/watch.py +1 -1
- adam/config.py +2 -1
- adam/embedded_params.py +1 -1
- adam/pod_exec_result.py +7 -2
- adam/repl.py +141 -89
- adam/repl_commands.py +21 -20
- adam/repl_state.py +167 -39
- adam/sql/sql_completer.py +89 -49
- adam/sql/sql_state_machine.py +518 -0
- adam/sql/term_completer.py +76 -0
- adam/sso/cred_cache.py +1 -1
- adam/sso/idp.py +1 -1
- adam/utils.py +0 -1
- adam/utils_audits.py +193 -0
- adam/{k8s_utils → utils_k8s}/cassandra_clusters.py +6 -8
- adam/{k8s_utils → utils_k8s}/cassandra_nodes.py +11 -4
- adam/{k8s_utils → utils_k8s}/deployment.py +2 -2
- adam/{k8s_utils → utils_k8s}/pods.py +33 -9
- adam/{k8s_utils → utils_k8s}/secrets.py +4 -0
- adam/{k8s_utils → utils_k8s}/statefulsets.py +4 -4
- adam/utils_net.py +24 -0
- adam/version.py +1 -1
- {kaqing-2.0.50.dist-info → kaqing-2.0.110.dist-info}/METADATA +1 -1
- kaqing-2.0.110.dist-info/RECORD +187 -0
- adam/commands/cql/cql_table_completer.py +0 -16
- adam/commands/describe/describe.py +0 -46
- adam/commands/describe/describe_keyspace.py +0 -60
- adam/commands/describe/describe_keyspaces.py +0 -50
- adam/commands/describe/describe_table.py +0 -60
- adam/commands/describe/describe_tables.py +0 -50
- adam/commands/postgres/psql_table_completer.py +0 -18
- adam/sql/any_completer.py +0 -84
- adam/sql/sql_utils.py +0 -5
- adam/sql/table_name_completer.py +0 -17
- kaqing-2.0.50.dist-info/RECORD +0 -185
- /adam/commands/{describe → audit}/__init__.py +0 -0
- /adam/{k8s_utils → utils_k8s}/__init__.py +0 -0
- /adam/{k8s_utils → utils_k8s}/config_maps.py +0 -0
- /adam/{k8s_utils → utils_k8s}/custom_resources.py +0 -0
- /adam/{k8s_utils → utils_k8s}/ingresses.py +0 -0
- /adam/{k8s_utils → utils_k8s}/jobs.py +0 -0
- /adam/{k8s_utils → utils_k8s}/kube_context.py +0 -0
- /adam/{k8s_utils → utils_k8s}/service_accounts.py +0 -0
- /adam/{k8s_utils → utils_k8s}/services.py +0 -0
- /adam/{k8s_utils → utils_k8s}/volumes.py +0 -0
- {kaqing-2.0.50.dist-info → kaqing-2.0.110.dist-info}/WHEEL +0 -0
- {kaqing-2.0.50.dist-info → kaqing-2.0.110.dist-info}/entry_points.txt +0 -0
- {kaqing-2.0.50.dist-info → kaqing-2.0.110.dist-info}/top_level.txt +0 -0
adam/commands/report.py
CHANGED
|
@@ -22,12 +22,18 @@ class Report(Command):
|
|
|
22
22
|
def command(self):
|
|
23
23
|
return Report.COMMAND
|
|
24
24
|
|
|
25
|
+
def required(self):
|
|
26
|
+
return ReplState.NON_L
|
|
27
|
+
|
|
25
28
|
def run(self, cmd: str, state: ReplState):
|
|
26
29
|
if not(args := self.args(cmd)):
|
|
27
30
|
return super().run(cmd, state)
|
|
28
31
|
|
|
29
32
|
output: dict[str, any] = {}
|
|
30
33
|
state, args = self.apply_state(args, state)
|
|
34
|
+
if not self.validate_state(state):
|
|
35
|
+
return state
|
|
36
|
+
|
|
31
37
|
if state.in_repl:
|
|
32
38
|
args, show = Command.extract_options(args, ['-s', '--show'])
|
|
33
39
|
|
adam/commands/restart.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
from adam.commands.command import Command
|
|
2
|
-
from adam.
|
|
3
|
-
from adam.
|
|
2
|
+
from adam.utils_k8s.pods import Pods
|
|
3
|
+
from adam.utils_k8s.statefulsets import StatefulSets
|
|
4
4
|
from adam.repl_state import ReplState, RequiredState
|
|
5
5
|
from adam.utils import log2
|
|
6
6
|
|
adam/commands/rollout.py
CHANGED
|
@@ -4,7 +4,7 @@ from kubernetes.client.rest import ApiException
|
|
|
4
4
|
|
|
5
5
|
from adam.commands.command import Command
|
|
6
6
|
from adam.commands.watch import Watch
|
|
7
|
-
from adam.
|
|
7
|
+
from adam.utils_k8s.statefulsets import StatefulSets
|
|
8
8
|
from adam.config import Config
|
|
9
9
|
from adam.repl_state import ReplState, RequiredState
|
|
10
10
|
from adam.utils import duration, log2
|
adam/commands/show/show.py
CHANGED
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
import click
|
|
2
2
|
|
|
3
|
+
from adam.commands.audit.show_last10 import ShowLast10
|
|
3
4
|
from adam.commands.command import Command
|
|
4
5
|
from adam.commands.medusa.medusa_show_backupjobs import MedusaShowBackupJobs
|
|
5
6
|
from adam.commands.medusa.medusa_show_restorejobs import MedusaShowRestoreJobs
|
|
6
7
|
from adam.commands.show.show_app_actions import ShowAppActions
|
|
7
8
|
from adam.commands.show.show_app_queues import ShowAppQueues
|
|
9
|
+
from adam.commands.show.show_host import ShowHost
|
|
8
10
|
from adam.commands.show.show_login import ShowLogin
|
|
9
11
|
from .show_params import ShowParams
|
|
10
12
|
from .show_app_id import ShowAppId
|
|
@@ -39,9 +41,10 @@ class Show(Command):
|
|
|
39
41
|
return super().intermediate_run(cmd, state, args, Show.cmd_list())
|
|
40
42
|
|
|
41
43
|
def cmd_list():
|
|
42
|
-
return [ShowAppActions(), ShowAppId(), ShowAppQueues(), ShowLogin(), ShowKubectlCommands(),
|
|
44
|
+
return [ShowAppActions(), ShowAppId(), ShowAppQueues(), ShowHost(), ShowLogin(), ShowKubectlCommands(),
|
|
43
45
|
ShowParams(), ShowProcesses(), ShowRepairs(), ShowStorage(), ShowAdam(),
|
|
44
|
-
ShowCassandraStatus(), ShowCassandraVersion(), MedusaShowRestoreJobs(), MedusaShowBackupJobs()
|
|
46
|
+
ShowCassandraStatus(), ShowCassandraVersion(), MedusaShowRestoreJobs(), MedusaShowBackupJobs(),
|
|
47
|
+
ShowLast10()]
|
|
45
48
|
|
|
46
49
|
def completion(self, state: ReplState):
|
|
47
50
|
return super().completion(state)
|
|
@@ -18,14 +18,15 @@ class ShowAppQueues(Command):
|
|
|
18
18
|
return ShowAppQueues.COMMAND
|
|
19
19
|
|
|
20
20
|
def required(self):
|
|
21
|
-
return RequiredState.
|
|
21
|
+
return RequiredState.APP_APP
|
|
22
22
|
|
|
23
23
|
def run(self, cmd: str, state: ReplState):
|
|
24
24
|
if not(args := self.args(cmd)):
|
|
25
25
|
return super().run(cmd, state)
|
|
26
26
|
|
|
27
27
|
state, args = self.apply_state(args, state)
|
|
28
|
-
if not self.validate_state(state, app_required=RequiredState.APP_APP):
|
|
28
|
+
# if not self.validate_state(state, app_required=RequiredState.APP_APP):
|
|
29
|
+
if not self.validate_state(state):
|
|
29
30
|
return state
|
|
30
31
|
|
|
31
32
|
_, forced = Command.extract_options(args, '--force')
|
|
@@ -8,9 +8,9 @@ from adam.columns.columns import Columns
|
|
|
8
8
|
from adam.commands.command import Command
|
|
9
9
|
from adam.commands.issues import Issues
|
|
10
10
|
from adam.config import Config
|
|
11
|
-
from adam.
|
|
12
|
-
from adam.
|
|
13
|
-
from adam.
|
|
11
|
+
from adam.utils_k8s.cassandra_nodes import CassandraNodes
|
|
12
|
+
from adam.utils_k8s.secrets import Secrets
|
|
13
|
+
from adam.utils_k8s.statefulsets import StatefulSets
|
|
14
14
|
from adam.repl_state import ReplState, RequiredState
|
|
15
15
|
from adam.utils import lines_to_tabular, log, log2
|
|
16
16
|
from adam.checks.status import parse_nodetool_status
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
from adam.commands.command import Command
|
|
2
|
-
from adam.
|
|
3
|
-
from adam.
|
|
4
|
-
from adam.
|
|
2
|
+
from adam.utils_k8s.cassandra_clusters import CassandraClusters
|
|
3
|
+
from adam.utils_k8s.cassandra_nodes import CassandraNodes
|
|
4
|
+
from adam.utils_k8s.secrets import Secrets
|
|
5
5
|
from adam.repl_state import ReplState, RequiredState
|
|
6
6
|
|
|
7
7
|
class ShowCassandraVersion(Command):
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
from adam.commands.command import Command
|
|
2
|
+
from adam.repl_state import ReplState
|
|
3
|
+
from adam.utils import log
|
|
4
|
+
from adam.utils_net import get_my_host
|
|
5
|
+
|
|
6
|
+
class ShowHost(Command):
|
|
7
|
+
COMMAND = 'show host'
|
|
8
|
+
|
|
9
|
+
# the singleton pattern
|
|
10
|
+
def __new__(cls, *args, **kwargs):
|
|
11
|
+
if not hasattr(cls, 'instance'): cls.instance = super(ShowHost, cls).__new__(cls)
|
|
12
|
+
|
|
13
|
+
return cls.instance
|
|
14
|
+
|
|
15
|
+
def __init__(self, successor: Command=None):
|
|
16
|
+
super().__init__(successor)
|
|
17
|
+
|
|
18
|
+
def command(self):
|
|
19
|
+
return ShowHost.COMMAND
|
|
20
|
+
|
|
21
|
+
def run(self, cmd: str, state: ReplState):
|
|
22
|
+
if not(args := self.args(cmd)):
|
|
23
|
+
return super().run(cmd, state)
|
|
24
|
+
|
|
25
|
+
log(get_my_host())
|
|
26
|
+
|
|
27
|
+
return state
|
|
28
|
+
|
|
29
|
+
def completion(self, state: ReplState):
|
|
30
|
+
return super().completion(state)
|
|
31
|
+
|
|
32
|
+
def help(self, _: ReplState):
|
|
33
|
+
return f'{ShowHost.COMMAND}\t show host'
|
adam/commands/show/show_login.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
from adam.commands.command import Command
|
|
2
2
|
from adam.commands.commands_utils import show_table
|
|
3
3
|
from adam.config import Config
|
|
4
|
-
from adam.
|
|
4
|
+
from adam.utils_k8s.statefulsets import StatefulSets
|
|
5
5
|
from adam.repl_state import ReplState, RequiredState
|
|
6
6
|
|
|
7
7
|
class ShowProcesses(Command):
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
from adam.commands.command import Command
|
|
2
|
-
from adam.
|
|
3
|
-
from adam.
|
|
2
|
+
from adam.utils_k8s.cassandra_clusters import CassandraClusters
|
|
3
|
+
from adam.utils_k8s.cassandra_nodes import CassandraNodes
|
|
4
4
|
from adam.repl_state import ReplState, RequiredState
|
|
5
5
|
|
|
6
6
|
class ShowRepairs(Command):
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
from adam.commands.command import Command
|
|
2
2
|
from adam.commands.commands_utils import show_table
|
|
3
3
|
from adam.config import Config
|
|
4
|
-
from adam.
|
|
4
|
+
from adam.utils_k8s.statefulsets import StatefulSets
|
|
5
5
|
from adam.repl_state import ReplState, RequiredState
|
|
6
6
|
|
|
7
7
|
class ShowStorage(Command):
|
adam/commands/watch.py
CHANGED
|
@@ -6,7 +6,7 @@ from typing import List
|
|
|
6
6
|
from adam.commands.command import Command
|
|
7
7
|
from adam.commands.commands_utils import show_pods, show_rollout
|
|
8
8
|
from adam.config import Config
|
|
9
|
-
from adam.
|
|
9
|
+
from adam.utils_k8s.statefulsets import StatefulSets
|
|
10
10
|
from adam.repl_state import ReplState, RequiredState
|
|
11
11
|
from adam.utils import convert_seconds, log2
|
|
12
12
|
|
adam/config.py
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import os
|
|
1
2
|
from typing import TypeVar, cast
|
|
2
3
|
import yaml
|
|
3
4
|
|
|
@@ -38,7 +39,7 @@ class Config:
|
|
|
38
39
|
return get_deep_keys(self.params)
|
|
39
40
|
|
|
40
41
|
def is_debug(self):
|
|
41
|
-
return Config().get('debug', False)
|
|
42
|
+
return os.getenv('QING_DEV', 'false').lower() == 'true' or Config().get('debug', False)
|
|
42
43
|
|
|
43
44
|
def debug(self, s: None):
|
|
44
45
|
if self.is_debug():
|
adam/embedded_params.py
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
def config():
|
|
2
|
-
return {'app': {'console-endpoint': 'https://{host}/{env}/{app}/static/console/index.html', 'cr': {'cluster-regex': '(.*?-.*?)-.*', 'group': 'ops.c3.ai', 'v': 'v2', 'plural': 'c3cassandras'}, 'label': 'c3__app_id-0', 'login': {'admin-group': '{host}/C3.ClusterAdmin', 'ingress': '{app_id}-k8singr-appleader-001', 'timeout': 5, 'session-check-url': 'https://{host}/{env}/{app}/api/8/C3/userSessionToken', 'cache-creds': True, 'cache-username': True, 'url': 'https://{host}/{env}/{app}', 'another': "You're logged in to {has}. However, for this app, you need to log in to {need}.", 'token-server-url': 'http://localhost:{port}', 'password-max-length': 128}, 'strip': '0'}, 'bash': {'workers': 32}, 'cassandra': {'service-name': 'all-pods-service'}, 'cql': {'workers': 32, 'samples': 3, 'secret': {'cluster-regex': '(.*?-.*?)-.*', 'name': '{cluster}-superuser', 'password-item': 'password'}, 'alter-tables': {'excludes': 'system_auth,system_traces,reaper_db,system_distributed,system_views,system,system_schema,system_virtual_schema', 'gc-grace-periods': '3600,86400,864000,7776000', 'batching': True}}, 'checks': {'compactions-threshold': 250, 'cpu-busy-threshold': 98.0, 'cpu-threshold': 0.0, 'cassandra-data-path': '/c3/cassandra', 'root-disk-threshold': 50, 'cassandra-disk-threshold': 50, 'snapshot-size-cmd': "ls /c3/cassandra/data/data/*/*/snapshots | grep snapshots | sed 's/:$//g' | xargs -I {} du -sk {} | awk '{print $1}' | awk '{s+=$1} END {print s}'", 'snapshot-size-threshold': '40G', 'table-sizes-cmd': "ls -Al /c3/cassandra/data/data/ | awk '{print $9}' | sed 's/\\^r//g' | xargs -I {} du -sk /c3/cassandra/data/data/{}"}, 'get-host-id': {'workers': 32}, 'idps': {'ad': {'email-pattern': '.*@c3.ai', 'uri': 'https://login.microsoftonline.com/53ad779a-93e7-485c-ba20-ac8290d7252b/oauth2/v2.0/authorize?response_type=id_token&response_mode=form_post&client_id=00ff94a8-6b0a-4715-98e0-95490012d818&scope=openid+email+profile&redirect_uri=https%3A%2F%2Fplat.c3ci.cloud%2Fc3%2Fc3%2Foidc%2Flogin&nonce={nonce}&state=EMPTY', 'jwks-uri': 'https://login.microsoftonline.com/common/discovery/keys', 'contact': 'Please contact ted.tran@c3.ai.', 'whitelist-file': '/kaqing/members'}, 'okta': {'default': True, 'email-pattern': '.*@c3iot.com', 'uri': 'https://c3energy.okta.com/oauth2/v1/authorize?response_type=id_token&response_mode=form_post&client_id={client_id}&scope=openid+email+profile+groups&redirect_uri=https%3A%2F%2F{host}%2Fc3%2Fc3%2Foidc%2Flogin&nonce={nonce}&state=EMPTY', 'jwks-uri': 'https://c3energy.okta.com/oauth2/v1/keys'}}, 'issues': {'workers': 32}, 'logs': {'path': '/c3/cassandra/logs/system.log'}, 'medusa': {'restore-auto-complete': False}, 'nodetool': {'workers': 32, 'samples': 3, 'commands_in_line': 40}, 'pg': {'name-pattern': '^{namespace}.*-k8spg-.*', 'excludes': '.helm., -admin-secret', 'agent': {'name': 'ops-pg-agent', 'just-in-time': False, 'timeout': 86400, 'image': 'seanahnsf/kaqing'}, 'default-db': 'postgres', 'default-schema': 'postgres', 'secret': {'endpoint-key': 'postgres-db-endpoint', 'port-key': 'postgres-db-port', 'username-key': 'postgres-admin-username', 'password-key': 'postgres-admin-password'}}, 'pod': {'name': 'ops', 'image': 'seanahnsf/kaqing-cloud', 'sa': {'name': 'ops', 'proto': 'c3', 'additional-cluster-roles': 'c3aiops-k8ssandra-operator'}, 'label-selector': 'run=ops'}, 'preview': {'rows': 10}, 'processes': {'columns': 'pod,cpu,mem', 'header': 'POD_NAME,CPU,MEM/LIMIT'}, 'reaper': {'service-name': 'reaper-service', 'port-forward': {'timeout': 86400, 'local-port': 9001}, 'abort-runs-batch': 10, 'show-runs-batch': 100, 'pod': {'cluster-regex': '(.*?-.*?-.*?-.*?)-.*', 'label-selector': 'k8ssandra.io/reaper={cluster}-reaper'}, 'secret': {'cluster-regex': '(.*?-.*?)-.*', 'name': '{cluster}-reaper-ui', 'password-item': 'password'}}, 'repair': {'log-path': '/home/cassrepair/logs/', 'image': 'ci-registry.c3iot.io/cloudops/cassrepair:2.0.14', 'secret': 'ciregistryc3iotio', 'env': {'interval': 24, 'timeout': 60, 'pr': False, 'runs': 1}}, 'repl': {'start-drive': 'a', 'auto-enter
|
|
2
|
+
return {'app': {'console-endpoint': 'https://{host}/{env}/{app}/static/console/index.html', 'cr': {'cluster-regex': '(.*?-.*?)-.*', 'group': 'ops.c3.ai', 'v': 'v2', 'plural': 'c3cassandras'}, 'label': 'c3__app_id-0', 'login': {'admin-group': '{host}/C3.ClusterAdmin', 'ingress': '{app_id}-k8singr-appleader-001', 'timeout': 5, 'session-check-url': 'https://{host}/{env}/{app}/api/8/C3/userSessionToken', 'cache-creds': True, 'cache-username': True, 'url': 'https://{host}/{env}/{app}', 'another': "You're logged in to {has}. However, for this app, you need to log in to {need}.", 'token-server-url': 'http://localhost:{port}', 'password-max-length': 128}, 'strip': '0'}, 'audit': {'endpoint': 'https://4psvtaxlcb.execute-api.us-west-2.amazonaws.com/prod/', 'workers': 3, 'timeout': 10, 'log-audit-queries': False, 'athena': {'auto-repair': {'elapsed_hours': 12}, 'region': 'us-west-2', 'catalog': 'AwsDataCatalog', 'database': 'audit', 'repair-partition-tables': 'audit', 'output': 's3://s3.ops--audit/ddl/results', 'repair-cluster-tables': 'cluster'}}, 'bash': {'workers': 32}, 'cassandra': {'service-name': 'all-pods-service'}, 'cql': {'workers': 32, 'samples': 3, 'secret': {'cluster-regex': '(.*?-.*?)-.*', 'name': '{cluster}-superuser', 'password-item': 'password'}, 'alter-tables': {'excludes': 'system_auth,system_traces,reaper_db,system_distributed,system_views,system,system_schema,system_virtual_schema', 'gc-grace-periods': '3600,86400,864000,7776000', 'batching': True}}, 'checks': {'compactions-threshold': 250, 'cpu-busy-threshold': 98.0, 'cpu-threshold': 0.0, 'cassandra-data-path': '/c3/cassandra', 'root-disk-threshold': 50, 'cassandra-disk-threshold': 50, 'snapshot-size-cmd': "ls /c3/cassandra/data/data/*/*/snapshots | grep snapshots | sed 's/:$//g' | xargs -I {} du -sk {} | awk '{print $1}' | awk '{s+=$1} END {print s}'", 'snapshot-size-threshold': '40G', 'table-sizes-cmd': "ls -Al /c3/cassandra/data/data/ | awk '{print $9}' | sed 's/\\^r//g' | xargs -I {} du -sk /c3/cassandra/data/data/{}"}, 'get-host-id': {'workers': 32}, 'idps': {'ad': {'email-pattern': '.*@c3.ai', 'uri': 'https://login.microsoftonline.com/53ad779a-93e7-485c-ba20-ac8290d7252b/oauth2/v2.0/authorize?response_type=id_token&response_mode=form_post&client_id=00ff94a8-6b0a-4715-98e0-95490012d818&scope=openid+email+profile&redirect_uri=https%3A%2F%2Fplat.c3ci.cloud%2Fc3%2Fc3%2Foidc%2Flogin&nonce={nonce}&state=EMPTY', 'jwks-uri': 'https://login.microsoftonline.com/common/discovery/keys', 'contact': 'Please contact ted.tran@c3.ai.', 'whitelist-file': '/kaqing/members'}, 'okta': {'default': True, 'email-pattern': '.*@c3iot.com', 'uri': 'https://c3energy.okta.com/oauth2/v1/authorize?response_type=id_token&response_mode=form_post&client_id={client_id}&scope=openid+email+profile+groups&redirect_uri=https%3A%2F%2F{host}%2Fc3%2Fc3%2Foidc%2Flogin&nonce={nonce}&state=EMPTY', 'jwks-uri': 'https://c3energy.okta.com/oauth2/v1/keys'}}, 'issues': {'workers': 32}, 'logs': {'path': '/c3/cassandra/logs/system.log'}, 'medusa': {'restore-auto-complete': False}, 'nodetool': {'workers': 32, 'samples': 3, 'commands_in_line': 40}, 'pg': {'name-pattern': '^{namespace}.*-k8spg-.*', 'excludes': '.helm., -admin-secret', 'agent': {'name': 'ops-pg-agent', 'just-in-time': False, 'timeout': 86400, 'image': 'seanahnsf/kaqing'}, 'default-db': 'postgres', 'default-schema': 'postgres', 'secret': {'endpoint-key': 'postgres-db-endpoint', 'port-key': 'postgres-db-port', 'username-key': 'postgres-admin-username', 'password-key': 'postgres-admin-password'}}, 'pod': {'name': 'ops', 'image': 'seanahnsf/kaqing-cloud', 'sa': {'name': 'ops', 'proto': 'c3', 'additional-cluster-roles': 'c3aiops-k8ssandra-operator'}, 'label-selector': 'run=ops'}, 'preview': {'rows': 10}, 'processes': {'columns': 'pod,cpu,mem', 'header': 'POD_NAME,CPU,MEM/LIMIT'}, 'reaper': {'service-name': 'reaper-service', 'port-forward': {'timeout': 86400, 'local-port': 9001}, 'abort-runs-batch': 10, 'show-runs-batch': 100, 'pod': {'cluster-regex': '(.*?-.*?-.*?-.*?)-.*', 'label-selector': 'k8ssandra.io/reaper={cluster}-reaper'}, 'secret': {'cluster-regex': '(.*?-.*?)-.*', 'name': '{cluster}-reaper-ui', 'password-item': 'password'}}, 'repair': {'log-path': '/home/cassrepair/logs/', 'image': 'ci-registry.c3iot.io/cloudops/cassrepair:2.0.14', 'secret': 'ciregistryc3iotio', 'env': {'interval': 24, 'timeout': 60, 'pr': False, 'runs': 1}}, 'repl': {'start-drive': 'a', 'a': {'auto-enter': 'c3/c3'}, 'c': {'auto-enter': 'cluster'}, 'history': {'push-cat-remote-log-file': True}, 'background-process': {'auto-nohup': True}}, 'status': {'columns': 'status,address,load,tokens,owns,host_id,gossip,compactions', 'header': '--,Address,Load,Tokens,Owns,Host ID,GOSSIP,COMPACTIONS'}, 'storage': {'columns': 'pod,volume_root,volume_cassandra,snapshots,data,compactions', 'header': 'POD_NAME,VOLUME /,VOLUME CASS,SNAPSHOTS,DATA,COMPACTIONS'}, 'watch': {'auto': 'rollout', 'timeout': 3600, 'interval': 10}, 'debug': False, 'debugs': {'timings': False, 'exit-on-error': False, 'show-parallelism': False}}
|
adam/pod_exec_result.py
CHANGED
|
@@ -15,12 +15,14 @@ class PodExecResult:
|
|
|
15
15
|
# ]
|
|
16
16
|
# }
|
|
17
17
|
# }
|
|
18
|
-
def __init__(self, stdout: str, stderr: str, command: str = None, error_output: str = None):
|
|
18
|
+
def __init__(self, stdout: str, stderr: str, command: str = None, error_output: str = None, pod: str = None, log_file: str = None):
|
|
19
19
|
self.stdout: str = stdout
|
|
20
20
|
self.stderr: str = stderr
|
|
21
21
|
self.command: str = command
|
|
22
22
|
if error_output:
|
|
23
23
|
self.error = yaml.safe_load(error_output)
|
|
24
|
+
self.pod = pod
|
|
25
|
+
self.log_file = log_file
|
|
24
26
|
|
|
25
27
|
def exit_code(self) -> int:
|
|
26
28
|
code = 0
|
|
@@ -30,4 +32,7 @@ class PodExecResult:
|
|
|
30
32
|
except:
|
|
31
33
|
pass
|
|
32
34
|
|
|
33
|
-
return code
|
|
35
|
+
return code
|
|
36
|
+
|
|
37
|
+
def __audit_extra__(self):
|
|
38
|
+
return self.log_file if self.log_file else None
|
adam/repl.py
CHANGED
|
@@ -3,6 +3,7 @@ import re
|
|
|
3
3
|
import time
|
|
4
4
|
import traceback
|
|
5
5
|
import click
|
|
6
|
+
import concurrent
|
|
6
7
|
from prompt_toolkit.completion import NestedCompleter
|
|
7
8
|
from prompt_toolkit.key_binding import KeyBindings
|
|
8
9
|
|
|
@@ -10,15 +11,15 @@ from adam.cli_group import cli
|
|
|
10
11
|
from adam.commands.command import Command
|
|
11
12
|
from adam.commands.command_helpers import ClusterCommandHelper
|
|
12
13
|
from adam.commands.help import Help
|
|
13
|
-
from adam.commands.postgres.
|
|
14
|
+
from adam.commands.postgres.postgres_context import PostgresContext
|
|
14
15
|
from adam.config import Config
|
|
15
|
-
from adam.
|
|
16
|
-
from adam.
|
|
16
|
+
from adam.utils_audits import Audits
|
|
17
|
+
from adam.utils_k8s.kube_context import KubeContext
|
|
18
|
+
from adam.utils_k8s.statefulsets import StatefulSets
|
|
17
19
|
from adam.log import Log
|
|
18
20
|
from adam.repl_commands import ReplCommands
|
|
19
21
|
from adam.repl_session import ReplSession
|
|
20
22
|
from adam.repl_state import ReplState
|
|
21
|
-
from adam.sql.sql_completer import SqlCompleter
|
|
22
23
|
from adam.utils import deep_merge_dicts, deep_sort_dict, lines_to_tabular, log2
|
|
23
24
|
from adam.apps import Apps
|
|
24
25
|
from . import __version__
|
|
@@ -37,7 +38,7 @@ def enter_repl(state: ReplState):
|
|
|
37
38
|
msg = ''
|
|
38
39
|
if state.device == ReplState.P:
|
|
39
40
|
msg = f'{ReplState.P}:'
|
|
40
|
-
pg =
|
|
41
|
+
pg: PostgresContext = PostgresContext.apply(state.namespace, state.pg_path) if state.pg_path else None
|
|
41
42
|
if pg and pg.db:
|
|
42
43
|
msg += pg.db
|
|
43
44
|
elif pg and pg.host:
|
|
@@ -48,6 +49,8 @@ def enter_repl(state: ReplState):
|
|
|
48
49
|
msg += state.app_env
|
|
49
50
|
if state.app_app:
|
|
50
51
|
msg += f'/{state.app_app}'
|
|
52
|
+
elif state.device == ReplState.L:
|
|
53
|
+
msg = f'{ReplState.L}:'
|
|
51
54
|
else:
|
|
52
55
|
msg = f'{ReplState.C}:'
|
|
53
56
|
if state.pod:
|
|
@@ -62,26 +65,36 @@ def enter_repl(state: ReplState):
|
|
|
62
65
|
return f"{msg}$ " if state.bash_session else f"{msg}> "
|
|
63
66
|
|
|
64
67
|
Log.log2(f'kaqing {__version__}')
|
|
65
|
-
ss = StatefulSets.list_sts_name_and_ns()
|
|
66
68
|
|
|
67
69
|
if state.device == ReplState.C:
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
state.
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
state.app_app = ea[1]
|
|
82
|
-
Config().wait_log(f'Moving to {state.app_env}/{state.app_app}...')
|
|
70
|
+
auto_enter = Config().get('repl.c.auto-enter', 'cluster')
|
|
71
|
+
if auto_enter and auto_enter in ['cluster', 'first-pod']:
|
|
72
|
+
ss = StatefulSets.list_sts_name_and_ns()
|
|
73
|
+
if not ss:
|
|
74
|
+
log2("No Cassandra clusters found.")
|
|
75
|
+
elif not state.sts and len(ss) == 1:
|
|
76
|
+
cluster = ss[0]
|
|
77
|
+
state.sts = cluster[0]
|
|
78
|
+
state.namespace = cluster[1]
|
|
79
|
+
if auto_enter == 'first-pod':
|
|
80
|
+
state.pod = f'{state.sts}-0'
|
|
81
|
+
if KubeContext().in_cluster_namespace:
|
|
82
|
+
Config().wait_log(f'Moving to the only Cassandra cluster: {state.sts}...')
|
|
83
83
|
else:
|
|
84
|
-
Config().wait_log(f'Moving to {state.
|
|
84
|
+
Config().wait_log(f'Moving to the only Cassandra cluster: {state.sts}@{state.namespace}...')
|
|
85
|
+
elif state.device == ReplState.A:
|
|
86
|
+
if not state.app_env:
|
|
87
|
+
if app := Config().get('repl.a.auto-enter-app', 'c3/c3'):
|
|
88
|
+
if app != 'no':
|
|
89
|
+
ea = app.split('/')
|
|
90
|
+
state.app_env = ea[0]
|
|
91
|
+
if len(ea) > 1:
|
|
92
|
+
state.app_app = ea[1]
|
|
93
|
+
Config().wait_log(f'Moving to {state.app_env}/{state.app_app}...')
|
|
94
|
+
else:
|
|
95
|
+
Config().wait_log(f'Moving to {state.app_env}...')
|
|
96
|
+
elif state.device == ReplState.P:
|
|
97
|
+
Config().wait_log('Inspecting postgres database instances...')
|
|
85
98
|
|
|
86
99
|
kb = KeyBindings()
|
|
87
100
|
|
|
@@ -89,73 +102,112 @@ def enter_repl(state: ReplState):
|
|
|
89
102
|
def _(event):
|
|
90
103
|
event.app.current_buffer.text = ''
|
|
91
104
|
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
105
|
+
with concurrent.futures.ThreadPoolExecutor(max_workers=Config().get('audit.workers', 3)) as executor:
|
|
106
|
+
# warm up AWS lambda - this log line may timeout and get lost, which is fine
|
|
107
|
+
executor.submit(Audits.log, 'entering kaqing repl', state.namespace, 'z', 0.0)
|
|
108
|
+
|
|
109
|
+
s0 = time.time()
|
|
110
|
+
|
|
111
|
+
# use sorted command list only for auto-completion
|
|
112
|
+
sorted_cmds = sorted(cmd_list, key=lambda cmd: cmd.command())
|
|
113
|
+
while True:
|
|
114
|
+
result = None
|
|
115
|
+
try:
|
|
116
|
+
completer = NestedCompleter.from_nested_dict({})
|
|
117
|
+
if not state.bash_session:
|
|
118
|
+
completions = {}
|
|
119
|
+
# app commands are available only on a: drive
|
|
120
|
+
if state.device == ReplState.A and state.app_app:
|
|
121
|
+
completions = Apps(path='apps.yaml').commands()
|
|
122
|
+
|
|
123
|
+
for cmd in sorted_cmds:
|
|
124
|
+
s1 = time.time()
|
|
125
|
+
try:
|
|
126
|
+
completions = deep_sort_dict(deep_merge_dicts(completions, cmd.completion(state)))
|
|
127
|
+
finally:
|
|
128
|
+
if Config().get('debugs.timings', False):
|
|
129
|
+
log2(f'Timing auto-completion-calc {cmd.command()}: {time.time() - s1:.2f}')
|
|
130
|
+
|
|
131
|
+
# print(json.dumps(completions, indent=4))
|
|
132
|
+
completer = NestedCompleter.from_nested_dict(completions)
|
|
133
|
+
|
|
134
|
+
cmd = session.prompt(prompt_msg(), completer=completer, key_bindings=kb)
|
|
135
|
+
s0 = time.time()
|
|
136
|
+
|
|
137
|
+
if state.bash_session:
|
|
138
|
+
if cmd.strip(' ') == 'exit':
|
|
139
|
+
state.exit_bash()
|
|
140
|
+
continue
|
|
141
|
+
|
|
142
|
+
cmd = f'bash {cmd}'
|
|
143
|
+
|
|
144
|
+
if cmd and cmd.strip(' ') and not (result := cmds.run(cmd, state)):
|
|
145
|
+
# not served by any command in the chain; try SQL query or C3 action
|
|
146
|
+
result = try_device_default_action(state, cmds, cmd_list, cmd)
|
|
147
|
+
except EOFError: # Handle Ctrl+D (EOF) for graceful exit
|
|
148
|
+
break
|
|
149
|
+
except Exception as e:
|
|
150
|
+
if Config().get('debugs.exit-on-error', False):
|
|
151
|
+
raise e
|
|
152
|
+
else:
|
|
153
|
+
log2(e)
|
|
154
|
+
Config().debug(traceback.format_exc())
|
|
155
|
+
finally:
|
|
156
|
+
Config().clear_wait_log_flag()
|
|
157
|
+
if Config().get('debugs.timings', False) and 'cmd' in locals() and 's0' in locals():
|
|
158
|
+
log2(f'Timing command {cmd}: {time.time() - s0:.2f}')
|
|
159
|
+
|
|
160
|
+
# offload audit logging
|
|
161
|
+
if cmd and (state.device != ReplState.L or Config().get('audit.log-audit-queries', False)):
|
|
162
|
+
executor.submit(Audits.log, cmd, state.namespace, state.device, time.time() - s0, get_audit_extra(result))
|
|
163
|
+
|
|
164
|
+
def try_device_default_action(state: ReplState, cmds: Command, cmd_list: list[Command], cmd: str):
|
|
165
|
+
result = None
|
|
166
|
+
|
|
167
|
+
c_sql_tried = False
|
|
168
|
+
if state.device == ReplState.P:
|
|
169
|
+
pg: PostgresContext = PostgresContext.apply(state.namespace, state.pg_path)
|
|
170
|
+
if pg.db:
|
|
171
|
+
c_sql_tried = True
|
|
172
|
+
cmd = f'pg {cmd}'
|
|
173
|
+
result = cmds.run(cmd, state)
|
|
174
|
+
elif state.device == ReplState.A:
|
|
175
|
+
if state.app_app:
|
|
176
|
+
c_sql_tried = True
|
|
177
|
+
cmd = f'app {cmd}'
|
|
178
|
+
result = cmds.run(cmd, state)
|
|
179
|
+
elif state.device == ReplState.L:
|
|
180
|
+
c_sql_tried = True
|
|
181
|
+
cmd = f'audit {cmd}'
|
|
182
|
+
result = cmds.run(cmd, state)
|
|
183
|
+
elif state.sts:
|
|
184
|
+
c_sql_tried = True
|
|
185
|
+
cmd = f'cql {cmd}'
|
|
186
|
+
result = cmds.run(cmd, state)
|
|
187
|
+
|
|
188
|
+
if not c_sql_tried:
|
|
189
|
+
log2(f'* Invalid command: {cmd}')
|
|
190
|
+
log2()
|
|
191
|
+
lines = [c.help(state) for c in cmd_list if c.help(state)]
|
|
192
|
+
log2(lines_to_tabular(lines, separator='\t'))
|
|
193
|
+
|
|
194
|
+
return result
|
|
195
|
+
|
|
196
|
+
def get_audit_extra(result: any):
|
|
197
|
+
if not result:
|
|
198
|
+
return None
|
|
199
|
+
|
|
200
|
+
if type(result) is list:
|
|
201
|
+
extras = set()
|
|
202
|
+
|
|
203
|
+
for r in result:
|
|
204
|
+
if hasattr(r, '__audit_extra__') and (x := r.__audit_extra__()):
|
|
205
|
+
extras.add(x)
|
|
206
|
+
|
|
207
|
+
return ','.join(list(extras))
|
|
208
|
+
|
|
209
|
+
if hasattr(result, '__audit_extra__') and (x := result.__audit_extra__()):
|
|
210
|
+
return x
|
|
159
211
|
|
|
160
212
|
@cli.command(context_settings=dict(ignore_unknown_options=True, allow_extra_args=True), cls=ClusterCommandHelper, help="Enter interactive shell.")
|
|
161
213
|
@click.option('--kubeconfig', '-k', required=False, metavar='path', help='path to kubeconfig file')
|