kaqing 2.0.50__py3-none-any.whl → 2.0.110__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of kaqing might be problematic. Click here for more details.

Files changed (123) hide show
  1. adam/apps.py +2 -2
  2. adam/batch.py +11 -15
  3. adam/checks/check_utils.py +4 -4
  4. adam/checks/compactionstats.py +1 -1
  5. adam/checks/cpu.py +2 -2
  6. adam/checks/disk.py +1 -1
  7. adam/checks/gossip.py +1 -1
  8. adam/checks/memory.py +3 -3
  9. adam/checks/status.py +1 -1
  10. adam/commands/alter_tables.py +3 -14
  11. adam/commands/app.py +2 -2
  12. adam/commands/app_ping.py +2 -2
  13. adam/commands/audit/audit.py +85 -0
  14. adam/commands/audit/audit_repair_tables.py +76 -0
  15. adam/commands/audit/audit_run.py +57 -0
  16. adam/commands/audit/show_last10.py +50 -0
  17. adam/commands/audit/show_slow10.py +49 -0
  18. adam/commands/audit/show_top10.py +48 -0
  19. adam/commands/audit/utils_show_top10.py +59 -0
  20. adam/commands/bash.py +76 -13
  21. adam/commands/cd.py +22 -13
  22. adam/commands/check.py +6 -0
  23. adam/commands/cli_commands.py +3 -3
  24. adam/commands/command.py +15 -11
  25. adam/commands/commands_utils.py +4 -5
  26. adam/commands/cql/cql_completions.py +7 -5
  27. adam/commands/cql/cql_utils.py +13 -10
  28. adam/commands/cql/cqlsh.py +6 -3
  29. adam/commands/deploy/code_utils.py +2 -2
  30. adam/commands/deploy/deploy.py +7 -1
  31. adam/commands/deploy/deploy_pg_agent.py +2 -2
  32. adam/commands/deploy/deploy_pod.py +6 -6
  33. adam/commands/deploy/deploy_utils.py +2 -2
  34. adam/commands/deploy/undeploy.py +7 -1
  35. adam/commands/deploy/undeploy_pg_agent.py +2 -2
  36. adam/commands/deploy/undeploy_pod.py +4 -4
  37. adam/commands/devices.py +29 -0
  38. adam/commands/help.py +10 -7
  39. adam/commands/issues.py +6 -0
  40. adam/commands/login.py +6 -3
  41. adam/commands/logs.py +2 -1
  42. adam/commands/ls.py +30 -24
  43. adam/commands/medusa/medusa_backup.py +2 -2
  44. adam/commands/medusa/medusa_restore.py +2 -2
  45. adam/commands/medusa/medusa_show_backupjobs.py +3 -2
  46. adam/commands/medusa/medusa_show_restorejobs.py +2 -2
  47. adam/commands/nodetool.py +5 -3
  48. adam/commands/postgres/postgres.py +3 -3
  49. adam/commands/postgres/{postgres_session.py → postgres_context.py} +29 -30
  50. adam/commands/postgres/postgres_utils.py +5 -5
  51. adam/commands/postgres/psql_completions.py +2 -3
  52. adam/commands/preview_table.py +17 -32
  53. adam/commands/pwd.py +5 -2
  54. adam/commands/reaper/reaper.py +3 -0
  55. adam/commands/reaper/reaper_restart.py +1 -1
  56. adam/commands/reaper/reaper_session.py +1 -1
  57. adam/commands/repair/repair.py +3 -3
  58. adam/commands/repair/repair_log.py +1 -1
  59. adam/commands/repair/repair_run.py +2 -2
  60. adam/commands/repair/repair_scan.py +1 -1
  61. adam/commands/repair/repair_stop.py +1 -1
  62. adam/commands/report.py +6 -0
  63. adam/commands/restart.py +2 -2
  64. adam/commands/rollout.py +1 -1
  65. adam/commands/show/show.py +5 -2
  66. adam/commands/show/show_app_actions.py +3 -0
  67. adam/commands/show/show_app_id.py +1 -1
  68. adam/commands/show/show_app_queues.py +3 -2
  69. adam/commands/show/show_cassandra_status.py +3 -3
  70. adam/commands/show/show_cassandra_version.py +3 -3
  71. adam/commands/show/show_host.py +33 -0
  72. adam/commands/show/show_login.py +3 -0
  73. adam/commands/show/show_processes.py +1 -1
  74. adam/commands/show/show_repairs.py +2 -2
  75. adam/commands/show/show_storage.py +1 -1
  76. adam/commands/watch.py +1 -1
  77. adam/config.py +2 -1
  78. adam/embedded_params.py +1 -1
  79. adam/pod_exec_result.py +7 -2
  80. adam/repl.py +141 -89
  81. adam/repl_commands.py +21 -20
  82. adam/repl_state.py +167 -39
  83. adam/sql/sql_completer.py +89 -49
  84. adam/sql/sql_state_machine.py +518 -0
  85. adam/sql/term_completer.py +76 -0
  86. adam/sso/cred_cache.py +1 -1
  87. adam/sso/idp.py +1 -1
  88. adam/utils.py +0 -1
  89. adam/utils_audits.py +193 -0
  90. adam/{k8s_utils → utils_k8s}/cassandra_clusters.py +6 -8
  91. adam/{k8s_utils → utils_k8s}/cassandra_nodes.py +11 -4
  92. adam/{k8s_utils → utils_k8s}/deployment.py +2 -2
  93. adam/{k8s_utils → utils_k8s}/pods.py +33 -9
  94. adam/{k8s_utils → utils_k8s}/secrets.py +4 -0
  95. adam/{k8s_utils → utils_k8s}/statefulsets.py +4 -4
  96. adam/utils_net.py +24 -0
  97. adam/version.py +1 -1
  98. {kaqing-2.0.50.dist-info → kaqing-2.0.110.dist-info}/METADATA +1 -1
  99. kaqing-2.0.110.dist-info/RECORD +187 -0
  100. adam/commands/cql/cql_table_completer.py +0 -16
  101. adam/commands/describe/describe.py +0 -46
  102. adam/commands/describe/describe_keyspace.py +0 -60
  103. adam/commands/describe/describe_keyspaces.py +0 -50
  104. adam/commands/describe/describe_table.py +0 -60
  105. adam/commands/describe/describe_tables.py +0 -50
  106. adam/commands/postgres/psql_table_completer.py +0 -18
  107. adam/sql/any_completer.py +0 -84
  108. adam/sql/sql_utils.py +0 -5
  109. adam/sql/table_name_completer.py +0 -17
  110. kaqing-2.0.50.dist-info/RECORD +0 -185
  111. /adam/commands/{describe → audit}/__init__.py +0 -0
  112. /adam/{k8s_utils → utils_k8s}/__init__.py +0 -0
  113. /adam/{k8s_utils → utils_k8s}/config_maps.py +0 -0
  114. /adam/{k8s_utils → utils_k8s}/custom_resources.py +0 -0
  115. /adam/{k8s_utils → utils_k8s}/ingresses.py +0 -0
  116. /adam/{k8s_utils → utils_k8s}/jobs.py +0 -0
  117. /adam/{k8s_utils → utils_k8s}/kube_context.py +0 -0
  118. /adam/{k8s_utils → utils_k8s}/service_accounts.py +0 -0
  119. /adam/{k8s_utils → utils_k8s}/services.py +0 -0
  120. /adam/{k8s_utils → utils_k8s}/volumes.py +0 -0
  121. {kaqing-2.0.50.dist-info → kaqing-2.0.110.dist-info}/WHEEL +0 -0
  122. {kaqing-2.0.50.dist-info → kaqing-2.0.110.dist-info}/entry_points.txt +0 -0
  123. {kaqing-2.0.50.dist-info → kaqing-2.0.110.dist-info}/top_level.txt +0 -0
adam/commands/report.py CHANGED
@@ -22,12 +22,18 @@ class Report(Command):
22
22
  def command(self):
23
23
  return Report.COMMAND
24
24
 
25
+ def required(self):
26
+ return ReplState.NON_L
27
+
25
28
  def run(self, cmd: str, state: ReplState):
26
29
  if not(args := self.args(cmd)):
27
30
  return super().run(cmd, state)
28
31
 
29
32
  output: dict[str, any] = {}
30
33
  state, args = self.apply_state(args, state)
34
+ if not self.validate_state(state):
35
+ return state
36
+
31
37
  if state.in_repl:
32
38
  args, show = Command.extract_options(args, ['-s', '--show'])
33
39
 
adam/commands/restart.py CHANGED
@@ -1,6 +1,6 @@
1
1
  from adam.commands.command import Command
2
- from adam.k8s_utils.pods import Pods
3
- from adam.k8s_utils.statefulsets import StatefulSets
2
+ from adam.utils_k8s.pods import Pods
3
+ from adam.utils_k8s.statefulsets import StatefulSets
4
4
  from adam.repl_state import ReplState, RequiredState
5
5
  from adam.utils import log2
6
6
 
adam/commands/rollout.py CHANGED
@@ -4,7 +4,7 @@ from kubernetes.client.rest import ApiException
4
4
 
5
5
  from adam.commands.command import Command
6
6
  from adam.commands.watch import Watch
7
- from adam.k8s_utils.statefulsets import StatefulSets
7
+ from adam.utils_k8s.statefulsets import StatefulSets
8
8
  from adam.config import Config
9
9
  from adam.repl_state import ReplState, RequiredState
10
10
  from adam.utils import duration, log2
@@ -1,10 +1,12 @@
1
1
  import click
2
2
 
3
+ from adam.commands.audit.show_last10 import ShowLast10
3
4
  from adam.commands.command import Command
4
5
  from adam.commands.medusa.medusa_show_backupjobs import MedusaShowBackupJobs
5
6
  from adam.commands.medusa.medusa_show_restorejobs import MedusaShowRestoreJobs
6
7
  from adam.commands.show.show_app_actions import ShowAppActions
7
8
  from adam.commands.show.show_app_queues import ShowAppQueues
9
+ from adam.commands.show.show_host import ShowHost
8
10
  from adam.commands.show.show_login import ShowLogin
9
11
  from .show_params import ShowParams
10
12
  from .show_app_id import ShowAppId
@@ -39,9 +41,10 @@ class Show(Command):
39
41
  return super().intermediate_run(cmd, state, args, Show.cmd_list())
40
42
 
41
43
  def cmd_list():
42
- return [ShowAppActions(), ShowAppId(), ShowAppQueues(), ShowLogin(), ShowKubectlCommands(),
44
+ return [ShowAppActions(), ShowAppId(), ShowAppQueues(), ShowHost(), ShowLogin(), ShowKubectlCommands(),
43
45
  ShowParams(), ShowProcesses(), ShowRepairs(), ShowStorage(), ShowAdam(),
44
- ShowCassandraStatus(), ShowCassandraVersion(), MedusaShowRestoreJobs(), MedusaShowBackupJobs()]
46
+ ShowCassandraStatus(), ShowCassandraVersion(), MedusaShowRestoreJobs(), MedusaShowBackupJobs(),
47
+ ShowLast10()]
45
48
 
46
49
  def completion(self, state: ReplState):
47
50
  return super().completion(state)
@@ -20,6 +20,9 @@ class ShowAppActions(Command):
20
20
  def command(self):
21
21
  return ShowAppActions.COMMAND
22
22
 
23
+ def required(self):
24
+ return ReplState.A
25
+
23
26
  def run(self, cmd: str, state: ReplState):
24
27
  if not self.args(cmd):
25
28
  return super().run(cmd, state)
@@ -1,5 +1,5 @@
1
1
  from adam.commands.command import Command
2
- from adam.k8s_utils.custom_resources import CustomResources
2
+ from adam.utils_k8s.custom_resources import CustomResources
3
3
  from adam.repl_state import ReplState, RequiredState
4
4
  from adam.utils import log
5
5
 
@@ -18,14 +18,15 @@ class ShowAppQueues(Command):
18
18
  return ShowAppQueues.COMMAND
19
19
 
20
20
  def required(self):
21
- return RequiredState.CLUSTER_OR_POD
21
+ return RequiredState.APP_APP
22
22
 
23
23
  def run(self, cmd: str, state: ReplState):
24
24
  if not(args := self.args(cmd)):
25
25
  return super().run(cmd, state)
26
26
 
27
27
  state, args = self.apply_state(args, state)
28
- if not self.validate_state(state, app_required=RequiredState.APP_APP):
28
+ # if not self.validate_state(state, app_required=RequiredState.APP_APP):
29
+ if not self.validate_state(state):
29
30
  return state
30
31
 
31
32
  _, forced = Command.extract_options(args, '--force')
@@ -8,9 +8,9 @@ from adam.columns.columns import Columns
8
8
  from adam.commands.command import Command
9
9
  from adam.commands.issues import Issues
10
10
  from adam.config import Config
11
- from adam.k8s_utils.cassandra_nodes import CassandraNodes
12
- from adam.k8s_utils.secrets import Secrets
13
- from adam.k8s_utils.statefulsets import StatefulSets
11
+ from adam.utils_k8s.cassandra_nodes import CassandraNodes
12
+ from adam.utils_k8s.secrets import Secrets
13
+ from adam.utils_k8s.statefulsets import StatefulSets
14
14
  from adam.repl_state import ReplState, RequiredState
15
15
  from adam.utils import lines_to_tabular, log, log2
16
16
  from adam.checks.status import parse_nodetool_status
@@ -1,7 +1,7 @@
1
1
  from adam.commands.command import Command
2
- from adam.k8s_utils.cassandra_clusters import CassandraClusters
3
- from adam.k8s_utils.cassandra_nodes import CassandraNodes
4
- from adam.k8s_utils.secrets import Secrets
2
+ from adam.utils_k8s.cassandra_clusters import CassandraClusters
3
+ from adam.utils_k8s.cassandra_nodes import CassandraNodes
4
+ from adam.utils_k8s.secrets import Secrets
5
5
  from adam.repl_state import ReplState, RequiredState
6
6
 
7
7
  class ShowCassandraVersion(Command):
@@ -0,0 +1,33 @@
1
+ from adam.commands.command import Command
2
+ from adam.repl_state import ReplState
3
+ from adam.utils import log
4
+ from adam.utils_net import get_my_host
5
+
6
+ class ShowHost(Command):
7
+ COMMAND = 'show host'
8
+
9
+ # the singleton pattern
10
+ def __new__(cls, *args, **kwargs):
11
+ if not hasattr(cls, 'instance'): cls.instance = super(ShowHost, cls).__new__(cls)
12
+
13
+ return cls.instance
14
+
15
+ def __init__(self, successor: Command=None):
16
+ super().__init__(successor)
17
+
18
+ def command(self):
19
+ return ShowHost.COMMAND
20
+
21
+ def run(self, cmd: str, state: ReplState):
22
+ if not(args := self.args(cmd)):
23
+ return super().run(cmd, state)
24
+
25
+ log(get_my_host())
26
+
27
+ return state
28
+
29
+ def completion(self, state: ReplState):
30
+ return super().completion(state)
31
+
32
+ def help(self, _: ReplState):
33
+ return f'{ShowHost.COMMAND}\t show host'
@@ -24,6 +24,9 @@ class ShowLogin(Command):
24
24
  def command(self):
25
25
  return ShowLogin.COMMAND
26
26
 
27
+ def required(self):
28
+ return ReplState.NON_L
29
+
27
30
  def run(self, cmd: str, state: ReplState):
28
31
  if not(args := self.args(cmd)):
29
32
  return super().run(cmd, state)
@@ -1,7 +1,7 @@
1
1
  from adam.commands.command import Command
2
2
  from adam.commands.commands_utils import show_table
3
3
  from adam.config import Config
4
- from adam.k8s_utils.statefulsets import StatefulSets
4
+ from adam.utils_k8s.statefulsets import StatefulSets
5
5
  from adam.repl_state import ReplState, RequiredState
6
6
 
7
7
  class ShowProcesses(Command):
@@ -1,6 +1,6 @@
1
1
  from adam.commands.command import Command
2
- from adam.k8s_utils.cassandra_clusters import CassandraClusters
3
- from adam.k8s_utils.cassandra_nodes import CassandraNodes
2
+ from adam.utils_k8s.cassandra_clusters import CassandraClusters
3
+ from adam.utils_k8s.cassandra_nodes import CassandraNodes
4
4
  from adam.repl_state import ReplState, RequiredState
5
5
 
6
6
  class ShowRepairs(Command):
@@ -1,7 +1,7 @@
1
1
  from adam.commands.command import Command
2
2
  from adam.commands.commands_utils import show_table
3
3
  from adam.config import Config
4
- from adam.k8s_utils.statefulsets import StatefulSets
4
+ from adam.utils_k8s.statefulsets import StatefulSets
5
5
  from adam.repl_state import ReplState, RequiredState
6
6
 
7
7
  class ShowStorage(Command):
adam/commands/watch.py CHANGED
@@ -6,7 +6,7 @@ from typing import List
6
6
  from adam.commands.command import Command
7
7
  from adam.commands.commands_utils import show_pods, show_rollout
8
8
  from adam.config import Config
9
- from adam.k8s_utils.statefulsets import StatefulSets
9
+ from adam.utils_k8s.statefulsets import StatefulSets
10
10
  from adam.repl_state import ReplState, RequiredState
11
11
  from adam.utils import convert_seconds, log2
12
12
 
adam/config.py CHANGED
@@ -1,3 +1,4 @@
1
+ import os
1
2
  from typing import TypeVar, cast
2
3
  import yaml
3
4
 
@@ -38,7 +39,7 @@ class Config:
38
39
  return get_deep_keys(self.params)
39
40
 
40
41
  def is_debug(self):
41
- return Config().get('debug', False)
42
+ return os.getenv('QING_DEV', 'false').lower() == 'true' or Config().get('debug', False)
42
43
 
43
44
  def debug(self, s: None):
44
45
  if self.is_debug():
adam/embedded_params.py CHANGED
@@ -1,2 +1,2 @@
1
1
  def config():
2
- return {'app': {'console-endpoint': 'https://{host}/{env}/{app}/static/console/index.html', 'cr': {'cluster-regex': '(.*?-.*?)-.*', 'group': 'ops.c3.ai', 'v': 'v2', 'plural': 'c3cassandras'}, 'label': 'c3__app_id-0', 'login': {'admin-group': '{host}/C3.ClusterAdmin', 'ingress': '{app_id}-k8singr-appleader-001', 'timeout': 5, 'session-check-url': 'https://{host}/{env}/{app}/api/8/C3/userSessionToken', 'cache-creds': True, 'cache-username': True, 'url': 'https://{host}/{env}/{app}', 'another': "You're logged in to {has}. However, for this app, you need to log in to {need}.", 'token-server-url': 'http://localhost:{port}', 'password-max-length': 128}, 'strip': '0'}, 'bash': {'workers': 32}, 'cassandra': {'service-name': 'all-pods-service'}, 'cql': {'workers': 32, 'samples': 3, 'secret': {'cluster-regex': '(.*?-.*?)-.*', 'name': '{cluster}-superuser', 'password-item': 'password'}, 'alter-tables': {'excludes': 'system_auth,system_traces,reaper_db,system_distributed,system_views,system,system_schema,system_virtual_schema', 'gc-grace-periods': '3600,86400,864000,7776000', 'batching': True}}, 'checks': {'compactions-threshold': 250, 'cpu-busy-threshold': 98.0, 'cpu-threshold': 0.0, 'cassandra-data-path': '/c3/cassandra', 'root-disk-threshold': 50, 'cassandra-disk-threshold': 50, 'snapshot-size-cmd': "ls /c3/cassandra/data/data/*/*/snapshots | grep snapshots | sed 's/:$//g' | xargs -I {} du -sk {} | awk '{print $1}' | awk '{s+=$1} END {print s}'", 'snapshot-size-threshold': '40G', 'table-sizes-cmd': "ls -Al /c3/cassandra/data/data/ | awk '{print $9}' | sed 's/\\^r//g' | xargs -I {} du -sk /c3/cassandra/data/data/{}"}, 'get-host-id': {'workers': 32}, 'idps': {'ad': {'email-pattern': '.*@c3.ai', 'uri': 'https://login.microsoftonline.com/53ad779a-93e7-485c-ba20-ac8290d7252b/oauth2/v2.0/authorize?response_type=id_token&response_mode=form_post&client_id=00ff94a8-6b0a-4715-98e0-95490012d818&scope=openid+email+profile&redirect_uri=https%3A%2F%2Fplat.c3ci.cloud%2Fc3%2Fc3%2Foidc%2Flogin&nonce={nonce}&state=EMPTY', 'jwks-uri': 'https://login.microsoftonline.com/common/discovery/keys', 'contact': 'Please contact ted.tran@c3.ai.', 'whitelist-file': '/kaqing/members'}, 'okta': {'default': True, 'email-pattern': '.*@c3iot.com', 'uri': 'https://c3energy.okta.com/oauth2/v1/authorize?response_type=id_token&response_mode=form_post&client_id={client_id}&scope=openid+email+profile+groups&redirect_uri=https%3A%2F%2F{host}%2Fc3%2Fc3%2Foidc%2Flogin&nonce={nonce}&state=EMPTY', 'jwks-uri': 'https://c3energy.okta.com/oauth2/v1/keys'}}, 'issues': {'workers': 32}, 'logs': {'path': '/c3/cassandra/logs/system.log'}, 'medusa': {'restore-auto-complete': False}, 'nodetool': {'workers': 32, 'samples': 3, 'commands_in_line': 40}, 'pg': {'name-pattern': '^{namespace}.*-k8spg-.*', 'excludes': '.helm., -admin-secret', 'agent': {'name': 'ops-pg-agent', 'just-in-time': False, 'timeout': 86400, 'image': 'seanahnsf/kaqing'}, 'default-db': 'postgres', 'default-schema': 'postgres', 'secret': {'endpoint-key': 'postgres-db-endpoint', 'port-key': 'postgres-db-port', 'username-key': 'postgres-admin-username', 'password-key': 'postgres-admin-password'}}, 'pod': {'name': 'ops', 'image': 'seanahnsf/kaqing-cloud', 'sa': {'name': 'ops', 'proto': 'c3', 'additional-cluster-roles': 'c3aiops-k8ssandra-operator'}, 'label-selector': 'run=ops'}, 'preview': {'rows': 10}, 'processes': {'columns': 'pod,cpu,mem', 'header': 'POD_NAME,CPU,MEM/LIMIT'}, 'reaper': {'service-name': 'reaper-service', 'port-forward': {'timeout': 86400, 'local-port': 9001}, 'abort-runs-batch': 10, 'show-runs-batch': 100, 'pod': {'cluster-regex': '(.*?-.*?-.*?-.*?)-.*', 'label-selector': 'k8ssandra.io/reaper={cluster}-reaper'}, 'secret': {'cluster-regex': '(.*?-.*?)-.*', 'name': '{cluster}-reaper-ui', 'password-item': 'password'}}, 'repair': {'log-path': '/home/cassrepair/logs/', 'image': 'ci-registry.c3iot.io/cloudops/cassrepair:2.0.14', 'secret': 'ciregistryc3iotio', 'env': {'interval': 24, 'timeout': 60, 'pr': False, 'runs': 1}}, 'repl': {'start-drive': 'a', 'auto-enter-app': 'c3/c3', 'auto-enter-only-cluster': True}, 'status': {'columns': 'status,address,load,tokens,owns,host_id,gossip,compactions', 'header': '--,Address,Load,Tokens,Owns,Host ID,GOSSIP,COMPACTIONS'}, 'storage': {'columns': 'pod,volume_root,volume_cassandra,snapshots,data,compactions', 'header': 'POD_NAME,VOLUME /,VOLUME CASS,SNAPSHOTS,DATA,COMPACTIONS'}, 'watch': {'auto': 'rollout', 'timeout': 3600, 'interval': 10}, 'debug': False, 'debugs': {'timings': False, 'exit-on-error': False, 'show-parallelism': False}}
2
+ return {'app': {'console-endpoint': 'https://{host}/{env}/{app}/static/console/index.html', 'cr': {'cluster-regex': '(.*?-.*?)-.*', 'group': 'ops.c3.ai', 'v': 'v2', 'plural': 'c3cassandras'}, 'label': 'c3__app_id-0', 'login': {'admin-group': '{host}/C3.ClusterAdmin', 'ingress': '{app_id}-k8singr-appleader-001', 'timeout': 5, 'session-check-url': 'https://{host}/{env}/{app}/api/8/C3/userSessionToken', 'cache-creds': True, 'cache-username': True, 'url': 'https://{host}/{env}/{app}', 'another': "You're logged in to {has}. However, for this app, you need to log in to {need}.", 'token-server-url': 'http://localhost:{port}', 'password-max-length': 128}, 'strip': '0'}, 'audit': {'endpoint': 'https://4psvtaxlcb.execute-api.us-west-2.amazonaws.com/prod/', 'workers': 3, 'timeout': 10, 'log-audit-queries': False, 'athena': {'auto-repair': {'elapsed_hours': 12}, 'region': 'us-west-2', 'catalog': 'AwsDataCatalog', 'database': 'audit', 'repair-partition-tables': 'audit', 'output': 's3://s3.ops--audit/ddl/results', 'repair-cluster-tables': 'cluster'}}, 'bash': {'workers': 32}, 'cassandra': {'service-name': 'all-pods-service'}, 'cql': {'workers': 32, 'samples': 3, 'secret': {'cluster-regex': '(.*?-.*?)-.*', 'name': '{cluster}-superuser', 'password-item': 'password'}, 'alter-tables': {'excludes': 'system_auth,system_traces,reaper_db,system_distributed,system_views,system,system_schema,system_virtual_schema', 'gc-grace-periods': '3600,86400,864000,7776000', 'batching': True}}, 'checks': {'compactions-threshold': 250, 'cpu-busy-threshold': 98.0, 'cpu-threshold': 0.0, 'cassandra-data-path': '/c3/cassandra', 'root-disk-threshold': 50, 'cassandra-disk-threshold': 50, 'snapshot-size-cmd': "ls /c3/cassandra/data/data/*/*/snapshots | grep snapshots | sed 's/:$//g' | xargs -I {} du -sk {} | awk '{print $1}' | awk '{s+=$1} END {print s}'", 'snapshot-size-threshold': '40G', 'table-sizes-cmd': "ls -Al /c3/cassandra/data/data/ | awk '{print $9}' | sed 's/\\^r//g' | xargs -I {} du -sk /c3/cassandra/data/data/{}"}, 'get-host-id': {'workers': 32}, 'idps': {'ad': {'email-pattern': '.*@c3.ai', 'uri': 'https://login.microsoftonline.com/53ad779a-93e7-485c-ba20-ac8290d7252b/oauth2/v2.0/authorize?response_type=id_token&response_mode=form_post&client_id=00ff94a8-6b0a-4715-98e0-95490012d818&scope=openid+email+profile&redirect_uri=https%3A%2F%2Fplat.c3ci.cloud%2Fc3%2Fc3%2Foidc%2Flogin&nonce={nonce}&state=EMPTY', 'jwks-uri': 'https://login.microsoftonline.com/common/discovery/keys', 'contact': 'Please contact ted.tran@c3.ai.', 'whitelist-file': '/kaqing/members'}, 'okta': {'default': True, 'email-pattern': '.*@c3iot.com', 'uri': 'https://c3energy.okta.com/oauth2/v1/authorize?response_type=id_token&response_mode=form_post&client_id={client_id}&scope=openid+email+profile+groups&redirect_uri=https%3A%2F%2F{host}%2Fc3%2Fc3%2Foidc%2Flogin&nonce={nonce}&state=EMPTY', 'jwks-uri': 'https://c3energy.okta.com/oauth2/v1/keys'}}, 'issues': {'workers': 32}, 'logs': {'path': '/c3/cassandra/logs/system.log'}, 'medusa': {'restore-auto-complete': False}, 'nodetool': {'workers': 32, 'samples': 3, 'commands_in_line': 40}, 'pg': {'name-pattern': '^{namespace}.*-k8spg-.*', 'excludes': '.helm., -admin-secret', 'agent': {'name': 'ops-pg-agent', 'just-in-time': False, 'timeout': 86400, 'image': 'seanahnsf/kaqing'}, 'default-db': 'postgres', 'default-schema': 'postgres', 'secret': {'endpoint-key': 'postgres-db-endpoint', 'port-key': 'postgres-db-port', 'username-key': 'postgres-admin-username', 'password-key': 'postgres-admin-password'}}, 'pod': {'name': 'ops', 'image': 'seanahnsf/kaqing-cloud', 'sa': {'name': 'ops', 'proto': 'c3', 'additional-cluster-roles': 'c3aiops-k8ssandra-operator'}, 'label-selector': 'run=ops'}, 'preview': {'rows': 10}, 'processes': {'columns': 'pod,cpu,mem', 'header': 'POD_NAME,CPU,MEM/LIMIT'}, 'reaper': {'service-name': 'reaper-service', 'port-forward': {'timeout': 86400, 'local-port': 9001}, 'abort-runs-batch': 10, 'show-runs-batch': 100, 'pod': {'cluster-regex': '(.*?-.*?-.*?-.*?)-.*', 'label-selector': 'k8ssandra.io/reaper={cluster}-reaper'}, 'secret': {'cluster-regex': '(.*?-.*?)-.*', 'name': '{cluster}-reaper-ui', 'password-item': 'password'}}, 'repair': {'log-path': '/home/cassrepair/logs/', 'image': 'ci-registry.c3iot.io/cloudops/cassrepair:2.0.14', 'secret': 'ciregistryc3iotio', 'env': {'interval': 24, 'timeout': 60, 'pr': False, 'runs': 1}}, 'repl': {'start-drive': 'a', 'a': {'auto-enter': 'c3/c3'}, 'c': {'auto-enter': 'cluster'}, 'history': {'push-cat-remote-log-file': True}, 'background-process': {'auto-nohup': True}}, 'status': {'columns': 'status,address,load,tokens,owns,host_id,gossip,compactions', 'header': '--,Address,Load,Tokens,Owns,Host ID,GOSSIP,COMPACTIONS'}, 'storage': {'columns': 'pod,volume_root,volume_cassandra,snapshots,data,compactions', 'header': 'POD_NAME,VOLUME /,VOLUME CASS,SNAPSHOTS,DATA,COMPACTIONS'}, 'watch': {'auto': 'rollout', 'timeout': 3600, 'interval': 10}, 'debug': False, 'debugs': {'timings': False, 'exit-on-error': False, 'show-parallelism': False}}
adam/pod_exec_result.py CHANGED
@@ -15,12 +15,14 @@ class PodExecResult:
15
15
  # ]
16
16
  # }
17
17
  # }
18
- def __init__(self, stdout: str, stderr: str, command: str = None, error_output: str = None):
18
+ def __init__(self, stdout: str, stderr: str, command: str = None, error_output: str = None, pod: str = None, log_file: str = None):
19
19
  self.stdout: str = stdout
20
20
  self.stderr: str = stderr
21
21
  self.command: str = command
22
22
  if error_output:
23
23
  self.error = yaml.safe_load(error_output)
24
+ self.pod = pod
25
+ self.log_file = log_file
24
26
 
25
27
  def exit_code(self) -> int:
26
28
  code = 0
@@ -30,4 +32,7 @@ class PodExecResult:
30
32
  except:
31
33
  pass
32
34
 
33
- return code
35
+ return code
36
+
37
+ def __audit_extra__(self):
38
+ return self.log_file if self.log_file else None
adam/repl.py CHANGED
@@ -3,6 +3,7 @@ import re
3
3
  import time
4
4
  import traceback
5
5
  import click
6
+ import concurrent
6
7
  from prompt_toolkit.completion import NestedCompleter
7
8
  from prompt_toolkit.key_binding import KeyBindings
8
9
 
@@ -10,15 +11,15 @@ from adam.cli_group import cli
10
11
  from adam.commands.command import Command
11
12
  from adam.commands.command_helpers import ClusterCommandHelper
12
13
  from adam.commands.help import Help
13
- from adam.commands.postgres.postgres_session import PostgresSession
14
+ from adam.commands.postgres.postgres_context import PostgresContext
14
15
  from adam.config import Config
15
- from adam.k8s_utils.kube_context import KubeContext
16
- from adam.k8s_utils.statefulsets import StatefulSets
16
+ from adam.utils_audits import Audits
17
+ from adam.utils_k8s.kube_context import KubeContext
18
+ from adam.utils_k8s.statefulsets import StatefulSets
17
19
  from adam.log import Log
18
20
  from adam.repl_commands import ReplCommands
19
21
  from adam.repl_session import ReplSession
20
22
  from adam.repl_state import ReplState
21
- from adam.sql.sql_completer import SqlCompleter
22
23
  from adam.utils import deep_merge_dicts, deep_sort_dict, lines_to_tabular, log2
23
24
  from adam.apps import Apps
24
25
  from . import __version__
@@ -37,7 +38,7 @@ def enter_repl(state: ReplState):
37
38
  msg = ''
38
39
  if state.device == ReplState.P:
39
40
  msg = f'{ReplState.P}:'
40
- pg = PostgresSession(state.namespace, state.pg_path) if state.pg_path else None
41
+ pg: PostgresContext = PostgresContext.apply(state.namespace, state.pg_path) if state.pg_path else None
41
42
  if pg and pg.db:
42
43
  msg += pg.db
43
44
  elif pg and pg.host:
@@ -48,6 +49,8 @@ def enter_repl(state: ReplState):
48
49
  msg += state.app_env
49
50
  if state.app_app:
50
51
  msg += f'/{state.app_app}'
52
+ elif state.device == ReplState.L:
53
+ msg = f'{ReplState.L}:'
51
54
  else:
52
55
  msg = f'{ReplState.C}:'
53
56
  if state.pod:
@@ -62,26 +65,36 @@ def enter_repl(state: ReplState):
62
65
  return f"{msg}$ " if state.bash_session else f"{msg}> "
63
66
 
64
67
  Log.log2(f'kaqing {__version__}')
65
- ss = StatefulSets.list_sts_name_and_ns()
66
68
 
67
69
  if state.device == ReplState.C:
68
- if not ss:
69
- raise Exception("no Cassandra clusters found")
70
- elif len(ss) == 1 and Config().get('repl.auto-enter-only-cluster', True):
71
- cluster = ss[0]
72
- state.sts = cluster[0]
73
- state.namespace = cluster[1]
74
- Config().wait_log(f'Moving to the only Cassandra cluster: {state.sts}@{state.namespace}...')
75
- elif state.device == ReplState.A:
76
- if app := Config().get('repl.auto-enter-app', 'c3/c3'):
77
- if app != 'no':
78
- ea = app.split('/')
79
- state.app_env = ea[0]
80
- if len(ea) > 1:
81
- state.app_app = ea[1]
82
- Config().wait_log(f'Moving to {state.app_env}/{state.app_app}...')
70
+ auto_enter = Config().get('repl.c.auto-enter', 'cluster')
71
+ if auto_enter and auto_enter in ['cluster', 'first-pod']:
72
+ ss = StatefulSets.list_sts_name_and_ns()
73
+ if not ss:
74
+ log2("No Cassandra clusters found.")
75
+ elif not state.sts and len(ss) == 1:
76
+ cluster = ss[0]
77
+ state.sts = cluster[0]
78
+ state.namespace = cluster[1]
79
+ if auto_enter == 'first-pod':
80
+ state.pod = f'{state.sts}-0'
81
+ if KubeContext().in_cluster_namespace:
82
+ Config().wait_log(f'Moving to the only Cassandra cluster: {state.sts}...')
83
83
  else:
84
- Config().wait_log(f'Moving to {state.app_env}...')
84
+ Config().wait_log(f'Moving to the only Cassandra cluster: {state.sts}@{state.namespace}...')
85
+ elif state.device == ReplState.A:
86
+ if not state.app_env:
87
+ if app := Config().get('repl.a.auto-enter-app', 'c3/c3'):
88
+ if app != 'no':
89
+ ea = app.split('/')
90
+ state.app_env = ea[0]
91
+ if len(ea) > 1:
92
+ state.app_app = ea[1]
93
+ Config().wait_log(f'Moving to {state.app_env}/{state.app_app}...')
94
+ else:
95
+ Config().wait_log(f'Moving to {state.app_env}...')
96
+ elif state.device == ReplState.P:
97
+ Config().wait_log('Inspecting postgres database instances...')
85
98
 
86
99
  kb = KeyBindings()
87
100
 
@@ -89,73 +102,112 @@ def enter_repl(state: ReplState):
89
102
  def _(event):
90
103
  event.app.current_buffer.text = ''
91
104
 
92
- # use sorted command list only for auto-completion
93
- sorted_cmds = sorted(cmd_list, key=lambda cmd: cmd.command())
94
- while True:
95
- try:
96
- completer = NestedCompleter.from_nested_dict({})
97
- if not state.bash_session:
98
- completions = {}
99
- # app commands are available only on a: drive
100
- if state.device == ReplState.A and state.app_app:
101
- completions = Apps(path='apps.yaml').commands()
102
-
103
- for cmd in sorted_cmds:
104
- s1 = time.time()
105
- try:
106
- completions = deep_sort_dict(deep_merge_dicts(completions, cmd.completion(state)))
107
- finally:
108
- if Config().get('debugs.timings', False):
109
- log2(f'Timing auto-completion-calc {cmd.command()}: {time.time() - s1:.2f}')
110
-
111
- # print(json.dumps(completions, indent=4))
112
- completer = SqlCompleter.from_nested_dict(completions)
113
-
114
- cmd = session.prompt(prompt_msg(), completer=completer, key_bindings=kb)
115
- s0 = time.time()
116
-
117
- if state.bash_session:
118
- if cmd.strip(' ') == 'exit':
119
- state.exit_bash()
120
- continue
121
-
122
- cmd = f'bash {cmd}'
123
-
124
- if cmd and cmd.strip(' ') and not cmds.run(cmd, state):
125
- c_sql_tried = False
126
- if state.device == ReplState.P:
127
- pg = PostgresSession(state.namespace, state.pg_path)
128
- if pg.db:
129
- c_sql_tried = True
130
- cmd = f'pg {cmd}'
131
- cmds.run(cmd, state)
132
- elif state.device == ReplState.A:
133
- if state.app_app:
134
- c_sql_tried = True
135
- cmd = f'app {cmd}'
136
- cmds.run(cmd, state)
137
- elif state.sts:
138
- c_sql_tried = True
139
- cmd = f'cql {cmd}'
140
- cmds.run(cmd, state)
141
-
142
- if not c_sql_tried:
143
- log2(f'* Invalid command: {cmd}')
144
- log2()
145
- lines = [c.help(state) for c in cmd_list if c.help(state)]
146
- log2(lines_to_tabular(lines, separator='\t'))
147
- except EOFError: # Handle Ctrl+D (EOF) for graceful exit
148
- break
149
- except Exception as e:
150
- if Config().get('debugs.exit-on-error', False):
151
- raise e
152
- else:
153
- log2(e)
154
- Config().debug(traceback.format_exc())
155
- finally:
156
- Config().clear_wait_log_flag()
157
- if Config().get('debugs.timings', False) and 'cmd' in locals() and 's0' in locals():
158
- log2(f'Timing command {cmd}: {time.time() - s0:.2f}')
105
+ with concurrent.futures.ThreadPoolExecutor(max_workers=Config().get('audit.workers', 3)) as executor:
106
+ # warm up AWS lambda - this log line may timeout and get lost, which is fine
107
+ executor.submit(Audits.log, 'entering kaqing repl', state.namespace, 'z', 0.0)
108
+
109
+ s0 = time.time()
110
+
111
+ # use sorted command list only for auto-completion
112
+ sorted_cmds = sorted(cmd_list, key=lambda cmd: cmd.command())
113
+ while True:
114
+ result = None
115
+ try:
116
+ completer = NestedCompleter.from_nested_dict({})
117
+ if not state.bash_session:
118
+ completions = {}
119
+ # app commands are available only on a: drive
120
+ if state.device == ReplState.A and state.app_app:
121
+ completions = Apps(path='apps.yaml').commands()
122
+
123
+ for cmd in sorted_cmds:
124
+ s1 = time.time()
125
+ try:
126
+ completions = deep_sort_dict(deep_merge_dicts(completions, cmd.completion(state)))
127
+ finally:
128
+ if Config().get('debugs.timings', False):
129
+ log2(f'Timing auto-completion-calc {cmd.command()}: {time.time() - s1:.2f}')
130
+
131
+ # print(json.dumps(completions, indent=4))
132
+ completer = NestedCompleter.from_nested_dict(completions)
133
+
134
+ cmd = session.prompt(prompt_msg(), completer=completer, key_bindings=kb)
135
+ s0 = time.time()
136
+
137
+ if state.bash_session:
138
+ if cmd.strip(' ') == 'exit':
139
+ state.exit_bash()
140
+ continue
141
+
142
+ cmd = f'bash {cmd}'
143
+
144
+ if cmd and cmd.strip(' ') and not (result := cmds.run(cmd, state)):
145
+ # not served by any command in the chain; try SQL query or C3 action
146
+ result = try_device_default_action(state, cmds, cmd_list, cmd)
147
+ except EOFError: # Handle Ctrl+D (EOF) for graceful exit
148
+ break
149
+ except Exception as e:
150
+ if Config().get('debugs.exit-on-error', False):
151
+ raise e
152
+ else:
153
+ log2(e)
154
+ Config().debug(traceback.format_exc())
155
+ finally:
156
+ Config().clear_wait_log_flag()
157
+ if Config().get('debugs.timings', False) and 'cmd' in locals() and 's0' in locals():
158
+ log2(f'Timing command {cmd}: {time.time() - s0:.2f}')
159
+
160
+ # offload audit logging
161
+ if cmd and (state.device != ReplState.L or Config().get('audit.log-audit-queries', False)):
162
+ executor.submit(Audits.log, cmd, state.namespace, state.device, time.time() - s0, get_audit_extra(result))
163
+
164
+ def try_device_default_action(state: ReplState, cmds: Command, cmd_list: list[Command], cmd: str):
165
+ result = None
166
+
167
+ c_sql_tried = False
168
+ if state.device == ReplState.P:
169
+ pg: PostgresContext = PostgresContext.apply(state.namespace, state.pg_path)
170
+ if pg.db:
171
+ c_sql_tried = True
172
+ cmd = f'pg {cmd}'
173
+ result = cmds.run(cmd, state)
174
+ elif state.device == ReplState.A:
175
+ if state.app_app:
176
+ c_sql_tried = True
177
+ cmd = f'app {cmd}'
178
+ result = cmds.run(cmd, state)
179
+ elif state.device == ReplState.L:
180
+ c_sql_tried = True
181
+ cmd = f'audit {cmd}'
182
+ result = cmds.run(cmd, state)
183
+ elif state.sts:
184
+ c_sql_tried = True
185
+ cmd = f'cql {cmd}'
186
+ result = cmds.run(cmd, state)
187
+
188
+ if not c_sql_tried:
189
+ log2(f'* Invalid command: {cmd}')
190
+ log2()
191
+ lines = [c.help(state) for c in cmd_list if c.help(state)]
192
+ log2(lines_to_tabular(lines, separator='\t'))
193
+
194
+ return result
195
+
196
+ def get_audit_extra(result: any):
197
+ if not result:
198
+ return None
199
+
200
+ if type(result) is list:
201
+ extras = set()
202
+
203
+ for r in result:
204
+ if hasattr(r, '__audit_extra__') and (x := r.__audit_extra__()):
205
+ extras.add(x)
206
+
207
+ return ','.join(list(extras))
208
+
209
+ if hasattr(result, '__audit_extra__') and (x := result.__audit_extra__()):
210
+ return x
159
211
 
160
212
  @cli.command(context_settings=dict(ignore_unknown_options=True, allow_extra_args=True), cls=ClusterCommandHelper, help="Enter interactive shell.")
161
213
  @click.option('--kubeconfig', '-k', required=False, metavar='path', help='path to kubeconfig file')