kaqing 2.0.145__py3-none-any.whl → 2.0.172__py3-none-any.whl

adam/embedded_params.py

@@ -1,2 +1,2 @@
 def config():
-    return {'app': {'console-endpoint': 'https://{host}/{env}/{app}/static/console/index.html', 'container-name': 'c3-server', 'cr': {'cluster-regex': '(.*?-.*?)-.*', 'group': 'ops.c3.ai', 'v': 'v2', 'plural': 'c3cassandras'}, 'label': 'c3__app_id-0', 'login': {'admin-group': '{host}/C3.ClusterAdmin', 'ingress': '{app_id}-k8singr-appleader-001', 'timeout': 5, 'session-check-url': 'https://{host}/{env}/{app}/api/8/C3/userSessionToken', 'cache-creds': True, 'cache-username': True, 'url': 'https://{host}/{env}/{app}', 'another': "You're logged in to {has}. However, for this app, you need to log in to {need}.", 'token-server-url': 'http://localhost:{port}', 'password-max-length': 128}, 'strip': '0'}, 'audit': {'endpoint': 'https://4psvtaxlcb.execute-api.us-west-2.amazonaws.com/prod/', 'workers': 3, 'timeout': 10, 'log-audit-queries': False, 'athena': {'auto-repair': {'elapsed_hours': 12}, 'region': 'us-west-2', 'catalog': 'AwsDataCatalog', 'database': 'audit', 'repair-partition-tables': 'audit', 'output': 's3://s3.ops--audit/ddl/results', 'repair-cluster-tables': 'cluster'}}, 'bash': {'workers': 32}, 'cassandra': {'service-name': 'all-pods-service'}, 'cql': {'workers': 32, 'samples': 3, 'secret': {'cluster-regex': '(.*?-.*?)-.*', 'name': '{cluster}-superuser', 'password-item': 'password'}, 'alter-tables': {'excludes': 'system_auth,system_traces,reaper_db,system_distributed,system_views,system,system_schema,system_virtual_schema', 'gc-grace-periods': '3600,86400,864000,7776000', 'batching': True}}, 'export': {'workers': 8, 'csv_dir': '/c3/cassandra/tmp', 'default-importer': 'sqlite', 'sqlite': {'workers': 8, 'columns': '<row-key>', 'local-db-dir': '/tmp/qing-db', 'column_counts_query': 'select id, count(id) as columns from {table} group by id order by columns desc limit 10'}, 'athena': {'workers': 8, 'columns': '<keys>', 'bucket': 'c3.ops--qing', 'column_counts_query': 'select id, count(id) as columns from {table} group by id order by columns desc limit 10'}, 'csv': {'workers': 8, 'columns': '<row-key>'}}, 'checks': {'compactions-threshold': 250, 'cpu-busy-threshold': 98.0, 'cpu-threshold': 0.0, 'cassandra-data-path': '/c3/cassandra', 'root-disk-threshold': 50, 'cassandra-disk-threshold': 50, 'snapshot-size-cmd': "ls /c3/cassandra/data/data/*/*/snapshots | grep snapshots | sed 's/:$//g' | xargs -I {} du -sk {} | awk '{print $1}' | awk '{s+=$1} END {print s}'", 'snapshot-size-threshold': '40G', 'table-sizes-cmd': "ls -Al /c3/cassandra/data/data/ | awk '{print $9}' | sed 's/\\^r//g' | xargs -I {} du -sk /c3/cassandra/data/data/{}"}, 'get-host-id': {'workers': 32}, 'idps': {'ad': {'email-pattern': '.*@c3.ai', 'uri': 'https://login.microsoftonline.com/53ad779a-93e7-485c-ba20-ac8290d7252b/oauth2/v2.0/authorize?response_type=id_token&response_mode=form_post&client_id=00ff94a8-6b0a-4715-98e0-95490012d818&scope=openid+email+profile&redirect_uri=https%3A%2F%2Fplat.c3ci.cloud%2Fc3%2Fc3%2Foidc%2Flogin&nonce={nonce}&state=EMPTY', 'jwks-uri': 'https://login.microsoftonline.com/common/discovery/keys', 'contact': 'Please contact ted.tran@c3.ai.', 'whitelist-file': '/kaqing/members'}, 'okta': {'default': True, 'email-pattern': '.*@c3iot.com', 'uri': 'https://c3energy.okta.com/oauth2/v1/authorize?response_type=id_token&response_mode=form_post&client_id={client_id}&scope=openid+email+profile+groups&redirect_uri=https%3A%2F%2F{host}%2Fc3%2Fc3%2Foidc%2Flogin&nonce={nonce}&state=EMPTY', 'jwks-uri': 'https://c3energy.okta.com/oauth2/v1/keys'}}, 'issues': {'workers': 32}, 'logs': {'path': '/c3/cassandra/logs/system.log'}, 'log-prefix': '/tmp/qing', 'medusa': {'restore-auto-complete': False}, 'nodetool': {'workers': 32, 'samples': 3, 'commands_in_line': 40}, 'pg': {'name-pattern': '^{namespace}.*-k8spg-.*', 'excludes': '.helm., -admin-secret', 'agent': {'name': 'ops-pg-agent', 'just-in-time': False, 'timeout': 86400, 'image': 'seanahnsf/kaqing'}, 'default-db': 'postgres', 'default-schema': 'postgres', 'secret': {'endpoint-key': 'postgres-db-endpoint', 'port-key': 'postgres-db-port', 'username-key': 'postgres-admin-username', 'password-key': 'postgres-admin-password'}}, 'pod': {'name': 'ops', 'image': 'seanahnsf/kaqing-cloud', 'sa': {'name': 'ops', 'proto': 'c3', 'additional-cluster-roles': 'c3aiops-k8ssandra-operator'}, 'label-selector': 'run=ops'}, 'preview': {'rows': 10}, 'processes': {'columns': 'pod,cpu,mem', 'header': 'POD_NAME,CPU,MEM/LIMIT'}, 'reaper': {'service-name': 'reaper-service', 'port-forward': {'timeout': 86400, 'local-port': 9001}, 'abort-runs-batch': 10, 'show-runs-batch': 100, 'pod': {'cluster-regex': '(.*?-.*?-.*?-.*?)-.*', 'label-selector': 'k8ssandra.io/reaper={cluster}-reaper'}, 'secret': {'cluster-regex': '(.*?-.*?)-.*', 'name': '{cluster}-reaper-ui', 'password-item': 'password'}}, 'repair': {'log-path': '/home/cassrepair/logs/', 'image': 'ci-registry.c3iot.io/cloudops/cassrepair:2.0.14', 'secret': 'ciregistryc3iotio', 'env': {'interval': 24, 'timeout': 60, 'pr': False, 'runs': 1}}, 'repl': {'start-drive': 'c', 'a': {'auto-enter': 'c3/c3/*'}, 'c': {'auto-enter': 'cluster'}, 'x': {'auto-enter': 'latest'}, 'history': {'push-cat-remote-log-file': True}, 'background-process': {'auto-nohup': True}}, 'status': {'columns': 'status,address,load,tokens,owns,host_id,gossip,compactions', 'header': '--,Address,Load,Tokens,Owns,Host ID,GOSSIP,COMPACTIONS'}, 'storage': {'columns': 'pod,volume_root,volume_cassandra,snapshots,data,compactions', 'header': 'POD_NAME,VOLUME /,VOLUME CASS,SNAPSHOTS,DATA,COMPACTIONS'}, 'watch': {'auto': 'rollout', 'timeout': 3600, 'interval': 10}, 'debug': False, 'debugs': {'timings': False, 'exit-on-error': False, 'show-parallelism': False}}
+    return {'app': {'console-endpoint': 'https://{host}/{env}/{app}/static/console/index.html', 'container-name': 'c3-server', 'cr': {'cluster-regex': '(.*?-.*?)-.*', 'group': 'ops.c3.ai', 'v': 'v2', 'plural': 'c3cassandras'}, 'label': 'c3__app_id-0', 'login': {'admin-group': '{host}/C3.ClusterAdmin', 'ingress': '{app_id}-k8singr-appleader-001', 'timeout': 5, 'session-check-url': 'https://{host}/{env}/{app}/api/8/C3/userSessionToken', 'cache-creds': True, 'cache-username': True, 'url': 'https://{host}/{env}/{app}', 'another': "You're logged in to {has}. However, for this app, you need to log in to {need}.", 'token-server-url': 'http://localhost:{port}', 'password-max-length': 128}, 'strip': '0'}, 'audit': {'endpoint': 'https://4psvtaxlcb.execute-api.us-west-2.amazonaws.com/prod/', 'workers': 3, 'timeout': 10, 'log-audit-queries': False, 'athena': {'auto-repair': {'elapsed_hours': 12}, 'region': 'us-west-2', 'catalog': 'AwsDataCatalog', 'database': 'audit', 'repair-partition-tables': 'audit', 'output': 's3://s3.ops--audit/ddl/results', 'repair-cluster-tables': 'cluster'}}, 'bash': {'workers': 32}, 'cassandra': {'service-name': 'all-pods-service'}, 'cql': {'workers': 32, 'samples': 3, 'secret': {'cluster-regex': '(.*?-.*?)-.*', 'name': '{cluster}-superuser', 'password-item': 'password'}, 'alter-tables': {'excludes': 'system_auth,system_traces,reaper_db,system_distributed,system_views,system,system_schema,system_virtual_schema', 'gc-grace-periods': '3600,86400,864000,7776000', 'batching': True}}, 'export': {'workers': 8, 'csv_dir': '/c3/cassandra/tmp', 'column_counts_query': 'select id, count(id) as columns from {table} group by id order by columns desc limit 10', 'default-importer': 'sqlite', 'sqlite': {'workers': 8, 'columns': '<row-key>', 'local-db-dir': '/tmp/qing-db'}, 'athena': {'workers': 8, 'columns': '<keys>', 'bucket': 'c3.ops--qing'}, 'csv': {'workers': 8, 'columns': '<row-key>'}, 'log-prefix': '/tmp/qing'}, 'checks': {'compactions-threshold': 250, 'cpu-busy-threshold': 98.0, 'cpu-threshold': 0.0, 'cassandra-data-path': '/c3/cassandra', 'root-disk-threshold': 50, 'cassandra-disk-threshold': 50, 'snapshot-size-cmd': "ls /c3/cassandra/data/data/*/*/snapshots | grep snapshots | sed 's/:$//g' | xargs -I {} du -sk {} | awk '{print $1}' | awk '{s+=$1} END {print s}'", 'snapshot-size-threshold': '40G', 'table-sizes-cmd': "ls -Al /c3/cassandra/data/data/ | awk '{print $9}' | sed 's/\\^r//g' | xargs -I {} du -sk /c3/cassandra/data/data/{}"}, 'get-host-id': {'workers': 32}, 'idps': {'ad': {'email-pattern': '.*@c3.ai', 'uri': 'https://login.microsoftonline.com/53ad779a-93e7-485c-ba20-ac8290d7252b/oauth2/v2.0/authorize?response_type=id_token&response_mode=form_post&client_id=00ff94a8-6b0a-4715-98e0-95490012d818&scope=openid+email+profile&redirect_uri=https%3A%2F%2Fplat.c3ci.cloud%2Fc3%2Fc3%2Foidc%2Flogin&nonce={nonce}&state=EMPTY', 'jwks-uri': 'https://login.microsoftonline.com/common/discovery/keys', 'contact': 'Please contact ted.tran@c3.ai.', 'whitelist-file': '/kaqing/members'}, 'okta': {'default': True, 'email-pattern': '.*@c3iot.com', 'uri': 'https://c3energy.okta.com/oauth2/v1/authorize?response_type=id_token&response_mode=form_post&client_id={client_id}&scope=openid+email+profile+groups&redirect_uri=https%3A%2F%2F{host}%2Fc3%2Fc3%2Foidc%2Flogin&nonce={nonce}&state=EMPTY', 'jwks-uri': 'https://c3energy.okta.com/oauth2/v1/keys'}}, 'issues': {'workers': 32}, 'logs': {'path': '/c3/cassandra/logs/system.log'}, 'log-prefix': '/tmp/qing', 'medusa': {'restore-auto-complete': False}, 'nodetool': {'workers': 32, 'samples': 3, 'commands_in_line': 40}, 'pg': {'name-pattern': '^{namespace}.*-k8spg-.*', 'excludes': '.helm., -admin-secret', 'agent': {'name': 'ops-pg-agent', 'just-in-time': False, 'timeout': 86400, 'image': 'seanahnsf/kaqing'}, 'default-db': 'postgres', 'default-schema': 'postgres', 'secret': {'endpoint-key': 'postgres-db-endpoint', 'port-key': 'postgres-db-port', 'username-key': 'postgres-admin-username', 'password-key': 'postgres-admin-password'}}, 'pod': {'name': 'ops', 'image': 'seanahnsf/kaqing-cloud', 'sa': {'name': 'ops', 'proto': 'c3', 'additional-cluster-roles': 'c3aiops-k8ssandra-operator'}, 'label-selector': 'run=ops'}, 'preview': {'rows': 10}, 'processes': {'columns': 'pod,cpu-metrics,mem', 'header': 'POD_NAME,M_CPU(USAGE/LIMIT),MEM/LIMIT'}, 'processes-qing': {'columns': 'pod,cpu,mem', 'header': 'POD_NAME,Q_CPU/TOTAL,MEM/LIMIT'}, 'reaper': {'service-name': 'reaper-service', 'port-forward': {'timeout': 86400, 'local-port': 9001}, 'abort-runs-batch': 10, 'show-runs-batch': 100, 'pod': {'cluster-regex': '(.*?-.*?-.*?-.*?)-.*', 'label-selector': 'k8ssandra.io/reaper={cluster}-reaper'}, 'secret': {'cluster-regex': '(.*?-.*?)-.*', 'name': '{cluster}-reaper-ui', 'password-item': 'password'}}, 'repair': {'log-path': '/home/cassrepair/logs/', 'image': 'ci-registry.c3iot.io/cloudops/cassrepair:2.0.14', 'secret': 'ciregistryc3iotio', 'env': {'interval': 24, 'timeout': 60, 'pr': False, 'runs': 1}}, 'repl': {'start-drive': 'c', 'a': {'auto-enter': 'c3/c3/*'}, 'c': {'auto-enter': 'cluster'}, 'x': {'auto-enter': 'latest'}, 'history': {'push-cat-log-file': True, 'push-cat-remote-log-file': True}, 'background-process': {'auto-nohup': True}}, 'status': {'columns': 'status,address,load,tokens,owns,host_id,gossip,compactions', 'header': '--,Address,Load,Tokens,Owns,Host ID,GOSSIP,COMPACTIONS'}, 'storage': {'columns': 'pod,volume_root,volume_cassandra,snapshots,data,compactions', 'header': 'POD_NAME,VOLUME /,VOLUME CASS,SNAPSHOTS,DATA,COMPACTIONS'}, 'watch': {'auto': 'rollout', 'timeout': 3600, 'interval': 10}, 'debug': False, 'debugs': {'timings': False, 'exit-on-error': False, 'show-parallelism': False}}

adam/log.py

@@ -4,6 +4,8 @@ import os
 import sys
 import click
 
+from adam.utils import log_exc
+
 class Log:
     DEBUG = False
 
@@ -28,7 +30,7 @@ class Log:
             print(file=sys.stderr)
 
     def log_to_file(config: dict[any, any]):
-        try:
+        with log_exc():
             base = f"/tmp/logs"
             os.makedirs(base, exist_ok=True)
 
@@ -42,6 +44,4 @@ class Log:
                     except:
                         f.write(config)
                 else:
-                        f.write(config)
-        except:
-            pass
+                        f.write(config)

adam/pod_exec_result.py

@@ -1,5 +1,7 @@
 import yaml
 
+from adam.utils import log_exc
+
 class PodExecResult:
     # {
     #   'metadata': {},
@@ -27,10 +29,8 @@ class PodExecResult:
     def exit_code(self) -> int:
         code = 0
 
-        try:
+        with log_exc(False):
             code = self.error['details']['causes'][0]['message']
-        except:
-            pass
 
         return code
 

adam/repl.py

@@ -1,26 +1,22 @@
 import os
 import time
-import traceback
 from typing import cast
 import click
-import concurrent
 from prompt_toolkit.key_binding import KeyBindings
 
 from adam.cli_group import cli
-from adam.commands.command import Command
+from adam.commands.command import Command, InvalidState
 from adam.commands.command_helpers import ClusterCommandHelper
 from adam.commands.devices.devices import Devices
 from adam.commands.help import Help
 from adam.config import Config
 from adam.utils_audits import Audits
-from adam.utils_k8s.app_pods import AppPods
 from adam.utils_k8s.kube_context import KubeContext
-from adam.utils_k8s.statefulsets import StatefulSets
 from adam.log import Log
 from adam.repl_commands import ReplCommands
 from adam.repl_session import ReplSession
 from adam.repl_state import ReplState
-from adam.utils import deep_merge_dicts, deep_sort_dict, lines_to_tabular, log2
+from adam.utils import clear_wait_log_flag, debug, debug_trace, deep_merge_dicts, deep_sort_dict, lines_to_tabular, log2, log_exc, log_timing
 from adam.apps import Apps
 from adam.utils_repl.repl_completer import ReplCompleter
 from . import __version__
@@ -50,34 +46,32 @@ def enter_repl(state: ReplState):
     def _(event):
         event.app.current_buffer.text = ''
 
-    with concurrent.futures.ThreadPoolExecutor(max_workers=Config().get('audit.workers', 3)) as executor:
+    with Audits.offload() as exec:
         # warm up AWS lambda - this log line may timeout and get lost, which is fine
-        executor.submit(Audits.log, 'entering kaqing repl', state.namespace, 'z', 0.0)
+        exec.submit(Audits.log, 'entering kaqing repl', state.namespace, 'z', 0.0)
 
         s0 = time.time()
 
         # use sorted command list only for auto-completion
         sorted_cmds = sorted(cmd_list, key=lambda cmd: cmd.command())
         while True:
+            cmd: str = None
             result = None
             try:
                 completer = ReplCompleter.from_nested_dict({})
                 if not state.bash_session:
-                    completions = {}
-                    # app commands are available only on a: drive
-                    if state.device == ReplState.A and state.app_app:
-                        completions = Apps(path='apps.yaml').commands()
-
-                    for cmd in sorted_cmds:
-                        s1 = time.time()
-                        try:
-                            completions = deep_sort_dict(deep_merge_dicts(completions, cmd.completion(state)))
-                        finally:
-                            if Config().get('debugs.timings', False):
-                                log2(f'Timing auto-completion-calc {cmd.command()}: {time.time() - s1:.2f}')
-
-                    # print(json.dumps(completions, indent=4))
-                    completer = ReplCompleter.from_nested_dict(completions)
+                    with log_timing('completion-calcs'):
+                        completions = {}
+                        # app commands are available only on a: drive
+                        if state.device == ReplState.A and state.app_app:
+                            completions = log_timing('actions', lambda: Apps(path='apps.yaml').commands())
+
+                        for c in sorted_cmds:
+                            with log_exc(f'* {c.command()} command returned None completions.'):
+                                completions = log_timing(c.command(), lambda: deep_sort_dict(deep_merge_dicts(completions, c.completion(state))))
+
+                        # print(json.dumps(completions, indent=4))
+                        completer = ReplCompleter.from_nested_dict(completions)
 
                 cmd = session.prompt(prompt_msg(), completer=completer, key_bindings=kb)
                 s0 = time.time()
@@ -112,8 +106,11 @@ def enter_repl(state: ReplState):
                     return (state, cmd)
 
                 target, cmd = targetted(state, cmd)
-                if cmd and cmd.strip(' ') and not (result := cmds.run(cmd, target)):
-                    result = try_device_default_action(target, cmds, cmd_list, cmd)
+                try:
+                    if cmd and cmd.strip(' ') and not (result := cmds.run(cmd, target)):
+                        result = try_device_default_action(target, cmds, cmd_list, cmd)
+                except InvalidState:
+                    pass
 
                 if result and type(result) is ReplState and (s := cast(ReplState, result).export_session) != state.export_session:
                     state.export_session = s
@@ -125,23 +122,23 @@ def enter_repl(state: ReplState):
                     raise e
                 else:
                     log2(e)
-                    Config().debug(traceback.format_exc())
+                    debug_trace()
             finally:
                 if not state.bash_session:
                     state.pop()
 
-                Config().clear_wait_log_flag()
-                if Config().get('debugs.timings', False) and 'cmd' in locals() and 's0' in locals():
-                    log2(f'Timing command {cmd}: {time.time() - s0:.2f}')
+                clear_wait_log_flag()
+                if cmd:
+                    log_timing(f'command {cmd}', s0=s0)
 
                 # offload audit logging
                 if cmd and (state.device != ReplState.L or Config().get('audit.log-audit-queries', False)):
-                    executor.submit(Audits.log, cmd, state.namespace, state.device, time.time() - s0, get_audit_extra(result))
+                    exec.submit(Audits.log, cmd, state.namespace, state.device, time.time() - s0, get_audit_extra(result))
 
 def try_device_default_action(state: ReplState, cmds: Command, cmd_list: list[Command], cmd: str):
-    default_action_tried, result = Devices.device(state).try_fallback_action(cmds, state, cmd)
+    action_taken, result = Devices.device(state).try_fallback_action(cmds, state, cmd)
 
-    if not default_action_tried:
+    if not action_taken:
         log2(f'* Invalid command: {cmd}')
         log2()
         lines = [c.help(state) for c in cmd_list if c.help(state)]

adam/repl_commands.py

@@ -1,6 +1,9 @@
 from adam.commands.alter_tables import AlterTables
-from adam.commands.app import App
-from adam.commands.app_ping import AppPing
+from adam.commands.app.app import App
+from adam.commands.app.app_ping import AppPing
+from adam.commands.app.show_app_actions import ShowAppActions
+from adam.commands.app.show_app_id import ShowAppId
+from adam.commands.app.show_app_queues import ShowAppQueues
 from adam.commands.audit.audit import Audit
 from adam.commands.cat import Cat
 from adam.commands.code import Code
@@ -22,8 +25,8 @@ from adam.commands.devices.device_postgres import DevicePostgres
 from adam.commands.export.drop_export_database import DropExportDatabase
 from adam.commands.export.export import ExportTables
 from adam.commands.export.import_session import ImportSession
-from adam.commands.export.clean_up_export_session import CleanUpExportSession
 from adam.commands.export.clean_up_export_sessions import CleanUpExportSessions
+from adam.commands.export.clean_up_all_export_sessions import CleanUpAllExportSessions
 from adam.commands.export.drop_export_databases import DropExportDatabases
 from adam.commands.export.export_select import ExportSelect
 from adam.commands.export.export_use import ExportUse
@@ -34,7 +37,6 @@ from adam.commands.export.show_export_session import ShowExportSession
 from adam.commands.export.show_export_sessions import ShowExportSessions
 from adam.commands.kubectl import Kubectl
 from adam.commands.shell import Shell
-from adam.commands.show.show_app_queues import ShowAppQueues
 from adam.commands.cp import ClipboardCopy
 from adam.commands.bash.bash import Bash
 from adam.commands.cd import Cd
@@ -57,8 +59,6 @@ from adam.commands.restart import Restart
 from adam.commands.rollout import RollOut
 from adam.commands.param_set import SetParam
 from adam.commands.show.show import Show
-from adam.commands.show.show_app_actions import ShowAppActions
-from adam.commands.show.show_app_id import ShowAppId
 from adam.commands.show.show_cassandra_status import ShowCassandraStatus
 from adam.commands.show.show_cassandra_version import ShowCassandraVersion
 from adam.commands.show.show_commands import ShowKubectlCommands
@@ -66,7 +66,7 @@ from adam.commands.show.show_host import ShowHost
 from adam.commands.show.show_login import ShowLogin
 from adam.commands.show.show_params import ShowParams
 from adam.commands.show.show_processes import ShowProcesses
-from adam.commands.show.show_repairs import ShowRepairs
+from adam.commands.show.show_cassandra_repairs import ShowCassandraRepairs
 from adam.commands.show.show_storage import ShowStorage
 from adam.commands.show.show_adam import ShowAdam
 from adam.commands.watch import Watch
@@ -97,13 +97,13 @@ class ReplCommands:
                 GetParam(), SetParam(), ShowParams(), ShowKubectlCommands(), ShowLogin(), ShowAdam(), ShowHost()]
 
     def cassandra_ops() -> list[Command]:
-        return [Cqlsh(), ShowCassandraStatus(), ShowCassandraVersion(), ShowRepairs(), ShowStorage(), ShowProcesses(),
+        return [Cqlsh(), ShowCassandraStatus(), ShowCassandraVersion(), ShowCassandraRepairs(), ShowStorage(), ShowProcesses(),
                 Check(), Issues(), NodeTool(), Report(), AlterTables(), Bash(),
                 ExportTables(), ExportSelect(), ExportUse(),  ShowExportDatabases(), ShowColumnCounts(),
                 DropExportDatabase(), DropExportDatabases(),
                 ShowExportSessions(), ShowExportSession(),
-                CleanUpExportSession(), CleanUpExportSessions(), ImportSession()] + \
-                Medusa.cmd_list() + [Restart(), RollOut(), Watch()] + Reaper.cmd_list() + Repair.cmd_list()
+                CleanUpExportSessions(), CleanUpAllExportSessions(), ImportSession()] + \
+                Medusa().cmd_list() + [Restart(), RollOut(), Watch()] + Reaper().cmd_list() + Repair().cmd_list()
 
     def postgres_ops() -> list[Command]:
         return [Postgres(), DeployPgAgent(), UndeployPgAgent(), PostgresPg()]
@@ -112,7 +112,7 @@ class ReplCommands:
         return [ShowAppActions(), ShowAppId(), ShowAppQueues(), AppPing(), App()]
 
     def audit_ops() -> list[Command]:
-        return [Audit()] + Audit.cmd_list()
+        return [Audit()] + Audit().cmd_list()
 
     def export_ops() -> list[Command]:
         return [ExportSelectX(), DropExportDatabase(), DropExportDatabases(), ShowColumnCounts()]

adam/repl_state.py

@@ -20,31 +20,8 @@ class BashSession:
     def pwd(self, state: 'ReplState'):
         command = f'cat /tmp/.qing-{self.session_id}'
 
-        if state.device == ReplState.A and state.app_app:
-            if state.app_pod:
-                rs = [AppPods.exec(state.app_pod, state.namespace, command, show_out=False)]
-            else:
-                pods = AppPods.pod_names(state.namespace, state.app_env, state.app_pod)
-                rs = AppClusters.exec(pods, state.namespace, command, show_out=False)
-        elif state.pod:
-            rs = [CassandraNodes.exec(state.pod, state.namespace, command, show_out=False)]
-        elif state.sts:
-            rs = CassandraClusters.exec(state.sts, state.namespace, command, action='bash', show_out=False)
-
-        dir = None
-        for r in rs:
-            if r.exit_code(): # if fails to read the session file, ignore
-                continue
-
-            dir0 = r.stdout.strip(' \r\n')
-            if dir:
-                if dir != dir0:
-                    log2('Inconsitent working dir found across multiple pods.')
-                    return None
-            else:
-                dir = dir0
-
-        return dir
+        with device(state) as pods:
+            return pods.exec(command, action='bash', show_out=False)
 
 class RequiredState(Enum):
     CLUSTER = 'cluster'
@@ -429,4 +406,53 @@ class ReplState:
             # self.export_session = o.export_session
             self.namespace = o.namespace
 
-            self.original_state = None
+            self.original_state = None
+
+class DevicePodService:
+    def __init__(self, handler: 'DeviceExecHandler'):
+        self.handler = handler
+
+    def exec(self, command: str, action='bash', show_out = True):
+        state = self.handler.state
+
+        rs = None
+        if state.device == ReplState.A and state.app_app:
+            if state.app_pod:
+                rs = [AppPods.exec(state.app_pod, state.namespace, command, show_out=show_out)]
+            else:
+                pods = AppPods.pod_names(state.namespace, state.app_env, state.app_app)
+                rs = AppClusters.exec(pods, state.namespace, command, show_out=show_out)
+        elif state.pod:
+            rs = [CassandraNodes.exec(state.pod, state.namespace, command, show_out=show_out)]
+        elif state.sts:
+            rs = CassandraClusters.exec(state.sts, state.namespace, command, action=action, show_out=show_out)
+        # assume that pg-agent or ops pod is single pod
+
+        dir = None
+        if rs:
+            for r in rs:
+                if r.exit_code(): # if fails to read the session file, ignore
+                    continue
+
+                dir0 = r.stdout.strip(' \r\n')
+                if dir:
+                    if dir != dir0:
+                        log2('Inconsitent working dir found across multiple pods.')
+                        return None
+                else:
+                    dir = dir0
+
+        return dir
+
+class DeviceExecHandler:
+    def __init__(self, state: ReplState):
+        self.state = state
+
+    def __enter__(self):
+        return DevicePodService(self)
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        return False
+
+def device(state: ReplState):
+    return DeviceExecHandler(state)

adam/sql/sql_completer.py

@@ -2,7 +2,7 @@ from enum import Enum
 from typing import Callable
 
 import sqlparse
-from sqlparse.sql import Statement, Token
+from sqlparse.sql import Token
 
 from adam.sql.term_completer import TermCompleter
 from adam.utils_repl.automata_completer import AutomataCompleter
@@ -26,11 +26,9 @@ class SqlCompleter(AutomataCompleter[Token]):
         tokens = []
 
         stmts = sqlparse.parse(text)
-        if not stmts:
-            tokens = []
-        else:
-            statement: Statement = stmts[0]
-            tokens = statement.tokens
+        if stmts:
+            for stmt in stmts:
+                tokens.extend(stmt.tokens)
 
         return tokens
 

adam/sql/sql_state_machine.py

@@ -2,6 +2,7 @@ from typing import Callable
 from sqlparse.sql import Token
 from sqlparse import tokens as TOKEN
 
+from adam.utils import log_exc
 from adam.utils_repl.state_machine import StateMachine, State
 
 __all__ = [
@@ -76,8 +77,8 @@ SQL_SPEC = [
     '-                               > ;                > ',
     'select_from_sq_                 > as               > select_from_x_as                        ^ as',
     'select_from_x_comma_            > name|audit       > select_from_x                           ^ tables',
-    'select_from_x_                                                                               ^ as,where,inner join,left outer join,right outer join,full outer join,group by,order by,limit',
-    'select_from_x_as_x_             > ,                > select_from_x_comma_                    ^ where,inner join,left outer join,right outer join,full outer join,group by,order by,limit',
+    'select_from_x_                                                                               ^ as,where,inner join,left outer join,right outer join,full outer join,group by,order by,limit,&',
+    'select_from_x_as_x_             > ,                > select_from_x_comma_                    ^ where,inner join,left outer join,right outer join,full outer join,group by,order by,limit,&',
     '-                               > as               > select_from_x_as',
     '-                               > where            > select_where',
     '-                               > order            > select_order',
@@ -96,6 +97,7 @@ SQL_SPEC = [
     '-                               > full             > select_from_x_full',
     '-                               > full outer join  > select_join',
     '-                               > ;                > ',
+    '-                               > &                > select_from_x$',
     'select_from_x_as_               > name             > select_from_x_as_x                      ^ x,y,z',
     'select_from_x_as_x              > ,                > select_from_x_as_x_comma_',
     '-                               > ;                > ',
@@ -118,34 +120,38 @@ SQL_SPEC = [
     'select_where_a_not_op           > name|single|num  > select_where_sc                         ^ single',
     'select_where_a_op               > name|single|num  > select_where_sc                         ^ single',
     'select_where_sc                 > ;                > ',
-    'select_where_sc_                > and|or           > select_where                            ^ and,or,order by,group by,limit',
+    'select_where_sc_                > and|or           > select_where                            ^ and,or,order by,group by,limit,&',
     '-                               > group            > select_group',
     '-                               > group by         > select_group_by',
     '-                               > order            > select_order',
     '-                               > order by         > select_order_by',
     '-                               > limit            > select_where_sc_limit',
     '-                               > ;                > ',
+    '-                               > &                > select_from_x$',
     'select_group_                   > by               > select_group_by                         ^ by',
     'select_group_by_                > name             > select_group_by_a                       ^ columns',
     'select_group_by_a               > ,                > select_group_by_a_comma_                ^ columns',
     '-                               > ;                > ',
     'select_group_by_a_comma_        > name             > select_group_by_a                       ^ columns',
-    'select_group_by_a_              > limit            > select_where_sc_limit                   ^ limit,order by',
+    'select_group_by_a_              > limit            > select_where_sc_limit                   ^ limit,order by,&',
     '-                               > order            > select_order',
     '-                               > order by         > select_order_by',
     '-                               > ;                > ',
+    '-                               > &                > select_from_x$',
     'select_order_                   > by               > select_order_by                         ^ by',
     'select_order_by_                > name             > select_order_by_a                       ^ columns',
     'select_order_by_a               > ,                > select_order_by_a_comma_',
     '-                               > ;                > ',
     'select_order_by_a_comma_        > name             > select_order_by_a                       ^ columns',
-    'select_order_by_a_              > desc|asc         > select_order_by_a_desc                  ^ desc,asc,limit',
+    'select_order_by_a_              > desc|asc         > select_order_by_a_desc                  ^ desc,asc,limit,&',
     '-                               > limit            > select_where_sc_limit',
     '-                               > ;                > ',
+    '-                               > &                > select_from_x$',
     'select_order_by_a_desc          > ,                > select_order_by_a_comma_',
     '-                               > ;                > ',
-    'select_order_by_a_desc_         > limit            > select_where_sc_limit                   ^ limit',
+    'select_order_by_a_desc_         > limit            > select_where_sc_limit                   ^ limit,&',
     '-                               > ;                > ',
+    '-                               > &                > select_from_x$',
     'select_where_sc_limit_          > num              > select_where_sc_limit_num               ^ 1',
     'select_where_sc_limit_num       > ;                > ',
     'select_where_sc_limit_num_rp__  > as               > select_from_x_as                        ^ as',
@@ -201,7 +207,9 @@ SQL_SPEC = [
     'insert_values                   > (                > insert_values_lp_',
     'insert_values_lp_               > name|single|num  > insert_values_lp_v                      ^ single',
     'insert_values_lp_v              > ,                > insert_values_lp_v_comma_',
+    '-                               > )                > insert_values_lp_v_rp_',
     'insert_values_lp_v_comma_       > name|single|num  > insert_values_lp_v',
+    'insert_values_lp_v_rp__         > &                > insert_values_lp_v_rp_$                 ^ &',
 
     # <update_statement> ::= UPDATE <table_name>
     #                        SET <set_clause_list>
@@ -222,8 +230,9 @@ SQL_SPEC = [
     'update_set_a_op                 > name|single|num  > update_set_sc                           ^ single',
     'update_set_sc                   > ,                > update_set_sc_comma_',
     'update_set_sc_comma_            > name             > update_set_a                            ^ id',
-    'update_set_sc_                  > ,                > update_set_sc_comma_                    ^ where',
+    'update_set_sc_                  > ,                > update_set_sc_comma_                    ^ where,&',
     '-                               > where            > update_where',
+    '-                               > &                > update_set_sc$                          ^ &',
     'update_where_                   > name             > update_where_a                          ^ id',
     'update_where_a                  > comparison       > update_where_a_op',
     'update_where_a_                 > comparison       > update_where_a_op                       ^ =,<,<=,>,>=,<>,like,not,in',
@@ -240,7 +249,7 @@ SQL_SPEC = [
     'update_where_a_in_lp_a_comma_   > name|single|num  > update_where_a_in_lp_a                  ^ single',
     'update_where_a_not_op           > name|single|num  > update_where_sc                         ^ single',
     'update_where_a_op               > name|single|num  > update_where_sc                         ^ single',
-    'update_where_sc_                > and|or           > update_where                            ^ and,or',
+    'update_where_sc_                > and|or           > update_where                            ^ and,or,&',
 
     # <delete_statement> ::= DELETE FROM <table_name> [ WHERE <search_condition> ]
 
@@ -421,12 +430,12 @@ CQL_SPEC = SQL_SPEC + [
 
     '                                > clean            > clean',
     'clean_                          > up               > clean_up                                ^ up all export sessions,up export session',
-    'clean_up_                       > all              > clean_up_all                            ^ all export sessions,export session',
+    'clean_up_                       > all              > clean_up_all                            ^ all export sessions,export sessions',
     '-                               > export           > clean_up_export',
     'clean_up_all_                   > export           > clean_up_all_export                     ^ export sessions',
     'clean_up_all_export_            > sessions         > clean_up_all_sessions                   ^ sessions',
-    'clean_up_export_                > session          > clean_up_export_session                 ^ session',
-    'clean_up_export_session_        > name             > clean_up_export_session$                ^ export-sessions',
+    'clean_up_export_                > sessions         > clean_up_export_sessions                ^ sessions',
+    'clean_up_export_sessions_       > name             > clean_up_export_sessions$               ^ export-sessions',
 ]
 
 CQL_KEYWORDS = SQL_KEYWORDS + [
@@ -546,7 +555,7 @@ class SqlStateMachine(StateMachine[Token]):
                 elif token.ttype == TOKEN.Operator.Comparison:
                     it = 'comparison'
 
-                try:
+                with log_exc(False):
                     # print(f'\n{state.to_s} > {it} > ', end='')
                     if comeback_state:
                         state = comeback_state
@@ -557,8 +566,6 @@ class SqlStateMachine(StateMachine[Token]):
 
                     if last_name:
                         state.context['last_name'] = last_name
-                except:
-                    pass
 
         return state
 

adam/sso/authn_ad.py

@@ -1,6 +1,5 @@
 import json
 import re
-import traceback
 import jwt
 import requests
 from urllib.parse import urlparse, parse_qs
@@ -8,6 +7,7 @@ from urllib.parse import urlparse, parse_qs
 from adam.log import Log
 from adam.sso.authenticator import Authenticator
 from adam.sso.id_token import IdToken
+from adam.utils import debug, log_exc
 from .idp_login import IdpLogin
 from adam.config import Config
 
@@ -33,7 +33,7 @@ class AdAuthenticator(Authenticator):
 
         session = requests.Session()
         r = session.get(idp_uri)
-        Config().debug(f'{r.status_code} {idp_uri}')
+        debug(f'{r.status_code} {idp_uri}')
 
         config = self.validate_and_return_config(r)
 
@@ -52,7 +52,7 @@ class AdAuthenticator(Authenticator):
         r = session.post(login_uri, data=body, headers={
             'Content-Type': 'application/x-www-form-urlencoded'
         })
-        Config().debug(f'{r.status_code} {login_uri}')
+        debug(f'{r.status_code} {login_uri}')
 
         config = self.validate_and_return_config(r)
 
@@ -69,7 +69,7 @@ class AdAuthenticator(Authenticator):
         r = session.post(kmsi_uri, data=body, headers={
             'Content-Type': 'application/x-www-form-urlencoded'
         })
-        Config().debug(f'{r.status_code} {kmsi_uri}')
+        debug(f'{r.status_code} {kmsi_uri}')
 
         if (config := self.extract_config_object(r.text)):
             if 'sErrorCode' in config and config['sErrorCode'] == '50058':
@@ -101,7 +101,7 @@ class AdAuthenticator(Authenticator):
 
     def validate_and_return_config(self, r: requests.Response):
         if r.status_code < 200 or r.status_code >= 300:
-            Config().debug(r.text)
+            debug(r.text)
 
             return None
 
@@ -138,7 +138,7 @@ class AdAuthenticator(Authenticator):
 
     def parse_id_token(self, id_token: str) -> IdToken:
         jwks_url = Config().get('idps.ad.jwks-uri', '')
-        try:
+        with log_exc():
             jwks_client = jwt.PyJWKClient(jwks_url, cache_jwk_set=True, lifespan=360)
             signing_key = jwks_client.get_signing_key_from_jwt(id_token)
             data = jwt.decode(
@@ -163,7 +163,5 @@ class AdAuthenticator(Authenticator):
                 nbf=data['nbf'] if 'nbf' in data else 0,
                 exp=data['exp'] if 'exp' in data else 0
             )
-        except:
-            Config().debug(traceback.format_exc())
 
         return None

adam/sso/authn_okta.py

@@ -8,7 +8,7 @@ from adam.sso.id_token import IdToken
 
 from .idp_login import IdpLogin
 from adam.config import Config
-from adam.utils import log2
+from adam.utils import debug, log2, log_exc
 
 class OktaException(Exception):
     pass
@@ -49,7 +49,7 @@ class OktaAuthenticator(Authenticator):
 
         session = requests.Session()
         response = session.post(authn_uri, headers=headers, data=json.dumps(payload))
-        Config().debug(f'{response.status_code} {authn_uri}')
+        debug(f'{response.status_code} {authn_uri}')
         auth_response = response.json()
 
         if 'sessionToken' not in auth_response:
@@ -59,7 +59,7 @@ class OktaAuthenticator(Authenticator):
 
         url = f'{idp_uri}&sessionToken={session_token}'
         r = session.get(url)
-        Config().debug(f'{r.status_code} {url}')
+        debug(f'{r.status_code} {url}')
 
         id_token = OktaAuthenticator().extract(r.text, r'.*name=\"id_token\" value=\"(.*?)\".*')
         if not id_token:
@@ -95,7 +95,7 @@ class OktaAuthenticator(Authenticator):
             return None
 
         jwks_url = Config().get('idps.okta.jwks-uri', 'https://c3energy.okta.com/oauth2/v1/keys')
-        try:
+        with log_exc():
             jwks_client = jwt.PyJWKClient(jwks_url, cache_jwk_set=True, lifespan=360)
             signing_key = jwks_client.get_signing_key_from_jwt(id_token)
             data = jwt.decode(
@@ -121,7 +121,5 @@ class OktaAuthenticator(Authenticator):
                 nbf=data['nbf'] if 'nbf' in data else 0,
                 exp=data['exp'] if 'exp' in data else 0
             )
-        except:
-            pass
 
         return None