kaqing 2.0.214__py3-none-any.whl → 2.0.227__py3-none-any.whl

adam/commands/repair/repair_stop.py

@@ -33,5 +33,5 @@ class RepairStop(Command):
     def completion(self, state: ReplState):
         return super().completion(state)
 
-    def help(self, _: ReplState):
-        return f'{RepairStop.COMMAND}\t delete a repair job'
+    def help(self, state: ReplState):
+        return super().help(state, 'delete a repair job')

adam/embedded_params.py

@@ -1,2 +1,2 @@
 def config():
-    return {'app': {'console-endpoint': 'https://{host}/{env}/{app}/static/console/index.html', 'container-name': 'c3-server', 'cr': {'cluster-regex': '(.*?-.*?)-.*', 'group': 'ops.c3.ai', 'v': 'v2', 'plural': 'c3cassandras'}, 'label': 'c3__app_id-0', 'login': {'admin-group': '{host}/C3.ClusterAdmin', 'ingress': '{app_id}-k8singr-appleader-001', 'timeout': 5, 'session-check-url': 'https://{host}/{env}/{app}/api/8/C3/userSessionToken', 'cache-creds': True, 'cache-username': True, 'url': 'https://{host}/{env}/{app}', 'another': "You're logged in to {has}. However, for this app, you need to log in to {need}.", 'token-server-url': 'http://localhost:{port}', 'password-max-length': 128}, 'strip': '0'}, 'audit': {'endpoint': 'https://4psvtaxlcb.execute-api.us-west-2.amazonaws.com/prod/', 'workers': 3, 'timeout': 10, 'log-audit-queries': False, 'athena': {'auto-repair': {'elapsed_hours': 12}, 'region': 'us-west-2', 'catalog': 'AwsDataCatalog', 'database': 'audit', 'repair-partition-tables': 'audit', 'output': 's3://s3.ops--audit/ddl/results', 'repair-cluster-tables': 'cluster'}, 'queries': {'last10': "SELECT * FROM audit\nWHERE drive <> 'z' and ({date_condition})\nORDER BY ts DESC LIMIT {limit}", 'slow10': "SELECT * FROM audit\nWHERE drive <> 'z' and ({date_condition})\nORDER BY CAST(duration AS REAL) DESC LIMIT {limit}", 'top10': "SELECT min(c) AS cluster, line, COUNT(*) AS cnt, avg(CAST(duration AS REAL)) AS duration\nFROM audit WHERE drive <> 'z' and ({date_condition})\nGROUP BY line ORDER BY cnt DESC LIMIT {limit}"}}, 'bash': {'workers': 32}, 'cassandra': {'service-name': 'all-pods-service'}, 'cql': {'workers': 32, 'samples': 3, 'secret': {'cluster-regex': '(.*?-.*?)-.*', 'name': '{cluster}-superuser', 'password-item': 'password'}, 'alter-tables': {'excludes': 'system_auth,system_traces,reaper_db,system_distributed,system_views,system,system_schema,system_virtual_schema', 'gc-grace-periods': '3600,86400,864000,7776000', 'batching': True}}, 'checks': {'compactions-threshold': 250, 'cpu-busy-threshold': 98.0, 'cpu-threshold': 0.0, 'cassandra-data-path': '/c3/cassandra', 'root-disk-threshold': 50, 'cassandra-disk-threshold': 50, 'snapshot-size-cmd': "ls /c3/cassandra/data/data/*/*/snapshots | grep snapshots | sed 's/:$//g' | xargs -I {} du -sk {} | awk '{print $1}' | awk '{s+=$1} END {print s}'", 'snapshot-size-threshold': '40G', 'table-sizes-cmd': "ls -Al /c3/cassandra/data/data/ | awk '{print $9}' | sed 's/\\^r//g' | xargs -I {} du -sk /c3/cassandra/data/data/{}"}, 'download': {'workers': 8}, 'export': {'workers': 8, 'csv_dir': '/c3/cassandra/tmp', 'column_counts_query': 'select id, count(id) as columns from {table} group by id order by columns desc limit 10', 'default-importer': 'sqlite', 'sqlite': {'workers': 8, 'columns': '<row-key>', 'local-db-dir': '/tmp/qing-db/q/export/db'}, 'athena': {'workers': 8, 'columns': '<keys>', 'bucket': 'c3.ops--qing'}, 'csv': {'workers': 8, 'columns': '<row-key>'}, 'log-dir': '/tmp/qing-db/q/export/logs'}, 'get-host-id': {'workers': 32}, 'idps': {'ad': {'email-pattern': '.*@c3.ai', 'uri': 'https://login.microsoftonline.com/53ad779a-93e7-485c-ba20-ac8290d7252b/oauth2/v2.0/authorize?response_type=id_token&response_mode=form_post&client_id=00ff94a8-6b0a-4715-98e0-95490012d818&scope=openid+email+profile&redirect_uri=https%3A%2F%2Fplat.c3ci.cloud%2Fc3%2Fc3%2Foidc%2Flogin&nonce={nonce}&state=EMPTY', 'jwks-uri': 'https://login.microsoftonline.com/common/discovery/keys', 'contact': 'Please contact ted.tran@c3.ai.', 'whitelist-file': '/kaqing/members'}, 'okta': {'default': True, 'email-pattern': '.*@c3iot.com', 'uri': 'https://c3energy.okta.com/oauth2/v1/authorize?response_type=id_token&response_mode=form_post&client_id={client_id}&scope=openid+email+profile+groups&redirect_uri=https%3A%2F%2F{host}%2Fc3%2Fc3%2Foidc%2Flogin&nonce={nonce}&state=EMPTY', 'jwks-uri': 'https://c3energy.okta.com/oauth2/v1/keys'}}, 'issues': {'workers': 32}, 'local-qing-dir': '/tmp/qing-db/q', 'local-downloads-dir': '/tmp/qing-db/q/downloads', 'logs': {'path': '/c3/cassandra/logs/system.log'}, 'log-dir': '/tmp/qing-db/q/logs', 'nodetool': {'workers': 96, 'commands_in_line': 40, 'status': {'workers': 32, 'samples': 3, 'commands_in_line': 40}}, 'pg': {'name-pattern': '^{namespace}.*-k8spg-.*', 'excludes': '.helm., -admin-secret', 'agent': {'name': 'ops-pg-agent', 'just-in-time': False, 'timeout': 86400, 'image': 'seanahnsf/kaqing'}, 'default-db': 'postgres', 'default-schema': 'postgres', 'secret': {'endpoint-key': 'postgres-db-endpoint', 'port-key': 'postgres-db-port', 'username-key': 'postgres-admin-username', 'password-key': 'postgres-admin-password'}}, 'pod': {'name': 'ops', 'image': 'seanahnsf/kaqing-cloud', 'sa': {'name': 'ops', 'proto': 'c3', 'additional-cluster-roles': 'c3aiops-k8ssandra-operator'}, 'label-selector': 'run=ops'}, 'preview': {'rows': 10}, 'processes': {'columns': 'pod,cpu-metrics,mem', 'header': 'POD_NAME,M_CPU(USAGE/LIMIT),MEM/LIMIT'}, 'processes-qing': {'columns': 'pod,cpu,mem', 'header': 'POD_NAME,Q_CPU/TOTAL,MEM/LIMIT'}, 'reaper': {'service-name': 'reaper-service', 'port-forward': {'timeout': 86400, 'local-port': 9001}, 'abort-runs-batch': 10, 'show-runs-batch': 100, 'pod': {'cluster-regex': '(.*?-.*?-.*?-.*?)-.*', 'label-selector': 'k8ssandra.io/reaper={cluster}-reaper'}, 'secret': {'cluster-regex': '(.*?-.*?)-.*', 'name': '{cluster}-reaper-ui', 'password-item': 'password'}}, 'repair': {'log-path': '/home/cassrepair/logs/', 'image': 'ci-registry.c3iot.io/cloudops/cassrepair:2.0.14', 'secret': 'ciregistryc3iotio', 'env': {'interval': 24, 'timeout': 60, 'pr': False, 'runs': 1}}, 'repl': {'start-drive': 'c', 'a': {'auto-enter': 'c3/c3'}, 'c': {'auto-enter': 'cluster'}, 'x': {'auto-enter': 'latest'}, 'history': {'push-cat-log-file': True, 'push-cat-remote-log-file': True}, 'background-process': {'auto-nohup': True}}, 'status': {'columns': 'status,address,load,tokens,owns,host_id,gossip,compactions', 'header': '--,Address,Load,Tokens,Owns,Host ID,GOSSIP,COMPACTIONS'}, 'storage': {'columns': 'pod,volume_root,volume_cassandra,snapshots,data,compactions', 'header': 'POD_NAME,VOLUME /,VOLUME CASS,SNAPSHOTS,DATA,COMPACTIONS'}, 'watch': {'auto': 'rollout', 'timeout': 3600, 'interval': 10}, 'auto-complete': {'c': {'tables': 'lazy', 'table-props-value': 'jit'}, 'x': {'tables': 'lazy'}, 'cli': {'cp': 'jit'}, 'export': {'databases': 'jit'}, 'medusa': {'backups': 'jit'}, 'reaper': {'schedules': 'jit'}}, 'debug': False, 'debugs': {'complete': False, 'timings': False, 'exit-on-error': False}}
+    return {'app': {'console-endpoint': 'https://{host}/{env}/{app}/static/console/index.html', 'container-name': 'c3-server', 'cr': {'cluster-regex': '(.*?-.*?)-.*', 'group': 'ops.c3.ai', 'v': 'v2', 'plural': 'c3cassandras'}, 'label': 'c3__app_id-0', 'login': {'admin-group': '{host}/C3.ClusterAdmin', 'ingress': '{app_id}-k8singr-appleader-001', 'timeout': 5, 'session-check-url': 'https://{host}/{env}/{app}/api/8/C3/userSessionToken', 'cache-creds': True, 'cache-username': True, 'url': 'https://{host}/{env}/{app}', 'another': "You're logged in to {has}. However, for this app, you need to log in to {need}.", 'token-server-url': 'http://localhost:{port}', 'password-max-length': 128}, 'strip': '0'}, 'audit': {'endpoint': 'https://4psvtaxlcb.execute-api.us-west-2.amazonaws.com/prod/', 'workers': 3, 'timeout': 10, 'log-audit-queries': False, 'athena': {'auto-repair': {'elapsed_hours': 12}, 'region': 'us-west-2', 'catalog': 'AwsDataCatalog', 'database': 'audit', 'repair-partition-tables': 'audit', 'output': 's3://s3.ops--audit/ddl/results', 'repair-cluster-tables': 'cluster'}, 'queries': {'last10': "SELECT * FROM audit\nWHERE drive <> 'z' and ({date_condition})\nORDER BY ts DESC LIMIT {limit}", 'slow10': "SELECT * FROM audit\nWHERE drive <> 'z' and ({date_condition})\nORDER BY CAST(duration AS REAL) DESC LIMIT {limit}", 'top10': "SELECT min(c) AS cluster, line, COUNT(*) AS cnt, avg(CAST(duration AS REAL)) AS duration\nFROM audit WHERE drive <> 'z' and ({date_condition})\nGROUP BY line ORDER BY cnt DESC LIMIT {limit}"}}, 'bash': {'workers': 32}, 'cassandra': {'service-name': 'all-pods-service'}, 'cql': {'workers': 32, 'samples': 3, 'secret': {'cluster-regex': '(.*?-.*?)-.*', 'name': '{cluster}-superuser', 'password-item': 'password'}, 'alter-tables': {'excludes': 'system_auth,system_traces,reaper_db,system_distributed,system_views,system,system_schema,system_virtual_schema', 'gc-grace-periods': '3600,86400,864000,7776000', 'batching': True}}, 'checks': {'compactions-threshold': 250, 'cpu-busy-threshold': 98.0, 'cpu-threshold': 0.0, 'cassandra-data-path': '/c3/cassandra', 'root-disk-threshold': 50, 'cassandra-disk-threshold': 50, 'snapshot-size-cmd': "ls /c3/cassandra/data/data/*/*/snapshots | grep snapshots | sed 's/:$//g' | xargs -I {} du -sk {} | awk '{print $1}' | awk '{s+=$1} END {print s}'", 'snapshot-size-threshold': '40G', 'table-sizes-cmd': "ls -Al /c3/cassandra/data/data/ | awk '{print $9}' | sed 's/\\^r//g' | xargs -I {} du -sk /c3/cassandra/data/data/{}"}, 'download': {'workers': 8}, 'export': {'workers': 8, 'csv_dir': '/c3/cassandra/tmp', 'column_counts_query': 'select id, count(id) as columns from {table} group by id order by columns desc limit 10', 'default-importer': 'sqlite', 'sqlite': {'workers': 8, 'columns': '<row-key>', 'local-db-dir': '/tmp/qing-db/q/export/db'}, 'athena': {'workers': 8, 'columns': '<keys>', 'bucket': 'c3.ops--qing'}, 'csv': {'workers': 8, 'columns': '<row-key>'}, 'log-dir': '/tmp/qing-db/q/export/logs', 'remote': {'log-dir': '/tmp/q/export/logs'}}, 'get-host-id': {'workers': 32}, 'idps': {'ad': {'email-pattern': '.*@c3.ai', 'uri': 'https://login.microsoftonline.com/53ad779a-93e7-485c-ba20-ac8290d7252b/oauth2/v2.0/authorize?response_type=id_token&response_mode=form_post&client_id=00ff94a8-6b0a-4715-98e0-95490012d818&scope=openid+email+profile&redirect_uri=https%3A%2F%2Fplat.c3ci.cloud%2Fc3%2Fc3%2Foidc%2Flogin&nonce={nonce}&state=EMPTY', 'jwks-uri': 'https://login.microsoftonline.com/common/discovery/keys', 'contact': 'Please contact ted.tran@c3.ai.', 'whitelist-file': '/kaqing/members'}, 'okta': {'default': True, 'email-pattern': '.*@c3iot.com', 'uri': 'https://c3energy.okta.com/oauth2/v1/authorize?response_type=id_token&response_mode=form_post&client_id={client_id}&scope=openid+email+profile+groups&redirect_uri=https%3A%2F%2F{host}%2Fc3%2Fc3%2Foidc%2Flogin&nonce={nonce}&state=EMPTY', 'jwks-uri': 'https://c3energy.okta.com/oauth2/v1/keys'}}, 'issues': {'workers': 32}, 'job': {'log-to-pods': True}, 'local-qing-dir': '/tmp/qing-db/q', 'local-downloads-dir': '/tmp/qing-db/q/downloads', 'logs': {'path': '/c3/cassandra/logs/system.log'}, 'log-dir': '/tmp/qing-db/q/logs', 'nodetool': {'workers': 96, 'commands_in_line': 40, 'grace-period-after-abort': 10, 'status': {'workers': 32, 'samples': 3, 'commands_in_line': 40}}, 'pg': {'name-pattern': '^{namespace}.*-k8spg-.*', 'excludes': '.helm., -admin-secret', 'agent': {'name': 'ops-pg-agent', 'just-in-time': False, 'timeout': 86400, 'image': 'seanahnsf/kaqing'}, 'default-db': 'postgres', 'default-schema': 'postgres', 'secret': {'endpoint-key': 'postgres-db-endpoint', 'port-key': 'postgres-db-port', 'username-key': 'postgres-admin-username', 'password-key': 'postgres-admin-password'}}, 'pod': {'name': 'ops', 'image': 'seanahnsf/kaqing-cloud', 'sa': {'name': 'ops', 'proto': 'c3', 'additional-cluster-roles': 'c3aiops-k8ssandra-operator'}, 'label-selector': 'run=ops'}, 'preview': {'rows': 10}, 'processes': {'columns': 'pod,cpu-metrics,mem', 'header': 'POD_NAME,M_CPU(USAGE/LIMIT),MEM/LIMIT'}, 'processes-mpstat': {'columns': 'pod,cpu,mem', 'header': 'POD_NAME,Q_CPU/TOTAL,MEM/LIMIT'}, 'reaper': {'service-name': 'reaper-service', 'port-forward': {'timeout': 86400, 'local-port': 9001}, 'abort-runs-batch': 10, 'show-runs-batch': 100, 'pod': {'cluster-regex': '(.*?-.*?-.*?-.*?)-.*', 'label-selector': 'k8ssandra.io/reaper={cluster}-reaper'}, 'secret': {'cluster-regex': '(.*?-.*?)-.*', 'name': '{cluster}-reaper-ui', 'password-item': 'password'}}, 'repair': {'log-path': '/home/cassrepair/logs/', 'image': 'ci-registry.c3iot.io/cloudops/cassrepair:2.0.14', 'secret': 'ciregistryc3iotio', 'env': {'interval': 24, 'timeout': 60, 'pr': False, 'runs': 1}}, 'repl': {'start-drive': 'c', 'a': {'auto-enter': 'c3/c3'}, 'c': {'auto-enter': 'cluster'}, 'x': {'auto-enter': 'latest'}, 'history': {'push-cat-log-file': True, 'push-cat-remote-log-file': True}, 'background-process': {'auto-nohup': True}}, 'status': {'columns': 'status,address,load,tokens,owns,host_id,gossip,compactions', 'header': '--,Address,Load,Tokens,Owns,Host ID,GOSSIP,COMPACTIONS'}, 'storage': {'columns': 'pod,volume_root,volume_cassandra,snapshots,data,compactions', 'header': 'POD_NAME,VOLUME /,VOLUME CASS,SNAPSHOTS,DATA,COMPACTIONS'}, 'watch': {'auto': 'rollout', 'timeout': 3600, 'interval': 10}, 'auto-complete': {'c': {'tables': 'lazy', 'table-props-value': 'jit'}, 'x': {'tables': 'lazy'}, 'cli': {'cp': 'jit'}, 'export': {'databases': 'jit'}, 'medusa': {'backups': 'jit'}, 'reaper': {'schedules': 'jit'}}, 'debug': False, 'debugs': {'complete': False, 'timings': False, 'exit-on-error': False}}

adam/repl.py

@@ -3,6 +3,7 @@ import time
 from typing import cast
 import click
 from prompt_toolkit.key_binding import KeyBindings
+from prompt_toolkit import HTML
 
 from adam.cli_group import cli
 from adam.commands.command import Command, InvalidArgumentsException, InvalidStateException
@@ -81,7 +82,7 @@ def enter_repl(state: ReplState):
                         # print(json.dumps(completions, indent=4))
                         completer = ReplCompleter.from_nested_dict(completions)
 
-                cmd = session.prompt(prompt_msg(), completer=completer, key_bindings=kb)
+                cmd = session.prompt(HTML(f'<ansibrightblue>{prompt_msg()}</ansibrightblue>'), completer=completer, key_bindings=kb)
                 s0 = time.time()
 
                 if state.bash_session:

adam/repl_commands.py

@@ -61,8 +61,13 @@ from adam.commands.export.show_export_sessions import ShowExportSessions
 from adam.commands.fs.find_files import FindLocalFiles
 from adam.commands.fs.find_processes import FindProcesses
 from adam.commands.fs.head import Head
+from adam.commands.fs.head_local import HeadLocal
 from adam.commands.fs.ls_local import LsLocal
 from adam.commands.fs.rm import RmLocal
+from adam.commands.fs.rm_logs import RmLogs
+from adam.commands.fs.show_last_results import ShowLastResults
+from adam.commands.fs.tail import Tail
+from adam.commands.fs.tail_local import TailLocal
 from adam.commands.kubectl import Kubectl
 from adam.commands.fs.shell import Shell
 from adam.commands.bash.bash import Bash
@@ -107,18 +112,28 @@ class ReplCommands:
         return deduped
 
     def navigation() -> list[Command]:
-        return [Ls(), LsLocal(), PreviewTable(), DeviceApp(), DevicePostgres(), DeviceCass(), DeviceAuditLog(), DeviceExport(),
-                Cd(), Cat(), CatLocal(), Head(), DownloadFile(), FindLocalFiles(), FindProcesses(), Pwd(), ClipboardCopy(),
-                GetParam(), SetParam(), ShowOffloadedCompletes(), ShowParams(), ShowKubectlCommands(), ShowLogin(), ShowAdam(), ShowHost()] + RmLocal().cmd_list()
+        return [Cd(), Cat(), CatLocal(), ClipboardCopy(),
+                DeviceApp(), DevicePostgres(), DeviceCass(), DeviceAuditLog(), DeviceExport(),
+                DownloadFile(), FindLocalFiles(), FindProcesses(), GetParam(),
+                Head(), HeadLocal(), Ls(), LsLocal(), PreviewTable(), Pwd(), RmLogs(),
+                SetParam(), ShowAdam(), ShowHost(), ShowKubectlCommands(), ShowLastResults(),
+                ShowLogin(), ShowOffloadedCompletes(), ShowParams(),
+                Tail(), TailLocal()] + \
+                RmLocal().cmd_list()
 
     def cassandra_ops() -> list[Command]:
-        return [Cqlsh(), DownloadCassandraLog(), ShowCassandraStatus(), ShowCassandraVersion(), ShowCassandraRepairs(), ShowStorage(), ShowProcesses(),
-                Check(), Issues(), NodeTool(), GenerateReport(), AlterTables(), Bash(),
-                ExportTables(), ExportXSelect(), ExportUse(),  ShowExportDatabases(), ShowColumnCounts(),
-                DropExportDatabase(), DropExportDatabases(),
-                ShowExportSessions(), ShowExportSession(), DownloadExportSession(),
-                CleanUpExportSessions(), CleanUpAllExportSessions(), ImportSession(), ImportCSVFiles()] + \
-                Medusa().cmd_list() + [RestartNodes(), RestartNode(), RestartCluster(), RollOut(), Watch()] + Reaper().cmd_list() + Repair().cmd_list() + Debug().cmd_list()
+        return [AlterTables(), Bash(), Check(), CleanUpExportSessions(), CleanUpAllExportSessions(), Cqlsh(),
+                DownloadCassandraLog(), DropExportDatabase(), DropExportDatabases(), DownloadExportSession(),
+                ExportTables(), ExportXSelect(), ExportUse(),
+                GenerateReport(), ImportSession(), ImportCSVFiles(), Issues(), NodeTool(),
+                RestartNodes(), RestartNode(), RestartCluster(), RollOut(),
+                ShowCassandraStatus(), ShowCassandraVersion(), ShowCassandraRepairs(), ShowColumnCounts(),
+                ShowStorage(), ShowExportDatabases(), ShowExportSessions(), ShowExportSession(), ShowProcesses(),
+                Watch()] + \
+                Debug().cmd_list() + \
+                Medusa().cmd_list() + \
+                Reaper().cmd_list() + \
+                Repair().cmd_list()
 
     def postgres_ops() -> list[Command]:
         return [Postgres(), DeployPgAgent(), UndeployPgAgent(), PostgresPg()]

adam/repl_session.py

@@ -1,8 +1,9 @@
+from typing import Union
 from prompt_toolkit import PromptSession
 from prompt_toolkit.auto_suggest import AutoSuggestFromHistory
 
 from adam.config import Config
-from adam.utils import ConfigHolder
+from adam.utils import ConfigHolder, LogFile
 
 class ReplSession:
     # the singleton pattern
@@ -16,6 +17,12 @@ class ReplSession:
             self.prompt_session = PromptSession(auto_suggest=AutoSuggestFromHistory())
             ConfigHolder().append_command_history = self.append_history
 
-    def append_history(self, entry: str):
-        if entry and  self.prompt_session and Config().get('repl.history.push-cat-remote-log-file', True):
+    def append_history(self, entry: Union[str, LogFile]):
+        if not entry:
+            return
+
+        if self.prompt_session and Config().get('repl.history.push-cat-remote-log-file', True):
+            if isinstance(entry, LogFile):
+                entry = entry.to_command()
+
             self.prompt_session.history.append_string(entry)

adam/sql/qingl.lark

@@ -4,19 +4,18 @@ start: statement_sequence
 WHITESPACE: (" " | /\t/ | /\n/ | /\r/ )+
 %ignore WHITESPACE
 
-# SEAN replaced with @host statement ( ";" statement )* ";"? "&"?
-statement_sequence: statement ( ";" statement )* ";"? "&"?
-
-statement: dcl_statement
-	| ddl_statement
-	| dml_statement
-	| dql_statement
-	| utility_statement
+statement_sequence: statement ( ";" statement )* ";"?
+
+statement: dcl_statement "&"?
+	| ddl_statement "&"?
+	| dml_statement "&"?
+	| dql_statement "&"?
+#	| utility_statement "&"?
     | qing_statement
 
 ddl_statement: create_statement
 	| drop_statement
-	| other_statement
+#	| other_statement
 
 # --------------------- SEAN added -------------------
 
@@ -27,8 +26,8 @@ host_name: IDENTIFIER
 
 # --------------------- p: -------------------
 
-qing_p_statement: preview_table_statement
-    | alter_table_statement
+qing_p_statement: preview_table_statement "&"?
+    | alter_table_statement "&"?
 
 preview_table_statement: "PREVIEW"i path
 alter_table_statement: "ALTER"i "TABLE"i keyspace_ref (add_column_action | drop_column_action | modify_column_action | rename_table_action | add_constraint_action)
@@ -50,11 +49,11 @@ column_name_list: column_name ("," column_name)*
 
 # --------------------- c: -------------------
 
-qing_c_statement: preview_table_statement
-    | alter_cql_table_statement
-    | alter_tables_statement
-    | describe_statement
-    | consistency_statement
+qing_c_statement: preview_table_statement "&"?
+    | alter_cql_table_statement "&"?
+    | alter_tables_statement "&"?
+    | describe_statement "&"?
+    | consistency_statement "&"?
     | drop_export_db_statement
     | use_export_db_statement
     | "SHOW"i show_c_subcommand
@@ -117,9 +116,9 @@ download_session_statement: "DOWNLOAD"i "EXPORT"i "SESSION"i export_session_name
 
 # --------------------- l: -------------------
 
-qing_l_statement: preview_table_statement
-    | alter_athena_table_statement
-    | describe_table_statement
+qing_l_statement: preview_table_statement "&"?
+    | alter_athena_table_statement "&"?
+    | describe_table_statement "&"?
     | "AUDIT"i ("REPAIR"i | "RUN"i)?
     | show_topn_statement
 
@@ -139,8 +138,8 @@ topn_window: IDENTIFIER
 
 # --------------------- x: -------------------
 
-qing_x_statement: preview_table_statement
-    | describe_table_statement
+qing_x_statement: preview_table_statement "&"?
+    | describe_table_statement "&"?
     | drop_export_db_statement
 
 qing_x0_statement: drop_export_db_statement
@@ -162,9 +161,9 @@ drop_statement: drop_scope
 	| drop_index
 	| drop_function
 
-other_statement: alter_index
-	| build_index
-	| execute_function
+# other_statement: alter_index
+#	| build_index
+#	| execute_function
 
 create_scope: "CREATE"i "SCOPE"i ( namespace_ref ":" )? bucket_ref "." scope_ref if_not_exists?
 
@@ -219,10 +218,10 @@ index_partition: "PARTITION"i "BY"i "HASH"i LPAREN partition_key_expr \
 
 partition_key_expr: expr
 
-alter_index: "ALTER"i "INDEX"i ( ( index_path "." index_name ) | ( index_name "ON"i keyspace_ref ) \
-	index_using? index_with )
+# alter_index: "ALTER"i "INDEX"i ( ( index_path "." index_name ) | ( index_name "ON"i keyspace_ref ) \
+#	index_using? index_with )
 
-build_index: "BUILD"i "INDEX"i "ON"i keyspace_ref LPAREN index_term ( "," index_term )* RPAREN index_using?
+# build_index: "BUILD"i "INDEX"i "ON"i keyspace_ref LPAREN index_term ( "," index_term )* RPAREN index_using?
 
 index_term: index_name | subquery_expr
 
@@ -283,11 +282,11 @@ sequence_with: "WITH"i expr
 
 drop_function: "DROP"i "FUNCTION"i function_ref if_exists?
 
-execute_function: "EXECUTE"i "FUNCTION"i function_ref LPAREN ( expr ( "," expr )* )? RPAREN
+# execute_function: "EXECUTE"i "FUNCTION"i function_ref LPAREN ( expr ( "," expr )* )? RPAREN
 
 dml_statement: delete_statement
 	| insert_statement
-	| merge_statement
+#	| merge_statement
 	| update_statement
 	| upsert_statement
 
@@ -307,44 +306,44 @@ insert_select: LPAREN "PRIMARY"i? "KEY"i key_expr ( "," "VALUE"i value_expr )? \
 
 options: obj
 
-merge_statement: "MERGE"i "INTO"i ( ansi_merge | lookup_merge ) limit_clause? returning_clause?
+# merge_statement: "MERGE"i "INTO"i ( ansi_merge | lookup_merge ) limit_clause? returning_clause?
 
-ansi_merge: target_keyspace use_index_clause "USING"i ansi_merge_source \
-	ansi_merge_predicate ansi_merge_actions
+# ansi_merge: target_keyspace use_index_clause "USING"i ansi_merge_source \
+#	ansi_merge_predicate ansi_merge_actions
 
-ansi_merge_source: ( merge_source_keyspace | merge_source_subquery | merge_source_expr ) \
-	ansi_join_hints?
+# ansi_merge_source: ( merge_source_keyspace | merge_source_subquery | merge_source_expr ) \
+#	ansi_join_hints?
 
-ansi_merge_predicate: "ON"i expr
+# ansi_merge_predicate: "ON"i expr
 
-ansi_merge_actions: merge_update? merge_delete? ansi_merge_insert?
+# ansi_merge_actions: merge_update? merge_delete? ansi_merge_insert?
 
-ansi_merge_insert: "WHEN"i "NOT"i "MATCHED"i "THEN"i "INSERT"i LPAREN "KEY"i? key_expr  \
-	( "," "VALUE"i? value_expr )? ( "," "OPTIONS"i? options )? RPAREN where_clause?
+# ansi_merge_insert: "WHEN"i "NOT"i "MATCHED"i "THEN"i "INSERT"i LPAREN "KEY"i? key_expr  \
+#	( "," "VALUE"i? value_expr )? ( "," "OPTIONS"i? options )? RPAREN where_clause?
 
 
-lookup_merge: target_keyspace "USING"i lookup_merge_source lookup_merge_predicate \
-	lookup_merge_actions
+# lookup_merge: target_keyspace "USING"i lookup_merge_source lookup_merge_predicate \
+#	lookup_merge_actions
 
-lookup_merge_source: ( merge_source_keyspace use_clause? )
-	| merge_source_subquery
-	| merge_source_expr
+# lookup_merge_source: ( merge_source_keyspace use_clause? )
+#	| merge_source_subquery
+#	| merge_source_expr
 
-lookup_merge_predicate: "ON"i "PRIMARY"i? "KEY"i expr
+# lookup_merge_predicate: "ON"i "PRIMARY"i? "KEY"i expr
 
-lookup_merge_actions: merge_update? merge_delete? lookup_merge_insert?
+# lookup_merge_actions: merge_update? merge_delete? lookup_merge_insert?
 
-lookup_merge_insert: "WHEN"i "NOT"i "MATCHED"i "THEN"i "INSERT"i expr where_clause?
+# lookup_merge_insert: "WHEN"i "NOT"i "MATCHED"i "THEN"i "INSERT"i expr where_clause?
 
-merge_source_keyspace: keyspace_ref ( "AS"i? alias )?
+# merge_source_keyspace: keyspace_ref ( "AS"i? alias )?
 
-merge_source_subquery: subquery_expr "AS"i? alias
+# merge_source_subquery: subquery_expr "AS"i? alias
 
-merge_source_expr: expr ( "AS"i? alias )?
+# merge_source_expr: expr ( "AS"i? alias )?
 
-merge_update: "WHEN"i "MATCHED"i "THEN"i "UPDATE"i set_clause? unset_clause? where_clause?
+# merge_update: "WHEN"i "MATCHED"i "THEN"i "UPDATE"i set_clause? unset_clause? where_clause?
 
-merge_delete: "WHEN"i "MATCHED"i "THEN"i "DELETE"i where_clause?
+# merge_delete: "WHEN"i "MATCHED"i "THEN"i "DELETE"i where_clause?
 
 update_statement: "UPDATE"i target_keyspace use_keys_clause? set_clause? unset_clause? \
 	where_clause? limit_clause? returning_clause?
@@ -373,7 +372,7 @@ key_expr: expr
 value_expr: expr
 
 dql_statement: select_statement
-    | infer_statement
+#    | infer_statement
     | update_statistics
 
 select_statement: select_term ( set_op select_term )* order_by_clause? limit_clause? offset_clause?
@@ -1006,16 +1005,16 @@ commit_transaction: "COMMIT"i ( "WORK"i | "TRAN"i | "TRANSACTION"i )?
 
 savepointname: identifier_ref
 
-utility_statement: advise_statement | explain_statement
+# utility_statement: advise_statement | explain_statement
 
-advise_statement: "ADVISE"i "INDEX"i? ( select_statement
-    | update_statement
-    | delete_statement
-    | merge_statement )
+# advise_statement: "ADVISE"i "INDEX"i? ( select_statement
+#    | update_statement
+#    | delete_statement
+#    | merge_statement )
 
-explain_statement: "INFER"i ( "COLLECTION"i | "KEYSPACE"i )? keyspace_ref ( "WITH"i options )?
+# explain_statement: "INFER"i ( "COLLECTION"i | "KEYSPACE"i )? keyspace_ref ( "WITH"i options )?
 
-infer_statement: "INFER"i ( "COLLECTION"i | "KEYSPACE"i )? keyspace_ref ( "WITH"i options )?
+# infer_statement: "INFER"i ( "COLLECTION"i | "KEYSPACE"i )? keyspace_ref ( "WITH"i options )?
 
 name: identifier_ref
 

adam/utils.py

@@ -4,22 +4,23 @@ from contextlib import redirect_stdout
 import copy
 import csv
 from datetime import datetime
+import html
 import importlib
 import io
-import json
 import os
 from pathlib import Path
 import random
 import string
 import threading
 import traceback
-from typing import Callable, Iterator, TextIO, TypeVar, Union
+from typing import Callable, Iterator, TypeVar, Union
 from dateutil import parser
 import subprocess
 import sys
 import time
 import click
 import yaml
+from prompt_toolkit import print_formatted_text, HTML
 from prompt_toolkit.completion import Completer
 
 from . import __version__
@@ -28,6 +29,9 @@ T = TypeVar('T')
 
 log_state = threading.local()
 
+class Color:
+    gray = 'gray'
+
 class ConfigReadable:
     def is_debug() -> bool:
         pass
@@ -117,19 +121,21 @@ def convert_seconds(total_seconds_float):
 def epoch(timestamp_string: str):
     return parser.parse(timestamp_string).timestamp()
 
-def log(s = None):
+def log(s = None, text_color: str = None):
     if not loggable():
         return False
 
     # want to print empty line for False or empty collection
     if s == None:
         print()
+    elif text_color:
+        print_formatted_text(HTML(f'<ansi{text_color}>{html.escape(s)}</ansi{text_color}>'))
     else:
         click.echo(s)
 
     return True
 
-def log2(s = None, nl = True, file: str = None):
+def log2(s = None, nl = True, file: str = None, text_color: str = None):
     if not loggable():
         return False
 
@@ -139,12 +145,16 @@ def log2(s = None, nl = True, file: str = None):
                 f.write(s)
                 if nl:
                     f.write('\n')
+        elif text_color:
+            print_formatted_text(HTML(f'<ansi{text_color}>{html.escape(s)}</ansi{text_color}>'))
         else:
             click.echo(s, err=True, nl=nl)
     else:
         if file:
             with open(file, 'at') as f:
                 f.write('\n')
+        elif text_color:
+            print_formatted_text(HTML(f'<ansi{text_color}>{html.escape(s)}</ansi{text_color}>'), file=sys.stderr)
         else:
             print(file=sys.stderr)
 
@@ -304,15 +314,14 @@ def is_lambda(func):
 
 def debug(s = None):
     if ConfigHolder().config.is_debug():
-        log2(f'DEBUG {s}')
+        log2(f'DEBUG {s}', text_color=Color.gray)
 
 def debug_complete(s = None):
     CommandLog.log(f'DEBUG {s}', config=ConfigHolder().config.get('debugs.complete', 'off'))
 
 def debug_trace():
     if ConfigHolder().config.is_debug():
-    # if LogConfig.is_debug():
-        log2(traceback.format_exc())
+        log2(traceback.format_exc(), text_color=Color.gray)
 
 def in_docker() -> bool:
     if os.path.exists('/.dockerenv'):
@@ -809,6 +818,12 @@ def kaqing_log_file_name(suffix = 'log'):
 def log_dir():
     return creating_dir(ConfigHolder().config.get('log-dir', '/tmp/qing-db/q/logs'))
 
+def log_to_pods():
+    return ConfigHolder().config.get('job.log-to-pods', True)
+
+def pod_log_dir():
+    return ConfigHolder().config.get('pod-log-dir', '/tmp/q/logs')
+
 class LogFileHandler:
     def __init__(self, suffix = 'log', condition=True):
         self.suffix = suffix
@@ -850,7 +865,7 @@ class CommandLog:
             except:
                 pass
         elif config == 'on':
-            log2(line)
+            log2(line, text_color=Color.gray)
 
     def close_log_file():
         if CommandLog.log_file:
@@ -880,3 +895,28 @@ def creating_dir(dir):
             os.makedirs(dir, exist_ok=True)
 
     return dir
+
+class LogFile(str):
+    def __init__(self, s: str):
+        super().__init__()
+
+    def __repr__(self):
+        return super().__repr__()
+
+    def to_command(self, cmd: str = ':tail'):
+        return f'{cmd} {self}'
+
+class PodLogFile(LogFile):
+    def __new__(cls, value, pod: str, size: str = None):
+        return super().__new__(cls, value)
+
+    def __init__(self, value, pod: str, size: str = None):
+         super().__init__(value)
+         self.pod = pod
+         self.size = size
+
+    def __repr__(self):
+        return super().__repr__()
+
+    def to_command(self, cmd: str = 'tail'):
+        return f'@{self.pod} {cmd} {self}'

adam/utils_async_job.py

@@ -0,0 +1,73 @@
+from datetime import datetime
+import re
+import traceback
+
+from adam.utils import log_dir
+
+class AsyncJobs:
+    _last_id: str = None
+    _last_command: str = None
+
+    def log_file(command: str, pod_name: str = None, job_id: str = None, pod_suffix: str = None, err = False, dir: str = None):
+        try:
+            if not job_id:
+                job_id = AsyncJobs.new_id()
+
+            AsyncJobs._last_id = job_id
+            AsyncJobs._last_command = command
+
+            AsyncJobs.write_last_command(job_id, command)
+
+            cmd_name = ''
+            if command.startswith('nodetool '):
+                command = command.strip(' &')
+                cmd_name = f".{'_'.join(command.split(' ')[5:])}"
+
+            if pod_suffix is None:
+                pod_suffix = '{pod}'
+                if pod_name:
+                    pod_suffix = pod_name
+                    if groups := re.match(r'.*-(.*)', pod_name):
+                        pod_suffix = f'-{groups[1]}'
+
+            if not dir:
+                dir = log_dir()
+
+            return f'{dir}/{job_id}{cmd_name}{pod_suffix}.{"err" if err else "log"}'
+        except:
+            traceback.print_exc()
+
+    def new_id(dt: datetime = None):
+        if not dt:
+            dt = datetime.now()
+
+        id = dt.strftime("%d%H%M%S")
+        AsyncJobs._last_id = id
+
+        return id
+
+    def last_command():
+        last_id = AsyncJobs._last_id
+        last_command = AsyncJobs._last_command
+
+        if not last_id or not last_command:
+            l0, l1 = AsyncJobs.read_last_command()
+            if not last_id:
+                last_id = l0
+            if not last_command:
+                last_command = l1
+
+        return last_id, last_command
+
+    def write_last_command(job_id: str, command: str):
+        with open(f'{log_dir()}/last', 'wt') as f:
+            f.write(job_id)
+            f.write('\n')
+            f.write(command)
+
+    def read_last_command():
+        with open(f'{log_dir()}/last', 'rt') as f:
+            job_id = f.readline().strip(' \r\n')
+            command = f.readline().strip(' \r\n')
+
+            return job_id, command

adam/utils_k8s/cassandra_clusters.py

@@ -2,9 +2,10 @@ import sys
 from typing import TypeVar
 
 from adam.config import Config
+from adam.utils_async_job import AsyncJobs
 from adam.utils_k8s.cassandra_nodes import CassandraNodes
 from adam.utils_k8s.pod_exec_result import PodExecResult
-from adam.utils import log, log2
+from adam.utils import log, log2, log_to_pods, pod_log_dir
 from adam.utils_k8s.pods import Pods
 from adam.utils_k8s.statefulsets import StatefulSets
 
@@ -22,23 +23,30 @@ class CassandraClusters:
              shell = '/bin/sh',
              backgrounded = False,
              log_file = None,
-             history=True) -> list[PodExecResult]:
+             history=True,
+             text_color: str = None) -> list[PodExecResult]:
 
         pods = StatefulSets.pod_names(sts, namespace)
         samples = 1 if on_any else sys.maxsize
         if (backgrounded or command.endswith(' &')) and not log_file:
-            log_file = Pods.log_file(command)
+            pod_suffix = None
+            dir = None
+            if log_to_pods():
+                pod_suffix = ''
+                dir = pod_log_dir()
+
+            log_file = AsyncJobs.log_file(command, job_id=AsyncJobs.new_id(), pod_suffix=pod_suffix, dir=dir)
 
         msg = 'd`Running|Ran ' + action + ' command onto {size} pods'
         with Pods.parallelize(pods, max_workers, samples, msg, action=action) as exec:
-            results: list[PodExecResult] = exec.map(lambda pod: CassandraNodes.exec(pod, namespace, command, False, False, shell, backgrounded, log_file, history))
+            results: list[PodExecResult] = exec.map(lambda pod: CassandraNodes.exec(pod, namespace, command, False, False, shell, backgrounded, log_file, history, text_color))
             for result in results:
                 if show_out and not Config().is_debug():
-                    log(result.command)
+                    log(result.command, text_color=text_color)
                     if result.stdout:
-                        log(result.stdout)
+                        log(result.stdout, text_color=text_color)
                     if result.stderr:
-                        log2(result.stderr)
+                        log2(result.stderr, text_color=text_color)
 
             return results
 

adam/utils_k8s/cassandra_nodes.py

@@ -14,13 +14,14 @@ class CassandraNodes:
              shell = '/bin/sh',
              backgrounded = False,
              log_file = None,
-             history = True) -> PodExecResult:
-        r = Pods.exec(pod_name, "cassandra", namespace, command, show_out = show_out, throw_err = throw_err, shell = shell, backgrounded = backgrounded, log_file=log_file)
+             history = True,
+             text_color: str = None) -> PodExecResult:
+        r: PodExecResult = Pods.exec(pod_name, "cassandra", namespace, command, show_out = show_out, throw_err = throw_err, shell = shell, backgrounded = backgrounded, log_file=log_file, text_color=text_color)
 
         if history and r and r.log_file:
-            entry = f':cat {r.log_file}'
+            # entry = f':tail {r.log_file}'
 
-            ReplSession().append_history(entry)
+            ReplSession().append_history(r.log_file)
 
         return r