kaqing 2.0.200__py3-none-any.whl → 2.0.211__py3-none-any.whl

adam/commands/export/importer_athena.py

@@ -19,9 +19,16 @@ class AthenaImporter(Importer):
     def prefix(self):
         return 'e'
 
-    def import_from_csv(self, state: ReplState, from_session: str,
-                        keyspace: str, table: str, target_table: str, columns: str,
-                        multi_tables = True, create_db = False):
+    def import_from_csv(self,
+                        state: ReplState,
+                        from_session: str,
+                        keyspace: str,
+                        table: str,
+                        target_table: str,
+                        columns: str,
+                        multi_tables = True,
+                        create_db = False,
+                        job_log: str = None):
         csv_file = self.csv_file(from_session, table, target_table)
         pod = state.pod
         namespace = state.namespace
@@ -47,12 +54,14 @@ class AthenaImporter(Importer):
             return to, to_session
         finally:
             if succeeded:
-                self.remove_csv(state, from_session, table, target_table, multi_tables)
+                self.remove_csv(state, from_session, table, target_table, multi_tables, job_log=job_log)
                 Athena.clear_cache()
 
-                if not multi_tables:
+                if multi_tables:
+                    log2(f'[{to_session}] {keyspace}.{target_table} OK', file=job_log)
+                else:
                     with export_db(state) as dbs:
-                        dbs.sql(f'select * from {database}.{target_table} limit 10')
+                        dbs.sql(f'select * from {keyspace}.{target_table} limit 10', backgrounded=True, export_log=job_log)
 
     def import_from_local_csv(self, state: ReplState,
                         keyspace: str, table: str, csv_file: str, multi_tables = True, create_db = False):

adam/commands/export/importer_sqlite.py

@@ -1,9 +1,10 @@
+from typing import TextIO
 import pandas
 
 from adam.commands.export.export_databases import export_db
 from adam.commands.export.importer import Importer
 from adam.repl_state import ReplState
-from adam.utils import GeneratorStream, bytes_generator_from_file, ing
+from adam.utils import GeneratorStream, bytes_generator_from_file, ing, log2
 from adam.utils_k8s.pods import Pods
 from adam.utils_sqlite import SQLite, sqlite
 
@@ -11,9 +12,17 @@ class SqliteImporter(Importer):
     def prefix(self):
         return 's'
 
-    def import_from_csv(self, state: ReplState, from_session: str,
-                        keyspace: str, table: str, target_table: str, columns: str,
-                        multi_tables = True, create_db = False):
+    def import_from_csv(self,
+                        state: ReplState,
+                        from_session: str,
+                        keyspace: str,
+                        table: str,
+                        target_table: str,
+                        columns: str,
+                        multi_tables = True,
+                        create_db = False,
+                        job_log: str = None):
+
         csv_file = self.csv_file(from_session, table, target_table)
         pod = state.pod
         namespace = state.namespace
@@ -21,7 +30,7 @@ class SqliteImporter(Importer):
 
         succeeded = False
         try:
-            with ing(f'[{to_session}] Uploading to Sqlite', suppress_log=multi_tables):
+            with ing(f'[{to_session}] Uploading to Sqlite', suppress_log=multi_tables, job_log=job_log):
                 # create a connection to single keyspace
                 with sqlite(to_session, keyspace) as conn:
                     bytes = Pods.read_file(pod, 'cassandra', namespace, csv_file)
@@ -35,12 +44,14 @@ class SqliteImporter(Importer):
             return to, to_session
         finally:
             if succeeded:
-                self.remove_csv(state, from_session, table, target_table, multi_tables)
+                self.remove_csv(state, from_session, table, target_table, multi_tables, job_log=job_log)
                 SQLite.clear_cache()
 
-                if not multi_tables:
+                if multi_tables:
+                    log2(f'[{to_session}] {keyspace}.{target_table} OK', file=job_log)
+                else:
                     with export_db(state) as dbs:
-                        dbs.sql(f'select * from {keyspace}.{target_table} limit 10')
+                        dbs.sql(f'select * from {keyspace}.{target_table} limit 10', backgrounded=True, export_log=job_log)
 
     def import_from_local_csv(self, state: ReplState,
                         keyspace: str, table: str, csv_file: str, multi_tables = True, create_db = False):

adam/commands/export/utils_export.py

@@ -1,12 +1,13 @@
 import io
+import os
 import re
 
 from adam.config import Config
-from adam.pod_exec_result import PodExecResult
+from adam.utils import ExecResult, creating_dir, log2
 from adam.repl_state import ReplState
 from adam.utils_k8s.cassandra_nodes import CassandraNodes
-from adam.utils_k8s.pods import log_prefix
 from adam.utils_k8s.statefulsets import StatefulSets
+from adam.utils_local import local_exec
 
 class ImportSpec:
     def __init__(self, table_name: str, session: str = None, files: list[str] = None, importer: str = None):
@@ -196,7 +197,7 @@ class ExportTableStatus:
         statuses: list[ExportTableStatus] = []
 
         status_in_whole = 'done'
-        log_files: list[str] = find_files(pod, namespace, f'{log_prefix()}-{export_session}_*.log*')
+        log_files: list[str] = find_files(pod, namespace, f'{export_log_dir()}/{export_session}_*.log*')
 
         for log_file in log_files:
             status: ExportTableStatus = ExportTableStatus.from_log_file(pod, namespace, export_session, log_file)
@@ -211,7 +212,7 @@ class ExportTableStatus:
         def get_csv_files_n_table(target_table: str):
             db = f'{copy_session}_{target_table}'
             csv_file = f'{csv_dir()}/{db}/*.csv'
-            csv_files: list[str] = find_files(pod, namespace, csv_file)
+            csv_files: list[str] = find_files(pod, namespace, csv_file, remote=True)
             if csv_files:
                 table = target_table
                 m = re.match(f'{csv_dir()}/{db}/(.*).csv', csv_files[0])
@@ -221,7 +222,7 @@ class ExportTableStatus:
 
             return csv_files, target_table
 
-        m = re.match(f'{log_prefix()}-{copy_session}_(.*?)\.(.*?)\.log(.*)', log_file)
+        m = re.match(f'{export_log_dir()}/{copy_session}_(.*?)\.(.*?)\.log(.*)', log_file)
         if m:
             keyspace = m.group(1)
             target_table = m.group(2)
@@ -234,7 +235,8 @@ class ExportTableStatus:
 
             # 4 rows exported to 1 files in 0 day, 0 hour, 0 minute, and 1.335 seconds.
             pattern = 'rows exported to'
-            r: PodExecResult = CassandraNodes.exec(pod, namespace, f"grep '{pattern}' {log_file}", show_out=Config().is_debug(), shell='bash')
+            r: ExecResult = local_exec(['grep', pattern, log_file], show_out=Config().is_debug())
+
             if r.exit_code() == 0:
                 csv_files, table = get_csv_files_n_table(target_table)
                 if csv_files:
@@ -246,17 +248,26 @@ class ExportTableStatus:
 
         return ExportTableStatus(None, None, 'unknown')
 
-def csv_dir():
-    return Config().get('export.csv_dir', '/c3/cassandra/tmp')
-
-def find_files(pod: str, namespace: str, pattern: str, mmin: int = 0):
-    if mmin:
-        r = CassandraNodes.exec(pod, namespace, f'find {pattern} -mmin -{mmin}', show_out=Config().is_debug(), shell='bash')
+def find_files(pod: str, namespace: str, pattern: str, mmin: int = 0, remote = False):
+    stdout = ''
+    if not remote:
+        # find . -maxdepth 1 -type f -name '*'
+        dir = os.path.dirname(pattern)
+        base = os.path.basename(pattern)
+        cmd = ['find', dir, '-name', base]
+        if mmin:
+                cmd += ['-mmin',  f'-{mmin}']
+
+        stdout = local_exec(cmd, show_out=Config().is_debug()).stdout
     else:
-        r = CassandraNodes.exec(pod, namespace, f'find {pattern}', show_out=Config().is_debug(), shell='bash')
+        cmd = f'find {pattern}'
+        if mmin:
+                cmd = f'{cmd} -mmin -{mmin}'
+
+        stdout = CassandraNodes.exec(pod, namespace, cmd, show_out=Config().is_debug(), shell='bash').stdout
 
     log_files = []
-    for line in r.stdout.split('\n'):
+    for line in stdout.split('\n'):
         line = line.strip(' \r')
         if line:
             log_files.append(line)
@@ -341,4 +352,15 @@ class PodPushHandler:
         return False
 
 def state_with_pod(state: ReplState, pod: str = None):
-    return PodPushHandler(state, pod=pod)
+    return PodPushHandler(state, pod=pod)
+
+def os_system_exec(cmd: str, show_out = False):
+    if show_out: log2(cmd)
+
+    os.system(cmd)
+
+def csv_dir():
+    return Config().get('export.csv_dir', '/c3/cassandra/tmp')
+
+def export_log_dir():
+    return creating_dir(Config().get('export.log-dir', '/tmp/qing-db/q/export/logs'))

adam/commands/postgres/postgres_databases.py

@@ -145,7 +145,7 @@ class PostgresDatabases:
 
             r = Pods.exec(pod_name, container_name, state.namespace, cmd, show_out=show_out, backgrounded=backgrounded, env_prefix=env_prefix)
             if r and r.log_file:
-                ReplSession().append_history(f'@{r.pod} cat {r.log_file}')
+                ReplSession().append_history(f':cat {r.log_file}')
 
             return r
 

adam/commands/postgres/postgres_ls.py

@@ -1,5 +1,5 @@
 from adam.commands.command import Command
-from adam.commands.ls import Ls
+from adam.commands.fs.ls import Ls
 from adam.repl_state import ReplState, RequiredState
 
 class PostgresLs(Command):

adam/commands/postgres/utils_postgres.py

@@ -1,6 +1,7 @@
 import functools
 
 from adam.commands.postgres.postgres_databases import PostgresDatabases, pg_path
+from adam.config import Config
 from adam.repl_state import ReplState
 from adam.utils import log2, wait_log
 from adam.utils_k8s.pods import Pods
@@ -62,7 +63,7 @@ class PostgresPodService:
         if isinstance(args, list):
             query = ' '.join(args)
 
-        PostgresDatabases.run_sql(state, query, backgrounded=backgrounded)
+        PostgresDatabases.run_sql(state, query, show_out=Config().is_debug(), backgrounded=backgrounded)
 
 class PostgresExecHandler:
     def __init__(self, state: ReplState, backgrounded=False):

adam/commands/show/show_cassandra_status.py

@@ -8,13 +8,14 @@ from adam.checks.gossip import Gossip
 from adam.columns.columns import Columns
 from adam.commands import extract_options, extract_trailing_options
 from adam.commands.command import Command
+from adam.commands.commands_utils import write_to_kaqing_log_file
 from adam.commands.cql.utils_cql import cassandra
 from adam.config import Config
 from adam.repl_session import ReplSession
 from adam.utils_issues import IssuesUtils
 from adam.utils_k8s.statefulsets import StatefulSets
 from adam.repl_state import ReplState, RequiredState
-from adam.utils import SORT, tabulize, log2, log_exc
+from adam.utils import SORT, log_dir, tabulize, log2, log_exc
 from adam.checks.status import parse_nodetool_status
 
 class ShowCassandraStatus(Command):
@@ -105,15 +106,7 @@ class ShowCassandraStatus(Command):
         r = tabulize(status, lambda s: ','.join([c.host_value(check_results, s) for c in columns]), header=header, separator=',', sorted=SORT, to = 0 if backgrounded else 1)
 
         if backgrounded:
-            log_prefix = Config().get('log-prefix', '/tmp/qing')
-            log_file = f'{log_prefix}-{datetime.now().strftime("%d%H%M%S")}.log'
-
-            with open(log_file, 'w') as f:
-                f.write(r)
-
-            ReplSession().append_history(f':sh cat {log_file}')
-
-            r = log_file
+            r = write_to_kaqing_log_file(r)
 
         IssuesUtils.show(check_results)
 

adam/commands/show/show_processes.py

@@ -37,7 +37,7 @@ class ShowProcesses(Command):
                             header = Config().get('processes-qing.header', 'POD_NAME,Q_CPU/TOTAL,MEM/LIMIT')
 
                     with cassandra(state) as pods:
-                        pods.display_table(cols, header, show_out=show_out, backgrounded=backgrounded)
+                        pods.display_table(cols, header, show_out=show_out, backgrounded=backgrounded, msg='Checking processes')
 
                     return state
 

adam/commands/show/show_storage.py

@@ -3,6 +3,7 @@ from adam.commands.command import Command
 from adam.commands.cql.utils_cql import cassandra
 from adam.config import Config
 from adam.repl_state import ReplState, RequiredState
+from adam.utils import ing
 
 class ShowStorage(Command):
     COMMAND = 'show storage'
@@ -32,7 +33,7 @@ class ShowStorage(Command):
                     cols = Config().get('storage.columns', 'pod,volume_root,volume_cassandra,snapshots,data,compactions')
                     header = Config().get('storage.header', 'POD_NAME,VOLUME /,VOLUME CASS,SNAPSHOTS,DATA,COMPACTIONS')
                     with cassandra(state) as pods:
-                        pods.display_table(cols, header, show_out=show_out, backgrounded=backgrounded)
+                        pods.display_table(cols, header, show_out=show_out, backgrounded=backgrounded, msg='Checking storage')
 
                     return state
 

adam/embedded_params.py

@@ -1,2 +1,2 @@
 def config():
-    return {'app': {'console-endpoint': 'https://{host}/{env}/{app}/static/console/index.html', 'container-name': 'c3-server', 'cr': {'cluster-regex': '(.*?-.*?)-.*', 'group': 'ops.c3.ai', 'v': 'v2', 'plural': 'c3cassandras'}, 'label': 'c3__app_id-0', 'login': {'admin-group': '{host}/C3.ClusterAdmin', 'ingress': '{app_id}-k8singr-appleader-001', 'timeout': 5, 'session-check-url': 'https://{host}/{env}/{app}/api/8/C3/userSessionToken', 'cache-creds': True, 'cache-username': True, 'url': 'https://{host}/{env}/{app}', 'another': "You're logged in to {has}. However, for this app, you need to log in to {need}.", 'token-server-url': 'http://localhost:{port}', 'password-max-length': 128}, 'strip': '0'}, 'audit': {'endpoint': 'https://4psvtaxlcb.execute-api.us-west-2.amazonaws.com/prod/', 'workers': 3, 'timeout': 10, 'log-audit-queries': False, 'athena': {'auto-repair': {'elapsed_hours': 12}, 'region': 'us-west-2', 'catalog': 'AwsDataCatalog', 'database': 'audit', 'repair-partition-tables': 'audit', 'output': 's3://s3.ops--audit/ddl/results', 'repair-cluster-tables': 'cluster'}, 'queries': {'last10': "SELECT * FROM audit\nWHERE drive <> 'z' and ({date_condition})\nORDER BY ts DESC LIMIT {limit}", 'slow10': "SELECT * FROM audit\nWHERE drive <> 'z' and ({date_condition})\nORDER BY CAST(duration AS REAL) DESC LIMIT {limit}", 'top10': "SELECT min(c) AS cluster, line, COUNT(*) AS cnt, avg(CAST(duration AS REAL)) AS duration\nFROM audit WHERE drive <> 'z' and ({date_condition})\nGROUP BY line ORDER BY cnt DESC LIMIT {limit}"}}, 'bash': {'workers': 32}, 'cassandra': {'service-name': 'all-pods-service'}, 'cql': {'workers': 32, 'samples': 3, 'secret': {'cluster-regex': '(.*?-.*?)-.*', 'name': '{cluster}-superuser', 'password-item': 'password'}, 'alter-tables': {'excludes': 'system_auth,system_traces,reaper_db,system_distributed,system_views,system,system_schema,system_virtual_schema', 'gc-grace-periods': '3600,86400,864000,7776000', 'batching': True}}, 'checks': {'compactions-threshold': 250, 'cpu-busy-threshold': 98.0, 'cpu-threshold': 0.0, 'cassandra-data-path': '/c3/cassandra', 'root-disk-threshold': 50, 'cassandra-disk-threshold': 50, 'snapshot-size-cmd': "ls /c3/cassandra/data/data/*/*/snapshots | grep snapshots | sed 's/:$//g' | xargs -I {} du -sk {} | awk '{print $1}' | awk '{s+=$1} END {print s}'", 'snapshot-size-threshold': '40G', 'table-sizes-cmd': "ls -Al /c3/cassandra/data/data/ | awk '{print $9}' | sed 's/\\^r//g' | xargs -I {} du -sk /c3/cassandra/data/data/{}"}, 'download': {'workers': 8}, 'export': {'workers': 8, 'csv_dir': '/c3/cassandra/tmp', 'column_counts_query': 'select id, count(id) as columns from {table} group by id order by columns desc limit 10', 'default-importer': 'sqlite', 'sqlite': {'workers': 8, 'columns': '<row-key>', 'local-db-dir': '/tmp/qing-db'}, 'athena': {'workers': 8, 'columns': '<keys>', 'bucket': 'c3.ops--qing'}, 'csv': {'workers': 8, 'columns': '<row-key>'}, 'log-prefix': '/tmp/qing'}, 'get-host-id': {'workers': 32}, 'idps': {'ad': {'email-pattern': '.*@c3.ai', 'uri': 'https://login.microsoftonline.com/53ad779a-93e7-485c-ba20-ac8290d7252b/oauth2/v2.0/authorize?response_type=id_token&response_mode=form_post&client_id=00ff94a8-6b0a-4715-98e0-95490012d818&scope=openid+email+profile&redirect_uri=https%3A%2F%2Fplat.c3ci.cloud%2Fc3%2Fc3%2Foidc%2Flogin&nonce={nonce}&state=EMPTY', 'jwks-uri': 'https://login.microsoftonline.com/common/discovery/keys', 'contact': 'Please contact ted.tran@c3.ai.', 'whitelist-file': '/kaqing/members'}, 'okta': {'default': True, 'email-pattern': '.*@c3iot.com', 'uri': 'https://c3energy.okta.com/oauth2/v1/authorize?response_type=id_token&response_mode=form_post&client_id={client_id}&scope=openid+email+profile+groups&redirect_uri=https%3A%2F%2F{host}%2Fc3%2Fc3%2Foidc%2Flogin&nonce={nonce}&state=EMPTY', 'jwks-uri': 'https://c3energy.okta.com/oauth2/v1/keys'}}, 'issues': {'workers': 32}, 'local-tmp-dir': '/tmp/qing-db', 'logs': {'path': '/c3/cassandra/logs/system.log'}, 'log-prefix': '/tmp/qing', 'nodetool': {'workers': 96, 'commands_in_line': 40, 'status': {'workers': 32, 'samples': 3, 'commands_in_line': 40}}, 'pg': {'name-pattern': '^{namespace}.*-k8spg-.*', 'excludes': '.helm., -admin-secret', 'agent': {'name': 'ops-pg-agent', 'just-in-time': False, 'timeout': 86400, 'image': 'seanahnsf/kaqing'}, 'default-db': 'postgres', 'default-schema': 'postgres', 'secret': {'endpoint-key': 'postgres-db-endpoint', 'port-key': 'postgres-db-port', 'username-key': 'postgres-admin-username', 'password-key': 'postgres-admin-password'}}, 'pod': {'name': 'ops', 'image': 'seanahnsf/kaqing-cloud', 'sa': {'name': 'ops', 'proto': 'c3', 'additional-cluster-roles': 'c3aiops-k8ssandra-operator'}, 'label-selector': 'run=ops'}, 'preview': {'rows': 10}, 'processes': {'columns': 'pod,cpu-metrics,mem', 'header': 'POD_NAME,M_CPU(USAGE/LIMIT),MEM/LIMIT'}, 'processes-qing': {'columns': 'pod,cpu,mem', 'header': 'POD_NAME,Q_CPU/TOTAL,MEM/LIMIT'}, 'reaper': {'service-name': 'reaper-service', 'port-forward': {'timeout': 86400, 'local-port': 9001}, 'abort-runs-batch': 10, 'show-runs-batch': 100, 'pod': {'cluster-regex': '(.*?-.*?-.*?-.*?)-.*', 'label-selector': 'k8ssandra.io/reaper={cluster}-reaper'}, 'secret': {'cluster-regex': '(.*?-.*?)-.*', 'name': '{cluster}-reaper-ui', 'password-item': 'password'}}, 'repair': {'log-path': '/home/cassrepair/logs/', 'image': 'ci-registry.c3iot.io/cloudops/cassrepair:2.0.14', 'secret': 'ciregistryc3iotio', 'env': {'interval': 24, 'timeout': 60, 'pr': False, 'runs': 1}}, 'repl': {'start-drive': 'c', 'a': {'auto-enter': 'c3/c3'}, 'c': {'auto-enter': 'cluster'}, 'x': {'auto-enter': 'latest'}, 'history': {'push-cat-log-file': True, 'push-cat-remote-log-file': True}, 'background-process': {'auto-nohup': True, 'via-sh': True}}, 'status': {'columns': 'status,address,load,tokens,owns,host_id,gossip,compactions', 'header': '--,Address,Load,Tokens,Owns,Host ID,GOSSIP,COMPACTIONS'}, 'storage': {'columns': 'pod,volume_root,volume_cassandra,snapshots,data,compactions', 'header': 'POD_NAME,VOLUME /,VOLUME CASS,SNAPSHOTS,DATA,COMPACTIONS'}, 'watch': {'auto': 'rollout', 'timeout': 3600, 'interval': 10}, 'auto-complete': {'c': {'tables': 'lazy'}, 'x': {'tables': 'lazy'}, 'cli': {'cp': 'jit'}, 'export': {'databases': 'jit'}, 'medusa': {'backups': 'jit'}, 'reaper': {'schedules': 'jit'}}, 'debug': False, 'debugs': {'complete': False, 'timings': False, 'exit-on-error': False}}
+    return {'app': {'console-endpoint': 'https://{host}/{env}/{app}/static/console/index.html', 'container-name': 'c3-server', 'cr': {'cluster-regex': '(.*?-.*?)-.*', 'group': 'ops.c3.ai', 'v': 'v2', 'plural': 'c3cassandras'}, 'label': 'c3__app_id-0', 'login': {'admin-group': '{host}/C3.ClusterAdmin', 'ingress': '{app_id}-k8singr-appleader-001', 'timeout': 5, 'session-check-url': 'https://{host}/{env}/{app}/api/8/C3/userSessionToken', 'cache-creds': True, 'cache-username': True, 'url': 'https://{host}/{env}/{app}', 'another': "You're logged in to {has}. However, for this app, you need to log in to {need}.", 'token-server-url': 'http://localhost:{port}', 'password-max-length': 128}, 'strip': '0'}, 'audit': {'endpoint': 'https://4psvtaxlcb.execute-api.us-west-2.amazonaws.com/prod/', 'workers': 3, 'timeout': 10, 'log-audit-queries': False, 'athena': {'auto-repair': {'elapsed_hours': 12}, 'region': 'us-west-2', 'catalog': 'AwsDataCatalog', 'database': 'audit', 'repair-partition-tables': 'audit', 'output': 's3://s3.ops--audit/ddl/results', 'repair-cluster-tables': 'cluster'}, 'queries': {'last10': "SELECT * FROM audit\nWHERE drive <> 'z' and ({date_condition})\nORDER BY ts DESC LIMIT {limit}", 'slow10': "SELECT * FROM audit\nWHERE drive <> 'z' and ({date_condition})\nORDER BY CAST(duration AS REAL) DESC LIMIT {limit}", 'top10': "SELECT min(c) AS cluster, line, COUNT(*) AS cnt, avg(CAST(duration AS REAL)) AS duration\nFROM audit WHERE drive <> 'z' and ({date_condition})\nGROUP BY line ORDER BY cnt DESC LIMIT {limit}"}}, 'bash': {'workers': 32}, 'cassandra': {'service-name': 'all-pods-service'}, 'cql': {'workers': 32, 'samples': 3, 'secret': {'cluster-regex': '(.*?-.*?)-.*', 'name': '{cluster}-superuser', 'password-item': 'password'}, 'alter-tables': {'excludes': 'system_auth,system_traces,reaper_db,system_distributed,system_views,system,system_schema,system_virtual_schema', 'gc-grace-periods': '3600,86400,864000,7776000', 'batching': True}}, 'checks': {'compactions-threshold': 250, 'cpu-busy-threshold': 98.0, 'cpu-threshold': 0.0, 'cassandra-data-path': '/c3/cassandra', 'root-disk-threshold': 50, 'cassandra-disk-threshold': 50, 'snapshot-size-cmd': "ls /c3/cassandra/data/data/*/*/snapshots | grep snapshots | sed 's/:$//g' | xargs -I {} du -sk {} | awk '{print $1}' | awk '{s+=$1} END {print s}'", 'snapshot-size-threshold': '40G', 'table-sizes-cmd': "ls -Al /c3/cassandra/data/data/ | awk '{print $9}' | sed 's/\\^r//g' | xargs -I {} du -sk /c3/cassandra/data/data/{}"}, 'download': {'workers': 8}, 'export': {'workers': 8, 'csv_dir': '/c3/cassandra/tmp', 'column_counts_query': 'select id, count(id) as columns from {table} group by id order by columns desc limit 10', 'default-importer': 'sqlite', 'sqlite': {'workers': 8, 'columns': '<row-key>', 'local-db-dir': '/tmp/qing-db/q/export/db'}, 'athena': {'workers': 8, 'columns': '<keys>', 'bucket': 'c3.ops--qing'}, 'csv': {'workers': 8, 'columns': '<row-key>'}, 'log-dir': '/tmp/qing-db/q/export/logs'}, 'get-host-id': {'workers': 32}, 'idps': {'ad': {'email-pattern': '.*@c3.ai', 'uri': 'https://login.microsoftonline.com/53ad779a-93e7-485c-ba20-ac8290d7252b/oauth2/v2.0/authorize?response_type=id_token&response_mode=form_post&client_id=00ff94a8-6b0a-4715-98e0-95490012d818&scope=openid+email+profile&redirect_uri=https%3A%2F%2Fplat.c3ci.cloud%2Fc3%2Fc3%2Foidc%2Flogin&nonce={nonce}&state=EMPTY', 'jwks-uri': 'https://login.microsoftonline.com/common/discovery/keys', 'contact': 'Please contact ted.tran@c3.ai.', 'whitelist-file': '/kaqing/members'}, 'okta': {'default': True, 'email-pattern': '.*@c3iot.com', 'uri': 'https://c3energy.okta.com/oauth2/v1/authorize?response_type=id_token&response_mode=form_post&client_id={client_id}&scope=openid+email+profile+groups&redirect_uri=https%3A%2F%2F{host}%2Fc3%2Fc3%2Foidc%2Flogin&nonce={nonce}&state=EMPTY', 'jwks-uri': 'https://c3energy.okta.com/oauth2/v1/keys'}}, 'issues': {'workers': 32}, 'local-qing-dir': '/tmp/qing-db/q', 'local-downloads-dir': '/tmp/qing-db/q/downloads', 'logs': {'path': '/c3/cassandra/logs/system.log'}, 'log-dir': '/tmp/qing-db/q/logs', 'nodetool': {'workers': 96, 'commands_in_line': 40, 'status': {'workers': 32, 'samples': 3, 'commands_in_line': 40}}, 'pg': {'name-pattern': '^{namespace}.*-k8spg-.*', 'excludes': '.helm., -admin-secret', 'agent': {'name': 'ops-pg-agent', 'just-in-time': False, 'timeout': 86400, 'image': 'seanahnsf/kaqing'}, 'default-db': 'postgres', 'default-schema': 'postgres', 'secret': {'endpoint-key': 'postgres-db-endpoint', 'port-key': 'postgres-db-port', 'username-key': 'postgres-admin-username', 'password-key': 'postgres-admin-password'}}, 'pod': {'name': 'ops', 'image': 'seanahnsf/kaqing-cloud', 'sa': {'name': 'ops', 'proto': 'c3', 'additional-cluster-roles': 'c3aiops-k8ssandra-operator'}, 'label-selector': 'run=ops'}, 'preview': {'rows': 10}, 'processes': {'columns': 'pod,cpu-metrics,mem', 'header': 'POD_NAME,M_CPU(USAGE/LIMIT),MEM/LIMIT'}, 'processes-qing': {'columns': 'pod,cpu,mem', 'header': 'POD_NAME,Q_CPU/TOTAL,MEM/LIMIT'}, 'reaper': {'service-name': 'reaper-service', 'port-forward': {'timeout': 86400, 'local-port': 9001}, 'abort-runs-batch': 10, 'show-runs-batch': 100, 'pod': {'cluster-regex': '(.*?-.*?-.*?-.*?)-.*', 'label-selector': 'k8ssandra.io/reaper={cluster}-reaper'}, 'secret': {'cluster-regex': '(.*?-.*?)-.*', 'name': '{cluster}-reaper-ui', 'password-item': 'password'}}, 'repair': {'log-path': '/home/cassrepair/logs/', 'image': 'ci-registry.c3iot.io/cloudops/cassrepair:2.0.14', 'secret': 'ciregistryc3iotio', 'env': {'interval': 24, 'timeout': 60, 'pr': False, 'runs': 1}}, 'repl': {'start-drive': 'c', 'a': {'auto-enter': 'c3/c3'}, 'c': {'auto-enter': 'cluster'}, 'x': {'auto-enter': 'latest'}, 'history': {'push-cat-log-file': True, 'push-cat-remote-log-file': True}, 'background-process': {'auto-nohup': True}}, 'status': {'columns': 'status,address,load,tokens,owns,host_id,gossip,compactions', 'header': '--,Address,Load,Tokens,Owns,Host ID,GOSSIP,COMPACTIONS'}, 'storage': {'columns': 'pod,volume_root,volume_cassandra,snapshots,data,compactions', 'header': 'POD_NAME,VOLUME /,VOLUME CASS,SNAPSHOTS,DATA,COMPACTIONS'}, 'watch': {'auto': 'rollout', 'timeout': 3600, 'interval': 10}, 'auto-complete': {'c': {'tables': 'lazy'}, 'x': {'tables': 'lazy'}, 'cli': {'cp': 'jit'}, 'export': {'databases': 'jit'}, 'medusa': {'backups': 'jit'}, 'reaper': {'schedules': 'jit'}}, 'debug': False, 'debugs': {'complete': False, 'timings': False, 'exit-on-error': False}}

adam/repl_commands.py

@@ -1,16 +1,16 @@
-from adam.commands.alter_tables import AlterTables
 from adam.commands.app.app import App
 from adam.commands.app.app_ping import AppPing
 from adam.commands.app.show_app_actions import ShowAppActions
 from adam.commands.app.show_app_id import ShowAppId
 from adam.commands.app.show_app_queues import ShowAppQueues
 from adam.commands.audit.audit import Audit
-from adam.commands.cat import Cat
+from adam.commands.fs.cat import Cat
 from adam.commands.code import Code
+from adam.commands.cql.alter_tables import AlterTables
 from adam.commands.debug.debug import Debug
-from adam.commands.debug.debug_timings import DebugTimings
 from adam.commands.download_cassandra_log import DownloadCassandraLog
-from adam.commands.download_file import DownloadFile
+from adam.commands.fs.cat_local import CatLocal
+from adam.commands.fs.download_file import DownloadFile
 from adam.commands.deploy.code_start import CodeStart
 from adam.commands.deploy.code_stop import CodeStop
 from adam.commands.deploy.deploy import Deploy
@@ -41,16 +41,17 @@ from adam.commands.export.show_column_counts import ShowColumnCounts
 from adam.commands.export.show_export_databases import ShowExportDatabases
 from adam.commands.export.show_export_session import ShowExportSession
 from adam.commands.export.show_export_sessions import ShowExportSessions
-from adam.commands.find_files import FindLocalFiles
-from adam.commands.find_processes import FindProcesses
-from adam.commands.head import Head
+from adam.commands.fs.find_files import FindLocalFiles
+from adam.commands.fs.find_processes import FindProcesses
+from adam.commands.fs.head import Head
+from adam.commands.fs.ls_local import LsLocal
 from adam.commands.kubectl import Kubectl
 from adam.commands.restart_cluster import RestartCluster
 from adam.commands.restart_node import RestartNode
-from adam.commands.shell import Shell
+from adam.commands.fs.shell import Shell
 from adam.commands.clipboard_copy import ClipboardCopy
 from adam.commands.bash.bash import Bash
-from adam.commands.cd import Cd
+from adam.commands.fs.cd import Cd
 from adam.commands.check import Check
 from adam.commands.command import Command
 from adam.commands.cql.cqlsh import Cqlsh
@@ -58,7 +59,7 @@ from adam.commands.exit import Exit
 from adam.commands.medusa.medusa import Medusa
 from adam.commands.param_get import GetParam
 from adam.commands.issues import Issues
-from adam.commands.ls import Ls
+from adam.commands.fs.ls import Ls
 from adam.commands.nodetool import NodeTool
 from adam.commands.postgres.postgres import Postgres, PostgresPg
 from adam.commands.preview_table import PreviewTable
@@ -104,8 +105,8 @@ class ReplCommands:
         return deduped
 
     def navigation() -> list[Command]:
-        return [Ls(), PreviewTable(), DeviceApp(), DevicePostgres(), DeviceCass(), DeviceAuditLog(), DeviceExport(),
-                Cd(), Cat(), Head(), DownloadFile(), FindLocalFiles(), FindProcesses(), Pwd(), ClipboardCopy(),
+        return [Ls(), LsLocal(), PreviewTable(), DeviceApp(), DevicePostgres(), DeviceCass(), DeviceAuditLog(), DeviceExport(),
+                Cd(), Cat(), CatLocal(), Head(), DownloadFile(), FindLocalFiles(), FindProcesses(), Pwd(), ClipboardCopy(),
                 GetParam(), SetParam(), ShowParams(), ShowKubectlCommands(), ShowLogin(), ShowAdam(), ShowHost()]
 
     def cassandra_ops() -> list[Command]:

adam/sso/cred_cache.py

@@ -2,8 +2,7 @@ import os
 from pathlib import Path
 from dotenv import load_dotenv
 
-from adam.config import Config
-from adam.utils import debug, log_exc
+from adam.utils import creating_dir, debug, log_exc
 from adam.utils_k8s.kube_context import KubeContext
 
 class CredCache:
@@ -15,7 +14,7 @@ class CredCache:
 
     def __init__(self):
         if not hasattr(self, 'env_f'):
-            self.dir = f'{Path.home()}/.kaqing'
+            self.dir = creating_dir(f'{Path.home()}/.kaqing')
             self.env_f = f'{self.dir}/.credentials'
             # immutable - cannot reload with different file content
             load_dotenv(dotenv_path=self.env_f)
@@ -44,8 +43,6 @@ class CredCache:
             updated.append(f'IDP_PASSWORD={password}')
 
         if updated:
-            if not os.path.exists(self.env_f):
-                os.makedirs(self.dir, exist_ok=True)
             with open(self.env_f, 'w') as file:
                 file.write('\n'.join(updated))