kaqing 2.0.200__py3-none-any.whl → 2.0.213__py3-none-any.whl

adam/commands/export/importer_athena.py

@@ -19,9 +19,16 @@ class AthenaImporter(Importer):
     def prefix(self):
         return 'e'
 
-    def import_from_csv(self, state: ReplState, from_session: str,
-                        keyspace: str, table: str, target_table: str, columns: str,
-                        multi_tables = True, create_db = False):
+    def import_from_csv(self,
+                        state: ReplState,
+                        from_session: str,
+                        keyspace: str,
+                        table: str,
+                        target_table: str,
+                        columns: str,
+                        multi_tables = True,
+                        create_db = False,
+                        job_log: str = None):
         csv_file = self.csv_file(from_session, table, target_table)
         pod = state.pod
         namespace = state.namespace
@@ -47,12 +54,14 @@ class AthenaImporter(Importer):
             return to, to_session
         finally:
             if succeeded:
-                self.remove_csv(state, from_session, table, target_table, multi_tables)
+                self.remove_csv(state, from_session, table, target_table, multi_tables, job_log=job_log)
                 Athena.clear_cache()
 
-                if not multi_tables:
+                if multi_tables:
+                    log2(f'[{to_session}] {keyspace}.{target_table} OK', file=job_log)
+                else:
                     with export_db(state) as dbs:
-                        dbs.sql(f'select * from {database}.{target_table} limit 10')
+                        dbs.sql(f'select * from {keyspace}.{target_table} limit 10', backgrounded=True, export_log=job_log)
 
     def import_from_local_csv(self, state: ReplState,
                         keyspace: str, table: str, csv_file: str, multi_tables = True, create_db = False):

adam/commands/export/importer_sqlite.py

@@ -1,9 +1,10 @@
+from typing import TextIO
 import pandas
 
 from adam.commands.export.export_databases import export_db
 from adam.commands.export.importer import Importer
 from adam.repl_state import ReplState
-from adam.utils import GeneratorStream, bytes_generator_from_file, ing
+from adam.utils import GeneratorStream, bytes_generator_from_file, ing, log2
 from adam.utils_k8s.pods import Pods
 from adam.utils_sqlite import SQLite, sqlite
 
@@ -11,9 +12,17 @@ class SqliteImporter(Importer):
     def prefix(self):
         return 's'
 
-    def import_from_csv(self, state: ReplState, from_session: str,
-                        keyspace: str, table: str, target_table: str, columns: str,
-                        multi_tables = True, create_db = False):
+    def import_from_csv(self,
+                        state: ReplState,
+                        from_session: str,
+                        keyspace: str,
+                        table: str,
+                        target_table: str,
+                        columns: str,
+                        multi_tables = True,
+                        create_db = False,
+                        job_log: str = None):
+
         csv_file = self.csv_file(from_session, table, target_table)
         pod = state.pod
         namespace = state.namespace
@@ -21,7 +30,7 @@ class SqliteImporter(Importer):
 
         succeeded = False
         try:
-            with ing(f'[{to_session}] Uploading to Sqlite', suppress_log=multi_tables):
+            with ing(f'[{to_session}] Uploading to Sqlite', suppress_log=multi_tables, job_log=job_log):
                 # create a connection to single keyspace
                 with sqlite(to_session, keyspace) as conn:
                     bytes = Pods.read_file(pod, 'cassandra', namespace, csv_file)
@@ -35,12 +44,14 @@ class SqliteImporter(Importer):
             return to, to_session
         finally:
             if succeeded:
-                self.remove_csv(state, from_session, table, target_table, multi_tables)
+                self.remove_csv(state, from_session, table, target_table, multi_tables, job_log=job_log)
                 SQLite.clear_cache()
 
-                if not multi_tables:
+                if multi_tables:
+                    log2(f'[{to_session}] {keyspace}.{target_table} OK', file=job_log)
+                else:
                     with export_db(state) as dbs:
-                        dbs.sql(f'select * from {keyspace}.{target_table} limit 10')
+                        dbs.sql(f'select * from {keyspace}.{target_table} limit 10', backgrounded=True, export_log=job_log)
 
     def import_from_local_csv(self, state: ReplState,
                         keyspace: str, table: str, csv_file: str, multi_tables = True, create_db = False):

adam/commands/export/utils_export.py

@@ -1,12 +1,13 @@
 import io
+import os
 import re
 
 from adam.config import Config
-from adam.pod_exec_result import PodExecResult
+from adam.utils import ExecResult, creating_dir, log2
 from adam.repl_state import ReplState
 from adam.utils_k8s.cassandra_nodes import CassandraNodes
-from adam.utils_k8s.pods import log_prefix
 from adam.utils_k8s.statefulsets import StatefulSets
+from adam.utils_local import local_exec
 
 class ImportSpec:
     def __init__(self, table_name: str, session: str = None, files: list[str] = None, importer: str = None):
@@ -196,7 +197,7 @@ class ExportTableStatus:
         statuses: list[ExportTableStatus] = []
 
         status_in_whole = 'done'
-        log_files: list[str] = find_files(pod, namespace, f'{log_prefix()}-{export_session}_*.log*')
+        log_files: list[str] = find_files(pod, namespace, f'{export_log_dir()}/{export_session}_*.log*')
 
         for log_file in log_files:
             status: ExportTableStatus = ExportTableStatus.from_log_file(pod, namespace, export_session, log_file)
@@ -211,7 +212,7 @@ class ExportTableStatus:
         def get_csv_files_n_table(target_table: str):
             db = f'{copy_session}_{target_table}'
             csv_file = f'{csv_dir()}/{db}/*.csv'
-            csv_files: list[str] = find_files(pod, namespace, csv_file)
+            csv_files: list[str] = find_files(pod, namespace, csv_file, remote=True)
             if csv_files:
                 table = target_table
                 m = re.match(f'{csv_dir()}/{db}/(.*).csv', csv_files[0])
@@ -221,7 +222,7 @@ class ExportTableStatus:
 
             return csv_files, target_table
 
-        m = re.match(f'{log_prefix()}-{copy_session}_(.*?)\.(.*?)\.log(.*)', log_file)
+        m = re.match(f'{export_log_dir()}/{copy_session}_(.*?)\.(.*?)\.log(.*)', log_file)
         if m:
             keyspace = m.group(1)
             target_table = m.group(2)
@@ -234,7 +235,8 @@ class ExportTableStatus:
 
             # 4 rows exported to 1 files in 0 day, 0 hour, 0 minute, and 1.335 seconds.
             pattern = 'rows exported to'
-            r: PodExecResult = CassandraNodes.exec(pod, namespace, f"grep '{pattern}' {log_file}", show_out=Config().is_debug(), shell='bash')
+            r: ExecResult = local_exec(['grep', pattern, log_file], show_out=Config().is_debug())
+
             if r.exit_code() == 0:
                 csv_files, table = get_csv_files_n_table(target_table)
                 if csv_files:
@@ -246,17 +248,26 @@ class ExportTableStatus:
 
         return ExportTableStatus(None, None, 'unknown')
 
-def csv_dir():
-    return Config().get('export.csv_dir', '/c3/cassandra/tmp')
-
-def find_files(pod: str, namespace: str, pattern: str, mmin: int = 0):
-    if mmin:
-        r = CassandraNodes.exec(pod, namespace, f'find {pattern} -mmin -{mmin}', show_out=Config().is_debug(), shell='bash')
+def find_files(pod: str, namespace: str, pattern: str, mmin: int = 0, remote = False):
+    stdout = ''
+    if not remote:
+        # find . -maxdepth 1 -type f -name '*'
+        dir = os.path.dirname(pattern)
+        base = os.path.basename(pattern)
+        cmd = ['find', dir, '-name', base]
+        if mmin:
+                cmd += ['-mmin',  f'-{mmin}']
+
+        stdout = local_exec(cmd, show_out=Config().is_debug()).stdout
     else:
-        r = CassandraNodes.exec(pod, namespace, f'find {pattern}', show_out=Config().is_debug(), shell='bash')
+        cmd = f'find {pattern}'
+        if mmin:
+                cmd = f'{cmd} -mmin -{mmin}'
+
+        stdout = CassandraNodes.exec(pod, namespace, cmd, show_out=Config().is_debug(), shell='bash').stdout
 
     log_files = []
-    for line in r.stdout.split('\n'):
+    for line in stdout.split('\n'):
         line = line.strip(' \r')
         if line:
             log_files.append(line)
@@ -341,4 +352,15 @@ class PodPushHandler:
         return False
 
 def state_with_pod(state: ReplState, pod: str = None):
-    return PodPushHandler(state, pod=pod)
+    return PodPushHandler(state, pod=pod)
+
+def os_system_exec(cmd: str, show_out = False):
+    if show_out: log2(cmd)
+
+    os.system(cmd)
+
+def csv_dir():
+    return Config().get('export.csv_dir', '/c3/cassandra/tmp')
+
+def export_log_dir():
+    return creating_dir(Config().get('export.log-dir', '/tmp/qing-db/q/export/logs'))

adam/commands/{os/cat.py → fs/cat_local.py}

@@ -1,14 +1,17 @@
+import os
+
 from adam.commands import validate_args
 from adam.commands.command import Command
-from adam.commands.devices.devices import Devices
 from adam.repl_state import ReplState, RequiredState
+from adam.utils import log2
+from adam.utils_local import find_local_files
 
-class Cat(Command):
-    COMMAND = 'cat'
+class CatLocal(Command):
+    COMMAND = ':cat'
 
     # the singleton pattern
     def __new__(cls, *args, **kwargs):
-        if not hasattr(cls, 'instance'): cls.instance = super(Cat, cls).__new__(cls)
+        if not hasattr(cls, 'instance'): cls.instance = super(CatLocal, cls).__new__(cls)
 
         return cls.instance
 
@@ -16,7 +19,7 @@ class Cat(Command):
         super().__init__(successor)
 
     def command(self):
-        return Cat.COMMAND
+        return CatLocal.COMMAND
 
     def required(self):
         return [RequiredState.CLUSTER_OR_POD, RequiredState.APP_APP, ReplState.P]
@@ -26,11 +29,14 @@ class Cat(Command):
             return super().run(cmd, state)
 
         with self.validate(args, state) as (args, state):
-            with validate_args(args, state, name='file'):
-                return Devices.of(state).bash(state, state, cmd.split(' '))
+            with validate_args(args, state, name='file') as args:
+                os.system(f'cat {args}')
+                log2()
+
+                return state
 
     def completion(self, state: ReplState):
-        return super().completion(state, lambda: {f: None for f in Devices.of(state).files(state)}, pods=Devices.of(state).pods(state, '-'), auto='jit')
+        return super().completion(state, lambda: {n: None for n in find_local_files(file_type='f', max_depth=1)}, auto='jit')
 
     def help(self, _: ReplState):
-        return f'{Cat.COMMAND} file [&]\t run cat command on the pod'
+        return f'{CatLocal.COMMAND} file\t run cat command on local system'

adam/commands/{os → fs}/download_file.py

@@ -2,7 +2,7 @@ from adam.commands import validate_args
 from adam.commands.command import Command
 from adam.commands.devices.devices import Devices
 from adam.config import Config
-from adam.pod_exec_result import PodExecResult
+from adam.utils_k8s.pod_exec_result import PodExecResult
 from adam.repl_state import ReplState, RequiredState
 from adam.utils import log2
 from adam.utils_k8s.pods import Pods

adam/commands/{os → fs}/find_files.py

@@ -3,10 +3,10 @@ import os
 from adam.commands.command import Command
 from adam.repl_state import ReplState
 from adam.utils import log2
-from adam.utils_local import local_tmp_dir
+from adam.utils_local import local_qing_dir
 
 class FindLocalFiles(Command):
-    COMMAND = 'find local'
+    COMMAND = ':find file'
 
     # the singleton pattern
     def __new__(cls, *args, **kwargs):
@@ -28,9 +28,9 @@ class FindLocalFiles(Command):
             cmd = 'find'
 
             if not args:
-                cmd = f'find {local_tmp_dir()}'
+                cmd = f'find {local_qing_dir()}'
             elif len(args) == 1:
-                cmd = f"find {local_tmp_dir()} -name '{args[0]}'"
+                cmd = f"find {local_qing_dir()} -name '{args[0]}'"
             else:
                 new_args = [f"'{arg}'" if '*' in arg else arg for arg in args]
                 cmd = 'find ' + ' '.join(new_args)

adam/commands/{shell.py → fs/ls_local.py}

@@ -1,16 +1,16 @@
 import os
 
-from adam.commands import validate_args
 from adam.commands.command import Command
 from adam.repl_state import ReplState
 from adam.utils import log2
+from adam.utils_local import local_qing_dir
 
-class Shell(Command):
-    COMMAND = ':sh'
+class LsLocal(Command):
+    COMMAND = ':ls'
 
     # the singleton pattern
     def __new__(cls, *args, **kwargs):
-        if not hasattr(cls, 'instance'): cls.instance = super(Shell, cls).__new__(cls)
+        if not hasattr(cls, 'instance'): cls.instance = super(LsLocal, cls).__new__(cls)
 
         return cls.instance
 
@@ -18,19 +18,18 @@ class Shell(Command):
         super().__init__(successor)
 
     def command(self):
-        return Shell.COMMAND
+        return LsLocal.COMMAND
 
     def run(self, cmd: str, state: ReplState):
         if not(args := self.args(cmd)):
             return super().run(cmd, state)
 
-        with self.validate(args, state) as (args, _):
-            with validate_args(args, state, at_least=0) as args_str:
-                if args_str:
-                    os.system(args_str)
-                    log2()
-                else:
-                    os.system('QING_DROPPED=true bash')
+        with self.validate(args, state) as (args, state):
+            if args:
+                os.system(f'ls {args}')
+            else:
+                os.system(f'ls {local_qing_dir()}')
+            log2()
 
             return state
 
@@ -38,4 +37,4 @@ class Shell(Command):
         return super().completion(state)
 
     def help(self, _: ReplState):
-        return f'{Shell.COMMAND}\t drop down to shell'
+        return f'{LsLocal.COMMAND} [dir]\t list files on local system'

adam/commands/fs/rm.py

@@ -0,0 +1,18 @@
+from adam.commands.fs.rm_downloads import RmDownloads
+from adam.commands.fs.rm_logs import RmLogs
+from adam.commands.intermediate_command import IntermediateCommand
+
+class RmLocal(IntermediateCommand):
+    COMMAND = ':rm'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(RmLocal, cls).__new__(cls)
+
+        return cls.instance
+
+    def command(self):
+        return RmLocal.COMMAND
+
+    def cmd_list(self):
+        return [RmDownloads(), RmLogs()]

adam/commands/fs/rm_downloads.py

@@ -0,0 +1,39 @@
+import os
+
+from adam.commands.command import Command
+from adam.repl_state import ReplState
+from adam.utils import log2
+from adam.utils_local import local_downloads_dir
+
+class RmDownloads(Command):
+    COMMAND = ':rm downloads'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(RmDownloads, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return RmDownloads.COMMAND
+
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        with self.validate(args, state) as (args, state):
+            cmd = f'rm -rf {local_downloads_dir()}/*'
+            log2(cmd)
+            os.system(cmd)
+            log2()
+
+            return state
+
+    def completion(self, state: ReplState):
+        return super().completion(state)
+
+    def help(self, _: ReplState):
+        return f'{RmDownloads.COMMAND}\t remove all downloads files under {local_downloads_dir()}'

adam/commands/fs/rm_logs.py

@@ -0,0 +1,38 @@
+import os
+
+from adam.commands.command import Command
+from adam.repl_state import ReplState
+from adam.utils import log2, log_dir
+
+class RmLogs(Command):
+    COMMAND = ':rm logs'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(RmLogs, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return RmLogs.COMMAND
+
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        with self.validate(args, state) as (args, state):
+            cmd = f'rm -rf {log_dir()}/*'
+            log2(cmd)
+            os.system(cmd)
+            log2()
+
+            return state
+
+    def completion(self, state: ReplState):
+        return super().completion(state)
+
+    def help(self, _: ReplState):
+        return f'{RmLogs.COMMAND}\t remove all qing log files under {log_dir()}'

adam/commands/postgres/postgres_databases.py

@@ -145,7 +145,7 @@ class PostgresDatabases:
 
             r = Pods.exec(pod_name, container_name, state.namespace, cmd, show_out=show_out, backgrounded=backgrounded, env_prefix=env_prefix)
             if r and r.log_file:
-                ReplSession().append_history(f'@{r.pod} cat {r.log_file}')
+                ReplSession().append_history(f':cat {r.log_file}')
 
             return r
 

adam/commands/postgres/postgres_ls.py

@@ -1,5 +1,5 @@
 from adam.commands.command import Command
-from adam.commands.ls import Ls
+from adam.commands.fs.ls import Ls
 from adam.repl_state import ReplState, RequiredState
 
 class PostgresLs(Command):

adam/commands/postgres/utils_postgres.py

@@ -1,6 +1,7 @@
 import functools
 
 from adam.commands.postgres.postgres_databases import PostgresDatabases, pg_path
+from adam.config import Config
 from adam.repl_state import ReplState
 from adam.utils import log2, wait_log
 from adam.utils_k8s.pods import Pods
@@ -62,7 +63,7 @@ class PostgresPodService:
         if isinstance(args, list):
             query = ' '.join(args)
 
-        PostgresDatabases.run_sql(state, query, backgrounded=backgrounded)
+        PostgresDatabases.run_sql(state, query, show_out=Config().is_debug(), backgrounded=backgrounded)
 
 class PostgresExecHandler:
     def __init__(self, state: ReplState, backgrounded=False):

adam/commands/show/show_cassandra_status.py

@@ -8,13 +8,14 @@ from adam.checks.gossip import Gossip
 from adam.columns.columns import Columns
 from adam.commands import extract_options, extract_trailing_options
 from adam.commands.command import Command
+from adam.commands.commands_utils import write_to_kaqing_log_file
 from adam.commands.cql.utils_cql import cassandra
 from adam.config import Config
 from adam.repl_session import ReplSession
 from adam.utils_issues import IssuesUtils
 from adam.utils_k8s.statefulsets import StatefulSets
 from adam.repl_state import ReplState, RequiredState
-from adam.utils import SORT, tabulize, log2, log_exc
+from adam.utils import SORT, log_dir, tabulize, log2, log_exc
 from adam.checks.status import parse_nodetool_status
 
 class ShowCassandraStatus(Command):
@@ -105,15 +106,7 @@ class ShowCassandraStatus(Command):
         r = tabulize(status, lambda s: ','.join([c.host_value(check_results, s) for c in columns]), header=header, separator=',', sorted=SORT, to = 0 if backgrounded else 1)
 
         if backgrounded:
-            log_prefix = Config().get('log-prefix', '/tmp/qing')
-            log_file = f'{log_prefix}-{datetime.now().strftime("%d%H%M%S")}.log'
-
-            with open(log_file, 'w') as f:
-                f.write(r)
-
-            ReplSession().append_history(f':sh cat {log_file}')
-
-            r = log_file
+            r = write_to_kaqing_log_file(r)
 
         IssuesUtils.show(check_results)
 

adam/commands/show/show_processes.py

@@ -37,7 +37,7 @@ class ShowProcesses(Command):
                             header = Config().get('processes-qing.header', 'POD_NAME,Q_CPU/TOTAL,MEM/LIMIT')
 
                     with cassandra(state) as pods:
-                        pods.display_table(cols, header, show_out=show_out, backgrounded=backgrounded)
+                        pods.display_table(cols, header, show_out=show_out, backgrounded=backgrounded, msg='Checking processes')
 
                     return state
 

adam/commands/show/show_storage.py

@@ -3,6 +3,7 @@ from adam.commands.command import Command
 from adam.commands.cql.utils_cql import cassandra
 from adam.config import Config
 from adam.repl_state import ReplState, RequiredState
+from adam.utils import ing
 
 class ShowStorage(Command):
     COMMAND = 'show storage'
@@ -32,7 +33,7 @@ class ShowStorage(Command):
                     cols = Config().get('storage.columns', 'pod,volume_root,volume_cassandra,snapshots,data,compactions')
                     header = Config().get('storage.header', 'POD_NAME,VOLUME /,VOLUME CASS,SNAPSHOTS,DATA,COMPACTIONS')
                     with cassandra(state) as pods:
-                        pods.display_table(cols, header, show_out=show_out, backgrounded=backgrounded)
+                        pods.display_table(cols, header, show_out=show_out, backgrounded=backgrounded, msg='Checking storage')
 
                     return state
 

adam/embedded_params.py

@@ -1,2 +1,2 @@
 def config():
-    return {'app': {'console-endpoint': 'https://{host}/{env}/{app}/static/console/index.html', 'container-name': 'c3-server', 'cr': {'cluster-regex': '(.*?-.*?)-.*', 'group': 'ops.c3.ai', 'v': 'v2', 'plural': 'c3cassandras'}, 'label': 'c3__app_id-0', 'login': {'admin-group': '{host}/C3.ClusterAdmin', 'ingress': '{app_id}-k8singr-appleader-001', 'timeout': 5, 'session-check-url': 'https://{host}/{env}/{app}/api/8/C3/userSessionToken', 'cache-creds': True, 'cache-username': True, 'url': 'https://{host}/{env}/{app}', 'another': "You're logged in to {has}. However, for this app, you need to log in to {need}.", 'token-server-url': 'http://localhost:{port}', 'password-max-length': 128}, 'strip': '0'}, 'audit': {'endpoint': 'https://4psvtaxlcb.execute-api.us-west-2.amazonaws.com/prod/', 'workers': 3, 'timeout': 10, 'log-audit-queries': False, 'athena': {'auto-repair': {'elapsed_hours': 12}, 'region': 'us-west-2', 'catalog': 'AwsDataCatalog', 'database': 'audit', 'repair-partition-tables': 'audit', 'output': 's3://s3.ops--audit/ddl/results', 'repair-cluster-tables': 'cluster'}, 'queries': {'last10': "SELECT * FROM audit\nWHERE drive <> 'z' and ({date_condition})\nORDER BY ts DESC LIMIT {limit}", 'slow10': "SELECT * FROM audit\nWHERE drive <> 'z' and ({date_condition})\nORDER BY CAST(duration AS REAL) DESC LIMIT {limit}", 'top10': "SELECT min(c) AS cluster, line, COUNT(*) AS cnt, avg(CAST(duration AS REAL)) AS duration\nFROM audit WHERE drive <> 'z' and ({date_condition})\nGROUP BY line ORDER BY cnt DESC LIMIT {limit}"}}, 'bash': {'workers': 32}, 'cassandra': {'service-name': 'all-pods-service'}, 'cql': {'workers': 32, 'samples': 3, 'secret': {'cluster-regex': '(.*?-.*?)-.*', 'name': '{cluster}-superuser', 'password-item': 'password'}, 'alter-tables': {'excludes': 'system_auth,system_traces,reaper_db,system_distributed,system_views,system,system_schema,system_virtual_schema', 'gc-grace-periods': '3600,86400,864000,7776000', 'batching': True}}, 'checks': {'compactions-threshold': 250, 'cpu-busy-threshold': 98.0, 'cpu-threshold': 0.0, 'cassandra-data-path': '/c3/cassandra', 'root-disk-threshold': 50, 'cassandra-disk-threshold': 50, 'snapshot-size-cmd': "ls /c3/cassandra/data/data/*/*/snapshots | grep snapshots | sed 's/:$//g' | xargs -I {} du -sk {} | awk '{print $1}' | awk '{s+=$1} END {print s}'", 'snapshot-size-threshold': '40G', 'table-sizes-cmd': "ls -Al /c3/cassandra/data/data/ | awk '{print $9}' | sed 's/\\^r//g' | xargs -I {} du -sk /c3/cassandra/data/data/{}"}, 'download': {'workers': 8}, 'export': {'workers': 8, 'csv_dir': '/c3/cassandra/tmp', 'column_counts_query': 'select id, count(id) as columns from {table} group by id order by columns desc limit 10', 'default-importer': 'sqlite', 'sqlite': {'workers': 8, 'columns': '<row-key>', 'local-db-dir': '/tmp/qing-db'}, 'athena': {'workers': 8, 'columns': '<keys>', 'bucket': 'c3.ops--qing'}, 'csv': {'workers': 8, 'columns': '<row-key>'}, 'log-prefix': '/tmp/qing'}, 'get-host-id': {'workers': 32}, 'idps': {'ad': {'email-pattern': '.*@c3.ai', 'uri': 'https://login.microsoftonline.com/53ad779a-93e7-485c-ba20-ac8290d7252b/oauth2/v2.0/authorize?response_type=id_token&response_mode=form_post&client_id=00ff94a8-6b0a-4715-98e0-95490012d818&scope=openid+email+profile&redirect_uri=https%3A%2F%2Fplat.c3ci.cloud%2Fc3%2Fc3%2Foidc%2Flogin&nonce={nonce}&state=EMPTY', 'jwks-uri': 'https://login.microsoftonline.com/common/discovery/keys', 'contact': 'Please contact ted.tran@c3.ai.', 'whitelist-file': '/kaqing/members'}, 'okta': {'default': True, 'email-pattern': '.*@c3iot.com', 'uri': 'https://c3energy.okta.com/oauth2/v1/authorize?response_type=id_token&response_mode=form_post&client_id={client_id}&scope=openid+email+profile+groups&redirect_uri=https%3A%2F%2F{host}%2Fc3%2Fc3%2Foidc%2Flogin&nonce={nonce}&state=EMPTY', 'jwks-uri': 'https://c3energy.okta.com/oauth2/v1/keys'}}, 'issues': {'workers': 32}, 'local-tmp-dir': '/tmp/qing-db', 'logs': {'path': '/c3/cassandra/logs/system.log'}, 'log-prefix': '/tmp/qing', 'nodetool': {'workers': 96, 'commands_in_line': 40, 'status': {'workers': 32, 'samples': 3, 'commands_in_line': 40}}, 'pg': {'name-pattern': '^{namespace}.*-k8spg-.*', 'excludes': '.helm., -admin-secret', 'agent': {'name': 'ops-pg-agent', 'just-in-time': False, 'timeout': 86400, 'image': 'seanahnsf/kaqing'}, 'default-db': 'postgres', 'default-schema': 'postgres', 'secret': {'endpoint-key': 'postgres-db-endpoint', 'port-key': 'postgres-db-port', 'username-key': 'postgres-admin-username', 'password-key': 'postgres-admin-password'}}, 'pod': {'name': 'ops', 'image': 'seanahnsf/kaqing-cloud', 'sa': {'name': 'ops', 'proto': 'c3', 'additional-cluster-roles': 'c3aiops-k8ssandra-operator'}, 'label-selector': 'run=ops'}, 'preview': {'rows': 10}, 'processes': {'columns': 'pod,cpu-metrics,mem', 'header': 'POD_NAME,M_CPU(USAGE/LIMIT),MEM/LIMIT'}, 'processes-qing': {'columns': 'pod,cpu,mem', 'header': 'POD_NAME,Q_CPU/TOTAL,MEM/LIMIT'}, 'reaper': {'service-name': 'reaper-service', 'port-forward': {'timeout': 86400, 'local-port': 9001}, 'abort-runs-batch': 10, 'show-runs-batch': 100, 'pod': {'cluster-regex': '(.*?-.*?-.*?-.*?)-.*', 'label-selector': 'k8ssandra.io/reaper={cluster}-reaper'}, 'secret': {'cluster-regex': '(.*?-.*?)-.*', 'name': '{cluster}-reaper-ui', 'password-item': 'password'}}, 'repair': {'log-path': '/home/cassrepair/logs/', 'image': 'ci-registry.c3iot.io/cloudops/cassrepair:2.0.14', 'secret': 'ciregistryc3iotio', 'env': {'interval': 24, 'timeout': 60, 'pr': False, 'runs': 1}}, 'repl': {'start-drive': 'c', 'a': {'auto-enter': 'c3/c3'}, 'c': {'auto-enter': 'cluster'}, 'x': {'auto-enter': 'latest'}, 'history': {'push-cat-log-file': True, 'push-cat-remote-log-file': True}, 'background-process': {'auto-nohup': True, 'via-sh': True}}, 'status': {'columns': 'status,address,load,tokens,owns,host_id,gossip,compactions', 'header': '--,Address,Load,Tokens,Owns,Host ID,GOSSIP,COMPACTIONS'}, 'storage': {'columns': 'pod,volume_root,volume_cassandra,snapshots,data,compactions', 'header': 'POD_NAME,VOLUME /,VOLUME CASS,SNAPSHOTS,DATA,COMPACTIONS'}, 'watch': {'auto': 'rollout', 'timeout': 3600, 'interval': 10}, 'auto-complete': {'c': {'tables': 'lazy'}, 'x': {'tables': 'lazy'}, 'cli': {'cp': 'jit'}, 'export': {'databases': 'jit'}, 'medusa': {'backups': 'jit'}, 'reaper': {'schedules': 'jit'}}, 'debug': False, 'debugs': {'complete': False, 'timings': False, 'exit-on-error': False}}
+    return {'app': {'console-endpoint': 'https://{host}/{env}/{app}/static/console/index.html', 'container-name': 'c3-server', 'cr': {'cluster-regex': '(.*?-.*?)-.*', 'group': 'ops.c3.ai', 'v': 'v2', 'plural': 'c3cassandras'}, 'label': 'c3__app_id-0', 'login': {'admin-group': '{host}/C3.ClusterAdmin', 'ingress': '{app_id}-k8singr-appleader-001', 'timeout': 5, 'session-check-url': 'https://{host}/{env}/{app}/api/8/C3/userSessionToken', 'cache-creds': True, 'cache-username': True, 'url': 'https://{host}/{env}/{app}', 'another': "You're logged in to {has}. However, for this app, you need to log in to {need}.", 'token-server-url': 'http://localhost:{port}', 'password-max-length': 128}, 'strip': '0'}, 'audit': {'endpoint': 'https://4psvtaxlcb.execute-api.us-west-2.amazonaws.com/prod/', 'workers': 3, 'timeout': 10, 'log-audit-queries': False, 'athena': {'auto-repair': {'elapsed_hours': 12}, 'region': 'us-west-2', 'catalog': 'AwsDataCatalog', 'database': 'audit', 'repair-partition-tables': 'audit', 'output': 's3://s3.ops--audit/ddl/results', 'repair-cluster-tables': 'cluster'}, 'queries': {'last10': "SELECT * FROM audit\nWHERE drive <> 'z' and ({date_condition})\nORDER BY ts DESC LIMIT {limit}", 'slow10': "SELECT * FROM audit\nWHERE drive <> 'z' and ({date_condition})\nORDER BY CAST(duration AS REAL) DESC LIMIT {limit}", 'top10': "SELECT min(c) AS cluster, line, COUNT(*) AS cnt, avg(CAST(duration AS REAL)) AS duration\nFROM audit WHERE drive <> 'z' and ({date_condition})\nGROUP BY line ORDER BY cnt DESC LIMIT {limit}"}}, 'bash': {'workers': 32}, 'cassandra': {'service-name': 'all-pods-service'}, 'cql': {'workers': 32, 'samples': 3, 'secret': {'cluster-regex': '(.*?-.*?)-.*', 'name': '{cluster}-superuser', 'password-item': 'password'}, 'alter-tables': {'excludes': 'system_auth,system_traces,reaper_db,system_distributed,system_views,system,system_schema,system_virtual_schema', 'gc-grace-periods': '3600,86400,864000,7776000', 'batching': True}}, 'checks': {'compactions-threshold': 250, 'cpu-busy-threshold': 98.0, 'cpu-threshold': 0.0, 'cassandra-data-path': '/c3/cassandra', 'root-disk-threshold': 50, 'cassandra-disk-threshold': 50, 'snapshot-size-cmd': "ls /c3/cassandra/data/data/*/*/snapshots | grep snapshots | sed 's/:$//g' | xargs -I {} du -sk {} | awk '{print $1}' | awk '{s+=$1} END {print s}'", 'snapshot-size-threshold': '40G', 'table-sizes-cmd': "ls -Al /c3/cassandra/data/data/ | awk '{print $9}' | sed 's/\\^r//g' | xargs -I {} du -sk /c3/cassandra/data/data/{}"}, 'download': {'workers': 8}, 'export': {'workers': 8, 'csv_dir': '/c3/cassandra/tmp', 'column_counts_query': 'select id, count(id) as columns from {table} group by id order by columns desc limit 10', 'default-importer': 'sqlite', 'sqlite': {'workers': 8, 'columns': '<row-key>', 'local-db-dir': '/tmp/qing-db/q/export/db'}, 'athena': {'workers': 8, 'columns': '<keys>', 'bucket': 'c3.ops--qing'}, 'csv': {'workers': 8, 'columns': '<row-key>'}, 'log-dir': '/tmp/qing-db/q/export/logs'}, 'get-host-id': {'workers': 32}, 'idps': {'ad': {'email-pattern': '.*@c3.ai', 'uri': 'https://login.microsoftonline.com/53ad779a-93e7-485c-ba20-ac8290d7252b/oauth2/v2.0/authorize?response_type=id_token&response_mode=form_post&client_id=00ff94a8-6b0a-4715-98e0-95490012d818&scope=openid+email+profile&redirect_uri=https%3A%2F%2Fplat.c3ci.cloud%2Fc3%2Fc3%2Foidc%2Flogin&nonce={nonce}&state=EMPTY', 'jwks-uri': 'https://login.microsoftonline.com/common/discovery/keys', 'contact': 'Please contact ted.tran@c3.ai.', 'whitelist-file': '/kaqing/members'}, 'okta': {'default': True, 'email-pattern': '.*@c3iot.com', 'uri': 'https://c3energy.okta.com/oauth2/v1/authorize?response_type=id_token&response_mode=form_post&client_id={client_id}&scope=openid+email+profile+groups&redirect_uri=https%3A%2F%2F{host}%2Fc3%2Fc3%2Foidc%2Flogin&nonce={nonce}&state=EMPTY', 'jwks-uri': 'https://c3energy.okta.com/oauth2/v1/keys'}}, 'issues': {'workers': 32}, 'local-qing-dir': '/tmp/qing-db/q', 'local-downloads-dir': '/tmp/qing-db/q/downloads', 'logs': {'path': '/c3/cassandra/logs/system.log'}, 'log-dir': '/tmp/qing-db/q/logs', 'nodetool': {'workers': 96, 'commands_in_line': 40, 'status': {'workers': 32, 'samples': 3, 'commands_in_line': 40}}, 'pg': {'name-pattern': '^{namespace}.*-k8spg-.*', 'excludes': '.helm., -admin-secret', 'agent': {'name': 'ops-pg-agent', 'just-in-time': False, 'timeout': 86400, 'image': 'seanahnsf/kaqing'}, 'default-db': 'postgres', 'default-schema': 'postgres', 'secret': {'endpoint-key': 'postgres-db-endpoint', 'port-key': 'postgres-db-port', 'username-key': 'postgres-admin-username', 'password-key': 'postgres-admin-password'}}, 'pod': {'name': 'ops', 'image': 'seanahnsf/kaqing-cloud', 'sa': {'name': 'ops', 'proto': 'c3', 'additional-cluster-roles': 'c3aiops-k8ssandra-operator'}, 'label-selector': 'run=ops'}, 'preview': {'rows': 10}, 'processes': {'columns': 'pod,cpu-metrics,mem', 'header': 'POD_NAME,M_CPU(USAGE/LIMIT),MEM/LIMIT'}, 'processes-qing': {'columns': 'pod,cpu,mem', 'header': 'POD_NAME,Q_CPU/TOTAL,MEM/LIMIT'}, 'reaper': {'service-name': 'reaper-service', 'port-forward': {'timeout': 86400, 'local-port': 9001}, 'abort-runs-batch': 10, 'show-runs-batch': 100, 'pod': {'cluster-regex': '(.*?-.*?-.*?-.*?)-.*', 'label-selector': 'k8ssandra.io/reaper={cluster}-reaper'}, 'secret': {'cluster-regex': '(.*?-.*?)-.*', 'name': '{cluster}-reaper-ui', 'password-item': 'password'}}, 'repair': {'log-path': '/home/cassrepair/logs/', 'image': 'ci-registry.c3iot.io/cloudops/cassrepair:2.0.14', 'secret': 'ciregistryc3iotio', 'env': {'interval': 24, 'timeout': 60, 'pr': False, 'runs': 1}}, 'repl': {'start-drive': 'c', 'a': {'auto-enter': 'c3/c3'}, 'c': {'auto-enter': 'cluster'}, 'x': {'auto-enter': 'latest'}, 'history': {'push-cat-log-file': True, 'push-cat-remote-log-file': True}, 'background-process': {'auto-nohup': True}}, 'status': {'columns': 'status,address,load,tokens,owns,host_id,gossip,compactions', 'header': '--,Address,Load,Tokens,Owns,Host ID,GOSSIP,COMPACTIONS'}, 'storage': {'columns': 'pod,volume_root,volume_cassandra,snapshots,data,compactions', 'header': 'POD_NAME,VOLUME /,VOLUME CASS,SNAPSHOTS,DATA,COMPACTIONS'}, 'watch': {'auto': 'rollout', 'timeout': 3600, 'interval': 10}, 'auto-complete': {'c': {'tables': 'lazy'}, 'x': {'tables': 'lazy'}, 'cli': {'cp': 'jit'}, 'export': {'databases': 'jit'}, 'medusa': {'backups': 'jit'}, 'reaper': {'schedules': 'jit'}}, 'debug': False, 'debugs': {'complete': False, 'timings': False, 'exit-on-error': False}}