kaqing 2.0.110__py3-none-any.whl → 2.0.214__py3-none-any.whl

adam/commands/cql/utils_cql.py

@@ -0,0 +1,306 @@
+import functools
+import re
+from typing import Union
+
+from adam.commands.command import Command
+from adam.commands.commands_utils import show_table
+from adam.utils_k8s.cassandra_clusters import CassandraClusters
+from adam.utils_k8s.cassandra_nodes import CassandraNodes
+from adam.utils_k8s.secrets import Secrets
+from adam.utils_k8s.pod_exec_result import PodExecResult
+from adam.repl_state import ReplState
+from adam.utils import ing, log2, log_timing, wait_log
+from adam.utils_k8s.statefulsets import StatefulSets
+
+def cd_dirs(state: ReplState) -> list[str]:
+    if state.pod:
+        return [".."]
+    elif state.sts:
+        return [".."] + StatefulSets.pod_names(state.sts, state.namespace)
+    else:
+        return StatefulSets.list_sts_names()
+
+@functools.lru_cache()
+def cassandra_keyspaces(state: ReplState, on_any=True):
+    if state.pod:
+        wait_log(f'Inspecting Cassandra Keyspaces on {state.pod}...')
+    else:
+        wait_log(f'Inspecting Cassandra Keyspaces...')
+
+    r: list[PodExecResult] = run_cql(state, 'describe keyspaces', show_out=False, on_any=on_any)
+    if not r:
+        log2('No pod is available')
+        return []
+
+    return parse_cql_desc_keyspaces(r.stdout if state.pod else r[0].stdout)
+
+def cassandra_table_names(state: ReplState, keyspace = None):
+    return [f'{k}.{t}' for k, ts in cassandra_tables(state, on_any=True).items() for t in ts if not keyspace or keyspace == '*' or k == keyspace]
+
+@functools.lru_cache()
+def cassandra_tables(state: ReplState, on_any=False) -> dict[str, list[str]]:
+    r: list[PodExecResult] = run_cql(state, 'describe tables', show_out=False, on_any=on_any)
+    if not r:
+        log2('No pod is available')
+        return {}
+
+    return parse_cql_desc_tables(r.stdout if state.pod else r[0].stdout)
+
+@functools.lru_cache()
+def table_spec(state: ReplState, table: str, on_any=False) -> 'TableSpec':
+    r: list[PodExecResult] = run_cql(state, f'describe table {table}', show_out=False, on_any=on_any)
+    if not r:
+        log2('No pod is available')
+        return None
+
+    return parse_cql_desc_table(r.stdout if state.pod else r[0].stdout)
+
+def run_cql(state: ReplState, cql: str, opts: list = [], show_out = False, show_query = False, use_single_quotes = False, on_any = False, backgrounded=False, log_file=None, history=True) -> list[PodExecResult]:
+    if show_query:
+        log2(cql)
+
+    command = None
+    with log_timing('Secrets.get_user_pass'):
+        user, pw = Secrets.get_user_pass(state.sts if state.sts else state.pod, state.namespace, secret_path='cql.secret')
+        if use_single_quotes:
+            command = f"cqlsh -u {user} -p {pw} {' '.join(opts)} -e '{cql}'"
+        else:
+            command = f'cqlsh -u {user} -p {pw} {" ".join(opts)} -e "{cql}"'
+
+    with log_timing(cql):
+        with cassandra(state) as pods:
+            return pods.exec(command, action='cql', show_out=show_out, on_any=on_any, backgrounded=backgrounded, log_file=log_file, history=history)
+
+def parse_cql_desc_tables(out: str):
+    # Keyspace data_endpoint_auth
+    # ---------------------------
+    # "token"
+
+    # Keyspace reaper_db
+    # ------------------
+    # repair_run                     schema_migration
+    # repair_run_by_cluster          schema_migration_leader
+
+    # Keyspace system
+    tables_by_keyspace: dict[str, list[str]] = {}
+    keyspace = None
+    state = 's0'
+    for line in out.split('\n'):
+        if state == 's0':
+            groups = re.match(r'^Keyspace (.*)$', line)
+            if groups:
+                keyspace = groups[1].strip(' \r')
+                state = 's1'
+        elif state == 's1':
+            if line.startswith('---'):
+                state = 's2'
+        elif state == 's2':
+            if not line.strip(' \r'):
+                state = 's0'
+            else:
+                for table in line.split(' '):
+                    if t := table.strip(' \r'):
+                        if not keyspace in tables_by_keyspace:
+                            tables_by_keyspace[keyspace] = []
+                        tables_by_keyspace[keyspace].append(t)
+
+    return tables_by_keyspace
+
+def parse_cql_desc_keyspaces(out: str) -> list[str]:
+    #
+    # Warning: Cannot create directory at `/home/cassandra/.cassandra`. Command history will not be saved. Please check what was the environment property CQL_HISTORY set to.
+    #
+    #
+    # Warning: Using a password on the command line interface can be insecure.
+    # Recommendation: use the credentials file to securely provide the password.
+    #
+    #
+    # azops88_db  system_auth         system_traces
+    # reaper_db   system_distributed  system_views
+    # system      system_schema       system_virtual_schema
+    #
+    kses = []
+    for line in out.split('\n'):
+        line = line.strip(' \r')
+        if not line:
+            continue
+        if line.startswith('Warning:'):
+            continue
+        if line.startswith('Recommendation:'):
+            continue
+
+        for ks in line.split(' '):
+            if s := ks.strip(' \r\t'):
+                kses.append(s)
+
+    return kses
+
+class ColumnSpec:
+    def __init__(self, name: str, type: str, key_index = -1):
+        self.name = name
+        self.type = type
+        self.key_index = key_index
+
+    def __eq__(self, other):
+        if not isinstance(other, ColumnSpec):
+            return NotImplemented
+
+        return self.name == other.name and self.type == other.type and self.key_index == other.key_index
+
+class TableSpec:
+    def __init__(self, columns: list[ColumnSpec]):
+        self.columns = columns
+
+    def row_key(self):
+        for c in self.columns:
+            if c.key_index == 0:
+                return c.name
+
+    def keys(self):
+        return [c.name for c in self.columns if c.key_index > -1]
+
+def parse_cql_desc_table(out: str) -> TableSpec:
+    # CREATE TABLE azops88_db.analyticscontainer_dfeevalhistory (
+    #     id text,
+    #     columnname text,
+    #     version bigint static,
+    #     contentb blob,
+    #     contentbool boolean,
+    #     contentn double,
+    #     contents text,
+    #     PRIMARY KEY (id, columnname)
+    # ) WITH CLUSTERING ORDER BY (columnname ASC)
+    #     AND additional_write_policy = '99p'
+    #     AND bloom_filter_fp_chance = 0.1
+    #     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
+    #     AND cdc = false
+    #     AND comment = ''
+    #     AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
+    #     AND compression = {'chunk_length_in_kb': '16', 'class': 'org.apache.cassandra.io.compress.SnappyCompressor'}
+    #     AND memtable = 'default'
+    #     AND crc_check_chance = 1.0
+    #     AND default_time_to_live = 0
+    #     AND extensions = {}
+    #     AND gc_grace_seconds = 3600
+    #     AND max_index_interval = 2048
+    #     AND memtable_flush_period_in_ms = 0
+    #     AND min_index_interval = 128
+    #     AND read_repair = 'BLOCKING'
+    #     AND speculative_retry = '99p';
+    pkeys = {}
+    columns: list[ColumnSpec] = []
+
+    state = 's0'
+    for line in out.split('\n'):
+        if state == 's0':
+            if line.startswith('CREATE TABLE'):
+                state = 's1'
+        elif state == 's1':
+            if line.startswith(')'):
+                state = 's2'
+                continue
+
+            groups = re.match(r'^\s*PRIMARY KEY\s*\((.*)\).*$', line)
+            if groups:
+                pkeys = {n.strip(' '): i for i, n in enumerate(groups[1].strip(' \r').split(','))}
+                continue
+
+            #  single key column - name text PRIMARY KEY,
+            groups = re.match(r'^\s*(\S*?)\s*(\S*?)\s*PRIMARY KEY,.*$', line)
+            if groups:
+                columns.append(ColumnSpec(groups[1], groups[2]))
+                pkeys[groups[1]] = 0
+            else:
+                groups = re.match(r'^\s*(\S*?)\s*(\S*?),.*$', line)
+                if groups:
+                    columns.append(ColumnSpec(groups[1], groups[2]))
+        elif state == 's2':
+            pass
+
+    for column in columns:
+        if column.name in pkeys.keys():
+            column.key_index = pkeys[column.name]
+
+    return TableSpec(columns)
+
+class CassandraPodService:
+    def __init__(self, handler: 'CassandraExecHandler'):
+        self.handler = handler
+
+    def exec(self, command: str, action='bash', show_out = True, on_any = False, throw_err = False, shell = '/bin/sh', backgrounded = False, log_file = None, history=True) -> Union[PodExecResult, list[PodExecResult]]:
+        state = self.handler.state
+        pod = self.handler.pod
+
+        if pod:
+            return CassandraNodes.exec(pod, state.namespace, command,
+                                    show_out=show_out, throw_err=throw_err, shell=shell, backgrounded=backgrounded, log_file=log_file, history=history)
+        elif state.sts:
+            return CassandraClusters.exec(state.sts, state.namespace, command, action=action,
+                                        show_out=show_out, on_any=on_any, shell=shell, backgrounded=backgrounded, log_file=log_file, history=history)
+
+        return []
+
+    def cql(self, args: list[str], opts: list = [], show_out = False, show_query = False, use_single_quotes = False, on_any = False, backgrounded=False, log_file: str = None):
+        state = self.handler.state
+        query: str = args
+
+        if isinstance(query, list):
+            opts = []
+            cqls = []
+            for arg in args:
+                if arg.startswith('--'):
+                    opts.append(arg)
+                elif arg != '-e':
+                    cqls.append(arg)
+            if not cqls:
+                if self.state.in_repl:
+                    log2('Please enter cql statement. e.g. select host_id from system.local')
+                else:
+                    log2('* CQL statement is missing.')
+                    log2()
+                    Command.display_help()
+
+                return 'no-cql'
+
+            query = ' '.join(cqls)
+            show_out = True
+
+        return run_cql(state, query, opts=opts, show_out=show_out, show_query=show_query, use_single_quotes=use_single_quotes, on_any=on_any, backgrounded=backgrounded, log_file=log_file)
+
+    def display_table(self, cols: str, header: str, show_out = True, backgrounded = False, msg: str = None):
+        state = self.handler.state
+
+        with ing(msg=msg, condition=backgrounded and msg):
+            if state.pod:
+                return show_table(state, [state.pod], cols, header, show_out=show_out, backgrounded = backgrounded)
+            elif state.sts:
+                pod_names = [pod.metadata.name for pod in StatefulSets.pods(state.sts, state.namespace)]
+                return show_table(state, pod_names, cols, header, show_out=show_out, backgrounded = backgrounded)
+
+    def nodetool(self, args: str, status = False, show_out = True, backgrounded = False) -> Union[PodExecResult, list[PodExecResult]]:
+        state = self.handler.state
+        pod = self.handler.pod
+
+        user, pw = state.user_pass()
+        command = f"nodetool -u {user} -pw {pw} {args}"
+
+        if pod:
+            return CassandraNodes.exec(pod, state.namespace, command, show_out=show_out, backgrounded=backgrounded)
+        else:
+            return CassandraClusters.exec(state.sts, state.namespace, command, action='nodetool.status' if status else 'nodetool', show_out=show_out, backgrounded=backgrounded)
+
+class CassandraExecHandler:
+    def __init__(self, state: ReplState, pod: str = None):
+        self.state = state
+        self.pod = pod
+        if not pod and state.pod:
+            self.pod = state.pod
+
+    def __enter__(self):
+        return CassandraPodService(self)
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        return False
+
+def cassandra(state: ReplState, pod: str=None):
+    return CassandraExecHandler(state, pod=pod)

adam/commands/debug/debug.py

@@ -0,0 +1,22 @@
+from adam.commands.command import Command
+from adam.commands.debug.debug_completes import DebugCompletes
+from adam.commands.debug.debug_timings import DebugTimings
+from adam.commands.intermediate_command import IntermediateCommand
+
+class Debug(IntermediateCommand):
+    COMMAND = 'debug'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(Debug, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return Debug.COMMAND
+
+    def cmd_list(self):
+        return [DebugTimings(), DebugCompletes()]

adam/commands/debug/debug_completes.py

@@ -0,0 +1,35 @@
+from adam.commands import validate_args
+from adam.commands.command import Command
+from adam.config import Config
+from adam.repl_state import ReplState
+
+class DebugCompletes(Command):
+    COMMAND = 'debug completes'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(DebugCompletes, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return DebugCompletes.COMMAND
+
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        with self.validate(args, state) as (args, state):
+            with validate_args(args, state, name='on, off or file') as args:
+                Config().set('debugs.complete', args)
+
+                return state
+
+    def completion(self, state: ReplState):
+        return super().completion(state, {f: None for f in ['on', 'off', 'file']})
+
+    def help(self, _: ReplState):
+        return f'{DebugCompletes.COMMAND} on|off|file\t turn auto complete debug on or off'

adam/commands/debug/debug_timings.py

@@ -0,0 +1,35 @@
+from adam.commands import validate_args
+from adam.commands.command import Command
+from adam.config import Config
+from adam.repl_state import ReplState
+
+class DebugTimings(Command):
+    COMMAND = 'debug timings'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(DebugTimings, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return DebugTimings.COMMAND
+
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        with self.validate(args, state) as (args, state):
+            with validate_args(args, state, name='on, off or file') as args:
+                Config().set('debugs.timings', args)
+
+                return state
+
+    def completion(self, state: ReplState):
+        return super().completion(state, {f: None for f in ['on', 'off', 'file']})
+
+    def help(self, _: ReplState):
+        return f'{DebugTimings.COMMAND} on|off|file\t turn timing debug on or off'

adam/commands/debug/show_offloaded_completes.py

@@ -0,0 +1,45 @@
+from adam.commands.command import Command
+from adam.repl_state import ReplState
+from adam.sql.async_executor import AsyncExecutor
+from adam.utils import tabulize
+
+class ShowOffloadedCompletes(Command):
+    COMMAND = 'show offloaded completes'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(ShowOffloadedCompletes, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return ShowOffloadedCompletes.COMMAND
+
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        with self.validate(args, state) as (args, state):
+            pending, processed, first, last = AsyncExecutor.entries_in_queue()
+            lines = []
+            for k, v in processed.items():
+                lines.append(f'{k}\t{v:2.2f}')
+            for k in pending:
+                lines.append(f'{k}\tpending')
+            lines += [
+                f'---------------------------',
+                f'duration\t{last-first:2.2f}'
+            ]
+
+            tabulize(lines, header='key\tduration(in sec)', separator='\t')
+
+            return state
+
+    def completion(self, state: ReplState):
+        return super().completion(state)
+
+    def help(self, _: ReplState):
+        return f'{ShowOffloadedCompletes.COMMAND}\t show offloaded completes'

adam/commands/deploy/code_start.py

@@ -25,19 +25,16 @@ class CodeStart(Command):
         if not(args := self.args(cmd)):
             return super().run(cmd, state)
 
-        state, args = self.apply_state(args, state)
-        if not self.validate_state(state):
-            return state
-
-        log2('This will support c3/c3 only for demo.')
+        with self.validate(args, state) as (args, state):
+            log2('This will support c3/c3 only for demo.')
 
-        stop_user_codes(state.namespace)
-        try:
-            start_user_code(state.namespace)
-        finally:
             stop_user_codes(state.namespace)
+            try:
+                start_user_code(state.namespace)
+            finally:
+                stop_user_codes(state.namespace)
 
-        return state
+            return state
 
     def completion(self, state: ReplState):
         if state.namespace:

adam/commands/deploy/code_stop.py

@@ -24,28 +24,11 @@ class CodeStop(Command):
         if not(args := self.args(cmd)):
             return super().run(cmd, state)
 
-        state, args = self.apply_state(args, state)
-        if not self.validate_state(state):
-            return state
-
-        _, dry = Command.extract_options(args, '--dry')
-        stop_user_codes(state.namespace, dry)
-
-        # if not args:
-        #     log2('Please specify <port>.')
-        #     return state
+        with self.validate(args, state) as (args, state):
+            _, dry = Command.extract_options(args, '--dry')
+            stop_user_codes(state.namespace, dry)
 
-        # port = args[0]
-        # name = f'ops-{port}'
-        # user = os.getenv("USER")
-        # label_selector=f'user={user}'
-        # Ingresses.delete_ingresses(state.namespace, label_selector=label_selector)
-        # Services.delete_services(state.namespace, label_selector=label_selector)
-
-        # pattern = f'/c3/c3/ops/code/{user}/'
-        # self.kill_process_by_pattern(pattern)
-
-        return state
+            return state
 
     def completion(self, state: ReplState):
         if state.namespace:

adam/commands/deploy/code_utils.py

@@ -7,7 +7,7 @@ from adam.apps import Apps
 from adam.config import Config
 from adam.utils_k8s.ingresses import Ingresses
 from adam.utils_k8s.services import Services
-from adam.utils import log2, random_alphanumeric
+from adam.utils import debug, log2, random_alphanumeric
 
 def start_user_code(namespace: str):
     try:
@@ -57,7 +57,7 @@ def kill_process_by_pattern(pattern, dry=False):
         pids = process.stdout.strip().split('\n')
 
         if not pids or pids == ['']:
-            Config().debug(f"No processes found matching pattern: '{pattern}'")
+            debug(f"No processes found matching pattern: '{pattern}'")
             return
 
         for pid in pids:
@@ -66,7 +66,7 @@ def kill_process_by_pattern(pattern, dry=False):
                     if not dry:
                         subprocess.run(f"kill -9 {pid}", shell=True, check=True)
 
-                    Config().debug(f"Killed process with PID: {pid} (matching pattern: '{pattern}')")
+                    debug(f"Killed process with PID: {pid} (matching pattern: '{pattern}')")
                 except subprocess.CalledProcessError as e:
                     log2(f"Error killing process {pid}: {e}")
 

adam/commands/deploy/deploy.py

@@ -1,14 +1,12 @@
 import click
 
-from adam.commands.command import Command
 from adam.commands.deploy.deploy_pg_agent import DeployPgAgent
 from adam.commands.deploy.deploy_pod import DeployPod
+from adam.commands.intermediate_command import IntermediateCommand
 from .deploy_frontend import DeployFrontend
-from adam.repl_state import ReplState, RequiredState
 
-class Deploy(Command):
+class Deploy(IntermediateCommand):
     COMMAND = 'deploy'
-    reaper_login = None
 
     # the singleton pattern
     def __new__(cls, *args, **kwargs):
@@ -16,33 +14,12 @@ class Deploy(Command):
 
         return cls.instance
 
-    def __init__(self, successor: Command=None):
-        super().__init__(successor)
-
     def command(self):
         return Deploy.COMMAND
 
-    def required(self):
-        return RequiredState.NAMESPACE
-
-    def run(self, cmd: str, state: ReplState):
-        if not(args := self.args(cmd)):
-            return super().run(cmd, state)
-
-        if not self.validate_state(state):
-            return state
-
-        return super().intermediate_run(cmd, state, args, Deploy.cmd_list())
-
-    def cmd_list():
+    def cmd_list(self):
         return [DeployFrontend(), DeployPod(), DeployPgAgent()]
 
-    def completion(self, state: ReplState):
-        if state.sts:
-            return super().completion(state)
-
-        return {}
-
 class DeployCommandHelper(click.Command):
     def get_help(self, ctx: click.Context):
-        Command.intermediate_help(super().get_help(ctx), Deploy.COMMAND, Deploy.cmd_list())
+        IntermediateCommand.intermediate_help(super().get_help(ctx), Deploy.COMMAND, Deploy().cmd_list())

adam/commands/deploy/deploy_frontend.py

@@ -26,24 +26,21 @@ class DeployFrontend(Command):
         if not(args := self.args(cmd)):
             return super().run(cmd, state)
 
-        state, args = self.apply_state(args, state)
-        if not self.validate_state(state):
-            return state
-
-        log2('This will support c3/c3 only for demo.')
+        with self.validate(args, state) as (args, state):
+            log2('This will support c3/c3 only for demo.')
+
+            pod_name = Config().get('pod.name', 'ops')
+            label_selector = Config().get('pod.label-selector', 'run=ops')
+            try:
+                uri = deploy_frontend(pod_name, state.namespace, label_selector)
+                log2(f'Ops pod is available at {uri}.')
+            except Exception as e:
+                if e.status == 409:
+                    log2(f"Error: '{pod_name}' already exists in namespace '{state.namespace}'.")
+                else:
+                    log2(f"Error creating ingress or service: {e}")
 
-        pod_name = Config().get('pod.name', 'ops')
-        label_selector = Config().get('pod.label-selector', 'run=ops')
-        try:
-            uri = deploy_frontend(pod_name, state.namespace, label_selector)
-            log2(f'Ops pod is available at {uri}.')
-        except Exception as e:
-            if e.status == 409:
-                log2(f"Error: '{pod_name}' already exists in namespace '{state.namespace}'.")
-            else:
-                log2(f"Error creating ingress or service: {e}")
-
-        return state
+            return state
 
     def completion(self, state: ReplState):
         return super().completion(state)

adam/commands/deploy/deploy_pg_agent.py

@@ -1,5 +1,5 @@
 from adam.commands.command import Command
-from adam.commands.postgres.postgres_context import PostgresContext
+from adam.commands.postgres.postgres_databases import PostgresDatabases
 from adam.config import Config
 from adam.repl_state import ReplState, RequiredState
 
@@ -25,11 +25,8 @@ class DeployPgAgent(Command):
         if not(args := self.args(cmd)):
             return super().run(cmd, state)
 
-        state, args = self.apply_state(args, state)
-        if not self.validate_state(state):
-            return state
-
-        PostgresContext.deploy_pg_agent(Config().get('pg.agent.name', 'ops-pg-agent'), state.namespace)
+        with self.validate(args, state) as (args, state):
+            PostgresDatabases.deploy_pg_agent(Config().get('pg.agent.name', 'ops-pg-agent'), state.namespace)
 
     def completion(self, state: ReplState):
         return super().completion(state)