kaqing 1.98.15__py3-none-any.whl → 2.0.145__py3-none-any.whl

adam/commands/command.py

@@ -3,8 +3,9 @@ import copy
 import subprocess
 import sys
 
-from adam.config import Config
+from adam.commands.command_helpers import ClusterCommandHelper
 from adam.repl_state import ReplState, RequiredState
+from adam.utils import is_lambda, lines_to_tabular, log, log2
 
 repl_cmds: list['Command'] = []
 
@@ -22,14 +23,18 @@ class Command:
     # Do not do child of child!!!
     @abstractmethod
     def run(self, cmd: str, state: ReplState):
-        Config().debug(cmd)
         if self._successor:
             return self._successor.run(cmd, state)
 
         return None
 
-    def completion(self, _: ReplState, leaf: dict[str, any] = None) -> dict[str, any]:
-        # COMMAND = 'reaper activate schedule'
+    def completion(self, state: ReplState, leaf: dict[str, any] = None) -> dict[str, any]:
+        if not self.validate_state(state, show_err=False):
+            return {}
+
+        if is_lambda(leaf):
+            leaf = leaf()
+
         d = leaf
         for t in reversed(self.command().split(' ')):
             d = {t: d}
@@ -39,12 +44,11 @@ class Command:
     def required(self) -> RequiredState:
         return None
 
-    def validate_state(self, state: ReplState, pg_required: RequiredState = None, app_required: RequiredState = None):
-        return state.validate(self.required(), pg_required=pg_required, app_required=app_required)
+    def validate_state(self, state: ReplState, show_err=True):
+        return state.validate(self.required(), show_err=show_err)
 
-    @abstractmethod
-    def help(self, state: ReplState) -> str:
-        pass
+    def help(self, _: ReplState) -> str:
+        return None
 
     def args(self, cmd: str):
         a = list(filter(None, cmd.split(' ')))
@@ -54,8 +58,11 @@ class Command:
 
         return a
 
-    def apply_state(self, args: list[str], state: ReplState, resolve_pg = True) -> tuple[ReplState, list[str]]:
-        return state.apply_args(args, cmd=self.command_tokens(), resolve_pg=resolve_pg)
+    def apply_state(self, args: list[str], state: ReplState, resolve_pg = True, args_to_check = 6) -> tuple[ReplState, list[str]]:
+        """
+        Applies any contextual arguments such as namespace or statefulset to the ReplState and returns any non-contextual arguments.
+        """
+        return state.apply_args(args, cmd=self.command_tokens(), resolve_pg=resolve_pg, args_to_check=args_to_check)
 
     def command_tokens(self):
         return self.command().split(' ')
@@ -103,4 +110,33 @@ class Command:
         while s := cmd._successor:
             print(f'-> {s.command()}', end = '')
             cmd = s
-        print()
+        print()
+
+    def intermediate_run(self, cmd: str, state: ReplState, args: list[str], cmds: list['Command'], separator='\t', display_help=True):
+        state, _ = self.apply_state(args, state)
+
+        if state.in_repl:
+            if display_help:
+                log(lines_to_tabular([c.help(state) for c in cmds], separator=separator))
+
+            return 'command-missing'
+        else:
+            # head with the Chain of Responsibility pattern
+            cmds = Command.chain(cmds)
+            if not cmds.run(cmd, state):
+                if display_help:
+                    log2('* Command is missing.')
+                    Command.display_help()
+                return 'command-missing'
+
+        return state
+
+    def intermediate_help(super_help: str, cmd: str, cmd_list: list['Command'], separator='\t', show_cluster_help=False):
+        log(super_help)
+        log()
+        log('Sub-Commands:')
+
+        log(lines_to_tabular([c.help(ReplState()).replace(f'{cmd} ', '  ', 1) for c in cmd_list], separator=separator))
+        if show_cluster_help:
+            log()
+            ClusterCommandHelper.cluster_help()

adam/commands/commands_utils.py

@@ -1,16 +1,15 @@
 from concurrent.futures import ThreadPoolExecutor
-import time
 from kubernetes import client
 from typing import List
 
 from adam.checks.check_utils import run_checks
 from adam.columns.columns import Columns, collect_checks
 from adam.commands.issues import Issues
-from adam.k8s_utils.cassandra_nodes import CassandraNodes
-from adam.k8s_utils.pods import Pods
-from adam.k8s_utils.statefulsets import StatefulSets
+from adam.utils_k8s.cassandra_nodes import CassandraNodes
+from adam.utils_k8s.pods import Pods
+from adam.utils_k8s.statefulsets import StatefulSets
 from adam.repl_state import ReplState
-from adam.utils import convert_seconds, duration, lines_to_tabular, log, log2
+from adam.utils import duration, lines_to_tabular, log, log2
 
 def show_pods(pods: List[client.V1Pod], ns: str, show_namespace = True, show_host_id = True):
     if len(pods) == 0:

adam/commands/cql/cql_completions.py

@@ -0,0 +1,28 @@
+from adam.commands.cql.cql_utils import cassandra_keyspaces, cassandra_table_names
+from adam.commands.export.exporter import Exporter
+from adam.commands.export.export_databases import ExportDatabases
+from adam.config import Config
+from adam.repl_state import ReplState
+from adam.sql.sql_completer import SqlCompleter, SqlVariant
+
+def cql_completions(state: ReplState) -> dict[str, any]:
+    ps = Config().get('cql.alter-tables.gc-grace-periods', '3600,86400,864000,7776000').split(',')
+    # warm up caches
+    cassandra_keyspaces(state)
+    cassandra_table_names(state)
+    ExportDatabases.database_names()
+
+    expandables = {
+        'keyspaces': lambda: cassandra_keyspaces(state),
+        'table-props': lambda: {
+            'GC_GRACE_SECONDS': ps
+        },
+        'export-dbs': lambda: ExportDatabases.database_names(),
+        'export-sessions': lambda: Exporter.export_session_names(state.sts, state.pod, state.namespace),
+        'export-sessions-incomplete': lambda: Exporter.export_session_names(state.sts, state.pod, state.namespace, export_state='pending_import'),
+    }
+
+    return SqlCompleter(
+        lambda: cassandra_table_names(state),
+        expandables=expandables,
+        variant=SqlVariant.CQL).completions_for_nesting()

adam/commands/cql/cql_utils.py

@@ -0,0 +1,209 @@
+import functools
+import re
+
+from adam.config import Config
+from adam.utils_k8s.cassandra_clusters import CassandraClusters
+from adam.utils_k8s.cassandra_nodes import CassandraNodes
+from adam.utils_k8s.secrets import Secrets
+from adam.pod_exec_result import PodExecResult
+from adam.repl_state import ReplState
+from adam.utils import log2
+
+@functools.lru_cache()
+def cassandra_keyspaces(state: ReplState, on_any=True):
+    if state.pod:
+        Config().wait_log(f'Inspecting Cassandra Keyspaces on {state.pod}...')
+    else:
+        Config().wait_log(f'Inspecting Cassandra Keyspaces...')
+
+    r: list[PodExecResult] = run_cql(state, 'describe keyspaces', show_out=False, on_any=on_any)
+    if not r:
+        log2('No pod is available')
+        return []
+
+    return parse_cql_desc_keyspaces(r.stdout if state.pod else r[0].stdout)
+
+def cassandra_table_names(state: ReplState, keyspace = None):
+    return [f'{k}.{t}' for k, ts in cassandra_tables(state, on_any=True).items() for t in ts if not keyspace or keyspace == '*' or k == keyspace]
+
+@functools.lru_cache()
+def cassandra_tables(state: ReplState, on_any=False) -> dict[str, list[str]]:
+    r: list[PodExecResult] = run_cql(state, 'describe tables', show_out=False, on_any=on_any)
+    if not r:
+        log2('No pod is available')
+        return {}
+
+    return parse_cql_desc_tables(r.stdout if state.pod else r[0].stdout)
+
+@functools.lru_cache()
+def table_spec(state: ReplState, table: str, on_any=False) -> 'TableSpec':
+    r: list[PodExecResult] = run_cql(state, f'describe table {table}', show_out=False, on_any=on_any)
+    if not r:
+        log2('No pod is available')
+        return None
+
+    return parse_cql_desc_table(r.stdout if state.pod else r[0].stdout)
+
+def run_cql(state: ReplState, cql: str, opts: list = [], show_out = False, use_single_quotes = False, on_any = False, background=False, log_file=None) -> list[PodExecResult]:
+    user, pw = Secrets.get_user_pass(state.sts if state.sts else state.pod, state.namespace, secret_path='cql.secret')
+    if use_single_quotes:
+        command = f"cqlsh -u {user} -p {pw} {' '.join(opts)} -e '{cql}'"
+    else:
+        command = f'cqlsh -u {user} -p {pw} {" ".join(opts)} -e "{cql}"'
+
+    if state.pod:
+        return CassandraNodes.exec(state.pod, state.namespace, command, show_out=show_out, background=background, log_file=log_file)
+    else:
+        return CassandraClusters.exec(state.sts, state.namespace, command, show_out=show_out, action='cql', on_any=on_any, background=background, log_file=log_file)
+
+def parse_cql_desc_tables(out: str):
+    # Keyspace data_endpoint_auth
+    # ---------------------------
+    # "token"
+
+    # Keyspace reaper_db
+    # ------------------
+    # repair_run                     schema_migration
+    # repair_run_by_cluster          schema_migration_leader
+
+    # Keyspace system
+    tables_by_keyspace: dict[str, list[str]] = {}
+    keyspace = None
+    state = 's0'
+    for line in out.split('\n'):
+        if state == 's0':
+            groups = re.match(r'^Keyspace (.*)$', line)
+            if groups:
+                keyspace = groups[1].strip(' \r')
+                state = 's1'
+        elif state == 's1':
+            if line.startswith('---'):
+                state = 's2'
+        elif state == 's2':
+            if not line.strip(' \r'):
+                state = 's0'
+            else:
+                for table in line.split(' '):
+                    if t := table.strip(' \r'):
+                        if not keyspace in tables_by_keyspace:
+                            tables_by_keyspace[keyspace] = []
+                        tables_by_keyspace[keyspace].append(t)
+
+    return tables_by_keyspace
+
+def parse_cql_desc_keyspaces(out: str) -> list[str]:
+    #
+    # Warning: Cannot create directory at `/home/cassandra/.cassandra`. Command history will not be saved. Please check what was the environment property CQL_HISTORY set to.
+    #
+    #
+    # Warning: Using a password on the command line interface can be insecure.
+    # Recommendation: use the credentials file to securely provide the password.
+    #
+    #
+    # azops88_db  system_auth         system_traces
+    # reaper_db   system_distributed  system_views
+    # system      system_schema       system_virtual_schema
+    #
+    kses = []
+    for line in out.split('\n'):
+        line = line.strip(' \r')
+        if not line:
+            continue
+        if line.startswith('Warning:'):
+            continue
+        if line.startswith('Recommendation:'):
+            continue
+
+        for ks in line.split(' '):
+            if s := ks.strip(' \r\t'):
+                kses.append(s)
+
+    return kses
+
+class ColumnSpec:
+    def __init__(self, name: str, type: str, key_index = -1):
+        self.name = name
+        self.type = type
+        self.key_index = key_index
+
+    def __eq__(self, other):
+        if not isinstance(other, ColumnSpec):
+            return NotImplemented
+
+        return self.name == other.name and self.type == other.type and self.key_index == other.key_index
+
+class TableSpec:
+    def __init__(self, columns: list[ColumnSpec]):
+        self.columns = columns
+
+    def row_key(self):
+        for c in self.columns:
+            if c.key_index == 0:
+                return c.name
+
+    def keys(self):
+        return [c.name for c in self.columns if c.key_index > -1]
+
+def parse_cql_desc_table(out: str) -> TableSpec:
+    # CREATE TABLE azops88_db.analyticscontainer_dfeevalhistory (
+    #     id text,
+    #     columnname text,
+    #     version bigint static,
+    #     contentb blob,
+    #     contentbool boolean,
+    #     contentn double,
+    #     contents text,
+    #     PRIMARY KEY (id, columnname)
+    # ) WITH CLUSTERING ORDER BY (columnname ASC)
+    #     AND additional_write_policy = '99p'
+    #     AND bloom_filter_fp_chance = 0.1
+    #     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
+    #     AND cdc = false
+    #     AND comment = ''
+    #     AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
+    #     AND compression = {'chunk_length_in_kb': '16', 'class': 'org.apache.cassandra.io.compress.SnappyCompressor'}
+    #     AND memtable = 'default'
+    #     AND crc_check_chance = 1.0
+    #     AND default_time_to_live = 0
+    #     AND extensions = {}
+    #     AND gc_grace_seconds = 3600
+    #     AND max_index_interval = 2048
+    #     AND memtable_flush_period_in_ms = 0
+    #     AND min_index_interval = 128
+    #     AND read_repair = 'BLOCKING'
+    #     AND speculative_retry = '99p';
+    pkeys = {}
+    columns: list[ColumnSpec] = []
+
+    state = 's0'
+    for line in out.split('\n'):
+        if state == 's0':
+            if line.startswith('CREATE TABLE'):
+                state = 's1'
+        elif state == 's1':
+            if line.startswith(')'):
+                state = 's2'
+                continue
+
+            groups = re.match(r'^\s*PRIMARY KEY\s*\((.*)\).*$', line)
+            if groups:
+                pkeys = {n.strip(' '): i for i, n in enumerate(groups[1].strip(' \r').split(','))}
+                continue
+
+            #  single key column - name text PRIMARY KEY,
+            groups = re.match(r'^\s*(\S*?)\s*(\S*?)\s*PRIMARY KEY,.*$', line)
+            if groups:
+                columns.append(ColumnSpec(groups[1], groups[2]))
+                pkeys[groups[1]] = 0
+            else:
+                groups = re.match(r'^\s*(\S*?)\s*(\S*?),.*$', line)
+                if groups:
+                    columns.append(ColumnSpec(groups[1], groups[2]))
+        elif state == 's2':
+            pass
+
+    for column in columns:
+        if column.name in pkeys.keys():
+            column.key_index = pkeys[column.name]
+
+    return TableSpec(columns)

adam/commands/{cqlsh.py → cql/cqlsh.py}

@@ -2,7 +2,9 @@ import click
 
 from adam.commands.command import Command
 from adam.commands.command_helpers import ClusterOrPodCommandHelper
-from adam.commands.cql_utils import run_cql
+from adam.commands.cql.cql_completions import cql_completions
+from adam.utils_k8s.statefulsets import StatefulSets
+from .cql_utils import run_cql
 from adam.repl_state import ReplState, RequiredState
 from adam.utils import log, log2
 
@@ -32,11 +34,14 @@ class Cqlsh(Command):
         if not self.validate_state(state):
             return state
 
+        background = False
         opts = []
         cqls = []
-        for arg in args:
+        for index, arg in enumerate(args):
             if arg.startswith('--'):
                 opts.append(arg)
+            elif index == len(args) -1 and arg == '&':
+                background = True
             elif arg != '-e':
                 cqls.append(arg)
         if not cqls:
@@ -50,21 +55,21 @@ class Cqlsh(Command):
             return 'no-cql'
 
         cql = ' '.join(cqls)
-        return run_cql(state, cql, opts, show_out=True)
+        return run_cql(state, cql, opts, show_out=True, background=background)
 
     def completion(self, state: ReplState) -> dict[str, any]:
+        if state.device != state.C:
+            # conflicts with psql completions
+            return {}
+
         if state.sts or state.pod:
-            return {Cqlsh.COMMAND: None} | {
-                'delete': {'from': None},
-                'insert': {'into': None},
-                'select': None,
-                'update': None,
-            }
+            return cql_completions(state) | \
+                   {f'@{p}': cql_completions(state) for p in StatefulSets.pod_names(state.sts, state.namespace)}
 
         return {}
 
     def help(self, _: ReplState) -> str:
-        return f'[{Cqlsh.COMMAND}] <cql-statements>\t run cqlsh with queries'
+        return f'<cql-statements> [&]\t run cqlsh with queries'
 
 class CqlCommandHelper(click.Command):
     def get_help(self, ctx: click.Context):

adam/commands/{frontend → deploy}/code_start.py

@@ -1,5 +1,5 @@
 from adam.commands.command import Command
-from adam.commands.frontend.code_utils import start_user_code, stop_user_codes
+from adam.commands.deploy.code_utils import start_user_code, stop_user_codes
 from adam.repl_state import ReplState, RequiredState
 from adam.utils import log2
 

adam/commands/{frontend → deploy}/code_stop.py

@@ -1,5 +1,5 @@
 from adam.commands.command import Command
-from adam.commands.frontend.code_utils import stop_user_codes
+from adam.commands.deploy.code_utils import stop_user_codes
 from adam.repl_state import ReplState, RequiredState
 
 class CodeStop(Command):

adam/commands/{frontend → deploy}/code_utils.py

@@ -5,8 +5,8 @@ import subprocess
 
 from adam.apps import Apps
 from adam.config import Config
-from adam.k8s_utils.ingresses import Ingresses
-from adam.k8s_utils.services import Services
+from adam.utils_k8s.ingresses import Ingresses
+from adam.utils_k8s.services import Services
 from adam.utils import log2, random_alphanumeric
 
 def start_user_code(namespace: str):

adam/commands/deploy/deploy.py

@@ -0,0 +1,48 @@
+import click
+
+from adam.commands.command import Command
+from adam.commands.deploy.deploy_pg_agent import DeployPgAgent
+from adam.commands.deploy.deploy_pod import DeployPod
+from .deploy_frontend import DeployFrontend
+from adam.repl_state import ReplState, RequiredState
+
+class Deploy(Command):
+    COMMAND = 'deploy'
+    reaper_login = None
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(Deploy, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return Deploy.COMMAND
+
+    def required(self):
+        return RequiredState.NAMESPACE
+
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        if not self.validate_state(state):
+            return state
+
+        return super().intermediate_run(cmd, state, args, Deploy.cmd_list())
+
+    def cmd_list():
+        return [DeployFrontend(), DeployPod(), DeployPgAgent()]
+
+    def completion(self, state: ReplState):
+        if state.sts:
+            return super().completion(state)
+
+        return {}
+
+class DeployCommandHelper(click.Command):
+    def get_help(self, ctx: click.Context):
+        Command.intermediate_help(super().get_help(ctx), Deploy.COMMAND, Deploy.cmd_list())

adam/commands/deploy/deploy_frontend.py

@@ -0,0 +1,52 @@
+from adam.commands.command import Command
+from adam.commands.deploy.deploy_utils import deploy_frontend
+from adam.config import Config
+from adam.repl_state import ReplState, RequiredState
+from adam.utils import log2
+
+class DeployFrontend(Command):
+    COMMAND = 'deploy frontend'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(DeployFrontend, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return DeployFrontend.COMMAND
+
+    def required(self):
+        return RequiredState.NAMESPACE
+
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        state, args = self.apply_state(args, state)
+        if not self.validate_state(state):
+            return state
+
+        log2('This will support c3/c3 only for demo.')
+
+        pod_name = Config().get('pod.name', 'ops')
+        label_selector = Config().get('pod.label-selector', 'run=ops')
+        try:
+            uri = deploy_frontend(pod_name, state.namespace, label_selector)
+            log2(f'Ops pod is available at {uri}.')
+        except Exception as e:
+            if e.status == 409:
+                log2(f"Error: '{pod_name}' already exists in namespace '{state.namespace}'.")
+            else:
+                log2(f"Error creating ingress or service: {e}")
+
+        return state
+
+    def completion(self, state: ReplState):
+        return super().completion(state)
+
+    def help(self, _: ReplState):
+        return f'{DeployFrontend.COMMAND}\t deploy Web frontend'

adam/commands/deploy/deploy_pg_agent.py

@@ -0,0 +1,38 @@
+from adam.commands.command import Command
+from adam.commands.postgres.postgres_context import PostgresContext
+from adam.config import Config
+from adam.repl_state import ReplState, RequiredState
+
+class DeployPgAgent(Command):
+    COMMAND = 'deploy pg-agent'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(DeployPgAgent, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return DeployPgAgent.COMMAND
+
+    def required(self):
+        return RequiredState.NAMESPACE
+
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        state, args = self.apply_state(args, state)
+        if not self.validate_state(state):
+            return state
+
+        PostgresContext.deploy_pg_agent(Config().get('pg.agent.name', 'ops-pg-agent'), state.namespace)
+
+    def completion(self, state: ReplState):
+        return super().completion(state)
+
+    def help(self, _: ReplState):
+        return f'{DeployPgAgent.COMMAND}\t deploy Postgres agent'