kaqing 1.77.0__py3-none-any.whl → 2.0.171__py3-none-any.whl

adam/commands/export/show_export_session.py

@@ -0,0 +1,48 @@
+from adam.commands.command import Command
+from adam.commands.export.export_databases import ExportDatabases
+from adam.commands.export.exporter import Exporter
+from adam.repl_state import ReplState, RequiredState
+from adam.utils import log2
+
+class ShowExportSession(Command):
+    COMMAND = 'show export session'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(ShowExportSession, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return ShowExportSession.COMMAND
+
+    def required(self):
+        return RequiredState.CLUSTER_OR_POD
+
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        with self.validate(args, state) as (args, state):
+            if not args:
+                if state.in_repl:
+                    log2('Specify export database name.')
+                else:
+                    log2('* Database name is missing.')
+
+                    Command.display_help()
+
+                return 'command-missing'
+
+            ExportDatabases.disply_export_session(state.sts, state.pod, state.namespace, args[0])
+
+            return state
+
+    def completion(self, state: ReplState):
+        return super().completion(state, {session: None for session in Exporter.export_session_names(state.sts, state.pod, state.namespace)})
+
+    def help(self, _: ReplState):
+        return f'{ShowExportSession.COMMAND} <export-session-name>\t show export session'

adam/commands/export/show_export_sessions.py

@@ -0,0 +1,44 @@
+from adam.commands.command import Command
+from adam.commands.export.exporter import Exporter
+from adam.repl_state import ReplState, RequiredState
+from adam.utils import lines_to_tabular, log
+from adam.utils_k8s.statefulsets import StatefulSets
+
+class ShowExportSessions(Command):
+    COMMAND = 'show export sessions'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(ShowExportSessions, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return ShowExportSessions.COMMAND
+
+    def required(self):
+        return RequiredState.CLUSTER_OR_POD
+
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        with self.validate(args, state) as (args, state):
+            pod = state.pod
+            if not pod:
+                pod = StatefulSets.pod_names(state.sts, state.namespace)[0]
+
+            sessions: dict[str, str] = Exporter.find_export_sessions(pod, state.namespace)
+            log(lines_to_tabular([f'{session}\t{export_state}' for session, export_state in sorted(sessions.items(), reverse=True)],
+                                    header='EXPORT_SESSION\tSTATUS', separator='\t'))
+
+            return state
+
+    def completion(self, state: ReplState):
+        return super().completion(state)
+
+    def help(self, _: ReplState):
+        return f'{ShowExportSessions.COMMAND}\t list export sessions'

adam/commands/export/utils_export.py

@@ -0,0 +1,314 @@
+import io
+import re
+
+from adam.config import Config
+from adam.pod_exec_result import PodExecResult
+from adam.repl_state import ReplState
+from adam.utils_k8s.cassandra_nodes import CassandraNodes
+from adam.utils_k8s.pods import log_prefix
+from adam.utils_k8s.statefulsets import StatefulSets
+
+class ImportSpec:
+    def __init__(self, session: str, importer: str):
+        self.session = session
+        self.importer = importer
+
+    def parse_specs(specs_str: str):
+        session: str = None
+        importer: str = None
+
+        if specs_str:
+            importer, session = ImportSpec._extract_importer(specs_str.strip(' '))
+
+        return ImportSpec(session, importer)
+
+    def _extract_importer(spec_str: str) -> tuple[str, str]:
+        importer = None
+        rest = spec_str
+
+        p = re.compile(r"(.*?)to\s+(.*)", re.IGNORECASE)
+        match = p.match(spec_str)
+        if match:
+            rest = match.group(1).strip(' ')
+            importer = match.group(2).strip(' ')
+
+        return importer, rest
+
+class ExportSpec(ImportSpec):
+    def __init__(self, keyspace: str, consistency: str, importer: str, tables: list['ExportTableSpec'], session: str = None):
+        super().__init__(None, importer)
+
+        self.keyspace = keyspace
+        self.consistency = consistency
+        self.tables = tables
+        self.session = session
+
+    def __str__(self):
+        return f'keyspace: {self.keyspace}, consistency: {self.consistency}, importer: {self.importer}, tables: {",".join([t.table for t in self.tables])}, session: {self.session}'
+
+    def __eq__(self, other):
+        if not isinstance(other, ExportSpec):
+            return NotImplemented
+
+        return self.keyspace == other.keyspace and self.tables == other.tables and self.consistency == other.consistency and self.importer == other.importer and self.session == other.session
+
+    def parse_specs(specs_str: str):
+        keyspace: str = None
+        consistency: str = None
+        importer: str = None
+        specs: list[ExportTableSpec] = None
+
+        if specs_str:
+            importer, specs_str = ExportSpec._extract_importer(specs_str.strip(' '))
+            keyspace, specs_str = ExportSpec._extract_keyspace(specs_str)
+            consistency, specs = ExportSpec._extract_consisteny(specs_str)
+
+        return ExportSpec(keyspace, consistency, importer, specs)
+
+    def _extract_keyspace(spec_str: str) -> tuple[str, str]:
+        keyspace = None
+        rest = spec_str
+
+        p = re.compile(r"\s*\*\s+in\s+(\S+)(.*)", re.IGNORECASE)
+        match = p.match(spec_str)
+        if match:
+            keyspace = match.group(1).strip(' ')
+            rest = match.group(2).strip(' ')
+        elif spec_str.startswith('*'):
+            keyspace = '*'
+            rest = spec_str[1:].strip(' ')
+
+        return keyspace, rest
+
+    def _extract_consisteny(spec_str: str) -> tuple[str, list['ExportTableSpec']]:
+        consistency = None
+
+        p = re.compile(r"(.*?)with\s+consistency\s+(.*)", re.IGNORECASE)
+        match = p.match(spec_str)
+        if match:
+            spec_str = match.group(1).strip(' ')
+            consistency = match.group(2)
+
+        if spec_str:
+            p = r",\s*(?![^()]*\))"
+            specs = re.split(p, spec_str)
+
+            return consistency, [ExportTableSpec.parse(spec) for spec in specs]
+
+        return consistency, None
+
+class ExportTableSpec:
+    def __init__(self, keyspace: str, table: str, columns: str = None, target_table: str = None):
+        self.keyspace = keyspace
+        self.table = table
+        self.columns = columns
+        self.target_table = target_table
+
+    def __str__(self):
+        return f'{self.keyspace}.{self.table}({self.columns}) AS {self.target_table}'
+
+    def __eq__(self, other):
+        if not isinstance(other, ExportTableSpec):
+            return NotImplemented
+
+        return self.keyspace == other.keyspace and self.table == other.table and self.columns == other.columns and self.target_table == other.target_table
+
+    def from_status(status: 'ExportTableStatus'):
+        return ExportTableSpec(status.keyspace, status.table, target_table=status.target_table)
+
+    def parse(spec_str: str) -> 'ExportTableSpec':
+        target = None
+
+        p = re.compile(r"(.*?)\s+as\s+(.*)", re.IGNORECASE)
+        match = p.match(spec_str)
+        if match:
+            spec_str = match.group(1)
+            target = match.group(2)
+
+        keyspace = None
+        table = spec_str
+        columns = None
+
+        p = re.compile('(.*?)\.(.*?)\((.*)\)')
+        match = p.match(spec_str)
+        if match:
+            keyspace = match.group(1)
+            table = match.group(2)
+            columns = match.group(3)
+        else:
+            p = re.compile('(.*?)\.(.*)')
+            match = p.match(spec_str)
+            if match:
+                keyspace = match.group(1)
+                table = match.group(2)
+
+        return ExportTableSpec(keyspace, table, columns, target)
+
+    def __eq__(self, other):
+        if isinstance(other, ExportTableSpec):
+            return self.keyspace == other.keyspace and self.table == other.table and self.columns == other.columns and self.target_table == other.target_table
+
+        return False
+
+    def __str__(self):
+        return f'{self.keyspace}.{self.table}({self.columns}) as {self.target_table}'
+
+class ExportTableStatus:
+    def __init__(self, keyspace: str, target_table: str, status: str, table: str = None):
+        self.keyspace = keyspace
+        self.target_table = target_table
+        self.status = status
+        self.table = table
+
+    def __str__(self):
+        return f'{self.keyspace}.{self.table} as {self.target_table} = {self.status}'
+
+    def __eq__(self, other):
+        if isinstance(other, ExportTableStatus):
+            return self.keyspace == other.keyspace and self.table == other.table and self.status == other.status and self.target_table == other.target_table
+
+        return False
+
+    def from_session(sts: str, pod: str, namespace: str, export_session: str):
+        statuses: list[ExportTableStatus] = []
+
+        status_in_whole = 'done'
+        log_files: list[str] = find_files(pod, namespace, f'{log_prefix()}-{export_session}_*.log*')
+
+        for log_file in log_files:
+            status: ExportTableStatus = ExportTableStatus.from_log_file(pod, namespace, export_session, log_file)
+            statuses.append(status)
+
+            if status.status != 'done':
+                status_in_whole = status.status
+
+        return statuses, status_in_whole
+
+    def from_log_file(pod: str, namespace: str, copy_session: str, log_file: str):
+        def get_csv_files_n_table(target_table: str):
+            db = f'{copy_session}_{target_table}'
+            csv_file = f'{csv_dir()}/{db}/*.csv'
+            csv_files: list[str] = find_files(pod, namespace, csv_file)
+            if csv_files:
+                table = target_table
+                m = re.match(f'{csv_dir()}/{db}/(.*).csv', csv_files[0])
+                if m:
+                    table = m.group(1)
+                return csv_files, table
+
+            return csv_files, target_table
+
+        m = re.match(f'{log_prefix()}-{copy_session}_(.*?)\.(.*?)\.log(.*)', log_file)
+        if m:
+            keyspace = m.group(1)
+            target_table = m.group(2)
+            state = m.group(3)
+            if state == '.pending_import':
+                _, table = get_csv_files_n_table(target_table)
+                return ExportTableStatus(keyspace, target_table, 'pending_import', table)
+            elif state == '.done':
+                return ExportTableStatus(keyspace, target_table, 'done', target_table)
+
+            # 4 rows exported to 1 files in 0 day, 0 hour, 0 minute, and 1.335 seconds.
+            pattern = 'rows exported to'
+            r: PodExecResult = CassandraNodes.exec(pod, namespace, f"grep '{pattern}' {log_file}", show_out=Config().is_debug(), shell='bash')
+            if r.exit_code() == 0:
+                csv_files, table = get_csv_files_n_table(target_table)
+                if csv_files:
+                    return ExportTableStatus(keyspace, target_table, 'exported', table)
+                else:
+                    return ExportTableStatus(keyspace, target_table, 'imported', target_table)
+            else:
+                return ExportTableStatus(keyspace, target_table, 'export_in_pregress')
+
+        return ExportTableStatus(None, None, 'unknown')
+
+def csv_dir():
+    return Config().get('export.csv_dir', '/c3/cassandra/tmp')
+
+def find_files(pod: str, namespace: str, pattern: str, mmin: int = 0):
+    if mmin:
+        r = CassandraNodes.exec(pod, namespace, f'find {pattern} -mmin -{mmin}', show_out=Config().is_debug(), shell='bash')
+    else:
+        r = CassandraNodes.exec(pod, namespace, f'find {pattern}', show_out=Config().is_debug(), shell='bash')
+
+    log_files = []
+    for line in r.stdout.split('\n'):
+        line = line.strip(' \r')
+        if line:
+            log_files.append(line)
+
+    return log_files
+
+class GeneratorStream(io.RawIOBase):
+    def __init__(self, generator):
+        self._generator = generator
+        self._buffer = b''  # Buffer to store leftover bytes from generator yields
+
+    def readable(self):
+        return True
+
+    def _read_from_generator(self):
+        try:
+            chunk = next(self._generator)
+            if isinstance(chunk, str):
+                chunk = chunk.encode('utf-8')  # Encode if generator yields strings
+            self._buffer += chunk
+        except StopIteration:
+            pass  # Generator exhausted
+
+    def readinto(self, b):
+        # Fill the buffer if necessary
+        while len(self._buffer) < len(b):
+            old_buffer_len = len(self._buffer)
+            self._read_from_generator()
+            if len(self._buffer) == old_buffer_len:  # Generator exhausted and buffer empty
+                break
+
+        bytes_to_read = min(len(b), len(self._buffer))
+        b[:bytes_to_read] = self._buffer[:bytes_to_read]
+        self._buffer = self._buffer[bytes_to_read:]
+        return bytes_to_read
+
+    def read(self, size=-1):
+        if size == -1:  # Read all remaining data
+            while True:
+                old_buffer_len = len(self._buffer)
+                self._read_from_generator()
+                if len(self._buffer) == old_buffer_len:
+                    break
+            data = self._buffer
+            self._buffer = b''
+            return data
+        else:
+            # Ensure enough data in buffer
+            while len(self._buffer) < size:
+                old_buffer_len = len(self._buffer)
+                self._read_from_generator()
+                if len(self._buffer) == old_buffer_len:
+                    break
+
+            data = self._buffer[:size]
+            self._buffer = self._buffer[size:]
+            return data
+
+class PodHandler:
+    def __init__(self, state: ReplState):
+        self.state = state
+
+    def __enter__(self):
+        state = self.state
+
+        if not state.pod:
+            state.push()
+            state.pod = StatefulSets.pod_names(state.sts, state.namespace)[0]
+
+        return state
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.state.pop()
+
+        return False
+
+def state_with_pod(state: ReplState):
+    return PodHandler(state)

{walker → adam}/commands/help.py

@@ -1,7 +1,7 @@
-from walker.commands.command import Command
-from walker.repl_commands import ReplCommands
-from walker.repl_state import ReplState
-from walker.utils import lines_to_tabular, log
+from adam.commands.command import Command
+from adam.repl_commands import ReplCommands
+from adam.repl_state import ReplState
+from adam.utils import lines_to_tabular, log
 
 class Help(Command):
     COMMAND = 'help'
@@ -23,20 +23,25 @@ class Help(Command):
             return super().run(cmd, state)
 
         def section(cmds : list[ReplCommands]):
-            return [f'  {c.help(state)}' for c in cmds if c.help(state)]
+            sorted_cmds = sorted(cmds, key=lambda cmd: cmd.command())
+            return [f'  {c.help(state)}' for c in sorted_cmds if c.help(state)]
 
         lines = []
-        lines.append('NAVIGATION & HELP')
-        lines.append('  a: | c: | p:\t switch to another operational device: App, Cassandra or Postgres')
+        lines.append('NAVIGATION')
+        lines.append('  a: | c: | l: | p: | x:\t switch to another operational device: App, Cassandra, Audit, Postgres or Export')
         lines.extend(section(ReplCommands.navigation()))
-        lines.append('CHECK CASSANDRA')
-        lines.extend(section(ReplCommands.cassandra_check()))
-        lines.append('CASSANDRA OPERATIONS')
+        lines.append('CASSANDRA')
         lines.extend(section(ReplCommands.cassandra_ops()))
+        lines.append('POSTGRES')
+        lines.extend(section(ReplCommands.postgres_ops()))
+        lines.append('APP')
+        lines.extend(section(ReplCommands.app_ops()))
+        lines.append('EXPORT DB')
+        lines.extend(section(ReplCommands.export_ops()))
+        lines.append('AUDIT')
+        lines.extend(section(ReplCommands.audit_ops()))
         lines.append('TOOLS')
         lines.extend(section(ReplCommands.tools()))
-        lines.append('APP')
-        lines.extend(section(ReplCommands.app()))
         lines.append('')
         lines.extend(section(ReplCommands.exit()))
 

adam/commands/intermediate_command.py

@@ -0,0 +1,49 @@
+from abc import abstractmethod
+
+from adam.commands.command import Command
+from adam.commands.command_helpers import ClusterCommandHelper
+from adam.repl_state import ReplState
+from adam.utils import lines_to_tabular, log, log2
+
+class IntermediateCommand(Command):
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        return self.intermediate_run(cmd, state, args, self.cmd_list())
+
+    @abstractmethod
+    def cmd_list(self):
+        pass
+
+    def intermediate_run(self, cmd: str, state: ReplState, args: list[str], cmds: list['Command'], separator='\t', display_help=True):
+        state, _ = self.apply_state(args, state)
+
+        if state.in_repl:
+            if display_help:
+                log(lines_to_tabular([c.help(state) for c in cmds], separator=separator))
+
+            return 'command-missing'
+        else:
+            # head with the Chain of Responsibility pattern
+            if not self.run_subcommand(cmd, state):
+                if display_help:
+                    log2('* Command is missing.')
+                    Command.display_help()
+                return 'command-missing'
+
+        return state
+
+    def run_subcommand(self, cmd: str, state: ReplState):
+        cmds = Command.chain(self.cmd_list())
+        return cmds.run(cmd, state)
+
+    def intermediate_help(super_help: str, cmd: str, cmd_list: list['Command'], separator='\t', show_cluster_help=False):
+        log(super_help)
+        log()
+        log('Sub-Commands:')
+
+        log(lines_to_tabular([c.help(ReplState()).replace(f'{cmd} ', '  ', 1) for c in cmd_list], separator=separator))
+        if show_cluster_help:
+            log()
+            ClusterCommandHelper.cluster_help()

adam/commands/issues.py

@@ -0,0 +1,43 @@
+from adam.checks.check_result import CheckResult
+from adam.checks.check_utils import run_checks
+from adam.commands import extract_options
+from adam.commands.command import Command
+from adam.repl_state import ReplState
+from adam.utils_issues import IssuesUtils
+
+class Issues(Command):
+    COMMAND = 'issues'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(Issues, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return Issues.COMMAND
+
+    def required(self):
+        return ReplState.NON_L
+
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        with self.validate(args, state) as (args, state):
+            with extract_options(args, ['-s', '--show']) as (args, show_out):
+                results = run_checks(state.sts, state.namespace, state.pod, show_out=show_out)
+
+                issues = CheckResult.collect_issues(results)
+                IssuesUtils.show_issues(issues, in_repl=state.in_repl)
+
+                return issues if issues else 'issues'
+
+    def completion(self, state: ReplState):
+        return super().completion(state, {'-s': None})
+
+    def help(self, _: ReplState):
+        return f'{Issues.COMMAND} [-s]\t find all issues'

adam/commands/kubectl.py

@@ -0,0 +1,38 @@
+import subprocess
+
+from adam.commands.command import Command
+from adam.repl_state import ReplState, RequiredState
+
+class Kubectl(Command):
+    COMMAND = 'k'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(Kubectl, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return Kubectl.COMMAND
+
+    def required(self):
+        return RequiredState.NAMESPACE
+
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        with self.validate(args, state) as (args, state):
+            subprocess.run(["kubectl"] + args)
+
+            return state
+
+    def completion(self, state: ReplState):
+        return super().completion(state)
+
+
+    def help(self, _: ReplState):
+        return f'{Kubectl.COMMAND} \t run a kubectl command'

adam/commands/login.py

@@ -0,0 +1,70 @@
+import os
+import signal
+import traceback
+
+from adam.app_session import AppSession
+from adam.apps import Apps
+from adam.commands import extract_options
+from adam.config import Config
+from adam.sso.idp import Idp
+from adam.sso.idp_login import IdpLogin
+from adam.commands.command import Command
+from adam.repl_state import ReplState, RequiredState
+from adam.utils import log, log2
+
+class Login(Command):
+    COMMAND = 'login'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(Login, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return Login.COMMAND
+
+    def required(self):
+        return ReplState.NON_L
+
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        def custom_handler(signum, frame):
+            AppSession.ctrl_c_entered = True
+
+        signal.signal(signal.SIGINT, custom_handler)
+
+        with self.validate(args, state) as (args, state):
+            with extract_options(args, 'd') as (args, debug):
+                if debug:
+                    Config().set('debug', True)
+
+                username: str = os.getenv('USERNAME')
+                if len(args) > 0:
+                    username = args[0]
+
+                login: IdpLogin = None
+                try:
+                    if not(host := Apps.app_host('c3', 'c3', state.namespace)):
+                        log2('Cannot locate ingress for app.')
+                        return state
+
+                    if not (login := Idp.login(host, username=username, use_token_from_env=False)):
+                        log2('Invalid username/password. Please try again.')
+                except:
+                    log2(traceback.format_exc())
+
+                log(f'IDP_TOKEN={login.ser() if login else ""}')
+
+                return state
+
+    def completion(self, state: ReplState):
+        return super().completion(state)
+
+    def help(self, _: ReplState):
+        return f'{Login.COMMAND}\t SSO login'

{walker → adam}/commands/logs.py

@@ -1,7 +1,7 @@
-from walker.commands.command import Command
-from walker.config import Config
-from walker.k8s_utils.cassandra_nodes import CassandraNodes
-from walker.repl_state import ReplState, RequiredState
+from adam.commands.command import Command
+from adam.config import Config
+from adam.utils_k8s.cassandra_nodes import CassandraNodes
+from adam.repl_state import ReplState, RequiredState
 
 class Logs(Command):
     COMMAND = 'logs'
@@ -25,14 +25,12 @@ class Logs(Command):
         if not(args := self.args(cmd)):
             return super().run(cmd, state)
 
-        state, args = self.apply_state(args, state)
-        if not self.validate_state(state):
-            return state
-
-        path = Config().get('logs.path', '/c3/cassandra/logs/system.log')
-        return CassandraNodes.exec(state.pod, state.namespace, f'cat {path}')
+        with self.validate(args, state) as (args, state):
+            path = Config().get('logs.path', '/c3/cassandra/logs/system.log')
+            return CassandraNodes.exec(state.pod, state.namespace, f'cat {path}')
 
     def completion(self, _: ReplState):
+        # available only on cli
         return {}
 
     def help(self, _: ReplState):