kaqing 2.0.14__py3-none-any.whl → 2.0.145__py3-none-any.whl

adam/commands/devices/device.py

@@ -0,0 +1,27 @@
+from abc import abstractmethod
+from adam.commands.command import Command
+from adam.repl_state import ReplState
+
+class Device:
+    @abstractmethod
+    def ls(self, cmd: str, state: ReplState):
+        pass
+
+    def ls_completion(self, cmd: str, state: ReplState, default: dict = {}):
+        return default
+
+    def cd(self, dir: str, state: ReplState):
+        pass
+
+    def cd_completion(self, cmd: str, state: ReplState, default: dict = {}):
+        return default
+
+    @abstractmethod
+    def pwd(self, state: ReplState):
+        pass
+
+    def try_fallback_action(self, chain: Command, state: ReplState, cmd: str):
+        return False, None
+
+    def enter(state: ReplState):
+        pass

adam/commands/devices/device_app.py

@@ -0,0 +1,146 @@
+from adam.apps import Apps
+from adam.commands.bash.bash import Bash
+from adam.commands.command import Command
+from adam.commands.devices.device import Device
+from adam.config import Config
+from adam.repl_state import ReplState
+from adam.utils import lines_to_tabular, log
+from adam.utils_k8s.app_pods import AppPods
+from adam.utils_k8s.ingresses import Ingresses
+
+class DeviceApp(Command, Device):
+    COMMAND = f'{ReplState.A}:'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(DeviceApp, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return DeviceApp.COMMAND
+
+    def run(self, cmd: str, state: ReplState):
+        if not self.args(cmd):
+            return super().run(cmd, state)
+
+        state.device = ReplState.A
+
+        return state
+
+    def completion(self, state: ReplState):
+        return super().completion(state)
+
+    def help(self, _: ReplState):
+        return f'{DeviceApp.COMMAND}\t move to App Operations device'
+
+    def ls(self, cmd: str, state: ReplState):
+        if state.app_pod:
+            return Bash().run('bash ' + cmd, state)
+        elif state.app_app:
+            pods = AppPods.pod_names(state.namespace, state.app_env, state.app_app)
+
+            log(lines_to_tabular(pods, 'POD_NAME'))
+        elif state.app_env:
+            def line(n: str, ns: str):
+                host = Ingresses.get_host(Config().get('app.login.ingress', '{app_id}-k8singr-appleader-001').replace('{app_id}', f'{ns}-{n}'), ns)
+                if not host:
+                    return None
+
+                endpoint = Config().get('app.login.url', 'https://{host}/{env}/{app}').replace('{host}', host).replace('{env}', state.app_env).replace('{app}', 'c3')
+                if not endpoint:
+                    return None
+
+                return f"{n.split('-')[1]},{Ingresses.get_host(f'{ns}-{n}-k8singr-appleader-001', ns)},{endpoint}"
+
+            svcs = [l for l in [line(n, ns) for n, ns in Apps.apps(state.app_env)] if l]
+
+            log(lines_to_tabular(svcs, 'APP,HOST,ENDPOINT', separator=','))
+        else:
+            svcs = [n for n, ns in Apps.envs()]
+
+            log(lines_to_tabular(svcs, 'ENV', separator=','))
+
+
+    def ls_completion(self, cmd, state, default: dict = {}):
+        if state.app_app:
+            def pod_names():
+                return [p for p in AppPods.pod_names(state.namespace, state.app_env, state.app_app)]
+
+            return super().completion(state) | {f'@{p}': {cmd: None} for p in pod_names()}
+
+        return default
+
+    def cd(self, dir: str, state: ReplState):
+        if dir == '':
+            state.app_env = None
+            state.app_app = None
+            state.app_pod = None
+        elif dir == '..':
+            if state.app_pod:
+                state.app_pod = None
+            elif state.app_app:
+                state.app_app = None
+            else:
+                state.app_env = None
+        else:
+            if state.app_app:
+                state.app_pod = dir
+            elif not state.app_env:
+                tks = dir.split('@')
+                if len(tks) > 1:
+                    state.namespace = tks[1]
+
+                state.app_env = dir.split('@')[0]
+            else:
+                state.app_app = dir
+
+    def cd_completion(self, cmd: str, state: ReplState, default: dict = {}):
+        if state.app_app:
+            return {cmd: {'..': None} | {pod: None for pod in AppPods.pod_names(state.namespace, state.app_env, state.app_app)}}
+        elif state.app_env:
+            return {cmd: {'..': None} | {app[0].split('-')[1]: None for app in Apps.apps(state.app_env)}}
+        else:
+            return {cmd: {'..': None} | {env[0]: None for env in Apps.envs()}}
+
+    def pwd(self, state: ReplState):
+        words = []
+
+        if state.app_env:
+            words.append(f'env/{state.app_env}')
+        if state.app_app:
+            words.append(f'app/{state.app_app}')
+
+        return '\t'.join([f'{ReplState.X}:>'] + (words if words else ['/']))
+
+    def try_fallback_action(self, chain: Command, state: ReplState, cmd: str):
+        if state.app_app:
+            return True, chain.run(f'app {cmd}', state)
+
+        return False, None
+
+    def enter(self, state: ReplState):
+        if not state.app_env:
+            if auto_enter := Config().get('repl.a.auto-enter', 'c3/c3/*'):
+                if auto_enter != 'no':
+                    ea = auto_enter.split('/')
+                    state.app_env = ea[0]
+                    if len(ea) > 2:
+                        state.app_app = ea[1]
+                        state.app_pod = ea[2]
+                        if state.app_pod == '*':
+                            if (pods := AppPods.pod_names(state.namespace, ea[0], ea[1])):
+                                state.app_pod = pods[0]
+                                Config().wait_log(f'Moving to {state.app_env}/{state.app_app}/{state.app_pod}...')
+                            else:
+                                Config().wait_log(f'No pods found, moving to {state.app_env}/{state.app_app}...')
+                        else:
+                            Config().wait_log(f'Moving to {state.app_env}/{state.app_app}/{state.app_pod}...')
+                    elif len(ea) > 1:
+                        state.app_app = ea[1]
+                        Config().wait_log(f'Moving to {state.app_env}/{state.app_app}...')
+                    else:
+                        Config().wait_log(f'Moving to {state.app_env}...')

adam/commands/devices/device_auit_log.py

@@ -0,0 +1,43 @@
+from adam.commands.command import Command
+from adam.commands.devices.device import Device
+from adam.repl_state import ReplState
+from adam.utils import lines_to_tabular, log
+from adam.utils_athena import Athena
+
+class DeviceAuditLog(Command, Device):
+    COMMAND = f'{ReplState.L}:'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(DeviceAuditLog, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return DeviceAuditLog.COMMAND
+
+    def run(self, cmd: str, state: ReplState):
+        if not self.args(cmd):
+            return super().run(cmd, state)
+
+        state.device = ReplState.L
+
+        return state
+
+    def completion(self, state: ReplState):
+        return super().completion(state)
+
+    def help(self, _: ReplState):
+        return f'{DeviceAuditLog.COMMAND}\t move to Audit Log Operations device'
+
+    def ls(self, cmd: str, _: ReplState):
+        log(lines_to_tabular(Athena.table_names(), 'NAME', separator=','))
+
+    def pwd(self, state: ReplState):
+        return '\t'.join([f'{ReplState.L}:>', '/'])
+
+    def try_fallback_action(self, chain: Command, state: ReplState, cmd: str):
+        return True, chain.run(f'audit {cmd}', state)

adam/commands/devices/device_cass.py

@@ -0,0 +1,145 @@
+from adam.commands.bash.bash import Bash
+from adam.commands.command import Command
+from adam.commands.commands_utils import show_pods, show_rollout
+from adam.commands.devices.device import Device
+from adam.config import Config
+from adam.repl_state import ReplState
+from adam.utils import lines_to_tabular, log, log2
+from adam.utils_k8s.cassandra_clusters import CassandraClusters
+from adam.utils_k8s.custom_resources import CustomResources
+from adam.utils_k8s.kube_context import KubeContext
+from adam.utils_k8s.statefulsets import StatefulSets
+
+class DeviceCass(Command, Device):
+    COMMAND = f'{ReplState.C}:'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(DeviceCass, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return DeviceCass.COMMAND
+
+    def run(self, cmd: str, state: ReplState):
+        if not self.args(cmd):
+            return super().run(cmd, state)
+
+        state.device = ReplState.C
+
+        return state
+
+    def completion(self, state: ReplState):
+        return super().completion(state)
+
+    def help(self, _: ReplState):
+        return f'{DeviceCass.COMMAND}\t move to Cassandra Operations device'
+
+    def ls(self, cmd: str, state: ReplState):
+        if state.pod:
+            return Bash().run('bash ' + cmd, state)
+        elif state.sts and state.namespace:
+            show_pods(StatefulSets.pods(state.sts, state.namespace), state.namespace, show_namespace=not KubeContext.in_cluster_namespace())
+            show_rollout(state.sts, state.namespace)
+        else:
+            self.show_statefulsets()
+
+    def ls_completion(self, cmd: str, state: ReplState, default: dict = {}):
+        def pod_names():
+            return [p for p in StatefulSets.pod_names(state.sts, state.namespace)]
+
+        if state.sts:
+            return super().completion(state) | {f'@{p}': {'ls': None} for p in pod_names()}
+        else:
+            return {cmd: {n: None for n in StatefulSets.list_sts_names()}}
+
+    def show_statefulsets(self):
+        ss = StatefulSets.list_sts_names()
+        if len(ss) == 0:
+            log2('No Cassandra clusters found.')
+            return
+
+        app_ids = CustomResources.get_app_ids()
+        list = []
+        for s in ss:
+            cr_name = CustomResources.get_cr_name(s)
+            app_id = 'Unknown'
+            if cr_name in app_ids:
+                app_id = app_ids[cr_name]
+            list.append(f"{s} {app_id}")
+
+        header = 'STATEFULSET_NAME@NAMESPACE APP_ID'
+        if KubeContext.in_cluster_namespace():
+            header = 'STATEFULSET_NAME APP_ID'
+        log(lines_to_tabular(list, header))
+
+    def cd(self, dir: str, state: ReplState):
+        if dir == '':
+            state.sts = None
+            state.pod = None
+        elif dir == '..':
+            if state.pod:
+                state.pod = None
+            else:
+                state.sts = None
+        else:
+            if not state.sts:
+                ss_and_ns = dir.split('@')
+                state.sts = ss_and_ns[0]
+                if len(ss_and_ns) > 1:
+                    state.namespace = ss_and_ns[1]
+            elif not state.pod:
+                p, _ = KubeContext.is_pod_name(dir)
+                if p:
+                    state.pod = p
+                else:
+                    names = CassandraClusters.pod_names_by_host_id(state.sts, state.namespace);
+                    if dir in names:
+                        state.pod = names[dir]
+                    else:
+                        log2('Not a valid pod name or host id.')
+
+    def cd_completion(self, cmd: str, state: ReplState, default: dict = {}):
+        if state.pod:
+            return {cmd: {'..': None}}
+        elif state.sts:
+            return {cmd: {'..': None} | {p: None for p in StatefulSets.pod_names(state.sts, state.namespace)}}
+        else:
+            return {cmd: {p: None for p in StatefulSets.list_sts_names()}}
+
+    def pwd(self, state: ReplState):
+        words = []
+
+        if state.sts:
+            words.append(f'sts/{state.sts}')
+        if state.pod:
+            words.append(f'pod/{state.pod}')
+
+        return '\t'.join([f'{ReplState.C}:>'] + (words if words else ['/']))
+
+    def try_fallback_action(self, chain: Command, state: ReplState, cmd: str):
+        if state.sts:
+            return True, chain.run(f'cql {cmd}', state)
+
+        return False, None
+
+    def enter(self, state: ReplState):
+        auto_enter = Config().get('repl.c.auto-enter', 'cluster')
+        if auto_enter and auto_enter in ['cluster', 'first-pod']:
+            sts = StatefulSets.list_sts_name_and_ns()
+            if not sts:
+                log2("No Cassandra clusters found.")
+            elif not state.sts and len(sts) == 1:
+                cluster = sts[0]
+                state.sts = cluster[0]
+                state.namespace = cluster[1]
+                if auto_enter == 'first-pod':
+                    state.pod = f'{state.sts}-0'
+                if KubeContext().in_cluster_namespace:
+                    Config().wait_log(f'Moving to the only Cassandra cluster: {state.sts}...')
+                else:
+                    Config().wait_log(f'Moving to the only Cassandra cluster: {state.sts}@{state.namespace}...')

adam/commands/devices/device_export.py

@@ -0,0 +1,86 @@
+from adam.commands.command import Command
+from adam.commands.devices.device import Device
+from adam.commands.export.export_databases import ExportDatabases
+from adam.config import Config
+from adam.repl_state import ReplState
+from adam.utils import lines_to_tabular, log, log2
+
+class DeviceExport(Command, Device):
+    COMMAND = f'{ReplState.X}:'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(DeviceExport, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return DeviceExport.COMMAND
+
+    def run(self, cmd: str, state: ReplState):
+        if not self.args(cmd):
+            return super().run(cmd, state)
+
+        state.device = ReplState.X
+
+        return state
+
+    def completion(self, state: ReplState):
+        return super().completion(state)
+
+    def help(self, _: ReplState):
+        return f'{DeviceExport.COMMAND}\t move to Export Database Operations device'
+
+    def ls(self, cmd: str, state: ReplState):
+        if state.export_session:
+            self.show_export_tables(state.export_session)
+        else:
+            self.show_export_databases()
+
+    def show_export_databases(self, importer: str = None):
+        lines = [f'{k}\t{v}' for k, v in ExportDatabases.database_names_with_keyspace_cnt(importer).items()]
+        log(lines_to_tabular(lines, 'NAME\tKEYSPACES', separator='\t'))
+
+    def show_export_tables(self, export_session: str):
+        log(lines_to_tabular(ExportDatabases.table_names(export_session), 'NAME', separator=','))
+
+    def cd(self, dir: str, state: ReplState):
+        if dir in ['', '..']:
+            state.export_session = None
+        else:
+            state.export_session = dir
+
+    def cd_completion(self, cmd: str, state: ReplState, default: dict = {}):
+        if state.export_session:
+            return {cmd: {'..': None} | {n: None for n in ExportDatabases.database_names()}}
+        else:
+            return {cmd: {n: None for n in ExportDatabases.database_names()}}
+
+    def pwd(self, state: ReplState):
+        words = []
+
+        if state.export_session:
+            words.append(state.export_session)
+
+        return '\t'.join([f'{ReplState.X}:>'] + (words if words else ['/']))
+
+    def try_fallback_action(self, chain: Command, state: ReplState, cmd: str):
+        result = chain.run(f'.{cmd}', state)
+        if type(result) is ReplState:
+            if state.export_session and not result.export_session:
+                state.export_session = None
+
+        return True, result
+
+    def enter(self, state: ReplState):
+        if auto_enter := Config().get('repl.x.auto-enter', 'no'):
+            if auto_enter == 'latest':
+                Config().wait_log(f'Moving to the latest export database...')
+                if dbs := ExportDatabases.database_names():
+                    state.export_session = sorted(dbs)[-1]
+                else:
+                    log2('No export database found.')
+

adam/commands/devices/device_postgres.py

@@ -0,0 +1,109 @@
+from adam.commands.command import Command
+from adam.commands.devices.device import Device
+from adam.commands.postgres.postgres_context import PostgresContext
+from adam.commands.postgres.postgres_utils import pg_database_names, pg_table_names
+from adam.config import Config
+from adam.repl_state import ReplState
+from adam.utils import lines_to_tabular, log
+
+class DevicePostgres(Command, Device):
+    COMMAND = f'{ReplState.P}:'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(DevicePostgres, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return DevicePostgres.COMMAND
+
+    def run(self, cmd: str, state: ReplState):
+        if not self.args(cmd):
+            return super().run(cmd, state)
+
+        state.device = ReplState.P
+
+        return state
+
+    def completion(self, state: ReplState):
+        return super().completion(state)
+
+    def help(self, _: ReplState):
+        return f'{DevicePostgres.COMMAND}\t move to Postgres Operations device'
+
+    def ls(self, cmd: str, state: ReplState):
+        if state.pg_path:
+            pg: PostgresContext = PostgresContext.apply(state.namespace, state.pg_path)
+            if pg.db:
+                self.show_pg_tables(pg)
+            else:
+                self.show_pg_databases(pg)
+        else:
+            self.show_pg_hosts(state)
+
+    def show_pg_hosts(self, state: ReplState):
+        if state.namespace:
+            def line(pg: PostgresContext):
+                return f'{pg.path()},{pg.endpoint()}:{pg.port()},{pg.username()},{pg.password()}'
+
+            lines = [line(PostgresContext.apply(state.namespace, pg)) for pg in PostgresContext.hosts(state.namespace)]
+
+            log(lines_to_tabular(lines, 'NAME,ENDPOINT,USERNAME,PASSWORD', separator=','))
+        else:
+            def line(pg: PostgresContext):
+                return f'{pg.path()},{pg.namespace},{pg.endpoint()}:{pg.port()},{pg.username()},{pg.password()}'
+
+            lines = [line(PostgresContext.apply(state.namespace, pg)) for pg in PostgresContext.hosts(state.namespace)]
+
+            log(lines_to_tabular(lines, 'NAME,NAMESPACE,ENDPOINT,USERNAME,PASSWORD', separator=','))
+
+    def show_pg_databases(self, pg: PostgresContext):
+        log(lines_to_tabular(pg_database_names(pg.namespace, pg.path()), 'DATABASE', separator=','))
+
+    def show_pg_tables(self, pg: PostgresContext):
+        log(lines_to_tabular(pg_table_names(pg.namespace, pg.path()), 'NAME', separator=','))
+
+    def cd(self, dir: str, state: ReplState):
+        if dir == '':
+            state.pg_path = None
+        else:
+            context: PostgresContext = PostgresContext.apply(state.namespace, state.pg_path, arg=dir)
+            # patch up state.namespace from pg cd
+            if not state.namespace and context.namespace:
+                state.namespace = context.namespace
+            state.pg_path = context.path()
+
+    def cd_completion(self, cmd: str, state: ReplState, default: dict = {}):
+        pg: PostgresContext = PostgresContext.apply(state.namespace, state.pg_path) if state.pg_path else None
+        if pg and pg.db:
+            return {cmd: {'..': None}}
+        elif pg and pg.host:
+            return {cmd: {'..': None} | {p: None for p in pg_database_names(state.namespace, pg.path())}}
+        else:
+            return {cmd: {p: None for p in PostgresContext.hosts(state.namespace)}}
+
+    def pwd(self, state: ReplState):
+        words = []
+
+        pg: PostgresContext = PostgresContext.apply(state.namespace, state.pg_path)
+
+        if pg.host:
+            words.append(f'host/{pg.host}')
+        if pg.db:
+            words.append(f'database/{pg.db}')
+
+        return '\t'.join([f'{ReplState.P}:>'] + (words if words else ['/']))
+
+    def try_fallback_action(self, chain: Command, state: ReplState, cmd: str):
+        pg: PostgresContext = PostgresContext.apply(state.namespace, state.pg_path)
+        if pg.db:
+            return True, chain.run(f'pg {cmd}', state)
+
+        return False, None
+
+    def enter(self, state: ReplState):
+        Config().wait_log('Inspecting postgres database instances...')

adam/commands/devices/devices.py

@@ -0,0 +1,25 @@
+from adam.commands.devices.device import Device
+from adam.commands.devices.device_app import DeviceApp
+from adam.commands.devices.device_auit_log import DeviceAuditLog
+from adam.commands.devices.device_cass import DeviceCass
+from adam.commands.devices.device_export import DeviceExport
+from adam.commands.devices.device_postgres import DevicePostgres
+from adam.repl_state import ReplState
+
+class Devices:
+    def device(state: ReplState) -> Device:
+        if state.device == ReplState.A:
+            return DeviceApp()
+        elif state.device == ReplState.C:
+            return DeviceCass()
+        elif state.device == ReplState.L:
+            return DeviceAuditLog()
+        elif state.device == ReplState.P:
+            return DevicePostgres()
+        elif state.device == ReplState.X:
+            return DeviceExport()
+
+        return DeviceCass()
+
+    def all():
+        return [DeviceApp(), DeviceCass(), DeviceAuditLog(), DevicePostgres(), DeviceExport()]

adam/commands/export/clean_up_export_session.py

@@ -0,0 +1,53 @@
+from adam.commands.command import Command
+from adam.commands.export.exporter import Exporter
+from adam.repl_state import ReplState, RequiredState
+from adam.utils import log, log2
+
+class CleanUpExportSession(Command):
+    COMMAND = 'clean up export session'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(CleanUpExportSession, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return CleanUpExportSession.COMMAND
+
+    def required(self):
+        return RequiredState.CLUSTER_OR_POD
+
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        state, args = self.apply_state(args, state)
+        if not self.validate_state(state):
+            return state
+
+        if not args:
+            if state.in_repl:
+                log2('Specify export session name.')
+            else:
+                log2('* Session name is missing.')
+
+                Command.display_help()
+
+            return 'command-missing'
+
+        csv_cnt, log_cnt = Exporter.clean_up_session(state.sts, state.pod, state.namespace, args[0])
+        log(f'Removed {csv_cnt} csv and {log_cnt} log files.')
+
+        Exporter.clear_export_session_cache()
+
+        return state
+
+    def completion(self, _: ReplState):
+        return {}
+
+    def help(self, _: ReplState):
+        return f'{CleanUpExportSession.COMMAND} <export-session-name>\t clean up export session'

adam/commands/export/clean_up_export_sessions.py

@@ -0,0 +1,40 @@
+from adam.commands.command import Command
+from adam.commands.export.exporter import Exporter
+from adam.repl_state import ReplState, RequiredState
+
+class CleanUpExportSessions(Command):
+    COMMAND = 'clean up all export sessions'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(CleanUpExportSessions, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return CleanUpExportSessions.COMMAND
+
+    def required(self):
+        return RequiredState.CLUSTER_OR_POD
+
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        state, args = self.apply_state(args, state)
+        if not self.validate_state(state):
+            return state
+
+        if Exporter.clean_up_all_sessions(state.sts, state.pod, state.namespace):
+            Exporter.clear_export_session_cache()
+
+        return state
+
+    def completion(self, _: ReplState):
+        return {}
+
+    def help(self, _: ReplState):
+        return f'{CleanUpExportSessions.COMMAND}\t clean up all export sessions'