kaqing 1.77.0__py3-none-any.whl → 2.0.171__py3-none-any.whl

walker/commands/postgres/postgres_session.py

@@ -1,225 +0,0 @@
-import functools
-import re
-import subprocess
-
-from walker.config import Config
-from walker.k8s_utils.kube_context import KubeContext
-from walker.k8s_utils.pods import Pods
-from walker.k8s_utils.secrets import Secrets
-from walker.utils import log2
-
-class PostgresSession:
-    def __init__(self, ns: str, path: str):
-        self.namespace = ns
-        self.conn_details = None
-        self.host = None
-        self.db = None
-
-        if path:
-            tks = path.split('/')
-            hn = tks[0].split('@')
-            self.host = hn[0]
-            if len(hn) > 1 and not ns:
-                self.namespace = hn[1]
-
-            if len(tks) > 1:
-                self.db = tks[1]
-
-    # work for databases()
-    def __eq__(self, other: 'PostgresSession'):
-        return self.host == other.host
-
-    def __hash__(self):
-        return hash(self.host)
-
-    def find_namespace(self, arg: str):
-        if arg:
-            tks = arg.split('@')
-            if len(tks) > 1:
-                return tks[1]
-
-        return None
-
-    def directory(self, arg: str = None):
-        if arg:
-            if arg == '..':
-                if self.db:
-                    self.db = None
-                else:
-                    self.host = None
-            else:
-                tks = arg.split('@')
-                arg = tks[0]
-                if not self.host:
-                    self.host = arg
-                else:
-                    self.db = arg
-
-        if not self.host:
-            return None
-
-        d = self.host
-        if not self.db:
-            return d
-
-        return f'{self.host}/{self.db}'
-
-    def hosts(ns: str):
-        return PostgresSession.hosts_for_namespace(ns)
-
-    @functools.lru_cache()
-    def hosts_for_namespace(ns: str):
-        ss = Secrets.list_secrets(ns, name_pattern=Config().get('pg.name-pattern', '^{namespace}.*k8spg.*'))
-
-        def excludes(name: str):
-            exs = Config().get('pg.excludes', '.helm., -admin-secret')
-            if exs:
-                for ex in exs.split(','):
-                    if ex.strip(' ') in name:
-                        return True
-
-            return False
-
-        return [s for s in ss if not excludes(s)]
-
-    @functools.lru_cache()
-    def databases(self):
-        dbs = []
-        #  List of databases
-        #                  Name                  |  Owner   | Encoding |   Collate   |    Ctype    | ICU Locale | Locale Provider |   Access privileges
-        # ---------------------------------------+----------+----------+-------------+-------------+------------+-----------------+-----------------------
-        #  postgres                              | postgres | UTF8     | en_US.UTF-8 | en_US.UTF-8 |            | libc            |
-        #  stgawsscpsr_c3_c3                     | postgres | UTF8     | C           | C           |            | libc            |
-        #  template1                             | postgres | UTF8     | en_US.UTF-8 | en_US.UTF-8 |            | libc            | =c/postgres          +
-        #                                        |          |          |             |             |            |                 | postgres=CTc/postgres
-        # (48 rows)
-        r = self.run_sql('\l', show_out=False)
-        s = 0
-        for line in r.stdout.split('\n'):
-            line: str = line.strip(' \r')
-            if s == 0:
-                if 'List of databases' in line:
-                    s = 1
-            elif s == 1:
-                if 'Name' in line and 'Owner' in line and 'Encoding' in line:
-                    s = 2
-            elif s == 2:
-                if line.startswith('---------'):
-                    s = 3
-            elif s == 3:
-                groups = re.match(r'^\s*(\S*)\s*\|\s*(\S*)\s*\|.*', line)
-                if groups and groups[1] != '|':
-                    dbs.append({'name': groups[1], 'owner': groups[2]})
-
-        return dbs
-
-    def tables(self):
-        dbs = []
-        #                                            List of relations
-        #   Schema  |                            Name                            | Type  |     Owner
-        # ----------+------------------------------------------------------------+-------+---------------
-        #  postgres | c3_2_admin_aclpriv                                         | table | postgres
-        #  postgres | c3_2_admin_aclpriv_a                                       | table | postgres
-        r = self.run_sql('\dt', show_out=False)
-        s = 0
-        for line in r.stdout.split('\n'):
-            line: str = line.strip(' \r')
-            if s == 0:
-                if 'List of relations' in line:
-                    s = 1
-            elif s == 1:
-                if 'Schema' in line and 'Name' in line and 'Type' in line:
-                    s = 2
-            elif s == 2:
-                if line.startswith('---------'):
-                    s = 3
-            elif s == 3:
-                groups = re.match(r'^\s*(\S*)\s*\|\s*(\S*)\s*\|.*', line)
-                if groups and groups[1] != '|':
-                    dbs.append({'schema': groups[1], 'name': groups[2]})
-
-        return dbs
-
-    def run_sql(self, sql: str, show_out = True):
-        db = self.db if self.db else PostgresSession.default_db()
-
-        if KubeContext.in_cluster():
-            cmd1 = f'env PGPASSWORD={self.password()} psql -h {self.endpoint()} -p {self.port()} -U {self.username()} {db} --pset pager=off -c'
-            log2(f'{cmd1} "{sql}"')
-            # remove double quotes from the sql argument
-            cmd = cmd1.split(' ') + [sql]
-            r = subprocess.run(cmd, capture_output=True, text=True)
-            if show_out:
-                log2(r.stdout)
-                log2(r.stderr)
-
-            return r
-        else:
-            ns = self.namespace
-            image = Config().get('pg.agent.image', 'seanahnsf/kaqing')
-            pod_name = Config().get('pg.agent.name', 'kaqing-agent')
-            timeout = Config().get('pg.agent.timeout', 3600)
-
-            try:
-                Pods.create(ns, pod_name, image, ['sleep', f'{timeout}'], env={'NAMESPACE': ns}, sa_name='c3')
-            except Exception as e:
-                if e.status == 409:
-                    if Pods.completed(ns, pod_name):
-                        try:
-                            Pods.delete(pod_name, ns)
-                            Pods.create(ns, pod_name, image, ['sleep', f'{timeout}'], env={'NAMESPACE': ns}, sa_name='c3')
-                        except Exception as e2:
-                            log2("Exception when calling BatchV1Api->create_pod: %s\n" % e2)
-
-                            return
-                else:
-                    log2("Exception when calling BatchV1Api->create_pod: %s\n" % e)
-
-                    return
-
-            Pods.wait_for_running(ns, pod_name)
-
-            cmd = f'PGPASSWORD="{self.password()}" psql -h {self.endpoint()} -p {self.port()} -U {self.username()} {db} --pset pager=off -c "{sql}"'
-
-            return Pods.exec(pod_name, pod_name, ns, cmd, show_out=show_out)
-
-    def endpoint(self):
-        if not self.conn_details:
-            self.conn_details = Secrets.get_data(self.namespace, self.host)
-
-        endpoint_key = Config().get('pg.secret.endpoint-key', 'postgres-db-endpoint')
-
-        return self.conn_details[endpoint_key] if endpoint_key in self.conn_details else ''
-
-    def port(self):
-        if not self.conn_details:
-            self.conn_details = Secrets.get_data(self.namespace, self.host)
-
-        port_key = Config().get('pg.secret.port-key', 'postgres-db-port')
-
-        return  self.conn_details[port_key] if port_key in self.conn_details else ''
-
-    def username(self):
-        if not self.conn_details:
-            self.conn_details = Secrets.get_data(self.namespace, self.host)
-
-        username_key = Config().get('pg.secret.username-key', 'postgres-admin-username')
-
-        return  self.conn_details[username_key] if username_key in self.conn_details else ''
-
-    def password(self):
-        if not self.conn_details:
-            self.conn_details = Secrets.get_data(self.namespace, self.host)
-
-        password_key = Config().get('pg.secret.password-key', 'postgres-admin-password')
-
-        return  self.conn_details[password_key] if password_key in self.conn_details else ''
-
-    def default_db():
-        return Config().get('pg.default-db', 'postgres')
-
-    def default_owner():
-        return Config().get('pg.default-owner', 'postgres')
-
-    def default_schema():
-        return Config().get('pg.default-schema', 'postgres')

walker/commands/preview_table.py

@@ -1,98 +0,0 @@
-import functools
-
-from walker.commands.command import Command
-from walker.commands.cql_utils import parse_cql_desc_tables, run_cql
-from walker.commands.postgres.postgres_session import PostgresSession
-from walker.config import Config
-from walker.pod_exec_result import PodExecResult
-from walker.repl_state import ReplState, RequiredState
-from walker.utils import lines_to_tabular, log, log2
-
-class PreviewTable(Command):
-    COMMAND = 'preview'
-
-    # the singleton pattern
-    def __new__(cls, *args, **kwargs):
-        if not hasattr(cls, 'instance'): cls.instance = super(PreviewTable, cls).__new__(cls)
-
-        return cls.instance
-
-    def __init__(self, successor: Command=None):
-        super().__init__(successor)
-
-    def command(self):
-        return PreviewTable.COMMAND
-
-    def required(self):
-        return RequiredState.CLUSTER_OR_POD
-
-    def run(self, cmd: str, state: ReplState):
-        if not(args := self.args(cmd)):
-            return super().run(cmd, state)
-
-        state, args = self.apply_state(args, state)
-        if state.device == ReplState.P:
-            if not self.validate_state(state, RequiredState.PG_DATABASE):
-                return state
-        else:
-            if not self.validate_state(state):
-                return state
-
-        if not args:
-            def show_tables():
-                if state.device == ReplState.P:
-                    pg = PostgresSession(state.namespace, state.pg_path)
-                    lines = [db["name"] for db in pg.tables() if db["schema"] == PostgresSession.default_schema()]
-                    log(lines_to_tabular(lines, separator=','))
-                else:
-                    run_cql(state, f'describe tables', show_out=True)
-
-            if state.in_repl:
-                log2('Table is required.')
-                log2()
-                log2('Tables:')
-                show_tables()
-            else:
-                log2('* Table is missing.')
-                show_tables()
-
-                Command.display_help()
-
-            return 'command-missing'
-
-        table = args[0]
-
-        rows = Config().get('preview.rows', 10)
-        if state.device == ReplState.P:
-            PostgresSession(state.namespace, state.pg_path).run_sql(f'select * from {table} limit {rows}')
-        else:
-            run_cql(state, f'select * from {table} limit {rows}', show_out=True, use_single_quotes=True)
-
-        return state
-
-    def completion(self, state: ReplState):
-        if state.device == ReplState.P:
-            if tables := PreviewTable.pg_tables(state.namespace, state.pg_path):
-                return {PreviewTable.COMMAND: {db["name"]: None for db in tables if db["schema"] == PostgresSession.default_schema()}}
-        else:
-            if state.pod:
-                tables = PreviewTable.cql_tables(state)
-                return {PreviewTable.COMMAND: {f'{k}.{t}': None for k, ts in tables.items() for t in ts}}
-
-        return {}
-
-    def help(self, _: ReplState):
-        return f'{PreviewTable.COMMAND} TABLE\t preview table'
-
-    @functools.lru_cache()
-    def cql_tables(state: ReplState):
-        r: PodExecResult =  run_cql(state, 'describe tables', show_out=False)
-        return parse_cql_desc_tables(r.stdout)
-
-    @functools.lru_cache()
-    def pg_tables(ns: str, pg_path: str):
-        pg = PostgresSession(ns, pg_path)
-        if pg.db:
-            return pg.tables()
-
-        return None

walker/commands/processes.py

@@ -1,53 +0,0 @@
-from walker.commands.command import Command
-from walker.commands.commands_utils import show_table
-from walker.config import Config
-from walker.k8s_utils.statefulsets import StatefulSets
-from walker.repl_state import ReplState, RequiredState
-
-class Processes(Command):
-    COMMAND = 'processes'
-
-    # the singleton pattern
-    def __new__(cls, *args, **kwargs):
-        if not hasattr(cls, 'instance'): cls.instance = super(Processes, cls).__new__(cls)
-
-        return cls.instance
-
-    def __init__(self, successor: Command=None):
-        super().__init__(successor)
-
-    def command(self):
-        return Processes.COMMAND
-
-    def required(self):
-        return RequiredState.CLUSTER_OR_POD
-
-    def run(self, cmd: str, state: ReplState):
-        if not(args := self.args(cmd)):
-            return super().run(cmd, state)
-
-        state, args = self.apply_state(args, state)
-        if not self.validate_state(state):
-            return state
-
-        args, show_output = Command.extract_options(args, ['-s', '--show'])
-
-        cols = Config().get('processes.columns', 'pod,cpu,mem')
-        header = Config().get('processes.header', 'POD_NAME,CPU,MEM/LIMIT')
-
-        if state.pod:
-            show_table(state, [state.pod], cols, header, show_output=show_output)
-        elif state.sts:
-            pod_names = [pod.metadata.name for pod in StatefulSets.pods(state.sts, state.namespace)]
-            show_table(state, pod_names, cols, header, show_output=show_output)
-
-        return state
-
-    def completion(self, state: ReplState):
-        if not state.sts:
-            return {}
-
-        return {Processes.COMMAND: None}
-
-    def help(self, _: ReplState):
-        return f'{Processes.COMMAND} [-s]\t show process overview  -s show commands on nodes'

walker/commands/pwd.py

@@ -1,64 +0,0 @@
-from walker.commands.command import Command
-from walker.commands.postgres.postgres_session import PostgresSession
-from walker.repl_state import ReplState
-from walker.utils import lines_to_tabular, log
-
-class Pwd(Command):
-    COMMAND = 'pwd'
-
-    # the singleton pattern
-    def __new__(cls, *args, **kwargs):
-        if not hasattr(cls, 'instance'): cls.instance = super(Pwd, cls).__new__(cls)
-
-        return cls.instance
-
-    def __init__(self, successor: Command=None):
-        super().__init__(successor)
-
-    def command(self):
-        return Pwd.COMMAND
-
-    def run(self, cmd: str, state: ReplState):
-        if not(args := self.args(cmd)):
-            return super().run(cmd, state)
-
-        state, args = self.apply_state(args, state)
-
-        def device_line(state: ReplState, device: str):
-            words = []
-
-            if device == ReplState.P:
-                pg = PostgresSession(state.namespace, state.pg_path)
-
-                if pg.host:
-                    words.append(f'host/{pg.host}')
-                if pg.db:
-                    words.append(f'database/{pg.db}')
-            elif device == ReplState.A:
-                if state.app_env:
-                    words.append(f'env/{state.app_env}')
-                if state.app_app:
-                    words.append(f'app/{state.app_app}')
-            else:
-                if state.sts:
-                    words.append(f'sts/{state.sts}')
-                if state.pod:
-                    words.append(f'pod/{state.pod}')
-
-            return '\t'.join([f'{device}:>'] + (words if words else ['/']))
-
-        log(lines_to_tabular([
-            device_line(state, ReplState.A),
-            device_line(state, ReplState.C),
-            device_line(state, ReplState.P),
-            f'NAMESPACE\t{state.namespace if state.namespace else "/"}'
-        ], 'DEVICE\tLOCATION', separator='\t'))
-        log()
-
-        return state
-
-    def completion(self, state: ReplState):
-        return super().completion(state)
-
-    def help(self, _: ReplState):
-        return f'{Pwd.COMMAND}\t print current working directories'

walker/commands/reaper/reaper.py

@@ -1,78 +0,0 @@
-import click
-
-from walker.commands.command import Command
-from walker.commands.command_helpers import ClusterCommandHelper
-from .reaper_forward import ReaperForward
-from .reaper_forward_stop import ReaperForwardStop
-from .reaper_restart import ReaperRestart
-from .reaper_run_abort import ReaperRunAbort
-from .reaper_runs import ReaperRuns
-from .reaper_runs_abort import ReaperRunsAbort
-from .reaper_schedule_activate import ReaperScheduleActivate
-from .reaper_schedule_start import ReaperScheduleStart
-from .reaper_schedule_stop import ReaperScheduleStop
-from .reaper_schedules import ReaperSchedules
-from .reaper_status import ReaperStatus
-from walker.repl_state import ReplState, RequiredState
-from walker.utils import lines_to_tabular, log, log2
-
-class Reaper(Command):
-    COMMAND = 'reaper'
-    reaper_login = None
-
-    # the singleton pattern
-    def __new__(cls, *args, **kwargs):
-        if not hasattr(cls, 'instance'): cls.instance = super(Reaper, cls).__new__(cls)
-
-        return cls.instance
-
-    def __init__(self, successor: Command=None):
-        super().__init__(successor)
-
-    def command(self):
-        return Reaper.COMMAND
-
-    def required(self):
-        return RequiredState.CLUSTER
-
-    def run(self, cmd: str, state: ReplState):
-        if not(args := self.args(cmd)):
-            return super().run(cmd, state)
-
-        state, args = self.apply_state(args, state)
-        if not self.validate_state(state):
-            return state
-
-        if state.in_repl:
-            log(lines_to_tabular([c.help(ReplState()) for c in Reaper.cmd_list()], separator=':'))
-
-            return 'command-missing'
-        else:
-            # head with the Chain of Responsibility pattern
-            cmds = Command.chain(Reaper.cmd_list())
-            if not cmds.run(cmd, state):
-                log2('* Command is missing.')
-                Command.display_help()
-
-    def cmd_list():
-        return [ReaperSchedules(), ReaperScheduleStop(), ReaperScheduleActivate(), ReaperScheduleStart(),
-                ReaperForwardStop(), ReaperForward(), ReaperRunAbort(), ReaperRunsAbort(), ReaperRestart(), ReaperRuns(), ReaperStatus()]
-
-    def completion(self, state: ReplState):
-        if state.sts:
-            return super().completion(state)
-
-        return {}
-
-    def help(self, _: ReplState):
-        return None
-
-class ReaperCommandHelper(click.Command):
-    def get_help(self, ctx: click.Context):
-        log(super().get_help(ctx))
-        log()
-        log('Sub-Commands:')
-
-        log(lines_to_tabular([c.help(ReplState()).replace(f'{Reaper.COMMAND} ', '  ', 1) for c in Reaper.cmd_list()], separator=':'))
-        log()
-        ClusterCommandHelper.cluster_help()

walker/commands/reaper/reaper_forward.py

@@ -1,100 +0,0 @@
-import threading
-import time
-
-from walker.commands.command import Command
-from .reaper_session import ReaperSession
-from walker.config import Config
-from walker.repl_session import ReplSession
-from walker.repl_state import ReplState, RequiredState
-from walker.utils import lines_to_tabular, log2
-
-class ReaperForward(Command):
-    COMMAND = 'reaper forward'
-    reaper_login = None
-
-    # the singleton pattern
-    def __new__(cls, *args, **kwargs):
-        if not hasattr(cls, 'instance'): cls.instance = super(ReaperForward, cls).__new__(cls)
-
-        return cls.instance
-
-    def __init__(self, successor: Command=None):
-        super().__init__(successor)
-
-    def command(self):
-        return ReaperForward.COMMAND
-
-    def required(self):
-        return RequiredState.CLUSTER
-
-    def run(self, cmd: str, state: ReplState):
-        if not(args := self.args(cmd)):
-            return super().run(cmd, state)
-
-        state, args = self.apply_state(args, state)
-        if not self.validate_state(state):
-            return state
-
-        if not(reaper := ReaperSession.create(state)):
-            return state
-
-        spec = reaper.reaper_spec(state)
-        if state.in_repl:
-            if ReaperSession.is_forwarding:
-                log2("Another port-forward is already running.")
-
-                return "already-running"
-
-            # make it a daemon to exit with a Ctrl-D
-            thread = threading.Thread(target=self.loop, args=(state, reaper), daemon=True)
-            thread.start()
-
-            while not ReaperSession.is_forwarding:
-                time.sleep(1)
-
-            d = {
-                'reaper-ui': spec["web-uri"],
-                'reaper-username': spec["username"],
-                'reaper-password': spec["password"]
-            }
-            log2()
-            log2(lines_to_tabular([f'{k},{v}' for k, v in d.items()], separator=','))
-
-            for k, v in d.items():
-                ReplSession().prompt_session.history.append_string(f'cp {k}')
-            log2()
-            log2(f'Use <Up> arrow key to copy the values to clipboard.')
-        else:
-            try:
-                log2(f'Click: {spec["web-uri"]}')
-                log2(f'username: {spec["username"]}')
-                log2(f'password: {spec["password"]}')
-                log2()
-                log2(f"Press Ctrl+C to break.")
-
-                time.sleep(Config().get('reaper.port-forward.timeout', 3600 * 24))
-            except KeyboardInterrupt:
-                pass
-
-        return state
-
-    def loop(self, state: ReplState, reaper: ReaperSession):
-        def body(uri: str, _: dict[str, str]):
-            ReaperSession.is_forwarding = True
-            try:
-                while not ReaperSession.stopping.is_set():
-                    time.sleep(1)
-            finally:
-                ReaperSession.stopping.clear()
-                ReaperSession.is_forwarding = False
-
-        return reaper.port_forwarded(state, 'webui', body)
-
-    def completion(self, state: ReplState):
-        if state.sts:
-            return super().completion(state)
-
-        return {}
-
-    def help(self, _: ReplState):
-        return f'{ReaperForward.COMMAND}\t port-forward to reaper'

walker/commands/reaper/reaper_run_abort.py

@@ -1,65 +0,0 @@
-import requests
-
-from walker.commands.command import Command
-from .reaper_session import ReaperSession
-from walker.repl_state import ReplState, RequiredState
-from walker.utils import log2
-
-class ReaperRunAbort(Command):
-    COMMAND = 'reaper abort run'
-    reaper_login = None
-
-    # the singleton pattern
-    def __new__(cls, *args, **kwargs):
-        if not hasattr(cls, 'instance'): cls.instance = super(ReaperRunAbort, cls).__new__(cls)
-
-        return cls.instance
-
-    def __init__(self, successor: Command=None):
-        super().__init__(successor)
-
-    def command(self):
-        return ReaperRunAbort.COMMAND
-
-    def required(self):
-        return RequiredState.CLUSTER
-
-    def run(self, cmd: str, state: ReplState):
-        if not(args := self.args(cmd)):
-            return super().run(cmd, state)
-
-        state, args = self.apply_state(args, state)
-        if not self.validate_state(state):
-            return state
-
-        if not args:
-            if state.in_repl:
-                log2('Specify run id to abort.')
-            else:
-                Command.display_help()
-
-            return state
-
-        if not(reaper := ReaperSession.create(state)):
-            return state
-
-        self.stop_run(state, reaper, args[0])
-
-        return state
-
-    def stop_run(self, state: ReplState, reaper: ReaperSession, run_id: str):
-        def body(uri: str, headers: dict[str, str]):
-            return requests.put(uri, headers=headers)
-
-        # PAUSED, RUNNING, ABORTED
-        # PUT /repair_run/{id}/state/{state}
-        reaper.port_forwarded(state, f'repair_run/{run_id}/state/ABORTED', body, method='PUT')
-
-    def completion(self, state: ReplState):
-        if state.sts:
-            return super().completion(state)
-
-        return {}
-
-    def help(self, _: ReplState):
-        return f'{ReaperRunAbort.COMMAND} <run-id>\t abort reaper run'