kaqing 1.77.0__py3-none-any.whl → 2.0.171__py3-none-any.whl

adam/commands/postgres/postgres_context.py

@@ -0,0 +1,274 @@
+from datetime import datetime
+import functools
+import re
+import subprocess
+
+from adam.config import Config
+from adam.repl_session import ReplSession
+from adam.utils_k8s.kube_context import KubeContext
+from adam.utils_k8s.pods import Pods
+from adam.utils_k8s.secrets import Secrets
+from adam.utils import log2
+
+class PostgresContext:
+    def apply(namespace: str, path: str, arg: str = None) -> 'PostgresContext':
+        context = PostgresContext(namespace, path)
+
+        if arg:
+            if arg == '..':
+                if context.db:
+                    context.db = None
+                else:
+                    context.host = None
+            else:
+                tks = arg.split('@')
+                if not context.host:
+                    context.host = tks[0]
+                else:
+                    context.db = tks[0]
+
+                if not namespace and tks[1]:
+                    context.namespace = tks[1]
+
+        return context
+
+    def __init__(self, ns: str, path: str):
+        self.namespace = ns
+        self.conn_details = None
+        self.host = None
+        self.db = None
+
+        if path:
+            tks = path.split('/')
+            hn = tks[0].split('@')
+            self.host = hn[0]
+            if len(hn) > 1 and not ns:
+                self.namespace = hn[1]
+
+            if len(tks) > 1:
+                self.db = tks[1]
+
+    def path(self):
+        if not self.host:
+            return None
+
+        d = self.host
+        if not self.db:
+            return d
+
+        return f'{self.host}/{self.db}'
+
+    def hosts(ns: str):
+        return PostgresContext.hosts_for_namespace(ns)
+
+    @functools.lru_cache()
+    def hosts_for_namespace(ns: str):
+        ss = Secrets.list_secrets(ns, name_pattern=Config().get('pg.name-pattern', '^{namespace}.*k8spg.*'))
+
+        def excludes(name: str):
+            exs = Config().get('pg.excludes', '.helm., -admin-secret')
+            if exs:
+                for ex in exs.split(','):
+                    if ex.strip(' ') in name:
+                        return True
+
+            return False
+
+        return [s for s in ss if not excludes(s)]
+
+    def databases(self):
+        dbs = []
+        #  List of databases
+        #                  Name                  |  Owner   | Encoding |   Collate   |    Ctype    | ICU Locale | Locale Provider |   Access privileges
+        # ---------------------------------------+----------+----------+-------------+-------------+------------+-----------------+-----------------------
+        #  postgres                              | postgres | UTF8     | en_US.UTF-8 | en_US.UTF-8 |            | libc            |
+        #  stgawsscpsr_c3_c3                     | postgres | UTF8     | C           | C           |            | libc            |
+        #  template1                             | postgres | UTF8     | en_US.UTF-8 | en_US.UTF-8 |            | libc            | =c/postgres          +
+        #                                        |          |          |             |             |            |                 | postgres=CTc/postgres
+        # (48 rows)
+        if r := self.run_sql('\l', show_out=False):
+            s = 0
+            for line in r.stdout.split('\n'):
+                line: str = line.strip(' \r')
+                if s == 0:
+                    if 'List of databases' in line:
+                        s = 1
+                elif s == 1:
+                    if 'Name' in line and 'Owner' in line and 'Encoding' in line:
+                        s = 2
+                elif s == 2:
+                    if line.startswith('---------'):
+                        s = 3
+                elif s == 3:
+                    groups = re.match(r'^\s*(\S*)\s*\|\s*(\S*)\s*\|.*', line)
+                    if groups and groups[1] != '|':
+                        dbs.append({'name': groups[1], 'owner': groups[2]})
+
+        return dbs
+
+    def tables(self):
+        dbs = []
+        #                                            List of relations
+        #   Schema  |                            Name                            | Type  |     Owner
+        # ----------+------------------------------------------------------------+-------+---------------
+        #  postgres | c3_2_admin_aclpriv                                         | table | postgres
+        #  postgres | c3_2_admin_aclpriv_a                                       | table | postgres
+        if r := self.run_sql('\dt', show_out=False):
+            s = 0
+            for line in r.stdout.split('\n'):
+                line: str = line.strip(' \r')
+                if s == 0:
+                    if 'List of relations' in line:
+                        s = 1
+                elif s == 1:
+                    if 'Schema' in line and 'Name' in line and 'Type' in line:
+                        s = 2
+                elif s == 2:
+                    if line.startswith('---------'):
+                        s = 3
+                elif s == 3:
+                    groups = re.match(r'^\s*(\S*)\s*\|\s*(\S*)\s*\|.*', line)
+                    if groups and groups[1] != '|':
+                        dbs.append({'schema': groups[1], 'name': groups[2]})
+
+        return dbs
+
+    def run_sql(self, sql: str, show_out = True, background = False):
+        db = self.db if self.db else PostgresContext.default_db()
+
+        if KubeContext.in_cluster():
+            cmd1 = f'env PGPASSWORD={self.password()} psql -h {self.endpoint()} -p {self.port()} -U {self.username()} {db} --pset pager=off -c'
+            log2(f'{cmd1} "{sql}"')
+            # remove double quotes from the sql argument
+            cmd = cmd1.split(' ') + [sql]
+
+            r = subprocess.run(cmd, capture_output=not background, text=True)
+            if show_out:
+                log2(r.stdout)
+                log2(r.stderr)
+
+            return r
+        else:
+            pod_name, container_name = PostgresContext.pod_and_container(self.namespace)
+            if not pod_name:
+                return
+
+            # ns = self.namespace
+            # pod_name = Config().get('pg.agent.name', 'ops-pg-agent')
+
+            # if Config().get('pg.agent.just-in-time', False):
+            #     if not PostgresContext.deploy_pg_agent(pod_name, ns):
+            #         return
+
+            # real_pod_name = pod_name
+            # try:
+            #     # try with dedicated pg agent pod name configured
+            #     Pods.get(ns, pod_name)
+            # except:
+            #     try:
+            #         # try with the ops pod
+            #         pod_name = Config().get('pod.name', 'ops')
+            #         real_pod_name = Pods.get_with_selector(ns, label_selector = Config().get('pod.label-selector', 'run=ops')).metadata.name
+            #     except:
+            #         log2(f"Could not locate {pod_name} pod.")
+            #         return None
+
+            cmd = f'psql -h {self.endpoint()} -p {self.port()} -U {self.username()} {db} --pset pager=off -c "{sql}"'
+            env_prefix = f'PGPASSWORD="{self.password()}"'
+
+            r = Pods.exec(pod_name, container_name, self.namespace, cmd, show_out=show_out, background=background, env_prefix=env_prefix)
+            if r and Config().get('repl.history.push-cat-remote-log-file', True):
+                if r.log_file and ReplSession().prompt_session:
+                    ReplSession().prompt_session.history.append_string(f'@{r.pod} cat {r.log_file}')
+
+            return r
+
+    def pod_and_container(ns: str):
+        container_name = Config().get('pg.agent.name', 'ops-pg-agent')
+
+        if Config().get('pg.agent.just-in-time', False):
+            if not PostgresContext.deploy_pg_agent(container_name, ns):
+                return None
+
+        pod_name = container_name
+        try:
+            # try with dedicated pg agent pod name configured
+            Pods.get(ns, container_name)
+        except:
+            try:
+                # try with the ops pod
+                container_name = Config().get('pod.name', 'ops')
+                pod_name = Pods.get_with_selector(ns, label_selector = Config().get('pod.label-selector', 'run=ops')).metadata.name
+            except:
+                log2(f"Could not locate {container_name} pod.")
+                return None
+
+        return pod_name, container_name
+
+    def deploy_pg_agent(pod_name: str, ns: str) -> str:
+        image = Config().get('pg.agent.image', 'seanahnsf/kaqing')
+        timeout = Config().get('pg.agent.timeout', 3600)
+        try:
+            Pods.create(ns, pod_name, image, ['sleep', f'{timeout}'], env={'NAMESPACE': ns}, sa_name='c3')
+        except Exception as e:
+            if e.status == 409:
+                if Pods.completed(ns, pod_name):
+                    try:
+                        Pods.delete(pod_name, ns)
+                        Pods.create(ns, pod_name, image, ['sleep', f'{timeout}'], env={'NAMESPACE': ns}, sa_name='c3')
+                    except Exception as e2:
+                        log2("Exception when calling BatchV1Api->create_pod: %s\n" % e2)
+
+                        return
+            else:
+                log2("Exception when calling BatchV1Api->create_pod: %s\n" % e)
+
+                return
+
+        Pods.wait_for_running(ns, pod_name)
+
+        return pod_name
+
+    def undeploy_pg_agent(pod_name: str, ns: str):
+        Pods.delete(pod_name, ns, grace_period_seconds=0)
+
+    def endpoint(self):
+        if not self.conn_details:
+            self.conn_details = Secrets.get_data(self.namespace, self.host)
+
+        endpoint_key = Config().get('pg.secret.endpoint-key', 'postgres-db-endpoint')
+
+        return self.conn_details[endpoint_key] if endpoint_key in self.conn_details else ''
+
+    def port(self):
+        if not self.conn_details:
+            self.conn_details = Secrets.get_data(self.namespace, self.host)
+
+        port_key = Config().get('pg.secret.port-key', 'postgres-db-port')
+
+        return  self.conn_details[port_key] if port_key in self.conn_details else ''
+
+    def username(self):
+        if not self.conn_details:
+            self.conn_details = Secrets.get_data(self.namespace, self.host)
+
+        username_key = Config().get('pg.secret.username-key', 'postgres-admin-username')
+
+        return  self.conn_details[username_key] if username_key in self.conn_details else ''
+
+    def password(self):
+        if not self.conn_details:
+            self.conn_details = Secrets.get_data(self.namespace, self.host)
+
+        password_key = Config().get('pg.secret.password-key', 'postgres-admin-password')
+
+        return  self.conn_details[password_key] if password_key in self.conn_details else ''
+
+    def default_db():
+        return Config().get('pg.default-db', 'postgres')
+
+    def default_owner():
+        return Config().get('pg.default-owner', 'postgres')
+
+    def default_schema():
+        return Config().get('pg.default-schema', 'postgres')

{walker → adam}/commands/postgres/postgres_ls.py

@@ -1,10 +1,9 @@
-from walker.commands.command import Command
-from walker.commands.ls import Ls
-from walker.repl_state import ReplState, RequiredState
+from adam.commands.command import Command
+from adam.commands.ls import Ls
+from adam.repl_state import ReplState, RequiredState
 
 class PostgresLs(Command):
     COMMAND = 'pg ls'
-    reaper_login = None
 
     # the singleton pattern
     def __new__(cls, *args, **kwargs):
@@ -25,15 +24,12 @@ class PostgresLs(Command):
         if not(args := self.args(cmd)):
             return super().run(cmd, state)
 
-        state, args = self.apply_state(args, state)
-        if not self.validate_state(state):
-            return state
-
-        state.device = ReplState.P
+        with self.validate(args, state) as (args, state):
+            state.device = ReplState.P
 
-        Ls().run('ls', state)
+            Ls().run('ls', state)
 
-        return state
+            return state
 
     def completion(self, state: ReplState):
         if state.sts:

{walker → adam}/commands/postgres/postgres_preview.py

@@ -1,11 +1,9 @@
-from walker.commands.command import Command
-from walker.commands.ls import Ls
-from walker.commands.preview_table import PreviewTable
-from walker.repl_state import ReplState, RequiredState
+from adam.commands.command import Command
+from adam.commands.preview_table import PreviewTable
+from adam.repl_state import ReplState, RequiredState
 
 class PostgresPreview(Command):
     COMMAND = 'pg preview'
-    reaper_login = None
 
     # the singleton pattern
     def __new__(cls, *args, **kwargs):
@@ -20,21 +18,18 @@ class PostgresPreview(Command):
         return PostgresPreview.COMMAND
 
     def required(self):
-        return RequiredState.NAMESPACE
+        return RequiredState.PG_DATABASE
 
     def run(self, cmd: str, state: ReplState):
         if not(args := self.args(cmd)):
             return super().run(cmd, state)
 
-        state, args = self.apply_state(args, state)
-        if not self.validate_state(state, RequiredState.PG_DATABASE):
-            return state
-
-        state.device = ReplState.P
+        with self.validate(args, state) as (args, state):
+            state.device = ReplState.P
 
-        PreviewTable().run(f'preview {" ".join(args)}', state)
+            PreviewTable().run(f'preview {" ".join(args)}', state)
 
-        return state
+            return state
 
     def completion(self, state: ReplState):
         if state.sts:

adam/commands/postgres/psql_completions.py

@@ -0,0 +1,10 @@
+from adam.commands.postgres.utils_postgres import pg_table_names
+from adam.sql.sql_completer import SqlCompleter
+
+def psql_completions(ns: str, pg_path: str):
+    return {
+        '\h': None,
+        '\d': None,
+        '\dt': None,
+        '\du': None
+    } | SqlCompleter(lambda: pg_table_names(ns, pg_path)).completions_for_nesting()

adam/commands/postgres/utils_postgres.py

@@ -0,0 +1,66 @@
+import functools
+
+from adam.commands.postgres.postgres_context import PostgresContext
+from adam.repl_state import ReplState
+from adam.utils import log2, wait_log
+from adam.utils_k8s.pods import Pods
+
+TestPG = [False]
+
+@functools.lru_cache()
+def pg_database_names(ns: str, pg_path: str):
+    if TestPG[0]:
+        return ['azops88_c3ai_c3']
+
+    wait_log('Inspecting Postgres Databases...')
+
+    pg = PostgresContext.apply(ns, pg_path)
+    return [db['name'] for db in pg.databases() if db['owner'] == PostgresContext.default_owner()]
+
+@functools.lru_cache()
+def pg_table_names(ns: str, pg_path: str):
+    if TestPG[0]:
+        return ['C3_2_XYZ1']
+
+    wait_log('Inspecting Postgres Database...')
+    return [table['name'] for table in pg_tables(ns, pg_path) if table['schema'] == PostgresContext.default_schema()]
+
+def pg_tables(ns: str, pg_path: str):
+    pg = PostgresContext.apply(ns, pg_path)
+    if pg.db:
+        return pg.tables()
+
+    return []
+
+class PostgresPodService:
+    def __init__(self, handler: 'PostgresExecHandler'):
+        self.handler = handler
+
+    def exec(self, command: str, show_out=True):
+        state = self.handler.state
+
+        pod, container = PostgresContext.pod_and_container(state.namespace)
+        if not pod:
+            log2('Cannot locate postgres agent or ops pod.')
+            return state
+
+        return Pods.exec(pod, container, state.namespace, command, show_out=show_out)
+
+    def sql(self, args: list[str], background=False):
+        state = self.handler.state
+
+        PostgresContext.apply(state.namespace, state.pg_path).run_sql(' '.join(args), background=background)
+
+class PostgresExecHandler:
+    def __init__(self, state: ReplState, background=False):
+        self.state = state
+        self.background = background
+
+    def __enter__(self):
+        return PostgresPodService(self)
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        return False
+
+def postgres(state: ReplState, background=False):
+    return PostgresExecHandler(state, background=background)

adam/commands/preview_table.py

@@ -0,0 +1,37 @@
+from adam.commands.command import Command
+from adam.commands.devices.devices import Devices
+from adam.repl_state import ReplState, RequiredState
+
+class PreviewTable(Command):
+    COMMAND = 'preview'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(PreviewTable, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return PreviewTable.COMMAND
+
+    def required(self):
+        return [RequiredState.CLUSTER_OR_POD, RequiredState.PG_DATABASE, ReplState.L]
+
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        with self.validate(args, state) as (args, state):
+            Devices.device(state).preview(args[0] if args else None, state)
+
+            return state
+
+    def completion(self, _: ReplState):
+        # taken care of by the sql completer
+        return {}
+
+    def help(self, _: ReplState):
+        return f'{PreviewTable.COMMAND} TABLE\t preview table'

adam/commands/pwd.py

@@ -0,0 +1,47 @@
+from adam.app_session import AppSession
+from adam.commands.command import Command
+from adam.commands.devices.devices import Devices
+from adam.repl_state import ReplState
+from adam.utils import lines_to_tabular, log
+
+class Pwd(Command):
+    COMMAND = 'pwd'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(Pwd, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return Pwd.COMMAND
+
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        with self.validate(args, state) as (_, state):
+            host = "unknown"
+            try:
+                app_session: AppSession = AppSession.create('c3', 'c3')
+                host = app_session.host
+            except:
+                pass
+
+            log(lines_to_tabular([device.pwd(state) for device in Devices.all()] + [
+                f'',
+                f'HOST\t{host}',
+                f'NAMESPACE\t{state.namespace if state.namespace else "/"}',
+            ], 'DEVICE\tLOCATION', separator='\t'))
+            log()
+
+            return state
+
+    def completion(self, state: ReplState):
+        return super().completion(state)
+
+    def help(self, _: ReplState):
+        return f'{Pwd.COMMAND}\t print current working directories'

adam/commands/reaper/reaper.py

@@ -0,0 +1,35 @@
+import click
+
+from adam.commands.intermediate_command import IntermediateCommand
+from .reaper_forward import ReaperForward
+from .reaper_forward_stop import ReaperForwardStop
+from .reaper_restart import ReaperRestart
+from .reaper_run_abort import ReaperRunAbort
+from .reaper_runs import ReaperRuns
+from .reaper_runs_abort import ReaperRunsAbort
+from .reaper_schedule_activate import ReaperScheduleActivate
+from .reaper_schedule_start import ReaperScheduleStart
+from .reaper_schedule_stop import ReaperScheduleStop
+from .reaper_schedules import ReaperSchedules
+from .reaper_status import ReaperStatus
+
+class Reaper(IntermediateCommand):
+    COMMAND = 'reaper'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(Reaper, cls).__new__(cls)
+
+        return cls.instance
+
+    def command(self):
+        return Reaper.COMMAND
+
+    def cmd_list(self):
+        return [ReaperSchedules(), ReaperScheduleStop(), ReaperScheduleActivate(), ReaperScheduleStart(),
+                ReaperForwardStop(), ReaperForward(), ReaperRunAbort(), ReaperRunsAbort(), ReaperRestart(),
+                ReaperRuns(), ReaperStatus()]
+
+class ReaperCommandHelper(click.Command):
+    def get_help(self, ctx: click.Context):
+        IntermediateCommand.intermediate_help(super().get_help(ctx), Reaper.COMMAND, Reaper().cmd_list(), show_cluster_help=True)

adam/commands/reaper/reaper_forward.py

@@ -0,0 +1,93 @@
+from functools import partial
+import threading
+import time
+
+from adam.commands.command import Command
+from adam.commands.reaper.reaper_forward_session import ReaperForwardSession
+from adam.commands.reaper.utils_reaper import Reapers, port_forwarding
+from adam.config import Config
+from adam.repl_session import ReplSession
+from adam.repl_state import ReplState, RequiredState
+from adam.utils import lines_to_tabular, log2
+
+class ReaperForward(Command):
+    COMMAND = 'reaper forward'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(ReaperForward, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return ReaperForward.COMMAND
+
+    def required(self):
+        return RequiredState.CLUSTER
+
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        with self.validate(args, state) as (args, state):
+            if not Reapers.pod_name(state):
+                return state
+
+            spec = Reapers.reaper_spec(state)
+            if state.in_repl:
+                if ReaperForwardSession.is_forwarding:
+                    log2("Another port-forward is already running.")
+
+                    return "already-running"
+
+                # make it a daemon to exit with a Ctrl-D
+                thread = threading.Thread(target=self.loop, args=(state,), daemon=True)
+                thread.start()
+
+                while not ReaperForwardSession.is_forwarding:
+                    time.sleep(1)
+
+                d = {
+                    'reaper-ui': spec["web-uri"],
+                    'reaper-username': spec["username"],
+                    'reaper-password': spec["password"]
+                }
+                log2()
+                log2(lines_to_tabular([f'{k},{v}' for k, v in d.items()], separator=','))
+
+                for k, v in d.items():
+                    ReplSession().prompt_session.history.append_string(f'cp {k}')
+                log2()
+                log2(f'Use <Up> arrow key to copy the values to clipboard.')
+            else:
+                try:
+                    log2(f'Click: {spec["web-uri"]}')
+                    log2(f'username: {spec["username"]}')
+                    log2(f'password: {spec["password"]}')
+                    log2()
+                    log2(f"Press Ctrl+C to break.")
+
+                    time.sleep(Config().get('reaper.port-forward.timeout', 3600 * 24))
+                except KeyboardInterrupt:
+                    pass
+
+            return state
+
+    def loop(self, state: ReplState):
+        with port_forwarding(state, Reapers.local_port(), partial(Reapers.svc_or_pod, state), Reapers.target_port()):
+            ReaperForwardSession.is_forwarding = True
+            try:
+                while not ReaperForwardSession.stopping.is_set():
+                    time.sleep(1)
+            finally:
+                ReaperForwardSession.stopping.clear()
+                ReaperForwardSession.is_forwarding = False
+
+    def completion(self, state: ReplState):
+        return super().completion(state)
+
+    def help(self, _: ReplState):
+        return f'{ReaperForward.COMMAND}\t port-forward to reaper'

adam/commands/reaper/reaper_forward_session.py

@@ -0,0 +1,6 @@
+import threading
+
+class ReaperForwardSession:
+    is_forwarding = False
+    stopping = threading.Event()
+    schedules_ids_by_cluster: dict[str, list[str]] = {}