kaqing 2.0.52__py3-none-any.whl → 2.0.184__py3-none-any.whl

adam/commands/postgres/postgres_session.py

@@ -1,240 +0,0 @@
-import functools
-import re
-import subprocess
-
-from adam.config import Config
-from adam.k8s_utils.kube_context import KubeContext
-from adam.k8s_utils.pods import Pods
-from adam.k8s_utils.secrets import Secrets
-from adam.utils import log2
-
-class PostgresSession:
-    def __init__(self, ns: str, path: str):
-        self.namespace = ns
-        self.conn_details = None
-        self.host = None
-        self.db = None
-
-        if path:
-            tks = path.split('/')
-            hn = tks[0].split('@')
-            self.host = hn[0]
-            if len(hn) > 1 and not ns:
-                self.namespace = hn[1]
-
-            if len(tks) > 1:
-                self.db = tks[1]
-
-    def find_namespace(self, arg: str):
-        if arg:
-            tks = arg.split('@')
-            if len(tks) > 1:
-                return tks[1]
-
-        return None
-
-    def directory(self, arg: str = None):
-        if arg:
-            if arg == '..':
-                if self.db:
-                    self.db = None
-                else:
-                    self.host = None
-            else:
-                tks = arg.split('@')
-                arg = tks[0]
-                if not self.host:
-                    self.host = arg
-                else:
-                    self.db = arg
-
-        if not self.host:
-            return None
-
-        d = self.host
-        if not self.db:
-            return d
-
-        return f'{self.host}/{self.db}'
-
-    def hosts(ns: str):
-        return PostgresSession.hosts_for_namespace(ns)
-
-    @functools.lru_cache()
-    def hosts_for_namespace(ns: str):
-        ss = Secrets.list_secrets(ns, name_pattern=Config().get('pg.name-pattern', '^{namespace}.*k8spg.*'))
-
-        def excludes(name: str):
-            exs = Config().get('pg.excludes', '.helm., -admin-secret')
-            if exs:
-                for ex in exs.split(','):
-                    if ex.strip(' ') in name:
-                        return True
-
-            return False
-
-        return [s for s in ss if not excludes(s)]
-
-    def databases(self):
-        dbs = []
-        #  List of databases
-        #                  Name                  |  Owner   | Encoding |   Collate   |    Ctype    | ICU Locale | Locale Provider |   Access privileges
-        # ---------------------------------------+----------+----------+-------------+-------------+------------+-----------------+-----------------------
-        #  postgres                              | postgres | UTF8     | en_US.UTF-8 | en_US.UTF-8 |            | libc            |
-        #  stgawsscpsr_c3_c3                     | postgres | UTF8     | C           | C           |            | libc            |
-        #  template1                             | postgres | UTF8     | en_US.UTF-8 | en_US.UTF-8 |            | libc            | =c/postgres          +
-        #                                        |          |          |             |             |            |                 | postgres=CTc/postgres
-        # (48 rows)
-        if r := self.run_sql('\l', show_out=False):
-            s = 0
-            for line in r.stdout.split('\n'):
-                line: str = line.strip(' \r')
-                if s == 0:
-                    if 'List of databases' in line:
-                        s = 1
-                elif s == 1:
-                    if 'Name' in line and 'Owner' in line and 'Encoding' in line:
-                        s = 2
-                elif s == 2:
-                    if line.startswith('---------'):
-                        s = 3
-                elif s == 3:
-                    groups = re.match(r'^\s*(\S*)\s*\|\s*(\S*)\s*\|.*', line)
-                    if groups and groups[1] != '|':
-                        dbs.append({'name': groups[1], 'owner': groups[2]})
-
-        return dbs
-
-    def tables(self):
-        dbs = []
-        #                                            List of relations
-        #   Schema  |                            Name                            | Type  |     Owner
-        # ----------+------------------------------------------------------------+-------+---------------
-        #  postgres | c3_2_admin_aclpriv                                         | table | postgres
-        #  postgres | c3_2_admin_aclpriv_a                                       | table | postgres
-        if r := self.run_sql('\dt', show_out=False):
-            s = 0
-            for line in r.stdout.split('\n'):
-                line: str = line.strip(' \r')
-                if s == 0:
-                    if 'List of relations' in line:
-                        s = 1
-                elif s == 1:
-                    if 'Schema' in line and 'Name' in line and 'Type' in line:
-                        s = 2
-                elif s == 2:
-                    if line.startswith('---------'):
-                        s = 3
-                elif s == 3:
-                    groups = re.match(r'^\s*(\S*)\s*\|\s*(\S*)\s*\|.*', line)
-                    if groups and groups[1] != '|':
-                        dbs.append({'schema': groups[1], 'name': groups[2]})
-
-        return dbs
-
-    def run_sql(self, sql: str, show_out = True):
-        db = self.db if self.db else PostgresSession.default_db()
-
-        if KubeContext.in_cluster():
-            cmd1 = f'env PGPASSWORD={self.password()} psql -h {self.endpoint()} -p {self.port()} -U {self.username()} {db} --pset pager=off -c'
-            log2(f'{cmd1} "{sql}"')
-            # remove double quotes from the sql argument
-            cmd = cmd1.split(' ') + [sql]
-            r = subprocess.run(cmd, capture_output=True, text=True)
-            if show_out:
-                log2(r.stdout)
-                log2(r.stderr)
-
-            return r
-        else:
-            ns = self.namespace
-            pod_name = Config().get('pg.agent.name', 'ops-pg-agent')
-
-            if Config().get('pg.agent.just-in-time', False):
-                if not PostgresSession.deploy_pg_agent(pod_name, ns):
-                    return
-
-            real_pod_name = pod_name
-            try:
-                # try with dedicated pg agent pod name configured
-                Pods.get(ns, pod_name)
-            except:
-                try:
-                    # try with the ops pod
-                    pod_name = Config().get('pod.name', 'ops')
-                    real_pod_name = Pods.get_with_selector(ns, label_selector = Config().get('pod.label-selector', 'run=ops')).metadata.name
-                except:
-                    log2(f"Could not locate {pod_name} pod.")
-                    return None
-
-            cmd = f'PGPASSWORD="{self.password()}" psql -h {self.endpoint()} -p {self.port()} -U {self.username()} {db} --pset pager=off -c "{sql}"'
-
-            return Pods.exec(real_pod_name, pod_name, ns, cmd, show_out=show_out)
-
-    def deploy_pg_agent(pod_name: str, ns: str) -> str:
-        image = Config().get('pg.agent.image', 'seanahnsf/kaqing')
-        timeout = Config().get('pg.agent.timeout', 3600)
-        try:
-            Pods.create(ns, pod_name, image, ['sleep', f'{timeout}'], env={'NAMESPACE': ns}, sa_name='c3')
-        except Exception as e:
-            if e.status == 409:
-                if Pods.completed(ns, pod_name):
-                    try:
-                        Pods.delete(pod_name, ns)
-                        Pods.create(ns, pod_name, image, ['sleep', f'{timeout}'], env={'NAMESPACE': ns}, sa_name='c3')
-                    except Exception as e2:
-                        log2("Exception when calling BatchV1Api->create_pod: %s\n" % e2)
-
-                        return
-            else:
-                log2("Exception when calling BatchV1Api->create_pod: %s\n" % e)
-
-                return
-
-        Pods.wait_for_running(ns, pod_name)
-
-        return pod_name
-
-    def undeploy_pg_agent(pod_name: str, ns: str):
-        Pods.delete(pod_name, ns, grace_period_seconds=0)
-
-    def endpoint(self):
-        if not self.conn_details:
-            self.conn_details = Secrets.get_data(self.namespace, self.host)
-
-        endpoint_key = Config().get('pg.secret.endpoint-key', 'postgres-db-endpoint')
-
-        return self.conn_details[endpoint_key] if endpoint_key in self.conn_details else ''
-
-    def port(self):
-        if not self.conn_details:
-            self.conn_details = Secrets.get_data(self.namespace, self.host)
-
-        port_key = Config().get('pg.secret.port-key', 'postgres-db-port')
-
-        return  self.conn_details[port_key] if port_key in self.conn_details else ''
-
-    def username(self):
-        if not self.conn_details:
-            self.conn_details = Secrets.get_data(self.namespace, self.host)
-
-        username_key = Config().get('pg.secret.username-key', 'postgres-admin-username')
-
-        return  self.conn_details[username_key] if username_key in self.conn_details else ''
-
-    def password(self):
-        if not self.conn_details:
-            self.conn_details = Secrets.get_data(self.namespace, self.host)
-
-        password_key = Config().get('pg.secret.password-key', 'postgres-admin-password')
-
-        return  self.conn_details[password_key] if password_key in self.conn_details else ''
-
-    def default_db():
-        return Config().get('pg.default-db', 'postgres')
-
-    def default_owner():
-        return Config().get('pg.default-owner', 'postgres')
-
-    def default_schema():
-        return Config().get('pg.default-schema', 'postgres')

adam/commands/postgres/postgres_utils.py

@@ -1,31 +0,0 @@
-import functools
-
-from adam.commands.postgres.postgres_session import PostgresSession
-from adam.config import Config
-
-TestPG = [False]
-
-@functools.lru_cache()
-def pg_database_names(ns: str, pg_path: str):
-    if TestPG[0]:
-        return ['azops88_c3ai_c3']
-
-    Config().wait_log('Inspecting Postgres Databases...')
-
-    pg = PostgresSession(ns, pg_path)
-    return [db['name'] for db in pg.databases() if db['owner'] == PostgresSession.default_owner()]
-
-@functools.lru_cache()
-def pg_table_names(ns: str, pg_path: str):
-    if TestPG[0]:
-        return ['C3_2_XYZ1']
-
-    Config().wait_log('Inspecting Postgres Database...')
-    return [table['name'] for table in pg_tables(ns, pg_path) if table['schema'] == PostgresSession.default_schema()]
-
-def pg_tables(ns: str, pg_path: str):
-    pg = PostgresSession(ns, pg_path)
-    if pg.db:
-        return pg.tables()
-
-    return []

adam/commands/postgres/psql_table_completer.py

@@ -1,11 +0,0 @@
-from adam.commands.postgres.postgres_utils import pg_table_names
-from adam.sql.term_completer import TermCompleter
-
-class PsqlTableNameCompleter(TermCompleter):
-    def __init__(self, namespace: str, pg_path: str, ignore_case: bool = True):
-        super().__init__(pg_table_names(namespace, pg_path), ignore_case=ignore_case)
-        self.namespace = namespace
-        self.pg_path = pg_path
-
-    def __repr__(self) -> str:
-        return "PsqlTableCompleter(%r, pg_path=%r)" % (self.namespace, self.pg_path)

adam/commands/reaper/reaper_session.py

@@ -1,159 +0,0 @@
-from collections.abc import Callable
-import threading
-from kubernetes import client
-import portforward
-import re
-import requests
-from typing import List, cast
-
-from adam.config import Config
-from adam.k8s_utils.kube_context import KubeContext
-from adam.repl_state import ReplState
-from adam.utils import lines_to_tabular, log2
-
-class ReaperSession:
-    is_forwarding = False
-    stopping = threading.Event()
-    schedules_ids_by_cluster: dict[str, list[str]] = {}
-
-    def __init__(self, pod: str, headers: dict[str, str] = None):
-        self.pod = pod
-        self.headers = headers
-
-    def login(self, state: ReplState, local_addr: str, remote_addr: str, show_output = True) -> str :
-        user, pw = state.user_pass(secret_path='reaper.secret')
-
-        response = requests.post(f'http://{local_addr}/login', headers={
-            'Accept': '*'
-        },data={
-            'username':user,
-            'password':pw})
-        if show_output:
-            log2(f'POST {remote_addr}/login')
-            log2(f'     username={user}&password={pw}')
-
-        if int(response.status_code / 100) != 2:
-            if show_output:
-                log2("login failed")
-            return None
-
-        return response.headers['Set-Cookie']
-
-    def port_forwarded(self, state: ReplState, path: str, body: Callable[[str, dict[str, str]], requests.Response], method: str = None, show_output = True):
-        local_port = Config().get('reaper.port-forward.local-port', 9001)
-        target_port = 8080
-
-        def f(local_addr: str, remote_addr: str):
-            if not self.headers:
-                self.headers = self.cookie_header(state, local_addr, remote_addr, show_output=show_output)
-
-            if show_output and method:
-                log2(f'{method} {remote_addr}/{path}')
-            response = body(f'http://{local_addr}/{path}', self.headers)
-
-            if response:
-                if int(response.status_code / 100) != 2:
-                    if show_output:
-                        log2(response.status_code)
-                    return response
-
-            if show_output:
-                log2()
-
-            return response if response else 'no-response'
-
-        if KubeContext.in_cluster():
-            # cs-a526330d23-cs-a526330d23-default-sts-0 ->
-            # curl http://cs-a526330d23-cs-a526330d23-reaper-service.stgawsscpsr.svc.cluster.local:8080
-            groups = re.match(r'^(.*?-.*?-.*?-.*?-).*', state.sts)
-            if groups:
-                svc_name = Config().get('reaper.service-name', 'reaper-service')
-                svc = f'{groups[1]}{svc_name}.{state.namespace}.svc.cluster.local:{target_port}'
-                return f(local_addr=svc, remote_addr=svc)
-            else:
-                return None
-        else:
-            with portforward.forward(state.namespace, self.pod, local_port, target_port):
-                return f(local_addr=f'localhost:{local_port}', remote_addr=f'{self.pod}:{target_port}')
-
-    def cookie_header(self, state: ReplState, local_addr, remote_addr, show_output = True):
-        return {'Cookie': self.login(state, local_addr, remote_addr, show_output=show_output)}
-
-    def create(state: ReplState) -> 'ReaperSession':
-        pods = ReaperSession.list_reaper_pods(state.sts if state.sts else state.pod, state.namespace)
-        if pods:
-            return ReaperSession(pods[0].metadata.name)
-        else:
-            log2('No reaper found.')
-
-            return None
-
-    def list_reaper_pods(sts_name: str, namespace: str) -> List[client.V1Pod]:
-        v1 = client.CoreV1Api()
-
-        # k8ssandra.io/reaper: cs-d0767a536f-cs-d0767a536f-reaper
-        groups = re.match(Config().get('reaper.pod.cluster-regex', r'(.*?-.*?-.*?-.*?)-.*'), sts_name)
-        label_selector = Config().get('reaper.pod.label-selector', 'k8ssandra.io/reaper={cluster}-reaper').replace('{cluster}', groups[1])
-
-        return cast(List[client.V1Pod], v1.list_namespaced_pod(namespace, label_selector=label_selector).items)
-
-    def show_schedules(self, state: ReplState, filter: Callable[[list[dict]], dict] = None):
-        schedules = self.list_schedules(state, filter=filter)
-        # forced refresh of schedule list
-        if not filter:
-            self.schedules_ids_by_cluster[state.sts] = [schedule['id'] for schedule in schedules]
-        self.show_schedules_tabular(schedules)
-
-    def schedule_ids(self, state: ReplState, show_output = True, filter: Callable[[list[dict]], dict] = None):
-        schedules = self.list_schedules(state, show_output=show_output, filter=filter)
-        return [schedule['id'] for schedule in schedules]
-
-    def list_schedules(self, state: ReplState, show_output = True, filter: Callable[[list[dict]], dict] = None) -> list[dict]:
-        def body(uri: str, headers: dict[str, str]):
-            return requests.get(uri, headers=headers)
-
-        response = self.port_forwarded(state, 'repair_schedule', body, method='GET', show_output=show_output)
-        if not response:
-            return
-
-        res = response.json()
-        if filter:
-            res = filter(res)
-
-        return res
-
-    def show_schedules_tabular(self, schedules: list[dict]):
-        log2(lines_to_tabular([f"{schedule['id']} {schedule['state']} {schedule['cluster_name']} {schedule['keyspace_name']}" for schedule in schedules], 'ID STATE CLUSTER KEYSPACE'))
-
-    def show_schedule(self, state: ReplState, schedule_id: str):
-        def filter(schedules: list[dict]):
-            return [schedule for schedule in schedules if schedule['id'] == schedule_id]
-
-        self.show_schedules(state, filter)
-
-    def reaper_spec(self, state: ReplState) -> dict[str, any]:
-        user, pw = state.user_pass(secret_path='reaper.secret')
-        local_port = Config().get('reaper.port-forward.local-port', 9001)
-
-        return {
-            'pod': self.pod,
-            'exec': f'kubectl exec -it {self.pod} -n {state.namespace} -- bash',
-            'forward': f'kubectl port-forward pods/{self.pod} -n {state.namespace} {local_port}:8080',
-            'web-uri': f'http://localhost:{local_port}/webui',
-            'username': user,
-            'password': pw
-        }
-
-    def cached_schedule_ids(state: ReplState) -> list[str]:
-        if state.sts in ReaperSession.schedules_ids_by_cluster:
-            return ReaperSession.schedules_ids_by_cluster[state.sts]
-
-        if reaper := ReaperSession.create(state):
-            Config().wait_log('Inspecting Cassandra Reaper...')
-
-            schedules = reaper.schedule_ids(state, show_output = False)
-            ReaperSession.schedules_ids_by_cluster[state.sts] = schedules
-
-            return schedules
-
-        return []

adam/commands/show/show_app_actions.py

@@ -1,53 +0,0 @@
-from adam.app_session import AppSession
-from adam.apps import AppAction, Apps
-from adam.commands.command import Command
-from adam.config import Config
-from adam.repl_state import ReplState
-from adam.utils import lines_to_tabular, log
-
-class ShowAppActions(Command):
-    COMMAND = 'show app actions'
-
-    # the singleton pattern
-    def __new__(cls, *args, **kwargs):
-        if not hasattr(cls, 'instance'): cls.instance = super(ShowAppActions, cls).__new__(cls)
-
-        return cls.instance
-
-    def __init__(self, successor: Command=None):
-        super().__init__(successor)
-
-    def command(self):
-        return ShowAppActions.COMMAND
-
-    def run(self, cmd: str, state: ReplState):
-        if not self.args(cmd):
-            return super().run(cmd, state)
-
-        lines = []
-        for typ in Apps().app_types():
-            for action in typ.actions:
-                a: AppAction = action
-                args = ','.join(a.arguments())
-                if args:
-                    line = f'{typ.name}.{a.name},{args}'
-                else:
-                    line = f'{typ.name}.{a.name},'
-                if a.help:
-                    line = f'{line},{a.help}'
-                lines.append(line)
-        log(lines_to_tabular(lines, 'ACTION,ARGS,DESCRIPTION', separator=','))
-        log()
-
-        app_session: AppSession = AppSession.create(state.app_env or 'c3', state.app_app or 'c3')
-        endpoint = Config().get('app.console-endpoint', 'https://{host}/{env}/{app}/static/console/index.html')
-        endpoint = endpoint.replace('{host}', app_session.host).replace('{env}', app_session.env).replace('{app}', state.app_app or 'c3')
-        log(lines_to_tabular([f'CONSOLE:,{endpoint}'], separator=','))
-
-        return lines
-
-    def completion(self, state: ReplState):
-        return super().completion(state)
-
-    def help(self, _: ReplState):
-        return f"{ShowAppActions.COMMAND}\t show app actions"

adam/commands/show/show_repairs.py

@@ -1,47 +0,0 @@
-from adam.commands.command import Command
-from adam.k8s_utils.cassandra_clusters import CassandraClusters
-from adam.k8s_utils.cassandra_nodes import CassandraNodes
-from adam.repl_state import ReplState, RequiredState
-
-class ShowRepairs(Command):
-    COMMAND = 'show cassandra repairs'
-
-    # the singleton pattern
-    def __new__(cls, *args, **kwargs):
-        if not hasattr(cls, 'instance'): cls.instance = super(ShowRepairs, cls).__new__(cls)
-
-        return cls.instance
-
-    def __init__(self, successor: Command=None):
-        super().__init__(successor)
-
-    def command(self):
-        return ShowRepairs.COMMAND
-
-    def required(self):
-        return RequiredState.CLUSTER_OR_POD
-
-    def run(self, cmd: str, state: ReplState):
-        if not(args := self.args(cmd)):
-            return super().run(cmd, state)
-
-        state, _ = state.apply_args(args)
-        if not self.validate_state(state):
-            return state
-
-        user, pw = state.user_pass()
-        command = f"nodetool -u {user} -pw {pw} repair_admin list"
-
-        if state.pod:
-            return CassandraNodes.exec(state.pod, state.namespace, command)
-        else:
-            return CassandraClusters.exec(state.sts, state.namespace, command, action='nodetool')
-
-    def completion(self, state: ReplState):
-        if state.sts:
-            return super().completion(state)
-
-        return {}
-
-    def help(self, _: ReplState):
-        return f'{ShowRepairs.COMMAND}\t show Cassandra repairs'

adam/k8s_utils/cassandra_clusters.py

@@ -1,35 +0,0 @@
-from collections.abc import Callable
-from concurrent.futures import ThreadPoolExecutor
-import sys
-from typing import TypeVar
-
-from adam.config import Config
-from adam.k8s_utils.cassandra_nodes import CassandraNodes
-from adam.pod_exec_result import PodExecResult
-from adam.utils import log2
-from .statefulsets import StatefulSets
-from .pods import Pods
-from .kube_context import KubeContext
-
-T = TypeVar('T')
-
-# utility collection on cassandra clusters; methods are all static
-class CassandraClusters:
-    def exec(statefulset: str, namespace: str, command: str, action: str = 'action', max_workers=0, show_out=True, on_any = False) -> list[PodExecResult]:
-        def body(executor: ThreadPoolExecutor, pod: str, namespace: str, show_out: bool):
-            if executor:
-                return executor.submit(CassandraNodes.exec, pod, namespace, command, False, False,)
-
-            return CassandraNodes.exec(pod, namespace, command, show_out=show_out)
-
-        def post(result, show_out: bool):
-            if KubeContext.show_out(show_out):
-                print(result.command)
-                if result.stdout:
-                    print(result.stdout)
-                if result.stderr:
-                    log2(result.stderr, file=sys.stderr)
-
-            return result
-
-        return StatefulSets.on_cluster(statefulset, namespace, body, post=post, action=action, max_workers=max_workers, show_out=show_out, on_any=on_any)

adam/sql/sql_utils.py

@@ -1,5 +0,0 @@
-import re
-
-def safe_terms(text: str):
-    tokens = re.findall(r'"[^"]+"|\b\w+\b|\S', text)
-    return tokens, len(tokens) > 1 or text.startswith(' ') or text.endswith(' ')