kaqing 2.0.171__py3-none-any.whl → 2.0.203__py3-none-any.whl

adam/commands/export/export_handlers.py

@@ -1,71 +0,0 @@
-from adam.commands.command import Command
-from adam.commands.export.export_databases import ExportDatabases
-from adam.commands.export.exporter import Exporter
-from adam.commands.export.utils_export import ExportSpec, state_with_pod
-from adam.repl_state import ReplState
-from adam.utils import log, log2
-
-class ExportService:
-    def __init__(self, handler: 'ExporterHandler'):
-        self.handler = handler
-
-    def export(self, args: list[str], export_only=False):
-        state = self.handler.state
-        export_session = state.export_session
-        spec: ExportSpec = None
-        try:
-            with state_with_pod(state) as state:
-                # --export-only for testing only
-                statuses, spec = Exporter.export_tables(args, state, export_only=export_only)
-                if not statuses:
-                    return state
-
-                Exporter.clear_export_session_cache()
-
-                if spec.importer == 'csv' or export_only:
-                    ExportDatabases.disply_export_session(state.sts, state.pod, state.namespace, spec.session)
-                else:
-                    log()
-                    ExportDatabases.display_export_db(state.export_session)
-        finally:
-            # if exporting to csv, do not bind the new session id to repl state
-            if spec and spec.importer == 'csv':
-                state.export_session = export_session
-
-        return state
-
-    def import_sesion(self, args: list[str]):
-        state = self.handler.state
-
-        if not args:
-            if state.in_repl:
-                log2('Specify export session name.')
-            else:
-                log2('* Export session name is missing.')
-
-                Command.display_help()
-
-            return 'command-missing'
-
-        with state_with_pod(state) as state:
-            tables, _ = Exporter.import_session(args, state)
-            if tables:
-                Exporter.clear_export_session_cache()
-
-                log()
-                ExportDatabases.display_export_db(state.export_session)
-
-        return state
-
-class ExporterHandler:
-    def __init__(self, state: ReplState):
-        self.state = state
-
-    def __enter__(self):
-        return ExportService(self)
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        return False
-
-def export(state: ReplState):
-    return ExporterHandler(state)

adam/commands/export/export_select_x.py

@@ -1,54 +0,0 @@
-from adam.commands.command import Command
-from adam.commands.export.export_databases import ExportDatabases
-from adam.repl_state import ReplState, RequiredState
-from adam.sql.sql_completer import SqlCompleter, SqlVariant
-from adam.utils_athena import Athena
-
-# No action body, only for a help entry and auto-completion
-class ExportSelectX(Command):
-    COMMAND = 'select_on_x'
-
-    # the singleton pattern
-    def __new__(cls, *args, **kwargs):
-        if not hasattr(cls, 'instance'): cls.instance = super(ExportSelectX, cls).__new__(cls)
-
-        return cls.instance
-
-    def __init__(self, successor: Command=None):
-        super().__init__(successor)
-
-    def command(self):
-        return ExportSelectX.COMMAND
-
-    def required(self):
-        return RequiredState.EXPORT_DB
-
-    def completion(self, state: ReplState):
-        completions = {}
-
-        if state.device == ReplState.X:
-            completions = {'drop': SqlCompleter(
-                lambda: ExportDatabases.table_names(state.export_session),
-                dml='drop',
-                expandables={
-                    'export-dbs': lambda: ExportDatabases.database_names(),
-                    'columns':lambda _: Athena.column_names(database=state.export_session, function='export'),
-                },
-                variant=SqlVariant.ATHENA
-            )}
-
-            if state.export_session:
-                completions |= {'select': SqlCompleter(
-                    lambda: ExportDatabases.table_names(state.export_session),
-                    dml='select',
-                    expandables={
-                        'export-dbs': lambda: ExportDatabases.database_names(),
-                        'columns':lambda _: Athena.column_names(database=state.export_session, function='export'),
-                    },
-                    variant=SqlVariant.ATHENA
-                )}
-
-        return completions
-
-    def help(self, _: ReplState):
-        return f'<sql-select-statements>\t run queries on export database'

adam/commands/logs.py

@@ -1,37 +0,0 @@
-from adam.commands.command import Command
-from adam.config import Config
-from adam.utils_k8s.cassandra_nodes import CassandraNodes
-from adam.repl_state import ReplState, RequiredState
-
-class Logs(Command):
-    COMMAND = 'logs'
-
-    # the singleton pattern
-    def __new__(cls, *args, **kwargs):
-        if not hasattr(cls, 'instance'): cls.instance = super(Logs, cls).__new__(cls)
-
-        return cls.instance
-
-    def __init__(self, successor: Command=None):
-        super().__init__(successor)
-
-    def command(self):
-        return Logs.COMMAND
-
-    def required(self):
-        return RequiredState.POD
-
-    def run(self, cmd: str, state: ReplState):
-        if not(args := self.args(cmd)):
-            return super().run(cmd, state)
-
-        with self.validate(args, state) as (args, state):
-            path = Config().get('logs.path', '/c3/cassandra/logs/system.log')
-            return CassandraNodes.exec(state.pod, state.namespace, f'cat {path}')
-
-    def completion(self, _: ReplState):
-        # available only on cli
-        return {}
-
-    def help(self, _: ReplState):
-        return f'{Logs.COMMAND}\t show cassandra system log'

adam/commands/postgres/postgres_context.py

@@ -1,274 +0,0 @@
-from datetime import datetime
-import functools
-import re
-import subprocess
-
-from adam.config import Config
-from adam.repl_session import ReplSession
-from adam.utils_k8s.kube_context import KubeContext
-from adam.utils_k8s.pods import Pods
-from adam.utils_k8s.secrets import Secrets
-from adam.utils import log2
-
-class PostgresContext:
-    def apply(namespace: str, path: str, arg: str = None) -> 'PostgresContext':
-        context = PostgresContext(namespace, path)
-
-        if arg:
-            if arg == '..':
-                if context.db:
-                    context.db = None
-                else:
-                    context.host = None
-            else:
-                tks = arg.split('@')
-                if not context.host:
-                    context.host = tks[0]
-                else:
-                    context.db = tks[0]
-
-                if not namespace and tks[1]:
-                    context.namespace = tks[1]
-
-        return context
-
-    def __init__(self, ns: str, path: str):
-        self.namespace = ns
-        self.conn_details = None
-        self.host = None
-        self.db = None
-
-        if path:
-            tks = path.split('/')
-            hn = tks[0].split('@')
-            self.host = hn[0]
-            if len(hn) > 1 and not ns:
-                self.namespace = hn[1]
-
-            if len(tks) > 1:
-                self.db = tks[1]
-
-    def path(self):
-        if not self.host:
-            return None
-
-        d = self.host
-        if not self.db:
-            return d
-
-        return f'{self.host}/{self.db}'
-
-    def hosts(ns: str):
-        return PostgresContext.hosts_for_namespace(ns)
-
-    @functools.lru_cache()
-    def hosts_for_namespace(ns: str):
-        ss = Secrets.list_secrets(ns, name_pattern=Config().get('pg.name-pattern', '^{namespace}.*k8spg.*'))
-
-        def excludes(name: str):
-            exs = Config().get('pg.excludes', '.helm., -admin-secret')
-            if exs:
-                for ex in exs.split(','):
-                    if ex.strip(' ') in name:
-                        return True
-
-            return False
-
-        return [s for s in ss if not excludes(s)]
-
-    def databases(self):
-        dbs = []
-        #  List of databases
-        #                  Name                  |  Owner   | Encoding |   Collate   |    Ctype    | ICU Locale | Locale Provider |   Access privileges
-        # ---------------------------------------+----------+----------+-------------+-------------+------------+-----------------+-----------------------
-        #  postgres                              | postgres | UTF8     | en_US.UTF-8 | en_US.UTF-8 |            | libc            |
-        #  stgawsscpsr_c3_c3                     | postgres | UTF8     | C           | C           |            | libc            |
-        #  template1                             | postgres | UTF8     | en_US.UTF-8 | en_US.UTF-8 |            | libc            | =c/postgres          +
-        #                                        |          |          |             |             |            |                 | postgres=CTc/postgres
-        # (48 rows)
-        if r := self.run_sql('\l', show_out=False):
-            s = 0
-            for line in r.stdout.split('\n'):
-                line: str = line.strip(' \r')
-                if s == 0:
-                    if 'List of databases' in line:
-                        s = 1
-                elif s == 1:
-                    if 'Name' in line and 'Owner' in line and 'Encoding' in line:
-                        s = 2
-                elif s == 2:
-                    if line.startswith('---------'):
-                        s = 3
-                elif s == 3:
-                    groups = re.match(r'^\s*(\S*)\s*\|\s*(\S*)\s*\|.*', line)
-                    if groups and groups[1] != '|':
-                        dbs.append({'name': groups[1], 'owner': groups[2]})
-
-        return dbs
-
-    def tables(self):
-        dbs = []
-        #                                            List of relations
-        #   Schema  |                            Name                            | Type  |     Owner
-        # ----------+------------------------------------------------------------+-------+---------------
-        #  postgres | c3_2_admin_aclpriv                                         | table | postgres
-        #  postgres | c3_2_admin_aclpriv_a                                       | table | postgres
-        if r := self.run_sql('\dt', show_out=False):
-            s = 0
-            for line in r.stdout.split('\n'):
-                line: str = line.strip(' \r')
-                if s == 0:
-                    if 'List of relations' in line:
-                        s = 1
-                elif s == 1:
-                    if 'Schema' in line and 'Name' in line and 'Type' in line:
-                        s = 2
-                elif s == 2:
-                    if line.startswith('---------'):
-                        s = 3
-                elif s == 3:
-                    groups = re.match(r'^\s*(\S*)\s*\|\s*(\S*)\s*\|.*', line)
-                    if groups and groups[1] != '|':
-                        dbs.append({'schema': groups[1], 'name': groups[2]})
-
-        return dbs
-
-    def run_sql(self, sql: str, show_out = True, background = False):
-        db = self.db if self.db else PostgresContext.default_db()
-
-        if KubeContext.in_cluster():
-            cmd1 = f'env PGPASSWORD={self.password()} psql -h {self.endpoint()} -p {self.port()} -U {self.username()} {db} --pset pager=off -c'
-            log2(f'{cmd1} "{sql}"')
-            # remove double quotes from the sql argument
-            cmd = cmd1.split(' ') + [sql]
-
-            r = subprocess.run(cmd, capture_output=not background, text=True)
-            if show_out:
-                log2(r.stdout)
-                log2(r.stderr)
-
-            return r
-        else:
-            pod_name, container_name = PostgresContext.pod_and_container(self.namespace)
-            if not pod_name:
-                return
-
-            # ns = self.namespace
-            # pod_name = Config().get('pg.agent.name', 'ops-pg-agent')
-
-            # if Config().get('pg.agent.just-in-time', False):
-            #     if not PostgresContext.deploy_pg_agent(pod_name, ns):
-            #         return
-
-            # real_pod_name = pod_name
-            # try:
-            #     # try with dedicated pg agent pod name configured
-            #     Pods.get(ns, pod_name)
-            # except:
-            #     try:
-            #         # try with the ops pod
-            #         pod_name = Config().get('pod.name', 'ops')
-            #         real_pod_name = Pods.get_with_selector(ns, label_selector = Config().get('pod.label-selector', 'run=ops')).metadata.name
-            #     except:
-            #         log2(f"Could not locate {pod_name} pod.")
-            #         return None
-
-            cmd = f'psql -h {self.endpoint()} -p {self.port()} -U {self.username()} {db} --pset pager=off -c "{sql}"'
-            env_prefix = f'PGPASSWORD="{self.password()}"'
-
-            r = Pods.exec(pod_name, container_name, self.namespace, cmd, show_out=show_out, background=background, env_prefix=env_prefix)
-            if r and Config().get('repl.history.push-cat-remote-log-file', True):
-                if r.log_file and ReplSession().prompt_session:
-                    ReplSession().prompt_session.history.append_string(f'@{r.pod} cat {r.log_file}')
-
-            return r
-
-    def pod_and_container(ns: str):
-        container_name = Config().get('pg.agent.name', 'ops-pg-agent')
-
-        if Config().get('pg.agent.just-in-time', False):
-            if not PostgresContext.deploy_pg_agent(container_name, ns):
-                return None
-
-        pod_name = container_name
-        try:
-            # try with dedicated pg agent pod name configured
-            Pods.get(ns, container_name)
-        except:
-            try:
-                # try with the ops pod
-                container_name = Config().get('pod.name', 'ops')
-                pod_name = Pods.get_with_selector(ns, label_selector = Config().get('pod.label-selector', 'run=ops')).metadata.name
-            except:
-                log2(f"Could not locate {container_name} pod.")
-                return None
-
-        return pod_name, container_name
-
-    def deploy_pg_agent(pod_name: str, ns: str) -> str:
-        image = Config().get('pg.agent.image', 'seanahnsf/kaqing')
-        timeout = Config().get('pg.agent.timeout', 3600)
-        try:
-            Pods.create(ns, pod_name, image, ['sleep', f'{timeout}'], env={'NAMESPACE': ns}, sa_name='c3')
-        except Exception as e:
-            if e.status == 409:
-                if Pods.completed(ns, pod_name):
-                    try:
-                        Pods.delete(pod_name, ns)
-                        Pods.create(ns, pod_name, image, ['sleep', f'{timeout}'], env={'NAMESPACE': ns}, sa_name='c3')
-                    except Exception as e2:
-                        log2("Exception when calling BatchV1Api->create_pod: %s\n" % e2)
-
-                        return
-            else:
-                log2("Exception when calling BatchV1Api->create_pod: %s\n" % e)
-
-                return
-
-        Pods.wait_for_running(ns, pod_name)
-
-        return pod_name
-
-    def undeploy_pg_agent(pod_name: str, ns: str):
-        Pods.delete(pod_name, ns, grace_period_seconds=0)
-
-    def endpoint(self):
-        if not self.conn_details:
-            self.conn_details = Secrets.get_data(self.namespace, self.host)
-
-        endpoint_key = Config().get('pg.secret.endpoint-key', 'postgres-db-endpoint')
-
-        return self.conn_details[endpoint_key] if endpoint_key in self.conn_details else ''
-
-    def port(self):
-        if not self.conn_details:
-            self.conn_details = Secrets.get_data(self.namespace, self.host)
-
-        port_key = Config().get('pg.secret.port-key', 'postgres-db-port')
-
-        return  self.conn_details[port_key] if port_key in self.conn_details else ''
-
-    def username(self):
-        if not self.conn_details:
-            self.conn_details = Secrets.get_data(self.namespace, self.host)
-
-        username_key = Config().get('pg.secret.username-key', 'postgres-admin-username')
-
-        return  self.conn_details[username_key] if username_key in self.conn_details else ''
-
-    def password(self):
-        if not self.conn_details:
-            self.conn_details = Secrets.get_data(self.namespace, self.host)
-
-        password_key = Config().get('pg.secret.password-key', 'postgres-admin-password')
-
-        return  self.conn_details[password_key] if password_key in self.conn_details else ''
-
-    def default_db():
-        return Config().get('pg.default-db', 'postgres')
-
-    def default_owner():
-        return Config().get('pg.default-owner', 'postgres')
-
-    def default_schema():
-        return Config().get('pg.default-schema', 'postgres')

adam/commands/postgres/psql_completions.py

@@ -1,10 +0,0 @@
-from adam.commands.postgres.utils_postgres import pg_table_names
-from adam.sql.sql_completer import SqlCompleter
-
-def psql_completions(ns: str, pg_path: str):
-    return {
-        '\h': None,
-        '\d': None,
-        '\dt': None,
-        '\du': None
-    } | SqlCompleter(lambda: pg_table_names(ns, pg_path)).completions_for_nesting()

adam/commands/report.py

@@ -1,61 +0,0 @@
-import click
-import json
-
-from adam.checks.check_result import CheckResult
-from adam.checks.check_utils import run_checks
-from adam.commands.command import Command
-from adam.repl_state import ReplState
-from adam.utils import log2
-
-class Report(Command):
-    COMMAND = 'report'
-
-    # the singleton pattern
-    def __new__(cls, *args, **kwargs):
-        if not hasattr(cls, 'instance'): cls.instance = super(Report, cls).__new__(cls)
-
-        return cls.instance
-
-    def __init__(self, successor: Command=None):
-        super().__init__(successor)
-
-    def command(self):
-        return Report.COMMAND
-
-    def required(self):
-        return ReplState.NON_L
-
-    def run(self, cmd: str, state: ReplState):
-        if not(args := self.args(cmd)):
-            return super().run(cmd, state)
-
-        with self.validate(args, state) as (args, state):
-            output: dict[str, any] = {}
-
-            if state.in_repl:
-                args, show = Command.extract_options(args, ['-s', '--show'])
-
-                args, redirect = Command.extract_options(args, ['>'])
-                if not redirect or not args:
-                    log2('Please specify file name: e.g. report > /tmp/report.log')
-                    return 'no-report-destination'
-
-                results = run_checks(state.sts, state.namespace, state.pod, show_out=show)
-                output = CheckResult.report(results)
-                with open(args[0], "w") as json_file:
-                    json.dump(output, json_file, indent=2)
-                    log2(f'Report stored in {args[0]}.')
-            else:
-                args, show = Command.extract_options(args, ['-s', '--show'])
-
-                results = run_checks(state.sts, state.namespace, state.pod, show_out=show)
-                output = CheckResult.report(results)
-                click.echo(json.dumps(output, indent=2))
-
-            return output
-
-    def completion(self, state: ReplState):
-        return super().completion(state, {">": None})
-
-    def help(self, _: ReplState):
-        return f"{Report.COMMAND} > <file-name>\t generate report"

adam/commands/restart.py

@@ -1,60 +0,0 @@
-from adam.commands import extract_options
-from adam.commands.command import Command
-from adam.utils_k8s.pods import Pods
-from adam.utils_k8s.statefulsets import StatefulSets
-from adam.repl_state import ReplState, RequiredState
-from adam.utils import log2
-
-class Restart(Command):
-    COMMAND = 'restart'
-
-    # the singleton pattern
-    def __new__(cls, *args, **kwargs):
-        if not hasattr(cls, 'instance'): cls.instance = super(Restart, cls).__new__(cls)
-
-        return cls.instance
-
-    def __init__(self, successor: Command=None):
-        super().__init__(successor)
-
-    def command(self):
-        return Restart.COMMAND
-
-    def required(self):
-        return RequiredState.CLUSTER_OR_POD
-
-    def run(self, cmd: str, state: ReplState):
-        if not(args := self.args(cmd)):
-            return super().run(cmd, state)
-
-        with self.validate(args, state) as (args, state):
-            with extract_options(args, '--force') as (args, forced):
-                if not args:
-                    if state.pod:
-                        log2(f'Restarting {state.pod}...')
-                        Pods.delete(state.pod, state.namespace)
-                    else:
-                        if not forced:
-                            log2('Please add --force for restarting all nodes in a cluster.')
-                            return 'force-needed'
-
-                        log2(f'Restarting all pods from {state.sts}...')
-                        for pod_name in StatefulSets.pod_names(state.sts, state.namespace):
-                            Pods.delete(pod_name, state.namespace)
-                else:
-                    for arg in args:
-                        Pods.delete(arg, state.namespace)
-
-                return state
-
-    def completion(self, state: ReplState):
-        if super().completion(state):
-            if state.pod:
-                return {Restart.COMMAND: None}
-            elif state.sts:
-                return {Restart.COMMAND: {p: None for p in StatefulSets.pod_names(state.sts, state.namespace)}}
-
-        return {}
-
-    def help(self, _: ReplState):
-        return f"{Restart.COMMAND} [<host-id>] [--force]\t restart the node you're in or all the nodes in the cluster"