kaqing 2.0.115__py3-none-any.whl → 2.0.172__py3-none-any.whl

adam/commands/cql/cql_utils.py

@@ -1,204 +0,0 @@
-import functools
-import re
-
-from adam.config import Config
-from adam.utils_k8s.cassandra_clusters import CassandraClusters
-from adam.utils_k8s.cassandra_nodes import CassandraNodes
-from adam.utils_k8s.secrets import Secrets
-from adam.pod_exec_result import PodExecResult
-from adam.repl_state import ReplState
-from adam.utils import log2
-
-@functools.lru_cache()
-def keyspaces(state: ReplState, on_any=False):
-    if state.pod:
-        Config().wait_log(f'Inspecting Cassandra Keyspaces on {state.pod}...')
-    else:
-        Config().wait_log(f'Inspecting Cassandra Keyspaces...')
-
-    r: list[PodExecResult] = run_cql(state, 'describe keyspaces', show_out=False, on_any=on_any)
-    if not r:
-        log2('No pod is available')
-        return []
-
-    return parse_cql_desc_keyspaces(r.stdout if state.pod else r[0].stdout)
-
-def cassandra_table_names(state: ReplState, keyspace = None):
-    return [f'{k}.{t}' for k, ts in tables(state, on_any=True).items() for t in ts if not keyspace or k == keyspace]
-
-@functools.lru_cache()
-def tables(state: ReplState, on_any=False) -> dict[str, list[str]]:
-    r: list[PodExecResult] = run_cql(state, 'describe tables', show_out=False, on_any=on_any)
-    if not r:
-        log2('No pod is available')
-        return {}
-
-    return parse_cql_desc_tables(r.stdout if state.pod else r[0].stdout)
-
-@functools.lru_cache()
-def table_spec(state: ReplState, table: str, on_any=False) -> 'TableSpec':
-    r: list[PodExecResult] = run_cql(state, f'describe table {table}', show_out=False, on_any=on_any)
-    if not r:
-        log2('No pod is available')
-        return None
-
-    return parse_cql_desc_table(r.stdout if state.pod else r[0].stdout)
-
-def run_cql(state: ReplState, cql: str, opts: list = [], show_out = False, use_single_quotes = False, on_any = False, background=False) -> list[PodExecResult]:
-    user, pw = Secrets.get_user_pass(state.sts if state.sts else state.pod, state.namespace, secret_path='cql.secret')
-    if use_single_quotes:
-        command = f"cqlsh -u {user} -p {pw} {' '.join(opts)} -e '{cql}'"
-    else:
-        command = f'cqlsh -u {user} -p {pw} {" ".join(opts)} -e "{cql}"'
-
-    if state.pod:
-        return CassandraNodes.exec(state.pod, state.namespace, command, show_out=show_out, background=background)
-    else:
-        return CassandraClusters.exec(state.sts, state.namespace, command, show_out=show_out, action='cql', on_any=on_any, background=background)
-
-def parse_cql_desc_tables(out: str):
-    # Keyspace data_endpoint_auth
-    # ---------------------------
-    # "token"
-
-    # Keyspace reaper_db
-    # ------------------
-    # repair_run                     schema_migration
-    # repair_run_by_cluster          schema_migration_leader
-
-    # Keyspace system
-    tables_by_keyspace: dict[str, list[str]] = {}
-    keyspace = None
-    state = 's0'
-    for line in out.split('\n'):
-        if state == 's0':
-            groups = re.match(r'^Keyspace (.*)$', line)
-            if groups:
-                keyspace = groups[1].strip(' \r')
-                state = 's1'
-        elif state == 's1':
-            if line.startswith('---'):
-                state = 's2'
-        elif state == 's2':
-            if not line.strip(' \r'):
-                state = 's0'
-            else:
-                for table in line.split(' '):
-                    if t := table.strip(' \r'):
-                        if not keyspace in tables_by_keyspace:
-                            tables_by_keyspace[keyspace] = []
-                        tables_by_keyspace[keyspace].append(t)
-
-    return tables_by_keyspace
-
-def parse_cql_desc_keyspaces(out: str) -> list[str]:
-    #
-    # Warning: Cannot create directory at `/home/cassandra/.cassandra`. Command history will not be saved. Please check what was the environment property CQL_HISTORY set to.
-    #
-    #
-    # Warning: Using a password on the command line interface can be insecure.
-    # Recommendation: use the credentials file to securely provide the password.
-    #
-    #
-    # azops88_db  system_auth         system_traces
-    # reaper_db   system_distributed  system_views
-    # system      system_schema       system_virtual_schema
-    #
-    kses = []
-    for line in out.split('\n'):
-        line = line.strip(' \r')
-        if not line:
-            continue
-        if line.startswith('Warning:'):
-            continue
-        if line.startswith('Recommendation:'):
-            continue
-
-        for ks in line.split(' '):
-            if s := ks.strip(' \r\t'):
-                kses.append(s)
-
-    return kses
-
-class ColumnSpec:
-    def __init__(self, name: str, type: str, key_index = -1):
-        self.name = name
-        self.type = type
-        self.key_index = key_index
-
-    def __eq__(self, other):
-        if not isinstance(other, ColumnSpec):
-            return NotImplemented  # Or raise TypeError, depending on desired strictness
-
-        # Then, compare the relevant attributes for equality
-        return self.name == other.name and self.type == other.type and self.key_index == other.key_index
-
-class TableSpec:
-    def __init__(self, columns: list[ColumnSpec]):
-        self.columns = columns
-
-    def row_key(self):
-        for c in self.columns:
-            if c.key_index == 0:
-                return c.name
-
-    def keys(self):
-        return [c.name for c in self.columns if c.key_index > -1]
-
-def parse_cql_desc_table(out: str) -> TableSpec:
-    # CREATE TABLE azops88_db.analyticscontainer_dfeevalhistory (
-    #     id text,
-    #     columnname text,
-    #     version bigint static,
-    #     contentb blob,
-    #     contentbool boolean,
-    #     contentn double,
-    #     contents text,
-    #     PRIMARY KEY (id, columnname)
-    # ) WITH CLUSTERING ORDER BY (columnname ASC)
-    #     AND additional_write_policy = '99p'
-    #     AND bloom_filter_fp_chance = 0.1
-    #     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
-    #     AND cdc = false
-    #     AND comment = ''
-    #     AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
-    #     AND compression = {'chunk_length_in_kb': '16', 'class': 'org.apache.cassandra.io.compress.SnappyCompressor'}
-    #     AND memtable = 'default'
-    #     AND crc_check_chance = 1.0
-    #     AND default_time_to_live = 0
-    #     AND extensions = {}
-    #     AND gc_grace_seconds = 3600
-    #     AND max_index_interval = 2048
-    #     AND memtable_flush_period_in_ms = 0
-    #     AND min_index_interval = 128
-    #     AND read_repair = 'BLOCKING'
-    #     AND speculative_retry = '99p';
-    pkeys = {}
-    columns: list[ColumnSpec] = []
-
-    state = 's0'
-    for line in out.split('\n'):
-        if state == 's0':
-            if line.startswith('CREATE TABLE'):
-                state = 's1'
-        elif state == 's1':
-            if line.startswith(')'):
-                state = 's2'
-                continue
-
-            groups = re.match(r'^\s*PRIMARY KEY\s*\((.*)\).*$', line)
-            if groups:
-                pkeys = {n.strip(' '): i for i, n in enumerate(groups[1].strip(' \r').split(','))}
-                continue
-
-            groups = re.match(r'^\s*(\S*?)\s*(\S*?),.*$', line)
-            if groups:
-                columns.append(ColumnSpec(groups[1], groups[2]))
-        elif state == 's2':
-            pass
-
-    for column in columns:
-        if column.name in pkeys.keys():
-            column.key_index = pkeys[column.name]
-
-    return TableSpec(columns)

adam/commands/devices.py

@@ -1,147 +0,0 @@
-from adam.commands.command import Command
-from adam.repl_state import ReplState
-
-class DeviceExport(Command):
-    COMMAND = f'{ReplState.X}:'
-
-    # the singleton pattern
-    def __new__(cls, *args, **kwargs):
-        if not hasattr(cls, 'instance'): cls.instance = super(DeviceExport, cls).__new__(cls)
-
-        return cls.instance
-
-    def __init__(self, successor: Command=None):
-        super().__init__(successor)
-
-    def command(self):
-        return DeviceExport.COMMAND
-
-    def run(self, cmd: str, state: ReplState):
-        if not self.args(cmd):
-            return super().run(cmd, state)
-
-        state.device = ReplState.X
-
-        return state
-
-    def completion(self, state: ReplState):
-        return super().completion(state)
-
-    def help(self, _: ReplState):
-        return f'{DeviceAuditLog.COMMAND}\t move to Export Database Operations device'
-
-class DeviceAuditLog(Command):
-    COMMAND = f'{ReplState.L}:'
-
-    # the singleton pattern
-    def __new__(cls, *args, **kwargs):
-        if not hasattr(cls, 'instance'): cls.instance = super(DeviceAuditLog, cls).__new__(cls)
-
-        return cls.instance
-
-    def __init__(self, successor: Command=None):
-        super().__init__(successor)
-
-    def command(self):
-        return DeviceAuditLog.COMMAND
-
-    def run(self, cmd: str, state: ReplState):
-        if not self.args(cmd):
-            return super().run(cmd, state)
-
-        state.device = ReplState.L
-
-        return state
-
-    def completion(self, state: ReplState):
-        return super().completion(state)
-
-    def help(self, _: ReplState):
-        return f'{DeviceAuditLog.COMMAND}\t move to Audit Log Operations device'
-
-class DeviceCass(Command):
-    COMMAND = f'{ReplState.C}:'
-
-    # the singleton pattern
-    def __new__(cls, *args, **kwargs):
-        if not hasattr(cls, 'instance'): cls.instance = super(DeviceCass, cls).__new__(cls)
-
-        return cls.instance
-
-    def __init__(self, successor: Command=None):
-        super().__init__(successor)
-
-    def command(self):
-        return DeviceCass.COMMAND
-
-    def run(self, cmd: str, state: ReplState):
-        if not self.args(cmd):
-            return super().run(cmd, state)
-
-        state.device = ReplState.C
-
-        return state
-
-    def completion(self, state: ReplState):
-        return super().completion(state)
-
-    def help(self, _: ReplState):
-        return f'{DeviceCass.COMMAND}\t move to Cassandra Operations device'
-
-class DevicePostgres(Command):
-    COMMAND = f'{ReplState.P}:'
-
-    # the singleton pattern
-    def __new__(cls, *args, **kwargs):
-        if not hasattr(cls, 'instance'): cls.instance = super(DevicePostgres, cls).__new__(cls)
-
-        return cls.instance
-
-    def __init__(self, successor: Command=None):
-        super().__init__(successor)
-
-    def command(self):
-        return DevicePostgres.COMMAND
-
-    def run(self, cmd: str, state: ReplState):
-        if not self.args(cmd):
-            return super().run(cmd, state)
-
-        state.device = ReplState.P
-
-        return state
-
-    def completion(self, state: ReplState):
-        return super().completion(state)
-
-    def help(self, _: ReplState):
-        return f'{DevicePostgres.COMMAND}\t move to Postgres Operations device'
-
-class DeviceApp(Command):
-    COMMAND = f'{ReplState.A}:'
-
-    # the singleton pattern
-    def __new__(cls, *args, **kwargs):
-        if not hasattr(cls, 'instance'): cls.instance = super(DeviceApp, cls).__new__(cls)
-
-        return cls.instance
-
-    def __init__(self, successor: Command=None):
-        super().__init__(successor)
-
-    def command(self):
-        return DeviceApp.COMMAND
-
-    def run(self, cmd: str, state: ReplState):
-        if not self.args(cmd):
-            return super().run(cmd, state)
-
-        state.device = ReplState.A
-
-        return state
-
-    def completion(self, state: ReplState):
-        return super().completion(state)
-
-    def help(self, _: ReplState):
-        return f'{DeviceApp.COMMAND}\t move to App Operations device'

adam/commands/export/export_on_x.py

@@ -1,76 +0,0 @@
-from adam.commands.command import Command
-from adam.repl_state import ReplState, RequiredState
-from adam.sql.sql_completer import SqlCompleter
-from adam.utils import log2
-from adam.utils_athena import Athena
-
-class ExportUseX(Command):
-    COMMAND = 'use'
-
-    # the singleton pattern
-    def __new__(cls, *args, **kwargs):
-        if not hasattr(cls, 'instance'): cls.instance = super(ExportUseX, cls).__new__(cls)
-
-        return cls.instance
-
-    def __init__(self, successor: Command=None):
-        super().__init__(successor)
-
-    def command(self):
-        return ExportUseX.COMMAND
-
-    def required(self):
-        return ReplState.X
-
-    def run(self, cmd: str, state: ReplState):
-        if not(args := self.args(cmd)):
-            return super().run(cmd, state)
-
-        state, args = self.apply_state(args, state)
-        if not self.validate_state(state):
-            return state
-
-        if not args:
-            if state.in_repl:
-                log2('Specify database to use.')
-            else:
-                log2('* database is missing.')
-
-                Command.display_help()
-
-            return 'command-missing'
-
-        state.export_session = args[0].replace('export_', '') if args[0].startswith('export_') else args[0]
-        Athena.clear_cache()
-
-        return state
-
-    def completion(self, state: ReplState):
-        dict = super().completion(state, {n: None for n in Athena.database_names()})
-
-        if state.export_session:
-            db = f'export_{state.export_session}'
-            dict = dict | {'select': SqlCompleter(
-                lambda: Athena.table_names(database=db, function='export'),
-                dml='select',
-                columns=lambda table: Athena.column_names(database=db, function='export'),
-                variant='athena'
-            )}
-
-        return dict
-
-    def help(self, _: ReplState):
-        return f'{ExportUseX.COMMAND} <export db name>\t use Export Database'
-
-# No action body, only for a help entry and auto-completion
-class ExportSelectX(Command):
-    COMMAND = 'select'
-
-    def command(self):
-        return ExportSelectX.COMMAND
-
-    def required(self):
-        return RequiredState.EXPORT_DB
-
-    def help(self, _: ReplState):
-        return f'<sql-select-statements>\t run queries on export database'

adam/commands/export/export_rmdbs.py

@@ -1,65 +0,0 @@
-import boto3
-
-from adam.commands.command import Command
-from adam.commands.export.utils_export import ing
-from adam.config import Config
-from adam.repl_state import ReplState, RequiredState
-from adam.utils import log2
-from adam.utils_athena import Athena
-
-class RemoveExportDatabases(Command):
-    COMMAND = '&rmdbs'
-
-    # the singleton pattern
-    def __new__(cls, *args, **kwargs):
-        if not hasattr(cls, 'instance'): cls.instance = super(RemoveExportDatabases, cls).__new__(cls)
-
-        return cls.instance
-
-    def __init__(self, successor: Command=None):
-        super().__init__(successor)
-
-    def command(self):
-        return RemoveExportDatabases.COMMAND
-
-    def required(self):
-        return RequiredState.CLUSTER_OR_POD
-
-    def run(self, cmd: str, state: ReplState):
-        if not(args := self.args(cmd)):
-            return super().run(cmd, state)
-
-        state, args = self.apply_state(args, state)
-        if not self.validate_state(state):
-            return state
-
-        dbs = Athena.database_names('export_')
-        def drop_all_exports():
-            for db in dbs:
-                query = f'DROP DATABASE {db} CASCADE'
-                if Config().is_debug():
-                    log2(query)
-                Athena.query(query)
-
-        if Config().is_debug():
-            drop_all_exports()
-        else:
-            ing(f'Droping {len(dbs)} databases', drop_all_exports)
-
-        def delete_s3_folder():
-            s3 = boto3.resource('s3')
-            bucket = s3.Bucket('c3.ops--qing')
-            bucket.objects.filter(Prefix='export/').delete()
-
-        if Config().is_debug():
-            delete_s3_folder()
-        else:
-            ing(f'Deleting s3 folder: export', delete_s3_folder)
-
-        return state
-
-    def completion(self, state: ReplState):
-        return super().completion(state)
-
-    def help(self, _: ReplState):
-        return f'{RemoveExportDatabases.COMMAND}\t remove all export databases'

adam/commands/postgres/postgres_utils.py

@@ -1,31 +0,0 @@
-import functools
-
-from adam.commands.postgres.postgres_context import PostgresContext
-from adam.config import Config
-
-TestPG = [False]
-
-@functools.lru_cache()
-def pg_database_names(ns: str, pg_path: str):
-    if TestPG[0]:
-        return ['azops88_c3ai_c3']
-
-    Config().wait_log('Inspecting Postgres Databases...')
-
-    pg = PostgresContext.apply(ns, pg_path)
-    return [db['name'] for db in pg.databases() if db['owner'] == PostgresContext.default_owner()]
-
-@functools.lru_cache()
-def pg_table_names(ns: str, pg_path: str):
-    if TestPG[0]:
-        return ['C3_2_XYZ1']
-
-    Config().wait_log('Inspecting Postgres Database...')
-    return [table['name'] for table in pg_tables(ns, pg_path) if table['schema'] == PostgresContext.default_schema()]
-
-def pg_tables(ns: str, pg_path: str):
-    pg = PostgresContext.apply(ns, pg_path)
-    if pg.db:
-        return pg.tables()
-
-    return []

adam/commands/reaper/reaper_session.py

@@ -1,159 +0,0 @@
-from collections.abc import Callable
-import threading
-from kubernetes import client
-import portforward
-import re
-import requests
-from typing import List, cast
-
-from adam.config import Config
-from adam.utils_k8s.kube_context import KubeContext
-from adam.repl_state import ReplState
-from adam.utils import lines_to_tabular, log2
-
-class ReaperSession:
-    is_forwarding = False
-    stopping = threading.Event()
-    schedules_ids_by_cluster: dict[str, list[str]] = {}
-
-    def __init__(self, pod: str, headers: dict[str, str] = None):
-        self.pod = pod
-        self.headers = headers
-
-    def login(self, state: ReplState, local_addr: str, remote_addr: str, show_output = True) -> str :
-        user, pw = state.user_pass(secret_path='reaper.secret')
-
-        response = requests.post(f'http://{local_addr}/login', headers={
-            'Accept': '*'
-        },data={
-            'username':user,
-            'password':pw})
-        if show_output:
-            log2(f'POST {remote_addr}/login')
-            log2(f'     username={user}&password={pw}')
-
-        if int(response.status_code / 100) != 2:
-            if show_output:
-                log2("login failed")
-            return None
-
-        return response.headers['Set-Cookie']
-
-    def port_forwarded(self, state: ReplState, path: str, body: Callable[[str, dict[str, str]], requests.Response], method: str = None, show_output = True):
-        local_port = Config().get('reaper.port-forward.local-port', 9001)
-        target_port = 8080
-
-        def f(local_addr: str, remote_addr: str):
-            if not self.headers:
-                self.headers = self.cookie_header(state, local_addr, remote_addr, show_output=show_output)
-
-            if show_output and method:
-                log2(f'{method} {remote_addr}/{path}')
-            response = body(f'http://{local_addr}/{path}', self.headers)
-
-            if response:
-                if int(response.status_code / 100) != 2:
-                    if show_output:
-                        log2(response.status_code)
-                    return response
-
-            if show_output:
-                log2()
-
-            return response if response else 'no-response'
-
-        if KubeContext.in_cluster():
-            # cs-a526330d23-cs-a526330d23-default-sts-0 ->
-            # curl http://cs-a526330d23-cs-a526330d23-reaper-service.stgawsscpsr.svc.cluster.local:8080
-            groups = re.match(r'^(.*?-.*?-.*?-.*?-).*', state.sts)
-            if groups:
-                svc_name = Config().get('reaper.service-name', 'reaper-service')
-                svc = f'{groups[1]}{svc_name}.{state.namespace}.svc.cluster.local:{target_port}'
-                return f(local_addr=svc, remote_addr=svc)
-            else:
-                return None
-        else:
-            with portforward.forward(state.namespace, self.pod, local_port, target_port):
-                return f(local_addr=f'localhost:{local_port}', remote_addr=f'{self.pod}:{target_port}')
-
-    def cookie_header(self, state: ReplState, local_addr, remote_addr, show_output = True):
-        return {'Cookie': self.login(state, local_addr, remote_addr, show_output=show_output)}
-
-    def create(state: ReplState) -> 'ReaperSession':
-        pods = ReaperSession.list_reaper_pods(state.sts if state.sts else state.pod, state.namespace)
-        if pods:
-            return ReaperSession(pods[0].metadata.name)
-        else:
-            log2('No reaper found.')
-
-            return None
-
-    def list_reaper_pods(sts_name: str, namespace: str) -> List[client.V1Pod]:
-        v1 = client.CoreV1Api()
-
-        # k8ssandra.io/reaper: cs-d0767a536f-cs-d0767a536f-reaper
-        groups = re.match(Config().get('reaper.pod.cluster-regex', r'(.*?-.*?-.*?-.*?)-.*'), sts_name)
-        label_selector = Config().get('reaper.pod.label-selector', 'k8ssandra.io/reaper={cluster}-reaper').replace('{cluster}', groups[1])
-
-        return cast(List[client.V1Pod], v1.list_namespaced_pod(namespace, label_selector=label_selector).items)
-
-    def show_schedules(self, state: ReplState, filter: Callable[[list[dict]], dict] = None):
-        schedules = self.list_schedules(state, filter=filter)
-        # forced refresh of schedule list
-        if not filter:
-            self.schedules_ids_by_cluster[state.sts] = [schedule['id'] for schedule in schedules]
-        self.show_schedules_tabular(schedules)
-
-    def schedule_ids(self, state: ReplState, show_output = True, filter: Callable[[list[dict]], dict] = None):
-        schedules = self.list_schedules(state, show_output=show_output, filter=filter)
-        return [schedule['id'] for schedule in schedules]
-
-    def list_schedules(self, state: ReplState, show_output = True, filter: Callable[[list[dict]], dict] = None) -> list[dict]:
-        def body(uri: str, headers: dict[str, str]):
-            return requests.get(uri, headers=headers)
-
-        response = self.port_forwarded(state, 'repair_schedule', body, method='GET', show_output=show_output)
-        if not response:
-            return
-
-        res = response.json()
-        if filter:
-            res = filter(res)
-
-        return res
-
-    def show_schedules_tabular(self, schedules: list[dict]):
-        log2(lines_to_tabular([f"{schedule['id']} {schedule['state']} {schedule['cluster_name']} {schedule['keyspace_name']}" for schedule in schedules], 'ID STATE CLUSTER KEYSPACE'))
-
-    def show_schedule(self, state: ReplState, schedule_id: str):
-        def filter(schedules: list[dict]):
-            return [schedule for schedule in schedules if schedule['id'] == schedule_id]
-
-        self.show_schedules(state, filter)
-
-    def reaper_spec(self, state: ReplState) -> dict[str, any]:
-        user, pw = state.user_pass(secret_path='reaper.secret')
-        local_port = Config().get('reaper.port-forward.local-port', 9001)
-
-        return {
-            'pod': self.pod,
-            'exec': f'kubectl exec -it {self.pod} -n {state.namespace} -- bash',
-            'forward': f'kubectl port-forward pods/{self.pod} -n {state.namespace} {local_port}:8080',
-            'web-uri': f'http://localhost:{local_port}/webui',
-            'username': user,
-            'password': pw
-        }
-
-    def cached_schedule_ids(state: ReplState) -> list[str]:
-        if state.sts in ReaperSession.schedules_ids_by_cluster:
-            return ReaperSession.schedules_ids_by_cluster[state.sts]
-
-        if reaper := ReaperSession.create(state):
-            Config().wait_log('Inspecting Cassandra Reaper...')
-
-            schedules = reaper.schedule_ids(state, show_output = False)
-            ReaperSession.schedules_ids_by_cluster[state.sts] = schedules
-
-            return schedules
-
-        return []