kaqing 1.77.0__py3-none-any.whl → 2.0.171__py3-none-any.whl

walker/commands/reaper/reaper_runs.py

@@ -1,97 +0,0 @@
-import requests
-
-from walker.commands.command import Command
-from .reaper_session import ReaperSession
-from walker.config import Config
-from walker.repl_state import ReplState, RequiredState
-from walker.utils import convert_seconds, epoch, lines_to_tabular, log, log2
-
-class ReaperRuns(Command):
-    COMMAND = 'reaper show runs'
-    reaper_login = None
-
-    # the singleton pattern
-    def __new__(cls, *args, **kwargs):
-        if not hasattr(cls, 'instance'): cls.instance = super(ReaperRuns, cls).__new__(cls)
-
-        return cls.instance
-
-    def __init__(self, successor: Command=None):
-        super().__init__(successor)
-
-    def command(self):
-        return ReaperRuns.COMMAND
-
-    def required(self):
-        return RequiredState.CLUSTER
-
-    def run(self, cmd: str, state: ReplState):
-        if not(args := self.args(cmd)):
-            return super().run(cmd, state)
-
-        state, args = self.apply_state(args, state)
-        if not self.validate_state(state):
-            return state
-
-        if not(reaper := ReaperSession.create(state)):
-            return state
-
-        self.show_runs(state, reaper)
-
-        return state
-
-    def show_runs(self, state: ReplState, reaper: ReaperSession):
-        def body(uri: str, headers: dict[str, str]):
-            return requests.get(uri, headers=headers, params={
-                'cluster_name': 'all',
-                'limit': Config().get('reaper.show-runs-batch', 10)
-            })
-
-        def line(run):
-            state = run['state']
-            start_time = run['start_time']
-            end_time = run['end_time']
-            duration = '-'
-            if state == 'DONE' and end_time:
-                hours, minutes, seconds = convert_seconds(epoch(end_time) - epoch(start_time))
-                if hours:
-                    duration = f"{hours:2d}h {minutes:2d}m {seconds:2d}s"
-                elif minutes:
-                    duration = f"{minutes:2d}m {seconds:2d}s"
-                else:
-                    duration = f"{seconds:2d}s"
-
-            return f"{start_time},{duration},{state},{run['cluster_name']},{run['keyspace_name']},{len(run['column_families'])},{run['segments_repaired']}/{run['total_segments']}"
-
-        # PAUSED, RUNNING, ABORTED
-        response = reaper.port_forwarded(state, 'repair_run?state=RUNNING', body, method='GET')
-        if not response:
-            return
-
-        header = 'Start,Duration,State,Cluster,Keyspace,Tables,Repaired'
-
-        runs = response.json()
-        if runs:
-            log(lines_to_tabular(sorted([line(run) for run in runs], reverse=True), header, separator=","))
-        else:
-            log2('No running runs found.')
-            log2()
-
-        response = reaper.port_forwarded(state, 'repair_run?state=PAUSED,ABORTED,DONE', body, method='GET')
-        if not response:
-            return
-
-        runs = response.json()
-        if runs:
-            log(lines_to_tabular(sorted([line(run) for run in runs], reverse=True), header, separator=","))
-        else:
-            log2('No runs found.')
-
-    def completion(self, state: ReplState):
-        if state.sts:
-            return super().completion(state)
-
-        return {}
-
-    def help(self, _: ReplState):
-        return f'{ReaperRuns.COMMAND}\t show reaper runs'

walker/commands/reaper/reaper_runs_abort.py

@@ -1,83 +0,0 @@
-import requests
-
-from walker.commands.command import Command
-from .reaper_session import ReaperSession
-from walker.config import Config
-from walker.repl_state import ReplState, RequiredState
-from walker.utils import log2
-
-class ReaperRunsAbort(Command):
-    COMMAND = 'reaper abort runs'
-    reaper_login = None
-
-    # the singleton pattern
-    def __new__(cls, *args, **kwargs):
-        if not hasattr(cls, 'instance'): cls.instance = super(ReaperRunsAbort, cls).__new__(cls)
-
-        return cls.instance
-
-    def __init__(self, successor: Command=None):
-        super().__init__(successor)
-
-    def command(self):
-        return ReaperRunsAbort.COMMAND
-
-    def required(self):
-        return RequiredState.CLUSTER
-
-    def run(self, cmd: str, state: ReplState):
-        if not(args := self.args(cmd)):
-            return super().run(cmd, state)
-
-        state, args = self.apply_state(args, state)
-        if not self.validate_state(state):
-            return state
-
-        if not(reaper := ReaperSession.create(state)):
-            return state
-
-        self.stop_runs(state, reaper)
-
-        return state
-
-    def stop_runs(self, state: ReplState, reaper: ReaperSession):
-        def body_list(uri: str, headers: dict[str, str]):
-            return requests.get(uri, headers=headers, params={
-                'cluster_name': 'all',
-                'limit': Config().get('reaper.abort-runs-batch', 10)
-            })
-
-        def body_abort(uri: str, headers: dict[str, str]):
-            return requests.put(uri, headers=headers)
-
-        # PAUSED, RUNNING, ABORTED
-        aborted = 0
-        while True == True:
-            response = reaper.port_forwarded(state, 'repair_run?state=RUNNING', body_list, method='GET')
-            if not response:
-                break
-
-            runs = response.json()
-            if not runs:
-                break
-
-            for run in runs:
-                run_id = run['id']
-                # PUT /repair_run/{id}/state/{state}
-                reaper.port_forwarded(state, f'repair_run/{run_id}/state/ABORTED', body_abort, method='PUT')
-                log2(f'Aborted {len(runs)} runs.')
-                aborted += 1
-
-        if aborted:
-            log2(f'Aborted {aborted} runs in total.')
-        else:
-            log2('No running repair runs found.')
-
-    def completion(self, state: ReplState):
-        if state.sts:
-            return super().completion(state)
-
-        return {}
-
-    def help(self, _: ReplState):
-        return f'{ReaperRunsAbort.COMMAND}\t abort all running reaper runs'

walker/commands/reaper/reaper_schedule_activate.py

@@ -1,64 +0,0 @@
-import requests
-
-from walker.commands.command import Command
-from .reaper_session import ReaperSession
-from walker.repl_state import ReplState, RequiredState
-from walker.utils import log2
-
-class ReaperScheduleActivate(Command):
-    COMMAND = 'reaper activate schedule'
-    reaper_login = None
-
-    # the singleton pattern
-    def __new__(cls, *args, **kwargs):
-        if not hasattr(cls, 'instance'): cls.instance = super(ReaperScheduleActivate, cls).__new__(cls)
-
-        return cls.instance
-
-    def __init__(self, successor: Command=None):
-        super().__init__(successor)
-
-    def command(self):
-        return ReaperScheduleActivate.COMMAND
-
-    def required(self):
-        return RequiredState.CLUSTER
-
-    def run(self, cmd: str, state: ReplState):
-        if not(args := self.args(cmd)):
-            return super().run(cmd, state)
-
-        state, args = self.apply_state(args, state)
-        if not self.validate_state(state):
-            return state
-
-        if not args:
-            log2('Specify schedule to activate.')
-
-            return state
-
-        schedule_id = args[0]
-        if not(reaper := ReaperSession.create(state)):
-            return state
-
-        self.activate_schedule(state, reaper, schedule_id)
-
-        return schedule_id
-
-    def activate_schedule(self, state: ReplState, reaper: ReaperSession, schedule_id: str):
-        def body(uri: str, headers: dict[str, str]):
-            return requests.put(uri, headers=headers)
-
-        reaper.port_forwarded(state, f'repair_schedule/{schedule_id}?state=ACTIVE', body, method='PUT')
-        reaper.show_schedule(state, schedule_id)
-
-    def completion(self, state: ReplState):
-        if state.sts:
-            leaf = {id: None for id in ReaperSession.cached_schedule_ids(state)}
-
-            return super().completion(state, leaf)
-
-        return {}
-
-    def help(self, _: ReplState):
-        return f'{ReaperScheduleActivate.COMMAND} <schedule-id>\t resume reaper schedule'

walker/commands/reaper/reaper_schedule_start.py

@@ -1,64 +0,0 @@
-import requests
-
-from walker.commands.command import Command
-from .reaper_session import ReaperSession
-from walker.repl_state import ReplState, RequiredState
-from walker.utils import log2
-
-class ReaperScheduleStart(Command):
-    COMMAND = 'reaper start schedule'
-    reaper_login = None
-
-    # the singleton pattern
-    def __new__(cls, *args, **kwargs):
-        if not hasattr(cls, 'instance'): cls.instance = super(ReaperScheduleStart, cls).__new__(cls)
-
-        return cls.instance
-
-    def __init__(self, successor: Command=None):
-        super().__init__(successor)
-
-    def command(self):
-        return ReaperScheduleStart.COMMAND
-
-    def required(self):
-        return RequiredState.CLUSTER
-
-    def run(self, cmd: str, state: ReplState):
-        if not(args := self.args(cmd)):
-            return super().run(cmd, state)
-
-        state, args = self.apply_state(args, state)
-        if not self.validate_state(state):
-            return state
-
-        if not args:
-            log2('Specify schedule to activate.')
-
-            return state
-
-        schedule_id = args[0]
-        if not(reaper := ReaperSession.create(state)):
-            return schedule_id
-
-        self.start_schedule(state, reaper, schedule_id)
-
-        return schedule_id
-
-    def start_schedule(self, state: ReplState, reaper: ReaperSession, schedule_id: str):
-        def body(uri: str, headers: dict[str, str]):
-            return requests.post(uri, headers=headers)
-
-        reaper.port_forwarded(state, f'repair_schedule/start/{schedule_id}', body, method='POST')
-        reaper.show_schedule(state, schedule_id)
-
-    def completion(self, state: ReplState):
-        if state.sts:
-            leaf = {id: None for id in ReaperSession.cached_schedule_ids(state)}
-
-            return super().completion(state, leaf)
-
-        return {}
-
-    def help(self, _: ReplState):
-        return f'{ReaperScheduleStart.COMMAND} <schedule-id>\t start reaper runs for schedule'

walker/commands/reaper/reaper_schedule_stop.py

@@ -1,64 +0,0 @@
-import requests
-
-from walker.commands.command import Command
-from .reaper_session import ReaperSession
-from walker.repl_state import ReplState, RequiredState
-from walker.utils import log2
-
-class ReaperScheduleStop(Command):
-    COMMAND = 'reaper stop schedule'
-    reaper_login = None
-
-    # the singleton pattern
-    def __new__(cls, *args, **kwargs):
-        if not hasattr(cls, 'instance'): cls.instance = super(ReaperScheduleStop, cls).__new__(cls)
-
-        return cls.instance
-
-    def __init__(self, successor: Command=None):
-        super().__init__(successor)
-
-    def command(self):
-        return ReaperScheduleStop.COMMAND
-
-    def required(self):
-        return RequiredState.CLUSTER
-
-    def run(self, cmd: str, state: ReplState):
-        if not(args := self.args(cmd)):
-            return super().run(cmd, state)
-
-        state, args = self.apply_state(args, state)
-        if not self.validate_state(state):
-            return state
-
-        if not args:
-            log2('Specify run schedule to stop.')
-
-            return state
-
-        schedule_id = args[0]
-        if not(reaper := ReaperSession.create(state)):
-            return schedule_id
-
-        self.stop_schedule(state, reaper, schedule_id)
-
-        return schedule_id
-
-    def stop_schedule(self, state: ReplState, reaper: ReaperSession, schedule_id: str):
-        def body(uri: str, headers: dict[str, str]):
-            return requests.put(uri, headers=headers)
-
-        reaper.port_forwarded(state, f'repair_schedule/{schedule_id}?state=PAUSED', body, method='PUT')
-        reaper.show_schedule(state, schedule_id)
-
-    def completion(self, state: ReplState):
-        if state.sts:
-            leaf = {id: None for id in ReaperSession.cached_schedule_ids(state)}
-
-            return super().completion(state, leaf)
-
-        return {}
-
-    def help(self, _: ReplState):
-        return f'{ReaperScheduleStop.COMMAND} <schedule-id>\t pause reaper schedule'

walker/commands/reaper/reaper_session.py

@@ -1,159 +0,0 @@
-from collections.abc import Callable
-import threading
-from kubernetes import client
-import portforward
-import re
-import requests
-from typing import List, cast
-
-from walker.config import Config
-from walker.k8s_utils.kube_context import KubeContext
-from walker.repl_state import ReplState
-from walker.utils import lines_to_tabular, log2
-
-class ReaperSession:
-    is_forwarding = False
-    stopping = threading.Event()
-    schedules_ids_by_cluster: dict[str, list[str]] = {}
-
-    def __init__(self, pod: str, headers: dict[str, str] = None):
-        self.pod = pod
-        self.headers = headers
-
-    def login(self, state: ReplState, local_addr: str, remote_addr: str, show_output = True) -> str :
-        user, pw = state.user_pass(secret_path='reaper.secret')
-
-        response = requests.post(f'http://{local_addr}/login', headers={
-            'Accept': '*'
-        },data={
-            'username':user,
-            'password':pw})
-        if show_output:
-            log2(f'POST {remote_addr}/login')
-            log2(f'     username={user}&password={pw}')
-
-        if int(response.status_code / 100) != 2:
-            if show_output:
-                log2("login failed")
-            return None
-
-        return response.headers['Set-Cookie']
-
-    def port_forwarded(self, state: ReplState, path: str, body: Callable[[str, dict[str, str]], requests.Response], method: str = None, show_output = True):
-        local_port = Config().get('reaper.port-forward.local-port', 9001)
-        target_port = 8080
-
-        def f(local_addr: str, remote_addr: str):
-            if not self.headers:
-                self.headers = self.cookie_header(state, local_addr, remote_addr, show_output=show_output)
-
-            if show_output and method:
-                log2(f'{method} {remote_addr}/{path}')
-            response = body(f'http://{local_addr}/{path}', self.headers)
-
-            if response:
-                if int(response.status_code / 100) != 2:
-                    if show_output:
-                        log2(response.status_code)
-                    return response
-
-            if show_output:
-                log2()
-
-            return response if response else 'no-response'
-
-        if KubeContext.in_cluster():
-            # cs-a526330d23-cs-a526330d23-default-sts-0 ->
-            # curl http://cs-a526330d23-cs-a526330d23-reaper-service.stgawsscpsr.svc.cluster.local:8080
-            groups = re.match(r'^(.*?-.*?-.*?-.*?-).*', state.sts)
-            if groups:
-                svc_name = Config().get('reaper.service-name', 'reaper-service')
-                svc = f'{groups[1]}{svc_name}.{state.namespace}.svc.cluster.local:{target_port}'
-                return f(local_addr=svc, remote_addr=svc)
-            else:
-                return None
-        else:
-            with portforward.forward(state.namespace, self.pod, local_port, target_port):
-                return f(local_addr=f'localhost:{local_port}', remote_addr=f'{self.pod}:{target_port}')
-
-    def cookie_header(self, state: ReplState, local_addr, remote_addr, show_output = True):
-        return {'Cookie': self.login(state, local_addr, remote_addr, show_output=show_output)}
-
-    def create(state: ReplState) -> 'ReaperSession':
-        pods = ReaperSession.list_reaper_pods(state.sts if state.sts else state.pod, state.namespace)
-        if pods:
-            return ReaperSession(pods[0].metadata.name)
-        else:
-            log2('No reaper found.')
-
-            return None
-
-    def list_reaper_pods(sts_name: str, namespace: str) -> List[client.V1Pod]:
-        v1 = client.CoreV1Api()
-
-        # k8ssandra.io/reaper: cs-d0767a536f-cs-d0767a536f-reaper
-        groups = re.match(Config().get('reaper.pod.cluster-regex', r'(.*?-.*?-.*?-.*?)-.*'), sts_name)
-        label_selector = Config().get('reaper.pod.label-selector', 'k8ssandra.io/reaper={cluster}-reaper').replace('{cluster}', groups[1])
-
-        return cast(List[client.V1Pod], v1.list_namespaced_pod(namespace, label_selector=label_selector).items)
-
-    def show_schedules(self, state: ReplState, filter: Callable[[list[dict]], dict] = None):
-        schedules = self.list_schedules(state, filter=filter)
-        # forced refresh of schedule list
-        if not filter:
-            self.schedules_ids_by_cluster[state.sts] = [schedule['id'] for schedule in schedules]
-        self.show_schedules_tabular(schedules)
-
-    def schedule_ids(self, state: ReplState, show_output = True, filter: Callable[[list[dict]], dict] = None):
-        schedules = self.list_schedules(state, show_output=show_output, filter=filter)
-        return [schedule['id'] for schedule in schedules]
-
-    def list_schedules(self, state: ReplState, show_output = True, filter: Callable[[list[dict]], dict] = None) -> list[dict]:
-        def body(uri: str, headers: dict[str, str]):
-            return requests.get(uri, headers=headers)
-
-        response = self.port_forwarded(state, 'repair_schedule', body, method='GET', show_output=show_output)
-        if not response:
-            return
-
-        res = response.json()
-        if filter:
-            res = filter(res)
-
-        return res
-
-    def show_schedules_tabular(self, schedules: list[dict]):
-        log2(lines_to_tabular([f"{schedule['id']} {schedule['state']} {schedule['cluster_name']} {schedule['keyspace_name']}" for schedule in schedules], 'ID STATE CLUSTER KEYSPACE'))
-
-    def show_schedule(self, state: ReplState, schedule_id: str):
-        def filter(schedules: list[dict]):
-            return [schedule for schedule in schedules if schedule['id'] == schedule_id]
-
-        self.show_schedules(state, filter)
-
-    def reaper_spec(self, state: ReplState) -> dict[str, any]:
-        user, pw = state.user_pass(secret_path='reaper.secret')
-        local_port = Config().get('reaper.port-forward.local-port', 9001)
-
-        return {
-            'pod': self.pod,
-            'exec': f'kubectl exec -it {self.pod} -n {state.namespace} -- bash',
-            'forward': f'kubectl port-forward pods/{self.pod} -n {state.namespace} {local_port}:8080',
-            'web-uri': f'http://localhost:{local_port}/webui',
-            'username': user,
-            'password': pw
-        }
-
-    def cached_schedule_ids(state: ReplState) -> list[str]:
-        if state.sts in ReaperSession.schedules_ids_by_cluster:
-            return ReaperSession.schedules_ids_by_cluster[state.sts]
-
-        if reaper := ReaperSession.create(state):
-            state.wait_log("Inspecting Cassandra Reaper...")
-
-            schedules = reaper.schedule_ids(state, show_output = False)
-            ReaperSession.schedules_ids_by_cluster[state.sts] = schedules
-
-            return schedules
-
-        return []

walker/commands/repair/repair.py

@@ -1,68 +0,0 @@
-import click
-
-from walker.commands.command import Command
-from walker.commands.command_helpers import ClusterCommandHelper
-from .repair_run import RepairRun
-from .repair_scan import RepairScan
-from .repair_stop import RepairStop
-from .repair_log import RepairLog
-from walker.repl_state import ReplState, RequiredState
-from walker.utils import lines_to_tabular, log, log2
-
-class Repair(Command):
-    COMMAND = 'repair'
-
-    # the singleton pattern
-    def __new__(cls, *args, **kwargs):
-        if not hasattr(cls, 'instance'): cls.instance = super(Repair, cls).__new__(cls)
-
-        return cls.instance
-
-    def __init__(self, successor: Command=None):
-        super().__init__(successor)
-
-    def command(self):
-        return Repair.COMMAND
-
-    def required(self):
-        return RequiredState.CLUSTER
-
-    def run(self, cmd: str, state: ReplState):
-        if not(args := self.args(cmd)):
-            return super().run(cmd, state)
-
-        state, args = self.apply_state(args, state)
-        if not self.validate_state(state):
-            return state
-
-        if state.in_repl:
-            log(lines_to_tabular([c.help(ReplState()) for c in Repair.cmd_list()], separator=':'))
-
-            return 'command-missing'
-        else:
-            # head with the Chain of Responsibility pattern
-            cmds = Command.chain(Repair.cmd_list())
-            if not cmds.run(cmd, state):
-                log2('* Command is missing.')
-                Command.display_help()
-
-    def cmd_list():
-        return [RepairRun(), RepairScan(), RepairStop(), RepairLog()]
-
-    def completion(self, state: ReplState):
-        if state.sts:
-            return super().completion(state)
-        return {}
-
-    def help(self, _: ReplState):
-        return None
-
-class RepairCommandHelper(click.Command):
-    def get_help(self, ctx: click.Context):
-        log(super().get_help(ctx))
-        log()
-        log('Sub-Commands:')
-
-        log(lines_to_tabular([c.help(ReplState()).replace(f'{Repair.COMMAND} ', '  ', 1) for c in Repair.cmd_list()], separator=':'))
-        log()
-        ClusterCommandHelper.cluster_help()

walker/commands/repair/repair_run.py

@@ -1,72 +0,0 @@
-from walker.commands.command import Command
-from walker.k8s_utils.jobs import Jobs
-from walker.k8s_utils.volumes import Volumes
-from walker.repl_state import ReplState, RequiredState
-from walker.config import Config
-from walker.commands.reaper.reaper_session import ReaperSession
-from walker.commands.reaper.reaper_runs_abort import ReaperRunsAbort
-from walker.commands.reaper.reaper_schedule_stop import ReaperScheduleStop
-from walker.utils import log2
-
-class RepairRun(Command):
-    COMMAND = 'repair run'
-
-    # the singleton pattern
-    def __new__(cls, *args, **kwargs):
-        if not hasattr(cls, 'instance'): cls.instance = super(RepairRun, cls).__new__(cls)
-
-        return cls.instance
-
-    def __init__(self, successor: Command=None):
-        super().__init__(successor)
-
-    def command(self):
-        return RepairRun.COMMAND
-
-    def required(self):
-        return RequiredState.CLUSTER
-
-    def run(self, cmd: str, state: ReplState):
-        if not(args := self.args(cmd)):
-            return super().run(cmd, state)
-
-        state, args = self.apply_state(args, state)
-        if not self.validate_state(state):
-            return state
-
-        replace = False
-        if len(args) == 1:
-            replace = args[0] == 'replace'
-
-        log2("Stopping all reaper schedules...")
-        reaper = ReaperSession.create(state)
-        schedules = reaper.schedule_ids(state)
-        for schedule_id in schedules:
-            ReaperScheduleStop().run(f'reaper stop schedule {schedule_id}', state)
-        log2("Aborting all reaper runs...")
-        state = ReaperRunsAbort().run('reaper abort runs', state)
-
-        image = Config().get('repair.image', 'ci-registry.c3iot.io/cloudops/cassrepair:2.0.11')
-        secret = Config().get('repair.secret', 'ciregistryc3iotio')
-        log_path = Config().get('repair.log-path', '/home/cassrepair/logs/')
-        user, _ = state.user_pass()
-        ns = state.namespace
-        env = Config().get('repair.env', {})
-        env["cluster"] = ns
-        env_from = {"username": user, "password": user}
-        pvc_name ='cassrepair-log-' + state.sts
-        Volumes.create_pvc(pvc_name, 30, ns)
-        if replace:
-            Jobs.delete('cassrepair-'+state.sts, ns)
-        Jobs.create('cassrepair-'+state.sts, ns, image, secret, env, env_from, 'cassrepair', pvc_name, log_path)
-
-        return state
-
-    def completion(self, state: ReplState):
-        if state.sts:
-            return super().completion(state)
-
-        return {}
-
-    def help(self, _: ReplState):
-        return f'{RepairRun.COMMAND} [replace]\t start a repair job, default not replacing'