kaqing 1.77.0__py3-none-any.whl → 2.0.171__py3-none-any.whl

adam/commands/reaper/utils_reaper.py

@@ -0,0 +1,196 @@
+from collections.abc import Callable
+from functools import partial
+from typing import List, cast
+from kubernetes import client
+import re
+
+import requests
+from adam.config import Config
+from adam.repl_state import ReplState
+from adam.utils import lines_to_tabular, log2, wait_log
+from adam.utils_k8s.k8s import port_forwarding
+
+class ReaperService:
+    def __init__(self, state: ReplState, local_addr: str, remote_addr: str, show_out = True):
+        self.state = state
+        self.local_addr = local_addr
+        self.remote_addr = remote_addr
+        self.show_out = show_out
+        self.headers = None
+
+    def get(self, path: str, params: dict[str, any] = {}):
+        with logging(self, 'GET', path) as (url, headers):
+            return requests.get(url, headers=headers, params=params)
+
+    def put(self, path: str, params: dict[str, any] = {}):
+        with logging(self, 'PUT', path) as (url, headers):
+            return requests.put(url, headers=headers, params=params)
+
+    def post(self, path: str, params: dict[str, any] = {}):
+        with logging(self, 'POST', path) as (url, headers):
+            return requests.post(url, headers=headers, params=params)
+
+class ReaperLogginHandler:
+    def __init__(self, svc: ReaperService, method: str, path: str):
+        self.svc = svc
+        self.method = method
+        self.path = path
+
+    def __enter__(self) -> tuple[str, dict[str, any]]:
+        if not self.svc.headers:
+            self.svc.headers = Reapers.cookie_header(self.svc.state, self.svc.local_addr, self.svc.remote_addr, show_output=self.svc.show_out)
+
+        if self.svc.show_out and self.method:
+            log2(f'{self.method} {self.svc.remote_addr}/{self.path}')
+
+        return (f'http://{self.svc.local_addr}/{self.path}', self.svc.headers)
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if exc_val and isinstance(exc_val, requests.Response):
+            if int(exc_val.status_code / 100) != 2:
+                if self.svc.show_out:
+                    log2(exc_val.status_code)
+
+        return False
+
+def logging(svc: ReaperService, method: str, path: str):
+    return ReaperLogginHandler(svc, method, path)
+
+class ReaperHandler:
+    def __init__(self, state: ReplState, show_out = True):
+        self.state = state
+        self.show_out = show_out
+        self.headers = None
+        self.forwarding = None
+
+    def __enter__(self):
+        self.forwarding = port_forwarding(self.state, Reapers.local_port(), partial(Reapers.svc_or_pod, self.state), Reapers.target_port())
+        (local_addr, remote_addr) = self.forwarding.__enter__()
+
+        return ReaperService(self.state, local_addr, remote_addr, show_out=self.show_out)
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if self.forwarding:
+            return self.forwarding.__exit__(exc_type, exc_val, exc_tb)
+
+        return False
+
+def reaper(state: ReplState, show_out = True):
+    return ReaperHandler(state, show_out=show_out)
+
+class Reapers:
+    schedules_ids_by_cluster: dict[str, list[str]] = {}
+
+    def pod_name(state: ReplState):
+        pods = Reapers.list_reaper_pods(state.sts if state.sts else state.pod, state.namespace)
+        if pods:
+            return pods[0].metadata.name
+
+        return None
+
+    def show_schedule(state: ReplState, schedule_id: str):
+        def filter(schedules: list[dict]):
+            return [schedule for schedule in schedules if schedule['id'] == schedule_id]
+
+        Reapers.show_schedules(state, filter)
+
+    def show_schedules(state: ReplState, filter: Callable[[list[dict]], dict] = None):
+        schedules = Reapers.list_schedules(state, filter=filter)
+
+        # forced refresh of schedule list
+        if not filter:
+            Reapers.schedules_ids_by_cluster[state.sts] = [schedule['id'] for schedule in schedules]
+        Reapers.show_schedules_tabular(schedules)
+
+    def schedule_ids(state: ReplState, show_output = True, filter: Callable[[list[dict]], dict] = None):
+        schedules = Reapers.list_schedules(state, show_output=show_output, filter=filter)
+        return [schedule['id'] for schedule in schedules]
+
+    def list_schedules(state: ReplState, show_output = True, filter: Callable[[list[dict]], dict] = None) -> list[dict]:
+        with reaper(state, show_out=show_output) as requests:
+            if not (response := requests.get('repair_schedule')):
+                return
+
+            res = response.json()
+            if filter:
+                res = filter(res)
+
+            return res
+
+    def show_schedules_tabular(schedules: list[dict]):
+        log2(lines_to_tabular([f"{schedule['id']} {schedule['state']} {schedule['cluster_name']} {schedule['keyspace_name']}" for schedule in schedules], 'ID STATE CLUSTER KEYSPACE'))
+
+    def list_reaper_pods(sts_name: str, namespace: str) -> List[client.V1Pod]:
+        v1 = client.CoreV1Api()
+
+        # k8ssandra.io/reaper: cs-d0767a536f-cs-d0767a536f-reaper
+        groups = re.match(Config().get('reaper.pod.cluster-regex', r'(.*?-.*?-.*?-.*?)-.*'), sts_name)
+        label_selector = Config().get('reaper.pod.label-selector', 'k8ssandra.io/reaper={cluster}-reaper').replace('{cluster}', groups[1])
+
+        return cast(List[client.V1Pod], v1.list_namespaced_pod(namespace, label_selector=label_selector).items)
+
+    def cookie_header(state: ReplState, local_addr, remote_addr, show_output = True):
+        return {'Cookie': Reapers.login(state, local_addr, remote_addr, show_output=show_output)}
+
+    def login(state: ReplState, local_addr: str, remote_addr: str, show_output = True) -> str :
+        user, pw = state.user_pass(secret_path='reaper.secret')
+
+        response = requests.post(f'http://{local_addr}/login', headers={
+            'Accept': '*'
+        },data={
+            'username':user,
+            'password':pw})
+        if show_output:
+            log2(f'POST {remote_addr}/login')
+            log2(f'     username={user}&password={pw}')
+
+        if int(response.status_code / 100) != 2:
+            if show_output:
+                log2("login failed")
+            return None
+
+        return response.headers['Set-Cookie']
+
+    def reaper_spec(state: ReplState) -> dict[str, any]:
+        if not (pod := Reapers.pod_name(state)):
+            return {}
+
+        user, pw = state.user_pass(secret_path='reaper.secret')
+
+        return {
+            'pod': pod,
+            'exec': f'kubectl exec -it {pod} -n {state.namespace} -- bash',
+            'forward': f'kubectl port-forward pods/{pod} -n {state.namespace} {Reapers.local_port()}:{Reapers.target_port()}',
+            'web-uri': f'http://localhost:{Reapers.local_port()}/webui',
+            'username': user,
+            'password': pw
+        }
+
+    def cached_schedule_ids(state: ReplState) -> list[str]:
+        if state.sts in Reapers.schedules_ids_by_cluster:
+            return Reapers.schedules_ids_by_cluster[state.sts]
+
+        if pod := Reapers.pod_name(state):
+            wait_log('Inspecting Cassandra Reaper...')
+
+            schedules = Reapers.schedule_ids(state, show_output = False)
+            Reapers.schedules_ids_by_cluster[state.sts] = schedules
+
+            return schedules
+
+        return []
+
+    def svc_name():
+        return Config().get('reaper.service-name', 'reaper-service')
+
+    def local_port():
+        return Config().get('reaper.port-forward.local-port', 9001)
+
+    def target_port():
+        return 8080
+
+    def svc_or_pod(state: ReplState, is_service: bool):
+        if is_service:
+            return Reapers.svc_name()
+        else:
+            return Reapers.pod_name(state)

adam/commands/repair/repair.py

@@ -0,0 +1,26 @@
+import click
+
+from adam.commands.intermediate_command import IntermediateCommand
+from .repair_run import RepairRun
+from .repair_scan import RepairScan
+from .repair_stop import RepairStop
+from .repair_log import RepairLog
+
+class Repair(IntermediateCommand):
+    COMMAND = 'repair'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(Repair, cls).__new__(cls)
+
+        return cls.instance
+
+    def command(self):
+        return Repair.COMMAND
+
+    def cmd_list(self):
+        return [RepairRun(), RepairScan(), RepairStop(), RepairLog()]
+
+class RepairCommandHelper(click.Command):
+    def get_help(self, ctx: click.Context):
+        IntermediateCommand.intermediate_help(super().get_help(ctx), Repair.COMMAND, Repair().cmd_list(), show_cluster_help=True)

{walker → adam}/commands/repair/repair_log.py

@@ -1,6 +1,6 @@
-from walker.commands.command import Command
-from walker.k8s_utils.jobs import Jobs
-from walker.repl_state import ReplState, RequiredState
+from adam.commands.command import Command
+from adam.utils_k8s.jobs import Jobs
+from adam.repl_state import ReplState, RequiredState
 
 class RepairLog(Command):
     COMMAND = 'repair log'
@@ -24,14 +24,11 @@ class RepairLog(Command):
         if not(args := self.args(cmd)):
             return super().run(cmd, state)
 
-        state, args = self.apply_state(args, state)
-        if not self.validate_state(state):
-            return state
-
-        ns = state.namespace
-        Jobs.get_logs('cassrepair-'+state.sts, ns)
+        with self.validate(args, state) as (args, state):
+            ns = state.namespace
+            Jobs.get_logs('cassrepair-'+state.sts, ns)
 
-        return state
+            return state
 
     def completion(self, state: ReplState):
         if state.sts:

adam/commands/repair/repair_run.py

@@ -0,0 +1,70 @@
+from adam.commands.command import Command
+from adam.commands.reaper.utils_reaper import Reapers
+from adam.utils_k8s.jobs import Jobs
+from adam.utils_k8s.volumes import Volumes
+from adam.repl_state import ReplState, RequiredState
+from adam.config import Config
+from adam.commands.reaper.reaper_runs_abort import ReaperRunsAbort
+from adam.commands.reaper.reaper_schedule_stop import ReaperScheduleStop
+from adam.utils import log2
+
+class RepairRun(Command):
+    COMMAND = 'repair run'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(RepairRun, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return RepairRun.COMMAND
+
+    def required(self):
+        return RequiredState.CLUSTER
+
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        with self.validate(args, state) as (args, state):
+            replace = False
+            if len(args) == 1:
+                replace = args[0] == 'replace'
+
+            try:
+                log2("Stopping all reaper schedules...")
+                for schedule_id in Reapers.cached_schedule_ids(state):
+                    ReaperScheduleStop().run(f'reaper stop schedule {schedule_id}', state)
+                log2("Aborting all reaper runs...")
+                state = ReaperRunsAbort().run('reaper abort runs', state)
+            except:
+                pass
+
+            image = Config().get('repair.image', 'ci-registry.c3iot.io/cloudops/cassrepair:2.0.11')
+            secret = Config().get('repair.secret', 'ciregistryc3iotio')
+            log_path = Config().get('repair.log-path', '/home/cassrepair/logs/')
+            user, _ = state.user_pass()
+            ns = state.namespace
+            env = Config().get('repair.env', {})
+            env["cluster"] = ns
+            env_from = {"username": user, "password": user}
+            pvc_name ='cassrepair-log-' + state.sts
+            Volumes.create_pvc(pvc_name, 30, ns)
+            if replace:
+                Jobs.delete('cassrepair-'+state.sts, ns)
+            Jobs.create('cassrepair-'+state.sts, ns, image, secret, env, env_from, 'cassrepair', pvc_name, log_path)
+
+            return state
+
+    def completion(self, state: ReplState):
+        if state.sts:
+            return super().completion(state)
+
+        return {}
+
+    def help(self, _: ReplState):
+        return f'{RepairRun.COMMAND} [replace]\t start a repair job, default not replacing'

adam/commands/repair/repair_scan.py

@@ -0,0 +1,71 @@
+import time
+
+from adam.commands.command import Command
+from adam.utils_k8s.pods import Pods
+from adam.repl_state import ReplState, RequiredState
+from adam.utils import log2
+from adam.config import Config
+
+class RepairScan(Command):
+    COMMAND = 'repair scan'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(RepairScan, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return RepairScan.COMMAND
+
+    def required(self):
+        return RequiredState.CLUSTER
+
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        with self.validate(args, state) as (args, state):
+            n = "7"
+            if len(args) == 1:
+                n = str(args[0])
+            image = Config().get('repair.image', 'ci-registry.c3iot.io/cloudops/cassrepair:2.0.11')
+            secret = Config().get('repair.secret', 'ciregistryc3iotio')
+            log_path = secret = Config().get('repair.log-path', '/home/cassrepair/logs/')
+            ns = state.namespace
+            pvc_name ='cassrepair-log-' + state.sts
+            pod_name = 'repair-scan'
+
+            try:
+                Pods.create(ns, pod_name, image, ["sh", "-c", "tail -f /dev/null"],
+                        secret=secret,
+                        env={},
+                        volume_name='cassrepair-log',
+                        pvc_name=pvc_name,
+                        mount_path='/home/cassrepair/logs/')
+            except Exception as e:
+                if e.status == 409:
+                    log2(f"Pod {pod_name} already exists")
+                else:
+                    log2("Exception when calling BatchV1Apii->create_namespaced_job: %s\n" % e)
+
+            Pods.wait_for_running(ns, pod_name, 'Waiting for the scanner pod to start up...')
+
+            try:
+                Pods.exec(pod_name, pod_name, ns, f"find {log_path} -type f -mtime -{n} -print0 | xargs -0 grep failed")
+            finally:
+                Pods.delete(pod_name, ns)
+
+            return state
+
+    def completion(self, state: ReplState):
+        if state.sts:
+            return super().completion(state)
+
+        return {}
+
+    def help(self, _: ReplState):
+        return f'{RepairScan.COMMAND} [n]\t scan last n days repair log, default 7 days'

{walker → adam}/commands/repair/repair_stop.py

@@ -1,7 +1,7 @@
-from walker.commands.command import Command
-from walker.k8s_utils.jobs import Jobs
-from walker.repl_state import ReplState, RequiredState
-from walker.config import Config
+from adam.commands.command import Command
+from adam.utils_k8s.jobs import Jobs
+from adam.repl_state import ReplState, RequiredState
+from adam.config import Config
 
 class RepairStop(Command):
     COMMAND = 'repair stop'
@@ -25,14 +25,11 @@ class RepairStop(Command):
         if not(args := self.args(cmd)):
             return super().run(cmd, state)
 
-        state, args = self.apply_state(args, state)
-        if not self.validate_state(state):
-            return state
-
-        ns = state.namespace
-        Jobs.delete('cassrepair-'+state.sts, ns)
+        with self.validate(args, state) as (args, state):
+            ns = state.namespace
+            Jobs.delete('cassrepair-'+state.sts, ns)
 
-        return state
+            return state
 
     def completion(self, state: ReplState):
         if state.sts:

adam/commands/report.py

@@ -0,0 +1,61 @@
+import click
+import json
+
+from adam.checks.check_result import CheckResult
+from adam.checks.check_utils import run_checks
+from adam.commands.command import Command
+from adam.repl_state import ReplState
+from adam.utils import log2
+
+class Report(Command):
+    COMMAND = 'report'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(Report, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return Report.COMMAND
+
+    def required(self):
+        return ReplState.NON_L
+
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        with self.validate(args, state) as (args, state):
+            output: dict[str, any] = {}
+
+            if state.in_repl:
+                args, show = Command.extract_options(args, ['-s', '--show'])
+
+                args, redirect = Command.extract_options(args, ['>'])
+                if not redirect or not args:
+                    log2('Please specify file name: e.g. report > /tmp/report.log')
+                    return 'no-report-destination'
+
+                results = run_checks(state.sts, state.namespace, state.pod, show_out=show)
+                output = CheckResult.report(results)
+                with open(args[0], "w") as json_file:
+                    json.dump(output, json_file, indent=2)
+                    log2(f'Report stored in {args[0]}.')
+            else:
+                args, show = Command.extract_options(args, ['-s', '--show'])
+
+                results = run_checks(state.sts, state.namespace, state.pod, show_out=show)
+                output = CheckResult.report(results)
+                click.echo(json.dumps(output, indent=2))
+
+            return output
+
+    def completion(self, state: ReplState):
+        return super().completion(state, {">": None})
+
+    def help(self, _: ReplState):
+        return f"{Report.COMMAND} > <file-name>\t generate report"

adam/commands/restart.py

@@ -0,0 +1,60 @@
+from adam.commands import extract_options
+from adam.commands.command import Command
+from adam.utils_k8s.pods import Pods
+from adam.utils_k8s.statefulsets import StatefulSets
+from adam.repl_state import ReplState, RequiredState
+from adam.utils import log2
+
+class Restart(Command):
+    COMMAND = 'restart'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(Restart, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return Restart.COMMAND
+
+    def required(self):
+        return RequiredState.CLUSTER_OR_POD
+
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        with self.validate(args, state) as (args, state):
+            with extract_options(args, '--force') as (args, forced):
+                if not args:
+                    if state.pod:
+                        log2(f'Restarting {state.pod}...')
+                        Pods.delete(state.pod, state.namespace)
+                    else:
+                        if not forced:
+                            log2('Please add --force for restarting all nodes in a cluster.')
+                            return 'force-needed'
+
+                        log2(f'Restarting all pods from {state.sts}...')
+                        for pod_name in StatefulSets.pod_names(state.sts, state.namespace):
+                            Pods.delete(pod_name, state.namespace)
+                else:
+                    for arg in args:
+                        Pods.delete(arg, state.namespace)
+
+                return state
+
+    def completion(self, state: ReplState):
+        if super().completion(state):
+            if state.pod:
+                return {Restart.COMMAND: None}
+            elif state.sts:
+                return {Restart.COMMAND: {p: None for p in StatefulSets.pod_names(state.sts, state.namespace)}}
+
+        return {}
+
+    def help(self, _: ReplState):
+        return f"{Restart.COMMAND} [<host-id>] [--force]\t restart the node you're in or all the nodes in the cluster"

{walker → adam}/commands/rollout.py

@@ -2,12 +2,13 @@ import datetime
 from kubernetes import client
 from kubernetes.client.rest import ApiException
 
-from walker.commands.command import Command
-from walker.commands.watch import Watch
-from walker.k8s_utils.statefulsets import StatefulSets
-from walker.config import Config
-from walker.repl_state import ReplState, RequiredState
-from walker.utils import duration, log2
+from adam.commands import extract_options
+from adam.commands.command import Command
+from adam.commands.watch import Watch
+from adam.utils_k8s.statefulsets import StatefulSets
+from adam.config import Config
+from adam.repl_state import ReplState, RequiredState
+from adam.utils import duration, log2
 
 class RollOut(Command):
     COMMAND = 'rollout'
@@ -31,32 +32,28 @@ class RollOut(Command):
         if not(args := self.args(cmd)):
             return super().run(cmd, state)
 
-        state, args = self.apply_state(args, state)
-        if not self.validate_state(state):
-            return state
+        with self.validate(args, state) as (args, state):
+            with extract_options(args, '--force') as (args, forced):
+                restarted, rollingout = StatefulSets.restarted_at(state.sts, state.namespace)
+                if rollingout and not forced:
+                    log2(f"* Cluster is being rolled out for {duration(restarted)}. Please wait until it's done or use --force.")
 
-        args, forced = Command.extract_options(args, '--force')
+                    return state
 
-        restarted, rollingout = StatefulSets.restarted_at(state.sts, state.namespace)
-        if rollingout and not forced:
-            log2(f"* Cluster is being rolled out for {duration(restarted)}. Please wait until it's done or use --force.")
+                self.rolling_restart(state.sts, state.namespace)
 
-            return state
+                auto_watch = False
+                if (auto_watch_cmds := Config().get('watch.auto', 'rollout')):
+                    cmds = [c.strip(' ') for c in auto_watch_cmds.split(',')]
+                    if self.command() in cmds:
+                        auto_watch = True
+                        log2('Rolling out cluster with auto watch...')
+                        Watch().run('watch', state)
 
-        self.rolling_restart(state.sts, state.namespace)
+                if not auto_watch:
+                    log2('Rolling out cluster...')
 
-        auto_watch = False
-        if (auto_watch_cmds := Config().get('watch.auto', 'rollout')):
-            cmds = [c.strip(' ') for c in auto_watch_cmds.split(',')]
-            if self.command() in cmds:
-                auto_watch = True
-                log2('Rolling out cluster with auto watch...')
-                Watch().run('watch', state)
-
-        if not auto_watch:
-            log2('Rolling out cluster...')
-
-        return state
+                return state
 
     def rolling_restart(self, statefulset, namespace):
         # kubectl rollout restart statefulset <statefulset-name>
@@ -82,9 +79,7 @@ class RollOut(Command):
             log2("Exception when calling AppsV1Api->read_namespaced_statefulset_status: %s\n" % e)
 
     def completion(self, state: ReplState):
-        if state.pod:
-            return {}
-        elif state.sts:
+        if super().completion(state):
             return {RollOut.COMMAND: None}
 
         return {}

adam/commands/shell.py

@@ -0,0 +1,34 @@
+import os
+
+from adam.commands.command import Command
+from adam.repl_state import ReplState
+
+class Shell(Command):
+    COMMAND = ':sh'
+
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(Shell, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self, successor: Command=None):
+        super().__init__(successor)
+
+    def command(self):
+        return Shell.COMMAND
+
+    def run(self, cmd: str, state: ReplState):
+        if not(args := self.args(cmd)):
+            return super().run(cmd, state)
+
+        with self.validate(args, state):
+            os.system('QING_DROPPED=true bash')
+
+            return state
+
+    def completion(self, state: ReplState):
+        return super().completion(state)
+
+    def help(self, _: ReplState):
+        return f'{Shell.COMMAND}\t drop down to shell'