kaqing 2.0.214__py3-none-any.whl → 2.0.227__py3-none-any.whl

adam/commands/fs/utils_fs.py

@@ -0,0 +1,192 @@
+import itertools
+import os
+import re
+
+from adam.commands.devices.devices import Devices
+from adam.config import Config
+from adam.repl_state import ReplState
+from adam.utils import Color, PodLogFile, log2, log_dir, log_to_pods, pod_log_dir, tabulize
+from adam.utils_k8s.pod_exec_result import PodExecResult
+from adam.utils_k8s.pods import Pods
+from adam.utils_local import find_local_files
+from adam.utils_async_job import AsyncJobs
+
+def show_last_results(state: ReplState):
+    last_id, last_command = AsyncJobs.last_command()
+    logs: list[str] = find_local_files(f'{log_dir()}/{last_id}*')
+
+    # /tmp/qing-db/q/logs/16145959.repair-0.err
+    # /tmp/qing-db/q/logs/16145959.repair-0.log
+
+    logs_by_n: dict[str, LogLine] = {}
+    for l in logs:
+        size = str(os.path.getsize(l))
+        n = l[len(f'{log_dir()}/{last_id}'):]
+        if n.startswith('.'):
+            n = n[1:]
+
+        if n.endswith('.log'):
+            n = n[:-4]
+            key = 'out'
+        else:
+            n = n[:-4]
+            key = 'err'
+
+        n = n.split('-')[-1]
+
+        if n not in logs_by_n:
+            logs_by_n[n] = LogLine(n, file=l.replace('.log', '.err'))
+
+        if key == 'out':
+            logs_by_n[n].out = size
+        else:
+            logs_by_n[n].err = size
+
+    if last_command:
+        keywords = last_command.strip(' &').split(' ')
+        if keywords and keywords[0] == 'nodetool':
+            # nodetool -u cs-a7b13e29bd-superuser -pw lDed6uXQAQP72kHOYuML repair &
+            keywords = keywords[-1:]
+
+        for ps in find_pids_for_pod(state.pod, Devices.of(state).default_container(state), state.namespace, keywords, match_last_arg=True):
+            n = ps.pod.split('-')[-1]
+            if n in logs_by_n:
+                logs_by_n[n].merge(ps)
+
+    log2(f'[{last_id}] {last_command}')
+    log2()
+    tabulize(sorted(logs_by_n.keys()), fn=lambda n: logs_by_n[n].table_line(), header=LogLine.header, separator='\t')
+
+def show_last_pod_results(state: ReplState):
+    last_id, last_command = AsyncJobs.last_command()
+
+    container = Devices.of(state).default_container(state)
+
+    if last_command:
+        keywords = last_command.strip(' &').split(' ')
+        if keywords and keywords[0] == 'nodetool':
+            # nodetool -u cs-a7b13e29bd-superuser -pw lDed6uXQAQP72kHOYuML repair &
+            keywords = keywords[-1:]
+
+    action = 'find-files'
+    msg = 'd`Running|Ran ' + action + ' onto {size} pods'
+    pods = Devices.of(state).pod_names(state)
+    with Pods.parallelize(pods, len(pods), msg=msg, action=action) as exec:
+        results: list[LogLine] = exec.map(lambda pod: log_line_for_pod(pod, container, state.namespace, pod_log_dir(), last_id, log_to_pods(), keywords))
+
+        log2(f'[{last_id}] {last_command}')
+        log2()
+        tabulize(sorted(results, key=lambda l: l.ordinal), fn=lambda l: l.table_line(), header=LogLine.header, separator='\t')
+
+def log_line_for_pod(pod: str, container: str, namespace: str, dir: str, last_id: str, remote: bool, keywords: list[str]):
+    logs: list[PodLogFile] = Pods.find_files(pod, container, namespace, f'{dir}/{last_id}*', remote=remote)
+
+    line = LogLine()
+
+    procs = []
+    if keywords:
+        procs = find_pids_for_pod(pod, container, namespace, keywords)
+
+    for log in logs:
+        l = str(log)
+        if l.endswith('.log'):
+            key = 'out'
+        else:
+            key = 'err'
+
+        line.ordinal = log.pod.split('-')[-1]
+        line.file = l.replace('.log', '.err')
+
+        if key == 'out':
+            line.out = log.size
+        else:
+            line.err = log.size
+            for proc in procs:
+                line.merge(proc)
+
+    return line
+
+def find_pids_for_cluster(state: ReplState, keywords: list[str], match_last_arg = False) -> list['ProcessInfo']:
+    container = Devices.of(state).default_container(state)
+
+    action = 'find-procs'
+    msg = 'd`Running|Ran ' + action + ' onto {size} pods'
+    pods = Devices.of(state).pod_names(state)
+    with Pods.parallelize(pods, len(pods), msg=msg, action=action) as exec:
+        r: list[list[ProcessInfo]] = exec.map(lambda pod: find_pids_for_pod(pod, container, state.namespace, keywords, match_last_arg=match_last_arg))
+
+        return list(itertools.chain.from_iterable(r))
+
+def find_pids_for_pod(pod: str, container: str, namespace: str, keywords: list[str], match_last_arg = False) -> list['ProcessInfo']:
+    r: PodExecResult = Pods.exec(pod, container, namespace, _find_procs_command(keywords), text_color=Color.gray)
+
+    return ProcessInfo.from_find_process_results(r, last_arg = keywords[-1] if match_last_arg else None)
+
+def _find_procs_command(keywords: list[str]):
+    regex_pattern = re.compile(r'[^\w\s]')
+
+    greps = []
+    for a in keywords:
+        a = a.strip('"\'').strip(' ')
+
+        if a and not regex_pattern.search(a):
+            greps.append(f'grep -- {a}')
+
+    awk = "awk '{ print $1, $2, $8, $NF }'"
+
+    return f"ps -ef | grep -v grep | {' | '.join(greps)} | {awk}"
+
+class ProcessInfo:
+    header = 'POD\tUSER\tPID\tCMD\tLAST_ARG'
+
+    def __init__(self, user: str, pid: str, cmd: str, last_arg: str, pod: str = None):
+        self.user = user
+        self.pid = pid
+        self.cmd = cmd
+        self.last_arg = last_arg
+        self.pod = pod
+
+    def from_find_process_results(rs: PodExecResult, last_arg: str = None):
+        processes: list[ProcessInfo] = []
+
+        for l in rs.stdout.split('\n'):
+            l = l.strip(' \t\r\n')
+            if not l:
+                continue
+
+            tokens = l.split(' ')
+            if last_arg and tokens[3] != last_arg:
+                continue
+
+            processes.append(ProcessInfo(tokens[0], tokens[1], tokens[2], tokens[3], pod=rs.pod))
+
+        return processes
+
+    def table_line(self):
+        return '\t'.join([self.pod, self.user, self.pid, self.cmd, self.last_arg])
+
+    def tabulize(processes: list['ProcessInfo']):
+        tabulize(processes, lambda p: p.table_line(), header = ProcessInfo.header, separator='\t')
+
+class LogLine(ProcessInfo):
+    header='ORDINAL\tPID\tCMD\tLAST_ARG\tOUT_SIZE\tERR_SIZE\tLOG(ERR)_FILES'
+
+    def __init__(self, ordinal: str = '-', out: str = '-', err: str = '-', file: str = '-', user: str = '-', pid: str = '-', cmd: str = '-', last_arg: str = '-', pod: str = '-'):
+        super().__init__(user, pid, cmd, last_arg, pod)
+        self.ordinal = ordinal
+        self.out = out
+        self.err = err
+        self.file = file
+
+    def __repr__(self):
+        return f"LogLine({', '.join([self.ordinal, self.pid, self.cmd, self.last_arg, self.out, self.err, self.file])})"
+
+    def table_line(self):
+        return '\t'.join([self.ordinal, self.pid, self.cmd, self.last_arg, self.out, self.err, self.file if self.err not in ['-', '0'] else self.file.replace('.err', '.log')])
+
+    def merge(self, process: ProcessInfo):
+        self.user = process.user
+        self.pid = process.pid
+        self.cmd = process.cmd
+        self.last_arg = process.last_arg
+        self.pod = process.pod

adam/commands/help.py

@@ -28,7 +28,7 @@ class Help(Command):
 
         lines = []
         lines.append('NAVIGATION')
-        lines.append('  a: | c: | l: | p: | x:\t switch to another operational device: App, Cassandra, Audit, Postgres or Export')
+        lines.append('  a: | c: | l: | p: | x:\tswitch to another operational device: App, Cassandra, Audit, Postgres or Export')
         lines.extend(section(ReplCommands.navigation()))
         lines.append('CASSANDRA')
         lines.extend(section(ReplCommands.cassandra_ops()))
@@ -52,5 +52,5 @@ class Help(Command):
     def completion(self, _: ReplState):
         return {Help.COMMAND: None}
 
-    def help(self, _: ReplState):
+    def help(self, state: ReplState):
         return None

adam/commands/kubectl.py

@@ -34,5 +34,5 @@ class Kubectl(Command):
         return super().completion(state)
 
 
-    def help(self, _: ReplState):
-        return f'{Kubectl.COMMAND} \t run a kubectl command'
+    def help(self, state: ReplState):
+        return super().help(state, 'run a kubectl command', args='[kubectl-args]')

adam/commands/medusa/medusa_backup.py

@@ -50,5 +50,5 @@ class MedusaBackup(Command):
     def completion(self, state: ReplState):
         return super().completion(state)
 
-    def help(self, _: ReplState):
-        return f'{MedusaBackup.COMMAND}\t start a backup job'
+    def help(self, state: ReplState):
+        return super().help(state, 'start a backup job')

adam/commands/medusa/medusa_restore.py

@@ -68,5 +68,5 @@ class MedusaRestore(Command):
     def completion(self, state: ReplState):
         return super().completion(state, lambda: {id: None for id in medusa_backup_names(state)}, auto_key='medusa.backups')
 
-    def help(self, _: ReplState):
-        return f'{MedusaRestore.COMMAND}\t start a restore job'
+    def help(self, state: ReplState):
+        return super().help(state, 'start a restore job')

adam/commands/medusa/medusa_show_backupjobs.py

@@ -47,5 +47,5 @@ class MedusaShowBackupJobs(Command):
     def completion(self, state: ReplState):
         return super().completion(state)
 
-    def help(self, _: ReplState):
-        return f'{MedusaShowBackupJobs.COMMAND}\t show Medusa backups'
+    def help(self, state: ReplState):
+        return super().help(state, 'show Medusa backups')

adam/commands/medusa/medusa_show_restorejobs.py

@@ -43,5 +43,5 @@ class MedusaShowRestoreJobs(Command):
     def completion(self, state: ReplState):
         return super().completion(state)
 
-    def help(self, _: ReplState):
-        return f'{MedusaShowRestoreJobs.COMMAND}\t show Medusa restores'
+    def help(self, state: ReplState):
+        return super().help(state, 'show Medusa restores')

adam/commands/nodetool/nodetool.py

@@ -1,13 +1,16 @@
+import time
 import click
 
+from adam.commands import extract_options
 from adam.commands.command import Command
 from adam.commands.command_helpers import ClusterOrPodCommandHelper
 from adam.commands.cql.utils_cql import cassandra
 from adam.commands.devices.devices import Devices
 from adam.commands.nodetool.nodetool_commands import NODETOOL_COMMANDS
+from adam.commands.nodetool.utils_nodetool import abort_nodetool_tasks, find_running_nodetool_tasks
 from adam.config import Config
 from adam.repl_state import ReplState, RequiredState
-from adam.utils import log
+from adam.utils import log, log2, tabulize
 
 class NodeTool(Command):
     COMMAND = 'nodetool'
@@ -32,16 +35,36 @@ class NodeTool(Command):
             return super().run(cmd, state)
 
         with self.validate(args, state) as (args, state):
-            with cassandra(state) as pods:
-                pods.nodetool(' '.join(args), status=(args[0] == 'status'))
+            with extract_options(args, '--force') as (args, forced):
+                with cassandra(state) as pods:
+                    if subcommand := args[0]:
+                        if subcommand in ['compact', 'repair']:
+                            ps = find_running_nodetool_tasks(subcommand, state)
+                            if ps:
+                                tabulize(ps, lambda p: '\t'.join(p), header='POD\tCMD\tID/PID\tLAST_ARG\tREAPER_RUN_STATE', separator='\t')
+                                log2()
 
-            return state
+                                if forced:
+                                    log2(f"* Found running instances of 'nodetool {subcommand}', aborting existing ones...")
+                                    abort_nodetool_tasks(state, subcommand, ps)
+
+                                    wait_duration = Config().get('nodetool.grace-period-after-abort', 10)
+                                    log2(f"* Scheduling new 'nodetool {subcommand}' in {wait_duration} secs...")
+                                    time.sleep(wait_duration)
+                                else:
+                                    log2(f"* Found running instances of 'nodetool {subcommand}', add --force to abort existing ones.")
+
+                                    return state
+
+                    pods.nodetool(' '.join(args), status=(args[0] == 'status'))
+
+                    return state
 
     def completion(self, state: ReplState):
-        return super().completion(state, {c: {'&': None} for c in NODETOOL_COMMANDS}, pods=Devices.of(state).pods(state, '-'))
+        return super().completion(state, {c: {'--force': {'&': None}, '&': None} for c in NODETOOL_COMMANDS}, pods=Devices.of(state).pods(state, '-'))
 
-    def help(self, _: ReplState):
-        return f'{NodeTool.COMMAND} <sub-command> [&]\t run nodetool with arguments'
+    def help(self, state: ReplState):
+        return super().help(state, 'run nodetool with arguments', args='<sub-command> [&]')
 
 class NodeToolCommandHelper(click.Command):
     def get_help(self, ctx: click.Context):

adam/commands/nodetool/utils_nodetool.py

@@ -0,0 +1,44 @@
+from adam.commands.devices.devices import Devices
+from adam.commands.export.utils_export import state_with_pod
+from adam.commands.fs.utils_fs import find_pids_for_cluster
+from adam.commands.reaper.utils_reaper import reaper
+from adam.config import Config
+from adam.repl_state import ReplState
+from adam.utils import log2
+
+def find_running_nodetool_tasks(subcommand: str, state: ReplState) -> list[list[str]]:
+    lines = []
+    processes = find_pids_for_cluster(state, [subcommand])
+    for p in processes:
+        l = [p.pod, p.cmd, p.pid, p.last_arg]
+        lines.append(l)
+
+    if subcommand == 'repair':
+        with reaper(state) as http:
+            response = http.get('repair_run?state=RUNNING', params={
+                'cluster_name': 'all',
+                'limit': Config().get('reaper.show-runs-batch', 10)
+            })
+
+            runs = response.json()
+            if runs:
+                for r in runs:
+                    l = ['reaper', 'reaper', r['id'], '', r['state']]
+                    lines.append(l)
+
+    return lines
+
+def abort_nodetool_tasks(state: ReplState, subcommand: str, processes: list[list[str]]):
+    for p in processes:
+        pod = p[0]
+        cmd = p[1]
+        id = p[2]
+
+        if pod == 'reaper':
+            with reaper(state) as http:
+                http.put(f'repair_run/{id}/state/ABORTED')
+        elif cmd == subcommand:
+            log2(f'@{pod} bash kill -9 {id}')
+
+            with state_with_pod(state, pod) as state1:
+                    Devices.of(state).bash(state, state1, ['kill', '-9', id])

adam/commands/postgres/postgres.py

@@ -67,8 +67,8 @@ class Postgres(IntermediateCommand):
 
         return {}
 
-    def help(self, _: ReplState):
-        return f'<sql-statements> [&]\t run queries on Postgres databases'
+    def help(self, state: ReplState):
+        return super().help(state, 'run queries on Postgres databases', command='[pg] <sql-statements>')
 
 class PostgresCommandHelper(click.Command):
     def get_help(self, ctx: click.Context):
@@ -83,7 +83,4 @@ class PostgresPg(Command):
     COMMAND = 'pg'
 
     def command(self):
-        return PostgresPg.COMMAND
-
-    def help(self, _: ReplState):
-        return f'pg <sql-statements>\t run queries on Postgres databases'
+        return PostgresPg.COMMAND

adam/commands/postgres/postgres_ls.py

@@ -37,5 +37,5 @@ class PostgresLs(Command):
 
         return {}
 
-    def help(self, _: ReplState):
-        return f'{PostgresLs.COMMAND}\t list postgres hosts, databases or tables'
+    def help(self, state: ReplState):
+        return super().help(state, 'list postgres hosts, databases or tables')

adam/commands/postgres/postgres_preview.py

@@ -37,5 +37,5 @@ class PostgresPreview(Command):
 
         return {}
 
-    def help(self, _: ReplState):
-        return f'{PostgresPreview.COMMAND}\t preview postgres table'
+    def help(self, state: ReplState):
+        return super().help(state, 'preview postgres table')

adam/commands/preview_table.py

@@ -32,8 +32,7 @@ class PreviewTable(Command):
             return state
 
     def completion(self, _: ReplState):
-        # taken care of by the sql completer
         return {}
 
-    def help(self, _: ReplState):
-        return f'{PreviewTable.COMMAND} TABLE\t preview table'
+    def help(self, state: ReplState):
+        return super().help(state, 'preview table', args='TABLE')

adam/commands/reaper/reaper_forward.py

@@ -89,5 +89,5 @@ class ReaperForward(Command):
     def completion(self, state: ReplState):
         return super().completion(state)
 
-    def help(self, _: ReplState):
-        return f'{ReaperForward.COMMAND}\t port-forward to reaper'
+    def help(self, state: ReplState):
+        return super().help(state, 'port-forward to reaper')

adam/commands/reaper/reaper_forward_stop.py

@@ -39,5 +39,5 @@ class ReaperForwardStop(Command):
     def completion(self, state: ReplState):
         return super().completion(state)
 
-    def help(self, _: ReplState):
-        return f'{ReaperForwardStop.COMMAND}\t stop port-forward to reaper'
+    def help(self, state: ReplState):
+        return super().help(state, 'stop port-forward to reaper')

adam/commands/reaper/reaper_restart.py

@@ -36,5 +36,5 @@ class ReaperRestart(Command):
     def completion(self, state: ReplState):
         return super().completion(state)
 
-    def help(self, _: ReplState):
-        return f'{ReaperRestart.COMMAND}\t restart reaper'
+    def help(self, state: ReplState):
+        return super().help(state, 'restart reaper')

adam/commands/reaper/reaper_run_abort.py

@@ -36,5 +36,5 @@ class ReaperRunAbort(Command):
     def completion(self, state: ReplState):
         return super().completion(state)
 
-    def help(self, _: ReplState):
-        return f'{ReaperRunAbort.COMMAND} <run-id>\t abort reaper run'
+    def help(self, state: ReplState):
+        return super().help(state, 'abort reaper run', args='<run-id>')

adam/commands/reaper/reaper_runs.py

@@ -1,8 +1,8 @@
 from adam.commands.command import Command
-from adam.commands.reaper.utils_reaper import reaper
+from adam.commands.reaper.utils_reaper import reaper, Reapers
 from adam.config import Config
 from adam.repl_state import ReplState, RequiredState
-from adam.utils import convert_seconds, epoch, tabulize, log, log2
+from adam.utils import convert_seconds, epoch, log2
 
 class ReaperRuns(Command):
     COMMAND = 'reaper show runs'
@@ -52,10 +52,11 @@ class ReaperRuns(Command):
                     'limit': Config().get('reaper.show-runs-batch', 10)
                 })
 
-                runs = response.json()
-                if runs:
-                    tabulize(sorted([line(run) for run in runs], reverse=True), header=header, separator=",")
-                else:
+                if not Reapers.tabulize_runs(state, response):
+                # runs = response.json()
+                # if runs:
+                #     tabulize(sorted([line(run) for run in runs], reverse=True), header=header, separator=",")
+                # else:
                     log2('No running runs found.')
                     log2()
 
@@ -64,10 +65,11 @@ class ReaperRuns(Command):
                     'limit': Config().get('reaper.show-runs-batch', 10)
                 })
 
-                runs = response.json()
-                if runs:
-                    tabulize(sorted([line(run) for run in runs], reverse=True), header=header, separator=",")
-                else:
+                if not Reapers.tabulize_runs(state, response):
+                # runs = response.json()
+                # if runs:
+                #     tabulize(sorted([line(run) for run in runs], reverse=True), header=header, separator=",")
+                # else:
                     log2('No runs found.')
 
             return state
@@ -78,5 +80,5 @@ class ReaperRuns(Command):
 
         return {}
 
-    def help(self, _: ReplState):
-        return f'{ReaperRuns.COMMAND}\t show reaper runs'
+    def help(self, state: ReplState):
+        return super().help(state, 'show reaper runs')

adam/commands/reaper/reaper_runs_abort.py

@@ -59,5 +59,5 @@ class ReaperRunsAbort(Command):
     def completion(self, state: ReplState):
         return super().completion(state)
 
-    def help(self, _: ReplState):
-        return f'{ReaperRunsAbort.COMMAND}\t abort all running reaper runs'
+    def help(self, state: ReplState):
+        return super().help(state, 'abort all running reaper runs')

adam/commands/reaper/reaper_schedule_activate.py

@@ -41,5 +41,5 @@ class ReaperScheduleActivate(Command):
     def completion(self, state: ReplState):
         return super().completion(state, lambda: {id: None for id in Reapers.cached_schedule_ids(state)}, auto_key='reaper.schedules')
 
-    def help(self, _: ReplState):
-        return f'{ReaperScheduleActivate.COMMAND} <schedule-id>\t resume reaper schedule'
+    def help(self, state: ReplState):
+        return super().help(state, 'resume reaper schedule', args='<schedule-id>')

adam/commands/reaper/reaper_schedule_start.py

@@ -36,5 +36,5 @@ class ReaperScheduleStart(Command):
     def completion(self, state: ReplState):
         return super().completion(state, lambda: {id: None for id in Reapers.cached_schedule_ids(state)}, auto_key='reaper.schedules')
 
-    def help(self, _: ReplState):
-        return f'{ReaperScheduleStart.COMMAND} <schedule-id>\t start reaper runs for schedule'
+    def help(self, state: ReplState):
+        return super().help(state, 'start reaper runs for schedule', args='<schedule-id>')

adam/commands/reaper/reaper_schedule_stop.py

@@ -36,5 +36,5 @@ class ReaperScheduleStop(Command):
     def completion(self, state: ReplState):
         return super().completion(state, lambda: {id: None for id in Reapers.cached_schedule_ids(state)}, auto_key='reaper.schedules')
 
-    def help(self, _: ReplState):
-        return f'{ReaperScheduleStop.COMMAND} <schedule-id>\t pause reaper schedule'
+    def help(self, state: ReplState):
+        return super().help(state, 'pause reaper schedule', args='<schedule-id>')

adam/commands/reaper/reaper_schedules.py

@@ -32,5 +32,5 @@ class ReaperSchedules(Command):
     def completion(self, state: ReplState):
         return super().completion(state)
 
-    def help(self, _: ReplState):
-        return f'{ReaperSchedules.COMMAND}\t show reaper schedules'
+    def help(self, state: ReplState):
+        return super().help(state, 'show reaper schedules')

adam/commands/reaper/reaper_status.py

@@ -52,5 +52,5 @@ class ReaperStatus(Command):
     def completion(self, state: ReplState):
         return super().completion(state)
 
-    def help(self, _: ReplState):
-        return f'{ReaperStatus.COMMAND}\t show reaper status'
+    def help(self, state: ReplState):
+        return super().help(state, 'show reaper status')

adam/commands/reaper/utils_reaper.py

@@ -7,7 +7,7 @@ import re
 import requests
 from adam.config import Config
 from adam.repl_state import ReplState
-from adam.utils import tabulize, log2, wait_log
+from adam.utils import Color, convert_seconds, epoch, tabulize, log2, wait_log
 from adam.utils_k8s.k8s import port_forwarding
 
 class ReaperService:
@@ -41,7 +41,7 @@ class ReaperLogginHandler:
             self.svc.headers = Reapers.cookie_header(self.svc.state, self.svc.local_addr, self.svc.remote_addr, show_output=self.svc.show_out)
 
         if self.svc.show_out and self.method:
-            log2(f'{self.method} {self.svc.remote_addr}/{self.path}')
+            log2(f'{self.method} {self.svc.remote_addr}/{self.path}', text_color=Color.gray)
 
         return (f'http://{self.svc.local_addr}/{self.path}', self.svc.headers)
 
@@ -139,12 +139,12 @@ class Reapers:
             'username':user,
             'password':pw})
         if show_output:
-            log2(f'POST {remote_addr}/login')
-            log2(f'     username={user}&password={pw}')
+            log2(f'POST {remote_addr}/login', text_color=Color.gray)
+            log2(f'      username={user}&password={pw}', text_color=Color.gray)
 
         if int(response.status_code / 100) != 2:
             if show_output:
-                log2("login failed")
+                log2("login failed", text_color=Color.gray)
             return None
 
         return response.headers['Set-Cookie']
@@ -201,3 +201,29 @@ class Reapers:
             leaf = {id: None for id in ids()}
 
         return (leaf, auto == 'lazy')
+
+    def tabulize_runs(state: ReplState, response: dict) -> dict[str, any]:
+        header = 'ID,START,DURATION,STATE,CLUSTER,KEYSPACE,TABLES,REPAIRED'
+
+        def line(run):
+            id = run['id']
+            state = run['state']
+            start_time = run['start_time']
+            end_time = run['end_time']
+            duration = '-'
+            if state == 'DONE' and end_time:
+                hours, minutes, seconds = convert_seconds(epoch(end_time) - epoch(start_time))
+                if hours:
+                    duration = f"{hours:2d}h {minutes:2d}m {seconds:2d}s"
+                elif minutes:
+                    duration = f"{minutes:2d}m {seconds:2d}s"
+                else:
+                    duration = f"{seconds:2d}s"
+
+            return f"{id},{start_time},{duration},{state},{run['cluster_name']},{run['keyspace_name']},{len(run['column_families'])},{run['segments_repaired']}/{run['total_segments']}"
+
+        runs = response.json()
+        if runs:
+            tabulize(sorted([line(run) for run in runs], reverse=True), header=header, separator=",")
+
+        return runs

adam/commands/repair/repair_log.py

@@ -33,5 +33,5 @@ class RepairLog(Command):
     def completion(self, state: ReplState):
         return super().completion(state)
 
-    def help(self, _: ReplState):
-        return f'{RepairLog.COMMAND}\t get repair job logs'
+    def help(self, state: ReplState):
+        return super().help(state, 'get repair job logs')

adam/commands/repair/repair_run.py

@@ -61,5 +61,5 @@ class RepairRun(Command):
     def completion(self, state: ReplState):
         return super().completion(state)
 
-    def help(self, _: ReplState):
-        return f'{RepairRun.COMMAND} [replace]\t start a repair job, default not replacing'
+    def help(self, state: ReplState):
+        return super().help(state, 'start a repair job, default not replacing', args='[replace]')

adam/commands/repair/repair_scan.py

@@ -62,5 +62,5 @@ class RepairScan(Command):
     def completion(self, state: ReplState):
         return super().completion(state)
 
-    def help(self, _: ReplState):
-        return f'{RepairScan.COMMAND} [n]\t scan last n days repair log, default 7 days'
+    def help(self, state: ReplState):
+        return super().help(state, 'scan last n days repair log, default 7 days')