kaqing 2.0.186__py3-none-any.whl → 2.0.200__py3-none-any.whl

adam/utils.py

@@ -19,7 +19,6 @@ import sys
 import time
 import click
 import yaml
-
 from prompt_toolkit.completion import Completer
 
 from . import __version__
@@ -28,11 +27,28 @@ T = TypeVar('T')
 
 log_state = threading.local()
 
-class LogConfig:
-    is_debug = lambda: False
-    is_debug_timing = lambda: False
-    is_debug_complete = lambda: False
-    is_display_help = True
+class ConfigReadable:
+    def is_debug() -> bool:
+        pass
+
+    def get(self, key: str, default: T) -> T:
+        pass
+
+class ConfigHolder:
+    # the singleton pattern
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'): cls.instance = super(ConfigHolder, cls).__new__(cls)
+
+        return cls.instance
+
+    def __init__(self):
+        if not hasattr(self, 'config'):
+            # set by Config
+            self.config: 'ConfigReadable' = None
+            # only for testing
+            self.is_display_help = True
+            # set by ReplSession
+            self.append_command_history = lambda entry: None
 
 NO_SORT = 0
 SORT = 1
@@ -164,7 +180,8 @@ def deep_merge_dicts(dict1, dict2):
         elif key not in merged_dict or value:
             # Otherwise, overwrite or add the value from dict2
             if key in merged_dict and isinstance(merged_dict[key], Completer):
-                print('SEAN completer found, ignoring', key, value)
+                pass
+                # print('SEAN completer found, ignoring', key, value)
             else:
                 merged_dict[key] = value
     return merged_dict
@@ -204,7 +221,7 @@ def get_deep_keys(d, current_path=""):
     return keys
 
 def display_help(replace_arg = False):
-    if not LogConfig.is_display_help:
+    if not ConfigHolder().is_display_help:
         return
 
     args = copy.copy(sys.argv)
@@ -293,15 +310,15 @@ def is_lambda(func):
     return callable(func) and hasattr(func, '__name__') and func.__name__ == '<lambda>'
 
 def debug(s = None):
-    if LogConfig.is_debug():
+    if ConfigHolder().config.is_debug():
         log2(f'DEBUG {s}')
 
 def debug_complete(s = None):
-    if LogConfig.is_debug_complete():
-        log2(f'DEBUG {s}')
+    CommandLog.log(f'DEBUG {s}', config=ConfigHolder().config.get('debugs.complete', 'off'))
 
 def debug_trace():
-    if LogConfig.is_debug():
+    if ConfigHolder().config.is_debug():
+    # if LogConfig.is_debug():
         log2(traceback.format_exc())
 
 def in_docker() -> bool:
@@ -329,7 +346,7 @@ class Ing:
 
         try:
             if not log_state.ing_cnt:
-                if not self.suppress_log and not LogConfig.is_debug():
+                if not self.suppress_log and not ConfigHolder().config.is_debug():
                     log2(f'{self.msg}...', nl=False)
 
             return None
@@ -339,7 +356,7 @@ class Ing:
     def __exit__(self, exc_type, exc_val, exc_tb):
         log_state.ing_cnt -= 1
         if not log_state.ing_cnt:
-            if not self.suppress_log and not LogConfig.is_debug():
+            if not self.suppress_log and not ConfigHolder().config.is_debug():
                 log2(' OK')
 
         return False
@@ -360,7 +377,7 @@ def ing(msg: str, body: Callable[[], None]=None, suppress_log=False):
     return r
 
 def loggable():
-    return LogConfig.is_debug() or not hasattr(log_state, 'ing_cnt') or not log_state.ing_cnt
+    return ConfigHolder().config and ConfigHolder().config.is_debug() or not hasattr(log_state, 'ing_cnt') or not log_state.ing_cnt
 
 class TimingNode:
     def __init__(self, depth: int, s0: time.time = time.time(), line: str = None):
@@ -388,7 +405,7 @@ class LogTiming:
         self.s0 = s0
 
     def __enter__(self):
-        if not LogConfig.is_debug_timing():
+        if (config := ConfigHolder().config.get('debugs.timings', 'off')) not in ['on', 'file']:
             return
 
         if not hasattr(log_state, 'timings'):
@@ -400,7 +417,7 @@ class LogTiming:
             self.s0 = time.time()
 
     def __exit__(self, exc_type, exc_val, exc_tb):
-        if not LogConfig.is_debug_timing():
+        if (config := ConfigHolder().config.get('debugs.timings', 'off')) not in ['on', 'file']:
             return False
 
         child = log_state.timings
@@ -411,7 +428,9 @@ class LogTiming:
         log_state.timings = self.me
 
         if not self.me.depth:
-            log2(self.me.tree())
+            # log timings finally
+            CommandLog.log(self.me.tree(), config)
+
             log_state.timings = TimingNode(0)
 
         return False
@@ -420,7 +439,7 @@ def log_timing(msg: str, body: Callable[[], None]=None, s0: time.time = None):
     if not s0 and not body:
         return LogTiming(msg, s0=s0)
 
-    if not LogConfig.is_debug_timing():
+    if not ConfigHolder().config.get('debugs.timings', False):
         if body:
             return body()
 
@@ -439,7 +458,6 @@ def log_timing(msg: str, body: Callable[[], None]=None, s0: time.time = None):
     return r
 
 def timing_log_line(depth: int, msg: str, s0: time.time):
-    # print('SEAN log timing', msg, threading.current_thread().name)
     elapsed = time.time() - s0
     offloaded = '-' if threading.current_thread().name.startswith('offload') or threading.current_thread().name.startswith('async') else '+'
     prefix = f'[{offloaded} timings] '
@@ -539,7 +557,7 @@ class LogTrace:
             elif self.err_msg is not False and self.err_msg:
                 log2(self.err_msg)
 
-            if self.err_msg is not False and LogConfig.is_debug():
+            if self.err_msg is not False and ConfigHolder().config.is_debug():
                 traceback.print_exception(exc_type, exc_val, exc_tb, file=sys.stderr)
 
         # swallow exception
@@ -588,7 +606,7 @@ class ParallelMapHandler:
         self.samples = samples
         self.msg = msg
         if msg and msg.startswith('d`'):
-            if LogConfig.is_debug():
+            if ConfigHolder().config.is_debug():
                 self.msg = msg.replace('d`', '', 1)
             else:
                 self.msg = None
@@ -756,4 +774,58 @@ class OffloadHandler(ParallelMapHandler):
             log2(f'{" ".join(self.begin)} with {self.workers} workers...')
 
 def offload(max_workers: int = 3, msg: str = None):
-    return OffloadHandler(max_workers, msg = msg)
+    return OffloadHandler(max_workers, msg = msg)
+
+def kaqing_log_file_name(suffix = 'log'):
+    return f"{ConfigHolder().config.get('log-prefix', '/tmp/qing')}-{datetime.now().strftime('%d%H%M%S')}.{suffix}"
+
+class LogFileHandler:
+    def __init__(self, suffix = 'log'):
+        self.suffix = suffix
+
+    def __enter__(self):
+        self.f = open(kaqing_log_file_name(), 'w')
+        self.f.__enter__()
+
+        return self.f
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.f.__exit__(exc_type, exc_val, exc_tb)
+
+        if ConfigHolder().append_command_history:
+            ConfigHolder().append_command_history(f':sh cat {self.f.name}')
+
+        return False
+
+def kaqing_log_file(suffix = 'log'):
+    return LogFileHandler(suffix = suffix)
+
+class CommandLog:
+    log_file = None
+
+    def log(line: str, config: str = 'off'):
+        if config == 'file':
+            if not CommandLog.log_file:
+                try:
+                    CommandLog.log_file = open(kaqing_log_file_name(suffix='cmd.log'), 'w')
+                except:
+                    pass
+
+            try:
+                CommandLog.log_file.write(line + '\n')
+            except:
+                pass
+        elif config == 'on':
+            log2(line)
+
+    def close_log_file():
+        if CommandLog.log_file:
+            try:
+                CommandLog.log_file.close()
+            except:
+                pass
+
+            if ConfigHolder().append_command_history:
+                ConfigHolder().append_command_history(f':sh cat {CommandLog.log_file.name}')
+
+            CommandLog.log_file = None

adam/utils_k8s/app_clusters.py

@@ -23,6 +23,6 @@ class AppClusters:
                     if result.stdout:
                         log(result.stdout)
                     if result.stderr:
-                        log2(result.stderr, file=sys.stderr)
+                        log2(result.stderr)
 
             return results

adam/utils_k8s/app_pods.py

@@ -29,8 +29,7 @@ class AppPods:
         container = Config().get('app.container-name', 'c3-server')
         r = Pods.exec(pod_name, container, namespace, command, show_out = show_out, throw_err = throw_err, shell = shell, backgrounded = backgrounded)
 
-        if r and Config().get('repl.history.push-cat-remote-log-file', True):
-            if r.log_file and ReplSession().prompt_session:
-                ReplSession().prompt_session.history.append_string(f'@{r.pod} cat {r.log_file}')
+        if r and r.log_file:
+            ReplSession().append_history(f'@{r.pod} cat {r.log_file}')
 
         return r

adam/utils_k8s/cassandra_clusters.py

@@ -21,20 +21,24 @@ class CassandraClusters:
              on_any = False,
              shell = '/bin/sh',
              backgrounded = False,
-             log_file = None) -> list[PodExecResult]:
+             log_file = None,
+             via_sh = True) -> list[PodExecResult]:
 
         pods = StatefulSets.pod_names(sts, namespace)
         samples = 1 if on_any else sys.maxsize
+        if via_sh and (backgrounded or command.endswith(' &')) and Config().get('repl.background-process.via-sh', True) and not log_file:
+            log_file = Pods.log_file(command)
+
         msg = 'd`Running|Ran ' + action + ' command onto {size} pods'
         with Pods.parallelize(pods, max_workers, samples, msg, action=action) as exec:
-            results: list[PodExecResult] = exec.map(lambda pod: CassandraNodes.exec(pod, namespace, command, False, False, shell, backgrounded, log_file))
+            results: list[PodExecResult] = exec.map(lambda pod: CassandraNodes.exec(pod, namespace, command, False, False, shell, backgrounded, log_file, via_sh=via_sh))
             for result in results:
                 if show_out and not Config().is_debug():
                     log(result.command)
                     if result.stdout:
                         log(result.stdout)
                     if result.stderr:
-                        log2(result.stderr, file=sys.stderr)
+                        log2(result.stderr)
 
             return results
 

adam/utils_k8s/cassandra_nodes.py

@@ -6,12 +6,15 @@ from adam.repl_session import ReplSession
 
 # utility collection on cassandra nodes; methods are all static
 class CassandraNodes:
-    def exec(pod_name: str, namespace: str, command: str, show_out = True, throw_err = False, shell = '/bin/sh', backgrounded = False, log_file = None) -> PodExecResult:
-        r = Pods.exec(pod_name, "cassandra", namespace, command, show_out = show_out, throw_err = throw_err, shell = shell, backgrounded = backgrounded, log_file=log_file)
+    def exec(pod_name: str, namespace: str, command: str, show_out = True, throw_err = False, shell = '/bin/sh', backgrounded = False, log_file = None, no_history = False, via_sh = True) -> PodExecResult:
+        r = Pods.exec(pod_name, "cassandra", namespace, command, show_out = show_out, throw_err = throw_err, shell = shell, backgrounded = backgrounded, log_file=log_file, via_sh=via_sh)
 
-        if r and Config().get('repl.history.push-cat-remote-log-file', True):
-            if r.log_file and ReplSession().prompt_session:
-                ReplSession().prompt_session.history.append_string(f'@{r.pod} cat {r.log_file}')
+        if not no_history and r and r.log_file:
+            entry = f'@{r.pod} cat {r.log_file}'
+            if Config().get('repl.background-process.via-sh', True):
+                entry = f':sh cat {r.log_file}'
+
+            ReplSession().append_history(entry)
 
         return r
 

adam/utils_k8s/kube_context.py

@@ -102,7 +102,4 @@ class KubeContext:
         return name if re.match(r"^(?!pg-).*-k8spg-.*$", name) else None
 
     def show_out(s: bool):
-        return s or Config().is_debug()
-
-    def show_parallelism():
-        return Config().get('debugs.show-parallelism', False)
+        return s or Config().is_debug()

adam/utils_k8s/pods.py

@@ -1,6 +1,8 @@
 from collections.abc import Callable
 from datetime import datetime
 import os
+import re
+import subprocess
 import sys
 import time
 from typing import TypeVar
@@ -9,6 +11,7 @@ from kubernetes.stream import stream
 from kubernetes.stream.ws_client import ERROR_CHANNEL, WSClient
 
 from adam.config import Config
+from adam.repl_session import ReplSession
 from adam.utils_k8s.volumes import ConfigMapMount
 from adam.pod_exec_result import PodExecResult
 from adam.utils import GeneratorStream, ParallelMapHandler, log2, debug, log_exc
@@ -60,12 +63,34 @@ class Pods:
              backgrounded = False,
              log_file = None,
              interaction: Callable[[any, list[str]], any] = None,
-             env_prefix: str = None):
+             env_prefix: str = None,
+             via_sh = True):
         if _TEST_POD_EXEC_OUTS:
             return _TEST_POD_EXEC_OUTS
 
         show_out = KubeContext.show_out(show_out)
 
+        if via_sh and (backgrounded or command.endswith(' &')) and Config().get('repl.background-process.via-sh', True):
+            command = command.strip(' &')
+
+            log_all_file = None
+            log_pod_file = None
+            if log_file:
+                log_pod_file = Pods.log_file_from_template(log_file, pod_name=pod_name)
+                if (a := Pods.log_file_from_template(log_file, pod_name='all')) != log_file:
+                    log_all_file = a
+            else:
+                log_pod_file = Pods.log_file(command, pod_name=pod_name)
+
+            command = command.replace('"', '\\"')
+            cmd = f'nohup kubectl exec {pod_name} -c {container} -- {shell} -c "{command} &" > {log_pod_file} 2>&1 &'
+            if log_all_file:
+                cmd = f'{cmd} >> {log_all_file}'
+
+            result = subprocess.run(cmd, capture_output=True, text=True, shell=True)
+
+            return PodExecResult(result.stdout, result.stderr, cmd, None, pod=pod_name, log_file=log_pod_file)
+
         api = client.CoreV1Api()
 
         tty = True
@@ -141,6 +166,33 @@ class Pods:
 
         return PodExecResult("".join(stdout), "".join(stderr), k_command, error_output, pod=pod_name, log_file=log_file)
 
+    def log_file(command: str, pod_name: str = None, dt: datetime = None):
+        cmd_name = ''
+        if command.startswith('nodetool '):
+            command = command.strip(' &')
+            cmd_name = f".{'_'.join(command.split(' ')[5:])}"
+
+        pod_suffix = '{pod}'
+        if pod_name:
+            pod_suffix = pod_name
+            if groups := re.match(r'.*-(.*)', pod_name):
+                pod_suffix = f'-{groups[1]}'
+
+        if not dt:
+            dt = datetime.now()
+
+        return f'{log_prefix()}-{dt.strftime("%d%H%M%S")}{cmd_name}{pod_suffix}.log'
+
+    def log_file_from_template(log_file: str, pod_name: str):
+        pod_suffix = pod_name
+        if pod_name and (groups := re.match(r'.*-(.*)', pod_name)):
+            pod_suffix = f'-{groups[1]}'
+
+        if not pod_suffix.startswith('-'):
+            pod_suffix = f'-{pod_suffix}'
+
+        return log_file.replace('{pod}', pod_suffix)
+
     def read_file(pod_name: str, container: str, namespace: str, file_path: str):
         v1 = client.CoreV1Api()
 
@@ -182,6 +234,8 @@ class Pods:
             for item in GeneratorStream(bytes):
                 f.write(item)
 
+        ReplSession().append_history(f':sh cat {to_path}')
+
         return to_path
 
     def get_container(namespace: str, pod_name: str, container_name: str):

adam/utils_repl/repl_completer.py

@@ -1,14 +1,11 @@
-from concurrent.futures import ThreadPoolExecutor
 import copy
-import inspect
 import re
-import threading
 import traceback
 from typing import Iterable, TypeVar, cast
-from prompt_toolkit.completion import CompleteEvent, Completer, Completion, NestedCompleter, WordCompleter
+from prompt_toolkit.completion import CompleteEvent, Completion, NestedCompleter, WordCompleter
 from prompt_toolkit.document import Document
 
-from adam.utils import debug, debug_complete, log2, log_timing
+from adam.utils import debug_complete, log2
 from adam.utils_repl.appendable_completer import AppendableCompleter
 
 import nest_asyncio
@@ -37,6 +34,8 @@ def merge_completions(dict1, dict2):
                         target[key] = merge_completions(target[key], value)
                     elif isinstance(target[key], AppendableCompleter):
                         cast(AppendableCompleter, target[key]).append_completions(key, value)
+                    elif isinstance(target[key], NestedCompleter):
+                        cast(NestedCompleter, target[key]).options = merge_completions(cast(NestedCompleter, target[key]).options, value)
                 elif isinstance(value, AppendableCompleter):
                     if isinstance(target[key], dict):
                         cast(AppendableCompleter, value).append_completions(key, target[key])
@@ -88,85 +87,3 @@ class ReplCompleter(NestedCompleter):
             )
             for c in completer.get_completions(document, complete_event):
                 yield c
-
-lock = threading.Lock()
-in_queue = set()
-
-def preload(action: callable, log_key: str = None):
-    with lock:
-        if not LazyNestedCompleter.loop:
-            LazyNestedCompleter.loop = asyncio.new_event_loop()
-            LazyNestedCompleter.async_exec = ThreadPoolExecutor(max_workers=6, thread_name_prefix='async')
-            LazyNestedCompleter.loop.set_default_executor(LazyNestedCompleter.async_exec)
-
-        # some lib does not handle asyncio loop properly, as sync exec submit does not work, use another async loop
-        async def a():
-            try:
-                arg_needed = len(action.__code__.co_varnames)
-
-                if log_key:
-                    with log_timing(log_key):
-                        r = action(None) if arg_needed else action()
-                else:
-                    r = action(None) if arg_needed else action()
-                if inspect.isawaitable(r):
-                    await r
-
-                in_queue.remove(log_key)
-            except Exception as e:
-                log2('preloading error', e, inspect.getsourcelines(action)[0][0])
-                traceback.print_exc()
-
-        if log_key not in in_queue:
-            in_queue.add(log_key)
-            LazyNestedCompleter.async_exec.submit(lambda: LazyNestedCompleter.loop.run_until_complete(a()))
-
-class LazyNestedCompleter(NestedCompleter):
-    loop: asyncio.AbstractEventLoop = None
-    async_exec: ThreadPoolExecutor = None
-
-    def __init__(self, name: str, options_lambda: callable, ignore_case: bool = True, auto: str = 'lazy') -> None:
-        super().__init__(None, ignore_case)
-        self.options_lambda = options_lambda
-        if auto == 'lazy':
-            preload(options_lambda, log_key=name)
-
-    def __repr__(self) -> str:
-        return "LazyNestedCompleter(%r, ignore_case=%r)" % (self.options, self.ignore_case)
-
-    def get_completions(
-        self, document: Document, complete_event: CompleteEvent
-    ) -> Iterable[Completion]:
-        if not self.options:
-            self.options = self.options_lambda()
-
-        # Split document.
-        text = document.text_before_cursor.lstrip()
-        stripped_len = len(document.text_before_cursor) - len(text)
-
-        # If there is a space, check for the first term, and use a
-        # subcompleter.
-        if " " in text:
-            first_term = text.split()[0]
-            completer = self.options.get(first_term)
-
-            # If we have a sub completer, use this for the completions.
-            if completer is not None:
-                remaining_text = text[len(first_term) :].lstrip()
-                move_cursor = len(text) - len(remaining_text) + stripped_len
-
-                new_document = Document(
-                    remaining_text,
-                    cursor_position=document.cursor_position - move_cursor,
-                )
-
-                for c in completer.get_completions(new_document, complete_event):
-                    yield c
-
-        # No space in the input: behave exactly like `WordCompleter`.
-        else:
-            completer = WordCompleter(
-                list(self.options.keys()), ignore_case=self.ignore_case
-            )
-            for c in completer.get_completions(document, complete_event):
-                yield c

adam/version.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-__version__ = "2.0.186"  #: the working version
+__version__ = "2.0.200"  #: the working version
 __release__ = "1.0.0"  #: the release version

{kaqing-2.0.186.dist-info → kaqing-2.0.200.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: kaqing
-Version: 2.0.186
+Version: 2.0.200
 Summary: UNKNOWN
 Home-page: UNKNOWN
 License: UNKNOWN