kaqing 2.0.188__py3-none-any.whl → 2.0.211__py3-none-any.whl

adam/utils_k8s/cassandra_nodes.py

@@ -1,17 +1,26 @@
 from adam.config import Config
 from adam.utils_k8s.pods import Pods
 from adam.utils_k8s.secrets import Secrets
-from adam.pod_exec_result import PodExecResult
+from adam.utils_k8s.pod_exec_result import PodExecResult
 from adam.repl_session import ReplSession
 
 # utility collection on cassandra nodes; methods are all static
 class CassandraNodes:
-    def exec(pod_name: str, namespace: str, command: str, show_out = True, throw_err = False, shell = '/bin/sh', backgrounded = False, log_file = None) -> PodExecResult:
+    def exec(pod_name: str,
+             namespace: str,
+             command: str,
+             show_out = True,
+             throw_err = False,
+             shell = '/bin/sh',
+             backgrounded = False,
+             log_file = None,
+             history = True) -> PodExecResult:
         r = Pods.exec(pod_name, "cassandra", namespace, command, show_out = show_out, throw_err = throw_err, shell = shell, backgrounded = backgrounded, log_file=log_file)
 
-        if r and Config().get('repl.history.push-cat-remote-log-file', True):
-            if r.log_file and ReplSession().prompt_session:
-                ReplSession().prompt_session.history.append_string(f'@{r.pod} cat {r.log_file}')
+        if history and r and r.log_file:
+            entry = f':cat {r.log_file}'
+
+            ReplSession().append_history(entry)
 
         return r
 

adam/utils_k8s/kube_context.py

@@ -102,7 +102,4 @@ class KubeContext:
         return name if re.match(r"^(?!pg-).*-k8spg-.*$", name) else None
 
     def show_out(s: bool):
-        return s or Config().is_debug()
-
-    def show_parallelism():
-        return Config().get('debugs.show-parallelism', False)
+        return s or Config().is_debug()

adam/{pod_exec_result.py → utils_k8s/pod_exec_result.py}

@@ -1,8 +1,8 @@
 import yaml
 
-from adam.utils import log_exc
+from adam.utils import ExecResult, log_exc
 
-class PodExecResult:
+class PodExecResult(ExecResult):
     # {
     #   'metadata': {},
     #   'status': 'Failure',
@@ -34,6 +34,12 @@ class PodExecResult:
 
         return code
 
+    def cat_log_file_cmd(self):
+        if self.pod and self.log_file:
+            return f'@{self.pod} cat {self.log_file}'
+
+        return None
+
     def __str__(self):
         return f'{"OK" if self.exit_code() == 0 else self.exit_code()} {self.command}'
 

adam/utils_k8s/pods.py

@@ -1,6 +1,8 @@
 from collections.abc import Callable
 from datetime import datetime
 import os
+import re
+import subprocess
 import sys
 import time
 from typing import TypeVar
@@ -9,10 +11,11 @@ from kubernetes.stream import stream
 from kubernetes.stream.ws_client import ERROR_CHANNEL, WSClient
 
 from adam.config import Config
+from adam.repl_session import ReplSession
 from adam.utils_k8s.volumes import ConfigMapMount
-from adam.pod_exec_result import PodExecResult
-from adam.utils import GeneratorStream, ParallelMapHandler, log2, debug, log_exc
-from adam.utils_local import local_tmp_dir
+from adam.utils_k8s.pod_exec_result import PodExecResult
+from adam.utils import GeneratorStream, ParallelMapHandler, log2, debug, log_dir, log_exc
+from adam.utils_local import local_downloads_dir
 from .kube_context import KubeContext
 
 from websocket._core import WebSocket
@@ -66,6 +69,33 @@ class Pods:
 
         show_out = KubeContext.show_out(show_out)
 
+        if backgrounded or command.endswith(' &'):
+            command = command.strip(' &')
+
+            log_all_file = None
+            log_pod_file = None
+            if log_file:
+                log_pod_file = Pods.log_file_from_template(log_file, pod_name=pod_name)
+                if (a := Pods.log_file_from_template(log_file, pod_name='all')) != log_file:
+                    log_all_file = a
+            else:
+                log_pod_file = Pods.log_file(command, pod_name=pod_name)
+
+            if env_prefix:
+                command = f'{env_prefix} {command}'
+
+            command = command.replace('"', '\\"')
+            cmd = f'nohup kubectl exec {pod_name} -c {container} -- {shell} -c "{command} &" > {log_pod_file} 2>&1 &'
+            if log_all_file:
+                cmd = f'{cmd} >> {log_all_file}'
+
+            if show_out:
+                log2(cmd)
+
+            result = subprocess.run(cmd, capture_output=True, text=True, shell=True)
+
+            return PodExecResult(result.stdout, result.stderr, cmd, None, pod=pod_name, log_file=log_pod_file)
+
         api = client.CoreV1Api()
 
         tty = True
@@ -73,22 +103,22 @@ class Pods:
         if env_prefix:
             exec_command = [shell, '-c', f'{env_prefix} {command}']
 
-        if backgrounded or command.endswith(' &'):
-            # should be false for starting a background process
-            tty = False
-
-            if Config().get('repl.background-process.auto-nohup', True):
-                command = command.strip(' &')
-                cmd_name = ''
-                if command.startswith('nodetool '):
-                    cmd_name = f".{'_'.join(command.split(' ')[5:])}"
-
-                if not log_file:
-                    log_file = f'{log_prefix()}-{datetime.now().strftime("%d%H%M%S")}{cmd_name}.log'
-                command = f"nohup {command} > {log_file} 2>&1 &"
-                if env_prefix:
-                    command = f'{env_prefix} {command}'
-                exec_command = [shell, '-c', command]
+        # if backgrounded or command.endswith(' &'):
+        #     # should be false for starting a background process
+        #     tty = False
+
+        #     if Config().get('repl.background-process.auto-nohup', True):
+        #         command = command.strip(' &')
+        #         cmd_name = ''
+        #         if command.startswith('nodetool '):
+        #             cmd_name = f".{'_'.join(command.split(' ')[5:])}"
+
+        #         if not log_file:
+        #             log_file = f'{log_prefix()}-{datetime.now().strftime("%d%H%M%S")}{cmd_name}.log'
+        #         command = f"nohup {command} > {log_file} 2>&1 &"
+        #         if env_prefix:
+        #             command = f'{env_prefix} {command}'
+        #         exec_command = [shell, '-c', command]
 
         k_command = f'kubectl exec {pod_name} -c {container} -n {namespace} -- {shell} -c "{command}"'
         debug(k_command)
@@ -141,6 +171,36 @@ class Pods:
 
         return PodExecResult("".join(stdout), "".join(stderr), k_command, error_output, pod=pod_name, log_file=log_file)
 
+    def log_file(command: str, pod_name: str = None, dt: datetime = None):
+        cmd_name = ''
+        if command.startswith('nodetool '):
+            command = command.strip(' &')
+            cmd_name = f".{'_'.join(command.split(' ')[5:])}"
+
+        pod_suffix = '{pod}'
+        if pod_name:
+            pod_suffix = pod_name
+            if groups := re.match(r'.*-(.*)', pod_name):
+                pod_suffix = f'-{groups[1]}'
+
+        return f'{log_dir()}/{Pods.job_id()}{cmd_name}{pod_suffix}.log'
+
+    def job_id(dt: datetime = None):
+        if not dt:
+            dt = datetime.now()
+
+        return dt.strftime("%d%H%M%S")
+
+    def log_file_from_template(log_file: str, pod_name: str):
+        pod_suffix = pod_name
+        if pod_name and (groups := re.match(r'.*-(.*)', pod_name)):
+            pod_suffix = f'-{groups[1]}'
+
+        if not pod_suffix.startswith('-'):
+            pod_suffix = f'-{pod_suffix}'
+
+        return log_file.replace('{pod}', pod_suffix)
+
     def read_file(pod_name: str, container: str, namespace: str, file_path: str):
         v1 = client.CoreV1Api()
 
@@ -175,13 +235,15 @@ class Pods:
 
     def download_file(pod_name: str, container: str, namespace: str, from_path: str, to_path: str = None):
         if not to_path:
-            to_path = f'{local_tmp_dir()}/{os.path.basename(from_path)}'
+            to_path = f'{local_downloads_dir()}/{os.path.basename(from_path)}'
 
         bytes = Pods.read_file(pod_name, container, namespace, from_path)
         with open(to_path, 'wb') as f:
             for item in GeneratorStream(bytes):
                 f.write(item)
 
+        ReplSession().append_history(f':cat {to_path}')
+
         return to_path
 
     def get_container(namespace: str, pod_name: str, container_name: str):
@@ -292,7 +354,4 @@ class Pods:
             log2(' Timed Out')
 
     def completed(namespace: str, pod_name: str):
-        return Pods.get(namespace, pod_name).status.phase in ['Succeeded', 'Failed']
-
-def log_prefix():
-    return Config().get('log-prefix', '/tmp/qing')
+        return Pods.get(namespace, pod_name).status.phase in ['Succeeded', 'Failed']

adam/utils_local.py

@@ -1,4 +1,80 @@
+import os
+import subprocess
+import traceback
+
 from adam.config import Config
+from adam.utils import ExecResult, creating_dir, debug
+
+def local_qing_dir():
+    return creating_dir(Config().get('local-qing-dir', '/tmp/qing-db/q'))
+
+def local_downloads_dir():
+    return creating_dir(Config().get('local-downloads-dir', '/tmp/qing-db/q/downloads'))
+
+class LocalExecResult(ExecResult):
+    def __init__(self, stdout: str, stderr: str, command: str = None, code = 0, log_file: str = None):
+        self.stdout: str = stdout
+        self.stderr: str = stderr
+        self.command: str = command
+        self.code = code
+        self.pod = 'local'
+        self.log_file = log_file
+
+    def exit_code(self) -> int:
+        return self.code
+
+    def cat_log_file_cmd(self):
+        if self.log_file:
+            return f':cat {self.log_file}'
+
+        return None
+
+    def __str__(self):
+        return f'{"OK" if self.exit_code() == 0 else self.exit_code()} {self.command}'
+
+    def __audit_extra__(self):
+        return self.log_file if self.log_file else None
+
+def local_exec(cmd: list[str], shell=False, show_out=False):
+    stdout = ''
+    stderr = ''
+    returncode = 0
+
+    try:
+        if show_out:
+            debug(' '.join(cmd))
+
+        r = subprocess.run(cmd, capture_output=True, text=True, shell=shell)
+        stdout = r.stdout
+        stderr = r.stderr
+        returncode = r.returncode
+    except FileNotFoundError as e:
+        pass
+
+    return LocalExecResult(stdout, stderr, ' '.join(cmd), returncode)
+
+def find_local_files(pattern: str = f'{local_qing_dir()}/*', file_type: str = None, max_depth = 0, mmin: int = 0):
+    # find . -maxdepth 1 -type f -name '*'
+    log_files = []
+    try:
+        dir = os.path.dirname(pattern)
+        base = os.path.basename(pattern)
+        cmd = ['find', dir]
+        if file_type:
+            cmd += ['-type', file_type]
+        if max_depth:
+            cmd += ['-maxdepth', str(max_depth)]
+        if mmin:
+                cmd += ['-mmin',  f'-{mmin}']
+        cmd += ['-name', base]
+
+        stdout = local_exec(cmd, show_out=Config().is_debug()).stdout
+
+        for line in stdout.split('\n'):
+            line = line.strip(' \r')
+            if line:
+                log_files.append(line)
+    except:
+        traceback.print_exc()
 
-def local_tmp_dir():
-    return Config().get('local-tmp-dir', '/tmp/qing-db')
+    return log_files

adam/utils_repl/repl_completer.py

@@ -1,14 +1,11 @@
-from concurrent.futures import ThreadPoolExecutor
 import copy
-import inspect
 import re
-import threading
 import traceback
 from typing import Iterable, TypeVar, cast
-from prompt_toolkit.completion import CompleteEvent, Completer, Completion, NestedCompleter, WordCompleter
+from prompt_toolkit.completion import CompleteEvent, Completion, NestedCompleter, WordCompleter
 from prompt_toolkit.document import Document
 
-from adam.utils import debug, debug_complete, log2, log_timing
+from adam.utils import debug_complete, log2
 from adam.utils_repl.appendable_completer import AppendableCompleter
 
 import nest_asyncio
@@ -37,6 +34,8 @@ def merge_completions(dict1, dict2):
                         target[key] = merge_completions(target[key], value)
                     elif isinstance(target[key], AppendableCompleter):
                         cast(AppendableCompleter, target[key]).append_completions(key, value)
+                    elif isinstance(target[key], NestedCompleter):
+                        cast(NestedCompleter, target[key]).options = merge_completions(cast(NestedCompleter, target[key]).options, value)
                 elif isinstance(value, AppendableCompleter):
                     if isinstance(target[key], dict):
                         cast(AppendableCompleter, value).append_completions(key, target[key])
@@ -77,8 +76,12 @@ class ReplCompleter(NestedCompleter):
                     cursor_position=document.cursor_position - move_cursor,
                 )
 
-                for c in completer.get_completions(new_document, complete_event):
-                    yield c
+                try:
+                    # potential thread racing
+                    for c in completer.get_completions(new_document, complete_event):
+                        yield c
+                except:
+                    pass
 
         # No space in the input: behave exactly like `WordCompleter`.
         else:
@@ -88,85 +91,3 @@ class ReplCompleter(NestedCompleter):
             )
             for c in completer.get_completions(document, complete_event):
                 yield c
-
-lock = threading.Lock()
-in_queue = set()
-
-def preload(action: callable, log_key: str = None):
-    with lock:
-        if not LazyNestedCompleter.loop:
-            LazyNestedCompleter.loop = asyncio.new_event_loop()
-            LazyNestedCompleter.async_exec = ThreadPoolExecutor(max_workers=6, thread_name_prefix='async')
-            LazyNestedCompleter.loop.set_default_executor(LazyNestedCompleter.async_exec)
-
-        # some lib does not handle asyncio loop properly, as sync exec submit does not work, use another async loop
-        async def a():
-            try:
-                arg_needed = len(action.__code__.co_varnames)
-
-                if log_key:
-                    with log_timing(log_key):
-                        r = action(None) if arg_needed else action()
-                else:
-                    r = action(None) if arg_needed else action()
-                if inspect.isawaitable(r):
-                    await r
-
-                in_queue.remove(log_key)
-            except Exception as e:
-                log2('preloading error', e, inspect.getsourcelines(action)[0][0])
-                traceback.print_exc()
-
-        if log_key not in in_queue:
-            in_queue.add(log_key)
-            LazyNestedCompleter.async_exec.submit(lambda: LazyNestedCompleter.loop.run_until_complete(a()))
-
-class LazyNestedCompleter(NestedCompleter):
-    loop: asyncio.AbstractEventLoop = None
-    async_exec: ThreadPoolExecutor = None
-
-    def __init__(self, name: str, options_lambda: callable, ignore_case: bool = True, auto: str = 'lazy') -> None:
-        super().__init__(None, ignore_case)
-        self.options_lambda = options_lambda
-        if auto == 'lazy':
-            preload(options_lambda, log_key=name)
-
-    def __repr__(self) -> str:
-        return "LazyNestedCompleter(%r, ignore_case=%r)" % (self.options, self.ignore_case)
-
-    def get_completions(
-        self, document: Document, complete_event: CompleteEvent
-    ) -> Iterable[Completion]:
-        if not self.options:
-            self.options = self.options_lambda()
-
-        # Split document.
-        text = document.text_before_cursor.lstrip()
-        stripped_len = len(document.text_before_cursor) - len(text)
-
-        # If there is a space, check for the first term, and use a
-        # subcompleter.
-        if " " in text:
-            first_term = text.split()[0]
-            completer = self.options.get(first_term)
-
-            # If we have a sub completer, use this for the completions.
-            if completer is not None:
-                remaining_text = text[len(first_term) :].lstrip()
-                move_cursor = len(text) - len(remaining_text) + stripped_len
-
-                new_document = Document(
-                    remaining_text,
-                    cursor_position=document.cursor_position - move_cursor,
-                )
-
-                for c in completer.get_completions(new_document, complete_event):
-                    yield c
-
-        # No space in the input: behave exactly like `WordCompleter`.
-        else:
-            completer = WordCompleter(
-                list(self.options.keys()), ignore_case=self.ignore_case
-            )
-            for c in completer.get_completions(document, complete_event):
-                yield c

adam/utils_sqlite.py

@@ -7,7 +7,7 @@ import sqlite3
 import pandas
 
 from adam.config import Config
-from adam.utils import tabulize, log, wait_log
+from adam.utils import creating_dir, tabulize, log, wait_log
 
 class CursorHandler:
     def __init__(self, conn: sqlite3.Connection):
@@ -53,7 +53,7 @@ class SQLite:
         return CursorHandler(conn)
 
     def local_db_dir():
-        return Config().get('export.sqlite.local-db-dir', '/tmp/qing-db')
+        return creating_dir(Config().get('export.sqlite.local-db-dir', '/tmp/qing-db/q/export/db'))
 
     def keyspace(database: str):
         return '_'.join(database.replace(".db", "").split('_')[1:])
@@ -95,8 +95,6 @@ class SQLite:
                conn.close()
 
     def connect(database: str, keyspace: str = None):
-        os.makedirs(SQLite.local_db_dir(), exist_ok=True)
-
         if keyspace:
             return sqlite3.connect(f'{SQLite.local_db_dir()}/{database}_{keyspace}.db')
         else:
@@ -131,7 +129,4 @@ class SQLite:
         return len(lines), log_file
 
     def query(conn, sql: str) -> tuple[str, str, list]:
-        return pandas.read_sql_query(sql, conn)
-
-def log_prefix():
-    return Config().get('export.log-prefix', '/tmp/qing')
+        return pandas.read_sql_query(sql, conn)

adam/version.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-__version__ = "2.0.188"  #: the working version
+__version__ = "2.0.211"  #: the working version
 __release__ = "1.0.0"  #: the release version

{kaqing-2.0.188.dist-info → kaqing-2.0.211.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: kaqing
-Version: 2.0.188
+Version: 2.0.211
 Summary: UNKNOWN
 Home-page: UNKNOWN
 License: UNKNOWN