kaqing 2.0.184__py3-none-any.whl → 2.0.214__py3-none-any.whl

adam/utils_k8s/cassandra_nodes.py

@@ -1,17 +1,26 @@
 from adam.config import Config
 from adam.utils_k8s.pods import Pods
 from adam.utils_k8s.secrets import Secrets
-from adam.pod_exec_result import PodExecResult
+from adam.utils_k8s.pod_exec_result import PodExecResult
 from adam.repl_session import ReplSession
 
 # utility collection on cassandra nodes; methods are all static
 class CassandraNodes:
-    def exec(pod_name: str, namespace: str, command: str, show_out = True, throw_err = False, shell = '/bin/sh', backgrounded = False, log_file = None) -> PodExecResult:
+    def exec(pod_name: str,
+             namespace: str,
+             command: str,
+             show_out = True,
+             throw_err = False,
+             shell = '/bin/sh',
+             backgrounded = False,
+             log_file = None,
+             history = True) -> PodExecResult:
         r = Pods.exec(pod_name, "cassandra", namespace, command, show_out = show_out, throw_err = throw_err, shell = shell, backgrounded = backgrounded, log_file=log_file)
 
-        if r and Config().get('repl.history.push-cat-remote-log-file', True):
-            if r.log_file and ReplSession().prompt_session:
-                ReplSession().prompt_session.history.append_string(f'@{r.pod} cat {r.log_file}')
+        if history and r and r.log_file:
+            entry = f':cat {r.log_file}'
+
+            ReplSession().append_history(entry)
 
         return r
 

adam/utils_k8s/k8s.py

@@ -1,4 +1,5 @@
 from collections.abc import Callable
+import inspect
 import re
 import portforward
 
@@ -49,6 +50,14 @@ class PortForwardHandler:
                 raise InvalidStateException(state)
 
             self.pod = pod
+
+            # pf = portforward.forward(state.namespace, pod, self.local_port + 1, self.target_port, log_level=portforward.LogLevel.DEBUG)
+            # print(inspect.getsource(pf.__enter__))
+            # print('test portforward START', state.namespace, pod, self.local_port + 1, self.target_port, pf.__enter__)
+            # with pf:
+            #     print('test portforward BODY')
+            # print('test portforward OK')
+
             self.forward_connection = portforward.forward(state.namespace, pod, self.local_port, self.target_port)
             if self.inc_connection_cnt() == 1:
                 self.forward_connection.__enter__()

adam/utils_k8s/kube_context.py

@@ -102,7 +102,4 @@ class KubeContext:
         return name if re.match(r"^(?!pg-).*-k8spg-.*$", name) else None
 
     def show_out(s: bool):
-        return s or Config().is_debug()
-
-    def show_parallelism():
-        return Config().get('debugs.show-parallelism', False)
+        return s or Config().is_debug()

adam/{pod_exec_result.py → utils_k8s/pod_exec_result.py}

@@ -1,8 +1,8 @@
 import yaml
 
-from adam.utils import log_exc
+from adam.utils import ExecResult, log_exc
 
-class PodExecResult:
+class PodExecResult(ExecResult):
     # {
     #   'metadata': {},
     #   'status': 'Failure',
@@ -34,6 +34,12 @@ class PodExecResult:
 
         return code
 
+    def cat_log_file_cmd(self):
+        if self.pod and self.log_file:
+            return f'@{self.pod} cat {self.log_file}'
+
+        return None
+
     def __str__(self):
         return f'{"OK" if self.exit_code() == 0 else self.exit_code()} {self.command}'
 

adam/utils_k8s/pods.py

@@ -1,6 +1,8 @@
 from collections.abc import Callable
 from datetime import datetime
 import os
+import re
+import subprocess
 import sys
 import time
 from typing import TypeVar
@@ -9,10 +11,11 @@ from kubernetes.stream import stream
 from kubernetes.stream.ws_client import ERROR_CHANNEL, WSClient
 
 from adam.config import Config
+from adam.repl_session import ReplSession
 from adam.utils_k8s.volumes import ConfigMapMount
-from adam.pod_exec_result import PodExecResult
-from adam.utils import GeneratorStream, ParallelMapHandler, log2, debug, log_exc
-from adam.utils_local import local_tmp_dir
+from adam.utils_k8s.pod_exec_result import PodExecResult
+from adam.utils import GeneratorStream, ParallelMapHandler, log2, debug, log_dir, log_exc
+from adam.utils_local import local_downloads_dir
 from .kube_context import KubeContext
 
 from websocket._core import WebSocket
@@ -66,6 +69,33 @@ class Pods:
 
         show_out = KubeContext.show_out(show_out)
 
+        if backgrounded or command.endswith(' &'):
+            command = command.strip(' &')
+
+            log_all_file = None
+            log_pod_file = None
+            if log_file:
+                log_pod_file = Pods.log_file_from_template(log_file, pod_name=pod_name)
+                if (a := Pods.log_file_from_template(log_file, pod_name='all')) != log_file:
+                    log_all_file = a
+            else:
+                log_pod_file = Pods.log_file(command, pod_name=pod_name)
+
+            if env_prefix:
+                command = f'{env_prefix} {command}'
+
+            command = command.replace('"', '\\"')
+            cmd = f'nohup kubectl exec {pod_name} -c {container} -- {shell} -c "{command} &" > {log_pod_file} 2>&1 &'
+            if log_all_file:
+                cmd = f'{cmd} >> {log_all_file}'
+
+            if show_out:
+                log2(cmd)
+
+            result = subprocess.run(cmd, capture_output=True, text=True, shell=True)
+
+            return PodExecResult(result.stdout, result.stderr, cmd, None, pod=pod_name, log_file=log_pod_file)
+
         api = client.CoreV1Api()
 
         tty = True
@@ -73,22 +103,22 @@ class Pods:
         if env_prefix:
             exec_command = [shell, '-c', f'{env_prefix} {command}']
 
-        if backgrounded or command.endswith(' &'):
-            # should be false for starting a background process
-            tty = False
-
-            if Config().get('repl.background-process.auto-nohup', True):
-                command = command.strip(' &')
-                cmd_name = ''
-                if command.startswith('nodetool '):
-                    cmd_name = f".{'_'.join(command.split(' ')[5:])}"
-
-                if not log_file:
-                    log_file = f'{log_prefix()}-{datetime.now().strftime("%d%H%M%S")}{cmd_name}.log'
-                command = f"nohup {command} > {log_file} 2>&1 &"
-                if env_prefix:
-                    command = f'{env_prefix} {command}'
-                exec_command = [shell, '-c', command]
+        # if backgrounded or command.endswith(' &'):
+        #     # should be false for starting a background process
+        #     tty = False
+
+        #     if Config().get('repl.background-process.auto-nohup', True):
+        #         command = command.strip(' &')
+        #         cmd_name = ''
+        #         if command.startswith('nodetool '):
+        #             cmd_name = f".{'_'.join(command.split(' ')[5:])}"
+
+        #         if not log_file:
+        #             log_file = f'{log_prefix()}-{datetime.now().strftime("%d%H%M%S")}{cmd_name}.log'
+        #         command = f"nohup {command} > {log_file} 2>&1 &"
+        #         if env_prefix:
+        #             command = f'{env_prefix} {command}'
+        #         exec_command = [shell, '-c', command]
 
         k_command = f'kubectl exec {pod_name} -c {container} -n {namespace} -- {shell} -c "{command}"'
         debug(k_command)
@@ -141,6 +171,36 @@ class Pods:
 
         return PodExecResult("".join(stdout), "".join(stderr), k_command, error_output, pod=pod_name, log_file=log_file)
 
+    def log_file(command: str, pod_name: str = None, dt: datetime = None):
+        cmd_name = ''
+        if command.startswith('nodetool '):
+            command = command.strip(' &')
+            cmd_name = f".{'_'.join(command.split(' ')[5:])}"
+
+        pod_suffix = '{pod}'
+        if pod_name:
+            pod_suffix = pod_name
+            if groups := re.match(r'.*-(.*)', pod_name):
+                pod_suffix = f'-{groups[1]}'
+
+        return f'{log_dir()}/{Pods.job_id()}{cmd_name}{pod_suffix}.log'
+
+    def job_id(dt: datetime = None):
+        if not dt:
+            dt = datetime.now()
+
+        return dt.strftime("%d%H%M%S")
+
+    def log_file_from_template(log_file: str, pod_name: str):
+        pod_suffix = pod_name
+        if pod_name and (groups := re.match(r'.*-(.*)', pod_name)):
+            pod_suffix = f'-{groups[1]}'
+
+        if not pod_suffix.startswith('-'):
+            pod_suffix = f'-{pod_suffix}'
+
+        return log_file.replace('{pod}', pod_suffix)
+
     def read_file(pod_name: str, container: str, namespace: str, file_path: str):
         v1 = client.CoreV1Api()
 
@@ -175,13 +235,15 @@ class Pods:
 
     def download_file(pod_name: str, container: str, namespace: str, from_path: str, to_path: str = None):
         if not to_path:
-            to_path = f'{local_tmp_dir()}/{os.path.basename(from_path)}'
+            to_path = f'{local_downloads_dir()}/{os.path.basename(from_path)}'
 
         bytes = Pods.read_file(pod_name, container, namespace, from_path)
         with open(to_path, 'wb') as f:
             for item in GeneratorStream(bytes):
                 f.write(item)
 
+        ReplSession().append_history(f':cat {to_path}')
+
         return to_path
 
     def get_container(namespace: str, pod_name: str, container_name: str):
@@ -292,7 +354,4 @@ class Pods:
             log2(' Timed Out')
 
     def completed(namespace: str, pod_name: str):
-        return Pods.get(namespace, pod_name).status.phase in ['Succeeded', 'Failed']
-
-def log_prefix():
-    return Config().get('log-prefix', '/tmp/qing')
+        return Pods.get(namespace, pod_name).status.phase in ['Succeeded', 'Failed']

adam/utils_k8s/statefulsets.py

@@ -56,8 +56,11 @@ class StatefulSets:
         return statefulset_pods
 
     @functools.lru_cache()
-    def pod_names(ss: str, ns: str):
-        return [pod.metadata.name for pod in StatefulSets.pods(ss, ns)]
+    def pod_names(sts: str, ns: str):
+        if not sts:
+            return []
+
+        return [pod.metadata.name for pod in StatefulSets.pods(sts, ns)]
 
     def restarted_at(ss: str, ns: str):
         # returns timestamp and if being rolled out

adam/utils_local.py

@@ -1,4 +1,80 @@
+import os
+import subprocess
+import traceback
+
 from adam.config import Config
+from adam.utils import ExecResult, creating_dir, debug
+
+def local_qing_dir():
+    return creating_dir(Config().get('local-qing-dir', '/tmp/qing-db/q'))
+
+def local_downloads_dir():
+    return creating_dir(Config().get('local-downloads-dir', '/tmp/qing-db/q/downloads'))
+
+class LocalExecResult(ExecResult):
+    def __init__(self, stdout: str, stderr: str, command: str = None, code = 0, log_file: str = None):
+        self.stdout: str = stdout
+        self.stderr: str = stderr
+        self.command: str = command
+        self.code = code
+        self.pod = 'local'
+        self.log_file = log_file
+
+    def exit_code(self) -> int:
+        return self.code
+
+    def cat_log_file_cmd(self):
+        if self.log_file:
+            return f':cat {self.log_file}'
+
+        return None
+
+    def __str__(self):
+        return f'{"OK" if self.exit_code() == 0 else self.exit_code()} {self.command}'
+
+    def __audit_extra__(self):
+        return self.log_file if self.log_file else None
+
+def local_exec(cmd: list[str], shell=False, show_out=False):
+    stdout = ''
+    stderr = ''
+    returncode = 0
+
+    try:
+        if show_out:
+            debug(' '.join(cmd))
+
+        r = subprocess.run(cmd, capture_output=True, text=True, shell=shell)
+        stdout = r.stdout
+        stderr = r.stderr
+        returncode = r.returncode
+    except FileNotFoundError as e:
+        pass
+
+    return LocalExecResult(stdout, stderr, ' '.join(cmd), returncode)
+
+def find_local_files(pattern: str = f'{local_qing_dir()}/*', file_type: str = None, max_depth = 0, mmin: int = 0):
+    # find . -maxdepth 1 -type f -name '*'
+    log_files = []
+    try:
+        dir = os.path.dirname(pattern)
+        base = os.path.basename(pattern)
+        cmd = ['find', dir]
+        if file_type:
+            cmd += ['-type', file_type]
+        if max_depth:
+            cmd += ['-maxdepth', str(max_depth)]
+        if mmin:
+                cmd += ['-mmin',  f'-{mmin}']
+        cmd += ['-name', base]
+
+        stdout = local_exec(cmd, show_out=Config().is_debug()).stdout
+
+        for line in stdout.split('\n'):
+            line = line.strip(' \r')
+            if line:
+                log_files.append(line)
+    except:
+        traceback.print_exc()
 
-def local_tmp_dir():
-    return Config().get('local-tmp-dir', '/tmp/qing-db')
+    return log_files

adam/utils_repl/appendable_completer.py

@@ -0,0 +1,6 @@
+from abc import abstractmethod
+
+class AppendableCompleter:
+    @abstractmethod
+    def append_completions(self, key: str, value: dict[str, any]):
+        pass

adam/utils_repl/repl_completer.py

@@ -1,14 +1,57 @@
+import copy
 import re
-from typing import Iterable, TypeVar
+import traceback
+from typing import Iterable, TypeVar, cast
 from prompt_toolkit.completion import CompleteEvent, Completion, NestedCompleter, WordCompleter
 from prompt_toolkit.document import Document
 
+from adam.utils import debug_complete, log2
+from adam.utils_repl.appendable_completer import AppendableCompleter
+
+import nest_asyncio
+nest_asyncio.apply()
+
+import asyncio
+
 __all__ = [
     "ReplCompleter",
 ]
 
 T = TypeVar('T')
 
+def merge_completions(dict1, dict2):
+    if isinstance(dict1, dict):
+        target = dict1.copy()
+    else:
+        target = copy.copy(dict1)
+
+    try:
+        for key, value in dict2.items():
+            if key in target:
+                debug_complete(f'[{key}] {type(dict2)} is being merged to {type(target[key])} completions')
+                if isinstance(value, dict):
+                    if isinstance(target[key], dict):
+                        target[key] = merge_completions(target[key], value)
+                    elif isinstance(target[key], AppendableCompleter):
+                        cast(AppendableCompleter, target[key]).append_completions(key, value)
+                    elif isinstance(target[key], NestedCompleter):
+                        cast(NestedCompleter, target[key]).options = merge_completions(cast(NestedCompleter, target[key]).options, value)
+                elif isinstance(value, AppendableCompleter):
+                    if isinstance(target[key], dict):
+                        cast(AppendableCompleter, value).append_completions(key, target[key])
+                        target[key] = value
+                    else:
+                        log2(f'* {key} of {type(value)} is overriding existing {type(target[key])} completions')
+                else:
+                    target[key] = value
+            else:
+                target[key] = value
+
+    except Exception as e:
+        traceback.print_exc()
+
+    return target
+
 class ReplCompleter(NestedCompleter):
     def get_completions(
         self, document: Document, complete_event: CompleteEvent
@@ -33,8 +76,12 @@ class ReplCompleter(NestedCompleter):
                     cursor_position=document.cursor_position - move_cursor,
                 )
 
-                for c in completer.get_completions(new_document, complete_event):
-                    yield c
+                try:
+                    # potential thread racing
+                    for c in completer.get_completions(new_document, complete_event):
+                        yield c
+                except:
+                    pass
 
         # No space in the input: behave exactly like `WordCompleter`.
         else:
@@ -43,4 +90,4 @@ class ReplCompleter(NestedCompleter):
                 list(self.options.keys()), ignore_case=self.ignore_case, pattern=re.compile(r"([a-zA-Z0-9_\.\@\&]+|[^a-zA-Z0-9_\.\@\&\s]+)")
             )
             for c in completer.get_completions(document, complete_event):
-                yield c
+                yield c

adam/utils_sqlite.py

@@ -7,7 +7,7 @@ import sqlite3
 import pandas
 
 from adam.config import Config
-from adam.utils import tabulize, log, wait_log
+from adam.utils import creating_dir, tabulize, log, wait_log
 
 class CursorHandler:
     def __init__(self, conn: sqlite3.Connection):
@@ -53,7 +53,7 @@ class SQLite:
         return CursorHandler(conn)
 
     def local_db_dir():
-        return Config().get('export.sqlite.local-db-dir', '/tmp/qing-db')
+        return creating_dir(Config().get('export.sqlite.local-db-dir', '/tmp/qing-db/q/export/db'))
 
     def keyspace(database: str):
         return '_'.join(database.replace(".db", "").split('_')[1:])
@@ -95,8 +95,6 @@ class SQLite:
                conn.close()
 
     def connect(database: str, keyspace: str = None):
-        os.makedirs(SQLite.local_db_dir(), exist_ok=True)
-
         if keyspace:
             return sqlite3.connect(f'{SQLite.local_db_dir()}/{database}_{keyspace}.db')
         else:
@@ -131,7 +129,4 @@ class SQLite:
         return len(lines), log_file
 
     def query(conn, sql: str) -> tuple[str, str, list]:
-        return pandas.read_sql_query(sql, conn)
-
-def log_prefix():
-    return Config().get('export.log-prefix', '/tmp/qing')
+        return pandas.read_sql_query(sql, conn)

adam/version.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-__version__ = "2.0.184"  #: the working version
+__version__ = "2.0.214"  #: the working version
 __release__ = "1.0.0"  #: the release version

{kaqing-2.0.184.dist-info → kaqing-2.0.214.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: kaqing
-Version: 2.0.184
+Version: 2.0.214
 Summary: UNKNOWN
 Home-page: UNKNOWN
 License: UNKNOWN