kaqing 2.0.171__py3-none-any.whl → 2.0.204__py3-none-any.whl

adam/utils_athena.py

@@ -1,10 +1,11 @@
+from collections.abc import Callable
 import functools
 import time
 import boto3
 import botocore
 
 from adam.config import Config
-from adam.utils import lines_to_tabular, log, log2, wait_log
+from adam.utils import tabulize, log, log2, log_exc, wait_log
 
 # no state utility class
 class Athena:
@@ -17,7 +18,7 @@ class Athena:
       if like:
          query = f"{query} AND schema_name like '{like}'"
 
-      try:
+      with log_exc():
          state, reason, rs = Athena.query(query)
          if rs:
             names = []
@@ -26,8 +27,6 @@ class Athena:
                   names.append(row_data[0])
 
             return names
-      except:
-         pass
 
       return []
 
@@ -64,7 +63,7 @@ class Athena:
 
    @functools.lru_cache()
    def column_names(tables: list[str] = [], database: str = None, function: str = 'audit', partition_cols_only = False):
-      try:
+      with log_exc():
          if not database:
             database = Config().get(f'{function}.athena.database', 'audit')
 
@@ -80,32 +79,33 @@ class Athena:
          _, _, rs = Athena.query(query)
          if rs:
             return [row['Data'][0].get('VarCharValue') for row in rs[1:]]
-      except:
-         # aws credentials not found
-         pass
 
       return []
 
-   def run_query(sql: str, database: str = None):
+   def run_query(sql: str, database: str = None, output: Callable[[str], str] = None):
       state, reason, rs = Athena.query(sql, database)
 
+      log_file = None
       if state == 'SUCCEEDED':
          if rs:
             column_info = rs[0]['Data']
             columns = [col.get('VarCharValue') for col in column_info]
-            lines = []
-            for row in rs[1:]:
-                  row_data = [col.get('VarCharValue') if col else '' for col in row['Data']]
-                  lines.append('\t'.join(row_data))
-
-            log(lines_to_tabular(lines, header='\t'.join(columns), separator='\t'))
-
-            return len(lines)
+            out = tabulize(rs[1:],
+                           lambda r: '\t'.join(col.get('VarCharValue') if col else '' for col in r['Data']),
+                           header='\t'.join(columns),
+                           separator='\t',
+                           to=0)
+            if output:
+               log_file = output(out)
+            else:
+               log(out)
+
+            return len(rs)-1, log_file
       else:
          log2(f"Query failed or was cancelled. State: {state}")
          log2(f"Reason: {reason}")
 
-      return 0
+      return 0, log_file
 
    def query(sql: str, database: str = None, function: str = 'audit') -> tuple[str, str, list]:
       region_name = Config().get(f'{function}.athena.region', 'us-west-2')

adam/utils_audits.py

@@ -4,7 +4,7 @@ import time
 import requests
 
 from adam.config import Config
-from adam.utils import OffloadHandler, debug, log2, offload
+from adam.utils import OffloadHandler, debug, log2, log_exc, offload
 from adam.utils_athena import Athena
 from adam.utils_net import get_my_host
 
@@ -46,12 +46,10 @@ class Audits:
       state, _, rs = Athena.query(f'select partitions_last_checked, clusters_last_checked from meta')
       if state == 'SUCCEEDED':
          if len(rs) > 1:
-            try:
+            with log_exc():
                row = rs[1]['Data']
                checked_in = float(row[0]['VarCharValue'])
                cluster_last_checked = float(row[1]['VarCharValue'])
-            except:
-               pass
 
       return AuditMeta(checked_in, cluster_last_checked)
 
@@ -88,10 +86,8 @@ class Audits:
       state, _, rs = Athena.query(query)
       if state == 'SUCCEEDED':
          if len(rs) > 1:
-               try:
+               with log_exc():
                   return [r['Data'][0]['VarCharValue'] for r in rs[1:]]
-               except:
-                  pass
 
       return []
 

adam/utils_issues.py

@@ -1,7 +1,7 @@
 from adam.checks.check_result import CheckResult
 from adam.checks.issue import Issue
 from adam.repl_session import ReplSession
-from adam.utils import lines_to_tabular, log, log2
+from adam.utils import tabulize, log2
 
 class IssuesUtils:
     def show(check_results: list[CheckResult], in_repl = False):
@@ -26,7 +26,7 @@ class IssuesUtils:
                     if in_repl:
                         ReplSession().prompt_session.history.append_string(issue.suggestion)
                         suggested += 1
-            log(lines_to_tabular(lines, separator='||'))
+            tabulize(lines, separator='||', to=2)
             if suggested:
                 log2()
                 log2(f'* {suggested} suggested commands are added to history. Press <Up> arrow to access them.')

adam/utils_k8s/app_clusters.py

@@ -2,7 +2,7 @@ import sys
 from typing import TypeVar
 
 from adam.utils_k8s.app_pods import AppPods
-from adam.pod_exec_result import PodExecResult
+from adam.utils_k8s.pod_exec_result import PodExecResult
 from adam.utils import log, log2
 from adam.utils_k8s.pods import Pods
 from .kube_context import KubeContext
@@ -12,17 +12,17 @@ T = TypeVar('T')
 # utility collection on app clusters; methods are all static
 class AppClusters:
     def exec(pods: list[str], namespace: str, command: str, action: str = 'action',
-             max_workers=0, show_out=True, on_any = False, shell = '/bin/sh', background = False) -> list[PodExecResult]:
+             max_workers=0, show_out=True, on_any = False, shell = '/bin/sh', backgrounded = False) -> list[PodExecResult]:
         samples = 1 if on_any else sys.maxsize
         msg = 'd`Running|Ran ' + action + ' command onto {size} pods'
         with Pods.parallelize(pods, max_workers, samples, msg, action=action) as exec:
-            results: list[PodExecResult] = exec.map(lambda pod: AppPods.exec(pod, namespace, command, False, False, shell, background))
+            results: list[PodExecResult] = exec.map(lambda pod: AppPods.exec(pod, namespace, command, False, False, shell, backgrounded))
             for result in results:
                 if KubeContext.show_out(show_out):
                     log(result.command)
                     if result.stdout:
                         log(result.stdout)
                     if result.stderr:
-                        log2(result.stderr, file=sys.stderr)
+                        log2(result.stderr)
 
             return results

adam/utils_k8s/app_pods.py

@@ -4,13 +4,16 @@ from kubernetes import client
 
 from adam.config import Config
 from adam.utils_k8s.pods import Pods
-from adam.pod_exec_result import PodExecResult
+from adam.utils_k8s.pod_exec_result import PodExecResult
 from adam.repl_session import ReplSession
 
 # utility collection on app pods; methods are all static
 class AppPods:
     @functools.lru_cache()
     def pod_names(namespace: str, env: str, app: str):
+        if not env or not app:
+            return []
+
         return [pod.metadata.name for pod in AppPods.app_pods(namespace, env, app)]
 
     def app_pods(namespace: str, env: str, app: str) -> List[client.V1Pod]:
@@ -22,12 +25,11 @@ class AppPods:
 
         return v1.list_namespaced_pod(namespace, label_selector=label_selector).items
 
-    def exec(pod_name: str, namespace: str, command: str, show_out = True, throw_err = False, shell = '/bin/sh', background = False) -> PodExecResult:
+    def exec(pod_name: str, namespace: str, command: str, show_out = True, throw_err = False, shell = '/bin/sh', backgrounded = False) -> PodExecResult:
         container = Config().get('app.container-name', 'c3-server')
-        r = Pods.exec(pod_name, container, namespace, command, show_out = show_out, throw_err = throw_err, shell = shell, background = background)
+        r = Pods.exec(pod_name, container, namespace, command, show_out = show_out, throw_err = throw_err, shell = shell, backgrounded = backgrounded)
 
-        if r and Config().get('repl.history.push-cat-remote-log-file', True):
-            if r.log_file and ReplSession().prompt_session:
-                ReplSession().prompt_session.history.append_string(f'@{r.pod} cat {r.log_file}')
+        if r and r.log_file:
+            ReplSession().append_history(f'@{r.pod} cat {r.log_file}')
 
         return r

adam/utils_k8s/cassandra_clusters.py

@@ -3,7 +3,7 @@ from typing import TypeVar
 
 from adam.config import Config
 from adam.utils_k8s.cassandra_nodes import CassandraNodes
-from adam.pod_exec_result import PodExecResult
+from adam.utils_k8s.pod_exec_result import PodExecResult
 from adam.utils import log, log2
 from adam.utils_k8s.pods import Pods
 from adam.utils_k8s.statefulsets import StatefulSets
@@ -12,21 +12,32 @@ T = TypeVar('T')
 
 # utility collection on cassandra clusters; methods are all static
 class CassandraClusters:
-    def exec(sts: str, namespace: str, command: str, action: str = 'action',
-             max_workers=0, show_out=True, on_any = False, shell = '/bin/sh', background = False, log_file = None) -> list[PodExecResult]:
+    def exec(sts: str,
+             namespace: str,
+             command: str,
+             action: str = 'action',
+             max_workers=0,
+             show_out=True,
+             on_any = False,
+             shell = '/bin/sh',
+             backgrounded = False,
+             log_file = None) -> list[PodExecResult]:
 
         pods = StatefulSets.pod_names(sts, namespace)
         samples = 1 if on_any else sys.maxsize
+        if (backgrounded or command.endswith(' &')) and not log_file:
+            log_file = Pods.log_file(command)
+
         msg = 'd`Running|Ran ' + action + ' command onto {size} pods'
         with Pods.parallelize(pods, max_workers, samples, msg, action=action) as exec:
-            results: list[PodExecResult] = exec.map(lambda pod: CassandraNodes.exec(pod, namespace, command, False, False, shell, background, log_file))
+            results: list[PodExecResult] = exec.map(lambda pod: CassandraNodes.exec(pod, namespace, command, False, False, shell, backgrounded, log_file))
             for result in results:
                 if show_out and not Config().is_debug():
                     log(result.command)
                     if result.stdout:
                         log(result.stdout)
                     if result.stderr:
-                        log2(result.stderr, file=sys.stderr)
+                        log2(result.stderr)
 
             return results
 

adam/utils_k8s/cassandra_nodes.py

@@ -1,17 +1,18 @@
 from adam.config import Config
 from adam.utils_k8s.pods import Pods
 from adam.utils_k8s.secrets import Secrets
-from adam.pod_exec_result import PodExecResult
+from adam.utils_k8s.pod_exec_result import PodExecResult
 from adam.repl_session import ReplSession
 
 # utility collection on cassandra nodes; methods are all static
 class CassandraNodes:
-    def exec(pod_name: str, namespace: str, command: str, show_out = True, throw_err = False, shell = '/bin/sh', background = False, log_file = None) -> PodExecResult:
-        r = Pods.exec(pod_name, "cassandra", namespace, command, show_out = show_out, throw_err = throw_err, shell = shell, background = background, log_file=log_file)
+    def exec(pod_name: str, namespace: str, command: str, show_out = True, throw_err = False, shell = '/bin/sh', backgrounded = False, log_file = None, no_history = False) -> PodExecResult:
+        r = Pods.exec(pod_name, "cassandra", namespace, command, show_out = show_out, throw_err = throw_err, shell = shell, backgrounded = backgrounded, log_file=log_file)
 
-        if r and Config().get('repl.history.push-cat-remote-log-file', True):
-            if r.log_file and ReplSession().prompt_session:
-                ReplSession().prompt_session.history.append_string(f'@{r.pod} cat {r.log_file}')
+        if not no_history and r and r.log_file:
+            entry = f':sh cat {r.log_file}'
+
+            ReplSession().append_history(entry)
 
         return r
 

adam/utils_k8s/custom_resources.py

@@ -1,11 +1,10 @@
 import functools
 import re
-import time
 from kubernetes import client
 
 from adam.config import Config
 from .kube_context import KubeContext
-from adam.utils import elapsed_time, lines_to_tabular, log2
+from adam.utils import log2, log_exc
 
 
 # utility collection; methods are all static
@@ -20,14 +19,12 @@ class CustomResources:
         strip = Config().get('app.strip', '0')
 
         v1 = client.CustomObjectsApi()
-        try:
+        with log_exc():
             c3cassandras = v1.list_cluster_custom_object(group=group, version=v, plural=plural)
             for c in c3cassandras.items():
                 if c[0] == 'items':
                     for item in c[1]:
                         app_ids_by_ss[f"{item['metadata']['name']}@{item['metadata']['namespace']}"] = item['metadata']['labels'][label].strip(strip)
-        except Exception:
-            pass
 
         return app_ids_by_ss
 
@@ -122,11 +119,10 @@ class CustomResources:
         body = bkspecs
         pretty = 'true'
 
-        try:
+        with log_exc(lambda e: "Exception when calling create_medusa_backupjob.create_namespaced_custom_object: %s\n" % e):
             api_instance.create_namespaced_custom_object(group, version, namespace, plural, body, pretty=pretty)
             log2(f"create_medusa_backupjob: created Full Backup {bkname}: {api_instance}")
-        except Exception as e:
-            log2("Exception when calling create_medusa_backupjob.create_namespaced_custom_object: %s\n" % e)
+
         return None
 
     def create_medusa_restorejob(restorejobname: str, bkname: str, dc: str, ns: str):
@@ -155,11 +151,10 @@ class CustomResources:
         body = rtspecs
         pretty = 'true'
 
-        try:
+        with log_exc(lambda e: "Exception when calling create_medusa_restorejob.create_namespaced_custom_object: %s\n" % e):
             api_instance.create_namespaced_custom_object(group, version, namespace, plural, body, pretty=pretty)
             log2(f"create_medusa_restorejob: created Restore Job {restorejobname}: {api_instance}")
-        except Exception as e:
-            log2("Exception when calling create_medusa_restorejob.create_namespaced_custom_object: %s\n" % e)
+
         return None
 
     def medusa_show_backup_names(dc: str, ns: str) -> list[dict]:
@@ -185,11 +180,10 @@ class CustomResources:
         pretty = 'true'
         label_selector = 'cassandra.datastax.com/datacenter=' + dc
 
-        try:
+        with log_exc(lambda e: "Exception when calling medusa_show_backupjobs.list_namespaced_custom_object: %s\n" % e):
             api_response = api_instance.list_namespaced_custom_object(group, version, namespace, plural, pretty=pretty, label_selector=label_selector)
             return api_response['items']
-        except Exception as e:
-            log2("Exception when calling medusa_show_backupjobs.list_namespaced_custom_object: %s\n" % e)
+
         return None
 
     def medusa_show_restorejobs(dc: str, ns: str):
@@ -201,11 +195,11 @@ class CustomResources:
         pretty = 'true'
         label_selector = 'cassandra.datastax.com/datacenter=' + dc
         rtlist = []
-        try:
+
+        with log_exc(lambda e: "Exception when calling medusa_show_restorejobs.list_namespaced_custom_object: %s\n" % e):
             api_response = api_instance.list_namespaced_custom_object(group, version, namespace, plural, pretty=pretty, label_selector=label_selector)
             for x in api_response['items']:
                 rtlist.append(f"{x['metadata']['name']}\t{x['metadata']['creationTimestamp']}\t{x['status'].get('finishTime', '')}")
             return rtlist
-        except Exception as e:
-            log2("Exception when calling medusa_show_restorejobs.list_namespaced_custom_object: %s\n" % e)
+
         return None

adam/utils_k8s/jobs.py

@@ -1,7 +1,7 @@
 from kubernetes import client
 from time import sleep
 from .pods import Pods
-from adam.utils import log2
+from adam.utils import log2, log_exc
 
 # utility collection on jobs; methods are all static
 class Jobs:
@@ -20,11 +20,10 @@ class Jobs:
             metadata=client.V1ObjectMeta(name=job_name),
             spec=spec)
 
-        try:
+        with log_exc(lambda e: "Exception when calling BatchV1Apii->create_namespaced_job: %s\n" % e):
             client.BatchV1Api().create_namespaced_job(body=job, namespace=namespace)
             log2(f"Job {job_name} created in {namespace}")
-        except Exception as e:
-            log2("Exception when calling BatchV1Apii->create_namespaced_job: %s\n" % e)
+
         return
 
     def get_job_pods(job_name: str, namespace: str):
@@ -32,7 +31,7 @@ class Jobs:
         return pods
 
     def delete(job_name: str, namespace: str, wait=True):
-        try:
+        with log_exc(lambda e: "Exception when calling BatchV1Apii->delete_namespaced_job: %s\n" % e):
             client.BatchV1Api().delete_namespaced_job(name=job_name, namespace=namespace, propagation_policy='Background')
             if wait:
                 while True:
@@ -41,14 +40,11 @@ class Jobs:
                         return
                     sleep(5)
             log2(f"Job {job_name} in {namespace} deleted.")
-        except Exception as e:
-            log2("Exception when calling BatchV1Apii->delete_namespaced_job: %s\n" % e)
+
         return
 
     def get_logs(job_name: str, namespace: str):
         v1 = client.CoreV1Api()
-        try:
+        with log_exc(lambda e: "Exception when calling CorV1Apii->list_namespaced_pod, cannot find job pod: %s\n" % e):
             pod_name = Jobs.get_job_pods(job_name, namespace).items[0].metadata.name
-            log2(v1.read_namespaced_pod_log(name=pod_name, namespace=namespace))
-        except Exception as e:
-            log2("Exception when calling CorV1Apii->list_namespaced_pod, cannot find job pod: %s\n" % e)
+            log2(v1.read_namespaced_pod_log(name=pod_name, namespace=namespace))

adam/utils_k8s/k8s.py

@@ -1,8 +1,9 @@
 from collections.abc import Callable
+import inspect
 import re
 import portforward
 
-from adam.commands.command import InvalidState
+from adam.commands.command import InvalidStateException
 from adam.repl_state import ReplState
 from adam.utils import log2
 from adam.utils_k8s.kube_context import KubeContext
@@ -24,14 +25,14 @@ class PortForwardHandler:
         if not self.svc_or_pod:
             log2('No service or pod found.')
 
-            raise InvalidState(state)
+            raise InvalidStateException(state)
 
         if KubeContext.in_cluster():
             svc_name = self.svc_or_pod(True)
             if not svc_name:
                 log2('No service found.')
 
-                raise InvalidState(state)
+                raise InvalidStateException(state)
 
             # cs-a526330d23-cs-a526330d23-default-sts-0 ->
             # curl http://cs-a526330d23-cs-a526330d23-reaper-service.stgawsscpsr.svc.cluster.local:8080
@@ -40,15 +41,23 @@ class PortForwardHandler:
                 svc = f'{groups[1]}{svc_name}.{state.namespace}.svc.cluster.local:{self.target_port}'
                 return (svc, svc)
             else:
-                raise InvalidState(state)
+                raise InvalidStateException(state)
         else:
             pod = self.svc_or_pod(False)
             if not pod:
                 log2('No pod found.')
 
-                raise InvalidState(state)
+                raise InvalidStateException(state)
 
             self.pod = pod
+
+            # pf = portforward.forward(state.namespace, pod, self.local_port + 1, self.target_port, log_level=portforward.LogLevel.DEBUG)
+            # print(inspect.getsource(pf.__enter__))
+            # print('test portforward START', state.namespace, pod, self.local_port + 1, self.target_port, pf.__enter__)
+            # with pf:
+            #     print('test portforward BODY')
+            # print('test portforward OK')
+
             self.forward_connection = portforward.forward(state.namespace, pod, self.local_port, self.target_port)
             if self.inc_connection_cnt() == 1:
                 self.forward_connection.__enter__()

adam/utils_k8s/kube_context.py

@@ -3,7 +3,7 @@ import re
 from kubernetes import config as kconfig
 
 from adam.config import Config
-from adam.utils import idp_token_from_env, lines_to_tabular, log2
+from adam.utils import idp_token_from_env, log2, tabulize
 
 class KubeContext:
     _in_cluster = False
@@ -56,7 +56,7 @@ class KubeContext:
             log2('Use -v <key>=<value> format.')
             log2()
             lines = [f'{key}\t{Config().get(key, None)}' for key in Config().keys()]
-            log2(lines_to_tabular(lines, separator='\t'))
+            tabulize(lines, separator='\t', to=2)
 
         for p in param_ovrs:
             tokens = p.split('=')
@@ -102,7 +102,4 @@ class KubeContext:
         return name if re.match(r"^(?!pg-).*-k8spg-.*$", name) else None
 
     def show_out(s: bool):
-        return s or Config().is_debug()
-
-    def show_parallelism():
-        return Config().get('debugs.show-parallelism', False)
+        return s or Config().is_debug()

adam/{pod_exec_result.py → utils_k8s/pod_exec_result.py}

@@ -1,6 +1,8 @@
 import yaml
 
-class PodExecResult:
+from adam.utils import ExecResult, log_exc
+
+class PodExecResult(ExecResult):
     # {
     #   'metadata': {},
     #   'status': 'Failure',
@@ -27,10 +29,8 @@ class PodExecResult:
     def exit_code(self) -> int:
         code = 0
 
-        try:
+        with log_exc(False):
             code = self.error['details']['causes'][0]['message']
-        except:
-            pass
 
         return code