kaqing 2.0.145__py3-none-any.whl → 2.0.172__py3-none-any.whl

adam/sso/cred_cache.py

@@ -1,9 +1,9 @@
 import os
 from pathlib import Path
-import traceback
 from dotenv import load_dotenv
 
 from adam.config import Config
+from adam.utils import debug, log_exc
 from adam.utils_k8s.kube_context import KubeContext
 
 class CredCache:
@@ -34,10 +34,8 @@ class CredCache:
     def cache(self, username: str, password: str = None):
         if os.path.exists(self.env_f):
             with open(self.env_f, 'w') as file:
-                try:
+                with log_exc():
                     file.truncate()
-                except:
-                    Config().debug(traceback.format_exc())
 
         updated = []
         updated.append(f'IDP_USERNAME={username}')
@@ -56,4 +54,4 @@ class CredCache:
             if password:
                 self.overrides['IDP_PASSWORD'] = password
 
-            Config().debug(f'Cached username: {username}, password: {password}, try load: {self.get_username()}')
+            debug(f'Cached username: {username}, password: {password}, try load: {self.get_username()}')

adam/sso/idp.py

@@ -3,7 +3,6 @@ import getpass
 import os
 import sys
 import termios
-import traceback
 from typing import Callable, TypeVar
 import requests
 from kubernetes import config
@@ -15,7 +14,7 @@ from .cred_cache import CredCache
 from .idp_session import IdpSession
 from .idp_login import IdpLogin
 from adam.config import Config
-from adam.utils import log, log2
+from adam.utils import debug, log, log_exc
 
 T = TypeVar('T')
 
@@ -31,6 +30,8 @@ class Idp:
     def login(app_host: str, username: str = None, idp_uri: str = None, forced = False, use_token_from_env = True, use_cached_creds = True, verify = True) -> IdpLogin:
         session: IdpSession = IdpSession.create(username, app_host, app_host, idp_uri=idp_uri)
 
+        debug(f'Idp.login({username})')
+
         if use_token_from_env:
             if l0 := session.login_from_env_var():
                 return l0
@@ -39,11 +40,9 @@ class Idp:
                 token_server = Config().get('app.login.token-server-url', 'http://localhost:{port}').replace('{port}', port)
                 res: requests.Response = requests.get(token_server)
                 if res.status_code == 200 and res.text:
-                    try:
+                    with log_exc():
                         # may fail if the idp token is not complete
                         return session.login_from_token(res.text)
-                    except:
-                        pass
 
         r: IdpLogin = None
         try:
@@ -57,10 +56,11 @@ class Idp:
                 default_user: str = None
                 if use_cached_creds:
                     default_user = CredCache().get_username()
-                    Config().debug(f'User read from cache: {default_user}')
+                    debug(f'User read from cache: {default_user}')
 
-                if from_env := os.getenv('USERNAME'):
-                    default_user = from_env
+                # no value in using USERNAME
+                # if from_env := os.getenv('USERNAME') and in_docker():
+                #     default_user = from_env
                 if default_user and default_user != username:
                     session = IdpSession.create(default_user, app_host, app_host)
 
@@ -125,7 +125,7 @@ class Idp:
             termios.tcsetattr(fd, termios.TCSADRAIN, old)
 
     def try_kubeconfig(username: str, kubeconfig: str):
-        try:
+        with log_exc():
             if kubeconfig[0] == '\t':
                 kubeconfig = kubeconfig[1:]
             kubeconfig_string = base64.b64decode(kubeconfig.encode('ascii') + b'==').decode('utf-8')
@@ -136,8 +136,5 @@ class Idp:
                 Secrets.list_secrets(os.getenv('NAMESPACE'))
 
                 return IdpLogin(None, None, None, username)
-        except:
-            Config().debug(traceback.format_exc())
-            pass
 
         return None

adam/utils.py

@@ -1,3 +1,4 @@
+from concurrent.futures import Future, ThreadPoolExecutor
 from contextlib import redirect_stdout
 import copy
 import csv
@@ -10,7 +11,8 @@ from pathlib import Path
 import random
 import string
 import threading
-from typing import Callable
+import traceback
+from typing import Callable, Iterator, TypeVar, Union
 from dateutil import parser
 import subprocess
 import sys
@@ -20,7 +22,12 @@ import yaml
 
 from . import __version__
 
-is_debug_holder = [lambda: False]
+log_state = threading.local()
+
+class LogConfig:
+    is_debug = lambda: False
+    is_debug_timing = lambda: False
+    is_display_help = True
 
 def to_tabular(lines: str, header: str = None, dashed_line = False):
     return lines_to_tabular(lines.split('\n'), header, dashed_line)
@@ -74,7 +81,7 @@ def epoch(timestamp_string: str):
 
 def log(s = None):
     if not loggable():
-        return
+        return False
 
     # want to print empty line for False or empty collection
     if s == None:
@@ -82,15 +89,19 @@ def log(s = None):
     else:
         click.echo(s)
 
+    return True
+
 def log2(s = None, nl = True):
     if not loggable():
-        return
+        return False
 
     if s:
         click.echo(s, err=True, nl=nl)
     else:
         print(file=sys.stderr)
 
+    return True
+
 def elapsed_time(start_time: float):
     end_time = time.time()
     elapsed_time = end_time - start_time
@@ -105,8 +116,8 @@ def duration(start_time: float, end_time: float = None):
         end_time = time.time()
     d = convert_seconds(end_time - start_time)
     t = []
-    if d[0]:
-        t.append(f'{d[0]}h')
+    if d:
+        t.append(f'{d}h')
     if t or d[1]:
         t.append(f'{d[1]}m')
     t.append(f'{d[2]}s')
@@ -169,6 +180,9 @@ def get_deep_keys(d, current_path=""):
     return keys
 
 def display_help(replace_arg = False):
+    if not LogConfig.is_display_help:
+        return
+
     args = copy.copy(sys.argv)
     if replace_arg:
         args[len(args) - 1] = '--help'
@@ -213,12 +227,13 @@ def json_to_csv(json_data: list[dict[any, any]], delimiter: str = ','):
         with redirect_stdout(body) as f:
             dict_writer = csv.DictWriter(f, keys, delimiter=delimiter)
             dict_writer.writerows(flattened_data)
+
         return header.getvalue().strip('\r\n'), [l.strip('\r') for l in body.getvalue().split('\n')]
     else:
         return None
 
 def log_to_file(config: dict[any, any]):
-    try:
+    with log_exc():
         base = f"/kaqing/logs"
         os.makedirs(base, exist_ok=True)
 
@@ -233,8 +248,6 @@ def log_to_file(config: dict[any, any]):
                     f.write(config)
             else:
                     f.write(config)
-    except:
-        pass
 
 def copy_config_file(rel_path: str, module: str, suffix: str = '.yaml', show_out = True):
     dir = f'{Path.home()}/.kaqing'
@@ -255,31 +268,50 @@ def idp_token_from_env():
 def is_lambda(func):
     return callable(func) and hasattr(func, '__name__') and func.__name__ == '<lambda>'
 
-class Ing:
-    state = threading.local()
+def debug(s = None):
+    if LogConfig.is_debug():
+        log2(f'DEBUG {s}')
+
+def debug_trace():
+    if LogConfig.is_debug():
+        log2(traceback.format_exc())
 
+def in_docker() -> bool:
+    if os.path.exists('/.dockerenv'):
+        return True
+
+    try:
+        with open('/proc/1/cgroup', 'rt') as f:
+            for line in f:
+                if 'docker' in line or 'lxc' in line:
+                    return True
+    except FileNotFoundError:
+        pass
+
+    return False
+
+class Ing:
     def __init__(self, msg: str, suppress_log=False):
         self.msg = msg
         self.suppress_log = suppress_log
-        self.nested = False
 
     def __enter__(self):
-        if not hasattr(Ing.state, 'ing_cnt'):
-            Ing.state.ing_cnt = 0
+        if not hasattr(log_state, 'ing_cnt'):
+            log_state.ing_cnt = 0
 
         try:
-            if not Ing.state.ing_cnt:
-                if not self.suppress_log and not is_debug_holder[0]():
+            if not log_state.ing_cnt:
+                if not self.suppress_log and not LogConfig.is_debug():
                     log2(f'{self.msg}...', nl=False)
 
             return None
         finally:
-            Ing.state.ing_cnt += 1
+            log_state.ing_cnt += 1
 
     def __exit__(self, exc_type, exc_val, exc_tb):
-        Ing.state.ing_cnt -= 1
-        if not Ing.state.ing_cnt:
-            if not self.suppress_log and not is_debug_holder[0]():
+        log_state.ing_cnt -= 1
+        if not log_state.ing_cnt:
+            if not self.suppress_log and not LogConfig.is_debug():
                 log2(' OK')
 
         return False
@@ -290,23 +322,351 @@ def ing(msg: str, body: Callable[[], None]=None, suppress_log=False):
 
     r = None
 
-    if not hasattr(Ing.state, 'ing_cnt'):
-        Ing.state.ing_cnt = 0
-
-    if not Ing.state.ing_cnt:
-        if not suppress_log and not is_debug_holder[0]():
-            log2(f'{msg}...', nl=False)
-
-    Ing.state.ing_cnt += 1
+    t = Ing(msg, suppress_log=suppress_log)
+    t.__enter__()
     try:
         r = body()
     finally:
-        Ing.state.ing_cnt -= 1
-        if not Ing.state.ing_cnt:
-            if not suppress_log and not is_debug_holder[0]():
-                log2(' OK')
+        t.__exit__(None, None, None)
 
     return r
 
 def loggable():
-    return is_debug_holder[0]() or not hasattr(Ing.state, 'ing_cnt') or not Ing.state.ing_cnt
+    return LogConfig.is_debug() or not hasattr(log_state, 'ing_cnt') or not log_state.ing_cnt
+
+class TimingNode:
+    def __init__(self, depth: int, s0: time.time = time.time(), line: str = None):
+        self.depth = depth
+        self.s0 = s0
+        self.line = line
+        self.children = []
+
+    def __str__(self):
+        return f'[{self.depth}: {self.line}, children={len(self.children)}]'
+
+    def tree(self):
+        lines = []
+        if self.line:
+            lines.append(self.line)
+
+        for child in self.children:
+            if child.line:
+                lines.append(child.tree())
+        return '\n'.join(lines)
+
+class LogTiming:
+    def __init__(self, msg: str, s0: time.time = None):
+        self.msg = msg
+        self.s0 = s0
+
+    def __enter__(self):
+        if not LogConfig.is_debug_timing():
+            return
+
+        if not hasattr(log_state, 'timings'):
+            log_state.timings = TimingNode(0)
+
+        self.me = log_state.timings
+        log_state.timings = TimingNode(self.me.depth+1)
+        if not self.s0:
+            self.s0 = time.time()
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if not LogConfig.is_debug_timing():
+            return False
+
+        child = log_state.timings
+        log_state.timings.line = timing_log_line(self.me.depth, self.msg, self.s0)
+
+        if child and child.line:
+            self.me.children.append(child)
+        log_state.timings = self.me
+
+        if not self.me.depth:
+            log2(self.me.tree())
+            log_state.timings = TimingNode(0)
+
+        return False
+
+def log_timing(msg: str, body: Callable[[], None]=None, s0: time.time = None):
+    if not s0 and not body:
+        return LogTiming(msg, s0=s0)
+
+    if not LogConfig.is_debug_timing():
+        if body:
+            return body()
+
+        return
+
+    r = None
+
+    t = LogTiming(msg, s0=s0)
+    t.__enter__()
+    try:
+        if body:
+            r = body()
+    finally:
+        t.__exit__(None, None, None)
+
+    return r
+
+def timing_log_line(depth: int, msg: str, s0: time.time):
+    elapsed = time.time() - s0
+    prefix = '[timings] '
+    if depth:
+        if elapsed > 0.01:
+            prefix = ('  ' * (depth-1)) + '* '
+        else:
+            prefix = '  ' * depth
+
+    return f'{prefix}{msg}: {elapsed:.2f} sec'
+
+class WaitLog:
+    wait_log_flag = False
+
+def wait_log(msg: str):
+    if not WaitLog.wait_log_flag:
+        log2(msg)
+        WaitLog.wait_log_flag = True
+
+def clear_wait_log_flag():
+    WaitLog.wait_log_flag = False
+
+class LogTrace:
+    def __init__(self, err_msg: Union[str, callable, bool] = None):
+        self.err_msg = err_msg
+
+    def __enter__(self):
+        return None
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if exc_type is not None:
+            if self.err_msg is True:
+                log2(str(exc_val))
+            elif callable(self.err_msg):
+                log2(self.err_msg(exc_val))
+            elif self.err_msg is not False and self.err_msg:
+                log2(self.err_msg)
+
+            if self.err_msg is not False and LogConfig.is_debug():
+                traceback.print_exception(exc_type, exc_val, exc_tb, file=sys.stderr)
+
+        # swallow exception
+        return True
+
+def log_exc(err_msg: Union[str, callable, bool] = None):
+    return LogTrace(err_msg=err_msg)
+
+T = TypeVar('T')
+
+class ParallelService:
+    def __init__(self, handler: 'ParallelMapHandler'):
+        self.handler = handler
+
+    def map(self, fn: Callable[..., T]) -> Iterator[T]:
+        executor = self.handler.executor
+        collection = self.handler.collection
+        collect = self.handler.collect
+        samples_cnt = self.handler.samples
+
+        iterator = None
+        if executor:
+            iterator = executor.map(fn, collection)
+        elif samples_cnt < sys.maxsize:
+            samples = []
+
+            for elem in collection:
+                if not samples_cnt:
+                    break
+
+                samples.append(fn(elem))
+                samples_cnt -= 1
+
+            iterator = iter(samples)
+        else:
+            iterator = map(fn, collection)
+
+        if collect:
+            return list(iterator)
+        else:
+            return iterator
+
+class ParallelMapHandler:
+    def __init__(self, collection: list, max_workers: int, samples: int = sys.maxsize, msg: str = None, collect = True):
+        self.collection = collection
+        self.max_workers = max_workers
+        self.executor = None
+        self.samples = samples
+        self.msg = msg
+        if msg and msg.startswith('d`'):
+            if LogConfig.is_debug():
+                self.msg = msg.replace('d`', '', 1)
+            else:
+                self.msg = None
+        self.collect = collect
+
+        self.begin = []
+        self.end = []
+        self.start_time = None
+
+    def __enter__(self):
+        self.calc_msgs()
+
+        if self.max_workers > 1 and (not self.size() or self.size()) and self.samples == sys.maxsize:
+            self.start_time = time.time()
+
+            self.executor = ThreadPoolExecutor(max_workers=self.max_workers)
+            self.executor.__enter__()
+
+        return ParallelService(self)
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if self.executor:
+            self.executor.__exit__(exc_type, exc_val, exc_tb)
+
+        if self.end:
+            log2(f'{" ".join(self.end)} in {elapsed_time(self.start_time)}.')
+
+        return False
+
+    def size(self):
+        if not self.collection:
+            return 0
+
+        return len(self.collection)
+
+    def calc_msgs(self):
+        if not self.msg:
+            return
+
+        size = self.size()
+            # return
+
+        offloaded = False
+        serially = False
+        sampling = False
+        if size == 0:
+            offloaded = True
+            self.msg = self.msg.replace('{size}', '1')
+        elif self.max_workers > 1 and size > 1 and self.samples == sys.maxsize:
+            self.msg = self.msg.replace('{size}', f'{size}')
+        elif self.samples < sys.maxsize:
+            sampling = True
+            if self.samples > size:
+                self.samples = size
+            self.msg = self.msg.replace('{size}', f'{self.samples}/{size} sample')
+        else:
+            serially = True
+            self.msg = self.msg.replace('{size}', f'{size}')
+            # return
+
+        for token in self.msg.split(' '):
+            if '|' in token:
+                self.begin.append(token.split('|')[0])
+                if not sampling and not serially and not offloaded:
+                    self.end.append(token.split('|')[1])
+            else:
+                self.begin.append(token)
+                if not sampling and not serially and not offloaded:
+                    self.end.append(token)
+
+        if offloaded:
+            log2(f'{" ".join(self.begin)} offloaded...')
+        elif sampling or serially:
+            log2(f'{" ".join(self.begin)} serially...')
+        else:
+            log2(f'{" ".join(self.begin)} with {self.max_workers} workers...')
+
+class OffloadService:
+    def __init__(self, handler: 'OffloadHandler'):
+        self.handler = handler
+
+    def submit(self, fn: Callable[..., T], /, *args, **kwargs) -> Future[T]:
+        executor = self.handler.executor
+
+        if executor:
+            return executor.submit(fn, *args, **kwargs)
+        else:
+            future = Future()
+
+            future.set_result(fn(*args, **kwargs))
+
+            return future
+
+class OffloadHandler(ParallelMapHandler):
+    def __init__(self, max_workers: int, msg: str = None):
+        super().__init__(None, max_workers, msg=msg, collect=False )
+
+    def __enter__(self):
+        self.calc_msgs()
+
+        if self.max_workers > 1 and (not self.size() or self.size()) and self.samples == sys.maxsize:
+            self.start_time = time.time()
+
+            self.executor = ThreadPoolExecutor(max_workers=self.max_workers)
+            self.executor.__enter__()
+
+        return OffloadService(self)
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if self.executor:
+            self.executor.__exit__(exc_type, exc_val, exc_tb)
+
+        if self.end:
+            log2(f'{" ".join(self.end)} in {elapsed_time(self.start_time)}.')
+
+        return False
+
+    def size(self):
+        if not self.collection:
+            return 0
+
+        return len(self.collection)
+
+    def calc_msgs(self):
+        if not self.msg:
+            return
+
+        size = self.size()
+            # return
+
+        offloaded = False
+        serially = False
+        sampling = False
+        if size == 0:
+            offloaded = True
+            self.msg = self.msg.replace('{size}', '1')
+        elif self.max_workers > 1 and size > 1 and self.samples == sys.maxsize:
+            self.msg = self.msg.replace('{size}', f'{size}')
+        elif self.samples < sys.maxsize:
+            sampling = True
+            if self.samples > size:
+                self.samples = size
+            self.msg = self.msg.replace('{size}', f'{self.samples}/{size} sample')
+        else:
+            serially = True
+            self.msg = self.msg.replace('{size}', f'{size}')
+            # return
+
+        for token in self.msg.split(' '):
+            if '|' in token:
+                self.begin.append(token.split('|')[0])
+                if not sampling and not serially and not offloaded:
+                    self.end.append(token.split('|')[1])
+            else:
+                self.begin.append(token)
+                if not sampling and not serially and not offloaded:
+                    self.end.append(token)
+
+        if offloaded:
+            log2(f'{" ".join(self.begin)} offloaded...')
+        elif sampling or serially:
+            log2(f'{" ".join(self.begin)} serially...')
+        else:
+            log2(f'{" ".join(self.begin)} with {self.max_workers} workers...')
+
+def parallelize(collection: list, max_workers: int = 0, samples = sys.maxsize, msg: str = None, collect = True):
+    return ParallelMapHandler(collection, max_workers, samples = samples, msg = msg, collect = collect)
+
+def offload(max_workers: int = 3, msg: str = None):
+    return OffloadHandler(max_workers, msg = msg)