kaqing 2.0.98__py3-none-any.whl → 2.0.203__py3-none-any.whl

adam/utils_k8s/jobs.py

@@ -1,7 +1,7 @@
 from kubernetes import client
 from time import sleep
 from .pods import Pods
-from adam.utils import log2
+from adam.utils import log2, log_exc
 
 # utility collection on jobs; methods are all static
 class Jobs:
@@ -20,11 +20,10 @@ class Jobs:
             metadata=client.V1ObjectMeta(name=job_name),
             spec=spec)
 
-        try:
+        with log_exc(lambda e: "Exception when calling BatchV1Apii->create_namespaced_job: %s\n" % e):
             client.BatchV1Api().create_namespaced_job(body=job, namespace=namespace)
             log2(f"Job {job_name} created in {namespace}")
-        except Exception as e:
-            log2("Exception when calling BatchV1Apii->create_namespaced_job: %s\n" % e)
+
         return
 
     def get_job_pods(job_name: str, namespace: str):
@@ -32,7 +31,7 @@ class Jobs:
         return pods
 
     def delete(job_name: str, namespace: str, wait=True):
-        try:
+        with log_exc(lambda e: "Exception when calling BatchV1Apii->delete_namespaced_job: %s\n" % e):
             client.BatchV1Api().delete_namespaced_job(name=job_name, namespace=namespace, propagation_policy='Background')
             if wait:
                 while True:
@@ -41,14 +40,11 @@ class Jobs:
                         return
                     sleep(5)
             log2(f"Job {job_name} in {namespace} deleted.")
-        except Exception as e:
-            log2("Exception when calling BatchV1Apii->delete_namespaced_job: %s\n" % e)
+
         return
 
     def get_logs(job_name: str, namespace: str):
         v1 = client.CoreV1Api()
-        try:
+        with log_exc(lambda e: "Exception when calling CorV1Apii->list_namespaced_pod, cannot find job pod: %s\n" % e):
             pod_name = Jobs.get_job_pods(job_name, namespace).items[0].metadata.name
-            log2(v1.read_namespaced_pod_log(name=pod_name, namespace=namespace))
-        except Exception as e:
-            log2("Exception when calling CorV1Apii->list_namespaced_pod, cannot find job pod: %s\n" % e)
+            log2(v1.read_namespaced_pod_log(name=pod_name, namespace=namespace))

adam/utils_k8s/k8s.py

@@ -0,0 +1,96 @@
+from collections.abc import Callable
+import inspect
+import re
+import portforward
+
+from adam.commands.command import InvalidStateException
+from adam.repl_state import ReplState
+from adam.utils import log2
+from adam.utils_k8s.kube_context import KubeContext
+
+class PortForwardHandler:
+    connections: dict[str, int] = {}
+
+    def __init__(self, state: ReplState, local_port: int, svc_or_pod: Callable[[bool],str], target_port: int):
+        self.state = state
+        self.local_port = local_port
+        self.svc_or_pod = svc_or_pod
+        self.target_port = target_port
+        self.forward_connection = None
+        self.pod = None
+
+    def __enter__(self) -> tuple[str, str]:
+        state = self.state
+
+        if not self.svc_or_pod:
+            log2('No service or pod found.')
+
+            raise InvalidStateException(state)
+
+        if KubeContext.in_cluster():
+            svc_name = self.svc_or_pod(True)
+            if not svc_name:
+                log2('No service found.')
+
+                raise InvalidStateException(state)
+
+            # cs-a526330d23-cs-a526330d23-default-sts-0 ->
+            # curl http://cs-a526330d23-cs-a526330d23-reaper-service.stgawsscpsr.svc.cluster.local:8080
+            groups = re.match(r'^(.*?-.*?-.*?-.*?-).*', state.sts)
+            if groups:
+                svc = f'{groups[1]}{svc_name}.{state.namespace}.svc.cluster.local:{self.target_port}'
+                return (svc, svc)
+            else:
+                raise InvalidStateException(state)
+        else:
+            pod = self.svc_or_pod(False)
+            if not pod:
+                log2('No pod found.')
+
+                raise InvalidStateException(state)
+
+            self.pod = pod
+
+            # pf = portforward.forward(state.namespace, pod, self.local_port + 1, self.target_port, log_level=portforward.LogLevel.DEBUG)
+            # print(inspect.getsource(pf.__enter__))
+            # print('test portforward START', state.namespace, pod, self.local_port + 1, self.target_port, pf.__enter__)
+            # with pf:
+            #     print('test portforward BODY')
+            # print('test portforward OK')
+
+            self.forward_connection = portforward.forward(state.namespace, pod, self.local_port, self.target_port)
+            if self.inc_connection_cnt() == 1:
+                self.forward_connection.__enter__()
+
+            return (f'localhost:{self.local_port}', f'{pod}:{self.target_port}')
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if self.forward_connection:
+            if not self.dec_connection_cnt():
+                return self.forward_connection.__exit__(exc_type, exc_val, exc_tb)
+
+        return False
+
+    def inc_connection_cnt(self):
+        id = self.connection_id(self.pod)
+        if id not in PortForwardHandler.connections:
+            PortForwardHandler.connections[id] = 1
+        else:
+            PortForwardHandler.connections[id] += 1
+
+        return PortForwardHandler.connections[id]
+
+    def dec_connection_cnt(self):
+        id = self.connection_id(self.pod)
+        if id not in PortForwardHandler.connections:
+            PortForwardHandler.connections[id] = 0
+        elif PortForwardHandler.connections[id] > 0:
+            PortForwardHandler.connections[id] -= 1
+
+        return PortForwardHandler.connections[id]
+
+    def connection_id(self, pod: str):
+        return f'{self.local_port}:{pod}:{self.target_port}'
+
+def port_forwarding(state: ReplState, local_port: int, svc_or_pod: Callable[[bool],str], target_port: int):
+    return PortForwardHandler(state, local_port, svc_or_pod, target_port)

adam/utils_k8s/kube_context.py

@@ -3,7 +3,7 @@ import re
 from kubernetes import config as kconfig
 
 from adam.config import Config
-from adam.utils import idp_token_from_env, lines_to_tabular, log2
+from adam.utils import idp_token_from_env, log2, tabulize
 
 class KubeContext:
     _in_cluster = False
@@ -56,7 +56,7 @@ class KubeContext:
             log2('Use -v <key>=<value> format.')
             log2()
             lines = [f'{key}\t{Config().get(key, None)}' for key in Config().keys()]
-            log2(lines_to_tabular(lines, separator='\t'))
+            tabulize(lines, separator='\t', to=2)
 
         for p in param_ovrs:
             tokens = p.split('=')
@@ -102,7 +102,4 @@ class KubeContext:
         return name if re.match(r"^(?!pg-).*-k8spg-.*$", name) else None
 
     def show_out(s: bool):
-        return s or Config().is_debug()
-
-    def show_parallelism():
-        return Config().get('debugs.show-parallelism', False)
+        return s or Config().is_debug()

adam/{pod_exec_result.py → utils_k8s/pod_exec_result.py}

@@ -1,6 +1,8 @@
 import yaml
 
-class PodExecResult:
+from adam.utils import ExecResult, log_exc
+
+class PodExecResult(ExecResult):
     # {
     #   'metadata': {},
     #   'status': 'Failure',
@@ -27,9 +29,13 @@ class PodExecResult:
     def exit_code(self) -> int:
         code = 0
 
-        try:
+        with log_exc(False):
             code = self.error['details']['causes'][0]['message']
-        except:
-            pass
 
-        return code
+        return code
+
+    def __str__(self):
+        return f'{"OK" if self.exit_code() == 0 else self.exit_code()} {self.command}'
+
+    def __audit_extra__(self):
+        return self.log_file if self.log_file else None

adam/utils_k8s/pods.py

@@ -1,24 +1,32 @@
 from collections.abc import Callable
-from concurrent.futures import ThreadPoolExecutor, as_completed
 from datetime import datetime
+import os
+import re
+import subprocess
 import sys
 import time
-from typing import TypeVar, cast
+from typing import TypeVar
 from kubernetes import client
 from kubernetes.stream import stream
-from kubernetes.stream.ws_client import ERROR_CHANNEL
+from kubernetes.stream.ws_client import ERROR_CHANNEL, WSClient
 
 from adam.config import Config
+from adam.repl_session import ReplSession
 from adam.utils_k8s.volumes import ConfigMapMount
-from adam.pod_exec_result import PodExecResult
-from adam.utils import elapsed_time, log2, random_alphanumeric
+from adam.utils_k8s.pod_exec_result import PodExecResult
+from adam.utils import GeneratorStream, ParallelMapHandler, log2, debug, log_exc
+from adam.utils_local import local_tmp_dir
 from .kube_context import KubeContext
 
+from websocket._core import WebSocket
+
 T = TypeVar('T')
 _TEST_POD_EXEC_OUTS: PodExecResult = None
 
 # utility collection on pods; methods are all static
 class Pods:
+    _TEST_POD_CLOSE_SOCKET: bool = False
+
     def set_test_pod_exec_outs(outs: PodExecResult):
         global _TEST_POD_EXEC_OUTS
         _TEST_POD_EXEC_OUTS = outs
@@ -26,11 +34,9 @@ class Pods:
         return _TEST_POD_EXEC_OUTS
 
     def delete(pod_name: str, namespace: str, grace_period_seconds: int = None):
-        try:
+        with log_exc(lambda e: "Exception when calling CoreV1Api->delete_namespaced_pod: %s\n" % e):
             v1 = client.CoreV1Api()
-            api_response = v1.delete_namespaced_pod(pod_name, namespace, grace_period_seconds=grace_period_seconds)
-        except Exception as e:
-            log2("Exception when calling CoreV1Api->delete_namespaced_pod: %s\n" % e)
+            v1.delete_namespaced_pod(pod_name, namespace, grace_period_seconds=grace_period_seconds)
 
     def delete_with_selector(namespace: str, label_selector: str, grace_period_seconds: int = None):
         v1 = client.CoreV1Api()
@@ -39,75 +45,61 @@ class Pods:
         for i in ret.items:
             v1.delete_namespaced_pod(name=i.metadata.name, namespace=namespace, grace_period_seconds=grace_period_seconds)
 
-    def on_pods(pods: list[str],
-                namespace: str,
-                body: Callable[[ThreadPoolExecutor, str, str, bool], T],
-                post: Callable[[T], T] = None,
-                action: str = 'action', max_workers=0, show_out=True, on_any = False) -> list[T]:
-        show_out = KubeContext.show_out(show_out)
-
+    def parallelize(collection: list, max_workers: int = 0, samples = sys.maxsize, msg: str = None, action: str = 'action'):
         if not max_workers:
             max_workers = Config().action_workers(action, 0)
-        if not on_any and max_workers > 0:
-            # if parallel, node sampling is suppressed
-            if KubeContext.show_parallelism():
-                log2(f'Executing on all nodes from statefulset in parallel...')
-            start_time = time.time()
-            try:
-                with ThreadPoolExecutor(max_workers=max_workers) as executor:
-                    # disable stdout from the pod_exec, then show the output in a for loop
-                    futures = [body(executor, pod, namespace, show_out) for pod in pods]
-                    if len(futures) == 0:
-                        return cast(list[T], [])
-
-                rs = [future.result() for future in as_completed(futures)]
-                if post:
-                    rs = [post(r, show_out=show_out) for r in rs]
-
-                return rs
-            finally:
-                if KubeContext.show_parallelism():
-                    log2(f"Parallel {action} elapsed time: {elapsed_time(start_time)} with {max_workers} workers")
-        else:
-            results: list[T] = []
-
-            samples = 1 if on_any else Config().action_node_samples(action, sys.maxsize)
-            l = min(len(pods), samples)
-            adj = 'all'
-            if l < len(pods):
-                adj = f'{l} sample'
-            if show_out:
-                log2(f'Executing on {adj} nodes from statefulset...')
-            for pod_name in pods:
-                try:
-                    # disable stdout from the pod_exec, then show the output in a for loop
-                    result = body(None, pod_name, namespace, False)
-                    if post:
-                        result = post(result, show_out=show_out)
-                    results.append(result)
-                    if result:
-                        l -= 1
-                        if not l:
-                            break
-                except Exception as e:
-                    log2(e)
-
-            return results
-
-    def exec(pod_name: str, container: str, namespace: str, command: str, show_out = True, throw_err = False, shell = '/bin/sh',
-             interaction: Callable[[any, list[str]], any] = None):
+        if samples == sys.maxsize:
+            samples = Config().action_node_samples(action, sys.maxsize)
+
+        return ParallelMapHandler(collection, max_workers, samples = samples, msg = msg)
+
+    def exec(pod_name: str,
+             container: str,
+             namespace: str,
+             command: str,
+             show_out = True,
+             throw_err = False,
+             shell = '/bin/sh',
+             backgrounded = False,
+             log_file = None,
+             interaction: Callable[[any, list[str]], any] = None,
+             env_prefix: str = None):
         if _TEST_POD_EXEC_OUTS:
             return _TEST_POD_EXEC_OUTS
 
         show_out = KubeContext.show_out(show_out)
 
+        if (backgrounded or command.endswith(' &')) and Config().get('repl.background-process.via-sh', True):
+            command = command.strip(' &')
+
+            log_all_file = None
+            log_pod_file = None
+            if log_file:
+                log_pod_file = Pods.log_file_from_template(log_file, pod_name=pod_name)
+                if (a := Pods.log_file_from_template(log_file, pod_name='all')) != log_file:
+                    log_all_file = a
+            else:
+                log_pod_file = Pods.log_file(command, pod_name=pod_name)
+
+            command = command.replace('"', '\\"')
+            cmd = f'nohup kubectl exec {pod_name} -c {container} -- {shell} -c "{command} &" > {log_pod_file} 2>&1 &'
+            if log_all_file:
+                cmd = f'{cmd} >> {log_all_file}'
+
+            result = subprocess.run(cmd, capture_output=True, text=True, shell=True)
+
+            return PodExecResult(result.stdout, result.stderr, cmd, None, pod=pod_name, log_file=log_pod_file)
+
         api = client.CoreV1Api()
 
-        log_file = None
         tty = True
         exec_command = [shell, '-c', command]
-        if command.endswith(' &'):
-            # should be false for starting a backgroud process
+        if env_prefix:
+            exec_command = [shell, '-c', f'{env_prefix} {command}']
+
+        if backgrounded or command.endswith(' &'):
+            print('!!!!SEAN backgrounded, but no via-sh!!!!!')
+            # should be false for starting a background process
             tty = False
 
             if Config().get('repl.background-process.auto-nohup', True):
@@ -116,15 +108,17 @@ class Pods:
                 if command.startswith('nodetool '):
                     cmd_name = f".{'_'.join(command.split(' ')[5:])}"
 
-                log_file = f'/tmp/qing-{datetime.now().strftime("%d%H%M%S")}{cmd_name}.log'
+                if not log_file:
+                    log_file = f'{log_prefix()}-{datetime.now().strftime("%d%H%M%S")}{cmd_name}.log'
                 command = f"nohup {command} > {log_file} 2>&1 &"
+                if env_prefix:
+                    command = f'{env_prefix} {command}'
                 exec_command = [shell, '-c', command]
 
         k_command = f'kubectl exec {pod_name} -c {container} -n {namespace} -- {shell} -c "{command}"'
-        if show_out:
-            print(k_command)
+        debug(k_command)
 
-        resp = stream(
+        resp: WSClient = stream(
             api.connect_get_namespaced_pod_exec,
             pod_name,
             namespace,
@@ -137,6 +131,7 @@ class Pods:
             _preload_content=False,
         )
 
+        s: WebSocket = resp.sock
         stdout = []
         stderr = []
         error_output = None
@@ -155,11 +150,9 @@ class Pods:
                     stderr.append(frag)
                     if show_out: print(frag, end="")
 
-            try:
+            with log_exc():
                 # get the exit code from server
                 error_output = resp.read_channel(ERROR_CHANNEL)
-            except Exception as e:
-                pass
         except Exception as e:
             if throw_err:
                 raise e
@@ -167,9 +160,84 @@ class Pods:
                 log2(e)
         finally:
             resp.close()
+            if s and s.sock and Pods._TEST_POD_CLOSE_SOCKET:
+                with log_exc():
+                    s.sock.close()
 
         return PodExecResult("".join(stdout), "".join(stderr), k_command, error_output, pod=pod_name, log_file=log_file)
 
+    def log_file(command: str, pod_name: str = None, dt: datetime = None):
+        cmd_name = ''
+        if command.startswith('nodetool '):
+            command = command.strip(' &')
+            cmd_name = f".{'_'.join(command.split(' ')[5:])}"
+
+        pod_suffix = '{pod}'
+        if pod_name:
+            pod_suffix = pod_name
+            if groups := re.match(r'.*-(.*)', pod_name):
+                pod_suffix = f'-{groups[1]}'
+
+        if not dt:
+            dt = datetime.now()
+
+        return f'{log_prefix()}-{dt.strftime("%d%H%M%S")}{cmd_name}{pod_suffix}.log'
+
+    def log_file_from_template(log_file: str, pod_name: str):
+        pod_suffix = pod_name
+        if pod_name and (groups := re.match(r'.*-(.*)', pod_name)):
+            pod_suffix = f'-{groups[1]}'
+
+        if not pod_suffix.startswith('-'):
+            pod_suffix = f'-{pod_suffix}'
+
+        return log_file.replace('{pod}', pod_suffix)
+
+    def read_file(pod_name: str, container: str, namespace: str, file_path: str):
+        v1 = client.CoreV1Api()
+
+        resp = stream(
+            v1.connect_get_namespaced_pod_exec,
+            name=pod_name,
+            namespace=namespace,
+            container=container,
+            command=["cat", file_path],
+            stderr=True, stdin=False,
+            stdout=True, tty=False,
+            _preload_content=False, # Important for streaming
+        )
+
+        s: WebSocket = resp.sock
+        try:
+            while resp.is_open():
+                resp.update(timeout=1)
+                if resp.peek_stdout():
+                    yield resp.read_stdout()
+
+            with log_exc():
+                # get the exit code from server
+                error_output = resp.read_channel(ERROR_CHANNEL)
+        except Exception as e:
+            raise e
+        finally:
+            resp.close()
+            if s and s.sock and Pods._TEST_POD_CLOSE_SOCKET:
+                with log_exc():
+                    s.sock.close()
+
+    def download_file(pod_name: str, container: str, namespace: str, from_path: str, to_path: str = None):
+        if not to_path:
+            to_path = f'{local_tmp_dir()}/{os.path.basename(from_path)}'
+
+        bytes = Pods.read_file(pod_name, container, namespace, from_path)
+        with open(to_path, 'wb') as f:
+            for item in GeneratorStream(bytes):
+                f.write(item)
+
+        ReplSession().append_history(f':sh cat {to_path}')
+
+        return to_path
+
     def get_container(namespace: str, pod_name: str, container_name: str):
         pod = Pods.get(namespace, pod_name)
         if not pod:
@@ -278,4 +346,7 @@ class Pods:
             log2(' Timed Out')
 
     def completed(namespace: str, pod_name: str):
-        return Pods.get(namespace, pod_name).status.phase in ['Succeeded', 'Failed']
+        return Pods.get(namespace, pod_name).status.phase in ['Succeeded', 'Failed']
+
+def log_prefix():
+    return Config().get('log-prefix', '/tmp/qing')

adam/utils_k8s/secrets.py

@@ -6,13 +6,13 @@ from kubernetes import client
 from kubernetes.client import V1Secret
 
 from adam.config import Config
-from adam.utils import log2
+from adam.utils import log2, wait_log
 
 # utility collection on secrets; methods are all static
 class Secrets:
     @functools.lru_cache()
     def list_secrets(namespace: str = None, name_pattern: str = None):
-        Config().wait_log('Inspecting Cassandra Instances...')
+        wait_log('Inspecting Cassandra Instances...')
 
         secrets_names = []
 
@@ -39,14 +39,14 @@ class Secrets:
 
         return secrets_names
 
-    def get_user_pass(ss_name: str, namespace: str, secret_path: str = 'cql.secret'):
+    def get_user_pass(sts_or_pod_name: str, namespace: str, secret_path: str = 'cql.secret'):
         # cs-d0767a536f-cs-d0767a536f-default-sts ->
         # cs-d0767a536f-superuser
         # cs-d0767a536f-reaper-ui
         user = 'superuser'
         if secret_path == 'reaper.secret':
             user = 'reaper-ui'
-        groups = re.match(Config().get(f'{secret_path}.cluster-regex', r'(.*?-.*?)-.*'), ss_name)
+        groups = re.match(Config().get(f'{secret_path}.cluster-regex', r'(.*?-.*?)-.*'), sts_or_pod_name)
         secret_name = Config().get(f'{secret_path}.name', '{cluster}-' + user).replace('{cluster}', groups[1], 1)
 
         secret = Secrets.get_data(namespace, secret_name)

adam/utils_k8s/service_accounts.py

@@ -1,6 +1,7 @@
 from kubernetes import client, config
 
 from adam.config import Config
+from adam.utils import debug
 
 # utility collection on service accounts; methods are all static
 class ServiceAccounts:
@@ -37,7 +38,7 @@ class ServiceAccounts:
             namespace=namespace,
             body=service_account
         )
-        Config().debug(f"Service Account '{api_response.metadata.name}' created in namespace '{namespace}'.")
+        debug(f"Service Account '{api_response.metadata.name}' created in namespace '{namespace}'.")
 
     def delete_service_account(namespace: str, label_selector: str) -> list:
         refs = []
@@ -45,7 +46,7 @@ class ServiceAccounts:
         v1 = client.CoreV1Api()
         sas = v1.list_namespaced_service_account(namespace=namespace, label_selector=label_selector).items
         for sa in sas:
-            Config().debug(f'delete {sa.metadata.name}')
+            debug(f'delete {sa.metadata.name}')
             v1.delete_namespaced_service_account(name=sa.metadata.name, namespace=namespace)
             refs.append(sa)
 
@@ -102,7 +103,7 @@ class ServiceAccounts:
         v1_rbac = client.RbacAuthorizationV1Api()
         cluster_role_bindings = v1_rbac.list_namespaced_role_binding(namespace=namespace, label_selector=label_selector).items
         for binding in cluster_role_bindings:
-            Config().debug(f'delete {binding.metadata.name}')
+            debug(f'delete {binding.metadata.name}')
             v1_rbac.delete_namespaced_role_binding(name=binding.metadata.name, namespace=namespace)
             refs.append(binding)
 
@@ -162,7 +163,7 @@ class ServiceAccounts:
         v1_rbac = client.RbacAuthorizationV1Api()
         cluster_role_bindings = v1_rbac.list_cluster_role_binding(label_selector=label_selector).items
         for binding in cluster_role_bindings:
-            Config().debug(f'delete {binding.metadata.name}')
+            debug(f'delete {binding.metadata.name}')
             v1_rbac.delete_cluster_role_binding(binding.metadata.name)
             refs.append(binding)
 

adam/utils_k8s/services.py

@@ -2,7 +2,7 @@ from typing import List
 from kubernetes import client
 
 from adam.config import Config
-from adam.utils import log2
+from adam.utils import debug, log2
 
 from .kube_context import KubeContext
 
@@ -71,7 +71,7 @@ class Services:
                     namespace=namespace,
                     body=delete_options
                 )
-                Config().debug(f"200 Service '{name}' in namespace '{namespace}' deleted successfully.")
+                debug(f"200 Service '{name}' in namespace '{namespace}' deleted successfully.")
         except client.ApiException as e:
             log2(f"Error deleting Service '{name}': {e}")
         except Exception as e:

adam/utils_k8s/statefulsets.py

@@ -1,12 +1,9 @@
-from collections.abc import Callable
-from concurrent.futures import ThreadPoolExecutor
 from datetime import datetime
 import functools
 import re
 from typing import List, TypeVar, cast
 from kubernetes import client
 
-from .pods import Pods
 from .kube_context import KubeContext
 from adam.utils import log2
 
@@ -58,18 +55,12 @@ class StatefulSets:
 
         return statefulset_pods
 
-    def on_cluster(statefulset: str,
-                namespace: str,
-                body: Callable[[ThreadPoolExecutor, str, str, bool], T],
-                post: Callable[[T], T] = None,
-                action: str = 'action', max_workers=0, show_out=True, on_any = False) -> list[T]:
-        pods = StatefulSets.pod_names(statefulset, namespace)
-
-        return Pods.on_pods(pods, namespace, body, post=post, action=action, max_workers=max_workers, show_out=show_out, on_any=on_any)
-
     @functools.lru_cache()
-    def pod_names(ss: str, ns: str):
-        return [pod.metadata.name for pod in StatefulSets.pods(ss, ns)]
+    def pod_names(sts: str, ns: str):
+        if not sts:
+            return []
+
+        return [pod.metadata.name for pod in StatefulSets.pods(sts, ns)]
 
     def restarted_at(ss: str, ns: str):
         # returns timestamp and if being rolled out
@@ -92,6 +83,7 @@ class StatefulSets:
 
         return restarted, False
 
+    @functools.lru_cache()
     def get_datacenter(sts: str, ns: str) -> str:
         v1 = client.AppsV1Api()
         namespace = ns