kaqing 1.98.139__tar.gz → 1.98.140__tar.gz

{kaqing-1.98.139 → kaqing-1.98.140}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: kaqing
-Version: 1.98.139
+Version: 1.98.140
 Summary: UNKNOWN
 Home-page: UNKNOWN
 License: UNKNOWN

{kaqing-1.98.139 → kaqing-1.98.140}/adam/commands/deploy/deploy.py

@@ -1,7 +1,6 @@
 import click
 
 from adam.commands.command import Command
-from adam.commands.command_helpers import ClusterCommandHelper
 from adam.commands.deploy.deploy_pod import DeployPod
 from .deploy_frontend import DeployFrontend
 from adam.repl_state import ReplState

{kaqing-1.98.139 → kaqing-1.98.140}/adam/commands/deploy/deploy_frontend.py

@@ -1,9 +1,6 @@
-from adam.app_session import AppSession
 from adam.commands.command import Command
 from adam.commands.deploy.deploy_utils import deploy_frontend
 from adam.config import Config
-from adam.k8s_utils.ingresses import Ingresses
-from adam.k8s_utils.services import Services
 from adam.repl_state import ReplState, RequiredState
 from adam.utils import log2
 

{kaqing-1.98.139 → kaqing-1.98.140}/adam/commands/deploy/deploy_pod.py

@@ -3,11 +3,10 @@ from kubernetes import client
 from adam.commands.command import Command
 from adam.commands.deploy.deploy_utils import deploy_frontend, gen_labels
 from adam.config import Config
-from adam.k8s_utils.ingresses import Ingresses
+from adam.k8s_utils.deployment import Deployments
 from adam.k8s_utils.kube_context import KubeContext
 from adam.k8s_utils.pods import Pods
 from adam.k8s_utils.service_accounts import ServiceAccounts
-from adam.k8s_utils.services import Services
 from adam.repl_state import ReplState, RequiredState
 from adam.utils import log2
 
@@ -55,11 +54,12 @@ class DeployPod(Command):
                 add=["SYS_PTRACE"]
             )
         )
-        Pods.create(state.namespace, pod_name, image, env={'NAMESPACE': state.namespace}, container_security_context=security_context, labels=labels, sa_name=sa_name)
+        Deployments.create(state.namespace, pod_name, image, env={'NAMESPACE': state.namespace}, container_security_context=security_context, labels=labels, sa_name=sa_name)
 
         uri = deploy_frontend(pod_name, state.namespace, label_selector)
 
-        Pods.wait_for_running(state.namespace, pod_name, msg=f'Ops pod is starting up; it will be available at {uri}.')
+        # Pods.wait_for_running(state.namespace, label_selector, msg=f'Ops pod is starting up; it will be available at {uri}.')
+        Pods.wait_for_running(state.namespace, pod_name, msg=f'Ops pod is starting up; it will be available at {uri}.', label_selector=label_selector)
 
         return state
 

{kaqing-1.98.139 → kaqing-1.98.140}/adam/commands/deploy/deploy_utils.py

@@ -1,14 +1,12 @@
 from adam.app_session import AppSession
 from adam.k8s_utils.ingresses import Ingresses
 from adam.k8s_utils.services import Services
-from adam.utils import log2
 
 def deploy_frontend(name: str, namespace: str, label_selector: str):
     app_session: AppSession = AppSession.create('c3', 'c3', namespace)
     port = 7678
     labels = gen_labels(label_selector)
     Services.create_service(name, namespace, port, labels, labels=labels)
-    # Services.create_service(name, namespace, port, {"run": "ops"})
     Ingresses.create_ingress(name, namespace, app_session.host, '/c3/c3/ops($|/)', port, annotations={
         'kubernetes.io/ingress.class': 'nginx',
         'nginx.ingress.kubernetes.io/use-regex': 'true',

{kaqing-1.98.139 → kaqing-1.98.140}/adam/commands/deploy/undeploy.py

@@ -1,10 +1,8 @@
 import click
 
 from adam.commands.command import Command
-from adam.commands.command_helpers import ClusterCommandHelper
 from adam.commands.deploy.undeploy_frontend import UndeployFrontend
 from adam.commands.deploy.undeploy_pod import UndeployPod
-from .deploy_frontend import DeployFrontend
 from adam.repl_state import ReplState
 from adam.utils import lines_to_tabular, log, log2
 

{kaqing-1.98.139 → kaqing-1.98.140}/adam/commands/deploy/undeploy_frontend.py

@@ -1,8 +1,6 @@
 from adam.commands.command import Command
 from adam.commands.deploy.deploy_utils import undeploy_frontend
 from adam.config import Config
-from adam.k8s_utils.ingresses import Ingresses
-from adam.k8s_utils.services import Services
 from adam.repl_state import ReplState, RequiredState
 
 class UndeployFrontend(Command):

{kaqing-1.98.139 → kaqing-1.98.140}/adam/commands/deploy/undeploy_pod.py

@@ -1,12 +1,10 @@
-from adam.app_session import AppSession
 from adam.commands.command import Command
-from adam.commands.deploy.deploy_utils import deploy_frontend, undeploy_frontend
+from adam.commands.deploy.deploy_utils import undeploy_frontend
 from adam.config import Config
-from adam.k8s_utils.ingresses import Ingresses
+from adam.k8s_utils.deployment import Deployments
 from adam.k8s_utils.kube_context import KubeContext
 from adam.k8s_utils.pods import Pods
 from adam.k8s_utils.service_accounts import ServiceAccounts
-from adam.k8s_utils.services import Services
 from adam.repl_state import ReplState, RequiredState
 from adam.utils import log2
 
@@ -41,9 +39,11 @@ class UndeployPod(Command):
             return state
 
         label_selector = Config().get('pod.label-selector', 'run=ops')
-        undeploy_frontend(state.namespace, label_selector)
-        Pods.delete_with_selector(state.namespace, label_selector, grace_period_seconds=0)
         ServiceAccounts.delete(state.namespace, label_selector=label_selector)
+        Deployments.delete_with_selector(state.namespace, label_selector, grace_period_seconds=0)
+        # instantly destroy the pod
+        Pods.delete_with_selector(state.namespace, label_selector, grace_period_seconds=0)
+        undeploy_frontend(state.namespace, label_selector)
 
         return state
 

kaqing-1.98.140/adam/k8s_utils/deployment.py

@@ -0,0 +1,219 @@
+from collections.abc import Callable
+from concurrent.futures import ThreadPoolExecutor, as_completed
+import sys
+import time
+from typing import TypeVar, cast
+from kubernetes import client
+from kubernetes.stream import stream
+from kubernetes.stream.ws_client import ERROR_CHANNEL
+
+from adam.config import Config
+from adam.k8s_utils.pods import Pods
+from adam.pod_exec_result import PodExecResult
+from adam.utils import elapsed_time, log2
+from .kube_context import KubeContext
+
+T = TypeVar('T')
+_TEST_POD_EXEC_OUTS: PodExecResult = None
+
+# utility collection on deployments; methods are all static
+class Deployments:
+    def set_test_pod_exec_outs(outs: PodExecResult):
+        global _TEST_POD_EXEC_OUTS
+        _TEST_POD_EXEC_OUTS = outs
+
+        return _TEST_POD_EXEC_OUTS
+
+    def delete(pod_name: str, namespace: str):
+        try:
+            v1 = client.CoreV1Api()
+            api_response = v1.delete_namespaced_pod(pod_name, namespace)
+        except Exception as e:
+            log2("Exception when calling CoreV1Api->delete_namespaced_pod: %s\n" % e)
+
+    def delete_with_selector(namespace: str, label_selector: str, grace_period_seconds: int = None):
+        v1 = client.AppsV1Api()
+
+        ret = v1.list_namespaced_deployment(namespace=namespace, label_selector=label_selector)
+        for i in ret.items:
+            v1.delete_namespaced_deployment(name=i.metadata.name, namespace=namespace, grace_period_seconds=grace_period_seconds)
+
+    def on_pods(pods: list[str],
+                namespace: str,
+                body: Callable[[ThreadPoolExecutor, str, str, bool], T],
+                post: Callable[[T], T] = None,
+                action: str = 'action', max_workers=0, show_out=True) -> list[T]:
+        show_out = KubeContext.show_out(show_out)
+
+        if not max_workers:
+            max_workers = Config().action_workers(action, 0)
+        if max_workers > 0:
+            # if parallel, node sampling is suppressed
+            if KubeContext.show_parallelism():
+                log2(f'Executing on all nodes from statefulset in parallel...')
+            start_time = time.time()
+            try:
+                with ThreadPoolExecutor(max_workers=max_workers) as executor:
+                    # disable stdout from the pod_exec, then show the output in a for loop
+                    futures = [body(executor, pod, namespace, show_out) for pod in pods]
+                    if len(futures) == 0:
+                        return cast(list[T], [])
+
+                rs = [future.result() for future in as_completed(futures)]
+                if post:
+                    rs = [post(r, show_out=show_out) for r in rs]
+
+                return rs
+            finally:
+                if KubeContext.show_parallelism():
+                    log2(f"Parallel {action} elapsed time: {elapsed_time(start_time)} with {max_workers} workers")
+        else:
+            results: list[T] = []
+
+            samples = Config().action_node_samples(action, sys.maxsize)
+            l = min(len(pods), samples)
+            adj = 'all'
+            if l < len(pods):
+                adj = f'{l} sample'
+            if show_out:
+                log2(f'Executing on {adj} nodes from statefulset...')
+            for pod_name in pods:
+                try:
+                    result = body(None, pod_name, namespace, show_out)
+                    if post:
+                        result = post(result, show_out=show_out)
+                    results.append(result)
+                    if result:
+                        l -= 1
+                        if not l:
+                            break
+                except Exception as e:
+                    log2(e)
+
+            return results
+
+    def exec(pod_name: str, container: str, namespace: str, command: str, show_out = True, throw_err = False, interaction: Callable[[any, list[str]], any] = None):
+        if _TEST_POD_EXEC_OUTS:
+            return _TEST_POD_EXEC_OUTS
+
+        show_out = KubeContext.show_out(show_out)
+
+        api = client.CoreV1Api()
+
+        exec_command = ["/bin/sh", "-c", command]
+        k_command = f'kubectl exec {pod_name} -c {container} -n {namespace} -- {command}'
+        if show_out:
+            print(k_command)
+
+        resp = stream(
+            api.connect_get_namespaced_pod_exec,
+            pod_name,
+            namespace,
+            command=exec_command,
+            container=container,
+            stderr=True,
+            stdin=True,
+            stdout=True,
+            tty=True,
+            _preload_content=False,
+        )
+
+        stdout = []
+        stderr = []
+        error_output = None
+        try:
+            while resp.is_open():
+                resp.update(timeout=1)
+                if resp.peek_stdout():
+                    frag = resp.read_stdout()
+                    stdout.append(frag)
+                    if show_out: print(frag, end="")
+
+                    if interaction:
+                        interaction(resp, stdout)
+                if resp.peek_stderr():
+                    frag = resp.read_stderr()
+                    stderr.append(frag)
+                    if show_out: print(frag, end="")
+
+            try:
+                # get the exit code from server
+                error_output = resp.read_channel(ERROR_CHANNEL)
+            except Exception:
+                pass
+        except Exception as e:
+            if throw_err:
+                raise e
+            else:
+                log2(e)
+        finally:
+            resp.close()
+
+        return PodExecResult("".join(stdout), "".join(stderr), k_command, error_output)
+
+    def get_container(namespace: str, pod_name: str, container_name: str):
+        pod = Pods.get(namespace, pod_name)
+        if not pod:
+            return None
+
+        for container in pod.spec.containers:
+            if container_name == container.name:
+                return container
+
+        return None
+
+    def get(namespace: str, pod_name: str):
+        v1 = client.CoreV1Api()
+        return v1.read_namespaced_pod(name=pod_name, namespace=namespace)
+
+    def create_deployment_spec(name: str, image: str, image_pull_secret: str,
+                               envs: list, container_security_context: client.V1SecurityContext,
+                               volume_name: str, pvc_name:str, mount_path:str,
+                               command: list[str]=None, sa_name=None, labels: dict[str, str] = {}):
+        return client.V1DeploymentSpec(
+            replicas=1,
+            selector=client.V1LabelSelector(match_labels=labels),
+            template=client.V1PodTemplateSpec(
+                metadata=client.V1ObjectMeta(labels=labels),
+                spec=Pods.create_pod_spec(name, image, image_pull_secret, envs, container_security_context,
+                                          volume_name, pvc_name, mount_path, command=command, sa_name=sa_name,
+                                          restart_policy="Always"),
+            ),
+        )
+
+    def create(namespace: str, deployment_name: str, image: str,
+               command: list[str] = None,
+               secret: str = None,
+               env: dict[str, any] = {},
+               container_security_context: client.V1SecurityContext = None,
+               labels: dict[str, str] = {},
+               volume_name: str = None,
+               pvc_name: str = None,
+               mount_path: str = None,
+               sa_name=None):
+        v1 = client.AppsV1Api()
+        envs = []
+        for k, v in env.items():
+            envs.append(client.V1EnvVar(name=str(k), value=str(v)))
+        deployment = Deployments.create_deployment_spec(deployment_name, image, secret, envs, container_security_context, volume_name, pvc_name, mount_path, command=command, sa_name=sa_name, labels=labels)
+        return v1.create_namespaced_deployment(
+            namespace=namespace,
+            body=client.V1Deployment(spec=deployment, metadata=client.V1ObjectMeta(
+                name=deployment_name,
+                labels=labels
+            ))
+        )
+
+    def wait_for_running(namespace: str, pod_name: str, msg: str=None):
+        msged = False
+
+        while Pods.get(namespace, pod_name).status.phase != 'Running':
+            if not msged:
+                if not msg:
+                    msg = f'Waiting for the {pod_name} pod to start up...'
+                log2(msg)
+                msged = True
+            time.sleep(5)
+
+    def completed(namespace: str, pod_name: str):
+        return Pods.get(namespace, pod_name).status.phase in ['Succeeded', 'Failed']

{kaqing-1.98.139 → kaqing-1.98.140}/adam/k8s_utils/pods.py

@@ -165,7 +165,17 @@ class Pods:
         v1 = client.CoreV1Api()
         return v1.read_namespaced_pod(name=pod_name, namespace=namespace)
 
-    def create_pod_spec(name: str, image: str, image_pull_secret: str, envs: list, container_security_context: client.V1SecurityContext, volume_name: str, pvc_name:str, mount_path:str, command: list[str]=None, sa_name=None):
+    def get_with_selector(namespace: str, label_selector: str):
+        v1 = client.CoreV1Api()
+
+        ret = v1.list_namespaced_pod(namespace=namespace, label_selector=label_selector)
+        for i in ret.items:
+            return v1.read_namespaced_pod(name=i.metadata.name, namespace=namespace)
+
+    def create_pod_spec(name: str, image: str, image_pull_secret: str,
+                        envs: list, container_security_context: client.V1SecurityContext,
+                        volume_name: str, pvc_name:str, mount_path:str,
+                        command: list[str]=None, sa_name=None, restart_policy="Never"):
         volume_mounts = []
         if volume_name and pvc_name and mount_path:
             volume_mounts=[client.V1VolumeMount(mount_path=mount_path, name=volume_name)]
@@ -182,7 +192,7 @@ class Pods:
             security_context=client.V1PodSecurityContext(run_as_user=1001, run_as_group=1001, fs_group=1001)
 
         return client.V1PodSpec(
-            restart_policy="Never",
+            restart_policy=restart_policy,
             containers=[container],
             image_pull_secrets=[client.V1LocalObjectReference(name=image_pull_secret)],
             security_context=security_context,
@@ -213,10 +223,10 @@ class Pods:
             ))
         )
 
-    def wait_for_running(namespace: str, pod_name: str, msg: str=None):
+    def wait_for_running(namespace: str, pod_name: str, msg: str=None, label_selector: str = None):
         msged = False
 
-        while Pods.get(namespace, pod_name).status.phase != 'Running':
+        while (Pods.get_with_selector(namespace, label_selector) if label_selector else Pods.get(namespace, pod_name)).status.phase != 'Running':
             if not msged:
                 if not msg:
                     msg = f'Waiting for the {pod_name} pod to start up...'

kaqing-1.98.140/adam/version.py

@@ -0,0 +1,5 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+__version__ = "1.98.140"  #: the working version
+__release__ = "1.0.0"  #: the release version

{kaqing-1.98.139 → kaqing-1.98.140}/kaqing.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: kaqing
-Version: 1.98.139
+Version: 1.98.140
 Summary: UNKNOWN
 Home-page: UNKNOWN
 License: UNKNOWN

{kaqing-1.98.139 → kaqing-1.98.140}/kaqing.egg-info/SOURCES.txt

@@ -139,6 +139,7 @@ adam/k8s_utils/__init__.py
 adam/k8s_utils/cassandra_clusters.py
 adam/k8s_utils/cassandra_nodes.py
 adam/k8s_utils/custom_resources.py
+adam/k8s_utils/deployment.py
 adam/k8s_utils/ingresses.py
 adam/k8s_utils/jobs.py
 adam/k8s_utils/kube_context.py

{kaqing-1.98.139 → kaqing-1.98.140}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='kaqing',
-    version='1.98.139',
+    version='1.98.140',
     packages=find_packages(),
     entry_points={
         'console_scripts': [

kaqing-1.98.139/adam/version.py

@@ -1,5 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-__version__ = "1.98.139"  #: the working version
-__release__ = "1.0.0"  #: the release version