outerbounds 0.3.180rc5__py3-none-any.whl → 0.3.182rc0__py3-none-any.whl

outerbounds/apps/_state_machine.py

@@ -119,7 +119,7 @@ class _capsuleDeployerStateMachine:
         return self.to_dot()
 
     def to_diagraph(self):
-        from graphviz import Digraph
+        from graphviz import Digraph  # type: ignore
 
         # Create a new Digraph
         dot = Digraph(name="StateMachine", format="png")
@@ -164,13 +164,23 @@ class CurrentWorkerInfo(TypedDict):
 
 class DEPLOYMENT_READY_CONDITIONS:
     """
-    Deployment ready conditions define what is considered a successful completion of a deployment.
+    Deployment ready conditions define what is considered a successful completion of the current deployment instance.
     This allows users or platform designers to configure the criteria for deployment readiness.
 
-    Reasons for different deployment modes include:
-    1) [at_least_one_running] Some endpoints may be deployed ephemerally and are considered ready when at least one instance is running; additional instances are for load management.
-    2) [all_running] Operators may require that all replicas are available, running, and only the current deployment version is serving traffic.
-    3) [fully_finished] Operators may only care that the minimum number of replicas for the current rollout are present in the cluster.
+    Why do we need deployment readiness conditions?
+    - Deployments might be taking place from a CI/CD esq environment, In these setups, the downstream build triggers might be depending on a specific criteria for deployment completion. Having readiness conditions allows the CI/CD systems to get a signal of when the deployment is ready.
+    - Users might be calling the deployment API under different conditions:
+        - Some users might want a cluster of workers ready before serving traffic while others might want just one worker ready to start serving traffic.
+
+    Some readiness conditions include:
+    1) [at_least_one_running] Atleast min(min_replicas, 1) workers of the current deployment instance's version have started running.
+        - Usecase: Some endpoints may be deployed ephemerally and are considered ready when at least one instance is running; additional instances are for load management.
+    2) [all_running] Atleast min_replicas number of workers are running for the deployment to be considered ready.
+        - Usecase: Operators may require that all replicas are available before traffic is routed. Needed when inference endpoints maybe under some SLA or require a larger load
+    3) [fully_finished] Atleast min_replicas number of workers are running for the deployment and there are no pending or crashlooping workers from previous versions lying around.
+        - Usecase: Ensuring endpoint is fully available and no other versions are running.
+    4) [async] The deployment will be assumed ready when the `upgradeInProgress` flag changes from `True` to `False`.
+        - Usecase: Operators may only care that the URL is minted for the deployment and the deployment might also be setup with 0 min workers.
     """
 
     # `ATLEAST_ONE_RUNNING` implies that atleast one worker of the current deployment instance's version has started running.
@@ -183,6 +193,9 @@ class DEPLOYMENT_READY_CONDITIONS:
     # `FULLY_FINISHED` implies that the deployment has the minimum number of replicas and all the workers are related to the current deployment instance's version.
     FULLY_FINISHED = "fully_finished"
 
+    # `ASYNC` implies that the deployment will be assumed ready after the URL is minted and the worker statuses are not checked.
+    ASYNC = "async"
+
     @classmethod
     def docstring(cls):
         return cls.__doc__
@@ -304,9 +317,11 @@ def _capsule_worker_semantic_status(
         for w in _workers:
             if w.get("phase") != phase:
                 continue
-            if w.get("version") not in xx:
-                xx[w.get("version")] = []
-            xx[w.get("version")].append(w)
+            worker_version = w.get("version")
+            if worker_version is not None:
+                if worker_version not in xx:
+                    xx[worker_version] = []
+                xx[worker_version].append(w)
         return xx
 
     pending_workers = _make_version_dict(workers, "Pending")

outerbounds/apps/app_cli.py

@@ -20,7 +20,7 @@ from typing import Dict, List, Any, Optional, Union
 # end up creating issues with click in general. So we need to figure a
 # way to figure the right import of click dynamically. a neat way to handle that would be
 # to have a function that can import the correct click based on the context in which stuff is being loaded.
-from metaflow._vendor import click
+from .click_importer import click
 from outerbounds._vendor import yaml
 from outerbounds.utils import metaflowconfig
 from .app_config import (
@@ -33,9 +33,6 @@ from .app_config import (
 from .perimeters import PerimeterExtractor
 from .cli_to_config import build_config_from_options
 from .utils import (
-    CommaSeparatedListType,
-    KVPairType,
-    KVDictType,
     MultiStepSpinner,
 )
 from . import experimental
@@ -56,6 +53,171 @@ import uuid
 from datetime import datetime
 
 
+class KeyValueDictPair(click.ParamType):
+    name = "KV-DICT-PAIR"  # type: ignore
+
+    def convert(self, value, param, ctx):
+        # Parse a string of the form KEY=VALUE into a dict {KEY: VALUE}
+        if len(value.split("=", 1)) != 2:
+            self.fail(
+                f"Invalid format for {value}. Expected format: KEY=VALUE", param, ctx
+            )
+
+        key, _value = value.split("=", 1)
+        try:
+            return {"key": key, "value": json.loads(_value)}
+        except json.JSONDecodeError:
+            return {"key": key, "value": _value}
+        except Exception as e:
+            self.fail(f"Invalid value for {value}. Error: {e}", param, ctx)
+
+    def __str__(self):
+        return repr(self)
+
+    def __repr__(self):
+        return "KV-PAIR"
+
+
+class KeyValuePair(click.ParamType):
+    name = "KV-PAIR"  # type: ignore
+
+    def convert(self, value, param, ctx):
+        # Parse a string of the form KEY=VALUE into a dict {KEY: VALUE}
+        if len(value.split("=", 1)) != 2:
+            self.fail(
+                f"Invalid format for {value}. Expected format: KEY=VALUE", param, ctx
+            )
+
+        key, _value = value.split("=", 1)
+        try:
+            return {key: json.loads(_value)}
+        except json.JSONDecodeError:
+            return {key: _value}
+        except Exception as e:
+            self.fail(f"Invalid value for {value}. Error: {e}", param, ctx)
+
+    def __str__(self):
+        return repr(self)
+
+    def __repr__(self):
+        return "KV-PAIR"
+
+
+class MountMetaflowArtifact(click.ParamType):
+    name = "MOUNT-METAFLOW-ARTIFACT"  # type: ignore
+
+    def convert(self, value, param, ctx):
+        """
+        Convert a string like "flow=MyFlow,artifact=my_model,path=/tmp/abc" or
+        "pathspec=MyFlow/123/foo/345/my_model,path=/tmp/abc" to a dict.
+        """
+        artifact_dict = {}
+        parts = value.split(",")
+
+        for part in parts:
+            if "=" not in part:
+                self.fail(
+                    f"Invalid format in part '{part}'. Expected 'key=value'", param, ctx
+                )
+
+            key, val = part.split("=", 1)
+            artifact_dict[key.strip()] = val.strip()
+
+        # Validate required fields
+        if "pathspec" in artifact_dict:
+            if "path" not in artifact_dict:
+                self.fail(
+                    "When using 'pathspec', you must also specify 'path'", param, ctx
+                )
+
+            # Return as pathspec format
+            return {
+                "pathspec": artifact_dict["pathspec"],
+                "path": artifact_dict["path"],
+            }
+        elif (
+            "flow" in artifact_dict
+            and "artifact" in artifact_dict
+            and "path" in artifact_dict
+        ):
+            # Return as flow/artifact format
+            result = {
+                "flow": artifact_dict["flow"],
+                "artifact": artifact_dict["artifact"],
+                "path": artifact_dict["path"],
+            }
+
+            # Add optional namespace if provided
+            if "namespace" in artifact_dict:
+                result["namespace"] = artifact_dict["namespace"]
+
+            return result
+        else:
+            self.fail(
+                "Invalid format. Must be either 'flow=X,artifact=Y,path=Z' or 'pathspec=X,path=Z'",
+                param,
+                ctx,
+            )
+
+    def __str__(self):
+        return repr(self)
+
+    def __repr__(self):
+        return "MOUNT-METAFLOW-ARTIFACT"
+
+
+class MountSecret(click.ParamType):
+    name = "MOUNT-SECRET"  # type: ignore
+
+    def convert(self, value, param, ctx):
+        """
+        Convert a string like "id=my_secret,path=/tmp/secret" to a dict.
+        """
+        secret_dict = {}
+        parts = value.split(",")
+
+        for part in parts:
+            if "=" not in part:
+                self.fail(
+                    f"Invalid format in part '{part}'. Expected 'key=value'", param, ctx
+                )
+
+            key, val = part.split("=", 1)
+            secret_dict[key.strip()] = val.strip()
+
+        # Validate required fields
+        if "id" in secret_dict and "path" in secret_dict:
+            return {"id": secret_dict["id"], "path": secret_dict["path"]}
+        else:
+            self.fail("Invalid format. Must be 'key=X,path=Y'", param, ctx)
+
+    def __str__(self):
+        return repr(self)
+
+    def __repr__(self):
+        return "MOUNT-SECRET"
+
+
+class CommaSeparatedList(click.ParamType):
+    name = "COMMA-SEPARATED-LIST"  # type: ignore
+
+    def convert(self, value, param, ctx):
+        return value.split(",")
+
+    def __str__(self):
+        return repr(self)
+
+    def __repr__(self):
+        return "COMMA-SEPARATED-LIST"
+
+
+KVPairType = KeyValuePair()
+MetaflowArtifactType = MountMetaflowArtifact()
+SecretMountType = MountSecret()
+CommaSeparatedListType = CommaSeparatedList()
+KVDictType = KeyValueDictPair()
+
+
 class ColorTheme:
     TIMESTAMP = "magenta"
     LOADING_COLOR = "cyan"

outerbounds/apps/app_config.py

@@ -40,7 +40,7 @@ class AppConfig:
         """Initialize configuration from a dictionary."""
         self.config = config_dict or {}
         self.schema = self._load_schema()
-        self._final_state = {}
+        self._final_state: Dict[str, Any] = {}
 
     def set_state(self, key, value):
         self._final_state[key] = value

outerbounds/apps/capsule.py

@@ -4,7 +4,7 @@ import pathlib
 import requests
 import time
 import shlex
-from typing import Optional, List, Dict, Any, Tuple
+from typing import Optional, List, Dict, Any, Tuple, Union
 from .utils import TODOException, safe_requests_wrapper, MaximumRetriesExceeded
 from .app_config import AppConfig, CAPSULE_DEBUG, AuthType
 from . import experimental
@@ -53,7 +53,7 @@ class CapsuleStateMachine:
 
     def __init__(self, capsule_id: str, current_deployment_instance_version: str):
         self._capsule_id = capsule_id
-        self._status_trail = []
+        self._status_trail: List[Dict[str, Any]] = []
         self._current_deployment_instance_version = current_deployment_instance_version
 
     def get_status_trail(self):
@@ -127,7 +127,7 @@ class CapsuleWorkersStateMachine:
         self._end_state_capsule_version = end_state_capsule_version
         self._deployment_mode = deployment_mode
         self._minimum_replicas = minimum_replicas
-        self._status_trail = []
+        self._status_trail: List[Dict[str, Union[float, List[WorkerStatus]]]] = []
 
     def get_status_trail(self):
         return self._status_trail
@@ -187,7 +187,7 @@ class CapsuleWorkersStateMachine:
 
     @property
     def current_status(self) -> List[WorkerStatus]:
-        return self._status_trail[-1].get("status")
+        return self._status_trail[-1].get("status")  # type: ignore
 
     def current_version_deployment_status(self) -> CapsuleWorkerSemanticStatus:
         return _capsule_worker_semantic_status(
@@ -663,7 +663,7 @@ class CapsuleDeployer:
         """
         if capsule_response.get("version", None) != current_deployment_instance_version:
             raise CapsuleDeploymentException(
-                self.identifier,
+                self.identifier,  # type: ignore
                 f"A capsule upgrade was triggered outside current deployment instance. Current deployment version was discarded. Current deployment version: {current_deployment_instance_version} and new version: {capsule_response.get('version', None)}",
             )
 

outerbounds/apps/click_importer.py

@@ -0,0 +1,24 @@
+"""
+The purpose of this file is a little bit of cleverness to allow us to use the CLI in this package across
+metaflow and outerbounds projects.
+
+The issue is that since outerbounds and metaflow both vendor click, we can't use object from one import path
+and expect them to work with objects created from the other import path.
+
+Meaning `outerbounds._vendor.click.Group` and `metaflow._vendor.click.Group` are different classes.
+So we need to ensure that based on when the import is taking place, we import the correct class.
+
+Overall, this ONLY affects constructs in click we are using to construct related to the  cli decorators but
+it doesn't affect any capabilities in click for logging.
+"""
+import os
+
+# Import Hacks
+if os.environ.get("APPS_CLI_LOADING_IN_METAFLOW", None):
+    from metaflow._vendor import click as metaflow_click
+
+    click = metaflow_click  # type: ignore
+else:
+    from outerbounds._vendor import click as outerbounds_click
+
+    click = outerbounds_click  # type: ignore

outerbounds/apps/dependencies.py

@@ -43,7 +43,7 @@ def bake_deployment_image(
     # 1. When the user has specified something like `pypi`/`conda`
     # 2, When the user has specified something like `from_requirements`/ `from_pyproject`
     # TODO: add parsers for the pyproject/requirements stuff.
-    from metaflow.ob_internal import bake_image
+    from metaflow.ob_internal import bake_image  # type: ignore
     from metaflow.plugins.pypi.parsers import (
         requirements_txt_parser,
         pyproject_toml_parser,
@@ -83,7 +83,7 @@ def bake_deployment_image(
     python_packages_exist = len(pypi_packages) > 0 or len(conda_packages) > 0
     if (not python_packages_exist) or app_config.get_state("skip_dependencies", False):
         # Inform the user that no dependencies are being used.
-        if app_config.get_state("skip_dependencies", False):
+        if app_config.get_state("skip_dependencies", False) and logger:
             logger(
                 "⏭️ Skipping baking dependencies into the image based on the --no-deps flag."
             )

outerbounds/apps/experimental/__init__.py

@@ -1,5 +1,5 @@
 from functools import wraps
-from metaflow._vendor import click
+from ..click_importer import click
 import os
 from typing import TYPE_CHECKING
 
@@ -78,7 +78,14 @@ def capsule_input_overrides(app_config: "AppConfig", capsule_input: dict):
 
 
 def _code_info(app_config: "AppConfig"):
-    from metaflow.metaflow_git import get_repository_info, _call_git
+    from metaflow.metaflow_git import get_repository_info
+
+    try:
+        from metaflow.metaflow_git import _call_git  # type: ignore
+    except ImportError:
+        # Fallback if _call_git is not available
+        def _call_git(args, path=None):
+            return "", 1, True
 
     repo_info = get_repository_info(app_config.get_state("packaging_directory", None))
     if len(repo_info) == 0:
@@ -88,16 +95,16 @@ def _code_info(app_config: "AppConfig"):
         ["log", "-1", "--pretty=%B"],
         path=app_config.get_state("packaging_directory", None),
     )
-    _url = (
-        repo_info["repo_url"]
-        if not repo_info["repo_url"].endswith(".git")
-        else repo_info["repo_url"].rstrip(".git")
-    )
+    repo_url = repo_info["repo_url"]
+    if isinstance(repo_url, str):
+        _url = repo_url if not repo_url.endswith(".git") else repo_url.rstrip(".git")
+    else:
+        _url = str(repo_url)
     _code_info = {
         "commitId": repo_info["commit_sha"],
-        "commitLink": os.path.join(_url, "commit", repo_info["commit_sha"]),
+        "commitLink": os.path.join(_url, "commit", str(repo_info["commit_sha"])),
     }
-    if not failed and returncode == 0:
+    if not failed and returncode == 0 and isinstance(git_log_info, str):
         _code_info["commitMessage"] = git_log_info.strip()
 
     return _code_info

outerbounds/apps/perimeters.py

@@ -1,11 +1,13 @@
 import os
 import json
-from typing import Tuple
+from typing import Tuple, Union
 
 
 class PerimeterExtractor:
     @classmethod
-    def for_ob_cli(cls, config_dir: str, profile: str) -> Tuple[str, str]:
+    def for_ob_cli(
+        cls, config_dir: str, profile: str
+    ) -> Union[Tuple[str, str], Tuple[None, None]]:
         """
         This function will be called when we are trying to extract the perimeter
         via the ob cli's execution. We will rely on the following logic:
@@ -35,17 +37,9 @@ class PerimeterExtractor:
                 config_dir, profile, "OBP_API_SERVER"
             )
 
-        return perimeter, api_server
-
-        pass
+        return perimeter, api_server  # type: ignore
 
     @classmethod
-    def during_metaflow_execution(cls):
-        pass  # todo: implement this
-
-
-def general_purpose_perimeter_extractor(config_dir: str, profile: str) -> str:
-    """
-    This function will rely on a mix of configuration variables and utilities with ob package to extract the perimeter
-    from the current environment.
-    """
+    def during_metaflow_execution(cls) -> str:
+        # TODO: implement this
+        return ""

outerbounds/apps/secrets.py

@@ -98,7 +98,7 @@ class SecretRetriever:
             raise OuterboundsSecretsException(f"Error decoding secret: {e}")
 
     def _get_secret_configs(self):
-        from metaflow_extensions.outerbounds.remote_config import init_config
+        from metaflow_extensions.outerbounds.remote_config import init_config  # type: ignore
         from os import environ
 
         conf = init_config()

outerbounds/apps/utils.py

@@ -3,6 +3,11 @@ import time
 import sys
 import json
 import requests
+from typing import Optional
+
+# This click import is not used to construct any ob
+# package cli. Its used only for printing stuff.
+# So we can use the static metaflow._vendor import path
 from metaflow._vendor import click
 from .app_config import CAPSULE_DEBUG
 import sys
@@ -10,6 +15,7 @@ import threading
 import time
 import logging
 import itertools
+from typing import Union, Callable, Any, List
 
 from outerbounds._vendor.spinner import (
     Spinners,
@@ -36,13 +42,13 @@ class MultiStepSpinner:
         self,
         spinner: Spinners = Spinners.dots,
         text: str = "",
-        color: str = None,
+        color: Optional[str] = None,
         align: str = "right",
         file=sys.stdout,
     ):
         cfg = spinner.value
         self.frames = cfg["frames"]
-        self.interval = cfg["interval"] / 1000.0
+        self.interval = float(cfg["interval"]) / 1000.0  # type: ignore
         self.text = text
         self.color = color
         if align not in ("left", "right"):
@@ -50,7 +56,7 @@ class MultiStepSpinner:
         self.align = align
         self._write_file = file
         # precompute clear length: max frame width + space + text length
-        max_frame = max(self.frames, key=len)
+        max_frame = max(self.frames, key=lambda x: len(x))  # type: ignore
         self.clear_len = len(self.main_text) + len(max_frame) + 1
 
         self._stop_evt = threading.Event()
@@ -150,171 +156,6 @@ class MaximumRetriesExceeded(Exception):
         return f"Maximum retries exceeded for {self.url}[{self.method}] {self.status_code} {self.text}"
 
 
-class KeyValueDictPair(click.ParamType):
-    name = "KV-DICT-PAIR"
-
-    def convert(self, value, param, ctx):
-        # Parse a string of the form KEY=VALUE into a dict {KEY: VALUE}
-        if len(value.split("=", 1)) != 2:
-            self.fail(
-                f"Invalid format for {value}. Expected format: KEY=VALUE", param, ctx
-            )
-
-        key, _value = value.split("=", 1)
-        try:
-            return {"key": key, "value": json.loads(_value)}
-        except json.JSONDecodeError:
-            return {"key": key, "value": _value}
-        except Exception as e:
-            self.fail(f"Invalid value for {value}. Error: {e}", param, ctx)
-
-    def __str__(self):
-        return repr(self)
-
-    def __repr__(self):
-        return "KV-PAIR"
-
-
-class KeyValuePair(click.ParamType):
-    name = "KV-PAIR"
-
-    def convert(self, value, param, ctx):
-        # Parse a string of the form KEY=VALUE into a dict {KEY: VALUE}
-        if len(value.split("=", 1)) != 2:
-            self.fail(
-                f"Invalid format for {value}. Expected format: KEY=VALUE", param, ctx
-            )
-
-        key, _value = value.split("=", 1)
-        try:
-            return {key: json.loads(_value)}
-        except json.JSONDecodeError:
-            return {key: _value}
-        except Exception as e:
-            self.fail(f"Invalid value for {value}. Error: {e}", param, ctx)
-
-    def __str__(self):
-        return repr(self)
-
-    def __repr__(self):
-        return "KV-PAIR"
-
-
-class MountMetaflowArtifact(click.ParamType):
-    name = "MOUNT-METAFLOW-ARTIFACT"
-
-    def convert(self, value, param, ctx):
-        """
-        Convert a string like "flow=MyFlow,artifact=my_model,path=/tmp/abc" or
-        "pathspec=MyFlow/123/foo/345/my_model,path=/tmp/abc" to a dict.
-        """
-        artifact_dict = {}
-        parts = value.split(",")
-
-        for part in parts:
-            if "=" not in part:
-                self.fail(
-                    f"Invalid format in part '{part}'. Expected 'key=value'", param, ctx
-                )
-
-            key, val = part.split("=", 1)
-            artifact_dict[key.strip()] = val.strip()
-
-        # Validate required fields
-        if "pathspec" in artifact_dict:
-            if "path" not in artifact_dict:
-                self.fail(
-                    "When using 'pathspec', you must also specify 'path'", param, ctx
-                )
-
-            # Return as pathspec format
-            return {
-                "pathspec": artifact_dict["pathspec"],
-                "path": artifact_dict["path"],
-            }
-        elif (
-            "flow" in artifact_dict
-            and "artifact" in artifact_dict
-            and "path" in artifact_dict
-        ):
-            # Return as flow/artifact format
-            result = {
-                "flow": artifact_dict["flow"],
-                "artifact": artifact_dict["artifact"],
-                "path": artifact_dict["path"],
-            }
-
-            # Add optional namespace if provided
-            if "namespace" in artifact_dict:
-                result["namespace"] = artifact_dict["namespace"]
-
-            return result
-        else:
-            self.fail(
-                "Invalid format. Must be either 'flow=X,artifact=Y,path=Z' or 'pathspec=X,path=Z'",
-                param,
-                ctx,
-            )
-
-    def __str__(self):
-        return repr(self)
-
-    def __repr__(self):
-        return "MOUNT-METAFLOW-ARTIFACT"
-
-
-class MountSecret(click.ParamType):
-    name = "MOUNT-SECRET"
-
-    def convert(self, value, param, ctx):
-        """
-        Convert a string like "id=my_secret,path=/tmp/secret" to a dict.
-        """
-        secret_dict = {}
-        parts = value.split(",")
-
-        for part in parts:
-            if "=" not in part:
-                self.fail(
-                    f"Invalid format in part '{part}'. Expected 'key=value'", param, ctx
-                )
-
-            key, val = part.split("=", 1)
-            secret_dict[key.strip()] = val.strip()
-
-        # Validate required fields
-        if "id" in secret_dict and "path" in secret_dict:
-            return {"id": secret_dict["id"], "path": secret_dict["path"]}
-        else:
-            self.fail("Invalid format. Must be 'key=X,path=Y'", param, ctx)
-
-    def __str__(self):
-        return repr(self)
-
-    def __repr__(self):
-        return "MOUNT-SECRET"
-
-
-class CommaSeparatedList(click.ParamType):
-    name = "COMMA-SEPARATED-LIST"
-
-    def convert(self, value, param, ctx):
-        return value.split(",")
-
-    def __str__(self):
-        return repr(self)
-
-    def __repr__(self):
-        return "COMMA-SEPARATED-LIST"
-
-
-KVPairType = KeyValuePair()
-MetaflowArtifactType = MountMetaflowArtifact()
-SecretMountType = MountSecret()
-CommaSeparatedListType = CommaSeparatedList()
-KVDictType = KeyValueDictPair()
-
-
 class TODOException(Exception):
     pass
 

outerbounds/command_groups/cli.py

@@ -7,6 +7,7 @@ from . import (
     tutorials_cli,
     fast_bakery_cli,
     secrets_cli,
+    kubernetes_cli,
 )
 
 
@@ -20,6 +21,7 @@ from . import (
         tutorials_cli.cli,
         fast_bakery_cli.cli,
         secrets_cli.cli,
+        kubernetes_cli.cli,
     ],
 )
 def cli(**kwargs):

outerbounds/command_groups/kubernetes_cli.py

@@ -0,0 +1,479 @@
+import json
+import base64
+import boto3
+import requests
+import os
+import time
+from datetime import datetime
+from functools import partial
+
+from os import path, environ
+from sys import exit
+from outerbounds._vendor import click
+
+from ..utils import metaflowconfig
+
+
+def _logger(
+    body="", system_msg=False, head="", bad=False, timestamp=True, nl=True, color=None
+):
+    if timestamp:
+        if timestamp is True:
+            dt = datetime.now()
+        else:
+            dt = timestamp
+        tstamp = dt.strftime("%Y-%m-%d %H:%M:%S.%f")[:-3]
+        click.secho(tstamp + " ", fg=ColorTheme.TIMESTAMP, nl=False)
+    if head:
+        click.secho(head, fg=ColorTheme.INFO_COLOR, nl=False)
+    click.secho(
+        body,
+        bold=system_msg,
+        fg=ColorTheme.BAD_COLOR if bad else color if color is not None else None,
+        nl=nl,
+    )
+
+
+class ColorTheme:
+    TIMESTAMP = "magenta"
+    LOADING_COLOR = "cyan"
+    BAD_COLOR = "red"
+    INFO_COLOR = "green"
+
+    TL_HEADER_COLOR = "magenta"
+    ROW_COLOR = "bright_white"
+
+    INFO_KEY_COLOR = "green"
+    INFO_VALUE_COLOR = "bright_white"
+
+
+def print_table(data, headers):
+    """Print data in a formatted table."""
+
+    if not data:
+        return
+
+    # Calculate column widths
+    col_widths = [len(h) for h in headers]
+
+    # Calculate actual widths based on data
+    for row in data:
+        for i, cell in enumerate(row):
+            col_widths[i] = max(col_widths[i], len(str(cell)))
+
+    # Print header
+    header_row = " | ".join(
+        [headers[i].ljust(col_widths[i]) for i in range(len(headers))]
+    )
+    click.secho("-" * len(header_row), fg=ColorTheme.TL_HEADER_COLOR)
+    click.secho(header_row, fg=ColorTheme.TL_HEADER_COLOR, bold=True)
+    click.secho("-" * len(header_row), fg=ColorTheme.TL_HEADER_COLOR)
+
+    # Print data rows
+    for row in data:
+        formatted_row = " | ".join(
+            [str(row[i]).ljust(col_widths[i]) for i in range(len(row))]
+        )
+        click.secho(formatted_row, fg=ColorTheme.ROW_COLOR, bold=True)
+    click.secho("-" * len(header_row), fg=ColorTheme.TL_HEADER_COLOR)
+
+
+def _get_kubernetes_client():
+    """Get kubernetes client from metaflow configuration."""
+    from metaflow.plugins.kubernetes.kubernetes_client import KubernetesClient
+
+    return KubernetesClient()
+
+
+def _get_current_user():
+    """Get current user from environment or metaflow config."""
+    # Try to get user from metaflow config first
+    try:
+        from metaflow.util import get_username
+
+        user = get_username()
+        if user:
+            return user
+    except:
+        pass
+
+    # Fallback to environment variables
+    raise click.ClickException("Failed to get current user")
+
+
+def _format_jobs_and_jobsets_table(
+    jobs_with_outcomes, jobsets_with_outcomes, filter_unchanged=True
+):
+    """Format jobs and jobsets into a table for display."""
+    headers = [
+        "Type",
+        "Name",
+        "Namespace",
+        "Status",
+        "Outcome",
+        "Created",
+        "Flow",
+        "Run ID",
+        "User",
+    ]
+    table_data = []
+
+    # Add jobs to table
+    for job, outcome in jobs_with_outcomes:
+        # Filter out unchanged resources if requested
+        if filter_unchanged and outcome == "leave_unchanged":
+            continue
+
+        annotations = job.metadata.annotations or {}
+
+        # Format creation timestamp
+        created_time = "N/A"
+        if job.metadata.creation_timestamp:
+            created_time = job.metadata.creation_timestamp.strftime("%Y-%m-%d %H:%M:%S")
+
+        table_data.append(
+            [
+                "Job",
+                job.metadata.name,
+                job.metadata.namespace,
+                str(job.status.active or 0) + " active"
+                if job.status.active
+                else "inactive",
+                outcome,
+                created_time,
+                annotations.get("metaflow/flow_name", "N/A"),
+                annotations.get("metaflow/run_id", "N/A"),
+                annotations.get("metaflow/user", "N/A"),
+            ]
+        )
+
+    # Add jobsets to table
+    for jobset, outcome in jobsets_with_outcomes:
+        # Filter out unchanged resources if requested
+        if filter_unchanged and outcome == "leave_unchanged":
+            continue
+
+        metadata = jobset.get("metadata", {})
+        annotations = metadata.get("annotations", {})
+        status = jobset.get("status", {})
+
+        # Format creation timestamp
+        created_time = "N/A"
+        creation_timestamp = metadata.get("creationTimestamp")
+        if creation_timestamp:
+            try:
+                from datetime import datetime
+
+                # Parse ISO timestamp
+                dt = datetime.fromisoformat(creation_timestamp.replace("Z", "+00:00"))
+                created_time = dt.strftime("%Y-%m-%d %H:%M:%S")
+            except:
+                created_time = (
+                    creation_timestamp[:19]
+                    if len(creation_timestamp) >= 19
+                    else creation_timestamp
+                )
+
+        table_data.append(
+            [
+                "JobSet",
+                metadata.get("name", "N/A"),
+                metadata.get("namespace", "N/A"),
+                "terminal" if status.get("terminalState") else "running",
+                outcome,
+                created_time,
+                annotations.get("metaflow/flow_name", "N/A"),
+                annotations.get("metaflow/run_id", "N/A"),
+                annotations.get("metaflow/user", "N/A"),
+            ]
+        )
+
+    return headers, table_data
+
+
+@click.group()
+def cli(**kwargs):
+    pass
+
+
+@click.group(help="Commands for interacting with Kubernetes.")
+def kubernetes(**kwargs):
+    pass
+
+
+@kubernetes.command(help="Kill pods/jobs/jobsets for a specific flow.")
+@click.option("--flow-name", required=True, help="Flow name to kill pods for")
+@click.option("--run-id", help="Specific run ID to kill pods for")
+@click.option("--my-runs", is_flag=True, help="Only kill runs by current user")
+@click.option(
+    "--dry-run", is_flag=True, help="Show what would be killed without actually killing"
+)
+@click.option("--auto-approve", is_flag=True, help="Skip confirmation prompt")
+@click.option(
+    "--clear-everything",
+    is_flag=True,
+    help="Force delete ALL matching resources regardless of their status (including terminal/completed ones)",
+)
+def kill(flow_name, run_id, my_runs, dry_run, auto_approve, clear_everything):
+    """Kill pods/jobs/jobsets for a specific flow."""
+    import warnings
+    from metaflow.ob_internal import PodKiller  # type: ignore
+
+    warnings.filterwarnings("ignore")
+
+    logger = partial(_logger, timestamp=True)
+
+    # Get kubernetes client
+    kubernetes_client = _get_kubernetes_client()
+
+    # Determine user filter
+    user = None
+    if my_runs:
+        user = _get_current_user()
+        logger(f"🔍 Filtering for runs by user: {user}", color=ColorTheme.INFO_COLOR)
+
+    pod_killer = PodKiller(
+        kubernetes_client=kubernetes_client.get(),
+        echo_func=lambda x: None,
+        namespace=kubernetes_client._namespace,
+    )
+
+    # Find matching jobs and jobsets
+    logger(
+        f"🔍 Searching for jobs and jobsets matching flow: {flow_name}",
+        color=ColorTheme.INFO_COLOR,
+    )
+    if run_id:
+        logger(f"🔍 Filtering by run ID: {run_id}", color=ColorTheme.INFO_COLOR)
+
+    try:
+        (
+            jobs_with_outcomes,
+            jobsets_with_outcomes,
+        ) = pod_killer.extract_matching_jobs_and_jobsets(
+            flow_name=flow_name, run_id=run_id, user=user
+        )
+    except Exception as e:
+        logger(f"Error finding matching resources: {e}", bad=True, system_msg=True)
+        exit(1)
+
+    # Check if anything was found
+    total_resources_found = len(jobs_with_outcomes) + len(jobsets_with_outcomes)
+    if total_resources_found == 0:
+        logger("✅ No matching jobs or jobsets found.", color=ColorTheme.INFO_COLOR)
+        return
+
+    # Calculate resources that will be processed
+    if clear_everything:
+        # Process ALL resources regardless of status
+        jobs_to_process = len(jobs_with_outcomes)
+        jobsets_to_process = len(jobsets_with_outcomes)
+        total_to_process = jobs_to_process + jobsets_to_process
+        filter_table = False
+
+        # Show warning for clear-everything mode
+        logger(
+            "⚠️  CLEAR EVERYTHING MODE: All matching resources will be force deleted regardless of status!",
+            color=ColorTheme.BAD_COLOR,
+            system_msg=True,
+        )
+    else:
+        # Normal mode: only process resources not in terminal state
+        jobs_to_process = len(
+            [j for j, outcome in jobs_with_outcomes if outcome != "leave_unchanged"]
+        )
+        jobsets_to_process = len(
+            [j for j, outcome in jobsets_with_outcomes if outcome != "leave_unchanged"]
+        )
+        total_to_process = jobs_to_process + jobsets_to_process
+        filter_table = True
+
+    # Display what will be affected
+    headers, table_data = _format_jobs_and_jobsets_table(
+        jobs_with_outcomes, jobsets_with_outcomes, filter_unchanged=filter_table
+    )
+
+    if total_to_process == 0:
+        logger(
+            "✅ All matching resources are already in terminal state. Nothing to do.",
+            color=ColorTheme.INFO_COLOR,
+        )
+        return
+
+    if dry_run:
+        logger(
+            "=== DRY RUN - The following resources would be affected ===",
+            color=ColorTheme.INFO_COLOR,
+            system_msg=True,
+            timestamp=False,
+        )
+    else:
+        logger(
+            "=== The following resources will be killed/deleted ===",
+            color=ColorTheme.BAD_COLOR,
+            system_msg=True,
+            timestamp=False,
+        )
+
+    print_table(table_data, headers)
+
+    # Show summary
+    logger(
+        "📊 Summary:",
+    )
+    logger(
+        f"  • Total resources found: {total_resources_found}",
+    )
+    logger(
+        f"  • Jobs to process: {jobs_to_process}",
+    )
+    logger(
+        f"  • JobSets to process: {jobsets_to_process}",
+    )
+    logger(
+        f"  • Resources to process: {total_to_process}",
+    )
+
+    if clear_everything:
+        logger(
+            "  • Mode: CLEAR EVERYTHING (forcing deletion of ALL resources)",
+            color=ColorTheme.BAD_COLOR,
+        )
+    else:
+        # Show how many are being skipped in normal mode
+        skipped_resources = total_resources_found - total_to_process
+        if skipped_resources > 0:
+            logger(
+                f"  • Resources already in terminal state (skipped): {skipped_resources}",
+                color=ColorTheme.INFO_COLOR,
+            )
+
+    if dry_run:
+        logger(
+            "🔍 Dry run completed. No resources were actually killed.",
+            color=ColorTheme.INFO_COLOR,
+            system_msg=True,
+        )
+        return
+
+    # Confirm before proceeding (unless auto-approve is set)
+    if not auto_approve:
+        confirm = click.prompt(
+            click.style(
+                f"⚠️  Are you sure you want to kill/delete {total_to_process} resources?",
+                fg=ColorTheme.BAD_COLOR,
+                bold=True,
+            ),
+            default="no",
+            type=click.Choice(["yes", "no"]),
+        )
+        if confirm == "no":
+            logger("❌ Operation cancelled.", color=ColorTheme.BAD_COLOR)
+            exit(1)
+
+    # Execute the kills/deletions
+    logger(
+        f"🚀 Processing {total_to_process} resources...",
+        color=ColorTheme.INFO_COLOR,
+        system_msg=True,
+    )
+
+    try:
+        progress_label = (
+            f"⚰️ Coffin: Deleting jobs and jobsets matching flow: {flow_name}"
+        )
+        if clear_everything:
+            progress_label = (
+                f"🔥 CLEAR ALL: Force deleting ALL resources for flow: {flow_name}"
+            )
+
+        __progress_bar = click.progressbar(
+            length=total_to_process,
+            label=click.style(
+                progress_label,
+                fg=ColorTheme.BAD_COLOR if clear_everything else ColorTheme.INFO_COLOR,
+                bold=True,
+            ),
+            fill_char=click.style(
+                "█",
+                fg=ColorTheme.BAD_COLOR if clear_everything else ColorTheme.INFO_COLOR,
+                bold=True,
+            ),
+            empty_char=click.style(
+                "░",
+                fg=ColorTheme.BAD_COLOR if clear_everything else ColorTheme.INFO_COLOR,
+                bold=True,
+            ),
+            item_show_func=lambda x: click.style(
+                x,
+                fg=ColorTheme.BAD_COLOR,
+                bold=True,
+            ),
+        )
+
+        pod_killer = PodKiller(
+            kubernetes_client=kubernetes_client.get(),
+            echo_func=lambda x: None,
+            namespace=kubernetes_client._namespace,
+            progress_bar=__progress_bar,
+        )
+
+        if clear_everything:
+            # Force delete everything mode
+            (
+                results,
+                jobs_processed,
+                jobsets_processed,
+            ) = pod_killer.process_matching_jobs_and_jobsets_force_all(
+                flow_name=flow_name, run_id=run_id, user=user
+            )
+        else:
+            # Normal mode
+            (
+                results,
+                jobs_processed,
+                jobsets_processed,
+            ) = pod_killer.process_matching_jobs_and_jobsets(
+                flow_name=flow_name, run_id=run_id, user=user
+            )
+
+        # Report results
+        successful_operations = sum(1 for r in results if r is True)
+        failed_operations = sum(1 for r in results if r is False)
+
+        logger(
+            "📊 Operation completed:",
+        )
+        logger(
+            f"  • Jobs processed: {jobs_processed}",
+        )
+        logger(
+            f"  • JobSets processed: {jobsets_processed}",
+        )
+        logger(
+            f"  • Successful operations: {successful_operations}",
+        )
+
+        if failed_operations > 0:
+            logger(
+                f"  • Failed operations: {failed_operations}",
+                color=ColorTheme.BAD_COLOR,
+            )
+            logger(
+                "⚠️  Some operations failed. Check the logs above for details.",
+                color=ColorTheme.BAD_COLOR,
+                system_msg=True,
+            )
+        else:
+            logger(
+                "✅ All operations completed successfully!",
+                color=ColorTheme.INFO_COLOR,
+                system_msg=True,
+            )
+
+    except Exception as e:
+        logger(f"Error during kill operation: {e}", bad=True, system_msg=True)
+        raise e
+
+
+cli.add_command(kubernetes, name="kubernetes")

{outerbounds-0.3.180rc5.dist-info → outerbounds-0.3.182rc0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: outerbounds
-Version: 0.3.180rc5
+Version: 0.3.182rc0
 Summary: More Data Science, Less Administration
 License: Proprietary
 Keywords: data science,machine learning,MLOps
@@ -29,8 +29,8 @@ Requires-Dist: google-cloud-secret-manager (>=2.20.0,<3.0.0) ; extra == "gcp"
 Requires-Dist: google-cloud-storage (>=2.14.0,<3.0.0) ; extra == "gcp"
 Requires-Dist: metaflow-checkpoint (==0.2.1)
 Requires-Dist: ob-metaflow (==2.15.17.1)
-Requires-Dist: ob-metaflow-extensions (==1.1.168rc5)
-Requires-Dist: ob-metaflow-stubs (==6.0.3.180rc5)
+Requires-Dist: ob-metaflow-extensions (==1.1.170rc0)
+Requires-Dist: ob-metaflow-stubs (==6.0.3.182rc0)
 Requires-Dist: opentelemetry-distro (>=0.41b0) ; extra == "otel"
 Requires-Dist: opentelemetry-exporter-otlp-proto-http (>=1.20.0) ; extra == "otel"
 Requires-Dist: opentelemetry-instrumentation-requests (>=0.41b0) ; extra == "otel"

{outerbounds-0.3.180rc5.dist-info → outerbounds-0.3.182rc0.dist-info}/RECORD

@@ -43,28 +43,30 @@ outerbounds/_vendor/yaml/scanner.py,sha256=ZcI8IngR56PaQ0m27WU2vxCqmDCuRjz-hr7pi
 outerbounds/_vendor/yaml/serializer.py,sha256=8wFZRy9SsQSktF_f9OOroroqsh4qVUe53ry07P9UgCc,4368
 outerbounds/_vendor/yaml/tokens.py,sha256=JBSu38wihGr4l73JwbfMA7Ks1-X84g8-NskTz7KwPmA,2578
 outerbounds/apps/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-outerbounds/apps/_state_machine.py,sha256=3hQF5O2zJdtQWdy9e5w393O85u6UjGApqTMlRU3UhFk,12964
-outerbounds/apps/app_cli.py,sha256=j_0vEQlsKGgvlPidog4bSZrY-_D1ne3JmHVafijL-iI,45354
-outerbounds/apps/app_config.py,sha256=KBmW9grhiuG9XZG-R0GZkM-024cjj6ztGzOX_2wZW34,11291
+outerbounds/apps/_state_machine.py,sha256=ixgL--jne3q71gQNnUeK-UdLP-Oc2kSGSsIW1fiHZPY,14469
+outerbounds/apps/app_cli.py,sha256=vTbIN43A9A_OHTM0I6cb28g3GtY4_jk1wmUi5wG09w0,50174
+outerbounds/apps/app_config.py,sha256=UHVK8JLIuW-OcGg5WxDm4QHeImPGtohD4KpJryZntC4,11307
 outerbounds/apps/artifacts.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-outerbounds/apps/capsule.py,sha256=YoOkUYGPrbXPaSUwS3teui1XPJDZCIxn-9tPpm1X0GM,30927
+outerbounds/apps/capsule.py,sha256=IYaD5X-IpSD9IsLpIJUaJ2B31up7PY7GhMMizkwLV7I,31039
 outerbounds/apps/cli_to_config.py,sha256=Thc5jXRxoU6Pr8kAVVOX-5Es5ha6y6Vh_GBzL__oI7Q,3299
+outerbounds/apps/click_importer.py,sha256=nnkPOR6TKrtIpc3a5Fna1zVJoQqDZvUXlNA9CdiNKFc,995
 outerbounds/apps/code_package/__init__.py,sha256=8McF7pgx8ghvjRnazp2Qktlxi9yYwNiwESSQrk-2oW8,68
 outerbounds/apps/code_package/code_packager.py,sha256=RWvM5BKjgLhu7icsO_n5SSYC57dwyST0dWpoWF88ovU,22881
 outerbounds/apps/code_package/examples.py,sha256=aF8qKIJxCVv_ugcShQjqUsXKKKMsm1oMkQIl8w3QKuw,4016
 outerbounds/apps/config_schema.yaml,sha256=j_mysTAPkIMSocItTg3aduMDfBs2teIhAErvpF0Elus,8826
-outerbounds/apps/dependencies.py,sha256=UucyQYZ5VjUPBb3XzAARa4fuiLHXuV7iMZ8OZ_nAuE8,3949
+outerbounds/apps/dependencies.py,sha256=03pZY-JRN-dYN-iyZ73zoEIEKmrOvbY4qld7RlRXYuw,3976
 outerbounds/apps/deployer.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-outerbounds/apps/experimental/__init__.py,sha256=ExPFIJSF8FcE1pKVyiNQnX8aBi3Rz8YFQ2_s5NVeU7I,3056
-outerbounds/apps/perimeters.py,sha256=DKWKMbkSx8WObf4yA74UolleT417B9c_6POioztMHxY,1766
-outerbounds/apps/secrets.py,sha256=sBMJzwjsARhtcuxgn18i66lagllpYOlh2oXR_hVAb1Y,6110
-outerbounds/apps/utils.py,sha256=NbEpSrzk-l95EBFoWuxil_0xjpIuFt0jjIXD2r1w71M,12118
+outerbounds/apps/experimental/__init__.py,sha256=RUZBAyqFnX3pRQxTjNmS1-qpgQcc9xQGQD2yJh4MA_M,3349
+outerbounds/apps/perimeters.py,sha256=1J1_-5legFPskv3HTRwQMpzTytE3TO8KRT2IvVOrWcQ,1584
+outerbounds/apps/secrets.py,sha256=aWzcAayQEJghQgFP_qp9w6jyvan_hoL4_ceqZ0ZjLd4,6126
+outerbounds/apps/utils.py,sha256=6REvD9PtJcLYzrxX5lZ5Dzzm-Sy2l-I1oSzQN9viYRs,7611
 outerbounds/apps/validations.py,sha256=kR2eXckx0XJ4kUOOLkMRepbTh0INtL1Z8aV4-fZpfc8,678
 outerbounds/cli_main.py,sha256=e9UMnPysmc7gbrimq2I4KfltggyU7pw59Cn9aEguVcU,74
 outerbounds/command_groups/__init__.py,sha256=QPWtj5wDRTINDxVUL7XPqG3HoxHNvYOg08EnuSZB2Hc,21
 outerbounds/command_groups/apps_cli.py,sha256=ecXyLhGxjbct62iqviP9qBX8s4d-XG56ICpTM2h2etk,20821
-outerbounds/command_groups/cli.py,sha256=de4_QY1UeoKX6y-IXIbmklAi6bz0DsdBSmAoCg6lq1o,482
+outerbounds/command_groups/cli.py,sha256=FTeeDrvyBb-qcs2xklTiCyVTN5I0tBPyBReqDIE4oWU,530
 outerbounds/command_groups/fast_bakery_cli.py,sha256=5kja7v6C651XAY6dsP_IkBPJQgfU4hA4S9yTOiVPhW0,6213
+outerbounds/command_groups/kubernetes_cli.py,sha256=2bxPKUp5g_gdwVo4lT-IeWvHxz6Jmj1KxG70nXNgX_M,14758
 outerbounds/command_groups/local_setup_cli.py,sha256=tuuqJRXQ_guEwOuQSIf9wkUU0yg8yAs31myGViAK15s,36364
 outerbounds/command_groups/perimeters_cli.py,sha256=iF_Uw7ROiSctf6FgoJEy30iDBLVE1j9FKuR3shgJRmc,19050
 outerbounds/command_groups/secrets_cli.py,sha256=Vgn_aiTo76a0s5hCJhNWEOrCVhyYeivD08ooQxz0y7c,2952
@@ -76,7 +78,7 @@ outerbounds/utils/metaflowconfig.py,sha256=l2vJbgPkLISU-XPGZFaC8ZKmYFyJemlD6bwB-
 outerbounds/utils/schema.py,sha256=lMUr9kNgn9wy-sO_t_Tlxmbt63yLeN4b0xQXbDUDj4A,2331
 outerbounds/utils/utils.py,sha256=4Z8cszNob_8kDYCLNTrP-wWads_S_MdL3Uj3ju4mEsk,501
 outerbounds/vendor.py,sha256=gRLRJNXtZBeUpPEog0LOeIsl6GosaFFbCxUvR4bW6IQ,5093
-outerbounds-0.3.180rc5.dist-info/METADATA,sha256=bhULcwevS8zjmFWtUdm_NqRqF1vYNqx1z-NGfBmKpb8,1846
-outerbounds-0.3.180rc5.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-outerbounds-0.3.180rc5.dist-info/entry_points.txt,sha256=AP6rZg7y5SK9e9a9iVq0Fi9Q2KPjPZSwtZ6R98rLw-8,56
-outerbounds-0.3.180rc5.dist-info/RECORD,,
+outerbounds-0.3.182rc0.dist-info/METADATA,sha256=Y51QdSpzQvPHckh4lLqNcic0u-gM1sIi4rX2K4RmUPA,1846
+outerbounds-0.3.182rc0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+outerbounds-0.3.182rc0.dist-info/entry_points.txt,sha256=AP6rZg7y5SK9e9a9iVq0Fi9Q2KPjPZSwtZ6R98rLw-8,56
+outerbounds-0.3.182rc0.dist-info/RECORD,,