outerbounds 0.3.179rc5__py3-none-any.whl → 0.3.180rc0__py3-none-any.whl

outerbounds/apps/capsule.py

@@ -8,62 +8,61 @@ from typing import Optional, List, Dict, Any, Tuple
 from .utils import TODOException, safe_requests_wrapper, MaximumRetriesExceeded
 from .app_config import AppConfig, CAPSULE_DEBUG, AuthType
 from . import experimental
+from ._state_machine import (
+    _capsule_worker_semantic_status,
+    _capsule_worker_status_diff,
+    CapsuleWorkerSemanticStatus,
+    WorkerStatus,
+    DEPLOYMENT_READY_CONDITIONS,
+)
 
 
 class CapsuleStateMachine:
     """
-    Since capsules are a kinda newer concept, we will treat the state transitions based on the conditions and the
-    availability of certain fields in the status dictionary.
-    """
+    - Every capsule create call will return a `identifier` and a `version` of the object.
+    - Each update call will return a new version.
+    - The status.currentlyServedVersion will be the version that is currently serving traffic.
+    - The status.updateInProgress will be True if an upgrade is in progress.
+
+    CapsuleState Transition:
+    - Every capsule create call will return a `identifier` and a `version` of the object.
+    - Happy Path:
+        - First time Create :
+            - wait for status.updateInProgress to be set to False
+                - (interleved) Poll the worker endpoints to check their status
+                    - showcase how many workers are coming up if things are on the cli side.
+                - If the user has set some flag like `--dont-wait-to-fully-finish` then we check the `status.currentlyServedVersion` to see if even one replica is ready to
+                serve traffic.
+            - once the status.updateInProgress is set to False, it means that the replicas are ready
+        - Upgrade:
+            - wait for status.updateInProgress to be set to False
+                - (interleved) Poll the worker endpoints to check their status and signal the user the number replicas coming up
+                - If the user has set some flag like `--dont-wait-to-fully-finish` then we check the `status.currentlyServedVersion` to see if even one replica is ready to
+                serve traffic.
+    - Unhappy Path:
+        - First time Create :
+            - wait for status.updateInProgress to be set to False,
+                - (interleved) Poll the workers to check their status.
+                    - If the worker pertaining the current deployment instance version is crashlooping then crash the deployment process with the error messages and logs.
+        - Upgrade:
+            - wait for status.updateInProgress to be set to False,
+                - (interleved) Poll the workers to check their status.
+                    - If the worker pertaining the current deployment instance version is crashlooping then crash the deployment process with the error messages and logs.
 
-    CONDITIONS = ["Ready", "DeploymentReplicasAvailable", "IngressObjectReady"]
+    """
 
-    def __init__(self, capsule_id: str):
+    def __init__(self, capsule_id: str, current_deployment_instance_version: str):
         self._capsule_id = capsule_id
         self._status_trail = []
-
-    def is_completely_new_capsule(self):
-        # This is a heuristic. Not a fully tested.
-        # If we create a completely new capsule then the status
-        # field might be a completely empty dictionary.
-        assert (
-            len(self._status_trail) > 0
-        ), "status_trail cannot be none to infer if its a new capsule"
-        return self._empty_status(self._status_trail[0].get("status"))
+        self._current_deployment_instance_version = current_deployment_instance_version
 
     def get_status_trail(self):
         return self._status_trail
 
-    @staticmethod
-    def _empty_status(status):
-        if json.dumps(status) == "{}":
-            return True
-        return False
-
-    @staticmethod
-    def _parse_conditions(conditions):
-        curr_conditons = {}
-        for condition in conditions:
-            curr_conditons[condition["type"]] = condition["status"]
-        return curr_conditons
-
     def add_status(self, status: dict):
         assert type(status) == dict, "TODO: Make this check somewhere else"
         self._status_trail.append({"timestamp": time.time(), "status": status})
 
-    @staticmethod
-    def _condition_change_emoji(previous_condition_status, current_condition_status):
-        if previous_condition_status == current_condition_status:
-            if previous_condition_status == "True":
-                return "✅"
-            else:
-                return "❌"
-        if previous_condition_status == "True" and current_condition_status == "False":
-            return "🔴 --> 🟢"
-        if previous_condition_status == "False" and current_condition_status == "True":
-            return "🚀"
-        return "🟡"
-
     @property
     def current_status(self):
         return self._status_trail[-1].get("status")
@@ -84,6 +83,14 @@ class CapsuleStateMachine:
             return f"https://{url}"
         return None
 
+    @property
+    def update_in_progress(self):
+        return self.current_status.get("updateInProgress", False)
+
+    @property
+    def currently_served_version(self):
+        return self.current_status.get("currentlyServedVersion", None)
+
     @property
     def ready_to_serve_traffic(self):
         if self.current_status.get("readyToServeTraffic", False):
@@ -97,17 +104,7 @@ class CapsuleStateMachine:
         return self.current_status.get("availableReplicas", 0)
 
     def report_current_status(self, logger):
-        if len(self._status_trail) < 2:
-            return
-        previous_status, current_status = self._status_trail[-2].get(
-            "status"
-        ), self._status_trail[-1].get("status")
-        if self._empty_status(current_status):
-            return
-
-        if self._empty_status(previous_status):
-            logger("💊 %s Deployment has started ... 🚀" % self._capsule_id)
-            return
+        pass
 
     def check_for_debug(self, state_dir: str):
         if CAPSULE_DEBUG:
@@ -118,6 +115,113 @@ class CapsuleStateMachine:
                 json.dump(self._status_trail, f, indent=4)
 
 
+class CapsuleWorkersStateMachine:
+    def __init__(
+        self,
+        capsule_id: str,
+        end_state_capsule_version: str,
+        deployment_mode: str = DEPLOYMENT_READY_CONDITIONS.ATLEAST_ONE_RUNNING,
+        minimum_replicas: int = 1,
+    ):
+        self._capsule_id = capsule_id
+        self._end_state_capsule_version = end_state_capsule_version
+        self._deployment_mode = deployment_mode
+        self._minimum_replicas = minimum_replicas
+        self._status_trail = []
+
+    def get_status_trail(self):
+        return self._status_trail
+
+    def add_status(self, worker_list_response: List[WorkerStatus]):
+        """
+        worker_list_response: List[Dict[str, Any]]
+            [
+                {
+                    "workerId": "c-4pqikm-659dd9ccdc-5hcwz",
+                    "phase": "Running",
+                    "activity": 0,
+                    "activityDataAvailable": false,
+                    "version": "0xhgaewiqb"
+                },
+                {
+                    "workerId": "c-4pqikm-b8559688b-xk2jh",
+                    "phase": "Pending",
+                    "activity": 0,
+                    "activityDataAvailable": false,
+                    "version": "421h48qh95"
+                }
+            ]
+        """
+        self._status_trail.append(
+            {"timestamp": time.time(), "status": worker_list_response}
+        )
+
+    def check_for_debug(self, state_dir: str):
+        if CAPSULE_DEBUG:
+            debug_path = os.path.join(
+                state_dir, f"debug_capsule_workers_{self._capsule_id}_trail.json"
+            )
+            with open(debug_path, "w") as f:
+                json.dump(self._status_trail, f, indent=4)
+
+            status_path = os.path.join(
+                state_dir, f"debug_capsule_workers_{self._capsule_id}_status.json"
+            )
+            with open(status_path, "w") as f:
+                json.dump(self.current_version_deployment_status(), f, indent=4)
+
+    def report_current_status(self, logger):
+        if len(self._status_trail) == 0:
+            return
+        older_status = None
+        if len(self._status_trail) >= 2:
+            older_status = _capsule_worker_semantic_status(
+                self._status_trail[-2].get("status"),
+                self._end_state_capsule_version,
+                self._minimum_replicas,
+            )
+        current_status = self.current_version_deployment_status()
+        changes = _capsule_worker_status_diff(current_status, older_status)
+        if len(changes) > 0:
+            logger(*changes)
+
+    @property
+    def current_status(self) -> List[WorkerStatus]:
+        return self._status_trail[-1].get("status")
+
+    def current_version_deployment_status(self) -> CapsuleWorkerSemanticStatus:
+        return _capsule_worker_semantic_status(
+            self.current_status, self._end_state_capsule_version, self._minimum_replicas
+        )
+
+    @property
+    def is_terminal_state(self):
+        return any(
+            s is not None for s in [self.is_success_state, self.is_failure_state]
+        )
+
+    @property
+    def is_success_state(self):
+        status = self.current_version_deployment_status()
+        if self._deployment_mode == DEPLOYMENT_READY_CONDITIONS.ATLEAST_ONE_RUNNING:
+            return status["status"]["at_least_one_running"]
+        elif self._deployment_mode == DEPLOYMENT_READY_CONDITIONS.ALL_RUNNING:
+            return status["status"]["all_running"]
+        elif self._deployment_mode == DEPLOYMENT_READY_CONDITIONS.FULLY_FINISHED:
+            return (
+                status["status"]["current_info"]["running"] == self._minimum_replicas
+                and status["status"]["current_info"]["pending"] == 0
+                and status["status"]["current_info"]["crashlooping"] == 0
+            )
+        else:
+            raise ValueError(f"Unknown deployment mode: {self._deployment_mode}")
+
+    @property
+    def is_failure_state(self):
+        status = self.current_version_deployment_status()
+        return status["status"]["at_least_one_crashlooping"]
+
+
 class CapsuleInput:
     @classmethod
     def construct_exec_command(cls, commands: list[str]):
@@ -220,6 +324,7 @@ class CapsuleInput:
             ],
             "port": app_config.get_state("port"),
             "displayName": app_config.get_state("name"),
+            "forceUpdate": app_config.get_state("force_upgrade", False),
         }
 
 
@@ -245,6 +350,15 @@ class CapsuleApiException(Exception):
         )
 
 
+class CapsuleDeploymentException(Exception):
+    def __init__(self, capsule_id: str, message: str):
+        self.capsule_id = capsule_id
+        self.message = message
+
+    def __str__(self):
+        return f"CapsuleDeploymentException: {self.capsule_id} \n\n {self.message}"
+
+
 class CapsuleApi:
     def __init__(self, base_url: str, perimeter: str):
         self._base_url = self._create_base_url(base_url, perimeter)
@@ -391,12 +505,18 @@ class CapsuleApi:
                 message="Capsule JSON decode failed",
             )
 
-    def logs(self, capsule_id: str, worker_id: str) -> List[str]:
+    def logs(
+        self, capsule_id: str, worker_id: str, previous: bool = False
+    ) -> List[str]:
         _url = os.path.join(self._base_url, capsule_id, "workers", worker_id, "logs")
+        options = None
+        if previous:
+            options = {"previous": True}
         response = self._wrapped_api_caller(
             requests.get,
             _url,
             retryable_status_codes=[409],  # todo : verify me
+            params=options,
         )
         try:
             return response.json().get("logs", []) or []
@@ -411,9 +531,9 @@ class CapsuleApi:
 
 
 def list_and_filter_capsules(
-    api_url, perimeter, project, branch, name, tags, auth_type, capsule_id
+    capsule_api: CapsuleApi, project, branch, name, tags, auth_type, capsule_id
 ):
-    capsules = CapsuleApi(api_url, perimeter).list()
+    capsules = capsule_api.list()
 
     def _tags_match(tags, key, value):
         for t in tags:
@@ -473,11 +593,20 @@ class CapsuleDeployer:
         base_url: str,
         create_timeout: int = 60 * 5,
         debug_dir: Optional[str] = None,
+        success_terminal_state_condition: str = DEPLOYMENT_READY_CONDITIONS.ATLEAST_ONE_RUNNING,
+        readiness_wait_time: int = 20,
     ):
         self._app_config = app_config
         self._capsule_api = CapsuleApi(base_url, app_config.get_state("perimeter"))
         self._create_timeout = create_timeout
         self._debug_dir = debug_dir
+        self._capsule_deploy_response = None
+        self._success_terminal_state_condition = success_terminal_state_condition
+        self._readiness_wait_time = readiness_wait_time
+
+    @property
+    def capsule_api(self):
+        return self._capsule_api
 
     @property
     def capsule_type(self):
@@ -498,41 +627,168 @@ class CapsuleDeployer:
             self._app_config, CapsuleInput.from_app_config(self._app_config)
         )
 
+    @property
+    def current_deployment_instance_version(self):
+        """
+        The backend `create` call returns a version of the object that will be
+        """
+        if self._capsule_deploy_response is None:
+            return None
+        return self._capsule_deploy_response.get("version", None)
+
     def create(self):
         capsule_response = self._capsule_api.create(self.create_input())
         self.identifier = capsule_response.get("id")
+        self._capsule_deploy_response = capsule_response
         return self.identifier
 
     def get(self):
         return self._capsule_api.get(self.identifier)
 
-    def wait_for_terminal_state(self, logger=print):
-        state_machine = CapsuleStateMachine(self.identifier)
-        logger(
-            "💊 Waiting for %s %s to be ready to serve traffic"
-            % (self.capsule_type.lower(), self.identifier)
+    def get_workers(self):
+        return self._capsule_api.get_workers(self.identifier)
+
+    def _backend_version_mismatch_check(
+        self, capsule_response: dict, current_deployment_instance_version: str
+    ):
+        """
+        - `capsule_response.version` contains the version of the object present in the database
+        - `current_deployment_instance_version` contains the version of the object that was deployed by this instance of the deployer.
+        In the situtation that the versions of the objects become a mismatch then it means that current deployment process is not giving the user the
+        output that they desire.
+        """
+        if capsule_response.get("version", None) != current_deployment_instance_version:
+            raise CapsuleDeploymentException(
+                self.identifier,
+                f"A capsule upgrade was triggered outside current deployment instance. Current deployment version was discarded. Current deployment version: {current_deployment_instance_version} and new version: {capsule_response.get('version', None)}",
+            )
+
+    def _monitor_worker_readiness(
+        self, workers_sm: "CapsuleWorkersStateMachine", logger=print
+    ):
+        """ """
+        for i in range(self._readiness_wait_time):
+            time.sleep(1)
+            workers_response = self.get_workers()
+            workers_sm.add_status(workers_response)
+            workers_sm.report_current_status(logger)
+
+    def _extract_logs_from_crashlooping_worker(
+        self, workers_sm: "CapsuleWorkersStateMachine"
+    ):
+        def _extract_worker_id_of_crashlooping_worker(
+            workers_status: List[WorkerStatus],
+        ):
+            for worker in workers_status:
+                if worker["phase"] == "CrashLoopBackOff":
+                    return worker["workerId"]
+            return None
+
+        worker_id = _extract_worker_id_of_crashlooping_worker(workers_sm.current_status)
+        if worker_id is None:
+            return None, None
+        logs = self.capsule_api.logs(self.identifier, worker_id, previous=True)
+        return logs, worker_id
+
+    def wait_for_terminal_state(
+        self,
+        logger=print,
+    ):
+        """ """
+        state_machine = CapsuleStateMachine(
+            self.identifier, self.current_deployment_instance_version
+        )
+        min_replicas = self._app_config.get_state("replicas", {}).get("min", 1)
+        workers_state_machine = CapsuleWorkersStateMachine(
+            self.identifier,
+            self.current_deployment_instance_version,
+            deployment_mode=self._success_terminal_state_condition,
+            minimum_replicas=min_replicas,
         )
         self.status = state_machine
         for i in range(self._create_timeout):
+            time.sleep(1)
             capsule_response = self.get()
+            workers_response = self.get_workers()
+
+            # We first need to check if someone has not upgraded the capsule under the hood and
+            # the current deployment instance is invalid.
+            self._backend_version_mismatch_check(
+                capsule_response, self.current_deployment_instance_version
+            )
             state_machine.add_status(capsule_response.get("status", {}))
-            time.sleep(1)
+            workers_state_machine.add_status(workers_response)
             state_machine.report_current_status(logger)
-            if state_machine.ready_to_serve_traffic:
+
+            workers_state_machine.report_current_status(logger)
+            # Deployment readiness checks will determine what is the terminal state
+            # of the workerstate machine. If we detect a terminal state in the workers,
+            # then even if the capsule upgrade is still in progress we will end up crashing
+            # the deployment.
+            if (
+                not state_machine.update_in_progress
+                and workers_state_machine.is_terminal_state
+            ) or (workers_state_machine.is_failure_state):
                 logger(
-                    "💊 %s %s is ready to serve traffic on the URL: %s"
+                    "💊 %s deployment status: %s | worker states: [success :%s | failure :%s ] "
                     % (
-                        self.capsule_type,
-                        self.identifier,
-                        state_machine.out_of_cluster_url,
-                    ),
+                        self.capsule_type.title(),
+                        "in progress"
+                        if state_machine.update_in_progress
+                        else "completed",
+                        workers_state_machine.is_success_state,
+                        workers_state_machine.is_failure_state,
+                    )
                 )
+                if workers_state_machine.is_success_state:
+                    # HACK : monitor the workers for N seconds to make sure they are healthy
+                    # this is a hack. Ideally we should implment a healtcheck as a first class citizen
+                    # but it will take some time to do that so in the meanwhile a timeout set on the cli
+                    # side will be really helpful.
+                    self._monitor_worker_readiness(workers_state_machine, logger)
+
+                elif workers_state_machine.is_failure_state:
+                    # hit the logs endpoint for the worker and get the logs
+                    # Print those logs out on the terminal
+                    # raise an exception that should be caught gracefully by the cli
+                    logs, worker_id = self._extract_logs_from_crashlooping_worker(
+                        workers_state_machine
+                    )
+                    if logs is not None:
+                        # todo: It would be really odd if the logs are not present and we discover something is crashlooping.
+                        # Handle that condition later
+                        logger(
+                            *(
+                                [
+                                    f"💥 Worker ID ({worker_id}) is crashlooping. Please check the following logs for more information: "
+                                ]
+                                + ["\t" + l["message"] for l in logs]
+                            )
+                        )
+                        raise CapsuleDeploymentException(
+                            self.identifier,
+                            f"Worker ID ({worker_id}) is crashlooping. Please check the logs for more information.",
+                        )
+
+                if state_machine.ready_to_serve_traffic:
+                    logger(
+                        "💊 %s %s is ready to serve traffic on the URL: %s"
+                        % (
+                            self.capsule_type,
+                            self.identifier,
+                            state_machine.out_of_cluster_url,
+                        ),
+                    )
+
                 break
+
             if self._debug_dir:
                 state_machine.check_for_debug(self._debug_dir)
+                workers_state_machine.check_for_debug(self._debug_dir)
 
         if not self.status.ready_to_serve_traffic:
-            raise TODOException(
-                f"Capsule {self.identifier} failed to be ready to serve traffic"
+            raise CapsuleDeploymentException(
+                self.identifier,
+                f"Capsule {self.identifier} failed to be ready to serve traffic",
             )
         return capsule_response

outerbounds/apps/cli_to_config.py

@@ -6,7 +6,15 @@ def build_config_from_options(options):
     config = {}
 
     # Set basic fields
-    for key in ["name", "port", "image", "compute_pools", "description", "app_type"]:
+    for key in [
+        "name",
+        "port",
+        "image",
+        "compute_pools",
+        "description",
+        "app_type",
+        "force_upgrade",
+    ]:
         if options.get(key):
             config[key] = options[key]
 

outerbounds/apps/config_schema.yaml

@@ -38,6 +38,11 @@ properties:
     type: string
     description: The description of the app to deploy.
     example: "This is a description of my app."
+  force_upgrade: # Only used in `deploy` command
+    allow_union: true
+    type: boolean
+    description: Whether to force upgrade the app even if it is currently being upgraded.
+    example: true
   app_type: # Only used in `deploy` command
     allow_union: true
     type: string

outerbounds/apps/utils.py

@@ -5,6 +5,138 @@ import json
 import requests
 from metaflow._vendor import click
 from .app_config import CAPSULE_DEBUG
+import sys
+import threading
+import time
+import logging
+import itertools
+
+from outerbounds._vendor.spinner import (
+    Spinners,
+)
+
+
+class MultiStepSpinner:
+    """
+    A spinner that supports multi-step progress and configurable alignment.
+
+    Parameters
+    ----------
+    spinner : Spinners
+        Which spinner frames/interval to use.
+    text : str
+        Static text to display beside the spinner.
+    color : str, optional
+        Click color name.
+    align : {'left','right'}
+        Whether to render the spinner to the left (default) or right of the text.
+    """
+
+    def __init__(
+        self,
+        spinner: Spinners = Spinners.dots,
+        text: str = "",
+        color: str = None,
+        align: str = "right",
+        file=sys.stdout,
+    ):
+        cfg = spinner.value
+        self.frames = cfg["frames"]
+        self.interval = cfg["interval"] / 1000.0
+        self.text = text
+        self.color = color
+        if align not in ("left", "right"):
+            raise ValueError("align must be 'left' or 'right'")
+        self.align = align
+        self._write_file = file
+        # precompute clear length: max frame width + space + text length
+        max_frame = max(self.frames, key=len)
+        self.clear_len = len(self.main_text) + len(max_frame) + 1
+
+        self._stop_evt = threading.Event()
+        self._pause_evt = threading.Event()
+        self._thread = None
+        self._write_lock = threading.Lock()
+
+    @property
+    def main_text(self):
+        # if self.text is a callable then call it
+        if callable(self.text):
+            return self.text()
+        return self.text
+
+    def _spin(self):
+        for frame in itertools.cycle(self.frames):
+            if self._stop_evt.is_set():
+                break
+            if self._pause_evt.is_set():
+                time.sleep(0.05)
+                continue
+
+            # ---- Core logging critical section ----
+            with self._write_lock:
+                symbol = click.style(frame, fg=self.color) if self.color else frame
+                if self.align == "left":
+                    msg = f"{symbol} {self.main_text}"
+                else:
+                    msg = f"{self.main_text} {symbol}"
+
+                click.echo(msg, nl=False, file=self._write_file)
+                click.echo("\r", nl=False, file=self._write_file)
+                self._write_file.flush()
+            # ---- End of critical section ----
+            time.sleep(self.interval)
+        # clear the line when done
+        self._clear_line()
+
+    def _clear_line(self):
+        with self._write_lock:
+            click.echo(" " * self.clear_len, nl=False, file=self._write_file)
+            click.echo("\r", nl=False, file=self._write_file)
+            self._write_file.flush()
+
+    def start(self):
+        if self._thread and self._thread.is_alive():
+            return
+        self._stop_evt.clear()
+        self._pause_evt.clear()
+        self._thread = threading.Thread(target=self._spin, daemon=True)
+        self._thread.start()
+
+    def stop(self):
+        self._stop_evt.set()
+        if self._thread:
+            self._thread.join()
+
+    def log(self, *messages: str):
+        """Pause the spinner, emit a ✔ + message, then resume."""
+        self._pause_evt.set()
+        self._clear_line()
+        # ---- Core logging critical section ----
+        with self._write_lock:
+            self._write_file.flush()
+            for message in messages:
+                click.echo(f"{message}", file=self._write_file, nl=True)
+            self._write_file.flush()
+        # ---- End of critical section ----
+        self._pause_evt.clear()
+
+    def __enter__(self):
+        self.start()
+        return self
+
+    def __exit__(self, exc_type, exc, tb):
+        self.stop()
+
+
+class SpinnerLogHandler(logging.Handler):
+    def __init__(self, spinner: MultiStepSpinner, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.spinner = spinner
+
+    def emit(self, record):
+        msg = self.format(record)
+        self.spinner.log(msg)
 
 
 class MaximumRetriesExceeded(Exception):