ob-metaflow-extensions 1.2.2__tar.gz → 1.2.3__tar.gz

{ob_metaflow_extensions-1.2.2 → ob_metaflow_extensions-1.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ob_metaflow_extensions
-Version: 1.2.2
+Version: 1.2.3
 Summary: Outerbounds Platform Extensions for Metaflow
 Author: Outerbounds, Inc.
 License: Commercial

{ob_metaflow_extensions-1.2.2 → ob_metaflow_extensions-1.2.3}/metaflow_extensions/outerbounds/plugins/apps/app_deploy_decorator.py

@@ -26,6 +26,9 @@ class AppDeployDecorator(StepDecorator):
     package_url = None
     package_sha = None
 
+    MAX_ENTROPY = 6
+    MAX_NAME_LENGTH = 15 - MAX_ENTROPY - 1  # -1 for the hyphen
+
     def step_init(self, flow, graph, step, decos, environment, flow_datastore, logger):
         self.logger = logger
         self.environment = environment
@@ -37,6 +40,23 @@ class AppDeployDecorator(StepDecorator):
             "METAFLOW_CODE_SHA", self.package_sha
         )
 
+    def _extract_project_info(self):
+        project = current.get("project_name")
+        branch = current.get("branch_name")
+        is_production = current.get("is_production")
+        return project, branch, is_production
+
+    def _resolve_default_image(self, flow):
+        # TODO : Resolve the default image over here.
+        pass
+
+    def _resolve_default_name_prefix(self, flow, step_name):
+        # TODO: Only tweek MAX_NAME_LENGTH as backend support allows longer names.
+        base_prefix = (flow.name + "-" + step_name).lower()
+        if len(base_prefix) > self.MAX_NAME_LENGTH:
+            base_prefix = "mf-app"
+        return base_prefix
+
     def task_pre_step(
         self,
         step_name,
@@ -58,22 +78,36 @@ class AppDeployDecorator(StepDecorator):
                 "METAFLOW_CODE_URL or METAFLOW_CODE_SHA is not set. "
                 "Please set METAFLOW_CODE_URL and METAFLOW_CODE_SHA in your environment."
             )
-        default_name = "-".join(current.pathspec.split("/")).lower()
         image = os.environ.get("FASTBAKERY_IMAGE", None)
 
-        hash_key = hashlib.sha256(package_url.encode()).hexdigest()[:6]
-
-        default_name = (
-            (current.flow_name + "-" + current.step_name)[:12] + "-" + hash_key
-        ).lower()
+        # TODO [Apps] - This is temporary. Backend will support longer names in the future.
+        default_name = self._resolve_default_name_prefix(flow, step_name)
+        project, branch, is_production = self._extract_project_info()
+        project_info = {}
+        if project is not None:
+            project_info["metaflow/project"] = project
+            project_info["metaflow/branch"] = branch
+            project_info["metaflow/is_production"] = is_production
+
+        default_tags = {
+            "metaflow/flow_name": flow.name,
+            "metaflow/step_name": step_name,
+            "metaflow/run_id": run_id,
+            "metaflow/task_id": task_id,
+            "metaflow/retry_count": retry_count,
+            "metaflow/pathspec": current.pathspec,
+            **project_info,
+        }
 
         AppDeployer._set_state(
             perimeter,
             api_server,
             code_package_url=package_url,
             code_package_key=package_sha,
-            name=default_name,
+            name_prefix=default_name,
             image=image,
+            max_entropy=self.MAX_ENTROPY,
+            default_tags=[{k: str(v)} for k, v in default_tags.items()],
         )
         current._update_env(
             {

{ob_metaflow_extensions-1.2.2 → ob_metaflow_extensions-1.2.3}/metaflow_extensions/outerbounds/plugins/apps/core/_state_machine.py

@@ -189,6 +189,10 @@ class CurrentWorkerInfo(TypedDict):
     crashlooping: int
 
 
+class LogLine(TypedDict):
+    message: str
+
+
 class DEPLOYMENT_READY_CONDITIONS:
     """
     Deployment ready conditions define what is considered a successful completion of the current deployment instance.

{ob_metaflow_extensions-1.2.2 → ob_metaflow_extensions-1.2.3}/metaflow_extensions/outerbounds/plugins/apps/core/capsule.py

@@ -18,6 +18,7 @@ from ._state_machine import (
     WorkerStatus,
     CapsuleStatus,
     DEPLOYMENT_READY_CONDITIONS,
+    LogLine,
 )
 
 
@@ -508,7 +509,7 @@ class CapsuleApi:
 
     def logs(
         self, capsule_id: str, worker_id: str, previous: bool = False
-    ) -> List[str]:
+    ) -> List[LogLine]:
         _url = os.path.join(self._base_url, capsule_id, "workers", worker_id, "logs")
         options = None
         if previous:
@@ -530,6 +531,20 @@ class CapsuleApi:
                 message="Capsule JSON decode failed",
             )
 
+    def patch(self, capsule_id: str, patch_input: dict):
+        capsule_response = self.get(capsule_id)
+        if "spec" not in capsule_response or len(capsule_response.get("spec", {})) == 0:
+            raise CapsuleApiException(
+                self._base_url,
+                "patch",
+                403,
+                "Capsule response of incorrect format",
+            )
+
+        spec = capsule_response.get("spec")
+        spec.update(patch_input)
+        return self.create(spec)
+
 
 def list_and_filter_capsules(
     capsule_api: CapsuleApi, project, branch, name, tags, auth_type, capsule_id

{ob_metaflow_extensions-1.2.2 → ob_metaflow_extensions-1.2.3}/metaflow_extensions/outerbounds/plugins/apps/core/config/__init__.py

@@ -9,4 +9,4 @@ from .config_utils import (
     RequiredFieldMissingException,
 )
 from . import schema_export
-from .typed_configs import TypedCoreConfig
+from .typed_configs import TypedCoreConfig, TypedDict

{ob_metaflow_extensions-1.2.2 → ob_metaflow_extensions-1.2.3}/metaflow_extensions/outerbounds/plugins/apps/core/config/unified_config.py

@@ -527,6 +527,10 @@ class PackageConfig(metaclass=ConfigMeta):
     )
 
 
+def everything_is_string(*args):
+    return all(isinstance(arg, str) for arg in args)
+
+
 class BasicAppValidations:
     @staticmethod
     def name(name):
@@ -534,7 +538,7 @@ class BasicAppValidations:
             return True
         regex = r"^[a-z0-9-]+$"  # Only allow lowercase letters, numbers, and hyphens
         validator = BasicValidations(CoreConfig, "name")
-        return validator.length_validation(20, name) and validator.regex_validation(
+        return validator.length_validation(15, name) and validator.regex_validation(
             regex, name
         )
 
@@ -548,12 +552,19 @@ class BasicAppValidations:
     def tags(tags):
         if tags is None:
             return True
-        if not all(isinstance(tag, dict) and len(tag) == 1 for tag in tags):
+        if not all(
+            isinstance(tag, dict)
+            and len(tag) == 1
+            and all(
+                [everything_is_string(*tag.keys()), everything_is_string(*tag.values())]
+            )
+            for tag in tags
+        ):
             raise ConfigValidationFailedException(
                 field_name="tags",
                 field_info=CoreConfig._get_field(CoreConfig, "tags"),  # type: ignore
                 current_value=tags,
-                message="Tags must be a list of dictionaries with one key. Currently they are set to %s "
+                message="Tags must be a list of dictionaries with one key and the value must be a string. Currently they are set to %s "
                 % (str(tags)),
             )
         return True

ob_metaflow_extensions-1.2.3/metaflow_extensions/outerbounds/plugins/apps/core/deployer.py

@@ -0,0 +1,309 @@
+from .config import TypedCoreConfig, TypedDict
+from .perimeters import PerimeterExtractor
+from .capsule import CapsuleApi
+import json
+from ._state_machine import DEPLOYMENT_READY_CONDITIONS, LogLine
+from .app_config import AppConfig, AppConfigError
+from .capsule import CapsuleDeployer, list_and_filter_capsules
+from functools import partial
+import sys
+import uuid
+from typing import Type, Dict, List
+from datetime import datetime
+
+
+class AppDeployer(TypedCoreConfig):
+    """ """
+
+    __init__ = TypedCoreConfig.__init__
+
+    _app_config: AppConfig
+
+    _state = {}
+
+    __state_items = [
+        "perimeter",
+        "api_url",
+        "code_package_url",
+        "code_package_key",
+        "image",
+        "project",
+        "branch",
+    ]
+
+    @property
+    def _deploy_config(self) -> AppConfig:
+        if not hasattr(self, "_app_config"):
+            self._app_config = AppConfig(self._config)
+        return self._app_config
+
+    # Things that need to be set before deploy
+    @classmethod
+    def _set_state(
+        cls,
+        perimeter: str,
+        api_url: str,
+        code_package_url: str = None,
+        code_package_key: str = None,
+        name_prefix: str = None,
+        image: str = None,
+        max_entropy: int = 4,
+        default_tags: List[Dict[str, str]] = None,
+        project: str = None,
+        branch: str = None,
+    ):
+        cls._state["perimeter"] = perimeter
+        cls._state["api_url"] = api_url
+        cls._state["code_package_url"] = code_package_url
+        cls._state["code_package_key"] = code_package_key
+        cls._state["name_prefix"] = name_prefix
+        cls._state["image"] = image
+        cls._state["max_entropy"] = max_entropy
+        cls._state["default_tags"] = default_tags
+        cls._state["project"] = project
+        cls._state["branch"] = branch
+
+        assert (
+            max_entropy > 0
+        ), "max_entropy must be greater than 0. Since AppDeployer's deploy fn can be called many time inside a step itself."
+
+    def deploy(
+        self,
+        readiness_condition=DEPLOYMENT_READY_CONDITIONS.ATLEAST_ONE_RUNNING,
+        max_wait_time=600,
+        readiness_wait_time=10,
+        logger_fn=partial(print, file=sys.stderr),
+        status_file=None,
+        no_loader=False,
+        **kwargs,
+    ) -> "DeployedApp":
+
+        # Name setting from top level if none is set in the code
+        if self._deploy_config._core_config.name is None:
+            name = self._state[
+                "name_prefix"
+            ]  # for now the name-prefix cannot be very large.
+            entropy = uuid.uuid4().hex[: self._state["max_entropy"]]
+            self._deploy_config._core_config.name = f"{name}-{entropy}"
+
+        if len(self._state["default_tags"]) > 0:
+            self._deploy_config._core_config.tags = (
+                self._deploy_config._core_config.tags or []
+            ) + self._state["default_tags"]
+
+        self._deploy_config.commit()
+        # Set any state that might have been passed down from the top level
+        for k in self.__state_items:
+            if self._deploy_config.get_state(k) is None:
+                self._deploy_config.set_state(k, self._state[k])
+
+        capsule = CapsuleDeployer(
+            self._deploy_config,
+            self._state["api_url"],
+            create_timeout=max_wait_time,
+            debug_dir=None,
+            success_terminal_state_condition=readiness_condition,
+            readiness_wait_time=readiness_wait_time,
+            logger_fn=logger_fn,
+        )
+
+        currently_present_capsules = list_and_filter_capsules(
+            capsule.capsule_api,
+            None,
+            None,
+            capsule.name,
+            None,
+            None,
+            None,
+        )
+
+        force_upgrade = self._deploy_config.get_state("force_upgrade", False)
+
+        if len(currently_present_capsules) > 0:
+            # Only update the capsule if there is no upgrade in progress
+            # Only update a "already updating" capsule if the `--force-upgrade` flag is provided.
+            _curr_cap = currently_present_capsules[0]
+            this_capsule_is_being_updated = _curr_cap.get("status", {}).get(
+                "updateInProgress", False
+            )
+
+            if this_capsule_is_being_updated and not force_upgrade:
+                _upgrader = _curr_cap.get("metadata", {}).get("lastModifiedBy", None)
+                message = f"{capsule.capsule_type} is currently being upgraded"
+                if _upgrader:
+                    message = (
+                        f"{capsule.capsule_type} is currently being upgraded. Upgrade was launched by {_upgrader}. "
+                        "If you wish to force upgrade, you can do so by providing the `--force-upgrade` flag."
+                    )
+                raise AppConfigError(message)
+
+            logger_fn(
+                f"🚀 {'' if not force_upgrade else 'Force'} Upgrading {capsule.capsule_type.lower()} `{capsule.name}`....",
+            )
+        else:
+            logger_fn(
+                f"🚀 Deploying {capsule.capsule_type.lower()} `{capsule.name}`....",
+            )
+
+        capsule.create()
+        final_status = capsule.wait_for_terminal_state()
+        return DeployedApp(
+            final_status["id"],
+            final_status["auth_type"],
+            final_status["public_url"],
+            final_status["available_replicas"],
+            final_status["name"],
+            final_status["deployed_version"],
+            final_status["deployed_at"],
+        )
+
+
+class DeployedApp:
+    def __init__(
+        self,
+        _id: str,
+        capsule_type: str,
+        public_url: str,
+        available_replicas: int,
+        name: str,
+        deployed_version: str,
+        deployed_at: str,
+    ):
+        self._id = _id
+        self._capsule_type = capsule_type
+        self._public_url = public_url
+        self._available_replicas = available_replicas
+        self._name = name
+        self._deployed_version = deployed_version
+        self._deployed_at = deployed_at
+
+    def _get_capsule_api(self) -> CapsuleApi:
+        perimeter, api_server = PerimeterExtractor.during_metaflow_execution()
+        return CapsuleApi(api_server, perimeter)
+
+    def logs(self, previous=False) -> Dict[str, List[LogLine]]:
+        """
+        Returns a dictionary of worker_id to logs.
+        If `previous` is True, it will return the logs from the previous execution of the workers. Useful when debugging a crashlooping worker.
+        """
+        capsule_api = self._get_capsule_api()
+        # extract workers from capsule
+        workers = capsule_api.get_workers(self._id)
+        # get logs from workers
+        logs = {
+            # worker_id: logs
+        }
+        for worker in workers:
+            # TODO: Handle exceptions better over here.
+            logs[worker["workerId"]] = capsule_api.logs(
+                self._id, worker["workerId"], previous=previous
+            )
+        return logs
+
+    def info(self) -> dict:
+        """
+        Returns a dictionary representing the deployed app.
+        """
+        capsule_api = self._get_capsule_api()
+        capsule = capsule_api.get(self._id)
+        return capsule
+
+    def scale_to_zero(self):
+        """
+        Scales the DeployedApp to 0 replicas.
+        """
+        capsule_api = self._get_capsule_api()
+        return capsule_api.patch(
+            self._id,
+            {
+                "autoscalingConfig": {
+                    "minReplicas": 0,
+                    "maxReplicas": 0,
+                }
+            },
+        )
+
+    def delete(self):
+        """
+        Deletes the DeployedApp.
+        """
+        capsule_api = self._get_capsule_api()
+        return capsule_api.delete(self._id)
+
+    @property
+    def id(self) -> str:
+        return self._id
+
+    @property
+    def auth_style(self) -> str:
+        # TODO : Fix naming here.
+        return self._capsule_type
+
+    @property
+    def public_url(self) -> str:
+        return self._public_url
+
+    @property
+    def available_replicas(self) -> int:
+        return self._available_replicas
+
+    @property
+    def name(self) -> str:
+        return self._name
+
+    @property
+    def deployed_version(self) -> str:
+        return self._deployed_version
+
+    def to_dict(self) -> dict:
+        return {
+            "id": self._id,
+            "auth_style": self.auth_style,  # TODO : Fix naming here.
+            "public_url": self._public_url,
+            "available_replicas": self._available_replicas,
+            "name": self._name,
+            "deployed_version": self._deployed_version,
+            "deployed_at": self._deployed_at,
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(
+            _id=data["id"],
+            capsule_type=data["capsule_type"],
+            public_url=data["public_url"],
+            available_replicas=data["available_replicas"],
+            name=data["name"],
+            deployed_version=data["deployed_version"],
+            deployed_at=data["deployed_at"],
+        )
+
+    @property
+    def deployed_at(self) -> datetime:
+        return datetime.fromisoformat(self._deployed_at)
+
+    def __repr__(self) -> str:
+        return (
+            f"DeployedApp(id='{self._id}', "
+            f"name='{self._name}', "
+            f"public_url='{self._public_url}', "
+            f"available_replicas={self._available_replicas}, "
+            f"deployed_version='{self._deployed_version}')"
+        )
+
+
+class apps:
+
+    _name_prefix = None
+
+    @classmethod
+    def set_name_prefix(cls, name_prefix: str):
+        cls._name_prefix = name_prefix
+
+    @property
+    def name_prefix(self) -> str:
+        return self._name_prefix
+
+    @property
+    def Deployer(self) -> Type[AppDeployer]:
+        return AppDeployer

{ob_metaflow_extensions-1.2.2 → ob_metaflow_extensions-1.2.3}/ob_metaflow_extensions.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ob-metaflow-extensions
-Version: 1.2.2
+Version: 1.2.3
 Summary: Outerbounds Platform Extensions for Metaflow
 Author: Outerbounds, Inc.
 License: Commercial

{ob_metaflow_extensions-1.2.2 → ob_metaflow_extensions-1.2.3}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_namespace_packages
 from pathlib import Path
 
 
-version = "1.2.2"
+version = "1.2.3"
 this_directory = Path(__file__).parent
 long_description = (this_directory / "README.md").read_text()
 

ob_metaflow_extensions-1.2.2/metaflow_extensions/outerbounds/plugins/apps/core/deployer.py

@@ -1,132 +0,0 @@
-from .config import TypedCoreConfig
-
-from ._state_machine import DEPLOYMENT_READY_CONDITIONS
-from .app_config import AppConfig, AppConfigError
-from .capsule import CapsuleDeployer, list_and_filter_capsules
-from functools import partial
-import sys
-from typing import Type
-
-
-class AppDeployer(TypedCoreConfig):
-    """ """
-
-    __init__ = TypedCoreConfig.__init__
-
-    _app_config: AppConfig
-
-    _state = {}
-
-    @property
-    def app_config(self) -> AppConfig:
-        if not hasattr(self, "_app_config"):
-            self._app_config = AppConfig(self._config)
-        return self._app_config
-
-    # Things that need to be set before deploy
-    @classmethod
-    def _set_state(
-        cls,
-        perimeter: str,
-        api_url: str,
-        code_package_url: str = None,
-        code_package_key: str = None,
-        name: str = None,
-        image: str = None,
-    ):
-        cls._state["perimeter"] = perimeter
-        cls._state["api_url"] = api_url
-        cls._state["code_package_url"] = code_package_url
-        cls._state["code_package_key"] = code_package_key
-        cls._state["name"] = name
-        cls._state["image"] = image
-
-    def deploy(
-        self,
-        readiness_condition=DEPLOYMENT_READY_CONDITIONS.ATLEAST_ONE_RUNNING,
-        max_wait_time=600,
-        readiness_wait_time=10,
-        logger_fn=partial(print, file=sys.stderr),
-        status_file=None,
-        no_loader=False,
-        **kwargs,
-    ):
-        # Name setting from top level if none is set in the code
-        if self.app_config._core_config.name is None:
-            self.app_config._core_config.name = self._state["name"]
-
-        self.app_config.commit()
-
-        # Set any state that might have been passed down from the top level
-        for k, v in self._state.items():
-            if self.app_config.get_state(k) is None:
-                self.app_config.set_state(k, v)
-
-        capsule = CapsuleDeployer(
-            self.app_config,
-            self._state["api_url"],
-            create_timeout=max_wait_time,
-            debug_dir=None,
-            success_terminal_state_condition=readiness_condition,
-            readiness_wait_time=readiness_wait_time,
-            logger_fn=logger_fn,
-        )
-
-        currently_present_capsules = list_and_filter_capsules(
-            capsule.capsule_api,
-            None,
-            None,
-            capsule.name,
-            None,
-            None,
-            None,
-        )
-
-        force_upgrade = self.app_config.get_state("force_upgrade", False)
-
-        if len(currently_present_capsules) > 0:
-            # Only update the capsule if there is no upgrade in progress
-            # Only update a "already updating" capsule if the `--force-upgrade` flag is provided.
-            _curr_cap = currently_present_capsules[0]
-            this_capsule_is_being_updated = _curr_cap.get("status", {}).get(
-                "updateInProgress", False
-            )
-
-            if this_capsule_is_being_updated and not force_upgrade:
-                _upgrader = _curr_cap.get("metadata", {}).get("lastModifiedBy", None)
-                message = f"{capsule.capsule_type} is currently being upgraded"
-                if _upgrader:
-                    message = (
-                        f"{capsule.capsule_type} is currently being upgraded. Upgrade was launched by {_upgrader}. "
-                        "If you wish to force upgrade, you can do so by providing the `--force-upgrade` flag."
-                    )
-                raise AppConfigError(message)
-
-            logger_fn(
-                f"🚀 {'' if not force_upgrade else 'Force'} Upgrading {capsule.capsule_type.lower()} `{capsule.name}`....",
-            )
-        else:
-            logger_fn(
-                f"🚀 Deploying {capsule.capsule_type.lower()} `{capsule.name}`....",
-            )
-
-        capsule.create()
-        final_status = capsule.wait_for_terminal_state()
-        return final_status
-
-
-class apps:
-
-    _name_prefix = None
-
-    @classmethod
-    def set_name_prefix(cls, name_prefix: str):
-        cls._name_prefix = name_prefix
-
-    @property
-    def name_prefix(self) -> str:
-        return self._name_prefix
-
-    @property
-    def Deployer(self) -> Type[AppDeployer]:
-        return AppDeployer