ob-metaflow-extensions 1.1.175rc2__py2.py3-none-any.whl → 1.1.175rc4__py2.py3-none-any.whl

metaflow_extensions/outerbounds/plugins/apps/core/config/unified_config.py

@@ -12,7 +12,7 @@ No external dependencies required - uses only Python standard library.
 import os
 import json
 from typing import Any, Dict, List, Optional, Union, Type
-
+import re
 
 from .config_utils import (
     ConfigField,
@@ -41,9 +41,142 @@ class AuthType:
         return [cls.BROWSER, cls.API]
 
 
+class UnitParser:
+    UNIT_FREE_REGEX = r"^\d+$"
+
+    metrics = {
+        "memory": {
+            "default_unit": "Mi",
+            "requires_unit": True,  # if a Unit free value is provided then we will add the default unit to it.
+            # Regex to match values with units (e.g., "512Mi", "4Gi", "1024Ki")
+            "unit_regex": r"^\d+(\.\d+)?(Ki|Mi|Gi|Ti|Pi|Ei)$",
+        },
+        "cpu": {
+            "default_unit": None,
+            "requires_unit": False,  # if a Unit free value is provided then we will not add the default unit to it.
+            # Accepts values like 400m, 4, 0.4, 1000n, etc.
+            # Regex to match values with units (e.g., "400m", "1000n", "2", "0.5")
+            "unit_regex": r"^(\d+(\.\d+)?(m|n)?|\d+(\.\d+)?)$",
+        },
+        "disk": {
+            "default_unit": "Mi",
+            "requires_unit": True,  # if a Unit free value is provided then we will add the default unit to it.
+            # Regex to match values with units (e.g., "100Mi", "1Gi", "500Ki")
+            "unit_regex": r"^\d+(\.\d+)?(Ki|Mi|Gi|Ti|Pi|Ei)$",
+        },
+        "gpu": {
+            "default_unit": None,
+            "requires_unit": False,
+            # Regex to match values with units (usually just integer count, e.g., "1", "2")
+            "unit_regex": r"^\d+$",
+        },
+    }
+
+    def __init__(self, metric_name: str):
+        self.metric_name = metric_name
+
+    def validate(self, value: str):
+        if self.metrics[self.metric_name]["requires_unit"]:
+            if not re.match(self.metrics[self.metric_name]["unit_regex"], value):
+                return False
+        return True
+
+    def process(self, value: str):
+        value = str(value)
+        if self.metrics[self.metric_name]["requires_unit"]:
+            if re.match(self.UNIT_FREE_REGEX, value):
+                # This means the value is unit free and we need to add the default unit to it.
+                value = "%s%s" % (
+                    value.strip(),
+                    self.metrics[self.metric_name]["default_unit"],
+                )
+                return value
+
+            if re.match(self.metrics[self.metric_name]["unit_regex"], value):
+                return value
+
+        return value
+
+    def parse(self, value: str):
+        if value is None:
+            return None
+        return self.process(value)
+
+    @staticmethod
+    def validation_wrapper_fn(
+        metric_name: str,
+    ):
+        def validation_fn(value: str):
+            if value is None:
+                return True
+            field_info = ResourceConfig._get_field(ResourceConfig, metric_name)  # type: ignore
+            parser = UnitParser(metric_name)
+            validation = parser.validate(value)
+            if not validation:
+                raise ConfigValidationFailedException(
+                    field_name=metric_name,
+                    field_info=field_info,
+                    current_value=value,
+                    message=f"Invalid value for `{metric_name}`. Must be of the format {parser.metrics[metric_name]['unit_regex']}.",
+                )
+            return validation
+
+        return validation_fn
+
+
+class BasicValidations:
+    def __init__(self, config_meta_class, field_name):
+        self.config_meta_class = config_meta_class
+        self.field_name = field_name
+
+    def _get_field(self):
+        return self.config_meta_class._get_field(self.config_meta_class, self.field_name)  # type: ignore
+
+    def enum_validation(self, enums: List[str], current_value):
+        if current_value not in enums:
+            raise ConfigValidationFailedException(
+                field_name=self.field_name,
+                field_info=self._get_field(),
+                current_value=current_value,
+                message=f"Configuration field {self.field_name} has invalid value {current_value}. Value must be one of: {' '.join(enums)}",
+            )
+        return True
+
+    def range_validation(self, min_value, max_value, current_value):
+        if current_value < min_value or current_value > max_value:
+            raise ConfigValidationFailedException(
+                field_name=self.field_name,
+                field_info=self._get_field(),
+                current_value=current_value,
+                message=f"Configuration field {self.field_name} has invalid value {current_value}. Value must be between {min_value} and {max_value}",
+            )
+        return True
+
+    def length_validation(self, max_length, current_value):
+        if len(current_value) > max_length:
+            raise ConfigValidationFailedException(
+                field_name=self.field_name,
+                field_info=self._get_field(),
+                current_value=current_value,
+                message=f"Configuration field {self.field_name} has invalid value {current_value}. Value must be less than {max_length}",
+            )
+        return True
+
+    def regex_validation(self, regex, current_value):
+        if not re.match(regex, current_value):
+            raise ConfigValidationFailedException(
+                field_name=self.field_name,
+                field_info=self._get_field(),
+                current_value=current_value,
+                message=f"Configuration field {self.field_name} has invalid value {current_value}. Value must match regex {regex}",
+            )
+        return True
+
+
 class ResourceConfig(metaclass=ConfigMeta):
     """Resource configuration for the app."""
 
+    # TODO: Add Unit Validation/Parsing Support for the Fields.
     cpu = ConfigField(
         default="1",
         cli_meta=CLIOption(
@@ -53,6 +186,8 @@ class ResourceConfig(metaclass=ConfigMeta):
         ),
         field_type=str,
         example="500m",
+        validation_fn=UnitParser.validation_wrapper_fn("cpu"),
+        parsing_fn=UnitParser("cpu").parse,
     )
     memory = ConfigField(
         default="4Gi",
@@ -63,6 +198,8 @@ class ResourceConfig(metaclass=ConfigMeta):
         ),
         field_type=str,
         example="512Mi",
+        validation_fn=UnitParser.validation_wrapper_fn("memory"),
+        parsing_fn=UnitParser("memory").parse,
     )
     gpu = ConfigField(
         cli_meta=CLIOption(
@@ -72,6 +209,8 @@ class ResourceConfig(metaclass=ConfigMeta):
         ),
         field_type=str,
         example="1",
+        validation_fn=UnitParser.validation_wrapper_fn("gpu"),
+        parsing_fn=UnitParser("gpu").parse,
     )
     disk = ConfigField(
         default="20Gi",
@@ -82,6 +221,8 @@ class ResourceConfig(metaclass=ConfigMeta):
         ),
         field_type=str,
         example="1Gi",
+        validation_fn=UnitParser.validation_wrapper_fn("disk"),
+        parsing_fn=UnitParser("disk").parse,
     )
 
 
@@ -156,14 +297,11 @@ class AuthConfig(metaclass=ConfigMeta):
 
     @staticmethod
     def validate(auth_config: "AuthConfig"):
-        if auth_config.type is not None and auth_config.type not in AuthType.choices():
-            raise ConfigValidationFailedException(
-                field_name="type",
-                field_info=auth_config._get_field("type"),
-                current_value=auth_config.type,
-                message=f"Invalid auth type: {auth_config.type}. Must be one of {AuthType.choices()}",
-            )
-        return True
+        if auth_config.type is None:
+            return True
+        return BasicValidations(AuthConfig, "type").enum_validation(
+            AuthType.choices(), auth_config.type
+        )
 
 
 class ReplicaConfig(metaclass=ConfigMeta):
@@ -207,12 +345,14 @@ class ReplicaConfig(metaclass=ConfigMeta):
                 replica_config.fixed is None,
             ]
         ):
+            # if nothing is set then set
             replica_config.fixed = 1
+        elif replica_config.min is not None and replica_config.max is None:
+            replica_config.max = replica_config.min
         return
 
     @staticmethod
     def validate(replica_config: "ReplicaConfig"):
-        # TODO: Have a better validation story.
         both_min_max_set = (
             replica_config.min is not None and replica_config.max is not None
         )
@@ -226,17 +366,17 @@ class ReplicaConfig(metaclass=ConfigMeta):
         def _greater_than_equals_zero(x):
             return x is not None and x >= 0
 
-        if both_min_max_set and replica_config.min > replica_config.max:
+        if both_min_max_set and replica_config.min > replica_config.max:  # type: ignore
             raise ConfigValidationFailedException(
                 field_name="min",
-                field_info=replica_config._get_field("min"),
+                field_info=replica_config._get_field("min"),  # type: ignore
                 current_value=replica_config.min,
                 message="Min replicas cannot be greater than max replicas",
             )
         if fixed_set and any_min_max_set:
             raise ConfigValidationFailedException(
                 field_name="fixed",
-                field_info=replica_config._get_field("fixed"),
+                field_info=replica_config._get_field("fixed"),  # type: ignore
                 current_value=replica_config.fixed,
                 message="Fixed replicas cannot be set when min or max replicas are set",
             )
@@ -244,15 +384,15 @@ class ReplicaConfig(metaclass=ConfigMeta):
         if max_is_set and not min_is_set:
             raise ConfigValidationFailedException(
                 field_name="min",
-                field_info=replica_config._get_field("min"),
+                field_info=replica_config._get_field("min"),  # type: ignore
                 current_value=replica_config.min,
                 message="If max replicas is set then min replicas must be set too.",
             )
 
-        if fixed_set and replica_config.fixed < 0:
+        if fixed_set and replica_config.fixed < 0:  # type: ignore
             raise ConfigValidationFailedException(
                 field_name="fixed",
-                field_info=replica_config._get_field("fixed"),
+                field_info=replica_config._get_field("fixed"),  # type: ignore
                 current_value=replica_config.fixed,
                 message="Fixed replicas cannot be less than 0",
             )
@@ -260,7 +400,7 @@ class ReplicaConfig(metaclass=ConfigMeta):
         if min_is_set and not _greater_than_equals_zero(replica_config.min):
             raise ConfigValidationFailedException(
                 field_name="min",
-                field_info=replica_config._get_field("min"),
+                field_info=replica_config._get_field("min"),  # type: ignore
                 current_value=replica_config.min,
                 message="Min replicas cannot be less than 0",
             )
@@ -268,7 +408,7 @@ class ReplicaConfig(metaclass=ConfigMeta):
         if max_is_set and not _greater_than_equals_zero(replica_config.max):
             raise ConfigValidationFailedException(
                 field_name="max",
-                field_info=replica_config._get_field("max"),
+                field_info=replica_config._get_field("max"),  # type: ignore
                 current_value=replica_config.max,
                 message="Max replicas cannot be less than 0",
             )
@@ -346,7 +486,7 @@ class DependencyConfig(metaclass=ConfigMeta):
         ):
             raise ConfigValidationFailedException(
                 field_name="from_requirements_file",
-                field_info=dependency_config._get_field("from_requirements_file"),
+                field_info=dependency_config._get_field("from_requirements_file"),  # type: ignore
                 current_value=dependency_config.from_requirements_file,
                 message="Cannot set from_requirements_file and from_pyproject_toml at the same time",
             )
@@ -358,7 +498,7 @@ class DependencyConfig(metaclass=ConfigMeta):
         ):
             raise ConfigValidationFailedException(
                 field_name="pypi" if dependency_config.pypi else "conda",
-                field_info=dependency_config._get_field(
+                field_info=dependency_config._get_field(  # type: ignore
                     "pypi" if dependency_config.pypi else "conda"
                 ),
                 current_value=dependency_config.pypi or dependency_config.conda,
@@ -395,27 +535,17 @@ class BasicAppValidations:
     def name(name):
         if name is None:
             return True
-        if len(name) > 128:
-            raise ConfigValidationFailedException(
-                field_name="name",
-                field_info=CoreConfig._get_field(CoreConfig, "name"),
-                current_value=name,
-                message="Name cannot be longer than 128 characters",
-            )
-        return True
+        regex = r"^[a-z0-9-]+$"  # Only allow lowercase letters, numbers, and hyphens
+        validator = BasicValidations(CoreConfig, "name")
+        return validator.length_validation(20, name) and validator.regex_validation(
+            regex, name
+        )
 
     @staticmethod
     def port(port):
         if port is None:
             return True
-        if port < 1 or port > 65535:
-            raise ConfigValidationFailedException(
-                field_name="port",
-                field_info=CoreConfig._get_field(CoreConfig, "port"),
-                current_value=port,
-                message="Port must be between 1 and 65535",
-            )
-        return True
+        return BasicValidations(CoreConfig, "port").range_validation(1, 65535, port)
 
     @staticmethod
     def tags(tags):
@@ -424,7 +554,7 @@ class BasicAppValidations:
         if not all(isinstance(tag, dict) and len(tag) == 1 for tag in tags):
             raise ConfigValidationFailedException(
                 field_name="tags",
-                field_info=CoreConfig._get_field(CoreConfig, "tags"),
+                field_info=CoreConfig._get_field(CoreConfig, "tags"),  # type: ignore
                 current_value=tags,
                 message="Tags must be a list of dictionaries with one key. Currently they are set to %s "
                 % (str(tags)),
@@ -439,9 +569,10 @@ class BasicAppValidations:
         if not isinstance(secrets, list):
             raise ConfigValidationFailedException(
                 field_name="secrets",
-                field_info=CoreConfig._get_field(CoreConfig, "secrets"),
+                field_info=CoreConfig._get_field(CoreConfig, "secrets"),  # type: ignore
                 current_value=secrets,
-                message="Secrets must be a list of strings",
+                message="Secrets must be a list of strings. Currently they are set to %s "
+                % (str(secrets)),
             )
         from ..validations import secrets_validator
 
@@ -450,7 +581,7 @@ class BasicAppValidations:
         except Exception as e:
             raise ConfigValidationFailedException(
                 field_name="secrets",
-                field_info=CoreConfig._get_field(CoreConfig, "secrets"),
+                field_info=CoreConfig._get_field(CoreConfig, "secrets"),  # type: ignore
                 current_value=secrets,
                 message=f"Secrets validation failed, {e}",
             )
@@ -460,14 +591,9 @@ class BasicAppValidations:
     def persistence(persistence):
         if persistence is None:
             return True
-        if persistence not in ["none", "postgres"]:
-            raise ConfigValidationFailedException(
-                field_name="persistence",
-                field_info=CoreConfig._get_field(CoreConfig, "persistence"),
-                current_value=persistence,
-                message=f"Persistence must be one of: {['none', 'postgres']}",
-            )
-        return True
+        return BasicValidations(CoreConfig, "persistence").enum_validation(
+            ["none", "postgres"], persistence
+        )
 
 
 class CoreConfig(metaclass=ConfigMeta):
@@ -787,7 +913,7 @@ How to read this schema:
         merged_config = CoreConfig()
 
         # Process each field according to its behavior
-        for field_name, field_info in CoreConfig._fields.items():
+        for field_name, field_info in CoreConfig._fields.items():  # type: ignore
             base_value = getattr(base_config, field_name, None)
             override_value = getattr(override_config, field_name, None)
 
@@ -852,7 +978,7 @@ How to read this schema:
                 for v in value:
                     _env_dict.update(v)
                 return _env_dict
-            if type(value) == tuple:
+            if type(value) == tuple or type(value) == list:
                 obj = list(x for x in source_data[key])
                 if len(obj) == 0:
                     return None  # Dont return Empty Lists so that we can set Nones

metaflow_extensions/outerbounds/plugins/apps/core/config_schema.yaml

@@ -94,24 +94,24 @@ properties:
     required: []
     properties:
       cpu:
-        description: CPU resource request and limit.
+        description: CPU resource request and limit. (validation applied)
         type: string
         default: '1'
         example: 500m
         mutation_behavior: union
       memory:
-        description: Memory resource request and limit.
+        description: Memory resource request and limit. (validation applied)
         type: string
         default: 4Gi
         example: 512Mi
         mutation_behavior: union
       gpu:
-        description: GPU resource request and limit.
+        description: GPU resource request and limit. (validation applied)
         type: string
         example: '1'
         mutation_behavior: union
       disk:
-        description: Storage resource request and limit.
+        description: Storage resource request and limit. (validation applied)
         type: string
         default: 20Gi
         example: 1Gi

metaflow_extensions/outerbounds/plugins/apps/core/deployer.py

@@ -0,0 +1,132 @@
+from .config import TypedCoreConfig
+
+from ._state_machine import DEPLOYMENT_READY_CONDITIONS
+from .app_config import AppConfig, AppConfigError
+from .capsule import CapsuleDeployer, list_and_filter_capsules
+from functools import partial
+import sys
+from typing import Type
+
+
+class AppDeployer(TypedCoreConfig):
+    """ """
+
+    __init__ = TypedCoreConfig.__init__
+
+    _app_config: AppConfig
+
+    _state = {}
+
+    @property
+    def app_config(self) -> AppConfig:
+        if not hasattr(self, "_app_config"):
+            self._app_config = AppConfig(self._config)
+        return self._app_config
+
+    # Things that need to be set before deploy
+    @classmethod
+    def _set_state(
+        cls,
+        perimeter: str,
+        api_url: str,
+        code_package_url: str = None,
+        code_package_key: str = None,
+        name: str = None,
+        image: str = None,
+    ):
+        cls._state["perimeter"] = perimeter
+        cls._state["api_url"] = api_url
+        cls._state["code_package_url"] = code_package_url
+        cls._state["code_package_key"] = code_package_key
+        cls._state["name"] = name
+        cls._state["image"] = image
+
+    def deploy(
+        self,
+        readiness_condition=DEPLOYMENT_READY_CONDITIONS.ATLEAST_ONE_RUNNING,
+        max_wait_time=600,
+        readiness_wait_time=10,
+        logger_fn=partial(print, file=sys.stderr),
+        status_file=None,
+        no_loader=False,
+        **kwargs,
+    ):
+        # Name setting from top level if none is set in the code
+        if self.app_config._core_config.name is None:
+            self.app_config._core_config.name = self._state["name"]
+
+        self.app_config.commit()
+
+        # Set any state that might have been passed down from the top level
+        for k, v in self._state.items():
+            if self.app_config.get_state(k) is None:
+                self.app_config.set_state(k, v)
+
+        capsule = CapsuleDeployer(
+            self.app_config,
+            self._state["api_url"],
+            create_timeout=max_wait_time,
+            debug_dir=None,
+            success_terminal_state_condition=readiness_condition,
+            readiness_wait_time=readiness_wait_time,
+            logger_fn=logger_fn,
+        )
+
+        currently_present_capsules = list_and_filter_capsules(
+            capsule.capsule_api,
+            None,
+            None,
+            capsule.name,
+            None,
+            None,
+            None,
+        )
+
+        force_upgrade = self.app_config.get_state("force_upgrade", False)
+
+        if len(currently_present_capsules) > 0:
+            # Only update the capsule if there is no upgrade in progress
+            # Only update a "already updating" capsule if the `--force-upgrade` flag is provided.
+            _curr_cap = currently_present_capsules[0]
+            this_capsule_is_being_updated = _curr_cap.get("status", {}).get(
+                "updateInProgress", False
+            )
+
+            if this_capsule_is_being_updated and not force_upgrade:
+                _upgrader = _curr_cap.get("metadata", {}).get("lastModifiedBy", None)
+                message = f"{capsule.capsule_type} is currently being upgraded"
+                if _upgrader:
+                    message = (
+                        f"{capsule.capsule_type} is currently being upgraded. Upgrade was launched by {_upgrader}. "
+                        "If you wish to force upgrade, you can do so by providing the `--force-upgrade` flag."
+                    )
+                raise AppConfigError(message)
+
+            logger_fn(
+                f"🚀 {'' if not force_upgrade else 'Force'} Upgrading {capsule.capsule_type.lower()} `{capsule.name}`....",
+            )
+        else:
+            logger_fn(
+                f"🚀 Deploying {capsule.capsule_type.lower()} `{capsule.name}`....",
+            )
+
+        capsule.create()
+        final_status = capsule.wait_for_terminal_state()
+        return final_status
+
+
+class apps:
+
+    _name_prefix = None
+
+    @classmethod
+    def set_name_prefix(cls, name_prefix: str):
+        cls._name_prefix = name_prefix
+
+    @property
+    def name_prefix(self) -> str:
+        return self._name_prefix
+
+    @property
+    def Deployer(self) -> Type[AppDeployer]:
+        return AppDeployer

metaflow_extensions/outerbounds/plugins/apps/core/perimeters.py

@@ -41,5 +41,47 @@ class PerimeterExtractor:
 
     @classmethod
     def during_metaflow_execution(cls) -> str:
-        # TODO: implement this
-        return ""
+        from metaflow.metaflow_config_funcs import init_config
+
+        clean_url = (
+            lambda url: f"https://{url}".rstrip("/")
+            if not url.startswith("https://")
+            else url
+        )
+
+        config = init_config()
+        api_server, perimeter, integrations_url = None, None, None
+        perimeter = config.get(
+            "OBP_PERIMETER", os.environ.get("OBP_PERIMETER", perimeter)
+        )
+        if perimeter is None:
+            raise RuntimeError(
+                "Perimeter not found in metaflow config or environment variables"
+            )
+
+        api_server = config.get(
+            "OBP_API_SERVER", os.environ.get("OBP_API_SERVER", api_server)
+        )
+
+        if api_server is not None and not api_server.startswith("https://"):
+            api_server = clean_url(api_server)
+
+        if api_server is not None:
+            return perimeter, api_server
+
+        integrations_url = config.get(
+            "OBP_INTEGRATIONS_URL", os.environ.get("OBP_INTEGRATIONS_URL", None)
+        )
+
+        if integrations_url is not None and not integrations_url.startswith("https://"):
+            integrations_url = clean_url(integrations_url)
+
+        if integrations_url is not None:
+            api_server = integrations_url.rstrip("/integrations")
+
+        if api_server is None:
+            raise RuntimeError(
+                "API server not found in metaflow config or environment variables"
+            )
+
+        return perimeter, api_server

metaflow_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.py

@@ -76,3 +76,4 @@ from ..plugins.snowflake import Snowflake
 from ..plugins.checkpoint_datastores import nebius_checkpoints, coreweave_checkpoints
 from ..plugins.aws import assume_role
 from . import ob_internal
+from .ob_internal import AppDeployer

metaflow_extensions/outerbounds/toplevel/ob_internal.py

@@ -1,3 +1,4 @@
 from ..plugins.kubernetes.pod_killer import PodKiller
 from ..plugins.fast_bakery.baker import bake_image
 from ..plugins.apps import core as app_core
+from ..plugins.apps.core import AppDeployer

{ob_metaflow_extensions-1.1.175rc2.dist-info → ob_metaflow_extensions-1.1.175rc4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ob-metaflow-extensions
-Version: 1.1.175rc2
+Version: 1.1.175rc4
 Summary: Outerbounds Platform Extensions for Metaflow
 Author: Outerbounds, Inc.
 License: Commercial