ob-metaflow-extensions 1.4.33__py2.py3-none-any.whl → 1.6.2__py2.py3-none-any.whl

metaflow_extensions/outerbounds/plugins/apps/core/capsule.py

@@ -20,16 +20,30 @@ from ._state_machine import (
     DEPLOYMENT_READY_CONDITIONS,
     LogLine,
 )
+from .exceptions import (
+    CapsuleApiException,
+    CapsuleConcurrentUpgradeException,
+    CapsuleCrashLoopException,
+    CapsuleDeletedDuringDeploymentException,
+    CapsuleDeploymentException,
+    CapsuleReadinessException,
+    OuterboundsBackendUnhealthyException,
+    OuterboundsForbiddenException,
+)
+
+STATE_REFRESH_FREQUENCY = 1  # in seconds
 
 
-def _format_url_string(url):
+def _format_url_string(url, is_https=True):
     if url is None:
         return None
 
     if url.startswith("http://") or url.startswith("https://"):
         return url
+    if is_https:
+        return f"https://{url}"
 
-    return f"https://{url}"
+    return f"http://{url}"
 
 
 class CapsuleStateMachine:
@@ -84,12 +98,12 @@ class CapsuleStateMachine:
     @property
     def out_of_cluster_url(self):
         access_info = self.current_status.get("accessInfo", {}) or {}
-        return _format_url_string(access_info.get("outOfClusterURL", None))
+        return _format_url_string(access_info.get("outOfClusterURL", None), True)
 
     @property
     def in_cluster_url(self):
         access_info = self.current_status.get("accessInfo", {}) or {}
-        return _format_url_string(access_info.get("inClusterURL", None))
+        return _format_url_string(access_info.get("inClusterURL", None), True)
 
     @property
     def update_in_progress(self):
@@ -328,46 +342,12 @@ class CapsuleInput:
         }
 
 
-class CapsuleApiException(Exception):
-    def __init__(
-        self,
-        url: str,
-        method: str,
-        status_code: int,
-        text: str,
-        message: Optional[str] = None,
-    ):
-        self.url = url
-        self.method = method
-        self.status_code = status_code
-        self.text = text
-        self.message = message
-
-    def __str__(self):
-        return (
-            f"CapsuleApiException: {self.url} [{self.method}]: Status Code: {self.status_code} \n\n {self.text}"
-            + (f"\n\n {self.message}" if self.message else "")
-        )
-
-
-class CapsuleDeploymentException(Exception):
-    def __init__(
-        self,
-        capsule_id: str,
-        message: str,
-    ):
-        self.capsule_id = capsule_id
-        self.message = message
-
-    def __str__(self):
-        return f"CapsuleDeploymentException: [{self.capsule_id}] :: {self.message}"
-
-
 class CapsuleApi:
-    def __init__(self, base_url: str, perimeter: str, logger_fn=None):
+    def __init__(self, base_url: str, perimeter: str, logger_fn=None, retry_500s=False):
         self._base_url = self._create_base_url(base_url, perimeter)
         from metaflow.metaflow_config import SERVICE_HEADERS
 
+        self._retry_500s = retry_500s
         self._logger_fn = logger_fn
         self._request_headers = {
             **{"Content-Type": "application/json", "Connection": "keep-alive"},
@@ -393,7 +373,23 @@ class CapsuleApi:
                 logger_fn=self._logger_fn,
                 **kwargs,
             )
+        # The CapsuleApi wraps every API call happening to the capsule
+        # API. We do this so that we can raise exceptions in a way that make
+        # it clearer to the end-user and the operator. since the safe_requests_wrapper
+        # can already retry 5xx errors too we should ensure that any time we hit max
+        # retries or if we hit 5xx without retries, we should raise a "special" exception
+        # and not the CapsuleApiException to notify the operator that the
+        # backend is not working right RN and thier application crashes. We can lift the
+        # exception to top level to make it importable so operators can deal with that condition
+        # how they like.
         except MaximumRetriesExceeded as e:
+            if e.status_code >= 500:
+                raise OuterboundsBackendUnhealthyException(
+                    e.url,
+                    e.method,
+                    e.status_code,
+                    e.text,
+                )
             raise CapsuleApiException(
                 e.url,
                 e.method,
@@ -401,7 +397,41 @@ class CapsuleApi:
                 e.text,
                 message=f"Maximum retries exceeded for {e.url} [{e.method}]",
             )
-        if response.status_code >= 400:
+        except requests.exceptions.ConnectionError as e:
+            # Network connectivity issues after retries exhausted
+            raise OuterboundsBackendUnhealthyException(
+                url=args[0] if args else "unknown",
+                method=method_func.__name__,
+                message=(
+                    f"Unable to reach Outerbounds backend at {args[0] if args else 'unknown'}. "
+                    "This could be due to network connectivity issues, DNS resolution failures, "
+                    "or the service being temporarily unavailable. "
+                    "Please check your network connection and retry. "
+                    "If the issue persists, contact Outerbounds support."
+                ),
+            ) from e
+
+        if response.status_code >= 500:
+            raise OuterboundsBackendUnhealthyException(
+                args[0],
+                method_func.__name__,
+                response.status_code,
+                response.text,
+                message=(
+                    f"Outerbounds backend returned an error (HTTP {response.status_code}). "
+                    "This is a server-side issue, not a problem with your configuration. "
+                    "Please retry your request. If the issue persists, contact Outerbounds support."
+                ),
+            )
+
+        elif response.status_code == 403:
+            raise OuterboundsForbiddenException(
+                args[0],
+                method_func.__name__,
+                response.text,
+            )
+
+        elif response.status_code >= 400:
             raise CapsuleApiException(
                 args[0],
                 method_func.__name__,
@@ -410,12 +440,39 @@ class CapsuleApi:
             )
         return response
 
+    def _retry_parameters(
+        self,
+        status_codes,
+        retries,
+    ):
+        """
+        All functions calling the wrapped_api_caller use this function
+        set the number of retries for the apis calls. It sets status codes
+        that are allowed to N retries (total including connection retries).
+        If no status codes are passed we should still always pass connnection
+        retries > 0 since DNS can be flaky in-frequently and we dont want to
+        trip up there.
+        """
+        kwargs = {}
+        if self._retry_500s:
+            kwargs = dict(
+                retryable_status_codes=status_codes
+                + [500, 502, 503, 504],  # todo : verify me
+                conn_error_retries=max(
+                    3, retries
+                ),  # connection retries + any other retries.
+            )
+        else:
+            kwargs = dict(
+                retryable_status_codes=status_codes,
+                conn_error_retries=retries,  # connection retries + any other retries.
+            )
+        return kwargs
+
     def create(self, capsule_input: dict):
         _data = json.dumps(capsule_input)
         response = self._wrapped_api_caller(
-            requests.post,
-            self._base_url,
-            data=_data,
+            requests.post, self._base_url, data=_data, **self._retry_parameters([], 3)
         )
         try:
             return response.json()
@@ -431,10 +488,7 @@ class CapsuleApi:
     def get(self, capsule_id: str) -> Dict[str, Any]:
         _url = os.path.join(self._base_url, capsule_id)
         response = self._wrapped_api_caller(
-            requests.get,
-            _url,
-            retryable_status_codes=[409, 404],  # todo : verify me
-            conn_error_retries=3,
+            requests.get, _url, **self._retry_parameters([409, 404], 3)
         )
         try:
             return response.json()
@@ -451,10 +505,7 @@ class CapsuleApi:
     def get_by_name(self, name: str, most_recent_only: bool = True):
         _url = os.path.join(self._base_url, f"?displayName={name}")
         response = self._wrapped_api_caller(
-            requests.get,
-            _url,
-            retryable_status_codes=[409],  # todo : verify me
-            conn_error_retries=3,
+            requests.get, _url, **self._retry_parameters([409], 3)
         )
         try:
             if most_recent_only:
@@ -478,10 +529,7 @@ class CapsuleApi:
 
     def list(self):
         response = self._wrapped_api_caller(
-            requests.get,
-            self._base_url,
-            retryable_status_codes=[409],  # todo : verify me
-            conn_error_retries=3,
+            requests.get, self._base_url, **self._retry_parameters([409], 3)
         )
         try:
             response_json = response.json()
@@ -506,9 +554,7 @@ class CapsuleApi:
     def delete(self, capsule_id: str):
         _url = os.path.join(self._base_url, capsule_id)
         response = self._wrapped_api_caller(
-            requests.delete,
-            _url,
-            retryable_status_codes=[409],  # todo : verify me
+            requests.delete, _url, **self._retry_parameters([409], 3)
         )
         if response.status_code >= 400:
             raise CapsuleApiException(
@@ -527,11 +573,10 @@ class CapsuleApi:
         response = self._wrapped_api_caller(
             requests.get,
             _url,
-            retryable_status_codes=[409, 404],  # todo : verify me
             # Adding 404s because sometimes we might even end up getting 404s if
             # the backend cache is not updated yet. So on consistent 404s we should
             # just crash out.
-            conn_error_retries=3,
+            **self._retry_parameters([409, 404], 3),
         )
         try:
             return response.json().get("workers", []) or []
@@ -552,10 +597,7 @@ class CapsuleApi:
         if previous:
             options = {"previous": True}
         response = self._wrapped_api_caller(
-            requests.get,
-            _url,
-            retryable_status_codes=[409],  # todo : verify me
-            params=options,
+            requests.get, _url, params=options, **self._retry_parameters([409], 3)
         )
         try:
             return response.json().get("logs", []) or []
@@ -655,6 +697,10 @@ class CapsuleDeployer:
             base_url,
             app_config.get_state("perimeter"),
             logger_fn=logger_fn or partial(print, file=sys.stderr),
+            retry_500s=True
+            # retry for 5xx because during the capsule deployer might even be used
+            # programmatically so any intermittent breakage shouldnt break the overall
+            # control flow unless the breakage is severe (maybe over 20s of complete outage)
         )
         self._create_timeout = create_timeout
         self._logger_fn = logger_fn
@@ -666,7 +712,7 @@ class CapsuleDeployer:
     @property
     def url(self):
         return _format_url_string(
-            ({} or self._capsule_deploy_response).get("outOfClusterUrl", None)
+            ({} or self._capsule_deploy_response).get("outOfClusterUrl", None), True
         )
 
     @property
@@ -723,9 +769,13 @@ class CapsuleDeployer:
         output that they desire.
         """
         if capsule_response.get("version", None) != current_deployment_instance_version:
-            raise CapsuleDeploymentException(
+            metadata = capsule_response.get("metadata", {}) or {}
+            raise CapsuleConcurrentUpgradeException(
                 self.identifier,  # type: ignore
-                f"A capsule upgrade was triggered outside current deployment instance. Current deployment version was discarded. Current deployment version: {current_deployment_instance_version} and new version: {capsule_response.get('version', None)}",
+                expected_version=current_deployment_instance_version,
+                actual_version=capsule_response.get("version", None),
+                modified_by=metadata.get("lastModifiedBy"),
+                modified_at=metadata.get("lastModifiedAt"),
             )
 
     def _update_capsule_and_worker_sm(
@@ -734,7 +784,26 @@ class CapsuleDeployer:
         workers_sm: "CapsuleWorkersStateMachine",
         logger: Callable[[str], None],
     ) -> Tuple[Dict[str, Any], List[Dict[str, Any]]]:
-        capsule_response = self.get()
+        try:
+            capsule_response = self.get()
+        except CapsuleApiException as e:
+            # At this point when the code is executing
+            # the CapsuleDeployer would already have created
+            # the capsule since this function is called within
+            # wait_for_terminal_state. Because of that if there
+            # is now a 404 then it means someone deleted the
+            # deployment WHILE this deployment instance is running
+            # We shoud notify the user that something funky has
+            # happened over here. We need to do this since Apps can
+            # now be programmatically created / deleted, we need to
+            # ensure that if some-one has done something concurrent-unsafe
+            # (foo deleting bar's deployment while bar is deploying)
+            # then for that circumstance we should raise an exception here
+            # that something funky has happened. Otherwise if the
+            # CapsuleApiException leaks out then it should be fine.
+            if e.status_code == 404:
+                raise CapsuleDeletedDuringDeploymentException(self.identifier) from e
+            raise
         capsule_sm.add_status(capsule_response.get("status", {}))  # type: ignore
 
         # We need to check if someone has not upgraded the capsule under the hood and
@@ -771,8 +840,10 @@ class CapsuleDeployer:
         """returns True if the worker is crashlooping, False otherwise"""
         logger = self._logger_fn or partial(print, file=sys.stderr)
         for i in range(self._readiness_wait_time):
-            time.sleep(1)
-            self._update_capsule_and_worker_sm(capsule_sm, workers_sm, logger)
+            time.sleep(STATE_REFRESH_FREQUENCY)
+            self._update_capsule_and_worker_sm(
+                capsule_sm, workers_sm, logger
+            )  # [2 API calls]
             if workers_sm.is_crashlooping:
                 return True
         return False
@@ -827,8 +898,8 @@ class CapsuleDeployer:
         # If we reach a teminal condition like described in `DEPLOYMENT_READY_CONDITIONS`, then
         # we will further check for readiness conditions.
         for i in range(self._create_timeout):
-            time.sleep(1)
-            capsule_response, _ = self._update_capsule_and_worker_sm(
+            time.sleep(STATE_REFRESH_FREQUENCY)
+            capsule_response, _ = self._update_capsule_and_worker_sm(  # [2 API calls]
                 state_machine, workers_state_machine, logger
             )
             # Deployment readiness checks will determine what is the terminal state
@@ -901,9 +972,10 @@ class CapsuleDeployer:
                                 + ["\t" + l["message"] for l in logs]
                             )
                         )
-                        raise CapsuleDeploymentException(
+                        raise CapsuleCrashLoopException(
                             self.identifier,
-                            f"Worker ID ({worker_id}) is crashlooping. Please check the logs for more information.",
+                            worker_id=worker_id,
+                            logs=logs,
                         )
 
                 if state_machine.ready_to_serve_traffic:
@@ -942,14 +1014,13 @@ class CapsuleDeployer:
             and not _is_async_readiness
             and not self.status.ready_to_serve_traffic
         ):
-            raise CapsuleDeploymentException(
+            raise CapsuleReadinessException(
                 self.identifier,
-                f"Capsule {self.identifier} failed to be ready to serve traffic",
             )
-
+        auth_type = self._app_config.get_state("auth", {}).get("type", AuthType.default)
         return dict(
             id=self.identifier,
-            auth_type=self.capsule_type,
+            auth_type=auth_type,
             public_url=self.url,
             available_replicas=self.status.available_replicas,
             name=self.name,

metaflow_extensions/outerbounds/plugins/apps/core/config/__init__.py

@@ -1,4 +1,4 @@
-from .unified_config import CoreConfig
+from .unified_config import CoreConfig, PackagedCode, BakedImage, AuthType
 from .cli_generator import auto_cli_options
 from .config_utils import (
     PureStringKVPairType,
@@ -7,6 +7,9 @@ from .config_utils import (
     MergingNotAllowedFieldsException,
     ConfigValidationFailedException,
     RequiredFieldMissingException,
+    ConfigFieldContext,
+    ConfigField,
+    FieldBehavior,
 )
 from . import schema_export
 from .typed_configs import TypedCoreConfig, TypedDict

metaflow_extensions/outerbounds/plugins/apps/core/config/cli_generator.py

@@ -54,6 +54,10 @@ class CLIGenerator:
         def _options_from_cfg_cls(_config_class):
             options = []
             for field_name, field_info in _config_class._fields.items():
+                # Skip fields not available in CLI context
+                if not field_info.is_available_in_cli():
+                    continue
+
                 if ConfigMeta.is_instance(field_info.field_type):
                     _subfield_options = _options_from_cfg_cls(field_info.field_type)
                     options.extend(_subfield_options)

metaflow_extensions/outerbounds/plugins/apps/core/config/config_utils.py

@@ -62,6 +62,67 @@ class FieldBehavior:
     NOT_ALLOWED = "not_allowed"
 
 
+class ConfigFieldContext:
+    """
+    Defines which interfaces a ConfigField is available in.
+
+    ConfigFieldContext controls whether a field appears in the CLI, programmatic API, or both.
+    This allows the same CoreConfig class to serve different interfaces while keeping
+    interface-specific fields properly scoped.
+
+    Context Types:
+
+    ALL (Default):
+        Field is available in both CLI and programmatic API.
+        Most configuration fields fall into this category.
+
+        Example:
+        ```python
+        name = ConfigField(
+            field_type=str,
+            # available_in=ConfigFieldContext.ALL is the default
+        )
+        ```
+
+    CLI:
+        Field is only available in the CLI interface.
+        Use for CLI-specific options that don't make sense programmatically.
+
+        Example:
+        ```python
+        # config_file path only makes sense when running from CLI
+        config_file = ConfigField(
+            cli_meta=CLIOption(name="config_file", cli_option_str="--config-file"),
+            field_type=str,
+            available_in=ConfigFieldContext.CLI,
+        )
+        ```
+
+    PROGRAMMATIC:
+        Field is only available in the programmatic API.
+        Use for fields that accept programmatic-only types (like PackagedCode)
+        or don't map well to CLI options.
+
+        Example:
+        ```python
+        # code_package accepts PackagedCode namedtuple from package_code()
+        code_package = ConfigField(
+            field_type=PackagedCode,
+            available_in=ConfigFieldContext.PROGRAMMATIC,
+        )
+        ```
+
+    Integration:
+        - CLI generators check `available_in` to decide whether to generate CLI options
+        - TypedConfig generators check `available_in` to decide whether to include in __init__
+        - Schema exporters can filter fields based on target interface
+    """
+
+    ALL = "all"  # Available in both CLI and programmatic API (default)
+    CLI = "cli"  # Only available in CLI
+    PROGRAMMATIC = "programmatic"  # Only available in programmatic API
+
+
 class CLIOption:
     """Metadata container for automatic CLI option generation from configuration fields.
 
@@ -225,6 +286,9 @@ class ConfigField:
         Optional function to validate field values.
     is_experimental : bool, optional
         Whether this field is experimental (for documentation).
+    available_in : str, optional
+        ConfigFieldContext specifying which interfaces this field is available in.
+        One of ConfigFieldContext.ALL (default), ConfigFieldContext.CLI, or ConfigFieldContext.PROGRAMMATIC.
     """
 
     def __init__(
@@ -240,6 +304,7 @@ class ConfigField:
         validation_fn: Optional[Callable] = None,
         is_experimental=False,  # This property is for bookkeeping purposes and for export in schema.
         parsing_fn: Optional[Callable] = None,
+        available_in: str = ConfigFieldContext.ALL,
     ):
         if behavior == FieldBehavior.NOT_ALLOWED and ConfigMeta.is_instance(field_type):
             raise ValueError(
@@ -264,6 +329,7 @@ class ConfigField:
         self.validation_fn = validation_fn
         self.is_experimental = is_experimental
         self.parsing_fn = parsing_fn
+        self.available_in = available_in
         self._qual_name_stack = []
 
     # This function allows config fields to be made aware of the
@@ -280,6 +346,17 @@ class ConfigField:
     def fully_qualified_name(self):
         return ".".join(self._qual_name_stack + [self.name])
 
+    def is_available_in_cli(self) -> bool:
+        """Check if this field should be available in the CLI interface."""
+        return self.available_in in (ConfigFieldContext.ALL, ConfigFieldContext.CLI)
+
+    def is_available_in_programmatic(self) -> bool:
+        """Check if this field should be available in the programmatic API."""
+        return self.available_in in (
+            ConfigFieldContext.ALL,
+            ConfigFieldContext.PROGRAMMATIC,
+        )
+
     def __set_name__(self, owner, name):
         self.name = name
 
@@ -431,7 +508,10 @@ class ConfigMeta(type):
 
     @staticmethod
     def is_instance(value) -> bool:
-        return hasattr(value, "_fields")
+        # Check for _fields attribute AND that it's a dict (not a tuple like namedtuples have)
+        return hasattr(value, "_fields") and isinstance(
+            getattr(value, "_fields", None), dict
+        )
 
     def __new__(mcs, name, bases, namespace):
         # Collect field metadata
@@ -619,9 +699,20 @@ def config_meta_to_dict(config_instance) -> Optional[Dict[str, Any]]:
     if config_instance is None:
         return None
 
+    # Helper to check if something is a namedtuple
+    def _is_namedtuple(obj):
+        return (
+            isinstance(obj, tuple)
+            and hasattr(obj, "_fields")
+            and isinstance(getattr(obj, "_fields", None), tuple)
+        )
+
     # Check if this is a ConfigMeta-based class
     if not ConfigMeta.is_instance(config_instance):
         # If it's not a config object, return as-is
+        # Handle namedtuples by converting to dict
+        if _is_namedtuple(config_instance):
+            return config_instance._asdict()
         return config_instance
 
     result = {}
@@ -635,16 +726,23 @@ def config_meta_to_dict(config_instance) -> Optional[Dict[str, Any]]:
         if value is not None and ConfigMeta.is_instance(value):
             # It's a nested config object
             result[field_name] = config_meta_to_dict(value)
+        elif _is_namedtuple(value):
+            # Handle namedtuples by converting to dict
+            result[field_name] = value._asdict()
         elif isinstance(value, list) and value:
-            # Handle lists that might contain config objects
+            # Handle lists that might contain config objects or namedtuples
             result[field_name] = [
-                config_meta_to_dict(item) if ConfigMeta.is_instance(item) else item
+                config_meta_to_dict(item)
+                if ConfigMeta.is_instance(item)
+                else (item._asdict() if _is_namedtuple(item) else item)
                 for item in value
             ]
         elif isinstance(value, dict) and value:
-            # Handle dictionaries that might contain config objects
+            # Handle dictionaries that might contain config objects or namedtuples
             result[field_name] = {
-                k: config_meta_to_dict(v) if ConfigMeta.is_instance(v) else v
+                k: config_meta_to_dict(v)
+                if ConfigMeta.is_instance(v)
+                else (v._asdict() if _is_namedtuple(v) else v)
                 for k, v in value.items()
             }
         else:

metaflow_extensions/outerbounds/plugins/apps/core/config/schema_export.py

@@ -165,7 +165,11 @@ def export_schema(
 
 # Private helper functions
 def _generate_openapi_schema(cls) -> Dict[str, Any]:
-    """Generate OpenAPI schema for a configuration class."""
+    """Generate OpenAPI schema for a configuration class.
+
+    Note: Schema is intended for CLI/config file usage, so PROGRAMMATIC-only
+    fields are excluded.
+    """
     # Clean up class docstring for better YAML formatting
     description = f"{cls.__name__} configuration"
     get_description = getattr(cls, "SCHEMA_DOC", None)
@@ -193,6 +197,13 @@ def _generate_openapi_schema(cls) -> Dict[str, Any]:
         if field_name.startswith("_"):
             continue
 
+        # Skip PROGRAMMATIC-only fields - schema is for CLI/config file usage
+        if (
+            hasattr(field_info, "is_available_in_cli")
+            and not field_info.is_available_in_cli()
+        ):
+            continue
+
         field_schema = _get_field_schema(field_info)
         schema["properties"][field_name] = field_schema