ob-metaflow-extensions 1.1.151__py2.py3-none-any.whl → 1.6.2__py2.py3-none-any.whl

metaflow_extensions/outerbounds/plugins/apps/core/exceptions.py

@@ -0,0 +1,341 @@
+import json
+from typing import Optional, List, Dict
+from ._state_machine import LogLine
+
+
+class OuterboundsBackendUnhealthyException(Exception):
+    """This exception is raised when the Outerbounds platform is unhealthy (5xx errors) or unreachable."""
+
+    def __init__(
+        self,
+        url: str,
+        method: str,
+        status_code: Optional[int] = None,
+        text: Optional[str] = None,
+        message: Optional[str] = None,
+    ):
+        self.url = url
+        self.method = method
+        self.status_code = status_code
+        self.text = text
+        self.message = message
+        super().__init__(self.message)
+
+
+class OuterboundsForbiddenException(Exception):
+    """This exception is raised when access to an Outerbounds API is forbidden (HTTP 403)."""
+
+    def __init__(
+        self,
+        url: str,
+        method: str,
+        text: str,
+    ):
+        self.url = url
+        self.method = method
+        self.text = text
+        self.message = (
+            f"Access forbidden (HTTP 403) when calling {url}. "
+            "This typically means your credentials lack permission for this operation. "
+            "Please verify that you outerbounds / metaflow configuration has access to the "
+            "correct perimeter.\n"
+            "If you believe this is an error, contact your Outerbounds administrator."
+        )
+        super().__init__(self.message)
+
+
+class OuterboundsConfigurationException(Exception):
+    """This exception is raised when Outerbounds configuration is missing or invalid."""
+
+    def __init__(self, missing_config: str):
+        self.missing_config = missing_config
+        self.message = (
+            f"Outerbounds configuration '{missing_config}' not found.\n\n"
+            "If running locally:\n"
+            "  - Run: outerbounds configure <magic-string-from-outerbounds-ui>\n"
+            "If running remotely (e.g., in a Metaflow task):\n"
+            "  - Ensure you have the Outerbounds distribution installed (pip install outerbounds)\n"
+            "    and not the open-source Metaflow. The Outerbounds fork injects configuration\n"
+            "    into remote tasks automatically which may not be present in open source metaflow.\n"
+        )
+        super().__init__(self.message)
+
+
+class CapsuleApiException(Exception):
+    def __init__(
+        self,
+        url: str,
+        method: str,
+        status_code: int,
+        text: str,
+        message: Optional[str] = None,
+    ):
+        self.url = url
+        self.method = method
+        self.status_code = status_code
+        self.text = text
+        self.message = message
+
+    def __str__(self):
+        return (
+            f"CapsuleApiException: {self.url} [{self.method}]: Status Code: {self.status_code} \n\n {self.text}"
+            + (f"\n\n {self.message}" if self.message else "")
+        )
+
+
+class CapsuleDeploymentException(Exception):
+    """Base exception for all capsule deployment failures."""
+
+    def __init__(
+        self,
+        capsule_id: str,
+        message: str,
+    ):
+        self.capsule_id = capsule_id
+        self.message = message
+        super().__init__(self.message)
+
+    def __str__(self):
+        return f"CapsuleDeploymentException: [{self.capsule_id}] :: {self.message}"
+
+
+class CapsuleCrashLoopException(CapsuleDeploymentException):
+    """Raised when a worker enters CrashLoopBackOff or Failed state."""
+
+    def __init__(
+        self,
+        capsule_id: str,
+        worker_id: str,
+        logs: Optional[List[LogLine]] = None,
+    ):
+        self.worker_id = worker_id
+        self.logs = logs or []
+        message = f"Worker ID ({worker_id}) is crashlooping. Please check the logs for more information."
+        super().__init__(capsule_id, message)
+
+    def __str__(self):
+        return f"CapsuleCrashLoopException: [{self.capsule_id}] :: {self.message}"
+
+
+class CapsuleReadinessException(CapsuleDeploymentException):
+    """Raised when capsule fails to meet readiness conditions within timeout."""
+
+    def __init__(
+        self,
+        capsule_id: str,
+        reason: Optional[str] = None,
+    ):
+        message = f"Capsule {capsule_id} failed to be ready to serve traffic"
+        if reason:
+            message += f": {reason}"
+        super().__init__(capsule_id, message)
+
+    def __str__(self):
+        return f"CapsuleReadinessException: [{self.capsule_id}] :: {self.message}"
+
+
+class CapsuleConcurrentUpgradeException(CapsuleDeploymentException):
+    """Raised when a concurrent upgrade invalidates the current deployment."""
+
+    def __init__(
+        self,
+        capsule_id: str,
+        expected_version: str,
+        actual_version: str,
+        modified_by: Optional[str] = None,
+        modified_at: Optional[str] = None,
+    ):
+        self.expected_version = expected_version
+        self.actual_version = actual_version
+        self.modified_by = modified_by
+        self.modified_at = modified_at
+        message = (
+            f"A capsule upgrade was triggered outside current deployment instance. "
+            f"Expected version: {expected_version}, actual version: {actual_version}"
+        )
+        if modified_by:
+            message += f". Modified by: {modified_by}"
+        if modified_at:
+            message += f" at {modified_at}"
+        super().__init__(capsule_id, message)
+
+    def __str__(self):
+        return (
+            f"CapsuleConcurrentUpgradeException: [{self.capsule_id}] :: {self.message}"
+        )
+
+
+class CapsuleDeletedDuringDeploymentException(CapsuleDeploymentException):
+    """Raised when a capsule is deleted while deployment is in progress."""
+
+    def __init__(self, capsule_id: str):
+        super().__init__(capsule_id, "Capsule was deleted during deployment")
+
+
+class CodePackagingException(Exception):
+    """Exception raised when code packaging fails."""
+
+    pass
+
+
+class AppNotFoundException(Exception):
+    pass
+
+
+class AppCreationFailedException(Exception):
+    """Raised when app deployment submission fails due to an API error."""
+
+    def __init__(
+        self,
+        app_name: str,
+        status_code: int,
+        error_text: str,
+    ):
+        self.status_code = status_code
+        self.error_text = error_text
+        message = f"Failed to submit app deployment: HTTP {status_code} - {error_text}"
+        if status_code == 400:
+            # 400 = validation error; the app configuration is invalid and must be fixed.
+            message = "Invalid app deployment configuration submitted: "
+            try:
+                reason_for_failure = json.loads(error_text).get("message", error_text)
+            except json.JSONDecodeError:
+                reason_for_failure = error_text
+            message += reason_for_failure
+            message += (
+                "\n\nCheck your config file or CLI parameters if deploying via CLI, "
+                "or AppDeployer parameters if deploying programmatically."
+            )
+
+        self.message = message
+        super().__init__(message)
+
+    def __str__(self):
+        return f"AppCreationFailedException: {self.message}"
+
+
+class AppDeploymentException(Exception):
+    """Base exception for all individual app deployment failures."""
+
+    def __init__(self, app_id: str, message: str):
+        self.app_id = app_id
+        self.message = message
+        self._deployed_app = None
+        super().__init__(self.message)
+
+    def __str__(self):
+        return f"AppDeploymentException: [{self.app_id}] :: {self.message}"
+
+    @property
+    def deployed_app(self):
+        from .deployer import DeployedApp
+
+        return DeployedApp._from_capsule_id(self.app_id)
+
+
+class AppCrashLoopException(AppDeploymentException):
+    """Raised when an app worker enters CrashLoopBackOff or Failed state."""
+
+    def __init__(
+        self,
+        app_id: str,
+        worker_id: str,
+        logs: Optional[List] = None,
+    ):
+        self.worker_id = worker_id
+        self.logs = logs or []
+        message = f"Worker ({worker_id}) is crashlooping. Please check the logs for more information."
+        super().__init__(app_id, message)
+
+    def __str__(self):
+        return f"AppCrashLoopException: [{self.app_id}] :: {self.message}"
+
+
+class AppReadinessException(AppDeploymentException):
+    """Raised when app fails to meet readiness conditions within timeout."""
+
+    def __init__(self, app_id: str, reason: Optional[str] = None):
+        message = f"App {app_id} failed to be ready to serve traffic"
+        if reason:
+            message += f": {reason}"
+        super().__init__(app_id, message)
+
+    def __str__(self):
+        return f"AppReadinessException: [{self.app_id}] :: {self.message}"
+
+
+class AppUpgradeInProgressException(AppDeploymentException):
+    """Raised when attempting to deploy while another upgrade is already in progress."""
+
+    def __init__(
+        self,
+        app_id: str,
+        upgrader: Optional[str] = None,
+    ):
+        self.upgrader = upgrader
+        if upgrader:
+            message = (
+                f"App {app_id} is currently being upgraded by {upgrader}. "
+                "Use force_upgrade=True in AppDeployer to override."
+            )
+        else:
+            message = (
+                f"App {app_id} is currently being upgraded. "
+                "Use force_upgrade=True in AppDeployer to override."
+            )
+        super().__init__(app_id, message)
+
+    def __str__(self):
+        return f"AppUpgradeInProgressException: [{self.app_id}] :: {self.message}"
+
+
+class AppConcurrentUpgradeException(AppDeploymentException):
+    """Raised when a concurrent upgrade invalidates the current deployment mid-flight."""
+
+    def __init__(
+        self,
+        app_id: str,
+        expected_version: str,
+        actual_version: str,
+        modified_by: Optional[str] = None,
+        modified_at: Optional[str] = None,
+    ):
+        self.expected_version = expected_version
+        self.actual_version = actual_version
+        self.modified_by = modified_by
+        self.modified_at = modified_at
+
+        modifier_info = ""
+        if modified_by:
+            modifier_info = f" by '{modified_by}'"
+            if modified_at:
+                modifier_info += f" at {modified_at}"
+
+        message = (
+            f"Another deployment was triggered{modifier_info} while this deployment was in progress.\n\n"
+            f"This deployment expected to be working with version '{expected_version}', but the app "
+            f"is now at version '{actual_version}'. The current deployment has been invalidated.\n\n"
+            "To avoid this in the future, you can either use a unique `name` for each deployment "
+            "or coordinate deployments to ensure concurrent upgrades to the same app don't overlap."
+        )
+        super().__init__(app_id, message)
+
+    def __str__(self):
+        return f"AppConcurrentUpgradeException: [{self.app_id}] :: {self.message}"
+
+
+class AppDeletedDuringDeploymentException(AppDeploymentException):
+    """Raised when an app is deleted while deployment is in progress."""
+
+    def __init__(self, app_id: str):
+        message = (
+            f"App '{app_id}' was deleted while this deployment was in progress.\n\n"
+            "This can happen when another process or user deletes the app during deployment. "
+            "Since apps can be programmatically created and deleted, concurrent operations "
+            "may conflict. If you did not intend to delete this app, check for other processes "
+            "or users that may be managing deployments in this perimeter."
+        )
+        super().__init__(app_id, message)
+
+    def __str__(self):
+        return f"AppDeletedDuringDeploymentException: [{self.app_id}] :: {self.message}"

metaflow_extensions/outerbounds/plugins/apps/core/experimental/__init__.py

@@ -0,0 +1,89 @@
+import os
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from ..app_config import AppConfig
+
+DEFAULT_BRANCH = "test"
+
+
+# Account for project / branch and the capsule input.
+def capsule_input_overrides(app_config: "AppConfig", capsule_input: dict):
+    project = app_config.get_state("project", None)
+    # Update the project/branch related configurations.
+    if project is not None:
+        branch = app_config.get_state("branch", DEFAULT_BRANCH)
+        capsule_input["tags"].extend(
+            [dict(key="project", value=project), dict(key="branch", value=branch)]
+        )
+
+    persistence = app_config.get_state("persistence", None)
+    if persistence is not None and persistence != "none":
+        capsule_input["persistence"] = persistence
+
+    capsule_input["generateStaticUrl"] = app_config.get_state(
+        "generate_static_url", False
+    )
+
+    model_asset_conf = app_config.get_state("models", None)
+    data_asset_conf = app_config.get_state("data", None)
+    code_info = _code_info(app_config)
+    # todo:fix me
+    _objects_key = "associatedObjects"
+    if model_asset_conf or data_asset_conf or code_info:
+        capsule_input[_objects_key] = {}
+
+        if model_asset_conf:
+            capsule_input[_objects_key]["models"] = [
+                {"assetId": x["asset_id"], "assetInstanceId": x["asset_instance_id"]}
+                for x in model_asset_conf
+            ]
+        if data_asset_conf:
+            capsule_input[_objects_key]["data"] = [
+                {"assetId": x["asset_id"], "assetInstanceId": x["asset_instance_id"]}
+                for x in data_asset_conf
+            ]
+        if code_info:
+            capsule_input[_objects_key]["code"] = code_info
+
+    return capsule_input
+
+
+def _code_info(app_config: "AppConfig"):
+    from metaflow.metaflow_git import get_repository_info
+
+    try:
+        from metaflow.metaflow_git import _call_git  # type: ignore
+    except ImportError:
+        # Fallback if _call_git is not available
+        def _call_git(args, path=None):
+            return "", 1, True
+
+    package_dirs = app_config.get_state("packaging_directories")
+    if package_dirs is None:
+        return None
+    for directory in package_dirs:
+        repo_info = get_repository_info(directory)
+        if len(repo_info) == 0:
+            continue
+        git_log_info, returncode, failed = _call_git(
+            ["log", "-1", "--pretty=%B"],
+            path=directory,
+        )
+        repo_url = repo_info["repo_url"]
+        if isinstance(repo_url, str):
+            _url = (
+                repo_url if not repo_url.endswith(".git") else repo_url.rstrip(".git")
+            )
+        else:
+            _url = str(repo_url)
+        _code_info = {
+            "commitId": repo_info["commit_sha"],
+            "commitLink": os.path.join(_url, "commit", str(repo_info["commit_sha"])),
+        }
+        if not failed and returncode == 0 and isinstance(git_log_info, str):
+            _code_info["commitMessage"] = git_log_info.strip()
+
+        return _code_info
+
+    return None

metaflow_extensions/outerbounds/plugins/apps/core/perimeters.py

@@ -0,0 +1,123 @@
+import os
+import json
+from typing import Tuple, Union
+
+import requests
+from .utils import safe_requests_wrapper
+from .exceptions import OuterboundsConfigurationException
+
+
+class PerimeterExtractor:
+
+    config = None
+
+    @classmethod
+    def for_ob_cli(
+        cls, config_dir: str, profile: str
+    ) -> Union[Tuple[str, str], Tuple[None, None]]:
+        """
+        This function will be called when we are trying to extract the perimeter
+        via the ob cli's execution. We will rely on the following logic:
+        1. check environment variables like OB_CURRENT_PERIMETER / OBP_PERIMETER
+        2. run init config to extract the perimeter related configurations.
+
+        Returns
+        -------
+            Tuple[str, str] : Tuple containing perimeter name , API server url.
+        """
+        from outerbounds.utils import metaflowconfig
+
+        perimeter = None
+        api_server = None
+        if os.environ.get("OB_CURRENT_PERIMETER") or os.environ.get("OBP_PERIMETER"):
+            perimeter = os.environ.get("OB_CURRENT_PERIMETER") or os.environ.get(
+                "OBP_PERIMETER"
+            )
+
+        if os.environ.get("OBP_API_SERVER"):
+            api_server = os.environ.get("OBP_API_SERVER")
+
+        if perimeter is None or api_server is None:
+            metaflow_config = metaflowconfig.init_config(config_dir, profile)
+            perimeter = metaflow_config.get("OBP_PERIMETER")
+            api_server = metaflowconfig.get_sanitized_url_from_config(
+                config_dir, profile, "OBP_API_SERVER"
+            )
+
+        return perimeter, api_server  # type: ignore
+
+    @classmethod
+    def during_programmatic_access(cls) -> Union[Tuple[str, str], Tuple[None, None]]:
+        from metaflow.metaflow_config_funcs import init_config
+
+        clean_url = (
+            lambda url: f"https://{url}".rstrip("/")
+            if not url.startswith("https://")
+            else url
+        )
+
+        config = init_config()
+        api_server, perimeter, integrations_url = None, None, None
+        perimeter = config.get(
+            "OBP_PERIMETER", os.environ.get("OBP_PERIMETER", perimeter)
+        )
+        if perimeter is None:
+            raise OuterboundsConfigurationException("OBP_PERIMETER")
+
+        api_server = config.get(
+            "OBP_API_SERVER", os.environ.get("OBP_API_SERVER", api_server)
+        )
+
+        if api_server is not None and not api_server.startswith("https://"):
+            api_server = clean_url(api_server)
+
+        if api_server is not None:
+            return perimeter, api_server
+
+        integrations_url = config.get(
+            "OBP_INTEGRATIONS_URL", os.environ.get("OBP_INTEGRATIONS_URL", None)
+        )
+
+        if integrations_url is not None and not integrations_url.startswith("https://"):
+            integrations_url = clean_url(integrations_url)
+
+        if integrations_url is not None:
+            api_server = integrations_url.rstrip("/integrations")
+
+        if api_server is None:
+            raise OuterboundsConfigurationException("OBP_API_SERVER")
+
+        return perimeter, api_server
+
+    @classmethod
+    def config_during_programmatic_access(cls) -> dict:
+        #!HACK: Resolving remote configs is a PITA (all the variable piping we need to do via metaflow)
+        # So instead we will just derive the URL. We are in this situation because its a pain
+        # to load configurations at arbitrary points in the runtime.
+        if cls.config is not None:
+            return json.loads(json.dumps(cls.config))  # Return fresh copy
+        from metaflow.metaflow_config import SERVICE_HEADERS
+
+        perimeter, api_server = cls.during_programmatic_access()
+        response = safe_requests_wrapper(
+            requests.get,
+            f"{api_server}/v1/perimeters/{perimeter}/metaflowconfigs/default",
+            headers=SERVICE_HEADERS,
+        )
+        if response.status_code >= 400:
+            raise RuntimeError(
+                f"Server error: {response.text}. Please reach out to your Outerbounds support team."
+            )
+        try:
+            remote_config = response.json()
+
+            if not remote_config.get("config"):
+                raise json.JSONDecodeError
+        except json.JSONDecodeError:
+            raise RuntimeError(
+                "Exception retrieving remote outerbounds configuration. "
+                "Please reach out to Outerbounds suport team with this stack trace."
+            )
+
+        cls.config = remote_config.get("config")
+        return json.loads(json.dumps(cls.config))  # Return fresh copy