ob-metaflow-extensions 1.1.45rc3__py2.py3-none-any.whl → 1.5.1__py2.py3-none-any.whl

metaflow_extensions/outerbounds/plugins/snowflake/snowflake.py

@@ -0,0 +1,378 @@
+"""
+This library is an abstraction layer for connecting to snowflake using Outerbounds
+OIDC tokens. It expects that a security integration that authenticates tokens minted
+by Outerbounds has already been configured in the target snowflake account.
+"""
+from metaflow.metaflow_config import SERVICE_URL
+from metaflow.metaflow_config_funcs import init_config
+from typing import Dict
+from os import environ
+import sys
+import json
+import requests
+import random
+import time
+
+
+class OuterboundsSnowflakeConnectorException(Exception):
+    pass
+
+
+class OuterboundsSnowflakeIntegrationSpecApiResponse:
+    def __init__(self, response):
+        self.response = response
+
+    @property
+    def account(self):
+        return self.response["account"]
+
+    @property
+    def user(self):
+        return self.response["user"]
+
+    @property
+    def default_role(self):
+        return self.response["default_role"]
+
+    @property
+    def warehouse(self):
+        return self.response["warehouse"]
+
+    @property
+    def database(self):
+        return self.response["database"]
+
+
+def get_snowflake_token(user: str = "", role: str = "", integration: str = "") -> str:
+    """
+    Uses the Outerbounds source token to request for a snowflake compatible OIDC
+    token. This token can then be used to connect to snowflake.
+    user: str
+        The user the token will be minted for
+    role: str
+        The role to which the token will be scoped to
+    integration: str
+        The name of the snowflake integration to use. If not set, an existing integration will be used provided that only one exists per perimeter. If integration is not set and more than one exists, then we raise an exception.
+    """
+    provisioner = SnowflakeIntegrationProvisioner(integration)
+    if not user or not role or not integration:
+        integration_spec = provisioner.get_snowflake_integration_spec()
+        if not user:
+            user = integration_spec.user
+
+        if not role:
+            role = integration_spec.default_role
+
+        if not integration:
+            integration = provisioner.get_integration_name()
+
+    snowflake_token_url = provisioner.get_snowflake_token_url()
+    perimeter = provisioner.get_perimeter()
+    payload = {
+        "perimeterName": perimeter,
+        "snowflakeUser": user,
+        "snowflakeRole": role,
+        "integrationName": integration,
+    }
+    json_payload = json.dumps(payload)
+    headers = provisioner.get_service_auth_header()
+    response = _api_server_get(
+        snowflake_token_url, data=json_payload, headers=headers, conn_error_retries=5
+    )
+    response.raise_for_status()
+    return response.json()["token"]
+
+
+def get_oauth_connection_params(
+    user: str = "", role: str = "", integration: str = "", **kwargs
+) -> Dict:
+    """
+    Get OAuth connection parameters for Snowflake authentication using Outerbounds integration.
+
+    This is a helper function that returns connection parameters dict that can be used
+    with both snowflake-connector-python and snowflake-snowpark-python.
+
+    user: str
+        The user name used to authenticate with snowflake
+    role: str
+        The role to request when connecting with snowflake
+    integration: str
+        The name of the snowflake integration to use. If not set, an existing integration
+        will be used provided that only one exists in the current perimeter.
+    kwargs: dict
+        Additional arguments to include in the connection parameters
+
+    Returns:
+        Dict with connection parameters including OAuth token
+    """
+    # ensure password is not set
+    if "password" in kwargs:
+        raise OuterboundsSnowflakeConnectorException(
+            "Password should not be set when using Outerbounds OAuth authentication."
+        )
+
+    provisioner = SnowflakeIntegrationProvisioner(integration)
+    get_defaults = any(
+        key not in kwargs for key in ["account", "warehouse", "database"]
+    )
+    if not user or not role or not integration or get_defaults:
+        integration_spec = provisioner.get_snowflake_integration_spec()
+        if not user:
+            user = integration_spec.user
+
+        if not role:
+            role = integration_spec.default_role
+
+        if not integration:
+            integration = provisioner.get_integration_name()
+
+        if "account" not in kwargs:
+            kwargs["account"] = integration_spec.account
+
+        if "warehouse" not in kwargs:
+            kwargs["warehouse"] = integration_spec.warehouse
+
+        # if the user is attempting to use a warehouse different from what is specified in the
+        # integration we will not set the database
+        if (
+            "database" not in kwargs
+            and kwargs["warehouse"] == integration_spec.warehouse
+        ):
+            kwargs["database"] = integration_spec.database
+
+    # get snowflake token
+    token = get_snowflake_token(user=user, role=role, integration=integration)
+    kwargs["token"] = token
+    kwargs["authenticator"] = "oauth"
+    kwargs["role"] = role
+    kwargs["user"] = user
+
+    return kwargs
+
+
+def connect(user: str = "", role: str = "", integration: str = "", **kwargs):
+    """
+    Connect to snowflake using the token minted by Outerbounds
+    user: str
+        The user name used to authenticate with snowflake
+    role: str
+        The role to request when connect with snowflake
+    integration: str
+        The name of the snowflake integration to use. If not set, an existing integration will be used provided that only one exists in the current perimeter. If integration is not set and more than one exists in the current perimeter, then we raise an exception.
+    kwargs: dict
+        Additional arguments to pass to the python snowflake connector
+    """
+    # Get OAuth connection params using the helper
+    connection_params = get_oauth_connection_params(
+        user=user, role=role, integration=integration, **kwargs
+    )
+
+    # connect to snowflake
+    try:
+        from snowflake.connector import connect
+
+        cn = connect(**connection_params)
+        return cn
+    except ImportError as ie:
+        raise OuterboundsSnowflakeConnectorException(
+            f"Error importing snowflake connector: {ie}.\nPlease make sure the 'snowflake-connector-python' package has been installed by running 'pip install -U \"outerbounds[snowflake]\"' or using the Metaflow decorators @pypi or @conda."
+        )
+    except Exception as e:
+        raise OuterboundsSnowflakeConnectorException(
+            f"Error connecting to snowflake: {e}"
+        )
+
+
+def _api_server_get(*args, conn_error_retries=2, **kwargs):
+    """
+    There are two categories of errors that we need to handle when dealing with any API server.
+    1. HTTP errors. These are are errors that are returned from the API server.
+        - How to handle retries for this case will be application specific.
+    2. Errors when the API server may not be reachable (DNS resolution / network issues)
+        - In this scenario, we know that something external to the API server is going wrong causing the issue.
+        - Failing pre-maturely in the case might not be the best course of action since critical user jobs might crash on intermittent issues.
+        - So in this case, we can just planely retry the request.
+
+    This function handles the second case. It's a simple wrapper to handle the retry logic for connection errors.
+    If this function is provided a `conn_error_retries` of 5, then the last retry will have waited 32 seconds.
+    Generally this is a safe enough number of retries after which we can assume that something is really broken. Until then,
+    there can be intermittent issues that would resolve themselves if we retry gracefully.
+    """
+    _num_retries = 0
+    noise = random.uniform(-0.5, 0.5)
+    while _num_retries < conn_error_retries:
+        try:
+            return requests.get(*args, **kwargs)
+        except requests.exceptions.ConnectionError:
+            if _num_retries <= conn_error_retries - 1:
+                # Exponential backoff with 2^(_num_retries+1) seconds
+                time.sleep((2 ** (_num_retries + 1)) + noise)
+                _num_retries += 1
+            else:
+                print(
+                    "[@snowflake] Failed to connect to the API server. ",
+                    file=sys.stderr,
+                )
+                raise
+
+
+class Snowflake:
+    def __init__(
+        self, user: str = "", role: str = "", integration: str = "", **kwargs
+    ) -> None:
+        self.cn = connect(user, role, integration, **kwargs)
+
+    def __enter__(self):
+        return self.cn
+
+    def __exit__(self, exception_type, exception_value, traceback):
+        self.cn.close()
+
+    def close(self):
+        self.cn.close()
+
+
+class SnowflakeIntegrationProvisioner:
+    def __init__(self, integration_name: str) -> None:
+        self.conf = init_config()
+        self.integration_name = integration_name
+
+    def get_snowflake_integration_spec(
+        self,
+    ) -> OuterboundsSnowflakeIntegrationSpecApiResponse:
+        integrations_url = self._get_integration_url()
+        perimeter = self.get_perimeter()
+        headers = {"Content-Type": "application/json", "Connection": "keep-alive"}
+        request_payload = {
+            "perimeter_name": perimeter,
+        }
+        # if integration is not set, list all integrations
+        if not self.integration_name:
+            list_snowflake_integrations_url = f"{integrations_url}/snowflake"
+            response = self._make_request(
+                list_snowflake_integrations_url, headers, request_payload
+            )
+            snowflake_integrations = response.get("integrations", [])
+            if not snowflake_integrations:
+                raise OuterboundsSnowflakeConnectorException(
+                    "No snowflake integrations found. Please make sure you have created a Snowflake integration on the Outerbounds UI first."
+                )
+
+            if len(snowflake_integrations) > 1:
+                raise OuterboundsSnowflakeConnectorException(
+                    f"Multiple snowflake integrations found. Please specify a specific integration name you would like to use."
+                )
+
+            self.integration_name = snowflake_integrations[0]["integration_name"]
+            return OuterboundsSnowflakeIntegrationSpecApiResponse(
+                snowflake_integrations[0]["integration_spec"]
+            )
+
+        get_snowflake_integration_url = (
+            f"{integrations_url}/snowflake/{self.integration_name}"
+        )
+        response = self._make_request(
+            get_snowflake_integration_url, headers, request_payload
+        )
+        self.integration_name = response["integration_name"]
+        return OuterboundsSnowflakeIntegrationSpecApiResponse(
+            response["integration_spec"]
+        )
+
+    def get_integration_name(self) -> str:
+        return self.integration_name
+
+    def get_perimeter(self) -> str:
+        if "OBP_PERIMETER" in self.conf:
+            perimeter = self.conf["OBP_PERIMETER"]
+        else:
+            # if the perimeter is not in metaflow config, try to get it from the environment
+            perimeter = environ.get("OBP_PERIMETER", "")
+        if not perimeter:
+            raise OuterboundsSnowflakeConnectorException(
+                "No perimeter set. Please make sure to run `outerbounds configure <...>` command which can be found on the Ourebounds UI or reach out to your Outerbounds support team."
+            )
+
+        return perimeter
+
+    def get_snowflake_token_url(self) -> str:
+        if "OBP_AUTH_SERVER" in self.conf:
+            auth_host = self.conf["OBP_AUTH_SERVER"]
+        else:
+            from urllib.parse import urlparse
+
+            auth_host = "auth." + urlparse(SERVICE_URL).hostname.split(".", 1)[1]
+
+        return "https://" + auth_host + "/generate/snowflake"
+
+    def get_service_auth_header(self) -> str:
+        if "METAFLOW_SERVICE_AUTH_KEY" in self.conf:
+            return {"x-api-key": self.conf["METAFLOW_SERVICE_AUTH_KEY"]}
+        else:
+            return json.loads(environ.get("METAFLOW_SERVICE_HEADERS"))
+
+    def _get_integration_url(self) -> str:
+        from metaflow_extensions.outerbounds.remote_config import init_config
+        from os import environ
+
+        if "OBP_INTEGRATIONS_URL" in self.conf:
+            integrations_url = self.conf["OBP_INTEGRATIONS_URL"]
+        else:
+            # if the integrations url is not in metaflow config, try to get it from the environment
+            integrations_url = environ.get("OBP_INTEGRATIONS_URL", "")
+
+        if not integrations_url:
+            raise OuterboundsSnowflakeConnectorException(
+                "No integrations url set. Please notify your Outerbounds support team about this issue."
+            )
+
+        return integrations_url
+
+    def _make_request(self, url, headers: Dict, payload: Dict) -> Dict:
+        try:
+            from metaflow.metaflow_config import SERVICE_HEADERS
+
+            request_headers = {**headers, **(SERVICE_HEADERS or {})}
+        except ImportError:
+            headers = headers
+
+        retryable_status_codes = [409]
+        json_payload = json.dumps(payload)
+        for attempt in range(2):  # 0 = initial attempt, 1-2 = retries
+            response = _api_server_get(
+                url, data=json_payload, headers=request_headers, conn_error_retries=5
+            )
+            if response.status_code not in retryable_status_codes:
+                break
+
+            if attempt < 2:  # Don't sleep after the last attempt
+                sleep_time = 0.5 * (attempt + 1)
+                time.sleep(sleep_time)
+
+        response = _api_server_get(
+            url, data=json_payload, headers=request_headers, conn_error_retries=5
+        )
+        self._handle_error_response(response)
+        return response.json()
+
+    @staticmethod
+    def _handle_error_response(response: requests.Response):
+        if response.status_code >= 500:
+            raise OuterboundsSnowflakeConnectorException(
+                f"Server error: {response.text}. Please reach out to your Outerbounds support team."
+            )
+
+        body = response.json()
+        status_code = body.get("error", {}).get("statusCode", response.status_code)
+        if status_code == 404:
+            raise OuterboundsSnowflakeConnectorException(f"Secret not found: {body}")
+
+        if status_code >= 400:
+            try:
+                raise OuterboundsSnowflakeConnectorException(
+                    f"status_code={status_code}\t*{body['error']['details']['kind']}*\n{body['error']['details']['message']}"
+                )
+            except KeyError:
+                raise OuterboundsSnowflakeConnectorException(
+                    f"status_code={status_code} Unexpected error: {body}"
+                )

metaflow_extensions/outerbounds/plugins/snowpark/snowpark.py

@@ -0,0 +1,309 @@
+import os
+import re
+import shlex
+import atexit
+import json
+import math
+import time
+import hashlib
+
+from metaflow import util
+
+from metaflow.metaflow_config import (
+    SERVICE_INTERNAL_URL,
+    SERVICE_HEADERS,
+    DEFAULT_METADATA,
+    DATASTORE_SYSROOT_S3,
+    DATATOOLS_S3ROOT,
+    KUBERNETES_SANDBOX_INIT_SCRIPT,
+    OTEL_ENDPOINT,
+    DEFAULT_SECRETS_BACKEND_TYPE,
+    AWS_SECRETS_MANAGER_DEFAULT_REGION,
+    S3_SERVER_SIDE_ENCRYPTION,
+    S3_ENDPOINT_URL,
+)
+from metaflow.metaflow_config_funcs import config_values
+
+from metaflow.mflog import (
+    export_mflog_env_vars,
+    bash_capture_logs,
+    BASH_SAVE_LOGS,
+    get_log_tailer,
+    tail_logs,
+)
+
+from .snowpark_client import SnowparkClient
+from .snowpark_exceptions import SnowparkException, SnowparkKilledException
+from .snowpark_job import SnowparkJob
+
+# Redirect structured logs to $PWD/.logs/
+LOGS_DIR = "$PWD/.logs"
+STDOUT_FILE = "mflog_stdout"
+STDERR_FILE = "mflog_stderr"
+STDOUT_PATH = os.path.join(LOGS_DIR, STDOUT_FILE)
+STDERR_PATH = os.path.join(LOGS_DIR, STDERR_FILE)
+
+
+class Snowpark(object):
+    def __init__(
+        self,
+        datastore,
+        metadata,
+        environment,
+        client_credentials,
+    ):
+        self.datastore = datastore
+        self.metadata = metadata
+        self.environment = environment
+        self.snowpark_client = SnowparkClient(**client_credentials)
+        atexit.register(lambda: self.job.kill() if hasattr(self, "job") else None)
+
+    def _job_name(self, user, flow_name, run_id, step_name, task_id, retry_count):
+        unique_str = (
+            "{user}-{flow_name}-{run_id}-{step_name}-{task_id}-{retry_count}".format(
+                user=user,
+                flow_name=flow_name,
+                run_id=str(run_id) if run_id is not None else "",
+                step_name=step_name,
+                task_id=str(task_id) if task_id is not None else "",
+                retry_count=str(retry_count) if retry_count is not None else "",
+            )
+        )
+        unique_hash = hashlib.md5(unique_str.encode("utf-8")).hexdigest()[:8]
+        raw_prefix = f"{flow_name}-{step_name}"
+        safe_prefix = re.sub(r"[^a-z0-9]", "-", raw_prefix.lower())
+        safe_prefix = safe_prefix[:54]
+        safe_prefix = safe_prefix.lstrip("-")
+        return f"{safe_prefix}-{unique_hash}"
+
+    def _command(self, environment, code_package_url, step_name, step_cmds, task_spec):
+        mflog_expr = export_mflog_env_vars(
+            datastore_type=self.datastore.TYPE,
+            stdout_path=STDOUT_PATH,
+            stderr_path=STDERR_PATH,
+            **task_spec,
+        )
+        init_cmds = environment.get_package_commands(
+            code_package_url, self.datastore.TYPE
+        )
+        init_expr = " && ".join(init_cmds)
+        step_expr = bash_capture_logs(
+            " && ".join(
+                environment.bootstrap_commands(step_name, self.datastore.TYPE)
+                + step_cmds
+            )
+        )
+
+        # construct an entry point that
+        # 1) initializes the mflog environment (mflog_expr)
+        # 2) bootstraps a metaflow environment (init_expr)
+        # 3) executes a task (step_expr)
+
+        # the `true` command is to make sure that the generated command
+        # plays well with docker containers which have entrypoint set as
+        # eval $@
+        cmd_str = "true && mkdir -p %s && %s && %s && %s; " % (
+            LOGS_DIR,
+            mflog_expr,
+            init_expr,
+            step_expr,
+        )
+        # after the task has finished, we save its exit code (fail/success)
+        # and persist the final logs. The whole entrypoint should exit
+        # with the exit code (c) of the task.
+        #
+        # Note that if step_expr OOMs, this tail expression is never executed.
+        # We lose the last logs in this scenario.
+        cmd_str += "c=$?; %s; exit $c" % BASH_SAVE_LOGS
+        # For supporting sandboxes, ensure that a custom script is executed before
+        # anything else is executed. The script is passed in as an env var.
+        cmd_str = (
+            '${METAFLOW_INIT_SCRIPT:+eval \\"${METAFLOW_INIT_SCRIPT}\\"} && %s'
+            % cmd_str
+        )
+        return shlex.split('bash -c "%s"' % cmd_str)
+
+    def create_job(
+        self,
+        step_name,
+        step_cli,
+        task_spec,
+        code_package_sha,
+        code_package_url,
+        code_package_ds,
+        image=None,
+        stage=None,
+        compute_pool=None,
+        volume_mounts=None,
+        external_integration=None,
+        cpu=None,
+        gpu=None,
+        memory=None,
+        run_time_limit=None,
+        env=None,
+        attrs=None,
+    ) -> SnowparkJob:
+        if env is None:
+            env = {}
+        if attrs is None:
+            attrs = {}
+
+        job_name = self._job_name(
+            attrs.get("metaflow.user"),
+            attrs.get("metaflow.flow_name"),
+            attrs.get("metaflow.run_id"),
+            attrs.get("metaflow.step_name"),
+            attrs.get("metaflow.task_id"),
+            attrs.get("metaflow.retry_count"),
+        )
+
+        snowpark_job = (
+            SnowparkJob(
+                client=self.snowpark_client,
+                name=job_name,
+                command=self._command(
+                    self.environment, code_package_url, step_name, [step_cli], task_spec
+                ),
+                step_name=step_name,
+                step_cli=step_cli,
+                task_spec=task_spec,
+                code_package_sha=code_package_sha,
+                code_package_url=code_package_url,
+                code_package_ds=code_package_ds,
+                image=image,
+                stage=stage,
+                compute_pool=compute_pool,
+                volume_mounts=volume_mounts,
+                external_integration=external_integration,
+                cpu=cpu,
+                gpu=gpu,
+                memory=memory,
+                run_time_limit=run_time_limit,
+                env=env,
+                attrs=attrs,
+            )
+            .environment_variable("METAFLOW_CODE_SHA", code_package_sha)
+            .environment_variable("METAFLOW_CODE_URL", code_package_url)
+            .environment_variable("METAFLOW_CODE_DS", code_package_ds)
+            .environment_variable("METAFLOW_USER", attrs["metaflow.user"])
+            .environment_variable("METAFLOW_SERVICE_URL", SERVICE_INTERNAL_URL)
+            .environment_variable(
+                "METAFLOW_SERVICE_HEADERS", json.dumps(SERVICE_HEADERS)
+            )
+            .environment_variable("METAFLOW_DATASTORE_SYSROOT_S3", DATASTORE_SYSROOT_S3)
+            .environment_variable("METAFLOW_DATATOOLS_S3ROOT", DATATOOLS_S3ROOT)
+            .environment_variable("METAFLOW_DEFAULT_DATASTORE", self.datastore.TYPE)
+            .environment_variable("METAFLOW_DEFAULT_METADATA", DEFAULT_METADATA)
+            .environment_variable("METAFLOW_SNOWPARK_WORKLOAD", 1)
+            .environment_variable("METAFLOW_RUNTIME_ENVIRONMENT", "snowpark")
+            .environment_variable(
+                "METAFLOW_INIT_SCRIPT", KUBERNETES_SANDBOX_INIT_SCRIPT
+            )
+            .environment_variable("METAFLOW_OTEL_ENDPOINT", OTEL_ENDPOINT)
+            .environment_variable(
+                "SNOWFLAKE_WAREHOUSE",
+                self.snowpark_client.connection_parameters.get("warehouse"),
+            )
+        )
+
+        for k, v in config_values():
+            if k.startswith("METAFLOW_CONDA_") or k.startswith("METAFLOW_DEBUG_"):
+                snowpark_job.environment_variable(k, v)
+
+        if DEFAULT_SECRETS_BACKEND_TYPE is not None:
+            snowpark_job.environment_variable(
+                "METAFLOW_DEFAULT_SECRETS_BACKEND_TYPE", DEFAULT_SECRETS_BACKEND_TYPE
+            )
+        if AWS_SECRETS_MANAGER_DEFAULT_REGION is not None:
+            snowpark_job.environment_variable(
+                "METAFLOW_AWS_SECRETS_MANAGER_DEFAULT_REGION",
+                AWS_SECRETS_MANAGER_DEFAULT_REGION,
+            )
+        if S3_SERVER_SIDE_ENCRYPTION is not None:
+            snowpark_job.environment_variable(
+                "METAFLOW_S3_SERVER_SIDE_ENCRYPTION", S3_SERVER_SIDE_ENCRYPTION
+            )
+        if S3_ENDPOINT_URL is not None:
+            snowpark_job.environment_variable(
+                "METAFLOW_S3_ENDPOINT_URL", S3_ENDPOINT_URL
+            )
+
+        for name, value in env.items():
+            snowpark_job.environment_variable(name, value)
+
+        return snowpark_job
+
+    def launch_job(self, **kwargs):
+        self.job = self.create_job(**kwargs).create().execute()
+
+    def wait(self, stdout_location, stderr_location, echo=None):
+        def update_delay(secs_since_start):
+            # this sigmoid function reaches
+            # - 0.1 after 11 minutes
+            # - 0.5 after 15 minutes
+            # - 1.0 after 23 minutes
+            # in other words, the user will see very frequent updates
+            # during the first 10 minutes
+            sigmoid = 1.0 / (1.0 + math.exp(-0.01 * secs_since_start + 9.0))
+            return 0.5 + sigmoid * 30.0
+
+        def wait_for_launch(job):
+            status = job.status
+
+            echo(
+                "Task is starting (%s)..." % status,
+                "stderr",
+                job_id=job.id,
+            )
+            t = time.time()
+            start_time = time.time()
+            while job.is_waiting:
+                new_status = job.status
+                if status != new_status or (time.time() - t) > 30:
+                    status = new_status
+                    echo(
+                        "Task is starting (%s)..." % status,
+                        "stderr",
+                        job_id=job.id,
+                    )
+                    t = time.time()
+                time.sleep(update_delay(time.time() - start_time))
+
+        _make_prefix = lambda: b"[%s] " % util.to_bytes(self.job.id)
+
+        stdout_tail = get_log_tailer(stdout_location, self.datastore.TYPE)
+        stderr_tail = get_log_tailer(stderr_location, self.datastore.TYPE)
+
+        # 1) Loop until the job has started
+        wait_for_launch(self.job)
+
+        # 2) Tail logs until the job has finished
+        tail_logs(
+            prefix=_make_prefix(),
+            stdout_tail=stdout_tail,
+            stderr_tail=stderr_tail,
+            echo=echo,
+            has_log_updates=lambda: self.job.is_running,
+        )
+
+        if self.job.has_failed:
+            msg = next(
+                msg
+                for msg in [
+                    self.job.message,
+                    "Task crashed.",
+                ]
+                if msg is not None
+            )
+            raise SnowparkException(
+                "%s " "This could be a transient error. " "Use @retry to retry." % msg
+            )
+        else:
+            if self.job.is_running:
+                # Kill the job if it is still running by throwing an exception.
+                raise SnowparkKilledException("Task failed!")
+            echo(
+                "Task finished with message '%s'." % self.job.message,
+                "stderr",
+                job_id=self.job.id,
+            )