ob-metaflow 2.11.13.1__py2.py3-none-any.whl → 2.19.7.1rc0__py2.py3-none-any.whl

metaflow/plugins/pypi/pip.py

@@ -8,11 +8,11 @@ from concurrent.futures import ThreadPoolExecutor
 from itertools import chain, product
 from urllib.parse import unquote
 
+from metaflow.debug import debug
 from metaflow.exception import MetaflowException
-from metaflow.util import which
 
 from .micromamba import Micromamba
-from .utils import pip_tags, wheel_tags
+from .utils import pip_tags, wheel_tags, markers_from_platform, conda_platform
 
 
 class PipException(MetaflowException):
@@ -25,6 +25,23 @@ class PipException(MetaflowException):
         super(PipException, self).__init__(msg)
 
 
+class PipPackageNotFound(Exception):
+    "Wrapper for pip package resolve errors."
+
+    def __init__(self, error):
+        self.error = error
+        try:
+            # Parse the package spec from error message:
+            # ERROR: ERROR: Could not find a version that satisfies the requirement pkg==0.0.1 (from versions: none)
+            # ERROR: No matching distribution found for pkg==0.0.1
+            self.package_spec = re.search(
+                "ERROR: No matching distribution found for (.*)", self.error
+            )[1]
+            self.package_name = re.match("\w*", self.package_spec)[0]
+        except Exception:
+            pass
+
+
 METADATA_FILE = "{prefix}/.pip/metadata"
 INSTALLATION_MARKER = "{prefix}/.pip/id"
 
@@ -34,10 +51,23 @@ INSTALLATION_MARKER = "{prefix}/.pip/id"
 
 
 class Pip(object):
-    def __init__(self, micromamba=None):
+    def __init__(self, micromamba=None, logger=None):
         # pip is assumed to be installed inside a conda environment managed by
         # micromamba. pip commands are executed using `micromamba run --prefix`
-        self.micromamba = micromamba or Micromamba()
+        self.micromamba = micromamba or Micromamba(logger)
+        if logger:
+            self.logger = logger
+        else:
+            self.logger = lambda *args, **kwargs: None  # No-op logger if not provided
+
+    def _get_resolved_python_version(self, prefix):
+        try:
+            result = self.micromamba._call(["list", "--prefix", prefix, "--json"])
+            for package in result:
+                if package.get("name") == "python":
+                    return package["version"]
+        except Exception:
+            return None
 
     def solve(self, id_, packages, python, platform):
         prefix = self.micromamba.path_to_environment(id_)
@@ -46,12 +76,19 @@ class Pip(object):
             msg += "for id {id}".format(id=id_)
             raise PipException(msg)
 
+        resolved_python = self._get_resolved_python_version(prefix)
+        if not resolved_python:
+            raise PipException(
+                "Could not determine Python version from conda environment"
+            )
+
+        debug.conda_exec("Solving packages for PyPI environment %s" % id_)
         with tempfile.TemporaryDirectory() as tmp_dir:
             report = "{tmp_dir}/report.json".format(tmp_dir=tmp_dir)
             implementations, platforms, abis = zip(
                 *[
                     (tag.interpreter, tag.platform, tag.abi)
-                    for tag in pip_tags(python, platform)
+                    for tag in pip_tags(resolved_python, platform)
                 ]
             )
             custom_index_url, extra_index_urls = self.indices(prefix)
@@ -81,7 +118,25 @@ class Pip(object):
                     cmd.append(f"{package}{version}")
                 else:
                     cmd.append(f"{package}=={version}")
-            self._call(prefix, cmd)
+            try:
+                env = {}
+                if conda_platform() != platform:
+                    # cross-platform resolving requires patching the machine and system info for pip to pick up all the relevant packages.
+                    marker_overrides = markers_from_platform(platform)
+                    env = {
+                        "PIP_CUSTOMIZE_OVERRIDES": json.dumps(marker_overrides),
+                        "PYTHONPATH": os.path.join(
+                            os.path.dirname(__file__), "pip_patcher"
+                        ),
+                    }
+                self._call(prefix, cmd, env)
+            except PipPackageNotFound as ex:
+                # pretty print package errors
+                raise PipException(
+                    "Unable to find a binary distribution compatible with %s for %s.\n\n"
+                    "Note: ***@pypi*** does not currently support source distributions"
+                    % (ex.package_spec, platform)
+                )
 
             def _format(dl_info):
                 res = {k: v for k, v in dl_info.items() if k in ["url"]}
@@ -96,9 +151,9 @@ class Pip(object):
                     res["url"] = "{vcs}+{url}@{commit_id}{subdir_str}".format(
                         **vcs_info,
                         **res,
-                        subdir_str="#subdirectory=%s" % subdirectory
-                        if subdirectory
-                        else ""
+                        subdir_str=(
+                            "#subdirectory=%s" % subdirectory if subdirectory else ""
+                        ),
                     )
                     # used to deduplicate the storage location in case wheel does not
                     # build with enough unique identifiers.
@@ -112,15 +167,26 @@ class Pip(object):
 
     def download(self, id_, packages, python, platform):
         prefix = self.micromamba.path_to_environment(id_)
+
+        resolved_python = self._get_resolved_python_version(prefix)
+        if not resolved_python:
+            raise PipException(
+                "Could not determine Python version from conda environment"
+            )
+
         metadata_file = METADATA_FILE.format(prefix=prefix)
         # download packages only if they haven't ever been downloaded before
         if os.path.isfile(metadata_file):
-            return
+            with open(metadata_file, "r") as file:
+                metadata = json.load(file)
+                if all(package["url"] in metadata for package in packages):
+                    return
 
         metadata = {}
         custom_index_url, extra_index_urls = self.indices(prefix)
 
         # build wheels if needed
+        debug.conda_exec("Building wheels for PyPI environment %s if necessary" % id_)
         with ThreadPoolExecutor() as executor:
 
             def _build(key, package):
@@ -158,7 +224,11 @@ class Pip(object):
                     if os.path.isfile(os.path.join(path, f)) and f.endswith(".whl")
                 ]
                 if (
-                    len(set(pip_tags(python, platform)).intersection(wheel_tags(wheel)))
+                    len(
+                        set(pip_tags(resolved_python, platform)).intersection(
+                            wheel_tags(wheel)
+                        )
+                    )
                     == 0
                 ):
                     raise PipException(
@@ -177,10 +247,11 @@ class Pip(object):
         implementations, platforms, abis = zip(
             *[
                 (tag.interpreter, tag.platform, tag.abi)
-                for tag in pip_tags(python, platform)
+                for tag in pip_tags(resolved_python, platform)
             ]
         )
 
+        debug.conda_exec("Downloading packages for PyPI environment %s" % id_)
         cmd = [
             "download",
             "--no-deps",
@@ -220,6 +291,7 @@ class Pip(object):
         # Pip can't install packages if the underlying virtual environment doesn't
         # share the same platform
         if self.micromamba.platform() == platform:
+            debug.conda_exec("Installing packages for local PyPI environment %s" % id_)
             cmd = [
                 "install",
                 "--no-compile",
@@ -302,11 +374,14 @@ class Pip(object):
                 .strip()
             )
         except subprocess.CalledProcessError as e:
+            errors = e.stderr.decode()
+            if "No matching distribution" in errors:
+                raise PipPackageNotFound(errors)
             raise PipException(
                 "command '{cmd}' returned error ({code}) {output}\n{stderr}".format(
                     cmd=" ".join(e.cmd),
                     code=e.returncode,
                     output=e.output.decode(),
-                    stderr=e.stderr.decode(),
+                    stderr=errors,
                 )
             )

metaflow/plugins/pypi/pypi_decorator.py

@@ -24,6 +24,13 @@ class PyPIStepDecorator(StepDecorator):
     name = "pypi"
     defaults = {"packages": {}, "python": None, "disabled": None}  # wheels
 
+    def __init__(self, attributes=None, statically_defined=False, inserted_by=None):
+        self._attributes_with_user_values = (
+            set(attributes.keys()) if attributes is not None else set()
+        )
+
+        super().__init__(attributes, statically_defined, inserted_by)
+
     def step_init(self, flow, graph, step, decos, environment, flow_datastore, logger):
         # The init_environment hook for Environment creates the relevant virtual
         # environments. The step_init hook sets up the relevant state for that hook to
@@ -34,7 +41,11 @@ class PyPIStepDecorator(StepDecorator):
 
         # Support flow-level decorator
         if "pypi_base" in self.flow._flow_decorators:
-            super_attributes = self.flow._flow_decorators["pypi_base"][0].attributes
+            pypi_base = self.flow._flow_decorators["pypi_base"][0]
+            super_attributes = pypi_base.attributes
+            self._attributes_with_user_values.update(
+                pypi_base._attributes_with_user_values
+            )
             self.attributes["packages"] = {
                 **super_attributes["packages"],
                 **self.attributes["packages"],
@@ -70,6 +81,10 @@ class PyPIStepDecorator(StepDecorator):
         # --environment=pypi to --environment=conda
         _supported_virtual_envs.extend(["pypi"])
 
+        # TODO: Hardcoded for now to support the fast bakery environment.
+        # We should introduce a more robust mechanism for appending supported environments, for example from within extensions.
+        _supported_virtual_envs.extend(["fast-bakery"])
+
         # The --environment= requirement ensures that valid virtual environments are
         # created for every step to execute it, greatly simplifying the @pypi
         # implementation.
@@ -83,6 +98,15 @@ class PyPIStepDecorator(StepDecorator):
                     ),
                 )
             )
+        # TODO: This code snippet can be done away with by altering the constructor of
+        #       MetaflowEnvironment. A good first-task exercise.
+        # Avoid circular import
+        from metaflow.plugins.datastores.local_storage import LocalStorage
+
+        environment.set_local_root(LocalStorage.get_datastore_root_from_config(logger))
+
+    def is_attribute_user_defined(self, name):
+        return name in self._attributes_with_user_values
 
 
 class PyPIFlowDecorator(FlowDecorator):
@@ -104,12 +128,20 @@ class PyPIFlowDecorator(FlowDecorator):
     name = "pypi_base"
     defaults = {"packages": {}, "python": None, "disabled": None}
 
+    def __init__(self, attributes=None, statically_defined=False, inserted_by=None):
+        self._attributes_with_user_values = (
+            set(attributes.keys()) if attributes is not None else set()
+        )
+
+        super().__init__(attributes, statically_defined, inserted_by)
+
     def flow_init(
         self, flow, graph, environment, flow_datastore, metadata, logger, echo, options
     ):
         from metaflow import decorators
 
         decorators._attach_decorators(flow, ["pypi"])
+        decorators._init(flow)
 
         # @pypi uses a conda environment to create a virtual environment.
         # The conda environment can be created through micromamba.
@@ -119,6 +151,10 @@ class PyPIFlowDecorator(FlowDecorator):
         # --environment=pypi to --environment=conda
         _supported_virtual_envs.extend(["pypi"])
 
+        # TODO: Hardcoded for now to support the fast bakery environment.
+        # We should introduce a more robust mechanism for appending supported environments, for example from within extensions.
+        _supported_virtual_envs.extend(["fast-bakery"])
+
         # The --environment= requirement ensures that valid virtual environments are
         # created for every step to execute it, greatly simplifying the @conda
         # implementation.

metaflow/plugins/pypi/utils.py

@@ -1,4 +1,3 @@
-import os
 import platform
 import sys
 
@@ -17,10 +16,13 @@ else:
     from metaflow._vendor.packaging import tags
     from metaflow._vendor.packaging.utils import parse_wheel_filename
 
-from urllib.parse import unquote, urlparse
+from urllib.parse import unquote
 
 from metaflow.exception import MetaflowException
 
+MICROMAMBA_URL = "https://micro.mamba.pm/api/micromamba/{platform}/{version}"
+MICROMAMBA_MIRROR_URL = "https://micromamba.outerbounds.sh/{platform}/{version}.tar.bz2"
+
 
 def conda_platform():
     # Returns the conda platform for the Python interpreter
@@ -39,6 +41,28 @@ def conda_platform():
             return "osx-64"
 
 
+def markers_from_platform(platform):
+    plat, mach = platform.split("-")
+
+    platform_system = {"osx": "Darwin", "linux": "Linux"}.get(plat)
+    platform_machine = {
+        "32": "x86",
+        "64": "x86_64",
+        "arm64": "aarch64",
+        "aarch64": "aarch64",
+    }.get(mach)
+
+    markers = {
+        k: v
+        for k, v in {
+            "platform_system": platform_system,
+            "platform_machine": platform_machine,
+        }.items()
+        if v is not None
+    }
+    return markers
+
+
 def wheel_tags(wheel):
     _, _, _, tags = parse_wheel_filename(wheel)
     return list(tags)
@@ -67,9 +91,31 @@ def pip_tags(python_version, mamba_platform):
                 "_2_25",
                 "_2_26",
                 "_2_27",
+                "_2_28",
+                "_2_29",
             )
         ]
         platforms.append("linux_x86_64")
+    elif mamba_platform == "linux-aarch64":
+        platforms = [
+            "manylinux%s_aarch64" % s
+            for s in (
+                "2014",
+                "_2_17",
+                "_2_18",
+                "_2_19",
+                "_2_20",
+                "_2_21",
+                "_2_23",
+                "_2_24",
+                "_2_25",
+                "_2_26",
+                "_2_27",
+                "_2_28",
+                "_2_29",
+            )
+        ]
+        platforms.append("linux_aarch64")
     elif mamba_platform == "osx-64":
         platforms = tags.mac_platforms(arch="x86_64")
     elif mamba_platform == "osx-arm64":

metaflow/plugins/resources_decorator.py

@@ -23,7 +23,7 @@ class ResourcesDecorator(StepDecorator):
     ----------
     cpu : int, default 1
         Number of CPUs required for this step.
-    gpu : int, default 0
+    gpu : int, optional, default None
         Number of GPUs required for this step.
     disk : int, optional, default None
         Disk size (in MB) required for this step. Only applies on Kubernetes.
@@ -37,7 +37,7 @@ class ResourcesDecorator(StepDecorator):
     name = "resources"
     defaults = {
         "cpu": "1",
-        "gpu": "0",
+        "gpu": None,
         "disk": None,
         "memory": "4096",
         "shared_memory": None,

metaflow/plugins/secrets/__init__.py

@@ -9,3 +9,6 @@ class SecretsProvider(abc.ABC):
     def get_secret_as_dict(self, secret_id, options={}, role=None) -> Dict[str, str]:
         """Retrieve the secret from secrets backend, and return a dictionary of
         environment variables."""
+
+
+from .secrets_func import get_secret

metaflow/plugins/secrets/secrets_decorator.py

@@ -1,182 +1,16 @@
 import os
-import re
 
 from metaflow.exception import MetaflowException
 from metaflow.decorators import StepDecorator
 from metaflow.metaflow_config import DEFAULT_SECRETS_ROLE
-from metaflow.unbounded_foreach import UBF_CONTROL
-
-from typing import Any, Dict, List, Union
-
-DISALLOWED_SECRETS_ENV_VAR_PREFIXES = ["METAFLOW_"]
-
-
-def get_default_secrets_backend_type():
-    from metaflow.metaflow_config import DEFAULT_SECRETS_BACKEND_TYPE
-
-    if DEFAULT_SECRETS_BACKEND_TYPE is None:
-        raise MetaflowException(
-            "No default secrets backend type configured, but needed by @secrets. "
-            "Set METAFLOW_DEFAULT_SECRETS_BACKEND_TYPE."
-        )
-    return DEFAULT_SECRETS_BACKEND_TYPE
-
-
-class SecretSpec:
-    def __init__(self, secrets_backend_type, secret_id, options={}, role=None):
-        self._secrets_backend_type = secrets_backend_type
-        self._secret_id = secret_id
-        self._options = options
-        self._role = role
-
-    @property
-    def secrets_backend_type(self):
-        return self._secrets_backend_type
-
-    @property
-    def secret_id(self):
-        return self._secret_id
-
-    @property
-    def options(self):
-        return self._options
-
-    @property
-    def role(self):
-        return self._role
-
-    def to_json(self):
-        """Mainly used for testing... not the same as the input dict in secret_spec_from_dict()!"""
-        return {
-            "secrets_backend_type": self.secrets_backend_type,
-            "secret_id": self.secret_id,
-            "options": self.options,
-            "role": self.role,
-        }
-
-    def __str__(self):
-        return "%s (%s)" % (self._secret_id, self._secrets_backend_type)
-
-    @staticmethod
-    def secret_spec_from_str(secret_spec_str, role):
-        # "." may be used in secret_id one day (provider specific). HOWEVER, it provides the best UX for
-        # non-conflicting cases (i.e. for secret ids that don't contain "."). This is true for all AWS
-        # Secrets Manager secrets.
-        #
-        # So we skew heavily optimize for best upfront UX for the present (1/2023).
-        #
-        # If/when a certain secret backend supports "." secret names, we can figure out a solution at that time.
-        # At a minimum, dictionary style secret spec may be used with no code changes (see secret_spec_from_dict()).
-        # Other options could be:
-        #  - accept and document that "." secret_ids don't work in Metaflow (across all possible providers)
-        #  - add a Metaflow config variable that specifies the separator (default ".")
-        #  - smarter spec parsing, that errors on secrets that look ambiguous. "aws-secrets-manager.XYZ" could mean:
-        #    + secret_id "XYZ" in aws-secrets-manager backend, OR
-        #    + secret_id "aws-secrets-manager.XYZ" default backend (if it is defined).
-        #    + in this case, user can simply set "azure-key-vault.aws-secrets-manager.XYZ" instead!
-        parts = secret_spec_str.split(".", maxsplit=1)
-        if len(parts) == 1:
-            secrets_backend_type = get_default_secrets_backend_type()
-            secret_id = parts[0]
-        else:
-            secrets_backend_type = parts[0]
-            secret_id = parts[1]
-        return SecretSpec(
-            secrets_backend_type, secret_id=secret_id, options={}, role=role
-        )
-
-    @staticmethod
-    def secret_spec_from_dict(secret_spec_dict, role):
-        if "type" not in secret_spec_dict:
-            secrets_backend_type = get_default_secrets_backend_type()
-        else:
-            secrets_backend_type = secret_spec_dict["type"]
-            if not isinstance(secrets_backend_type, str):
-                raise MetaflowException(
-                    "Bad @secrets specification - 'type' must be a string - found %s"
-                    % type(secrets_backend_type)
-                )
-        secret_id = secret_spec_dict.get("id")
-        if not isinstance(secret_id, str):
-            raise MetaflowException(
-                "Bad @secrets specification - 'id' must be a string - found %s"
-                % type(secret_id)
-            )
-        options = secret_spec_dict.get("options", {})
-        if not isinstance(options, dict):
-            raise MetaflowException(
-                "Bad @secrets specification - 'option' must be a dict - found %s"
-                % type(options)
-            )
-        role_for_source = secret_spec_dict.get("role", None)
-        if role_for_source is not None:
-            if not isinstance(role_for_source, str):
-                raise MetaflowException(
-                    "Bad @secrets specification - 'role' must be a str - found %s"
-                    % type(role_for_source)
-                )
-            role = role_for_source
-        return SecretSpec(
-            secrets_backend_type, secret_id=secret_id, options=options, role=role
-        )
-
-
-def validate_env_vars_across_secrets(all_secrets_env_vars):
-    vars_injected_by = {}
-    for secret_spec, env_vars in all_secrets_env_vars:
-        for k in env_vars:
-            if k in vars_injected_by:
-                raise MetaflowException(
-                    "Secret '%s' will inject '%s' as env var, and it is also added by '%s'"
-                    % (secret_spec, k, vars_injected_by[k])
-                )
-            vars_injected_by[k] = secret_spec
-
-
-def validate_env_vars_vs_existing_env(all_secrets_env_vars):
-    for secret_spec, env_vars in all_secrets_env_vars:
-        for k in env_vars:
-            if k in os.environ:
-                raise MetaflowException(
-                    "Secret '%s' will inject '%s' as env var, but it already exists in env"
-                    % (secret_spec, k)
-                )
-
-
-def validate_env_vars(env_vars):
-    for k, v in env_vars.items():
-        if not isinstance(k, str):
-            raise MetaflowException("Found non string key %s (%s)" % (str(k), type(k)))
-        if not isinstance(v, str):
-            raise MetaflowException(
-                "Found non string value %s (%s)" % (str(v), type(v))
-            )
-        if not re.fullmatch("[a-zA-Z_][a-zA-Z0-9_]*", k):
-            raise MetaflowException("Found invalid env var name '%s'." % k)
-        for disallowed_prefix in DISALLOWED_SECRETS_ENV_VAR_PREFIXES:
-            if k.startswith(disallowed_prefix):
-                raise MetaflowException(
-                    "Found disallowed env var name '%s' (starts with '%s')."
-                    % (k, disallowed_prefix)
-                )
-
-
-def get_secrets_backend_provider(secrets_backend_type):
-    from metaflow.plugins import SECRETS_PROVIDERS
-
-    try:
-        provider_cls = [
-            pc for pc in SECRETS_PROVIDERS if pc.TYPE == secrets_backend_type
-        ][0]
-        return provider_cls()
-    except IndexError:
-        raise MetaflowException(
-            "Unknown secrets backend type %s (available types: %s)"
-            % (
-                secrets_backend_type,
-                ", ".join(pc.TYPE for pc in SECRETS_PROVIDERS if pc.TYPE != "inline"),
-            )
-        )
+from metaflow.plugins.secrets.secrets_spec import SecretSpec
+from metaflow.plugins.secrets.utils import (
+    get_secrets_backend_provider,
+    validate_env_vars,
+    validate_env_vars_across_secrets,
+    validate_env_vars_vs_existing_env,
+)
+from metaflow.unbounded_foreach import UBF_TASK
 
 
 class SecretsDecorator(StepDecorator):
@@ -188,13 +22,14 @@ class SecretsDecorator(StepDecorator):
     ----------
     sources : List[Union[str, Dict[str, Any]]], default: []
         List of secret specs, defining how the secrets are to be retrieved
+    role : str, optional, default: None
+        Role to use for fetching secrets
+    allow_override : bool, optional, default: False
+        Toggle whether secrets can replace existing environment variables.
     """
 
     name = "secrets"
-    defaults = {
-        "sources": [],
-        "role": None,
-    }
+    defaults = {"sources": [], "role": None, "allow_override": False}
 
     def task_pre_step(
         self,
@@ -210,8 +45,17 @@ class SecretsDecorator(StepDecorator):
         ubf_context,
         inputs,
     ):
-        if ubf_context == UBF_CONTROL:
-            """control tasks (as used in "unbounded for each") don't need secrets"""
+        if (
+            ubf_context
+            and ubf_context == UBF_TASK
+            and os.environ.get("METAFLOW_RUNTIME_ENVIRONMENT", "local") == "local"
+        ):
+            # We will skip the secret injection for "locally" launched UBF_TASK (worker) tasks
+            # When we "locally" run @parallel tasks, the control task will create the worker tasks and the environment variables
+            # of the control task are inherited by the worker tasks. If we don't skip setting secrets in the worker task then the
+            # worker tasks will try to set the environment variables again which will cause a clash with the control task's env vars,
+            # causing the @secrets' `task_pre_step` to fail. In remote settings, (e.g. AWS Batch/Kubernetes), the worker task and
+            # control task are independently created so there is no chances of an env var clash.
             return
         # List of pairs (secret_spec, env_vars_from_this_spec)
         all_secrets_env_vars = []
@@ -267,7 +111,8 @@ class SecretsDecorator(StepDecorator):
             all_secrets_env_vars.append((secret_spec, env_vars_for_secret))
 
         validate_env_vars_across_secrets(all_secrets_env_vars)
-        validate_env_vars_vs_existing_env(all_secrets_env_vars)
+        if not self.attributes["allow_override"]:
+            validate_env_vars_vs_existing_env(all_secrets_env_vars)
 
         # By this point
         # all_secrets_env_vars contains a list of dictionaries... env maps.

metaflow/plugins/secrets/secrets_func.py

@@ -0,0 +1,49 @@
+from typing import Any, Dict, Optional, Union
+
+from metaflow.metaflow_config import DEFAULT_SECRETS_ROLE
+from metaflow.exception import MetaflowException
+from metaflow.plugins.secrets.secrets_spec import SecretSpec
+from metaflow.plugins.secrets.utils import get_secrets_backend_provider
+
+
+def get_secret(
+    source: Union[str, Dict[str, Any]], role: Optional[str] = None
+) -> Dict[str, str]:
+    """
+    Get secret from source
+
+    Parameters
+    ----------
+    source : Union[str, Dict[str, Any]]
+        Secret spec, defining how the secret is to be retrieved
+    role : str, optional
+        Role to use for fetching secrets
+    """
+    if role is None:
+        role = DEFAULT_SECRETS_ROLE
+
+    secret_spec = None
+
+    if isinstance(source, str):
+        secret_spec = SecretSpec.secret_spec_from_str(source, role=role)
+    elif isinstance(source, dict):
+        secret_spec = SecretSpec.secret_spec_from_dict(source, role=role)
+    else:
+        raise MetaflowException(
+            "get_secrets sources items must be either a string or a dict"
+        )
+
+    secrets_backend_provider = get_secrets_backend_provider(
+        secret_spec.secrets_backend_type
+    )
+    try:
+        dict_for_secret = secrets_backend_provider.get_secret_as_dict(
+            secret_spec.secret_id,
+            options=secret_spec.options,
+            role=secret_spec.role,
+        )
+        return dict_for_secret
+    except Exception as e:
+        raise MetaflowException(
+            "Failed to retrieve secret '%s': %s" % (secret_spec.secret_id, e)
+        )