ob-metaflow 2.15.18.1__py2.py3-none-any.whl → 2.16.0.1__py2.py3-none-any.whl

metaflow/plugins/pypi/conda_decorator.py

@@ -1,5 +1,3 @@
-import importlib
-import json
 import os
 import platform
 import re
@@ -7,12 +5,9 @@ import sys
 import tempfile
 
 from metaflow.decorators import FlowDecorator, StepDecorator
-from metaflow.extension_support import EXT_PKG
 from metaflow.metadata_provider import MetaDatum
 from metaflow.metaflow_environment import InvalidEnvironmentException
-from metaflow.util import get_metaflow_root
-
-from ...info_file import INFO_FILE
+from metaflow.packaging_sys import ContentType
 
 
 class CondaStepDecorator(StepDecorator):
@@ -45,26 +40,31 @@ class CondaStepDecorator(StepDecorator):
         "python": None,
         "disabled": None,
     }
+
+    _metaflow_home = None
+    _addl_env_vars = None
+
     # To define conda channels for the whole solve, users can specify
     # CONDA_CHANNELS in their environment. For pinning specific packages to specific
     # conda channels, users can specify channel::package as the package name.
 
-    def __init__(self, attributes=None, statically_defined=False):
+    def __init__(self, attributes=None, statically_defined=False, inserted_by=None):
         self._attributes_with_user_values = (
             set(attributes.keys()) if attributes is not None else set()
         )
 
-        super(CondaStepDecorator, self).__init__(attributes, statically_defined)
+        super(CondaStepDecorator, self).__init__(
+            attributes, statically_defined, inserted_by
+        )
 
     def init(self):
-        super(CondaStepDecorator, self).init()
-
         # Support legacy 'libraries=' attribute for the decorator.
         self.attributes["packages"] = {
             **self.attributes["libraries"],
             **self.attributes["packages"],
         }
-        del self.attributes["libraries"]
+        # Keep because otherwise make_decorator_spec will fail
+        self.attributes["libraries"] = {}
         if self.attributes["packages"]:
             self._attributes_with_user_values.add("packages")
 
@@ -152,67 +152,17 @@ class CondaStepDecorator(StepDecorator):
     def runtime_init(self, flow, graph, package, run_id):
         if self.disabled:
             return
-        # Create a symlink to metaflow installed outside the virtual environment.
-        self.metaflow_dir = tempfile.TemporaryDirectory(dir="/tmp")
-        os.symlink(
-            os.path.join(get_metaflow_root(), "metaflow"),
-            os.path.join(self.metaflow_dir.name, "metaflow"),
-        )
-
-        info = os.path.join(get_metaflow_root(), os.path.basename(INFO_FILE))
-        # Symlink the INFO file as well to properly propagate down the Metaflow version
-        if os.path.isfile(info):
-            os.symlink(
-                info, os.path.join(self.metaflow_dir.name, os.path.basename(INFO_FILE))
+        # We need to make all the code package available to the user code in
+        # a temporary directory which will be added to the PYTHONPATH.
+        if self.__class__._metaflow_home is None:
+            # Do this ONCE per flow
+            self.__class__._metaflow_home = tempfile.TemporaryDirectory(dir="/tmp")
+            package.extract_into(
+                self.__class__._metaflow_home.name, ContentType.ALL_CONTENT
+            )
+            self.__class__._addl_env_vars = package.get_post_extract_env_vars(
+                package.package_metadata, self.__class__._metaflow_home.name
             )
-        else:
-            # If there is no info file, we will actually create one in this new
-            # place because we won't be able to properly resolve the EXT_PKG extensions
-            # the same way as outside conda (looking at distributions, etc.). In a
-            # Conda environment, as shown below (where we set self.addl_paths), all
-            # EXT_PKG extensions are PYTHONPATH extensions. Instead of re-resolving,
-            # we use the resolved information that is written out to the INFO file.
-            with open(
-                os.path.join(self.metaflow_dir.name, os.path.basename(INFO_FILE)),
-                mode="wt",
-                encoding="utf-8",
-            ) as f:
-                f.write(
-                    json.dumps(
-                        self.environment.get_environment_info(include_ext_info=True)
-                    )
-                )
-
-        # Support metaflow extensions.
-        self.addl_paths = None
-        try:
-            m = importlib.import_module(EXT_PKG)
-        except ImportError:
-            # No additional check needed because if we are here, we already checked
-            # for other issues when loading at the toplevel.
-            pass
-        else:
-            custom_paths = list(set(m.__path__))
-            # For some reason, at times, unique paths appear multiple times. We
-            # simplify to avoid un-necessary links.
-
-            if len(custom_paths) == 1:
-                # Regular package; we take a quick shortcut here.
-                os.symlink(
-                    custom_paths[0],
-                    os.path.join(self.metaflow_dir.name, EXT_PKG),
-                )
-            else:
-                # This is a namespace package, we therefore create a bunch of
-                # directories so that we can symlink in those separately, and we will
-                # add those paths to the PYTHONPATH for the interpreter. Note that we
-                # don't symlink to the parent of the package because that could end up
-                # including more stuff we don't want
-                self.addl_paths = []
-                for p in custom_paths:
-                    temp_dir = tempfile.mkdtemp(dir=self.metaflow_dir.name)
-                    os.symlink(p, os.path.join(temp_dir, EXT_PKG))
-                    self.addl_paths.append(temp_dir)
 
         # # Also install any environment escape overrides directly here to enable
         # # the escape to work even in non metaflow-created subprocesses
@@ -291,11 +241,15 @@ class CondaStepDecorator(StepDecorator):
         if self.disabled:
             return
         # Ensure local installation of Metaflow is visible to user code
-        python_path = self.metaflow_dir.name
-        if self.addl_paths is not None:
-            addl_paths = os.pathsep.join(self.addl_paths)
-            python_path = os.pathsep.join([addl_paths, python_path])
-        cli_args.env["PYTHONPATH"] = python_path
+        python_path = self.__class__._metaflow_home.name
+        addl_env_vars = {}
+        if self.__class__._addl_env_vars is not None:
+            for key, value in self.__class__._addl_env_vars.items():
+                if key == "PYTHONPATH":
+                    addl_env_vars[key] = os.pathsep.join([value, python_path])
+                else:
+                    addl_env_vars[key] = value
+        cli_args.env.update(addl_env_vars)
         if self.interpreter:
             # https://github.com/conda/conda/issues/7707
             # Also ref - https://github.com/Netflix/metaflow/pull/178
@@ -306,7 +260,9 @@ class CondaStepDecorator(StepDecorator):
     def runtime_finished(self, exception):
         if self.disabled:
             return
-        self.metaflow_dir.cleanup()
+        if self.__class__._metaflow_home is not None:
+            self.__class__._metaflow_home.cleanup()
+            self.__class__._metaflow_home = None
 
 
 class CondaFlowDecorator(FlowDecorator):
@@ -339,22 +295,23 @@ class CondaFlowDecorator(FlowDecorator):
         "disabled": None,
     }
 
-    def __init__(self, attributes=None, statically_defined=False):
+    def __init__(self, attributes=None, statically_defined=False, inserted_by=None):
         self._attributes_with_user_values = (
             set(attributes.keys()) if attributes is not None else set()
         )
 
-        super(CondaFlowDecorator, self).__init__(attributes, statically_defined)
+        super(CondaFlowDecorator, self).__init__(
+            attributes, statically_defined, inserted_by
+        )
 
     def init(self):
-        super(CondaFlowDecorator, self).init()
-
         # Support legacy 'libraries=' attribute for the decorator.
         self.attributes["packages"] = {
             **self.attributes["libraries"],
             **self.attributes["packages"],
         }
-        del self.attributes["libraries"]
+        # Keep because otherwise make_decorator_spec will fail
+        self.attributes["libraries"] = {}
         if self.attributes["python"]:
             self.attributes["python"] = str(self.attributes["python"])
 

metaflow/plugins/pypi/conda_environment.py

@@ -17,6 +17,7 @@ from metaflow.debug import debug
 from metaflow.exception import MetaflowException
 from metaflow.metaflow_config import get_pinned_conda_libs
 from metaflow.metaflow_environment import MetaflowEnvironment
+from metaflow.packaging_sys import ContentType
 
 from . import MAGIC_FILE, _datastore_packageroot
 from .utils import conda_platform
@@ -35,6 +36,7 @@ class CondaEnvironment(MetaflowEnvironment):
     _force_rebuild = False
 
     def __init__(self, flow):
+        super().__init__(flow)
         self.flow = flow
 
     def set_local_root(self, local_root):
@@ -335,7 +337,7 @@ class CondaEnvironment(MetaflowEnvironment):
                     environment[decorator.name] = {
                         k: copy.deepcopy(decorator.attributes[k])
                         for k in decorator.attributes
-                        if k != "disabled"
+                        if k not in ("disabled", "libraries")
                     }
                 else:
                     return {}
@@ -477,7 +479,9 @@ class CondaEnvironment(MetaflowEnvironment):
         files = []
         manifest = self.get_environment_manifest_path()
         if os.path.exists(manifest):
-            files.append((manifest, os.path.basename(manifest)))
+            files.append(
+                (manifest, os.path.basename(manifest), ContentType.OTHER_CONTENT)
+            )
         return files
 
     def bootstrap_commands(self, step_name, datastore_type):

metaflow/plugins/pypi/pypi_decorator.py

@@ -24,12 +24,12 @@ class PyPIStepDecorator(StepDecorator):
     name = "pypi"
     defaults = {"packages": {}, "python": None, "disabled": None}  # wheels
 
-    def __init__(self, attributes=None, statically_defined=False):
+    def __init__(self, attributes=None, statically_defined=False, inserted_by=None):
         self._attributes_with_user_values = (
             set(attributes.keys()) if attributes is not None else set()
         )
 
-        super().__init__(attributes, statically_defined)
+        super().__init__(attributes, statically_defined, inserted_by)
 
     def step_init(self, flow, graph, step, decos, environment, flow_datastore, logger):
         # The init_environment hook for Environment creates the relevant virtual
@@ -128,12 +128,12 @@ class PyPIFlowDecorator(FlowDecorator):
     name = "pypi_base"
     defaults = {"packages": {}, "python": None, "disabled": None}
 
-    def __init__(self, attributes=None, statically_defined=False):
+    def __init__(self, attributes=None, statically_defined=False, inserted_by=None):
         self._attributes_with_user_values = (
             set(attributes.keys()) if attributes is not None else set()
         )
 
-        super().__init__(attributes, statically_defined)
+        super().__init__(attributes, statically_defined, inserted_by)
 
     def flow_init(
         self, flow, graph, environment, flow_datastore, metadata, logger, echo, options

metaflow/plugins/secrets/__init__.py

@@ -9,3 +9,6 @@ class SecretsProvider(abc.ABC):
     def get_secret_as_dict(self, secret_id, options={}, role=None) -> Dict[str, str]:
         """Retrieve the secret from secrets backend, and return a dictionary of
         environment variables."""
+
+
+from .secrets_func import get_secret

metaflow/plugins/secrets/secrets_decorator.py

@@ -1,183 +1,17 @@
 import os
-import re
 
 from metaflow.exception import MetaflowException
 from metaflow.decorators import StepDecorator
 from metaflow.metaflow_config import DEFAULT_SECRETS_ROLE
+from metaflow.plugins.secrets.secrets_spec import SecretSpec
+from metaflow.plugins.secrets.utils import (
+    get_secrets_backend_provider,
+    validate_env_vars,
+    validate_env_vars_across_secrets,
+    validate_env_vars_vs_existing_env,
+)
 from metaflow.unbounded_foreach import UBF_TASK
 
-from typing import Any, Dict, List, Union
-
-DISALLOWED_SECRETS_ENV_VAR_PREFIXES = ["METAFLOW_"]
-
-
-def get_default_secrets_backend_type():
-    from metaflow.metaflow_config import DEFAULT_SECRETS_BACKEND_TYPE
-
-    if DEFAULT_SECRETS_BACKEND_TYPE is None:
-        raise MetaflowException(
-            "No default secrets backend type configured, but needed by @secrets. "
-            "Set METAFLOW_DEFAULT_SECRETS_BACKEND_TYPE."
-        )
-    return DEFAULT_SECRETS_BACKEND_TYPE
-
-
-class SecretSpec:
-    def __init__(self, secrets_backend_type, secret_id, options={}, role=None):
-        self._secrets_backend_type = secrets_backend_type
-        self._secret_id = secret_id
-        self._options = options
-        self._role = role
-
-    @property
-    def secrets_backend_type(self):
-        return self._secrets_backend_type
-
-    @property
-    def secret_id(self):
-        return self._secret_id
-
-    @property
-    def options(self):
-        return self._options
-
-    @property
-    def role(self):
-        return self._role
-
-    def to_json(self):
-        """Mainly used for testing... not the same as the input dict in secret_spec_from_dict()!"""
-        return {
-            "secrets_backend_type": self.secrets_backend_type,
-            "secret_id": self.secret_id,
-            "options": self.options,
-            "role": self.role,
-        }
-
-    def __str__(self):
-        return "%s (%s)" % (self._secret_id, self._secrets_backend_type)
-
-    @staticmethod
-    def secret_spec_from_str(secret_spec_str, role):
-        # "." may be used in secret_id one day (provider specific). HOWEVER, it provides the best UX for
-        # non-conflicting cases (i.e. for secret ids that don't contain "."). This is true for all AWS
-        # Secrets Manager secrets.
-        #
-        # So we skew heavily optimize for best upfront UX for the present (1/2023).
-        #
-        # If/when a certain secret backend supports "." secret names, we can figure out a solution at that time.
-        # At a minimum, dictionary style secret spec may be used with no code changes (see secret_spec_from_dict()).
-        # Other options could be:
-        #  - accept and document that "." secret_ids don't work in Metaflow (across all possible providers)
-        #  - add a Metaflow config variable that specifies the separator (default ".")
-        #  - smarter spec parsing, that errors on secrets that look ambiguous. "aws-secrets-manager.XYZ" could mean:
-        #    + secret_id "XYZ" in aws-secrets-manager backend, OR
-        #    + secret_id "aws-secrets-manager.XYZ" default backend (if it is defined).
-        #    + in this case, user can simply set "azure-key-vault.aws-secrets-manager.XYZ" instead!
-        parts = secret_spec_str.split(".", maxsplit=1)
-        if len(parts) == 1:
-            secrets_backend_type = get_default_secrets_backend_type()
-            secret_id = parts[0]
-        else:
-            secrets_backend_type = parts[0]
-            secret_id = parts[1]
-        return SecretSpec(
-            secrets_backend_type, secret_id=secret_id, options={}, role=role
-        )
-
-    @staticmethod
-    def secret_spec_from_dict(secret_spec_dict, role):
-        if "type" not in secret_spec_dict:
-            secrets_backend_type = get_default_secrets_backend_type()
-        else:
-            secrets_backend_type = secret_spec_dict["type"]
-            if not isinstance(secrets_backend_type, str):
-                raise MetaflowException(
-                    "Bad @secrets specification - 'type' must be a string - found %s"
-                    % type(secrets_backend_type)
-                )
-        secret_id = secret_spec_dict.get("id")
-        if not isinstance(secret_id, str):
-            raise MetaflowException(
-                "Bad @secrets specification - 'id' must be a string - found %s"
-                % type(secret_id)
-            )
-        options = secret_spec_dict.get("options", {})
-        if not isinstance(options, dict):
-            raise MetaflowException(
-                "Bad @secrets specification - 'option' must be a dict - found %s"
-                % type(options)
-            )
-        role_for_source = secret_spec_dict.get("role", None)
-        if role_for_source is not None:
-            if not isinstance(role_for_source, str):
-                raise MetaflowException(
-                    "Bad @secrets specification - 'role' must be a str - found %s"
-                    % type(role_for_source)
-                )
-            role = role_for_source
-        return SecretSpec(
-            secrets_backend_type, secret_id=secret_id, options=options, role=role
-        )
-
-
-def validate_env_vars_across_secrets(all_secrets_env_vars):
-    vars_injected_by = {}
-    for secret_spec, env_vars in all_secrets_env_vars:
-        for k in env_vars:
-            if k in vars_injected_by:
-                raise MetaflowException(
-                    "Secret '%s' will inject '%s' as env var, and it is also added by '%s'"
-                    % (secret_spec, k, vars_injected_by[k])
-                )
-            vars_injected_by[k] = secret_spec
-
-
-def validate_env_vars_vs_existing_env(all_secrets_env_vars):
-    for secret_spec, env_vars in all_secrets_env_vars:
-        for k in env_vars:
-            if k in os.environ:
-                raise MetaflowException(
-                    "Secret '%s' will inject '%s' as env var, but it already exists in env"
-                    % (secret_spec, k)
-                )
-
-
-def validate_env_vars(env_vars):
-    for k, v in env_vars.items():
-        if not isinstance(k, str):
-            raise MetaflowException("Found non string key %s (%s)" % (str(k), type(k)))
-        if not isinstance(v, str):
-            raise MetaflowException(
-                "Found non string value %s (%s)" % (str(v), type(v))
-            )
-        if not re.fullmatch("[a-zA-Z_][a-zA-Z0-9_]*", k):
-            raise MetaflowException("Found invalid env var name '%s'." % k)
-        for disallowed_prefix in DISALLOWED_SECRETS_ENV_VAR_PREFIXES:
-            if k.startswith(disallowed_prefix):
-                raise MetaflowException(
-                    "Found disallowed env var name '%s' (starts with '%s')."
-                    % (k, disallowed_prefix)
-                )
-
-
-def get_secrets_backend_provider(secrets_backend_type):
-    from metaflow.plugins import SECRETS_PROVIDERS
-
-    try:
-        provider_cls = [
-            pc for pc in SECRETS_PROVIDERS if pc.TYPE == secrets_backend_type
-        ][0]
-        return provider_cls()
-    except IndexError:
-        raise MetaflowException(
-            "Unknown secrets backend type %s (available types: %s)"
-            % (
-                secrets_backend_type,
-                ", ".join(pc.TYPE for pc in SECRETS_PROVIDERS if pc.TYPE != "inline"),
-            )
-        )
-
 
 class SecretsDecorator(StepDecorator):
     """
@@ -188,6 +22,8 @@ class SecretsDecorator(StepDecorator):
     ----------
     sources : List[Union[str, Dict[str, Any]]], default: []
         List of secret specs, defining how the secrets are to be retrieved
+    role : str, optional, default: None
+        Role to use for fetching secrets
     """
 
     name = "secrets"

metaflow/plugins/secrets/secrets_func.py

@@ -0,0 +1,49 @@
+from typing import Any, Dict, Optional, Union
+
+from metaflow.metaflow_config import DEFAULT_SECRETS_ROLE
+from metaflow.exception import MetaflowException
+from metaflow.plugins.secrets.secrets_spec import SecretSpec
+from metaflow.plugins.secrets.utils import get_secrets_backend_provider
+
+
+def get_secret(
+    source: Union[str, Dict[str, Any]], role: Optional[str] = None
+) -> Dict[str, str]:
+    """
+    Get secret from source
+
+    Parameters
+    ----------
+    source : Union[str, Dict[str, Any]]
+        Secret spec, defining how the secret is to be retrieved
+    role : str, optional
+        Role to use for fetching secrets
+    """
+    if role is None:
+        role = DEFAULT_SECRETS_ROLE
+
+    secret_spec = None
+
+    if isinstance(source, str):
+        secret_spec = SecretSpec.secret_spec_from_str(source, role=role)
+    elif isinstance(source, dict):
+        secret_spec = SecretSpec.secret_spec_from_dict(source, role=role)
+    else:
+        raise MetaflowException(
+            "get_secrets sources items must be either a string or a dict"
+        )
+
+    secrets_backend_provider = get_secrets_backend_provider(
+        secret_spec.secrets_backend_type
+    )
+    try:
+        dict_for_secret = secrets_backend_provider.get_secret_as_dict(
+            secret_spec.secret_id,
+            options=secret_spec.options,
+            role=secret_spec.role,
+        )
+        return dict_for_secret
+    except Exception as e:
+        raise MetaflowException(
+            "Failed to retrieve secret '%s': %s" % (secret_spec.secret_id, e)
+        )

metaflow/plugins/secrets/secrets_spec.py

@@ -0,0 +1,101 @@
+from metaflow.exception import MetaflowException
+from metaflow.plugins.secrets.utils import get_default_secrets_backend_type
+
+
+class SecretSpec:
+    def __init__(self, secrets_backend_type, secret_id, options={}, role=None):
+        self._secrets_backend_type = secrets_backend_type
+        self._secret_id = secret_id
+        self._options = options
+        self._role = role
+
+    @property
+    def secrets_backend_type(self):
+        return self._secrets_backend_type
+
+    @property
+    def secret_id(self):
+        return self._secret_id
+
+    @property
+    def options(self):
+        return self._options
+
+    @property
+    def role(self):
+        return self._role
+
+    def to_json(self):
+        """Mainly used for testing... not the same as the input dict in secret_spec_from_dict()!"""
+        return {
+            "secrets_backend_type": self.secrets_backend_type,
+            "secret_id": self.secret_id,
+            "options": self.options,
+            "role": self.role,
+        }
+
+    def __str__(self):
+        return "%s (%s)" % (self._secret_id, self._secrets_backend_type)
+
+    @staticmethod
+    def secret_spec_from_str(secret_spec_str, role):
+        # "." may be used in secret_id one day (provider specific). HOWEVER, it provides the best UX for
+        # non-conflicting cases (i.e. for secret ids that don't contain "."). This is true for all AWS
+        # Secrets Manager secrets.
+        #
+        # So we skew heavily optimize for best upfront UX for the present (1/2023).
+        #
+        # If/when a certain secret backend supports "." secret names, we can figure out a solution at that time.
+        # At a minimum, dictionary style secret spec may be used with no code changes (see secret_spec_from_dict()).
+        # Other options could be:
+        #  - accept and document that "." secret_ids don't work in Metaflow (across all possible providers)
+        #  - add a Metaflow config variable that specifies the separator (default ".")
+        #  - smarter spec parsing, that errors on secrets that look ambiguous. "aws-secrets-manager.XYZ" could mean:
+        #    + secret_id "XYZ" in aws-secrets-manager backend, OR
+        #    + secret_id "aws-secrets-manager.XYZ" default backend (if it is defined).
+        #    + in this case, user can simply set "azure-key-vault.aws-secrets-manager.XYZ" instead!
+        parts = secret_spec_str.split(".", maxsplit=1)
+        if len(parts) == 1:
+            secrets_backend_type = get_default_secrets_backend_type()
+            secret_id = parts[0]
+        else:
+            secrets_backend_type = parts[0]
+            secret_id = parts[1]
+        return SecretSpec(
+            secrets_backend_type, secret_id=secret_id, options={}, role=role
+        )
+
+    @staticmethod
+    def secret_spec_from_dict(secret_spec_dict, role):
+        if "type" not in secret_spec_dict:
+            secrets_backend_type = get_default_secrets_backend_type()
+        else:
+            secrets_backend_type = secret_spec_dict["type"]
+            if not isinstance(secrets_backend_type, str):
+                raise MetaflowException(
+                    "Bad @secrets specification - 'type' must be a string - found %s"
+                    % type(secrets_backend_type)
+                )
+        secret_id = secret_spec_dict.get("id")
+        if not isinstance(secret_id, str):
+            raise MetaflowException(
+                "Bad @secrets specification - 'id' must be a string - found %s"
+                % type(secret_id)
+            )
+        options = secret_spec_dict.get("options", {})
+        if not isinstance(options, dict):
+            raise MetaflowException(
+                "Bad @secrets specification - 'option' must be a dict - found %s"
+                % type(options)
+            )
+        role_for_source = secret_spec_dict.get("role", None)
+        if role_for_source is not None:
+            if not isinstance(role_for_source, str):
+                raise MetaflowException(
+                    "Bad @secrets specification - 'role' must be a str - found %s"
+                    % type(role_for_source)
+                )
+            role = role_for_source
+        return SecretSpec(
+            secrets_backend_type, secret_id=secret_id, options=options, role=role
+        )