ob-metaflow 2.10.7.4__py2.py3-none-any.whl → 2.10.9.1__py2.py3-none-any.whl

metaflow/plugins/cards/exception.py

@@ -116,7 +116,7 @@ class UnresolvableDatastoreException(MetaflowException):
         super(UnresolvableDatastoreException, self).__init__(msg)
 
 
-class IncorrectArguementException(MetaflowException):
+class IncorrectArgumentException(MetaflowException):
     headline = (
         "`get_cards` function requires a `Task` object or pathspec as an argument"
     )
@@ -138,3 +138,22 @@ class IncorrectPathspecException(MetaflowException):
             % pthspec
         )
         super().__init__(msg=msg, lineno=None)
+
+
+class ComponentOverwriteNotSupportedException(MetaflowException):
+    headline = "Component overwrite is not supported"
+
+    def __init__(self, component_id, card_id, card_type):
+        id_str = ""
+        if card_id is not None:
+            id_str = "id='%s'" % card_id
+        msg = (
+            "Card component overwrite is not supported. "
+            "Component with id %s already exists in the @card(type='%s', %s). \n"
+            "Instead of calling `current.card.components[ID] = MyComponent`. "
+            "You can overwrite the entire component Array by calling "
+            "`current.card.components = [MyComponent]`"
+        ) % (component_id, card_type, id_str)
+        super().__init__(
+            msg=msg,
+        )

metaflow/plugins/datastores/azure_storage.py

@@ -250,9 +250,8 @@ class _AzureRootClient(object):
 class AzureStorage(DataStoreStorage):
     TYPE = "azure"
 
+    @check_azure_deps
     def __init__(self, root=None):
-        # cannot decorate __init__... invoke it with dummy decoratee
-        check_azure_deps(lambda: 0)
         super(AzureStorage, self).__init__(root)
         self._tmproot = ARTIFACT_LOCALROOT
         self._default_scope_token = None

metaflow/plugins/datastores/gs_storage.py

@@ -145,9 +145,8 @@ class _GSRootClient(object):
 class GSStorage(DataStoreStorage):
     TYPE = "gs"
 
+    @check_gs_deps
     def __init__(self, root=None):
-        # cannot decorate __init__... invoke it with dummy decoratee
-        check_gs_deps(lambda: 0)
         super(GSStorage, self).__init__(root)
         self._tmproot = ARTIFACT_LOCALROOT
         self._root_client = None

metaflow/plugins/datastores/s3_storage.py

@@ -2,7 +2,7 @@ import os
 
 from itertools import starmap
 
-from metaflow.plugins.datatools.s3.s3 import S3, S3Client, S3PutObject
+from metaflow.plugins.datatools.s3.s3 import S3, S3Client, S3PutObject, check_s3_deps
 from metaflow.metaflow_config import DATASTORE_SYSROOT_S3, ARTIFACT_LOCALROOT
 from metaflow.datastore.datastore_storage import CloseAfterUse, DataStoreStorage
 
@@ -18,6 +18,7 @@ except:
 class S3Storage(DataStoreStorage):
     TYPE = "s3"
 
+    @check_s3_deps
     def __init__(self, root=None):
         super(S3Storage, self).__init__(root)
         self.s3_client = S3Client()

metaflow/plugins/datatools/s3/s3.py

@@ -51,15 +51,25 @@ from .s3util import (
 if TYPE_CHECKING:
     from metaflow.client import Run
 
-try:
-    import boto3
-    from boto3.s3.transfer import TransferConfig
 
-    DOWNLOAD_FILE_THRESHOLD = 2 * TransferConfig().multipart_threshold
-    DOWNLOAD_MAX_CHUNK = 2 * 1024 * 1024 * 1024 - 1
-    boto_found = True
-except:
-    boto_found = False
+def _check_and_init_s3_deps():
+    try:
+        import boto3
+        from boto3.s3.transfer import TransferConfig
+    except (ImportError, ModuleNotFoundError):
+        raise MetaflowException("You need to install 'boto3' in order to use S3.")
+
+
+def check_s3_deps(func):
+    """The decorated function checks S3 dependencies (as needed for AWS S3 storage backend).
+    This includes boto3.
+    """
+
+    def _inner_func(*args, **kwargs):
+        _check_and_init_s3_deps()
+        return func(*args, **kwargs)
+
+    return _inner_func
 
 
 TEST_INJECT_RETRYABLE_FAILURES = int(
@@ -498,6 +508,7 @@ class S3(object):
     def get_root_from_config(cls, echo, create_on_absent=True):
         return DATATOOLS_S3ROOT
 
+    @check_s3_deps
     def __init__(
         self,
         tmproot: str = TEMPDIR,
@@ -508,9 +519,6 @@ class S3(object):
         encryption: Optional[str] = S3_SERVER_SIDE_ENCRYPTION,
         **kwargs
     ):
-        if not boto_found:
-            raise MetaflowException("You need to install 'boto3' in order to use S3.")
-
         if run:
             # 1. use a (current) run ID with optional customizations
             if DATATOOLS_S3ROOT is None:
@@ -875,6 +883,11 @@ class S3(object):
         `S3Object`
             An S3Object corresponding to the object requested.
         """
+        from boto3.s3.transfer import TransferConfig
+
+        DOWNLOAD_FILE_THRESHOLD = 2 * TransferConfig().multipart_threshold
+        DOWNLOAD_MAX_CHUNK = 2 * 1024 * 1024 * 1024 - 1
+
         url, r = self._url_and_range(key)
         src = urlparse(url)
 

metaflow/plugins/env_escape/client.py

@@ -37,6 +37,7 @@ from .data_transferer import DataTransferer, ObjReference
 from .exception_transferer import load_exception
 from .override_decorators import LocalAttrOverride, LocalException, LocalOverride
 from .stub import create_class
+from .utils import get_canonical_name
 
 BIND_TIMEOUT = 0.1
 BIND_RETRY = 0
@@ -336,7 +337,7 @@ class Client(object):
     def get_local_class(self, name, obj_id=None):
         # Gets (and creates if needed), the class mapping to the remote
         # class of name 'name'.
-        name = self._get_canonical_name(name)
+        name = get_canonical_name(name, self._aliases)
         if name == "function":
             # Special handling of pickled functions. We create a new class that
             # simply has a __call__ method that will forward things back to
@@ -398,17 +399,6 @@ class Client(object):
             local_instance = local_class(self, remote_class_name, obj_id)
         return local_instance
 
-    def _get_canonical_name(self, name):
-        # We look at the aliases looking for the most specific match first
-        base_name = self._aliases.get(name)
-        if base_name is not None:
-            return base_name
-        for idx in reversed([pos for pos, char in enumerate(name) if char == "."]):
-            base_name = self._aliases.get(name[:idx])
-            if base_name is not None:
-                return ".".join([base_name, name[idx + 1 :]])
-        return name
-
     def _communicate(self, msg):
         if os.getpid() != self._active_pid:
             raise RuntimeError(

metaflow/plugins/env_escape/client_modules.py

@@ -8,6 +8,7 @@ import sys
 from .consts import OP_CALLFUNC, OP_GETVAL, OP_SETVAL
 from .client import Client
 from .override_decorators import LocalException
+from .utils import get_canonical_name
 
 
 def _clean_client(client):
@@ -23,6 +24,7 @@ class _WrappedModule(object):
             r"^%s\.([a-zA-Z_][a-zA-Z0-9_]*)$" % prefix.replace(".", r"\.")  # noqa W605
         )
         self._exports = {}
+        self._aliases = exports["aliases"]
         for k in ("classes", "functions", "values"):
             result = []
             for item in exports[k]:
@@ -43,6 +45,11 @@ class _WrappedModule(object):
             return self._prefix
         if name in ("__file__", "__path__"):
             return self._client.name
+
+        # Make the name canonical because the prefix is also canonical.
+        name = get_canonical_name(self._prefix + "." + name, self._aliases)[
+            len(self._prefix) + 1 :
+        ]
         if name in self._exports["classes"]:
             # We load classes lazily
             return self._client.get_local_class("%s.%s" % (self._prefix, name))
@@ -87,6 +94,7 @@ class _WrappedModule(object):
             "_client",
             "_exports",
             "_exception_classes",
+            "_aliases",
         ):
             object.__setattr__(self, name, value)
             return
@@ -95,6 +103,11 @@ class _WrappedModule(object):
             # module when loading
             object.__setattr__(self, name, value)
             return
+
+        # Make the name canonical because the prefix is also canonical.
+        name = get_canonical_name(self._prefix + "." + name, self._aliases)[
+            len(self._prefix) + 1 :
+        ]
         if name in self._exports["values"]:
             self._client.stub_request(
                 None, OP_SETVAL, "%s.%s" % (self._prefix, name), value
@@ -126,7 +139,7 @@ class ModuleImporter(object):
 
     def find_module(self, fullname, path=None):
         if self._handled_modules is not None:
-            if fullname in self._handled_modules:
+            if get_canonical_name(fullname, self._aliases) in self._handled_modules:
                 return self
             return None
         if any([fullname.startswith(prefix) for prefix in self._module_prefixes]):
@@ -224,24 +237,15 @@ class ModuleImporter(object):
                 self._handled_modules[prefix] = _WrappedModule(
                     self, prefix, exports, formed_exception_classes, self._client
                 )
-        fullname = self._get_canonical_name(fullname)
-        module = self._handled_modules.get(fullname)
+        canonical_fullname = get_canonical_name(fullname, self._aliases)
+        # Modules are created canonically but we need to return something for any
+        # of the aliases.
+        module = self._handled_modules.get(canonical_fullname)
         if module is None:
             raise ImportError
         sys.modules[fullname] = module
         return module
 
-    def _get_canonical_name(self, name):
-        # We look at the aliases looking for the most specific match first
-        base_name = self._aliases.get(name)
-        if base_name is not None:
-            return base_name
-        for idx in reversed([pos for pos, char in enumerate(name) if char == "."]):
-            base_name = self._aliases.get(name[:idx])
-            if base_name is not None:
-                return ".".join([base_name, name[idx + 1 :]])
-        return name
-
 
 def create_modules(python_executable, pythonpath, max_pickle_version, path, prefixes):
     # This is an extra verification to make sure we are not trying to use the

metaflow/plugins/env_escape/server.py

@@ -53,7 +53,7 @@ from .override_decorators import (
     RemoteExceptionSerializer,
 )
 from .exception_transferer import dump_exception
-from .utils import get_methods
+from .utils import get_methods, get_canonical_name
 
 BIND_TIMEOUT = 0.1
 BIND_RETRY = 1
@@ -61,7 +61,6 @@ BIND_RETRY = 1
 
 class Server(object):
     def __init__(self, config_dir, max_pickle_version):
-
         self._max_pickle_version = data_transferer.defaultProtocol = max_pickle_version
         try:
             mappings = importlib.import_module(".server_mappings", package=config_dir)
@@ -108,6 +107,11 @@ class Server(object):
             for alias in aliases:
                 a = self._aliases.setdefault(alias, base_name)
                 if a != base_name:
+                    # Technically we could have a that aliases b and b that aliases c
+                    # and then a that aliases c. This would error out in that case
+                    # even though it is valid. It is easy for the user to get around
+                    # this by listing aliases in the same order so we don't support
+                    # it for now.
                     raise ValueError(
                         "%s is an alias to both %s and %s" % (alias, base_name, a)
                     )
@@ -155,12 +159,13 @@ class Server(object):
         parent_to_child = {}
 
         for ex_name, ex_cls in self._known_exceptions.items():
+            ex_name_canonical = get_canonical_name(ex_name, self._aliases)
             parents = []
             for base in ex_cls.__mro__[1:]:
                 if base is object:
                     raise ValueError(
-                        "Exported exceptions not rooted in a builtin exception are not supported: %s"
-                        % ex_name
+                        "Exported exceptions not rooted in a builtin exception "
+                        "are not supported: %s." % ex_name
                     )
                 if base.__module__ == "builtins":
                     # We found our base exception
@@ -168,17 +173,19 @@ class Server(object):
                     break
                 else:
                     fqn = ".".join([base.__module__, base.__name__])
-                    if fqn in self._known_exceptions:
-                        parents.append(fqn)
-                        children = parent_to_child.setdefault(fqn, [])
-                        children.append(ex_name)
+                    canonical_fqn = get_canonical_name(fqn, self._aliases)
+                    if canonical_fqn in self._known_exceptions:
+                        parents.append(canonical_fqn)
+                        children = parent_to_child.setdefault(canonical_fqn, [])
+                        children.append(ex_name_canonical)
                     else:
                         raise ValueError(
                             "Exported exception %s has non exported and non builtin parent "
-                            "exception: %s" % (ex_name, fqn)
+                            "exception: %s. Known exceptions: %s"
+                            % (ex_name, fqn, str(self._known_exceptions))
                         )
-            name_to_parent_count[ex_name] = len(parents) - 1
-            name_to_parents[ex_name] = parents
+            name_to_parent_count[ex_name_canonical] = len(parents) - 1
+            name_to_parents[ex_name_canonical] = parents
 
         # We now form the exceptions and put them in self._known_exceptions in
         # the proper order (topologically)

metaflow/plugins/env_escape/utils.py

@@ -20,3 +20,15 @@ def get_methods(class_object):
         elif isinstance(attribute, classmethod):
             all_methods["___c___%s" % name] = inspect.getdoc(attribute)
     return all_methods
+
+
+def get_canonical_name(name, aliases):
+    # We look at the aliases looking for the most specific match first
+    base_name = aliases.get(name)
+    if base_name is not None:
+        return base_name
+    for idx in reversed([pos for pos, char in enumerate(name) if char == "."]):
+        base_name = aliases.get(name[:idx])
+        if base_name is not None:
+            return ".".join([base_name, name[idx + 1 :]])
+    return name

metaflow/plugins/gcp/gs_exceptions.py

@@ -2,4 +2,4 @@ from metaflow.exception import MetaflowException
 
 
 class MetaflowGSPackageError(MetaflowException):
-    headline = "Missing required packages google-cloud-storage google-auth"
+    headline = "Missing required packages 'google-cloud-storage' and 'google-auth'"

metaflow/plugins/gcp/gs_utils.py

@@ -34,7 +34,7 @@ def _check_and_init_gs_deps():
 
 
 def check_gs_deps(func):
-    """The decorated function checks GS dependencies (as needed for Azure storage backend). This includes
+    """The decorated function checks GS dependencies (as needed for Google Cloud storage backend). This includes
     various GCP SDK packages, as well as a Python version of >=3.7
     """
 

metaflow/plugins/pypi/conda_environment.py

@@ -20,7 +20,7 @@ from metaflow.metaflow_environment import MetaflowEnvironment
 from metaflow.metaflow_profile import profile
 
 from . import MAGIC_FILE, _datastore_packageroot
-from .utils import conda_platform
+from .utils import conda_platform, generate_cache_path, parse_filename_from_url
 
 
 class CondaEnvironmentException(MetaflowException):
@@ -107,7 +107,7 @@ class CondaEnvironment(MetaflowEnvironment):
             local_packages = {
                 url: {
                     # Path to package in datastore.
-                    "path": urlparse(url).netloc + urlparse(url).path,
+                    "path": generate_cache_path(url, local_path),
                     # Path to package on local disk.
                     "local_path": local_path,
                 }
@@ -122,9 +122,8 @@ class CondaEnvironment(MetaflowEnvironment):
                         # Cache only those packages that manifest is unaware of
                         local_packages.pop(package["url"], None)
                     else:
-                        package["path"] = (
-                            urlparse(package["url"]).netloc
-                            + urlparse(package["url"]).path
+                        package["path"] = generate_cache_path(
+                            package["url"], parse_filename_from_url(package["url"])
                         )
                         dirty.add(id_)
 
@@ -187,7 +186,7 @@ class CondaEnvironment(MetaflowEnvironment):
             if decorator.name in ["conda", "pypi"]:
                 # handle @conda/@pypi(disabled=True)
                 disabled = decorator.attributes["disabled"]
-                return disabled or str(disabled).lower() != "false"
+                return str(disabled).lower() == "true"
         return False
 
     @functools.lru_cache(maxsize=None)

metaflow/plugins/pypi/pip.py

@@ -9,7 +9,7 @@ from metaflow.exception import MetaflowException
 from metaflow.util import which
 
 from .micromamba import Micromamba
-from .utils import pip_tags
+from .utils import parse_filename_from_url, pip_tags
 
 
 class PipException(MetaflowException):
@@ -118,7 +118,7 @@ class Pip(object):
         for package in packages:
             cmd.append("{url}".format(**package))
             metadata["{url}".format(**package)] = "{prefix}/.pip/wheels/{wheel}".format(
-                prefix=prefix, wheel=package["url"].split("/")[-1]
+                prefix=prefix, wheel=parse_filename_from_url(package["url"])
             )
         self._call(prefix, cmd)
         # write the url to wheel mappings in a magic location

metaflow/plugins/pypi/utils.py

@@ -1,3 +1,4 @@
+import os
 import platform
 import sys
 
@@ -13,6 +14,7 @@ else:
     from metaflow._vendor.packaging import tags
 
 from metaflow.exception import MetaflowException
+from urllib.parse import unquote, urlparse
 
 
 def conda_platform():
@@ -73,3 +75,16 @@ def pip_tags(python_version, mamba_platform):
     supported.extend(tags.cpython_tags(py_version, abis, platforms))
     supported.extend(tags.compatible_tags(py_version, interpreter, platforms))
     return supported
+
+
+def parse_filename_from_url(url):
+    # Separate method as it might require additional checks for the parsing.
+    filename = url.split("/")[-1]
+    return unquote(filename)
+
+
+def generate_cache_path(url, local_path):
+    base, _ = os.path.split(urlparse(url).path)
+    _, localfile = os.path.split(local_path)
+    unquoted_base = unquote(base)
+    return urlparse(url).netloc + os.path.join(unquoted_base, localfile)

metaflow/task.py

@@ -576,6 +576,7 @@ class MetaflowTask(object):
                     inputs,
                 )
 
+            for deco in decorators:
                 # decorators can actually decorate the step function,
                 # or they can replace it altogether. This functionality
                 # is used e.g. by catch_decorator which switches to a

metaflow/version.py

@@ -1 +1 @@
-metaflow_version = "2.10.7.4"
+metaflow_version = "2.10.9.1"

{ob_metaflow-2.10.7.4.dist-info → ob_metaflow-2.10.9.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ob-metaflow
-Version: 2.10.7.4
+Version: 2.10.9.1
 Summary: Metaflow: More Data Science, Less Engineering
 Author: Netflix, Outerbounds & the Metaflow Community
 Author-email: help@outerbounds.co