metaflow 2.15.14__py2.py3-none-any.whl → 2.15.16__py2.py3-none-any.whl

metaflow/__init__.py

@@ -128,8 +128,8 @@ _import_tl_plugins(globals())
 # this auto-generates decorator functions from Decorator objects
 # in the top-level metaflow namespace
 _import_plugin_decorators(globals())
-# Setting card import for only python 3.4
-if sys.version_info[0] >= 3 and sys.version_info[1] >= 4:
+# Setting card import for only python 3.6
+if sys.version_info[0] >= 3 and sys.version_info[1] >= 6:
     from . import cards
 
 # Client

metaflow/_vendor/click/core.py

@@ -719,7 +719,7 @@ class BaseCommand(object):
         prog_name=None,
         complete_var=None,
         standalone_mode=True,
-        **extra
+        **extra,
     ):
         """This is the way to invoke a script with all the bells and
         whistles as a command line application.  This will always terminate
@@ -1101,7 +1101,7 @@ class MultiCommand(Command):
         subcommand_metavar=None,
         chain=False,
         result_callback=None,
-        **attrs
+        **attrs,
     ):
         Command.__init__(self, name, **attrs)
         if no_args_is_help is None:
@@ -1463,6 +1463,7 @@ class Parameter(object):
         parameter. The old callback format will still work, but it will
         raise a warning to give you a chance to migrate the code easier.
     """
+
     param_type_name = "parameter"
 
     def __init__(
@@ -1708,7 +1709,7 @@ class Option(Parameter):
         hidden=False,
         show_choices=True,
         show_envvar=False,
-        **attrs
+        **attrs,
     ):
         default_is_missing = attrs.get("default", _missing) is _missing
         Parameter.__init__(self, param_decls, type=type, **attrs)

metaflow/cmd/develop/stub_generator.py

@@ -488,9 +488,6 @@ class StubGenerator:
                 self._imports.add(name)
 
         def _add_to_typing_check(name, is_module=False):
-            # if name != self._current_module_name:
-            #    self._typing_imports.add(name)
-            #
             if name == "None":
                 return
             if is_module:
@@ -504,6 +501,24 @@ class StubGenerator:
                     # the current file
                     self._typing_imports.add(splits[0])
 
+        def _format_qualified_class_name(cls: type) -> str:
+            """Helper to format a class with its qualified module name"""
+            # Special case for NoneType - return None
+            if cls.__name__ == "NoneType":
+                return "None"
+
+            module = inspect.getmodule(cls)
+            if (
+                module
+                and module.__name__ != "builtins"
+                and module.__name__ != "__main__"
+            ):
+                module_name = self._get_module_name_alias(module.__name__)
+                _add_to_typing_check(module_name, is_module=True)
+                return f"{module_name}.{cls.__name__}"
+            else:
+                return cls.__name__
+
         if isinstance(element, str):
             # Special case for self referential things (particularly in a class)
             if element == self._current_name:
@@ -557,19 +572,15 @@ class StubGenerator:
                 return element.__name__
         elif isinstance(element, type(Ellipsis)):
             return "..."
-        # elif (
-        #    isinstance(element, typing._GenericAlias)
-        #    and hasattr(element, "_name")
-        #    and element._name in ("List", "Tuple", "Dict", "Set")
-        # ):
-        #    # 3.7 has these as _GenericAlias but they don't behave like the ones in 3.10
-        #    _add_to_import("typing")
-        #    return str(element)
         elif isinstance(element, typing._GenericAlias):
             # We need to check things recursively in __args__ if it exists
             args_str = []
             for arg in getattr(element, "__args__", []):
-                args_str.append(self._get_element_name_with_module(arg))
+                # Special handling for class objects in type arguments
+                if isinstance(arg, type):
+                    args_str.append(_format_qualified_class_name(arg))
+                else:
+                    args_str.append(self._get_element_name_with_module(arg))
 
             _add_to_import("typing")
             if element._name:
@@ -584,12 +595,15 @@ class StubGenerator:
                         args_str = [call_args, args_str[-1]]
                 return "typing.%s[%s]" % (element._name, ", ".join(args_str))
             else:
-                return "%s[%s]" % (element.__origin__, ", ".join(args_str))
+                # Handle the case where we have a generic type without a _name
+                origin = element.__origin__
+                if isinstance(origin, type):
+                    origin_str = _format_qualified_class_name(origin)
+                else:
+                    origin_str = str(origin)
+                return "%s[%s]" % (origin_str, ", ".join(args_str))
         elif isinstance(element, ForwardRef):
             f_arg = self._get_module_name_alias(element.__forward_arg__)
-            # if f_arg in ("Run", "Task"):  # HACK -- forward references in current.py
-            #    _add_to_import("metaflow")
-            #    f_arg = "metaflow.%s" % f_arg
             _add_to_typing_check(f_arg)
             return '"%s"' % f_arg
         elif inspect.getmodule(element) == inspect.getmodule(typing):

metaflow/cmd/develop/stubs.py

@@ -12,25 +12,13 @@ from . import develop
 from .stub_generator import StubGenerator
 
 _py_ver = sys.version_info[:2]
-_metadata_package = None
 
-
-def _check_stubs_supported():
-    global _metadata_package
-    if _metadata_package is not None:
-        return _metadata_package
-    else:
-        if _py_ver >= (3, 4):
-            if _py_ver >= (3, 8):
-                from importlib import metadata
-            elif _py_ver >= (3, 7):
-                from metaflow._vendor.v3_7 import importlib_metadata as metadata
-            elif _py_ver >= (3, 6):
-                from metaflow._vendor.v3_6 import importlib_metadata as metadata
-            else:
-                from metaflow._vendor.v3_5 import importlib_metadata as metadata
-            _metadata_package = metadata
-        return _metadata_package
+if _py_ver >= (3, 8):
+    from importlib import metadata
+elif _py_ver >= (3, 7):
+    from metaflow._vendor.v3_7 import importlib_metadata as metadata
+else:
+    from metaflow._vendor.v3_6 import importlib_metadata as metadata
 
 
 @develop.group(short_help="Stubs management")
@@ -45,12 +33,6 @@ def stubs(ctx: Any):
     This CLI provides utilities to check and generate stubs for your current Metaflow
     installation.
     """
-    if _check_stubs_supported() is None:
-        raise click.UsageError(
-            "Building and installing stubs are not supported on Python %d.%d "
-            "(3.4 minimum required)" % _py_ver,
-            ctx=ctx,
-        )
 
 
 @stubs.command(short_help="Check validity of stubs")
@@ -187,7 +169,7 @@ setup(
     packages=find_namespace_packages(),
     package_data={{"metaflow-stubs": ["generated_for.txt", "py.typed", "**/*.pyi"]}},
     install_requires=["metaflow=={mf_version}"],
-    python_requires=">=3.5.2",
+    python_requires=">=3.6.1",
 )
                 """
             )
@@ -330,14 +312,14 @@ def get_packages_for_stubs() -> Tuple[List[Tuple[str, str]], List[str]]:
     # some reason it shows up multiple times.
     interesting_dists = [
         d
-        for d in _metadata_package.distributions()
+        for d in metadata.distributions()
         if any(
             [
                 p == "metaflow-stubs"
                 for p in (d.read_text("top_level.txt") or "").split()
             ]
         )
-        and isinstance(d, _metadata_package.PathDistribution)
+        and isinstance(d, metadata.PathDistribution)
     ]
 
     for dist in interesting_dists:

metaflow/datastore/task_datastore.py

@@ -118,7 +118,7 @@ class TaskDataStore(object):
         # The GZIP encodings are for backward compatibility
         self._encodings = {"pickle-v2", "gzip+pickle-v2"}
         ver = sys.version_info[0] * 10 + sys.version_info[1]
-        if ver >= 34:
+        if ver >= 36:
             self._encodings.add("pickle-v4")
             self._encodings.add("gzip+pickle-v4")
 
@@ -289,7 +289,7 @@ class TaskDataStore(object):
                     except (SystemError, OverflowError) as e:
                         raise DataException(
                             "Artifact *%s* is very large (over 2GB). "
-                            "You need to use Python 3.4 or newer if you want to "
+                            "You need to use Python 3.6 or newer if you want to "
                             "serialize large objects." % name
                         ) from e
                     except TypeError as e:
@@ -352,7 +352,7 @@ class TaskDataStore(object):
                 encode_type = "gzip+pickle-v2"
             if encode_type not in self._encodings:
                 raise DataException(
-                    "Python 3.4 or later is required to load artifact '%s'" % name
+                    "Python 3.6 or later is required to load artifact '%s'" % name
                 )
             else:
                 to_load[self._objects[name]].append(name)

metaflow/decorators.py

@@ -152,7 +152,7 @@ class Decorator(object):
         # Note that by design, later values override previous ones.
         self.attributes, new_user_attributes = unpack_delayed_evaluator(self.attributes)
         self._user_defined_attributes.update(new_user_attributes)
-        self.attributes = resolve_delayed_evaluator(self.attributes)
+        self.attributes = resolve_delayed_evaluator(self.attributes, to_dict=True)
 
         self._ran_init = True
 
@@ -638,7 +638,7 @@ StepFlag = NewType("StepFlag", bool)
 
 @overload
 def step(
-    f: Callable[[FlowSpecDerived], None]
+    f: Callable[[FlowSpecDerived], None],
 ) -> Callable[[FlowSpecDerived, StepFlag], None]: ...
 
 
@@ -649,7 +649,7 @@ def step(
 
 
 def step(
-    f: Union[Callable[[FlowSpecDerived], None], Callable[[FlowSpecDerived, Any], None]]
+    f: Union[Callable[[FlowSpecDerived], None], Callable[[FlowSpecDerived, Any], None]],
 ):
     """
     Marks a method in a FlowSpec as a Metaflow Step. Note that this

metaflow/extension_support/__init__.py

@@ -103,9 +103,6 @@ def load_module(module_name):
 
 def get_modules(extension_point):
     modules_to_load = []
-    if not _mfext_supported:
-        _ext_debug("Not supported for your Python version -- 3.4+ is needed")
-        return []
     if extension_point not in _extension_points:
         raise RuntimeError(
             "Metaflow extension point '%s' not supported" % extension_point
@@ -146,9 +143,6 @@ def dump_module_info(all_packages=None, pkgs_per_extension_point=None):
 
 
 def get_extensions_in_dir(d):
-    if not _mfext_supported:
-        _ext_debug("Not supported for your Python version -- 3.4+ is needed")
-        return None, None
     return _get_extension_packages(ignore_info_file=True, restrict_to_directories=[d])
 
 
@@ -312,21 +306,16 @@ def multiload_all(modules, extension_point, dst_globals):
 
 
 _py_ver = sys.version_info[:2]
-_mfext_supported = False
 _aliased_modules = []
 
-if _py_ver >= (3, 4):
-    import importlib.util
+import importlib.util
 
-    if _py_ver >= (3, 8):
-        from importlib import metadata
-    elif _py_ver >= (3, 7):
-        from metaflow._vendor.v3_7 import importlib_metadata as metadata
-    elif _py_ver >= (3, 6):
-        from metaflow._vendor.v3_6 import importlib_metadata as metadata
-    else:
-        from metaflow._vendor.v3_5 import importlib_metadata as metadata
-    _mfext_supported = True
+if _py_ver >= (3, 8):
+    from importlib import metadata
+elif _py_ver >= (3, 7):
+    from metaflow._vendor.v3_7 import importlib_metadata as metadata
+else:
+    from metaflow._vendor.v3_6 import importlib_metadata as metadata
 
 # Extension points are the directories that can be present in a EXT_PKG to
 # contribute to that extension point. For example, if you have
@@ -355,10 +344,6 @@ def _ext_debug(*args, **kwargs):
 
 
 def _get_extension_packages(ignore_info_file=False, restrict_to_directories=None):
-    if not _mfext_supported:
-        _ext_debug("Not supported for your Python version -- 3.4+ is needed")
-        return {}, {}
-
     # If we have an INFO file with the appropriate information (if running from a saved
     # code package for example), we use that directly
     # Pre-compute on _extension_points
@@ -381,12 +366,11 @@ def _get_extension_packages(ignore_info_file=False, restrict_to_directories=None
     try:
         extensions_module = importlib.import_module(EXT_PKG)
     except ImportError as e:
-        if _py_ver >= (3, 6):
-            # e.name is set to the name of the package that fails to load
-            # so don't error ONLY IF the error is importing this module (but do
-            # error if there is a transitive import error)
-            if not (isinstance(e, ModuleNotFoundError) and e.name == EXT_PKG):
-                raise
+        # e.name is set to the name of the package that fails to load
+        # so don't error ONLY IF the error is importing this module (but do
+        # error if there is a transitive import error)
+        if not (isinstance(e, ModuleNotFoundError) and e.name == EXT_PKG):
+            raise
         return {}, {}
 
     if restrict_to_directories:
@@ -894,20 +878,19 @@ def _attempt_load_module(module_name):
     try:
         extension_module = importlib.import_module(module_name)
     except ImportError as e:
-        if _py_ver >= (3, 6):
-            # e.name is set to the name of the package that fails to load
-            # so don't error ONLY IF the error is importing this module (but do
-            # error if there is a transitive import error)
-            errored_names = [EXT_PKG]
-            parts = module_name.split(".")
-            for p in parts[1:]:
-                errored_names.append("%s.%s" % (errored_names[-1], p))
-            if not (isinstance(e, ModuleNotFoundError) and e.name in errored_names):
-                print(
-                    "The following exception occurred while trying to load '%s' ('%s')"
-                    % (EXT_PKG, module_name)
-                )
-                raise
+        # e.name is set to the name of the package that fails to load
+        # so don't error ONLY IF the error is importing this module (but do
+        # error if there is a transitive import error)
+        errored_names = [EXT_PKG]
+        parts = module_name.split(".")
+        for p in parts[1:]:
+            errored_names.append("%s.%s" % (errored_names[-1], p))
+        if not (isinstance(e, ModuleNotFoundError) and e.name in errored_names):
+            print(
+                "The following exception occurred while trying to load '%s' ('%s')"
+                % (EXT_PKG, module_name)
+            )
+            raise
         _ext_debug("        Unknown error when loading '%s': %s" % (module_name, e))
         return None
     else:

metaflow/parameters.py

@@ -373,7 +373,7 @@ class Parameter(object):
         # Do it one item at a time so errors are ignored at that level (as opposed to
         # at the entire kwargs level)
         self.kwargs = {
-            k: resolve_delayed_evaluator(v, ignore_errors=ignore_errors)
+            k: resolve_delayed_evaluator(v, ignore_errors=ignore_errors, to_dict=True)
             for k, v in self.kwargs.items()
         }
 
@@ -382,7 +382,7 @@ class Parameter(object):
         for key, value in self._override_kwargs.items():
             if value is not None:
                 self.kwargs[key] = resolve_delayed_evaluator(
-                    value, ignore_errors=ignore_errors
+                    value, ignore_errors=ignore_errors, to_dict=True
                 )
         # Set two default values if no-one specified them
         self.kwargs.setdefault("required", False)

metaflow/plugins/argo/argo_workflows_cli.py

@@ -344,20 +344,20 @@ def create(
 
 
 def check_python_version(obj):
-    # argo-workflows integration for Metaflow isn't supported for Py versions below 3.5.
+    # argo-workflows integration for Metaflow isn't supported for Py versions below 3.6.
     # This constraint can very well be lifted if desired.
-    if sys.version_info < (3, 5):
+    if sys.version_info < (3, 6):
         obj.echo("")
         obj.echo(
             "Metaflow doesn't support Argo Workflows for Python %s right now."
             % platform.python_version()
         )
         obj.echo(
-            "Please upgrade your Python interpreter to version 3.5 (or higher) or "
+            "Please upgrade your Python interpreter to version 3.6 (or higher) or "
             "reach out to us at slack.outerbounds.co for more help."
         )
         raise UnsupportedPythonVersion(
-            "Try again with a more recent version of Python (>=3.5)."
+            "Try again with a more recent version of Python (>=3.6)."
         )
 
 

metaflow/plugins/argo/argo_workflows_deployer_objects.py

@@ -9,59 +9,16 @@ from metaflow.exception import MetaflowException
 from metaflow.plugins.argo.argo_client import ArgoClient
 from metaflow.metaflow_config import KUBERNETES_NAMESPACE
 from metaflow.plugins.argo.argo_workflows import ArgoWorkflows
-from metaflow.runner.deployer import Deployer, DeployedFlow, TriggeredRun
+from metaflow.runner.deployer import (
+    Deployer,
+    DeployedFlow,
+    TriggeredRun,
+    generate_fake_flow_file_contents,
+)
 
 from metaflow.runner.utils import get_lower_level_group, handle_timeout, temporary_fifo
 
 
-def generate_fake_flow_file_contents(
-    flow_name: str, param_info: dict, project_name: Optional[str] = None
-):
-    params_code = ""
-    for _, param_details in param_info.items():
-        param_python_var_name = param_details["python_var_name"]
-        param_name = param_details["name"]
-        param_type = param_details["type"]
-        param_help = param_details["description"]
-        param_required = param_details["is_required"]
-
-        if param_type == "JSON":
-            params_code += (
-                f"    {param_python_var_name} = Parameter('{param_name}', "
-                f"type=JSONType, help='''{param_help}''', required={param_required})\n"
-            )
-        elif param_type == "FilePath":
-            is_text = param_details.get("is_text", True)
-            encoding = param_details.get("encoding", "utf-8")
-            params_code += (
-                f"    {param_python_var_name} = IncludeFile('{param_name}', "
-                f"is_text={is_text}, encoding='{encoding}', help='''{param_help}''', "
-                f"required={param_required})\n"
-            )
-        else:
-            params_code += (
-                f"    {param_python_var_name} = Parameter('{param_name}', "
-                f"type={param_type}, help='''{param_help}''', required={param_required})\n"
-            )
-
-    project_decorator = f"@project(name='{project_name}')\n" if project_name else ""
-
-    contents = f"""\
-from metaflow import FlowSpec, Parameter, IncludeFile, JSONType, step, project
-{project_decorator}class {flow_name}(FlowSpec):
-{params_code}
-    @step
-    def start(self):
-        self.next(self.end)
-    @step
-    def end(self):
-        pass
-if __name__ == '__main__':
-    {flow_name}()
-"""
-    return contents
-
-
 class ArgoWorkflowsTriggeredRun(TriggeredRun):
     """
     A class representing a triggered Argo Workflow execution.

metaflow/plugins/aws/aws_client.py

@@ -35,6 +35,12 @@ class Boto3ClientProvider(object):
                 "Could not import module 'boto3'. Install boto3 first."
             )
 
+        # Convert dictionary config to Config object if needed
+        if "config" in client_params and not isinstance(
+            client_params["config"], Config
+        ):
+            client_params["config"] = Config(**client_params["config"])
+
         if module == "s3" and (
             "config" not in client_params or client_params["config"].retries is None
         ):

metaflow/plugins/cards/card_modules/chevron/renderer.py

@@ -61,7 +61,7 @@ def _get_key(key, scopes, warn, keep, def_ldel, def_rdel):
     # Loop through the scopes
     for scope in scopes:
         try:
-            # For every dot seperated key
+            # For every dot separated key
             for child in key.split("."):
                 # Move into the scope
                 try:

metaflow/plugins/cards/card_modules/test_cards.py

@@ -34,7 +34,7 @@ class TestPathSpecCard(MetaflowCard):
 class TestEditableCard(MetaflowCard):
     type = "test_editable_card"
 
-    seperator = "$&#!!@*"
+    separator = "$&#!!@*"
 
     ALLOW_USER_COMPONENTS = True
 
@@ -42,13 +42,13 @@ class TestEditableCard(MetaflowCard):
         self._components = components
 
     def render(self, task):
-        return self.seperator.join([str(comp) for comp in self._components])
+        return self.separator.join([str(comp) for comp in self._components])
 
 
 class TestEditableCard2(MetaflowCard):
     type = "test_editable_card_2"
 
-    seperator = "$&#!!@*"
+    separator = "$&#!!@*"
 
     ALLOW_USER_COMPONENTS = True
 
@@ -56,19 +56,19 @@ class TestEditableCard2(MetaflowCard):
         self._components = components
 
     def render(self, task):
-        return self.seperator.join([str(comp) for comp in self._components])
+        return self.separator.join([str(comp) for comp in self._components])
 
 
 class TestNonEditableCard(MetaflowCard):
     type = "test_non_editable_card"
 
-    seperator = "$&#!!@*"
+    separator = "$&#!!@*"
 
     def __init__(self, components=[], **kwargs):
         self._components = components
 
     def render(self, task):
-        return self.seperator.join([str(comp) for comp in self._components])
+        return self.separator.join([str(comp) for comp in self._components])
 
 
 class TestMockCard(MetaflowCard):

metaflow/plugins/cards/component_serializer.py

@@ -57,15 +57,8 @@ class ComponentStore:
         The `_component_map` attribute is supposed to be a dictionary so that we can access the components by their ids.
         But we also want to maintain order in which components are inserted since all of these components are going to be visible on a UI.
         Since python3.6 dictionaries are ordered by default so we can use the default python `dict`.
-        For python3.5 and below we need to use an OrderedDict since `dict`'s are not ordered by default.
         """
-        python_version = int(platform.python_version_tuple()[0]) * 10 + int(
-            platform.python_version_tuple()[1]
-        )
-        if python_version < 36:
-            self._component_map = OrderedDict()
-        else:
-            self._component_map = {}
+        self._component_map = {}
 
     def __init__(self, logger, card_type=None, components=None, user_set_id=None):
         self._logger = logger

metaflow/plugins/datatools/s3/s3op.py

@@ -131,7 +131,7 @@ def normalize_client_error(err):
     except ValueError:
         if error_code in ("AccessDenied", "AllAccessDisabled", "InvalidAccessKeyId"):
             return 403
-        if error_code == "NoSuchKey":
+        if error_code in ("NoSuchKey", "NoSuchBucket"):
             return 404
         if error_code == "InvalidRange":
             return 416

metaflow/plugins/metadata_providers/service.py

@@ -72,14 +72,18 @@ class ServiceMetadataProvider(MetadataProvider):
     @classmethod
     def compute_info(cls, val):
         v = val.rstrip("/")
-        try:
-            resp = cls._session.get(
-                os.path.join(v, "ping"), headers=SERVICE_HEADERS.copy()
-            )
-            resp.raise_for_status()
-        except:  # noqa E722
-            raise ValueError("Metaflow service [%s] unreachable." % v)
-        return v
+        for i in range(SERVICE_RETRY_COUNT):
+            try:
+                resp = cls._session.get(
+                    os.path.join(v, "ping"), headers=SERVICE_HEADERS.copy()
+                )
+                resp.raise_for_status()
+            except:  # noqa E722
+                time.sleep(2 ** (i - 1))
+            else:
+                return v
+
+        raise ValueError("Metaflow service [%s] unreachable." % v)
 
     @classmethod
     def default_info(cls):

metaflow/plugins/package_cli.py

@@ -38,14 +38,24 @@ def info(obj):
 
 
 @package.command(help="List files included in the code package.")
+@click.option(
+    "--archive/--no-archive",
+    default=False,
+    help="If True, lists the file paths as present in the code package archive; "
+    "otherwise, lists the files on your filesystem included in the code package",
+    show_default=True,
+)
 @click.pass_obj
-def list(obj):
+def list(obj, archive=False):
     obj.echo(
         "Files included in the code package " "(change with --package-suffixes):",
         fg="magenta",
         bold=False,
     )
-    obj.echo_always("\n".join(path for path, _ in obj.package.path_tuples()))
+    if archive:
+        obj.echo_always("\n".join(path for _, path in obj.package.path_tuples()))
+    else:
+        obj.echo_always("\n".join(path for path, _ in obj.package.path_tuples()))
 
 
 @package.command(help="Save the current code package in a tar file")

metaflow/plugins/pypi/bootstrap.py

@@ -14,7 +14,7 @@ from urllib.request import urlopen
 from metaflow.metaflow_config import DATASTORE_LOCAL_DIR, CONDA_USE_FAST_INIT
 from metaflow.plugins import DATASTORES
 from metaflow.plugins.pypi.utils import MICROMAMBA_MIRROR_URL, MICROMAMBA_URL
-from metaflow.util import which
+from metaflow.util import which, get_metaflow_root
 from urllib.request import Request
 import warnings
 
@@ -366,7 +366,7 @@ if __name__ == "__main__":
         # Move MAGIC_FILE inside local datastore.
         os.makedirs(manifest_dir, exist_ok=True)
         shutil.move(
-            os.path.join(os.getcwd(), MAGIC_FILE),
+            os.path.join(get_metaflow_root(), MAGIC_FILE),
             os.path.join(manifest_dir, MAGIC_FILE),
         )
         with open(os.path.join(manifest_dir, MAGIC_FILE)) as f: