metaflow 2.15.14__py2.py3-none-any.whl → 2.15.15__py2.py3-none-any.whl

metaflow/__init__.py

@@ -128,8 +128,8 @@ _import_tl_plugins(globals())
 # this auto-generates decorator functions from Decorator objects
 # in the top-level metaflow namespace
 _import_plugin_decorators(globals())
-# Setting card import for only python 3.4
-if sys.version_info[0] >= 3 and sys.version_info[1] >= 4:
+# Setting card import for only python 3.6
+if sys.version_info[0] >= 3 and sys.version_info[1] >= 6:
     from . import cards
 
 # Client

metaflow/_vendor/click/core.py

@@ -719,7 +719,7 @@ class BaseCommand(object):
         prog_name=None,
         complete_var=None,
         standalone_mode=True,
-        **extra
+        **extra,
     ):
         """This is the way to invoke a script with all the bells and
         whistles as a command line application.  This will always terminate
@@ -1101,7 +1101,7 @@ class MultiCommand(Command):
         subcommand_metavar=None,
         chain=False,
         result_callback=None,
-        **attrs
+        **attrs,
     ):
         Command.__init__(self, name, **attrs)
         if no_args_is_help is None:
@@ -1463,6 +1463,7 @@ class Parameter(object):
         parameter. The old callback format will still work, but it will
         raise a warning to give you a chance to migrate the code easier.
     """
+
     param_type_name = "parameter"
 
     def __init__(
@@ -1708,7 +1709,7 @@ class Option(Parameter):
         hidden=False,
         show_choices=True,
         show_envvar=False,
-        **attrs
+        **attrs,
     ):
         default_is_missing = attrs.get("default", _missing) is _missing
         Parameter.__init__(self, param_decls, type=type, **attrs)

metaflow/cmd/develop/stubs.py

@@ -12,25 +12,13 @@ from . import develop
 from .stub_generator import StubGenerator
 
 _py_ver = sys.version_info[:2]
-_metadata_package = None
 
-
-def _check_stubs_supported():
-    global _metadata_package
-    if _metadata_package is not None:
-        return _metadata_package
-    else:
-        if _py_ver >= (3, 4):
-            if _py_ver >= (3, 8):
-                from importlib import metadata
-            elif _py_ver >= (3, 7):
-                from metaflow._vendor.v3_7 import importlib_metadata as metadata
-            elif _py_ver >= (3, 6):
-                from metaflow._vendor.v3_6 import importlib_metadata as metadata
-            else:
-                from metaflow._vendor.v3_5 import importlib_metadata as metadata
-            _metadata_package = metadata
-        return _metadata_package
+if _py_ver >= (3, 8):
+    from importlib import metadata
+elif _py_ver >= (3, 7):
+    from metaflow._vendor.v3_7 import importlib_metadata as metadata
+else:
+    from metaflow._vendor.v3_6 import importlib_metadata as metadata
 
 
 @develop.group(short_help="Stubs management")
@@ -45,12 +33,6 @@ def stubs(ctx: Any):
     This CLI provides utilities to check and generate stubs for your current Metaflow
     installation.
     """
-    if _check_stubs_supported() is None:
-        raise click.UsageError(
-            "Building and installing stubs are not supported on Python %d.%d "
-            "(3.4 minimum required)" % _py_ver,
-            ctx=ctx,
-        )
 
 
 @stubs.command(short_help="Check validity of stubs")
@@ -187,7 +169,7 @@ setup(
     packages=find_namespace_packages(),
     package_data={{"metaflow-stubs": ["generated_for.txt", "py.typed", "**/*.pyi"]}},
     install_requires=["metaflow=={mf_version}"],
-    python_requires=">=3.5.2",
+    python_requires=">=3.6.1",
 )
                 """
             )
@@ -330,14 +312,14 @@ def get_packages_for_stubs() -> Tuple[List[Tuple[str, str]], List[str]]:
     # some reason it shows up multiple times.
     interesting_dists = [
         d
-        for d in _metadata_package.distributions()
+        for d in metadata.distributions()
         if any(
             [
                 p == "metaflow-stubs"
                 for p in (d.read_text("top_level.txt") or "").split()
             ]
         )
-        and isinstance(d, _metadata_package.PathDistribution)
+        and isinstance(d, metadata.PathDistribution)
     ]
 
     for dist in interesting_dists:

metaflow/datastore/task_datastore.py

@@ -118,7 +118,7 @@ class TaskDataStore(object):
         # The GZIP encodings are for backward compatibility
         self._encodings = {"pickle-v2", "gzip+pickle-v2"}
         ver = sys.version_info[0] * 10 + sys.version_info[1]
-        if ver >= 34:
+        if ver >= 36:
             self._encodings.add("pickle-v4")
             self._encodings.add("gzip+pickle-v4")
 
@@ -289,7 +289,7 @@ class TaskDataStore(object):
                     except (SystemError, OverflowError) as e:
                         raise DataException(
                             "Artifact *%s* is very large (over 2GB). "
-                            "You need to use Python 3.4 or newer if you want to "
+                            "You need to use Python 3.6 or newer if you want to "
                             "serialize large objects." % name
                         ) from e
                     except TypeError as e:
@@ -352,7 +352,7 @@ class TaskDataStore(object):
                 encode_type = "gzip+pickle-v2"
             if encode_type not in self._encodings:
                 raise DataException(
-                    "Python 3.4 or later is required to load artifact '%s'" % name
+                    "Python 3.6 or later is required to load artifact '%s'" % name
                 )
             else:
                 to_load[self._objects[name]].append(name)

metaflow/decorators.py

@@ -152,7 +152,7 @@ class Decorator(object):
         # Note that by design, later values override previous ones.
         self.attributes, new_user_attributes = unpack_delayed_evaluator(self.attributes)
         self._user_defined_attributes.update(new_user_attributes)
-        self.attributes = resolve_delayed_evaluator(self.attributes)
+        self.attributes = resolve_delayed_evaluator(self.attributes, to_dict=True)
 
         self._ran_init = True
 
@@ -638,7 +638,7 @@ StepFlag = NewType("StepFlag", bool)
 
 @overload
 def step(
-    f: Callable[[FlowSpecDerived], None]
+    f: Callable[[FlowSpecDerived], None],
 ) -> Callable[[FlowSpecDerived, StepFlag], None]: ...
 
 
@@ -649,7 +649,7 @@ def step(
 
 
 def step(
-    f: Union[Callable[[FlowSpecDerived], None], Callable[[FlowSpecDerived, Any], None]]
+    f: Union[Callable[[FlowSpecDerived], None], Callable[[FlowSpecDerived, Any], None]],
 ):
     """
     Marks a method in a FlowSpec as a Metaflow Step. Note that this

metaflow/extension_support/__init__.py

@@ -103,9 +103,6 @@ def load_module(module_name):
 
 def get_modules(extension_point):
     modules_to_load = []
-    if not _mfext_supported:
-        _ext_debug("Not supported for your Python version -- 3.4+ is needed")
-        return []
     if extension_point not in _extension_points:
         raise RuntimeError(
             "Metaflow extension point '%s' not supported" % extension_point
@@ -146,9 +143,6 @@ def dump_module_info(all_packages=None, pkgs_per_extension_point=None):
 
 
 def get_extensions_in_dir(d):
-    if not _mfext_supported:
-        _ext_debug("Not supported for your Python version -- 3.4+ is needed")
-        return None, None
     return _get_extension_packages(ignore_info_file=True, restrict_to_directories=[d])
 
 
@@ -312,21 +306,16 @@ def multiload_all(modules, extension_point, dst_globals):
 
 
 _py_ver = sys.version_info[:2]
-_mfext_supported = False
 _aliased_modules = []
 
-if _py_ver >= (3, 4):
-    import importlib.util
+import importlib.util
 
-    if _py_ver >= (3, 8):
-        from importlib import metadata
-    elif _py_ver >= (3, 7):
-        from metaflow._vendor.v3_7 import importlib_metadata as metadata
-    elif _py_ver >= (3, 6):
-        from metaflow._vendor.v3_6 import importlib_metadata as metadata
-    else:
-        from metaflow._vendor.v3_5 import importlib_metadata as metadata
-    _mfext_supported = True
+if _py_ver >= (3, 8):
+    from importlib import metadata
+elif _py_ver >= (3, 7):
+    from metaflow._vendor.v3_7 import importlib_metadata as metadata
+else:
+    from metaflow._vendor.v3_6 import importlib_metadata as metadata
 
 # Extension points are the directories that can be present in a EXT_PKG to
 # contribute to that extension point. For example, if you have
@@ -355,10 +344,6 @@ def _ext_debug(*args, **kwargs):
 
 
 def _get_extension_packages(ignore_info_file=False, restrict_to_directories=None):
-    if not _mfext_supported:
-        _ext_debug("Not supported for your Python version -- 3.4+ is needed")
-        return {}, {}
-
     # If we have an INFO file with the appropriate information (if running from a saved
     # code package for example), we use that directly
     # Pre-compute on _extension_points
@@ -381,12 +366,11 @@ def _get_extension_packages(ignore_info_file=False, restrict_to_directories=None
     try:
         extensions_module = importlib.import_module(EXT_PKG)
     except ImportError as e:
-        if _py_ver >= (3, 6):
-            # e.name is set to the name of the package that fails to load
-            # so don't error ONLY IF the error is importing this module (but do
-            # error if there is a transitive import error)
-            if not (isinstance(e, ModuleNotFoundError) and e.name == EXT_PKG):
-                raise
+        # e.name is set to the name of the package that fails to load
+        # so don't error ONLY IF the error is importing this module (but do
+        # error if there is a transitive import error)
+        if not (isinstance(e, ModuleNotFoundError) and e.name == EXT_PKG):
+            raise
         return {}, {}
 
     if restrict_to_directories:
@@ -894,20 +878,19 @@ def _attempt_load_module(module_name):
     try:
         extension_module = importlib.import_module(module_name)
     except ImportError as e:
-        if _py_ver >= (3, 6):
-            # e.name is set to the name of the package that fails to load
-            # so don't error ONLY IF the error is importing this module (but do
-            # error if there is a transitive import error)
-            errored_names = [EXT_PKG]
-            parts = module_name.split(".")
-            for p in parts[1:]:
-                errored_names.append("%s.%s" % (errored_names[-1], p))
-            if not (isinstance(e, ModuleNotFoundError) and e.name in errored_names):
-                print(
-                    "The following exception occurred while trying to load '%s' ('%s')"
-                    % (EXT_PKG, module_name)
-                )
-                raise
+        # e.name is set to the name of the package that fails to load
+        # so don't error ONLY IF the error is importing this module (but do
+        # error if there is a transitive import error)
+        errored_names = [EXT_PKG]
+        parts = module_name.split(".")
+        for p in parts[1:]:
+            errored_names.append("%s.%s" % (errored_names[-1], p))
+        if not (isinstance(e, ModuleNotFoundError) and e.name in errored_names):
+            print(
+                "The following exception occurred while trying to load '%s' ('%s')"
+                % (EXT_PKG, module_name)
+            )
+            raise
         _ext_debug("        Unknown error when loading '%s': %s" % (module_name, e))
         return None
     else:

metaflow/parameters.py

@@ -373,7 +373,7 @@ class Parameter(object):
         # Do it one item at a time so errors are ignored at that level (as opposed to
         # at the entire kwargs level)
         self.kwargs = {
-            k: resolve_delayed_evaluator(v, ignore_errors=ignore_errors)
+            k: resolve_delayed_evaluator(v, ignore_errors=ignore_errors, to_dict=True)
             for k, v in self.kwargs.items()
         }
 
@@ -382,7 +382,7 @@ class Parameter(object):
         for key, value in self._override_kwargs.items():
             if value is not None:
                 self.kwargs[key] = resolve_delayed_evaluator(
-                    value, ignore_errors=ignore_errors
+                    value, ignore_errors=ignore_errors, to_dict=True
                 )
         # Set two default values if no-one specified them
         self.kwargs.setdefault("required", False)

metaflow/plugins/argo/argo_workflows_cli.py

@@ -344,20 +344,20 @@ def create(
 
 
 def check_python_version(obj):
-    # argo-workflows integration for Metaflow isn't supported for Py versions below 3.5.
+    # argo-workflows integration for Metaflow isn't supported for Py versions below 3.6.
     # This constraint can very well be lifted if desired.
-    if sys.version_info < (3, 5):
+    if sys.version_info < (3, 6):
         obj.echo("")
         obj.echo(
             "Metaflow doesn't support Argo Workflows for Python %s right now."
             % platform.python_version()
         )
         obj.echo(
-            "Please upgrade your Python interpreter to version 3.5 (or higher) or "
+            "Please upgrade your Python interpreter to version 3.6 (or higher) or "
             "reach out to us at slack.outerbounds.co for more help."
         )
         raise UnsupportedPythonVersion(
-            "Try again with a more recent version of Python (>=3.5)."
+            "Try again with a more recent version of Python (>=3.6)."
         )
 
 

metaflow/plugins/cards/card_modules/chevron/renderer.py

@@ -61,7 +61,7 @@ def _get_key(key, scopes, warn, keep, def_ldel, def_rdel):
     # Loop through the scopes
     for scope in scopes:
         try:
-            # For every dot seperated key
+            # For every dot separated key
             for child in key.split("."):
                 # Move into the scope
                 try:

metaflow/plugins/cards/card_modules/test_cards.py

@@ -34,7 +34,7 @@ class TestPathSpecCard(MetaflowCard):
 class TestEditableCard(MetaflowCard):
     type = "test_editable_card"
 
-    seperator = "$&#!!@*"
+    separator = "$&#!!@*"
 
     ALLOW_USER_COMPONENTS = True
 
@@ -42,13 +42,13 @@ class TestEditableCard(MetaflowCard):
         self._components = components
 
     def render(self, task):
-        return self.seperator.join([str(comp) for comp in self._components])
+        return self.separator.join([str(comp) for comp in self._components])
 
 
 class TestEditableCard2(MetaflowCard):
     type = "test_editable_card_2"
 
-    seperator = "$&#!!@*"
+    separator = "$&#!!@*"
 
     ALLOW_USER_COMPONENTS = True
 
@@ -56,19 +56,19 @@ class TestEditableCard2(MetaflowCard):
         self._components = components
 
     def render(self, task):
-        return self.seperator.join([str(comp) for comp in self._components])
+        return self.separator.join([str(comp) for comp in self._components])
 
 
 class TestNonEditableCard(MetaflowCard):
     type = "test_non_editable_card"
 
-    seperator = "$&#!!@*"
+    separator = "$&#!!@*"
 
     def __init__(self, components=[], **kwargs):
         self._components = components
 
     def render(self, task):
-        return self.seperator.join([str(comp) for comp in self._components])
+        return self.separator.join([str(comp) for comp in self._components])
 
 
 class TestMockCard(MetaflowCard):

metaflow/plugins/cards/component_serializer.py

@@ -57,15 +57,8 @@ class ComponentStore:
         The `_component_map` attribute is supposed to be a dictionary so that we can access the components by their ids.
         But we also want to maintain order in which components are inserted since all of these components are going to be visible on a UI.
         Since python3.6 dictionaries are ordered by default so we can use the default python `dict`.
-        For python3.5 and below we need to use an OrderedDict since `dict`'s are not ordered by default.
         """
-        python_version = int(platform.python_version_tuple()[0]) * 10 + int(
-            platform.python_version_tuple()[1]
-        )
-        if python_version < 36:
-            self._component_map = OrderedDict()
-        else:
-            self._component_map = {}
+        self._component_map = {}
 
     def __init__(self, logger, card_type=None, components=None, user_set_id=None):
         self._logger = logger

metaflow/plugins/package_cli.py

@@ -38,14 +38,24 @@ def info(obj):
 
 
 @package.command(help="List files included in the code package.")
+@click.option(
+    "--archive/--no-archive",
+    default=False,
+    help="If True, lists the file paths as present in the code package archive; "
+    "otherwise, lists the files on your filesystem included in the code package",
+    show_default=True,
+)
 @click.pass_obj
-def list(obj):
+def list(obj, archive=False):
     obj.echo(
         "Files included in the code package " "(change with --package-suffixes):",
         fg="magenta",
         bold=False,
     )
-    obj.echo_always("\n".join(path for path, _ in obj.package.path_tuples()))
+    if archive:
+        obj.echo_always("\n".join(path for _, path in obj.package.path_tuples()))
+    else:
+        obj.echo_always("\n".join(path for path, _ in obj.package.path_tuples()))
 
 
 @package.command(help="Save the current code package in a tar file")

metaflow/plugins/pypi/bootstrap.py

@@ -14,7 +14,7 @@ from urllib.request import urlopen
 from metaflow.metaflow_config import DATASTORE_LOCAL_DIR, CONDA_USE_FAST_INIT
 from metaflow.plugins import DATASTORES
 from metaflow.plugins.pypi.utils import MICROMAMBA_MIRROR_URL, MICROMAMBA_URL
-from metaflow.util import which
+from metaflow.util import which, get_metaflow_root
 from urllib.request import Request
 import warnings
 
@@ -366,7 +366,7 @@ if __name__ == "__main__":
         # Move MAGIC_FILE inside local datastore.
         os.makedirs(manifest_dir, exist_ok=True)
         shutil.move(
-            os.path.join(os.getcwd(), MAGIC_FILE),
+            os.path.join(get_metaflow_root(), MAGIC_FILE),
             os.path.join(manifest_dir, MAGIC_FILE),
         )
         with open(os.path.join(manifest_dir, MAGIC_FILE)) as f:

metaflow/plugins/uv/bootstrap.py

@@ -4,6 +4,7 @@ import sys
 import time
 
 from metaflow.util import which
+from metaflow.info_file import read_info_file
 from metaflow.metaflow_config import get_pinned_conda_libs
 from urllib.request import Request, urlopen
 from urllib.error import URLError
@@ -78,11 +79,27 @@ if __name__ == "__main__":
         # return only dependency names instead of pinned versions
         return pinned.keys()
 
+    def skip_metaflow_dependencies():
+        skip_pkgs = ["metaflow"]
+        info = read_info_file()
+        if info is not None:
+            try:
+                skip_pkgs.extend([ext_name for ext_name in info["ext_info"][0].keys()])
+            except Exception:
+                print(
+                    "Failed to read INFO. Metaflow-related packages might get installed during runtime."
+                )
+
+        return skip_pkgs
+
     def sync_uv_project(datastore_type):
         print("Syncing uv project...")
         dependencies = " ".join(get_dependencies(datastore_type))
+        skip_pkgs = " ".join(
+            [f"--no-install-package {dep}" for dep in skip_metaflow_dependencies()]
+        )
         cmd = f"""set -e;
-            uv sync --frozen --no-install-package metaflow;
+            uv sync --frozen {skip_pkgs};
             uv pip install {dependencies} --strict
             """
         run_cmd(cmd)

metaflow/plugins/uv/uv_environment.py

@@ -22,7 +22,7 @@ class UVEnvironment(MetaflowEnvironment):
         self.logger("Bootstrapping uv...")
 
     def executable(self, step_name, default=None):
-        return "uv run python"
+        return "uv run --no-sync python"
 
     def add_to_package(self):
         # NOTE: We treat uv.lock and pyproject.toml as regular project assets and ship these along user code as part of the code package

metaflow/runner/click_api.py

@@ -467,9 +467,14 @@ class MetaflowAPI(object):
                     config_file = defaults.get("config")
 
                 if config_file:
-                    config_file = map(
-                        lambda x: (x[0], ConvertPath.convert_value(x[1], is_default)),
-                        config_file,
+                    config_file = dict(
+                        map(
+                            lambda x: (
+                                x[0],
+                                ConvertPath.convert_value(x[1], is_default),
+                            ),
+                            config_file,
+                        )
                     )
 
                 is_default = False
@@ -479,12 +484,14 @@ class MetaflowAPI(object):
                     config_value = defaults.get("config_value")
 
                 if config_value:
-                    config_value = map(
-                        lambda x: (
-                            x[0],
-                            ConvertDictOrStr.convert_value(x[1], is_default),
-                        ),
-                        config_value,
+                    config_value = dict(
+                        map(
+                            lambda x: (
+                                x[0],
+                                ConvertDictOrStr.convert_value(x[1], is_default),
+                            ),
+                            config_value,
+                        )
                     )
 
                 if (config_file is None) ^ (config_value is None):

metaflow/runner/deployer_impl.py

@@ -5,8 +5,10 @@ import sys
 
 from typing import Any, ClassVar, Dict, Optional, TYPE_CHECKING, Type
 
+from metaflow.metaflow_config import CLICK_API_PROCESS_CONFIG
+
 from .subprocess_manager import SubprocessManager
-from .utils import get_lower_level_group, handle_timeout, temporary_fifo
+from .utils import get_lower_level_group, handle_timeout, temporary_fifo, with_dir
 
 if TYPE_CHECKING:
     import metaflow.runner.deployer
@@ -88,7 +90,7 @@ class DeployerImpl(object):
         self.show_output = show_output
         self.profile = profile
         self.env = env
-        self.cwd = cwd
+        self.cwd = cwd or os.getcwd()
         self.file_read_timeout = file_read_timeout
 
         self.env_vars = os.environ.copy()
@@ -140,9 +142,19 @@ class DeployerImpl(object):
     ) -> "metaflow.runner.deployer.DeployedFlow":
         with temporary_fifo() as (attribute_file_path, attribute_file_fd):
             # every subclass needs to have `self.deployer_kwargs`
-            command = get_lower_level_group(
-                self.api, self.top_level_kwargs, self.TYPE, self.deployer_kwargs
-            ).create(deployer_attribute_file=attribute_file_path, **kwargs)
+            # TODO: Get rid of CLICK_API_PROCESS_CONFIG in the near future
+            if CLICK_API_PROCESS_CONFIG:
+                # We need to run this in the cwd because configs depend on files
+                # that may be located in paths relative to the directory the user
+                # wants to run in
+                with with_dir(self.cwd):
+                    command = get_lower_level_group(
+                        self.api, self.top_level_kwargs, self.TYPE, self.deployer_kwargs
+                    ).create(deployer_attribute_file=attribute_file_path, **kwargs)
+            else:
+                command = get_lower_level_group(
+                    self.api, self.top_level_kwargs, self.TYPE, self.deployer_kwargs
+                ).create(deployer_attribute_file=attribute_file_path, **kwargs)
 
             pid = self.spm.run_command(
                 [sys.executable, *command],