metaflow 2.14.1__py2.py3-none-any.whl → 2.14.3__py2.py3-none-any.whl

metaflow/cli.py

@@ -16,7 +16,6 @@ from .exception import CommandException, MetaflowException
 from .flowspec import _FlowState
 from .graph import FlowGraph
 from .metaflow_config import (
-    DECOSPECS,
     DEFAULT_DATASTORE,
     DEFAULT_ENVIRONMENT,
     DEFAULT_EVENT_LOGGER,
@@ -106,26 +105,6 @@ def logger(body="", system_msg=False, head="", bad=False, timestamp=True, nl=Tru
     click.secho(body, bold=system_msg, fg=LOGGER_BAD_COLOR if bad else None, nl=nl)
 
 
-def config_merge_cb(ctx, param, value):
-    # Callback to:
-    #  - read  the Click auto_envvar variable from both the
-    #    environment AND the configuration
-    #  - merge that value with the value passed in the command line (value)
-    #  - return the value as a tuple
-    # Note that this function gets called even if there is no option passed on the
-    # command line.
-    # NOTE: Assumes that ctx.auto_envvar_prefix is set to METAFLOW (same as in
-    # from_conf)
-
-    # Special case where DECOSPECS and value are the same. This happens
-    # when there is no --with option at the TL and DECOSPECS is read from
-    # the env var. In this case, click also passes it as value
-    splits = DECOSPECS.split()
-    if len(splits) == len(value) and all([a == b for (a, b) in zip(splits, value)]):
-        return value
-    return tuple(list(value) + DECOSPECS.split())
-
-
 @click.group(
     cls=LazyGroup,
     lazy_subcommands={
@@ -284,7 +263,6 @@ def version(obj):
     multiple=True,
     help="Add a decorator to all steps. You can specify this option "
     "multiple times to attach multiple decorators in steps.",
-    callback=config_merge_cb,
 )
 @click.option(
     "--pylint/--no-pylint",

metaflow/cli_components/run_cmds.py

@@ -8,6 +8,7 @@ from .. import decorators, namespace, parameters, tracing
 from ..exception import CommandException
 from ..graph import FlowGraph
 from ..metaflow_current import current
+from ..metaflow_config import DEFAULT_DECOSPECS
 from ..package import MetaflowPackage
 from ..runtime import NativeRuntime
 from ..system import _system_logger
@@ -70,6 +71,23 @@ def write_file(file_path, content):
             f.write(str(content))
 
 
+def config_merge_cb(ctx, param, value):
+    # Callback to:
+    #  - read  the Click auto_envvar variable from both the
+    #    environment AND the configuration
+    #  - merge that value with the value passed in the command line (value)
+    #  - return the value as a tuple
+    # Note that this function gets called even if there is no option passed on the
+    # command line.
+    # NOTE: Assumes that ctx.auto_envvar_prefix is set to METAFLOW (same as in
+    # from_conf)
+
+    # Read decospecs options from the environment (METAFLOW_DEFAULT_DECOSPECS=...)
+    # and merge them with the one provided as --with.
+    splits = DEFAULT_DECOSPECS.split()
+    return tuple(list(value) + splits)
+
+
 def common_run_options(func):
     @click.option(
         "--tag",
@@ -109,6 +127,7 @@ def common_run_options(func):
         help="Add a decorator to all steps. You can specify this "
         "option multiple times to attach multiple decorators "
         "in steps.",
+        callback=config_merge_cb,
     )
     @click.option(
         "--run-id-file",

metaflow/metaflow_config.py

@@ -284,7 +284,7 @@ CONTACT_INFO = from_conf(
 ###
 # Format is a space separated string of decospecs (what is passed
 # using --with)
-DECOSPECS = from_conf("DECOSPECS", "")
+DEFAULT_DECOSPECS = from_conf("DEFAULT_DECOSPECS", "")
 
 ###
 # AWS Batch configuration
@@ -432,6 +432,9 @@ CONDA_PACKAGE_GSROOT = from_conf("CONDA_PACKAGE_GSROOT")
 # should result in an appreciable speedup in flow environment initialization.
 CONDA_DEPENDENCY_RESOLVER = from_conf("CONDA_DEPENDENCY_RESOLVER", "conda")
 
+# Default to not using fast init binary.
+CONDA_USE_FAST_INIT = from_conf("CONDA_USE_FAST_INIT", False)
+
 ###
 # Escape hatch configuration
 ###
@@ -577,9 +580,9 @@ try:
                 _TOGGLE_DECOSPECS.extend(o)
             elif not n.startswith("__") and not isinstance(o, types.ModuleType):
                 globals()[n] = o
-    # If DECOSPECS is set, use that, else extrapolate from extensions
-    if not DECOSPECS:
-        DECOSPECS = " ".join(_TOGGLE_DECOSPECS)
+    # If DEFAULT_DECOSPECS is set, use that, else extrapolate from extensions
+    if not DEFAULT_DECOSPECS:
+        DEFAULT_DECOSPECS = " ".join(_TOGGLE_DECOSPECS)
 
 finally:
     # Erase all temporary names to avoid leaking things

metaflow/plugins/pypi/bootstrap.py

@@ -10,7 +10,7 @@ import tarfile
 import time
 from urllib.error import URLError
 from urllib.request import urlopen
-from metaflow.metaflow_config import DATASTORE_LOCAL_DIR
+from metaflow.metaflow_config import DATASTORE_LOCAL_DIR, CONDA_USE_FAST_INIT
 from metaflow.plugins import DATASTORES
 from metaflow.plugins.pypi.utils import MICROMAMBA_MIRROR_URL, MICROMAMBA_URL
 from metaflow.util import which
@@ -329,6 +329,8 @@ if __name__ == "__main__":
 
     @timer
     def fast_setup_environment(architecture, storage, env, prefix, pkgs_dir):
+        install_fast_initializer(architecture)
+
         # Get package urls
         conda_pkgs = env["conda"]
         pypi_pkgs = env.get("pypi", [])
@@ -379,9 +381,7 @@ if __name__ == "__main__":
         with open(os.path.join(manifest_dir, MAGIC_FILE)) as f:
             env = json.load(f)[id_][architecture]
 
-        if datastore_type == "s3":
-            # TODO: Remove this once fast-initializer is ready for all datastores
-            install_fast_initializer(architecture)
+        if CONDA_USE_FAST_INIT:
             fast_setup_environment(architecture, storage, env, prefix, pkgs_dir)
         else:
             setup_environment(

metaflow/version.py

@@ -1 +1 @@
-metaflow_version = "2.14.1"
+metaflow_version = "2.14.3"

{metaflow-2.14.1.dist-info → metaflow-2.14.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: metaflow
-Version: 2.14.1
+Version: 2.14.3
 Summary: Metaflow: More Data Science, Less Engineering
 Author: Metaflow Developers
 Author-email: help@metaflow.org
@@ -26,7 +26,7 @@ License-File: LICENSE
 Requires-Dist: requests
 Requires-Dist: boto3
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.14.1; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.14.3; extra == "stubs"
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier

{metaflow-2.14.1.dist-info → metaflow-2.14.3.dist-info}/RECORD

@@ -1,7 +1,7 @@
 metaflow/R.py,sha256=CqVfIatvmjciuICNnoyyNGrwE7Va9iXfLdFbQa52hwA,3958
 metaflow/__init__.py,sha256=fbhdWiWnEoAX4KnzRHMY_iQcT-uYlMWhzrXPKvK0i5g,5832
 metaflow/cards.py,sha256=IbRmredvmFEU0V6JL7DR8wCESwVmmZJubr6x24bo7U4,442
-metaflow/cli.py,sha256=mFTp2QzKoi_5rhf9_ht-GHj8-sVBbVdXqOXEMIjnktA,21846
+metaflow/cli.py,sha256=RU-yXpT-Lfl3xGyFNtL742e9KEqcRxEnQ-4mwXrXhvo,20928
 metaflow/cli_args.py,sha256=hDsdWdRmfXYifVGq6b6FDfgoWxtIG2nr_lU6EBV0Pnk,3584
 metaflow/clone_util.py,sha256=LSuVbFpPUh92UW32DBcnZbL0FFw-4w3CLa0tpEbCkzk,2066
 metaflow/cmd_with_io.py,sha256=kl53HkAIyv0ecpItv08wZYczv7u3msD1VCcciqigqf0,588
@@ -16,7 +16,7 @@ metaflow/includefile.py,sha256=kWKDSlzVcRVNGG9PV5eB3o2ynrzqhVsfaLtkqjshn7Q,20948
 metaflow/info_file.py,sha256=wtf2_F0M6dgiUu74AFImM8lfy5RrUw5Yj7Rgs2swKRY,686
 metaflow/integrations.py,sha256=LlsaoePRg03DjENnmLxZDYto3NwWc9z_PtU6nJxLldg,1480
 metaflow/lint.py,sha256=x4p6tnRzYqNNniCGXyrUW0WuYfTUgnaOMRivxvnxask,11661
-metaflow/metaflow_config.py,sha256=t9x39B64B3OWmWRexKseHkb0lnaEaYYY4UAzE7Uyjls,23289
+metaflow/metaflow_config.py,sha256=jCyJhKrBePqygpfZ4C-gmU14FlTesvFmmimnTP3t_U8,23433
 metaflow/metaflow_config_funcs.py,sha256=5GlvoafV6SxykwfL8D12WXSfwjBN_NsyuKE_Q3gjGVE,6738
 metaflow/metaflow_current.py,sha256=pfkXmkyHeMJhxIs6HBJNBEaBDpcl5kz9Wx5mW6F_3qo,7164
 metaflow/metaflow_environment.py,sha256=gs_UpYpuOKEEfFM0z0tnwje7zAVsQ5Ck7Dp2M9_1utQ,8065
@@ -36,7 +36,7 @@ metaflow/tuple_util.py,sha256=_G5YIEhuugwJ_f6rrZoelMFak3DqAR2tt_5CapS1XTY,830
 metaflow/unbounded_foreach.py,sha256=p184WMbrMJ3xKYHwewj27ZhRUsSj_kw1jlye5gA9xJk,387
 metaflow/util.py,sha256=mJBkV5tShIyCsLDeM1zygQGeciQVMrVPm_qI8Oi33G0,14656
 metaflow/vendor.py,sha256=FchtA9tH22JM-eEtJ2c9FpUdMn8sSb1VHuQS56EcdZk,5139
-metaflow/version.py,sha256=fyDZXks2VX1Hb8fXY7qYRCSj4E6RJ_8skOF33Cnea4w,28
+metaflow/version.py,sha256=WyNmkJuZnJ4inDdNJ_Tpr0MjMGRGQA51T55iBVMg3tc,28
 metaflow/_vendor/__init__.py,sha256=y_CiwUD3l4eAKvTVDZeqgVujMy31cAM1qjAB-HfI-9s,353
 metaflow/_vendor/typing_extensions.py,sha256=0nUs5p1A_UrZigrAVBoOEM6TxU37zzPDUtiij1ZwpNc,110417
 metaflow/_vendor/zipp.py,sha256=ajztOH-9I7KA_4wqDYygtHa6xUBVZgFpmZ8FE74HHHI,8425
@@ -113,7 +113,7 @@ metaflow/_vendor/v3_6/importlib_metadata/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCe
 metaflow/cli_components/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 metaflow/cli_components/dump_cmd.py,sha256=SZEX51BWNd1o3H2uHDkYA8KRvou5X8g5rTwpdu5vnNQ,2704
 metaflow/cli_components/init_cmd.py,sha256=Er-BO59UEUV1HIsg81bRtZWT2D2IZNMp93l-AoZLCls,1519
-metaflow/cli_components/run_cmds.py,sha256=FHXqYyGXlr4m2CVOoBagfPgKny4nu2L_QReVgBGVRVA,10530
+metaflow/cli_components/run_cmds.py,sha256=gAJeiuFt5y1IcVclwH6TV9p-JCy6-rDtEKUj2ioMO0s,11304
 metaflow/cli_components/step_cmd.py,sha256=zGJgTv7wxrv34nWDi__CHaC2eS6kItR95EdVGJX803w,4766
 metaflow/cli_components/utils.py,sha256=gpoDociadjnJD7MuiJup_MDR02ZJjjleejr0jPBu29c,6057
 metaflow/client/__init__.py,sha256=1GtQB4Y_CBkzaxg32L1syNQSlfj762wmLrfrDxGi1b8,226
@@ -300,7 +300,7 @@ metaflow/plugins/metadata_providers/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uz
 metaflow/plugins/metadata_providers/local.py,sha256=Z0CXaGZJbAkj4II3WspJi-uCCtShH64yaXZQ5i9Ym7g,24390
 metaflow/plugins/metadata_providers/service.py,sha256=K0Ym6lcmegX6wBC5uZbeAFQJSDFc8e6DzJiCB1VIqjc,22554
 metaflow/plugins/pypi/__init__.py,sha256=0YFZpXvX7HCkyBFglatual7XGifdA1RwC3U4kcizyak,1037
-metaflow/plugins/pypi/bootstrap.py,sha256=bG9bHf5C_2a5AsDiiauTX7S4BH3C-GSXCKDEQdgDEJU,14959
+metaflow/plugins/pypi/bootstrap.py,sha256=oRfJkAp99R338nYX2wq4FrV0Ax5h4QqqifNmAXrM3CY,14892
 metaflow/plugins/pypi/conda_decorator.py,sha256=piFcE4uGmWhhbGlxMK0GHd7BGEyqy6r9BFy8Mjoi80Q,15937
 metaflow/plugins/pypi/conda_environment.py,sha256=d5BAiY_aJJdlJ5h3N5nGSDmVoOY-8BVKqEbA5nrCpCY,22113
 metaflow/plugins/pypi/micromamba.py,sha256=LLJ2dGGOEyld07W8iI6dtE01h2Y1PQnBhU-dMBssZ3c,16502
@@ -360,9 +360,9 @@ metaflow/user_configs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3h
 metaflow/user_configs/config_decorators.py,sha256=qCKVAvd0NKgaCxQ2OThes5-DYHXq6A1HqURubYNeFdw,20481
 metaflow/user_configs/config_options.py,sha256=m6jccSpzI4qUJ7vyYkYBIf8G3V0Caunxg_k7zg4Zlqg,21067
 metaflow/user_configs/config_parameters.py,sha256=oeJGVKu1ao_YQX6Lg6P2FEv5k5-_F4sARLlVpTW9ezM,15502
-metaflow-2.14.1.dist-info/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
-metaflow-2.14.1.dist-info/METADATA,sha256=s4WRKmdEujHhp-jJSvjY2JBNVuwTmHQlomqc7PHrJB8,6121
-metaflow-2.14.1.dist-info/WHEEL,sha256=9Hm2OB-j1QcCUq9Jguht7ayGIIZBRTdOXD1qg9cCgPM,109
-metaflow-2.14.1.dist-info/entry_points.txt,sha256=IKwTN1T3I5eJL3uo_vnkyxVffcgnRdFbKwlghZfn27k,57
-metaflow-2.14.1.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
-metaflow-2.14.1.dist-info/RECORD,,
+metaflow-2.14.3.dist-info/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
+metaflow-2.14.3.dist-info/METADATA,sha256=rHc2E2ro1WrUOXzDZvDnLOtEGk8tbknNlSo-GbpIwII,6121
+metaflow-2.14.3.dist-info/WHEEL,sha256=9Hm2OB-j1QcCUq9Jguht7ayGIIZBRTdOXD1qg9cCgPM,109
+metaflow-2.14.3.dist-info/entry_points.txt,sha256=IKwTN1T3I5eJL3uo_vnkyxVffcgnRdFbKwlghZfn27k,57
+metaflow-2.14.3.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
+metaflow-2.14.3.dist-info/RECORD,,