metaflow 2.15.17__py2.py3-none-any.whl → 2.15.18__py2.py3-none-any.whl

metaflow/cli.py

@@ -1,5 +1,6 @@
 import functools
 import inspect
+import os
 import sys
 import traceback
 from datetime import datetime
@@ -242,6 +243,14 @@ def version(obj):
     type=click.Choice(["local"] + [m.TYPE for m in ENVIRONMENTS]),
     help="Execution environment type",
 )
+@click.option(
+    "--force-rebuild-environments/--no-force-rebuild-environments",
+    is_flag=True,
+    default=False,
+    hidden=True,
+    type=bool,
+    help="Explicitly rebuild the execution environments",
+)
 # See comment for --quiet
 @click.option(
     "--datastore",
@@ -300,6 +309,7 @@ def start(
     quiet=False,
     metadata=None,
     environment=None,
+    force_rebuild_environments=False,
     datastore=None,
     datastore_root=None,
     decospecs=None,
@@ -435,6 +445,8 @@ def start(
     ctx.obj.environment = [
         e for e in ENVIRONMENTS + [MetaflowEnvironment] if e.TYPE == environment
     ][0](ctx.obj.flow)
+    # set force rebuild flag for environments that support it.
+    ctx.obj.environment._force_rebuild = force_rebuild_environments
     ctx.obj.environment.validate_environment(ctx.obj.logger, datastore)
 
     ctx.obj.event_logger = LOGGING_SIDECARS[event_logger](

metaflow/metaflow_config.py

@@ -214,8 +214,6 @@ CARD_GSROOT = from_conf(
 )
 CARD_NO_WARNING = from_conf("CARD_NO_WARNING", False)
 
-SKIP_CARD_DUALWRITE = from_conf("SKIP_CARD_DUALWRITE", False)
-
 RUNTIME_CARD_RENDER_INTERVAL = from_conf("RUNTIME_CARD_RENDER_INTERVAL", 60)
 
 # Azure storage account URL

metaflow/plugins/cards/card_datastore.py

@@ -16,7 +16,6 @@ from metaflow.metaflow_config import (
     CARD_SUFFIX,
     CARD_AZUREROOT,
     CARD_GSROOT,
-    SKIP_CARD_DUALWRITE,
 )
 import metaflow.metaflow_config as metaflow_config
 
@@ -231,23 +230,6 @@ class CardDatastore(object):
 
     def save_card(self, uuid, card_type, card_html, card_id=None, overwrite=True):
         card_file_name = card_type
-        # TEMPORARY_WORKAROUND: FIXME (LATER) : Fix the duplication of below block in a few months.
-        # Check file blame to understand the age of this temporary workaround.
-
-        # This function will end up saving cards at two locations.
-        # Thereby doubling the number of cards. (Which is a temporary fix)
-        # Why do this ? :
-        # When cards were introduced there was an assumption made about task-ids being unique.
-        # This assumption was incorrect.
-        # Only the pathspec needs to be unique but there is no such guarantees about task-ids.
-        # When task-ids are non-unique, card read would result in finding incorrect cards.
-        # This happens because cards were stored based on task-ids.
-        # If we immediately switch from storing based on task-ids to a step-name abstraction folder,
-        # then card reading will crash for many users.
-        # It would especially happen for users who are accessing cards created by a newer
-        # MF client from an older version of MF client.
-        # It will also easily end up breaking the metaflow-ui (which maybe using a client from an older version).
-        # Hence, we are writing cards to both paths so that we can introduce breaking changes later in the future.
         card_path_with_steps = self.get_card_location(
             self._get_card_write_path(),
             card_file_name,
@@ -255,24 +237,10 @@ class CardDatastore(object):
             card_id=card_id,
             suffix=CardNameSuffix.CARD,
         )
-        if SKIP_CARD_DUALWRITE:
-            self._backend.save_bytes(
-                [(card_path_with_steps, BytesIO(bytes(card_html, "utf-8")))],
-                overwrite=overwrite,
-            )
-        else:
-            card_path_without_steps = self.get_card_location(
-                self._get_card_read_path(with_steps=False),
-                card_file_name,
-                uuid,
-                card_id=card_id,
-                suffix=CardNameSuffix.CARD,
-            )
-            for cp in [card_path_with_steps, card_path_without_steps]:
-                self._backend.save_bytes(
-                    [(cp, BytesIO(bytes(card_html, "utf-8")))], overwrite=overwrite
-                )
-
+        self._backend.save_bytes(
+            [(card_path_with_steps, BytesIO(bytes(card_html, "utf-8")))],
+            overwrite=overwrite,
+        )
         return self.info_from_path(card_path_with_steps, suffix=CardNameSuffix.CARD)
 
     def _list_card_paths(self, card_type=None, card_hash=None, card_id=None):
@@ -283,6 +251,10 @@ class CardDatastore(object):
         )
 
         if len(card_paths_with_steps) == 0:
+            # The listing logic is reading the cards with steps and without steps
+            # because earlier versions of clients (ones that wrote cards before June 2022),
+            # would have written cards without steps. So as a fallback we will try to check for the
+            # cards without steps.
             card_paths_without_steps = self._backend.list_content(
                 [self._get_card_read_path(with_steps=False)]
             )

metaflow/plugins/pypi/conda_environment.py

@@ -32,6 +32,7 @@ class CondaEnvironmentException(MetaflowException):
 class CondaEnvironment(MetaflowEnvironment):
     TYPE = "conda"
     _filecache = None
+    _force_rebuild = False
 
     def __init__(self, flow):
         self.flow = flow
@@ -71,7 +72,7 @@ class CondaEnvironment(MetaflowEnvironment):
         self.logger = make_thread_safe(logger)
 
         # TODO: Wire up logging
-        micromamba = Micromamba(self.logger)
+        micromamba = Micromamba(self.logger, self._force_rebuild)
         self.solvers = {"conda": micromamba, "pypi": Pip(micromamba, self.logger)}
 
     def init_environment(self, echo, only_steps=None):
@@ -107,7 +108,10 @@ class CondaEnvironment(MetaflowEnvironment):
             return (
                 id_,
                 (
-                    self.read_from_environment_manifest([id_, platform, type_])
+                    (
+                        not self._force_rebuild
+                        and self.read_from_environment_manifest([id_, platform, type_])
+                    )
                     or self.write_to_environment_manifest(
                         [id_, platform, type_],
                         self.solvers[type_].solve(id_, **environment),
@@ -153,7 +157,7 @@ class CondaEnvironment(MetaflowEnvironment):
             _meta = copy.deepcopy(local_packages)
             for id_, packages, _, _ in results:
                 for package in packages:
-                    if package.get("path"):
+                    if package.get("path") and not self._force_rebuild:
                         # Cache only those packages that manifest is unaware of
                         local_packages.pop(package["url"], None)
                     else:
@@ -186,7 +190,7 @@ class CondaEnvironment(MetaflowEnvironment):
             storage.save_bytes(
                 list_of_path_and_filehandle,
                 len_hint=len(list_of_path_and_filehandle),
-                # overwrite=True,
+                overwrite=self._force_rebuild,
             )
             for id_, packages, _, platform in results:
                 if id_ in dirty:

metaflow/plugins/pypi/micromamba.py

@@ -2,6 +2,7 @@ import functools
 import json
 import os
 import re
+import shutil
 import subprocess
 import tempfile
 import time
@@ -30,7 +31,7 @@ _double_equal_match = re.compile("==(?=[<=>!~])")
 
 
 class Micromamba(object):
-    def __init__(self, logger=None):
+    def __init__(self, logger=None, force_rebuild=False):
         # micromamba is a tiny version of the mamba package manager and comes with
         # metaflow specific performance enhancements.
 
@@ -60,6 +61,8 @@ class Micromamba(object):
         # which causes a race condition in case micromamba needs to be installed first.
         self.install_mutex = Lock()
 
+        self.force_rebuild = force_rebuild
+
     @property
     def bin(self) -> str:
         "Defer installing Micromamba until when the binary path is actually requested"
@@ -152,6 +155,11 @@ class Micromamba(object):
             keyword="metaflow",  # indicates metaflow generated environment
             id=id_,
         )
+        # If we are forcing a rebuild of the environment, we make sure to remove existing files beforehand.
+        # This is to ensure that no irrelevant packages get bundled relative to the resolved environment.
+        # NOTE: download always happens before create, so we want to do the cleanup here instead.
+        if self.force_rebuild:
+            shutil.rmtree(self.path_to_environment(id_, platform), ignore_errors=True)
 
         # cheap check
         if os.path.exists(f"{prefix}/fake.done"):

metaflow/version.py

@@ -1 +1 @@
-metaflow_version = "2.15.17"
+metaflow_version = "2.15.18"

{metaflow-2.15.17.dist-info → metaflow-2.15.18.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: metaflow
-Version: 2.15.17
+Version: 2.15.18
 Summary: Metaflow: More AI and ML, Less Engineering
 Author: Metaflow Developers
 Author-email: help@metaflow.org
@@ -26,7 +26,7 @@ License-File: LICENSE
 Requires-Dist: requests
 Requires-Dist: boto3
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.15.17; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.15.18; extra == "stubs"
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier

{metaflow-2.15.17.dist-info → metaflow-2.15.18.dist-info}/RECORD

@@ -1,7 +1,7 @@
 metaflow/R.py,sha256=CqVfIatvmjciuICNnoyyNGrwE7Va9iXfLdFbQa52hwA,3958
 metaflow/__init__.py,sha256=aCCgR992PUA5Urd-pb06c1afjKbfaNPDWnH8kKQRnNk,5937
 metaflow/cards.py,sha256=IbRmredvmFEU0V6JL7DR8wCESwVmmZJubr6x24bo7U4,442
-metaflow/cli.py,sha256=5nqEJf0R9MDdJUpIUpOxbc2dovyw_3HlbJqXmHYVCjc,21323
+metaflow/cli.py,sha256=ype7221pgVp3sFXTT4YpLmEx0yLahYD9UuHizLCbIT0,21714
 metaflow/cli_args.py,sha256=hDsdWdRmfXYifVGq6b6FDfgoWxtIG2nr_lU6EBV0Pnk,3584
 metaflow/clone_util.py,sha256=LSuVbFpPUh92UW32DBcnZbL0FFw-4w3CLa0tpEbCkzk,2066
 metaflow/cmd_with_io.py,sha256=kl53HkAIyv0ecpItv08wZYczv7u3msD1VCcciqigqf0,588
@@ -16,7 +16,7 @@ metaflow/includefile.py,sha256=RtISGl1V48qjkJBakUZ9yPpHV102h7pOIFiKP8PLHpc,20927
 metaflow/info_file.py,sha256=wtf2_F0M6dgiUu74AFImM8lfy5RrUw5Yj7Rgs2swKRY,686
 metaflow/integrations.py,sha256=LlsaoePRg03DjENnmLxZDYto3NwWc9z_PtU6nJxLldg,1480
 metaflow/lint.py,sha256=x4p6tnRzYqNNniCGXyrUW0WuYfTUgnaOMRivxvnxask,11661
-metaflow/metaflow_config.py,sha256=wrWSdPV6pAWFHo0Ao6uMnK2_zCaTD3S8-CYvxC5onPo,24045
+metaflow/metaflow_config.py,sha256=wVREyuJvQ8EyWDhkQ8c_A11dh1CcIA5HFjohp7CEsHE,23982
 metaflow/metaflow_config_funcs.py,sha256=5GlvoafV6SxykwfL8D12WXSfwjBN_NsyuKE_Q3gjGVE,6738
 metaflow/metaflow_current.py,sha256=pfkXmkyHeMJhxIs6HBJNBEaBDpcl5kz9Wx5mW6F_3qo,7164
 metaflow/metaflow_environment.py,sha256=CWG90qpfz9iJ6hHhFlAmMVNALn2v_5eTVk3mFbQR4Pw,8379
@@ -37,7 +37,7 @@ metaflow/tuple_util.py,sha256=_G5YIEhuugwJ_f6rrZoelMFak3DqAR2tt_5CapS1XTY,830
 metaflow/unbounded_foreach.py,sha256=p184WMbrMJ3xKYHwewj27ZhRUsSj_kw1jlye5gA9xJk,387
 metaflow/util.py,sha256=MCXCjcGwpuR7y9euBf1GTNRHPtlh6fCpdPMEtbULefw,14554
 metaflow/vendor.py,sha256=EDZokNMrx1PU07jNMiWFMFtC7TL03pMXZ1kKn13k-2g,5139
-metaflow/version.py,sha256=NkuKBUnu2oRi4r7ARbML2YLGujb0iUN9knslDNptHkM,29
+metaflow/version.py,sha256=aFD_zYUJd11BSUf4DN7nX_rYimfrx-sNGZXun2I_e-g,29
 metaflow/_vendor/__init__.py,sha256=y_CiwUD3l4eAKvTVDZeqgVujMy31cAM1qjAB-HfI-9s,353
 metaflow/_vendor/typing_extensions.py,sha256=q9zxWa6p6CzF1zZvSkygSlklduHf_b3K7MCxGz7MJRc,134519
 metaflow/_vendor/zipp.py,sha256=ajztOH-9I7KA_4wqDYygtHa6xUBVZgFpmZ8FE74HHHI,8425
@@ -248,7 +248,7 @@ metaflow/plugins/cards/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3
 metaflow/plugins/cards/card_cli.py,sha256=XyiU5f6i_xjFHPyowWGs9mKQm9fm3dK6id-IKsoe9cY,35177
 metaflow/plugins/cards/card_client.py,sha256=30dFBoC3axc261GeV7QCIs_V1OHhRtS31S0wEWsjw90,9501
 metaflow/plugins/cards/card_creator.py,sha256=vRz1EUFa4xQ1fUIWzqyACViC6D7KGFaq5XlLIEssXTI,7741
-metaflow/plugins/cards/card_datastore.py,sha256=3K19wE0CZVvOpuYUytftIYYnHHn3pMZJE87FMD6OYlM,14244
+metaflow/plugins/cards/card_datastore.py,sha256=bPLjBFWNlAgIjONAb0IjYXMmp9vMYlh3EYhRjAlDA0U,12714
 metaflow/plugins/cards/card_decorator.py,sha256=nU-uZ4_ubU7bCsSb6uT61YnN8ruo43_FzqZ24MsZpKg,12070
 metaflow/plugins/cards/card_resolver.py,sha256=bjyujYpGUFbLJNwXNGHlHhL4f-gVdVKebl7XW1vWDtE,717
 metaflow/plugins/cards/card_server.py,sha256=DHv0RcepaPULWbkDahiEMrU5A281Cfb0DvHLUYd8JsU,11772
@@ -327,8 +327,8 @@ metaflow/plugins/metadata_providers/service.py,sha256=5UlK0R5M9_nq2J6MgJgCZwqAC3
 metaflow/plugins/pypi/__init__.py,sha256=0YFZpXvX7HCkyBFglatual7XGifdA1RwC3U4kcizyak,1037
 metaflow/plugins/pypi/bootstrap.py,sha256=NaQboUVNJc90gmUa69-cK_T3G7piHNNsUSaN96IJoic,14745
 metaflow/plugins/pypi/conda_decorator.py,sha256=N0HGiaS1mRsa6qT4eYzu2C3DHtas22QIXowW4vEl44M,15961
-metaflow/plugins/pypi/conda_environment.py,sha256=C7WolJm1dgBQofd6vOk8KCr3U754TyAr4bOpkhosxfM,24582
-metaflow/plugins/pypi/micromamba.py,sha256=XAuZulI7O2Q_Zgtx28yjAqHco6tHiPQxoy_vsf1kWm8,16843
+metaflow/plugins/pypi/conda_environment.py,sha256=6QgHphwgXBob_C_ObpKL-4i2R7dN3XPn0ddlPCzu5tc,24771
+metaflow/plugins/pypi/micromamba.py,sha256=UltfY8NmLphfZ-AbpaMFIdxIeOXLdTYDrMrabvPrYVU,17352
 metaflow/plugins/pypi/parsers.py,sha256=gpOOG2Ph95wI73MWCAi7XjpK0gYhv5k5YIGBs73QPuE,8556
 metaflow/plugins/pypi/pip.py,sha256=WhPyA18RoVT40sqbZRJdCEij4euL9PZwLfm1SrAKqmU,14333
 metaflow/plugins/pypi/pypi_decorator.py,sha256=ybNgo-T5Z_0W2KNuED0pdjyI0qygZ4a1MXAzKqdHt_E,7250
@@ -389,12 +389,12 @@ metaflow/user_configs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3h
 metaflow/user_configs/config_decorators.py,sha256=n4OTbfuaXejPVLHnXXgVlNJNcrSvWrVOdXCMcqvQlik,20512
 metaflow/user_configs/config_options.py,sha256=yyaiKLB4QWlR5AaTg9I7GoBv-OlffOFdGPe1BL85Llc,21241
 metaflow/user_configs/config_parameters.py,sha256=Eyiqcz4YV_z4algDHAh2gaejGFgIdHk8Vix9AUdPSh0,20989
-metaflow-2.15.17.data/data/share/metaflow/devtools/Makefile,sha256=5n89OGIC_kE4wxtEI66VCucN-b-1w5bqvGeZYmeRGz8,13737
-metaflow-2.15.17.data/data/share/metaflow/devtools/Tiltfile,sha256=P5_rn_F3xYLN1_cEAQ9mNeS22HG2rb8beKIz2RIK6fU,20634
-metaflow-2.15.17.data/data/share/metaflow/devtools/pick_services.sh,sha256=DCnrMXwtApfx3B4S-YiZESMyAFHbXa3VuNL0MxPLyiE,2196
-metaflow-2.15.17.dist-info/licenses/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
-metaflow-2.15.17.dist-info/METADATA,sha256=7DfiQK2zg33YgaYjF6Oc5hEHfRgf7js9w-22xvqojFE,6742
-metaflow-2.15.17.dist-info/WHEEL,sha256=JNWh1Fm1UdwIQV075glCn4MVuCRs0sotJIq-J6rbxCU,109
-metaflow-2.15.17.dist-info/entry_points.txt,sha256=RvEq8VFlgGe_FfqGOZi0D7ze1hLD0pAtXeNyGfzc_Yc,103
-metaflow-2.15.17.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
-metaflow-2.15.17.dist-info/RECORD,,
+metaflow-2.15.18.data/data/share/metaflow/devtools/Makefile,sha256=5n89OGIC_kE4wxtEI66VCucN-b-1w5bqvGeZYmeRGz8,13737
+metaflow-2.15.18.data/data/share/metaflow/devtools/Tiltfile,sha256=P5_rn_F3xYLN1_cEAQ9mNeS22HG2rb8beKIz2RIK6fU,20634
+metaflow-2.15.18.data/data/share/metaflow/devtools/pick_services.sh,sha256=DCnrMXwtApfx3B4S-YiZESMyAFHbXa3VuNL0MxPLyiE,2196
+metaflow-2.15.18.dist-info/licenses/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
+metaflow-2.15.18.dist-info/METADATA,sha256=SNMazx7rUE5-fmaqxAv1-7KxPXKDAojZyRqLBCzLZEo,6742
+metaflow-2.15.18.dist-info/WHEEL,sha256=JNWh1Fm1UdwIQV075glCn4MVuCRs0sotJIq-J6rbxCU,109
+metaflow-2.15.18.dist-info/entry_points.txt,sha256=RvEq8VFlgGe_FfqGOZi0D7ze1hLD0pAtXeNyGfzc_Yc,103
+metaflow-2.15.18.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
+metaflow-2.15.18.dist-info/RECORD,,