metaflow 2.12.23__py2.py3-none-any.whl → 2.12.25__py2.py3-none-any.whl

metaflow/plugins/argo/argo_workflows.py

@@ -2633,6 +2633,9 @@ class ArgoWorkflows(object):
 
         return (
             DaemonTemplate("heartbeat-daemon")
+            # NOTE: Even though a retry strategy does not work for Argo daemon containers,
+            # this has the side-effect of protecting the exit hooks of the workflow from failing in case the daemon container errors out.
+            .retry_strategy(10, 1)
             .service_account_name(resources["service_account"])
             .container(
                 to_camelcase(
@@ -3304,6 +3307,15 @@ class DaemonTemplate(object):
         self.payload["serviceAccountName"] = service_account_name
         return self
 
+    def retry_strategy(self, times, minutes_between_retries):
+        if times > 0:
+            self.payload["retryStrategy"] = {
+                "retryPolicy": "Always",
+                "limit": times,
+                "backoff": {"duration": "%sm" % minutes_between_retries},
+            }
+        return self
+
     def to_json(self):
         return self.payload
 

metaflow/plugins/argo/argo_workflows_cli.py

@@ -179,7 +179,7 @@ def argo_workflows(obj, name=None):
 )
 @click.option(
     "--enable-heartbeat-daemon/--no-enable-heartbeat-daemon",
-    default=True,
+    default=False,
     show_default=True,
     help="Use a daemon container to broadcast heartbeats.",
 )

metaflow/plugins/kubernetes/kubernetes_client.py

@@ -121,10 +121,7 @@ class KubernetesClient(object):
         job_api = self._client.BatchV1Api()
         pods = self._find_active_pods(flow_name, run_id, user)
 
-        active_pods = False
-
         def _kill_pod(pod):
-            active_pods = True
             echo("Killing Kubernetes pod %s\n" % pod.metadata.name)
             try:
                 stream(
@@ -158,10 +155,10 @@ class KubernetesClient(object):
                     echo("failed to kill pod %s - %s" % (pod.metadata.name, str(e)))
 
         with ThreadPoolExecutor() as executor:
-            executor.map(_kill_pod, pods)
+            operated_pods = list(executor.map(_kill_pod, pods))
 
-        if not active_pods:
-            echo("No active Kubernetes pods found for run *%s*" % run_id)
+            if not operated_pods:
+                echo("No active Kubernetes pods found for run *%s*" % run_id)
 
     def jobset(self, **kwargs):
         return KubernetesJobSet(self, **kwargs)

metaflow/plugins/pypi/micromamba.py

@@ -270,15 +270,22 @@ class Micromamba(object):
                         except ValueError:
                             vpkg_version = None
                         raise MicromambaException(
-                            "Please set the environment variable CONDA_OVERRIDE_{var} to a specific version{version} of {name}.\n"
-                            "Here is an example of supplying environment variables through the command line -\n\n"
+                            "{msg}\n\n"
+                            "*Please set the environment variable CONDA_OVERRIDE_{var} to a specific version{version} of {name}.*\n\n"
+                            "Here is an example of supplying environment variables through the command line\n"
                             "CONDA_OVERRIDE_{var}=<{name}-version> python flow.py <args>".format(
+                                msg=msg.format(
+                                    cmd=" ".join(e.cmd),
+                                    code=e.returncode,
+                                    output=e.output.decode(),
+                                    stderr=error,
+                                ),
                                 var=vpkg_name.upper(),
                                 version=(
-                                    "" if not vpkg_version else (" (%s)" % vpkg_version)
+                                    "" if not vpkg_version else f" ({vpkg_version})"
                                 ),
                                 name=vpkg_name,
-                            ),
+                            )
                         )
                     err.append(error)
                 raise MicromambaException(

metaflow/plugins/pypi/pip.py

@@ -140,7 +140,10 @@ class Pip(object):
         metadata_file = METADATA_FILE.format(prefix=prefix)
         # download packages only if they haven't ever been downloaded before
         if os.path.isfile(metadata_file):
-            return
+            with open(metadata_file, "r") as file:
+                metadata = json.load(file)
+                if all(package["url"] in metadata for package in packages):
+                    return
 
         metadata = {}
         custom_index_url, extra_index_urls = self.indices(prefix)

metaflow/task.py

@@ -180,6 +180,10 @@ class MetaflowTask(object):
         # 2) join - pop the topmost frame from the stack
         # 3) step following a split - push a new frame in the stack
 
+        # We have a non-modifying case (case 4)) where we propagate the
+        # foreach-stack information to all tasks in the foreach. This is
+        # then used later to write the foreach-stack metadata for that task
+
         # case 1) - reset the stack
         if step_name == "start":
             self.flow._foreach_stack = []
@@ -264,6 +268,9 @@ class MetaflowTask(object):
             stack = inputs[0]["_foreach_stack"]
             stack.append(frame)
             self.flow._foreach_stack = stack
+        # case 4) - propagate in the foreach nest
+        elif "_foreach_stack" in inputs[0]:
+            self.flow._foreach_stack = inputs[0]["_foreach_stack"]
 
     def _clone_flow(self, datastore):
         x = self.flow.__class__(use_cli=False)

metaflow/version.py

@@ -1 +1 @@
-metaflow_version = "2.12.23"
+metaflow_version = "2.12.25"

{metaflow-2.12.23.dist-info → metaflow-2.12.25.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: metaflow
-Version: 2.12.23
+Version: 2.12.25
 Summary: Metaflow: More Data Science, Less Engineering
 Author: Metaflow Developers
 Author-email: help@metaflow.org
@@ -26,7 +26,7 @@ License-File: LICENSE
 Requires-Dist: requests
 Requires-Dist: boto3
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.12.23; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.12.25; extra == "stubs"
 
 ![Metaflow_Logo_Horizontal_FullColor_Ribbon_Dark_RGB](https://user-images.githubusercontent.com/763451/89453116-96a57e00-d713-11ea-9fa6-82b29d4d6eff.png)
 

{metaflow-2.12.23.dist-info → metaflow-2.12.25.dist-info}/RECORD

@@ -31,12 +31,12 @@ metaflow/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 metaflow/pylint_wrapper.py,sha256=zzBY9YaSUZOGH-ypDKAv2B_7XcoyMZj-zCoCrmYqNRc,2865
 metaflow/runtime.py,sha256=fbBObJJciagHWPzR3T7x9e_jez_RBnLZIHsXMvYnW_M,68875
 metaflow/tagging_util.py,sha256=ctyf0Q1gBi0RyZX6J0e9DQGNkNHblV_CITfy66axXB4,2346
-metaflow/task.py,sha256=uJHl8K4n3jNllWHSsG1vAZtDza0U2QbQcdg9GS_YPBE,28660
+metaflow/task.py,sha256=5-Qy6wAm7dt7DnhnX1KVhVAKl4DWb3IZWIN5YCdRGIg,29043
 metaflow/tuple_util.py,sha256=_G5YIEhuugwJ_f6rrZoelMFak3DqAR2tt_5CapS1XTY,830
 metaflow/unbounded_foreach.py,sha256=p184WMbrMJ3xKYHwewj27ZhRUsSj_kw1jlye5gA9xJk,387
 metaflow/util.py,sha256=olAvJK3y1it_k99MhLulTaAJo7OFVt5rnrD-ulIFLCU,13616
 metaflow/vendor.py,sha256=FchtA9tH22JM-eEtJ2c9FpUdMn8sSb1VHuQS56EcdZk,5139
-metaflow/version.py,sha256=4Kj4wLDHQlSoABrTJYqXZSqf1PE_WoOTlHFEMNeYCyI,29
+metaflow/version.py,sha256=YNnU6aijCl2Es_80zvwrPAhbV1OF5FlO1xdfXYRyWuA,29
 metaflow/_vendor/__init__.py,sha256=y_CiwUD3l4eAKvTVDZeqgVujMy31cAM1qjAB-HfI-9s,353
 metaflow/_vendor/typing_extensions.py,sha256=0nUs5p1A_UrZigrAVBoOEM6TxU37zzPDUtiij1ZwpNc,110417
 metaflow/_vendor/zipp.py,sha256=ajztOH-9I7KA_4wqDYygtHa6xUBVZgFpmZ8FE74HHHI,8425
@@ -175,8 +175,8 @@ metaflow/plugins/airflow/sensors/s3_sensor.py,sha256=iDReG-7FKnumrtQg-HY6cCUAAqN
 metaflow/plugins/argo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 metaflow/plugins/argo/argo_client.py,sha256=KTUpP0DmnmNsMp4tbdNyKX_zOdTFRVpUkrf7Vv79d-o,16011
 metaflow/plugins/argo/argo_events.py,sha256=_C1KWztVqgi3zuH57pInaE9OzABc2NnncC-zdwOMZ-w,5909
-metaflow/plugins/argo/argo_workflows.py,sha256=a-ew9ZwBq9nw9W9fCTLIcOlhnuKZZBTbBcyyofkjqWk,173003
-metaflow/plugins/argo/argo_workflows_cli.py,sha256=E_PyhOtxuS2F8DwhBANsRZCMxpeZ5rfID8eksfSOPm8,37231
+metaflow/plugins/argo/argo_workflows.py,sha256=2todjgM06mkqkXn4_-7ume7AknIqr3ehZNTikJU4bBM,173579
+metaflow/plugins/argo/argo_workflows_cli.py,sha256=0qAGo0YlC1Y9-1zqYAzhVCpCcITotfOI421VOIRpseM,37232
 metaflow/plugins/argo/argo_workflows_decorator.py,sha256=yprszMdbE3rBTcEA9VR0IEnPjTprUauZBc4SBb-Q7sA,7878
 metaflow/plugins/argo/argo_workflows_deployer.py,sha256=wSSZtThn_VPvE_Wu6NB1L0Q86LmBJh9g009v_lpvBPM,8125
 metaflow/plugins/argo/capture_error.py,sha256=Ys9dscGrTpW-ZCirLBU0gD9qBM0BjxyxGlUMKcwewQc,1852
@@ -282,7 +282,7 @@ metaflow/plugins/kubernetes/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMp
 metaflow/plugins/kubernetes/kube_utils.py,sha256=fYDlvqi8jYPsWijDwT6Z2qhQswyFqv7tiwtic_I80Vg,749
 metaflow/plugins/kubernetes/kubernetes.py,sha256=bKBqgZXnIDkoa4xKtKoV6InPtYQy4CujfvcbQ3Pvsbc,31305
 metaflow/plugins/kubernetes/kubernetes_cli.py,sha256=sFZ9Zrjef85vCO0MGpUF-em8Pw3dePFb3hbX3PtAH4I,13463
-metaflow/plugins/kubernetes/kubernetes_client.py,sha256=y_CswSO_OdDW8eC56lwKwC69JqIfo9tYVw0njXc_sj8,6519
+metaflow/plugins/kubernetes/kubernetes_client.py,sha256=tuvXP-QKpdeSmzVolB2R_TaacOr5DIb0j642eKcjsiM,6491
 metaflow/plugins/kubernetes/kubernetes_decorator.py,sha256=xz2tEIapYWMd9rRiOe8qcYvjRIKT5piWnq-twdySpD8,26031
 metaflow/plugins/kubernetes/kubernetes_job.py,sha256=Cfkee8LbXC17jSXWoeNdomQRvF_8YSeXNg1gvxm6E_M,31806
 metaflow/plugins/kubernetes/kubernetes_jobsets.py,sha256=wb0sK1OxW7pRbKdj6bWB4JsskXDsoKKqjyUWo4N9Y6E,41196
@@ -293,8 +293,8 @@ metaflow/plugins/pypi/__init__.py,sha256=0YFZpXvX7HCkyBFglatual7XGifdA1RwC3U4kci
 metaflow/plugins/pypi/bootstrap.py,sha256=FI-itExqIz7DUzLnnkGwoB60rFBviygpIFThUtqk_4E,5227
 metaflow/plugins/pypi/conda_decorator.py,sha256=fPeXxvmg51oSFTnlguNlcWUIdXHA9OuMnp9ElaxQPFo,15695
 metaflow/plugins/pypi/conda_environment.py,sha256=--q-8lypKupCdGsASpqABNpNqRxtQi6UCDgq8iHDFe4,19476
-metaflow/plugins/pypi/micromamba.py,sha256=HQIxsixkLjqs0ukWGTlATNu5DrbisReOr39Qd21_GZo,13737
-metaflow/plugins/pypi/pip.py,sha256=7B06mPOs5MvY33xbzPVYZlBr1iKMYaN-n8uulL9zSVg,13649
+metaflow/plugins/pypi/micromamba.py,sha256=QaZYMy5w4esW2w_Lb9kZdWU07EtZD_Ky00MVlA4FJw0,14079
+metaflow/plugins/pypi/pip.py,sha256=Uewmt6-meLyPhNLiAOAkDdfd1P4Go07bkQUD0uE5VIs,13827
 metaflow/plugins/pypi/pypi_decorator.py,sha256=rDMbHl7r81Ye7-TuIlKAVJ_CDnfjl9jV44ZPws-UsTY,7229
 metaflow/plugins/pypi/pypi_environment.py,sha256=FYMg8kF3lXqcLfRYWD83a9zpVjcoo_TARqMGZ763rRk,230
 metaflow/plugins/pypi/utils.py,sha256=ds1Mnv_DaxGnLAYp7ozg_K6oyguGyNhvHfE-75Ia1YA,2836
@@ -345,9 +345,9 @@ metaflow/tutorials/07-worldview/README.md,sha256=5vQTrFqulJ7rWN6r20dhot9lI2sVj9W
 metaflow/tutorials/07-worldview/worldview.ipynb,sha256=ztPZPI9BXxvW1QdS2Tfe7LBuVzvFvv0AToDnsDJhLdE,2237
 metaflow/tutorials/08-autopilot/README.md,sha256=GnePFp_q76jPs991lMUqfIIh5zSorIeWznyiUxzeUVE,1039
 metaflow/tutorials/08-autopilot/autopilot.ipynb,sha256=DQoJlILV7Mq9vfPBGW-QV_kNhWPjS5n6SJLqePjFYLY,3191
-metaflow-2.12.23.dist-info/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
-metaflow-2.12.23.dist-info/METADATA,sha256=VWGnCbg946pb6ZQOT9i0GtBrF2dvLDawKzulk-CY2zI,5906
-metaflow-2.12.23.dist-info/WHEEL,sha256=AHX6tWk3qWuce7vKLrj7lnulVHEdWoltgauo8bgCXgU,109
-metaflow-2.12.23.dist-info/entry_points.txt,sha256=IKwTN1T3I5eJL3uo_vnkyxVffcgnRdFbKwlghZfn27k,57
-metaflow-2.12.23.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
-metaflow-2.12.23.dist-info/RECORD,,
+metaflow-2.12.25.dist-info/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
+metaflow-2.12.25.dist-info/METADATA,sha256=DHT3QDndQ40-jE3c6GyPf9QsX5JP9_QfZtIVKKZMV7o,5906
+metaflow-2.12.25.dist-info/WHEEL,sha256=AHX6tWk3qWuce7vKLrj7lnulVHEdWoltgauo8bgCXgU,109
+metaflow-2.12.25.dist-info/entry_points.txt,sha256=IKwTN1T3I5eJL3uo_vnkyxVffcgnRdFbKwlghZfn27k,57
+metaflow-2.12.25.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
+metaflow-2.12.25.dist-info/RECORD,,