metaflow 2.12.23__tar.gz → 2.12.24__tar.gz

{metaflow-2.12.23/metaflow.egg-info → metaflow-2.12.24}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: metaflow
-Version: 2.12.23
+Version: 2.12.24
 Summary: Metaflow: More Data Science, Less Engineering
 Author: Metaflow Developers
 Author-email: help@metaflow.org
@@ -26,7 +26,7 @@ License-File: LICENSE
 Requires-Dist: requests
 Requires-Dist: boto3
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.12.23; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.12.24; extra == "stubs"
 
 ![Metaflow_Logo_Horizontal_FullColor_Ribbon_Dark_RGB](https://user-images.githubusercontent.com/763451/89453116-96a57e00-d713-11ea-9fa6-82b29d4d6eff.png)
 

{metaflow-2.12.23 → metaflow-2.12.24}/metaflow/plugins/argo/argo_workflows.py

@@ -2633,6 +2633,9 @@ class ArgoWorkflows(object):
 
         return (
             DaemonTemplate("heartbeat-daemon")
+            # NOTE: Even though a retry strategy does not work for Argo daemon containers,
+            # this has the side-effect of protecting the exit hooks of the workflow from failing in case the daemon container errors out.
+            .retry_strategy(10, 1)
             .service_account_name(resources["service_account"])
             .container(
                 to_camelcase(
@@ -3304,6 +3307,15 @@ class DaemonTemplate(object):
         self.payload["serviceAccountName"] = service_account_name
         return self
 
+    def retry_strategy(self, times, minutes_between_retries):
+        if times > 0:
+            self.payload["retryStrategy"] = {
+                "retryPolicy": "Always",
+                "limit": times,
+                "backoff": {"duration": "%sm" % minutes_between_retries},
+            }
+        return self
+
     def to_json(self):
         return self.payload
 

{metaflow-2.12.23 → metaflow-2.12.24}/metaflow/plugins/argo/argo_workflows_cli.py

@@ -179,7 +179,7 @@ def argo_workflows(obj, name=None):
 )
 @click.option(
     "--enable-heartbeat-daemon/--no-enable-heartbeat-daemon",
-    default=True,
+    default=False,
     show_default=True,
     help="Use a daemon container to broadcast heartbeats.",
 )

{metaflow-2.12.23 → metaflow-2.12.24}/metaflow/plugins/kubernetes/kubernetes_client.py

@@ -121,10 +121,7 @@ class KubernetesClient(object):
         job_api = self._client.BatchV1Api()
         pods = self._find_active_pods(flow_name, run_id, user)
 
-        active_pods = False
-
         def _kill_pod(pod):
-            active_pods = True
             echo("Killing Kubernetes pod %s\n" % pod.metadata.name)
             try:
                 stream(
@@ -158,10 +155,10 @@ class KubernetesClient(object):
                     echo("failed to kill pod %s - %s" % (pod.metadata.name, str(e)))
 
         with ThreadPoolExecutor() as executor:
-            executor.map(_kill_pod, pods)
+            operated_pods = list(executor.map(_kill_pod, pods))
 
-        if not active_pods:
-            echo("No active Kubernetes pods found for run *%s*" % run_id)
+            if not operated_pods:
+                echo("No active Kubernetes pods found for run *%s*" % run_id)
 
     def jobset(self, **kwargs):
         return KubernetesJobSet(self, **kwargs)

{metaflow-2.12.23 → metaflow-2.12.24}/metaflow/task.py

@@ -180,6 +180,10 @@ class MetaflowTask(object):
         # 2) join - pop the topmost frame from the stack
         # 3) step following a split - push a new frame in the stack
 
+        # We have a non-modifying case (case 4)) where we propagate the
+        # foreach-stack information to all tasks in the foreach. This is
+        # then used later to write the foreach-stack metadata for that task
+
         # case 1) - reset the stack
         if step_name == "start":
             self.flow._foreach_stack = []
@@ -264,6 +268,9 @@ class MetaflowTask(object):
             stack = inputs[0]["_foreach_stack"]
             stack.append(frame)
             self.flow._foreach_stack = stack
+        # case 4) - propagate in the foreach nest
+        elif "_foreach_stack" in inputs[0]:
+            self.flow._foreach_stack = inputs[0]["_foreach_stack"]
 
     def _clone_flow(self, datastore):
         x = self.flow.__class__(use_cli=False)

metaflow-2.12.24/metaflow/version.py

@@ -0,0 +1 @@
+metaflow_version = "2.12.24"

{metaflow-2.12.23 → metaflow-2.12.24/metaflow.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: metaflow
-Version: 2.12.23
+Version: 2.12.24
 Summary: Metaflow: More Data Science, Less Engineering
 Author: Metaflow Developers
 Author-email: help@metaflow.org
@@ -26,7 +26,7 @@ License-File: LICENSE
 Requires-Dist: requests
 Requires-Dist: boto3
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.12.23; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.12.24; extra == "stubs"
 
 ![Metaflow_Logo_Horizontal_FullColor_Ribbon_Dark_RGB](https://user-images.githubusercontent.com/763451/89453116-96a57e00-d713-11ea-9fa6-82b29d4d6eff.png)
 

{metaflow-2.12.23 → metaflow-2.12.24}/metaflow.egg-info/requires.txt

@@ -2,4 +2,4 @@ requests
 boto3
 
 [stubs]
-metaflow-stubs==2.12.23
+metaflow-stubs==2.12.24

metaflow-2.12.23/metaflow/version.py

@@ -1 +0,0 @@
-metaflow_version = "2.12.23"