ob-metaflow 2.12.23.1__tar.gz → 2.12.24.1__tar.gz

{ob_metaflow-2.12.23.1/ob_metaflow.egg-info → ob_metaflow-2.12.24.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ob-metaflow
-Version: 2.12.23.1
+Version: 2.12.24.1
 Summary: Metaflow: More Data Science, Less Engineering
 Author: Netflix, Outerbounds & the Metaflow Community
 Author-email: help@outerbounds.co
@@ -12,7 +12,7 @@ Requires-Dist: boto3
 Requires-Dist: pylint
 Requires-Dist: kubernetes
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.12.23.1; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.12.24.1; extra == "stubs"
 
 ![Metaflow_Logo_Horizontal_FullColor_Ribbon_Dark_RGB](https://user-images.githubusercontent.com/763451/89453116-96a57e00-d713-11ea-9fa6-82b29d4d6eff.png)
 

{ob_metaflow-2.12.23.1 → ob_metaflow-2.12.24.1}/metaflow/plugins/argo/argo_workflows.py

@@ -51,7 +51,7 @@ from metaflow.metaflow_config import (
     UI_URL,
     PAGERDUTY_TEMPLATE_URL,
 )
-from metaflow.metaflow_config_funcs import config_values
+from metaflow.metaflow_config_funcs import config_values, init_config
 from metaflow.mflog import BASH_SAVE_LOGS, bash_capture_logs, export_mflog_env_vars
 from metaflow.parameters import deploy_time_eval
 from metaflow.plugins.kubernetes.kubernetes import (
@@ -1822,6 +1822,21 @@ class ArgoWorkflows(object):
                 and k not in set(ARGO_WORKFLOWS_ENV_VARS_TO_SKIP.split(","))
             }
 
+            # get initial configs
+            initial_configs = init_config()
+            for entry in ["OBP_PERIMETER", "OBP_INTEGRATIONS_SECRETS_METADATA_URL"]:
+                if entry not in initial_configs:
+                    raise ArgoWorkflowsException(
+                        f"{entry} was not found in metaflow config. Please make sure to run `outerbounds configure <...>` command which can be found on the Ourebounds UI or reach out to your Outerbounds support team."
+                    )
+
+            additional_obp_configs = {
+                "OBP_PERIMETER": initial_configs["OBP_PERIMETER"],
+                "OBP_INTEGRATIONS_SECRETS_METADATA_URL": initial_configs[
+                    "OBP_INTEGRATIONS_SECRETS_METADATA_URL"
+                ],
+            }
+
             # Tmpfs variables
             use_tmpfs = resources["use_tmpfs"]
             tmpfs_size = resources["tmpfs_size"]
@@ -2083,6 +2098,13 @@ class ArgoWorkflows(object):
                                         "METAFLOW_KUBERNETES_SERVICE_ACCOUNT_NAME": "spec.serviceAccountName",
                                         "METAFLOW_KUBERNETES_NODE_IP": "status.hostIP",
                                     }.items()
+                                ]
+                                + [
+                                    kubernetes_sdk.V1EnvVar(
+                                        name=k,
+                                        value=v,
+                                    )
+                                    for k, v in additional_obp_configs.items()
                                 ],
                                 image=resources["image"],
                                 image_pull_policy=resources["image_pull_policy"],
@@ -2642,6 +2664,9 @@ class ArgoWorkflows(object):
 
         return (
             DaemonTemplate("heartbeat-daemon")
+            # NOTE: Even though a retry strategy does not work for Argo daemon containers,
+            # this has the side-effect of protecting the exit hooks of the workflow from failing in case the daemon container errors out.
+            .retry_strategy(10, 1)
             .service_account_name(resources["service_account"])
             .container(
                 to_camelcase(
@@ -3313,6 +3338,15 @@ class DaemonTemplate(object):
         self.payload["serviceAccountName"] = service_account_name
         return self
 
+    def retry_strategy(self, times, minutes_between_retries):
+        if times > 0:
+            self.payload["retryStrategy"] = {
+                "retryPolicy": "Always",
+                "limit": times,
+                "backoff": {"duration": "%sm" % minutes_between_retries},
+            }
+        return self
+
     def to_json(self):
         return self.payload
 

{ob_metaflow-2.12.23.1 → ob_metaflow-2.12.24.1}/metaflow/plugins/argo/argo_workflows_cli.py

@@ -179,7 +179,7 @@ def argo_workflows(obj, name=None):
 )
 @click.option(
     "--enable-heartbeat-daemon/--no-enable-heartbeat-daemon",
-    default=True,
+    default=False,
     show_default=True,
     help="Use a daemon container to broadcast heartbeats.",
 )

{ob_metaflow-2.12.23.1 → ob_metaflow-2.12.24.1}/metaflow/plugins/kubernetes/kubernetes.py

@@ -45,7 +45,7 @@ from metaflow.metaflow_config import (
     SERVICE_INTERNAL_URL,
 )
 from metaflow.unbounded_foreach import UBF_CONTROL, UBF_TASK
-from metaflow.metaflow_config_funcs import config_values
+from metaflow.metaflow_config_funcs import config_values, init_config
 from metaflow.mflog import (
     BASH_SAVE_LOGS,
     bash_capture_logs,
@@ -311,6 +311,22 @@ class Kubernetes(object):
         ) + KUBERNETES_SECRETS.split(","):
             jobset.secret(k)
 
+        initial_configs = init_config()
+        for entry in ["OBP_PERIMETER", "OBP_INTEGRATIONS_SECRETS_METADATA_URL"]:
+            if entry not in initial_configs:
+                raise KubernetesException(
+                    f"{entry} was not found in metaflow config. Please make sure to run `outerbounds configure <...>` command which can be found on the Ourebounds UI or reach out to your Outerbounds support team."
+                )
+
+        additional_obp_configs = {
+            "OBP_PERIMETER": initial_configs["OBP_PERIMETER"],
+            "OBP_INTEGRATIONS_SECRETS_METADATA_URL": initial_configs[
+                "OBP_INTEGRATIONS_SECRETS_METADATA_URL"
+            ],
+        }
+        for k, v in additional_obp_configs.items():
+            jobset.environment_variable(k, v)
+
         jobset.environment_variables_from_selectors(
             {
                 "METAFLOW_KUBERNETES_NAMESPACE": "metadata.namespace",

{ob_metaflow-2.12.23.1 → ob_metaflow-2.12.24.1}/metaflow/plugins/kubernetes/kubernetes_client.py

@@ -121,10 +121,7 @@ class KubernetesClient(object):
         job_api = self._client.BatchV1Api()
         pods = self._find_active_pods(flow_name, run_id, user)
 
-        active_pods = False
-
         def _kill_pod(pod):
-            active_pods = True
             echo("Killing Kubernetes pod %s\n" % pod.metadata.name)
             try:
                 stream(
@@ -158,10 +155,10 @@ class KubernetesClient(object):
                     echo("failed to kill pod %s - %s" % (pod.metadata.name, str(e)))
 
         with ThreadPoolExecutor() as executor:
-            executor.map(_kill_pod, pods)
+            operated_pods = list(executor.map(_kill_pod, pods))
 
-        if not active_pods:
-            echo("No active Kubernetes pods found for run *%s*" % run_id)
+            if not operated_pods:
+                echo("No active Kubernetes pods found for run *%s*" % run_id)
 
     def jobset(self, **kwargs):
         return KubernetesJobSet(self, **kwargs)

{ob_metaflow-2.12.23.1 → ob_metaflow-2.12.24.1}/metaflow/plugins/kubernetes/kubernetes_job.py

@@ -9,6 +9,7 @@ from metaflow.exception import MetaflowException
 from metaflow.metaflow_config import KUBERNETES_SECRETS
 from metaflow.tracing import inject_tracing_vars
 from metaflow.unbounded_foreach import UBF_CONTROL, UBF_TASK
+from metaflow.metaflow_config_funcs import init_config
 
 CLIENT_REFRESH_INTERVAL_SECONDS = 300
 from .kubernetes_jobsets import (
@@ -74,6 +75,20 @@ class KubernetesJob(object):
             if self._kwargs["shared_memory"]
             else None
         )
+        initial_configs = init_config()
+        for entry in ["OBP_PERIMETER", "OBP_INTEGRATIONS_SECRETS_METADATA_URL"]:
+            if entry not in initial_configs:
+                raise KubernetesJobException(
+                    f"{entry} was not found in metaflow config. Please make sure to run `outerbounds configure <...>` command which can be found on the Ourebounds UI or reach out to your Outerbounds support team."
+                )
+
+        additional_obp_configs = {
+            "OBP_PERIMETER": initial_configs["OBP_PERIMETER"],
+            "OBP_INTEGRATIONS_SECRETS_METADATA_URL": initial_configs[
+                "OBP_INTEGRATIONS_SECRETS_METADATA_URL"
+            ],
+        }
+
         return client.V1JobSpec(
             # Retries are handled by Metaflow when it is responsible for
             # executing the flow. The responsibility is moved to Kubernetes
@@ -135,6 +150,13 @@ class KubernetesJob(object):
                                     "METAFLOW_KUBERNETES_NODE_IP": "status.hostIP",
                                 }.items()
                             ]
+                            + [
+                                client.V1EnvVar(
+                                    name=k,
+                                    value=v,
+                                )
+                                for k, v in additional_obp_configs.items()
+                            ]
                             + [
                                 client.V1EnvVar(name=k, value=str(v))
                                 for k, v in inject_tracing_vars({}).items()

{ob_metaflow-2.12.23.1 → ob_metaflow-2.12.24.1}/metaflow/task.py

@@ -180,6 +180,10 @@ class MetaflowTask(object):
         # 2) join - pop the topmost frame from the stack
         # 3) step following a split - push a new frame in the stack
 
+        # We have a non-modifying case (case 4)) where we propagate the
+        # foreach-stack information to all tasks in the foreach. This is
+        # then used later to write the foreach-stack metadata for that task
+
         # case 1) - reset the stack
         if step_name == "start":
             self.flow._foreach_stack = []
@@ -264,6 +268,9 @@ class MetaflowTask(object):
             stack = inputs[0]["_foreach_stack"]
             stack.append(frame)
             self.flow._foreach_stack = stack
+        # case 4) - propagate in the foreach nest
+        elif "_foreach_stack" in inputs[0]:
+            self.flow._foreach_stack = inputs[0]["_foreach_stack"]
 
     def _clone_flow(self, datastore):
         x = self.flow.__class__(use_cli=False)

ob_metaflow-2.12.24.1/metaflow/version.py

@@ -0,0 +1 @@
+metaflow_version = "2.12.24.1"

{ob_metaflow-2.12.23.1 → ob_metaflow-2.12.24.1/ob_metaflow.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ob-metaflow
-Version: 2.12.23.1
+Version: 2.12.24.1
 Summary: Metaflow: More Data Science, Less Engineering
 Author: Netflix, Outerbounds & the Metaflow Community
 Author-email: help@outerbounds.co
@@ -12,7 +12,7 @@ Requires-Dist: boto3
 Requires-Dist: pylint
 Requires-Dist: kubernetes
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.12.23.1; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.12.24.1; extra == "stubs"
 
 ![Metaflow_Logo_Horizontal_FullColor_Ribbon_Dark_RGB](https://user-images.githubusercontent.com/763451/89453116-96a57e00-d713-11ea-9fa6-82b29d4d6eff.png)
 

{ob_metaflow-2.12.23.1 → ob_metaflow-2.12.24.1}/ob_metaflow.egg-info/requires.txt

@@ -4,4 +4,4 @@ pylint
 kubernetes
 
 [stubs]
-metaflow-stubs==2.12.23.1
+metaflow-stubs==2.12.24.1

ob_metaflow-2.12.23.1/metaflow/version.py

@@ -1 +0,0 @@
-metaflow_version = "2.12.23.1"