ob-metaflow 2.15.13.1__py2.py3-none-any.whl → 2.19.7.1rc0__py2.py3-none-any.whl

metaflow/plugins/airflow/airflow_cli.py

@@ -7,6 +7,7 @@ from hashlib import sha1
 from metaflow import current, decorators
 from metaflow._vendor import click
 from metaflow.exception import MetaflowException, MetaflowInternalError
+from metaflow.metaflow_config import FEAT_ALWAYS_UPLOAD_CODE_PACKAGE
 from metaflow.package import MetaflowPackage
 from metaflow.plugins.aws.step_functions.production_token import (
     load_token,
@@ -288,20 +289,30 @@ def make_flow(
     decorators._init_step_decorators(
         obj.flow, obj.graph, obj.environment, obj.flow_datastore, obj.logger
     )
-
+    obj.graph = obj.flow._graph
     # Save the code package in the flow datastore so that both user code and
     # metaflow package can be retrieved during workflow execution.
     obj.package = MetaflowPackage(
-        obj.flow, obj.environment, obj.echo, obj.package_suffixes
+        obj.flow,
+        obj.environment,
+        obj.echo,
+        suffixes=obj.package_suffixes,
+        flow_datastore=obj.flow_datastore if FEAT_ALWAYS_UPLOAD_CODE_PACKAGE else None,
     )
-    package_url, package_sha = obj.flow_datastore.save_data(
-        [obj.package.blob], len_hint=1
-    )[0]
+    # This blocks until the package is created
+    if FEAT_ALWAYS_UPLOAD_CODE_PACKAGE:
+        package_url = obj.package.package_url()
+        package_sha = obj.package.package_sha()
+    else:
+        package_url, package_sha = obj.flow_datastore.save_data(
+            [obj.package.blob], len_hint=1
+        )[0]
 
     return Airflow(
         dag_name,
         obj.graph,
         obj.flow,
+        obj.package.package_metadata,
         package_sha,
         package_url,
         obj.metadata,

metaflow/plugins/argo/argo_client.py

@@ -1,5 +1,6 @@
 import json
 
+from metaflow.metaflow_config import ARGO_EVENTS_SENSOR_NAMESPACE
 from metaflow.exception import MetaflowException
 from metaflow.plugins.kubernetes.kubernetes_client import KubernetesClient
 
@@ -58,21 +59,38 @@ class ArgoClient(object):
                 json.loads(e.body)["message"] if e.body is not None else e.reason
             )
 
-    def get_workflow_templates(self):
+    def get_workflow_templates(self, page_size=100):
         client = self._client.get()
-        try:
-            return client.CustomObjectsApi().list_namespaced_custom_object(
-                group=self._group,
-                version=self._version,
-                namespace=self._namespace,
-                plural="workflowtemplates",
-            )["items"]
-        except client.rest.ApiException as e:
-            if e.status == 404:
-                return None
-            raise ArgoClientException(
-                json.loads(e.body)["message"] if e.body is not None else e.reason
-            )
+        continue_token = None
+
+        while True:
+            try:
+                params = {"limit": page_size}
+                if continue_token:
+                    params["_continue"] = continue_token
+
+                response = client.CustomObjectsApi().list_namespaced_custom_object(
+                    group=self._group,
+                    version=self._version,
+                    namespace=self._namespace,
+                    plural="workflowtemplates",
+                    **params,
+                )
+
+                for item in response.get("items", []):
+                    yield item
+
+                metadata = response.get("metadata", {})
+                continue_token = metadata.get("continue")
+
+                if not continue_token:
+                    break
+            except client.rest.ApiException as e:
+                if e.status == 404:
+                    return None
+                raise ArgoClientException(
+                    json.loads(e.body)["message"] if e.body is not None else e.reason
+                )
 
     def register_workflow_template(self, name, workflow_template):
         # Unfortunately, Kubernetes client does not handle optimistic
@@ -307,6 +325,7 @@ class ArgoClient(object):
                 "failedJobsHistoryLimit": 10000,  # default is unfortunately 1
                 "successfulJobsHistoryLimit": 10000,  # default is unfortunately 3
                 "workflowSpec": {"workflowTemplateRef": {"name": name}},
+                "startingDeadlineSeconds": 3540,  # configuring this to 59 minutes so a failed trigger of cron workflow can succeed at most 59 mins after scheduled execution
             },
         }
         try:
@@ -360,12 +379,15 @@ class ArgoClient(object):
                 json.loads(e.body)["message"] if e.body is not None else e.reason
             )
 
-    def register_sensor(self, name, sensor=None):
+    def register_sensor(
+        self, name, sensor=None, sensor_namespace=ARGO_EVENTS_SENSOR_NAMESPACE
+    ):
         if sensor is None:
             sensor = {}
         # Unfortunately, Kubernetes client does not handle optimistic
         # concurrency control by itself unlike kubectl
         client = self._client.get()
+
         if not sensor:
             sensor["metadata"] = {}
 
@@ -375,7 +397,7 @@ class ArgoClient(object):
             ] = client.CustomObjectsApi().get_namespaced_custom_object(
                 group=self._group,
                 version=self._version,
-                namespace=self._namespace,
+                namespace=sensor_namespace,
                 plural="sensors",
                 name=name,
             )[
@@ -390,7 +412,7 @@ class ArgoClient(object):
                     return client.CustomObjectsApi().create_namespaced_custom_object(
                         group=self._group,
                         version=self._version,
-                        namespace=self._namespace,
+                        namespace=sensor_namespace,
                         plural="sensors",
                         body=sensor,
                     )
@@ -408,7 +430,7 @@ class ArgoClient(object):
             return client.CustomObjectsApi().replace_namespaced_custom_object(
                 group=self._group,
                 version=self._version,
-                namespace=self._namespace,
+                namespace=sensor_namespace,
                 plural="sensors",
                 body=sensor,
                 name=name,
@@ -418,7 +440,7 @@ class ArgoClient(object):
                 json.loads(e.body)["message"] if e.body is not None else e.reason
             )
 
-    def delete_sensor(self, name):
+    def delete_sensor(self, name, sensor_namespace):
         """
         Issues an API call for deleting a sensor
 
@@ -430,7 +452,7 @@ class ArgoClient(object):
             return client.CustomObjectsApi().delete_namespaced_custom_object(
                 group=self._group,
                 version=self._version,
-                namespace=self._namespace,
+                namespace=sensor_namespace,
                 plural="sensors",
                 name=name,
             )

metaflow/plugins/argo/argo_events.py

@@ -11,6 +11,7 @@ from metaflow.metaflow_config import (
     ARGO_EVENTS_WEBHOOK_AUTH,
     ARGO_EVENTS_WEBHOOK_URL,
     SERVICE_HEADERS,
+    SERVICE_RETRY_COUNT,
 )
 
 
@@ -127,12 +128,11 @@ class ArgoEvent(object):
                     headers={"Content-Type": "application/json", **headers},
                     data=json.dumps(data).encode("utf-8"),
                 )
-                retries = 3
-                backoff_factor = 2
 
-                for i in range(retries):
+                for i in range(SERVICE_RETRY_COUNT):
                     try:
-                        urllib.request.urlopen(request, timeout=10.0)
+                        # we do not want to wait indefinitely for a response on the event broadcast, as this will keep the task running.
+                        urllib.request.urlopen(request, timeout=60)
                         print(
                             "Argo Event (%s) published." % self._name, file=sys.stderr
                         )
@@ -141,10 +141,10 @@ class ArgoEvent(object):
                         # TODO: Retry retryable HTTP error codes
                         raise e
                     except urllib.error.URLError as e:
-                        if i == retries - 1:
+                        if i == SERVICE_RETRY_COUNT - 1:
                             raise e
                         else:
-                            time.sleep(backoff_factor**i)
+                            time.sleep(2**i)
             except Exception as e:
                 msg = "Unable to publish Argo Event (%s): %s" % (self._name, e)
                 if ignore_errors: