metaflow 2.13.5__py2.py3-none-any.whl → 2.13.6__py2.py3-none-any.whl

metaflow/plugins/argo/argo_workflows.py

@@ -624,6 +624,16 @@ class ArgoWorkflows(object):
             for event in trigger_on_finish_deco.triggers:
                 # Actual filters are deduced here since we don't have access to
                 # the current object in the @trigger_on_finish decorator.
+                project_name = event.get("project") or current.get("project_name")
+                branch_name = event.get("branch") or current.get("branch_name")
+                # validate that we have complete project info for an event name
+                if project_name or branch_name:
+                    if not (project_name and branch_name):
+                        # if one of the two is missing, we would end up listening to an event that will never be broadcast.
+                        raise ArgoWorkflowsException(
+                            "Incomplete project info. Please specify both 'project' and 'project_branch' or use the @project decorator"
+                        )
+
                 triggers.append(
                     {
                         # Make sure this remains consistent with the event name format
@@ -632,18 +642,16 @@ class ArgoWorkflows(object):
                         % ".".join(
                             v
                             for v in [
-                                event.get("project") or current.get("project_name"),
-                                event.get("branch") or current.get("branch_name"),
+                                project_name,
+                                branch_name,
                                 event["flow"],
                             ]
                             if v
                         ),
                         "filters": {
                             "auto-generated-by-metaflow": True,
-                            "project_name": event.get("project")
-                            or current.get("project_name"),
-                            "branch_name": event.get("branch")
-                            or current.get("branch_name"),
+                            "project_name": project_name,
+                            "branch_name": branch_name,
                             # TODO: Add a time filters to guard against cached events
                         },
                         "type": "run",
@@ -841,8 +849,15 @@ class ArgoWorkflows(object):
                     Metadata()
                     .labels(self._base_labels)
                     .label("app.kubernetes.io/name", "metaflow-task")
-                    .annotations(annotations)
-                    .annotations(self._base_annotations)
+                    .annotations(
+                        {
+                            **annotations,
+                            **self._base_annotations,
+                            **{
+                                "metaflow/run_id": "argo-{{workflow.name}}"
+                            },  # we want pods of the workflow to have the run_id as an annotation as well
+                        }
+                    )
                 )
                 # Set the entrypoint to flow name
                 .entrypoint(self.flow.name)

metaflow/plugins/events_decorator.py

@@ -398,111 +398,23 @@ class TriggerOnFinishDecorator(FlowDecorator):
             )
         elif self.attributes["flow"]:
             # flow supports the format @trigger_on_finish(flow='FooFlow')
-            if is_stringish(self.attributes["flow"]):
-                self.triggers.append(
-                    {
-                        "fq_name": self.attributes["flow"],
-                    }
-                )
-            elif isinstance(self.attributes["flow"], dict):
-                if "name" not in self.attributes["flow"]:
-                    raise MetaflowException(
-                        "The *flow* attribute for *@trigger_on_finish* is missing the "
-                        "*name* key."
-                    )
-                flow_name = self.attributes["flow"]["name"]
-
-                if not is_stringish(flow_name) or "." in flow_name:
-                    raise MetaflowException(
-                        "The *name* attribute of the *flow* is not a valid string"
-                    )
-                result = {"fq_name": flow_name}
-                if "project" in self.attributes["flow"]:
-                    if is_stringish(self.attributes["flow"]["project"]):
-                        result["project"] = self.attributes["flow"]["project"]
-                    else:
-                        raise MetaflowException(
-                            "The *project* attribute of the *flow* is not a string"
-                        )
-                if "project_branch" in self.attributes["flow"]:
-                    if is_stringish(self.attributes["flow"]["project_branch"]):
-                        result["branch"] = self.attributes["flow"]["project_branch"]
-                    else:
-                        raise MetaflowException(
-                            "The *project_branch* attribute of the *flow* is not a string"
-                        )
-                self.triggers.append(result)
-            elif callable(self.attributes["flow"]) and not isinstance(
+            flow = self.attributes["flow"]
+            if callable(flow) and not isinstance(
                 self.attributes["flow"], DeployTimeField
             ):
-                trig = DeployTimeField(
-                    "fq_name", [str, dict], None, self.attributes["flow"], False
-                )
+                trig = DeployTimeField("fq_name", [str, dict], None, flow, False)
                 self.triggers.append(trig)
             else:
-                raise MetaflowException(
-                    "Incorrect type for *flow* attribute in *@trigger_on_finish* "
-                    " decorator. Supported type is string or Dict[str, str] - \n"
-                    "@trigger_on_finish(flow='FooFlow') or "
-                    "@trigger_on_finish(flow={'name':'FooFlow', 'project_branch': 'branch'})"
-                )
+                self.triggers.extend(self._parse_static_triggers([flow]))
         elif self.attributes["flows"]:
             # flows attribute supports the following formats -
             #     1. flows=['FooFlow', 'BarFlow']
-            if isinstance(self.attributes["flows"], list):
-                for flow in self.attributes["flows"]:
-                    if is_stringish(flow):
-                        self.triggers.append(
-                            {
-                                "fq_name": flow,
-                            }
-                        )
-                    elif isinstance(flow, dict):
-                        if "name" not in flow:
-                            raise MetaflowException(
-                                "One or more flows in the *flows* attribute for "
-                                "*@trigger_on_finish* is missing the "
-                                "*name* key."
-                            )
-                        flow_name = flow["name"]
-
-                        if not is_stringish(flow_name) or "." in flow_name:
-                            raise MetaflowException(
-                                "The *name* attribute '%s' is not a valid string"
-                                % str(flow_name)
-                            )
-                        result = {"fq_name": flow_name}
-                        if "project" in flow:
-                            if is_stringish(flow["project"]):
-                                result["project"] = flow["project"]
-                            else:
-                                raise MetaflowException(
-                                    "The *project* attribute of the *flow* '%s' is not "
-                                    "a string" % flow_name
-                                )
-                        if "project_branch" in flow:
-                            if is_stringish(flow["project_branch"]):
-                                result["branch"] = flow["project_branch"]
-                            else:
-                                raise MetaflowException(
-                                    "The *project_branch* attribute of the *flow* %s "
-                                    "is not a string" % flow_name
-                                )
-                        self.triggers.append(result)
-                    else:
-                        raise MetaflowException(
-                            "One or more flows in *flows* attribute in "
-                            "*@trigger_on_finish* decorator have an incorrect type. "
-                            "Supported type is string or Dict[str, str]- \n"
-                            "@trigger_on_finish(flows=['FooFlow', 'BarFlow']"
-                        )
-            elif callable(self.attributes["flows"]) and not isinstance(
-                self.attributes["flows"], DeployTimeField
-            ):
-                trig = DeployTimeField(
-                    "flows", list, None, self.attributes["flows"], False
-                )
+            flows = self.attributes["flows"]
+            if callable(flows) and not isinstance(flows, DeployTimeField):
+                trig = DeployTimeField("flows", list, None, flows, False)
                 self.triggers.append(trig)
+            elif isinstance(flows, list):
+                self.triggers.extend(self._parse_static_triggers(flows))
             else:
                 raise MetaflowException(
                     "Incorrect type for *flows* attribute in *@trigger_on_finish* "
@@ -519,26 +431,7 @@ class TriggerOnFinishDecorator(FlowDecorator):
         for trigger in self.triggers:
             if isinstance(trigger, DeployTimeField):
                 continue
-            if trigger["fq_name"].count(".") == 0:
-                # fully qualified name is just the flow name
-                trigger["flow"] = trigger["fq_name"]
-            elif trigger["fq_name"].count(".") >= 2:
-                # fully qualified name is of the format - project.branch.flow_name
-                trigger["project"], tail = trigger["fq_name"].split(".", maxsplit=1)
-                trigger["branch"], trigger["flow"] = tail.rsplit(".", maxsplit=1)
-            else:
-                raise MetaflowException(
-                    "Incorrect format for *flow* in *@trigger_on_finish* "
-                    "decorator. Specify either just the *flow_name* or a fully "
-                    "qualified name like *project_name.branch_name.flow_name*."
-                )
-            # TODO: Also sanity check project and branch names
-            if not re.match(r"^[A-Za-z0-9_]+$", trigger["flow"]):
-                raise MetaflowException(
-                    "Invalid flow name *%s* in *@trigger_on_finish* "
-                    "decorator. Only alphanumeric characters and "
-                    "underscores(_) are allowed." % trigger["flow"]
-                )
+            self._parse_fq_name(trigger)
 
         self.options = self.attributes["options"]
 
@@ -593,9 +486,67 @@ class TriggerOnFinishDecorator(FlowDecorator):
                 run_objs.append(run_obj)
             current._update_env({"trigger": Trigger.from_runs(run_objs)})
 
+    @staticmethod
+    def _parse_static_triggers(flows):
+        results = []
+        for flow in flows:
+            if is_stringish(flow):
+                results.append(
+                    {
+                        "fq_name": flow,
+                    }
+                )
+            elif isinstance(flow, dict):
+                if "name" not in flow:
+                    if len(flows) > 1:
+                        raise MetaflowException(
+                            "One or more flows in the *flows* attribute for "
+                            "*@trigger_on_finish* is missing the "
+                            "*name* key."
+                        )
+                    raise MetaflowException(
+                        "The *flow* attribute for *@trigger_on_finish* is missing the "
+                        "*name* key."
+                    )
+                flow_name = flow["name"]
+
+                if not is_stringish(flow_name) or "." in flow_name:
+                    raise MetaflowException(
+                        f"The *name* attribute of the *flow* {flow_name} is not a valid string"
+                    )
+                result = {"fq_name": flow_name}
+                if "project" in flow:
+                    if is_stringish(flow["project"]):
+                        result["project"] = flow["project"]
+                    else:
+                        raise MetaflowException(
+                            f"The *project* attribute of the *flow* {flow_name} is not a string"
+                        )
+                if "project_branch" in flow:
+                    if is_stringish(flow["project_branch"]):
+                        result["branch"] = flow["project_branch"]
+                    else:
+                        raise MetaflowException(
+                            f"The *project_branch* attribute of the *flow* {flow_name} is not a string"
+                        )
+                results.append(result)
+            else:
+                if len(flows) > 1:
+                    raise MetaflowException(
+                        "One or more flows in the *flows* attribute for "
+                        "*@trigger_on_finish* decorator have an incorrect type. "
+                        "Supported type is string or Dict[str, str]- \n"
+                        "@trigger_on_finish(flows=['FooFlow', 'BarFlow']"
+                    )
+                raise MetaflowException(
+                    "Incorrect type for *flow* attribute in *@trigger_on_finish* "
+                    " decorator. Supported type is string or Dict[str, str] - \n"
+                    "@trigger_on_finish(flow='FooFlow') or "
+                    "@trigger_on_finish(flow={'name':'FooFlow', 'project_branch': 'branch'})"
+                )
+        return results
+
     def _parse_fq_name(self, trigger):
-        if isinstance(trigger, DeployTimeField):
-            trigger["fq_name"] = deploy_time_eval(trigger["fq_name"])
         if trigger["fq_name"].count(".") == 0:
             # fully qualified name is just the flow name
             trigger["flow"] = trigger["fq_name"]
@@ -615,32 +566,18 @@ class TriggerOnFinishDecorator(FlowDecorator):
                 "decorator. Only alphanumeric characters and "
                 "underscores(_) are allowed." % trigger["flow"]
             )
-        return trigger
 
     def format_deploytime_value(self):
-        for trigger in self.triggers:
-            # Case were trigger is a function that returns a list
-            # Need to do this bc we need to iterate over list and process
-            if isinstance(trigger, DeployTimeField):
-                deploy_value = deploy_time_eval(trigger)
-                if isinstance(deploy_value, list):
-                    self.triggers = deploy_value
+        if len(self.triggers) == 1 and isinstance(self.triggers[0], DeployTimeField):
+            deploy_value = deploy_time_eval(self.triggers[0])
+            if isinstance(deploy_value, list):
+                self.triggers = deploy_value
             else:
-                break
-        for trigger in self.triggers:
-            # Entire trigger is a function (returns either string or dict)
-            old_trig = trigger
-            if isinstance(trigger, DeployTimeField):
-                trigger = deploy_time_eval(trigger)
-            if isinstance(trigger, dict):
-                trigger["fq_name"] = trigger.get("name")
-                trigger["project"] = trigger.get("project")
-                trigger["branch"] = trigger.get("project_branch")
-            # We also added this bc it won't be formatted yet
-            if isinstance(trigger, str):
-                trigger = {"fq_name": trigger}
-                trigger = self._parse_fq_name(trigger)
-            self.triggers[self.triggers.index(old_trig)] = trigger
+                self.triggers = [deploy_value]
+            triggers = self._parse_static_triggers(self.triggers)
+            for trigger in triggers:
+                self._parse_fq_name(trigger)
+            self.triggers = triggers
 
     def get_top_level_options(self):
         return list(self._option_values.items())

metaflow/plugins/kubernetes/kubernetes.py

@@ -669,15 +669,6 @@ class Kubernetes(object):
         for name, value in system_annotations.items():
             job.annotation(name, value)
 
-        (
-            job.annotation("metaflow/run_id", run_id)
-            .annotation("metaflow/step_name", step_name)
-            .annotation("metaflow/task_id", task_id)
-            .annotation("metaflow/attempt", attempt)
-            .label("app.kubernetes.io/name", "metaflow-task")
-            .label("app.kubernetes.io/part-of", "metaflow")
-        )
-
         return job
 
     def create_k8sjob(self, job):

metaflow/version.py

@@ -1 +1 @@
-metaflow_version = "2.13.5"
+metaflow_version = "2.13.6"

{metaflow-2.13.5.dist-info → metaflow-2.13.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: metaflow
-Version: 2.13.5
+Version: 2.13.6
 Summary: Metaflow: More Data Science, Less Engineering
 Author: Metaflow Developers
 Author-email: help@metaflow.org
@@ -26,7 +26,7 @@ License-File: LICENSE
 Requires-Dist: requests
 Requires-Dist: boto3
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.13.5; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.13.6; extra == "stubs"
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier

{metaflow-2.13.5.dist-info → metaflow-2.13.6.dist-info}/RECORD

@@ -36,7 +36,7 @@ metaflow/tuple_util.py,sha256=_G5YIEhuugwJ_f6rrZoelMFak3DqAR2tt_5CapS1XTY,830
 metaflow/unbounded_foreach.py,sha256=p184WMbrMJ3xKYHwewj27ZhRUsSj_kw1jlye5gA9xJk,387
 metaflow/util.py,sha256=hKjHl6NYJkKBSU2tzdVbddfOX1zWK73T4GCO42A0XB4,14666
 metaflow/vendor.py,sha256=FchtA9tH22JM-eEtJ2c9FpUdMn8sSb1VHuQS56EcdZk,5139
-metaflow/version.py,sha256=3DskisNYGb1X_79clFayeGjmYoYe6Nydsy4YBI-w0Ew,28
+metaflow/version.py,sha256=p2wlGBdWzsaPFEcG9sZp37O29SjTggTsTxrhb2gRZvc,28
 metaflow/_vendor/__init__.py,sha256=y_CiwUD3l4eAKvTVDZeqgVujMy31cAM1qjAB-HfI-9s,353
 metaflow/_vendor/typing_extensions.py,sha256=0nUs5p1A_UrZigrAVBoOEM6TxU37zzPDUtiij1ZwpNc,110417
 metaflow/_vendor/zipp.py,sha256=ajztOH-9I7KA_4wqDYygtHa6xUBVZgFpmZ8FE74HHHI,8425
@@ -154,7 +154,7 @@ metaflow/plugins/catch_decorator.py,sha256=UOM2taN_OL2RPpuJhwEOA9ZALm0-hHD0XS2Hn
 metaflow/plugins/debug_logger.py,sha256=mcF5HYzJ0NQmqCMjyVUk3iAP-heroHRIiVWQC6Ha2-I,879
 metaflow/plugins/debug_monitor.py,sha256=Md5X_sDOSssN9pt2D8YcaIjTK5JaQD55UAYTcF6xYF0,1099
 metaflow/plugins/environment_decorator.py,sha256=6m9j2B77d-Ja_l_9CTJ__0O6aB2a8Qt_lAZu6UjAcUA,587
-metaflow/plugins/events_decorator.py,sha256=WsLcuy-FmXpQ6mvm431deTB2hE-fPYILgbVSWHRXslQ,28121
+metaflow/plugins/events_decorator.py,sha256=T_YSK-DlgZhd3ge9PlpTRNaMi15GK0tKZMZl1NdV9DQ,24403
 metaflow/plugins/logs_cli.py,sha256=77W5UNagU2mOKSMMvrQxQmBLRzvmjK-c8dWxd-Ygbqs,11410
 metaflow/plugins/package_cli.py,sha256=-J6D4cupHfWSZ4GEFo2yy9Je9oL3owRWm5pEJwaiqd4,1649
 metaflow/plugins/parallel_decorator.py,sha256=GR6LKIW7_S7AoU50Ar2_0nndVtO2epdn3LuthE0vKMQ,9127
@@ -181,7 +181,7 @@ metaflow/plugins/airflow/sensors/s3_sensor.py,sha256=iDReG-7FKnumrtQg-HY6cCUAAqN
 metaflow/plugins/argo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 metaflow/plugins/argo/argo_client.py,sha256=PS_cYGnPw9h4X7TP_plObDH3clMw4reOsBLkkGPTd0Y,16282
 metaflow/plugins/argo/argo_events.py,sha256=_C1KWztVqgi3zuH57pInaE9OzABc2NnncC-zdwOMZ-w,5909
-metaflow/plugins/argo/argo_workflows.py,sha256=HgreJyYibFiWScq9mvd0p0bM8NJPX49n0gdnhUVGoHI,175591
+metaflow/plugins/argo/argo_workflows.py,sha256=72zuqG23uBHDhoCJl52jHmbLE_w1lvZV5EnrkVggBJg,176389
 metaflow/plugins/argo/argo_workflows_cli.py,sha256=11_8l4IrtkwviKsijInTZPt7YK5TZzClREnw_Cf4D5o,36706
 metaflow/plugins/argo/argo_workflows_decorator.py,sha256=ogCSBmwsC2C3eusydrgjuAJd4qK18f1sI4jJwA4Fd-o,7800
 metaflow/plugins/argo/argo_workflows_deployer.py,sha256=6kHxEnYXJwzNCM9swI8-0AckxtPWqwhZLerYkX8fxUM,4444
@@ -288,7 +288,7 @@ metaflow/plugins/gcp/gs_utils.py,sha256=ZmIGFse1qYyvAVrwga23PQUzF6dXEDLLsZ2F-YRm
 metaflow/plugins/gcp/includefile_support.py,sha256=OQO0IVWv4ObboL0VqEZwcDOyj9ORLdur66JToxQ84vU,3887
 metaflow/plugins/kubernetes/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 metaflow/plugins/kubernetes/kube_utils.py,sha256=jdFMGbEmIow-oli26v31W9CmbZXigx06b3D_xIobpk0,4140
-metaflow/plugins/kubernetes/kubernetes.py,sha256=7yaa1TL3TcC-Js6_kAi0HGFLbXesMw3WiKWPlN9yIxo,30028
+metaflow/plugins/kubernetes/kubernetes.py,sha256=g_E5jBhoMSDzGFnP5PDQiVPPllUr1wulVcG3tz247G8,29668
 metaflow/plugins/kubernetes/kubernetes_cli.py,sha256=o_o0BDEJFpTuga7txRmkvZH8OIuTb5kI4UaG6xbzf84,13929
 metaflow/plugins/kubernetes/kubernetes_client.py,sha256=tuvXP-QKpdeSmzVolB2R_TaacOr5DIb0j642eKcjsiM,6491
 metaflow/plugins/kubernetes/kubernetes_decorator.py,sha256=OwIuB9MAeO_fmTv3_IurDnbL_szXH7et3TwEva4PCfc,30853
@@ -360,9 +360,9 @@ metaflow/user_configs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3h
 metaflow/user_configs/config_decorators.py,sha256=Tj0H88UT8Q6pylXxHXgiA6cqnNlw4d3mR7M8J9g3ZUg,20139
 metaflow/user_configs/config_options.py,sha256=Knpiax_YGmYAdR3zKmaepN8puW1MyL9g6-eMGAkcylo,20942
 metaflow/user_configs/config_parameters.py,sha256=T0Zz18o9zKEV7mMcKotFWvXixhJpotLRBVrKx6ENErQ,15416
-metaflow-2.13.5.dist-info/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
-metaflow-2.13.5.dist-info/METADATA,sha256=8_zAjFu6yf75apsgNeoWmncpzpEhuji4C-JYe3w8HBg,6121
-metaflow-2.13.5.dist-info/WHEEL,sha256=9Hm2OB-j1QcCUq9Jguht7ayGIIZBRTdOXD1qg9cCgPM,109
-metaflow-2.13.5.dist-info/entry_points.txt,sha256=IKwTN1T3I5eJL3uo_vnkyxVffcgnRdFbKwlghZfn27k,57
-metaflow-2.13.5.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
-metaflow-2.13.5.dist-info/RECORD,,
+metaflow-2.13.6.dist-info/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
+metaflow-2.13.6.dist-info/METADATA,sha256=7bPykdKjdrtqd0D8QhWa-ItxZbzsh_JAOp4gpTvE6v4,6121
+metaflow-2.13.6.dist-info/WHEEL,sha256=9Hm2OB-j1QcCUq9Jguht7ayGIIZBRTdOXD1qg9cCgPM,109
+metaflow-2.13.6.dist-info/entry_points.txt,sha256=IKwTN1T3I5eJL3uo_vnkyxVffcgnRdFbKwlghZfn27k,57
+metaflow-2.13.6.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
+metaflow-2.13.6.dist-info/RECORD,,