metaflow 2.13.5__tar.gz → 2.13.7__tar.gz

{metaflow-2.13.5/metaflow.egg-info → metaflow-2.13.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: metaflow
-Version: 2.13.5
+Version: 2.13.7
 Summary: Metaflow: More Data Science, Less Engineering
 Author: Metaflow Developers
 Author-email: help@metaflow.org
@@ -26,7 +26,7 @@ License-File: LICENSE
 Requires-Dist: requests
 Requires-Dist: boto3
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.13.5; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.13.7; extra == "stubs"
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier

{metaflow-2.13.5 → metaflow-2.13.7}/metaflow/decorators.py

@@ -591,9 +591,13 @@ def _init_flow_decorators(
                 )
         else:
             # Each "non-multiple" flow decorator is only allowed to have one set of options
+            # Note that there may be no deco_options if a MutableFlow config injected
+            # the decorator.
             deco_flow_init_options = {
-                option: deco_options[option.replace("-", "_")]
-                for option in deco.options
+                option: deco_options.get(
+                    option.replace("-", "_"), option_info["default"]
+                )
+                for option, option_info in deco.options.items()
             }
         for deco in decorators:
             deco.flow_init(

{metaflow-2.13.5 → metaflow-2.13.7}/metaflow/flowspec.py

@@ -86,6 +86,11 @@ class FlowSpecMeta(type):
         super().__init__(name, bases, attrs)
         if name == "FlowSpec":
             return
+
+        from .decorators import (
+            DuplicateFlowDecoratorException,
+        )  # Prevent circular import
+
         # We store some state in the flow class itself. This is primarily used to
         # attach global state to a flow. It is *not* an actual global because of
         # Runner/NBRunner. This is also created here in the meta class to avoid it being
@@ -98,6 +103,31 @@ class FlowSpecMeta(type):
         # Keys are _FlowState enum values
         cls._flow_state = {}
 
+        # We inherit stuff from our parent classes as well -- we need to be careful
+        # in terms of the order; we will follow the MRO with the following rules:
+        #  - decorators (cls._flow_decorators) will cause an error if they do not
+        #    support multiple and we see multiple instances of the same
+        #  - config decorators will be joined
+        #  - configs will be added later directly by the class; base class configs will
+        #    be taken into account as they would be inherited.
+
+        # We only need to do this for the base classes since the current class will
+        # get updated as decorators are parsed.
+        for base in cls.__mro__:
+            if base != cls and base != FlowSpec and issubclass(base, FlowSpec):
+                # Take care of decorators
+                for deco_name, deco in base._flow_decorators.items():
+                    if deco_name in cls._flow_decorators and not deco.allow_multiple:
+                        raise DuplicateFlowDecoratorException(deco_name)
+                    cls._flow_decorators.setdefault(deco_name, []).extend(deco)
+
+                # Take care of configs and config decorators
+                base_configs = base._flow_state.get(_FlowState.CONFIG_DECORATORS)
+                if base_configs:
+                    cls._flow_state.setdefault(_FlowState.CONFIG_DECORATORS, []).extend(
+                        base_configs
+                    )
+
         cls._init_attrs()
 
     def _init_attrs(cls):

{metaflow-2.13.5 → metaflow-2.13.7}/metaflow/parameters.py

@@ -316,7 +316,7 @@ class Parameter(object):
     help : str, optional, default None
         Help text to show in `run --help`.
     required : bool, optional, default None
-        Require that the user specified a value for the parameter. Note that if
+        Require that the user specifies a value for the parameter. Note that if
         a default is provide, the required flag is ignored.
         A value of None is equivalent to False.
     show_default : bool, optional, default None

{metaflow-2.13.5 → metaflow-2.13.7}/metaflow/plugins/argo/argo_workflows.py

@@ -624,6 +624,16 @@ class ArgoWorkflows(object):
             for event in trigger_on_finish_deco.triggers:
                 # Actual filters are deduced here since we don't have access to
                 # the current object in the @trigger_on_finish decorator.
+                project_name = event.get("project") or current.get("project_name")
+                branch_name = event.get("branch") or current.get("branch_name")
+                # validate that we have complete project info for an event name
+                if project_name or branch_name:
+                    if not (project_name and branch_name):
+                        # if one of the two is missing, we would end up listening to an event that will never be broadcast.
+                        raise ArgoWorkflowsException(
+                            "Incomplete project info. Please specify both 'project' and 'project_branch' or use the @project decorator"
+                        )
+
                 triggers.append(
                     {
                         # Make sure this remains consistent with the event name format
@@ -632,18 +642,16 @@ class ArgoWorkflows(object):
                         % ".".join(
                             v
                             for v in [
-                                event.get("project") or current.get("project_name"),
-                                event.get("branch") or current.get("branch_name"),
+                                project_name,
+                                branch_name,
                                 event["flow"],
                             ]
                             if v
                         ),
                         "filters": {
                             "auto-generated-by-metaflow": True,
-                            "project_name": event.get("project")
-                            or current.get("project_name"),
-                            "branch_name": event.get("branch")
-                            or current.get("branch_name"),
+                            "project_name": project_name,
+                            "branch_name": branch_name,
                             # TODO: Add a time filters to guard against cached events
                         },
                         "type": "run",
@@ -841,8 +849,15 @@ class ArgoWorkflows(object):
                     Metadata()
                     .labels(self._base_labels)
                     .label("app.kubernetes.io/name", "metaflow-task")
-                    .annotations(annotations)
-                    .annotations(self._base_annotations)
+                    .annotations(
+                        {
+                            **annotations,
+                            **self._base_annotations,
+                            **{
+                                "metaflow/run_id": "argo-{{workflow.name}}"
+                            },  # we want pods of the workflow to have the run_id as an annotation as well
+                        }
+                    )
                 )
                 # Set the entrypoint to flow name
                 .entrypoint(self.flow.name)

{metaflow-2.13.5 → metaflow-2.13.7}/metaflow/plugins/events_decorator.py

@@ -398,111 +398,23 @@ class TriggerOnFinishDecorator(FlowDecorator):
             )
         elif self.attributes["flow"]:
             # flow supports the format @trigger_on_finish(flow='FooFlow')
-            if is_stringish(self.attributes["flow"]):
-                self.triggers.append(
-                    {
-                        "fq_name": self.attributes["flow"],
-                    }
-                )
-            elif isinstance(self.attributes["flow"], dict):
-                if "name" not in self.attributes["flow"]:
-                    raise MetaflowException(
-                        "The *flow* attribute for *@trigger_on_finish* is missing the "
-                        "*name* key."
-                    )
-                flow_name = self.attributes["flow"]["name"]
-
-                if not is_stringish(flow_name) or "." in flow_name:
-                    raise MetaflowException(
-                        "The *name* attribute of the *flow* is not a valid string"
-                    )
-                result = {"fq_name": flow_name}
-                if "project" in self.attributes["flow"]:
-                    if is_stringish(self.attributes["flow"]["project"]):
-                        result["project"] = self.attributes["flow"]["project"]
-                    else:
-                        raise MetaflowException(
-                            "The *project* attribute of the *flow* is not a string"
-                        )
-                if "project_branch" in self.attributes["flow"]:
-                    if is_stringish(self.attributes["flow"]["project_branch"]):
-                        result["branch"] = self.attributes["flow"]["project_branch"]
-                    else:
-                        raise MetaflowException(
-                            "The *project_branch* attribute of the *flow* is not a string"
-                        )
-                self.triggers.append(result)
-            elif callable(self.attributes["flow"]) and not isinstance(
+            flow = self.attributes["flow"]
+            if callable(flow) and not isinstance(
                 self.attributes["flow"], DeployTimeField
             ):
-                trig = DeployTimeField(
-                    "fq_name", [str, dict], None, self.attributes["flow"], False
-                )
+                trig = DeployTimeField("fq_name", [str, dict], None, flow, False)
                 self.triggers.append(trig)
             else:
-                raise MetaflowException(
-                    "Incorrect type for *flow* attribute in *@trigger_on_finish* "
-                    " decorator. Supported type is string or Dict[str, str] - \n"
-                    "@trigger_on_finish(flow='FooFlow') or "
-                    "@trigger_on_finish(flow={'name':'FooFlow', 'project_branch': 'branch'})"
-                )
+                self.triggers.extend(self._parse_static_triggers([flow]))
         elif self.attributes["flows"]:
             # flows attribute supports the following formats -
             #     1. flows=['FooFlow', 'BarFlow']
-            if isinstance(self.attributes["flows"], list):
-                for flow in self.attributes["flows"]:
-                    if is_stringish(flow):
-                        self.triggers.append(
-                            {
-                                "fq_name": flow,
-                            }
-                        )
-                    elif isinstance(flow, dict):
-                        if "name" not in flow:
-                            raise MetaflowException(
-                                "One or more flows in the *flows* attribute for "
-                                "*@trigger_on_finish* is missing the "
-                                "*name* key."
-                            )
-                        flow_name = flow["name"]
-
-                        if not is_stringish(flow_name) or "." in flow_name:
-                            raise MetaflowException(
-                                "The *name* attribute '%s' is not a valid string"
-                                % str(flow_name)
-                            )
-                        result = {"fq_name": flow_name}
-                        if "project" in flow:
-                            if is_stringish(flow["project"]):
-                                result["project"] = flow["project"]
-                            else:
-                                raise MetaflowException(
-                                    "The *project* attribute of the *flow* '%s' is not "
-                                    "a string" % flow_name
-                                )
-                        if "project_branch" in flow:
-                            if is_stringish(flow["project_branch"]):
-                                result["branch"] = flow["project_branch"]
-                            else:
-                                raise MetaflowException(
-                                    "The *project_branch* attribute of the *flow* %s "
-                                    "is not a string" % flow_name
-                                )
-                        self.triggers.append(result)
-                    else:
-                        raise MetaflowException(
-                            "One or more flows in *flows* attribute in "
-                            "*@trigger_on_finish* decorator have an incorrect type. "
-                            "Supported type is string or Dict[str, str]- \n"
-                            "@trigger_on_finish(flows=['FooFlow', 'BarFlow']"
-                        )
-            elif callable(self.attributes["flows"]) and not isinstance(
-                self.attributes["flows"], DeployTimeField
-            ):
-                trig = DeployTimeField(
-                    "flows", list, None, self.attributes["flows"], False
-                )
+            flows = self.attributes["flows"]
+            if callable(flows) and not isinstance(flows, DeployTimeField):
+                trig = DeployTimeField("flows", list, None, flows, False)
                 self.triggers.append(trig)
+            elif isinstance(flows, list):
+                self.triggers.extend(self._parse_static_triggers(flows))
             else:
                 raise MetaflowException(
                     "Incorrect type for *flows* attribute in *@trigger_on_finish* "
@@ -519,26 +431,7 @@ class TriggerOnFinishDecorator(FlowDecorator):
         for trigger in self.triggers:
             if isinstance(trigger, DeployTimeField):
                 continue
-            if trigger["fq_name"].count(".") == 0:
-                # fully qualified name is just the flow name
-                trigger["flow"] = trigger["fq_name"]
-            elif trigger["fq_name"].count(".") >= 2:
-                # fully qualified name is of the format - project.branch.flow_name
-                trigger["project"], tail = trigger["fq_name"].split(".", maxsplit=1)
-                trigger["branch"], trigger["flow"] = tail.rsplit(".", maxsplit=1)
-            else:
-                raise MetaflowException(
-                    "Incorrect format for *flow* in *@trigger_on_finish* "
-                    "decorator. Specify either just the *flow_name* or a fully "
-                    "qualified name like *project_name.branch_name.flow_name*."
-                )
-            # TODO: Also sanity check project and branch names
-            if not re.match(r"^[A-Za-z0-9_]+$", trigger["flow"]):
-                raise MetaflowException(
-                    "Invalid flow name *%s* in *@trigger_on_finish* "
-                    "decorator. Only alphanumeric characters and "
-                    "underscores(_) are allowed." % trigger["flow"]
-                )
+            self._parse_fq_name(trigger)
 
         self.options = self.attributes["options"]
 
@@ -593,9 +486,67 @@ class TriggerOnFinishDecorator(FlowDecorator):
                 run_objs.append(run_obj)
             current._update_env({"trigger": Trigger.from_runs(run_objs)})
 
+    @staticmethod
+    def _parse_static_triggers(flows):
+        results = []
+        for flow in flows:
+            if is_stringish(flow):
+                results.append(
+                    {
+                        "fq_name": flow,
+                    }
+                )
+            elif isinstance(flow, dict):
+                if "name" not in flow:
+                    if len(flows) > 1:
+                        raise MetaflowException(
+                            "One or more flows in the *flows* attribute for "
+                            "*@trigger_on_finish* is missing the "
+                            "*name* key."
+                        )
+                    raise MetaflowException(
+                        "The *flow* attribute for *@trigger_on_finish* is missing the "
+                        "*name* key."
+                    )
+                flow_name = flow["name"]
+
+                if not is_stringish(flow_name) or "." in flow_name:
+                    raise MetaflowException(
+                        f"The *name* attribute of the *flow* {flow_name} is not a valid string"
+                    )
+                result = {"fq_name": flow_name}
+                if "project" in flow:
+                    if is_stringish(flow["project"]):
+                        result["project"] = flow["project"]
+                    else:
+                        raise MetaflowException(
+                            f"The *project* attribute of the *flow* {flow_name} is not a string"
+                        )
+                if "project_branch" in flow:
+                    if is_stringish(flow["project_branch"]):
+                        result["branch"] = flow["project_branch"]
+                    else:
+                        raise MetaflowException(
+                            f"The *project_branch* attribute of the *flow* {flow_name} is not a string"
+                        )
+                results.append(result)
+            else:
+                if len(flows) > 1:
+                    raise MetaflowException(
+                        "One or more flows in the *flows* attribute for "
+                        "*@trigger_on_finish* decorator have an incorrect type. "
+                        "Supported type is string or Dict[str, str]- \n"
+                        "@trigger_on_finish(flows=['FooFlow', 'BarFlow']"
+                    )
+                raise MetaflowException(
+                    "Incorrect type for *flow* attribute in *@trigger_on_finish* "
+                    " decorator. Supported type is string or Dict[str, str] - \n"
+                    "@trigger_on_finish(flow='FooFlow') or "
+                    "@trigger_on_finish(flow={'name':'FooFlow', 'project_branch': 'branch'})"
+                )
+        return results
+
     def _parse_fq_name(self, trigger):
-        if isinstance(trigger, DeployTimeField):
-            trigger["fq_name"] = deploy_time_eval(trigger["fq_name"])
         if trigger["fq_name"].count(".") == 0:
             # fully qualified name is just the flow name
             trigger["flow"] = trigger["fq_name"]
@@ -615,32 +566,18 @@ class TriggerOnFinishDecorator(FlowDecorator):
                 "decorator. Only alphanumeric characters and "
                 "underscores(_) are allowed." % trigger["flow"]
             )
-        return trigger
 
     def format_deploytime_value(self):
-        for trigger in self.triggers:
-            # Case were trigger is a function that returns a list
-            # Need to do this bc we need to iterate over list and process
-            if isinstance(trigger, DeployTimeField):
-                deploy_value = deploy_time_eval(trigger)
-                if isinstance(deploy_value, list):
-                    self.triggers = deploy_value
+        if len(self.triggers) == 1 and isinstance(self.triggers[0], DeployTimeField):
+            deploy_value = deploy_time_eval(self.triggers[0])
+            if isinstance(deploy_value, list):
+                self.triggers = deploy_value
             else:
-                break
-        for trigger in self.triggers:
-            # Entire trigger is a function (returns either string or dict)
-            old_trig = trigger
-            if isinstance(trigger, DeployTimeField):
-                trigger = deploy_time_eval(trigger)
-            if isinstance(trigger, dict):
-                trigger["fq_name"] = trigger.get("name")
-                trigger["project"] = trigger.get("project")
-                trigger["branch"] = trigger.get("project_branch")
-            # We also added this bc it won't be formatted yet
-            if isinstance(trigger, str):
-                trigger = {"fq_name": trigger}
-                trigger = self._parse_fq_name(trigger)
-            self.triggers[self.triggers.index(old_trig)] = trigger
+                self.triggers = [deploy_value]
+            triggers = self._parse_static_triggers(self.triggers)
+            for trigger in triggers:
+                self._parse_fq_name(trigger)
+            self.triggers = triggers
 
     def get_top_level_options(self):
         return list(self._option_values.items())

{metaflow-2.13.5 → metaflow-2.13.7}/metaflow/plugins/kubernetes/kubernetes.py

@@ -669,15 +669,6 @@ class Kubernetes(object):
         for name, value in system_annotations.items():
             job.annotation(name, value)
 
-        (
-            job.annotation("metaflow/run_id", run_id)
-            .annotation("metaflow/step_name", step_name)
-            .annotation("metaflow/task_id", task_id)
-            .annotation("metaflow/attempt", attempt)
-            .label("app.kubernetes.io/name", "metaflow-task")
-            .label("app.kubernetes.io/part-of", "metaflow")
-        )
-
         return job
 
     def create_k8sjob(self, job):

{metaflow-2.13.5 → metaflow-2.13.7}/metaflow/runner/click_api.py

@@ -371,6 +371,8 @@ class MetaflowAPI(object):
                     else:
                         components.append(v)
                 for k, v in options.items():
+                    if v is None:
+                        continue
                     if isinstance(v, list):
                         for i in v:
                             if isinstance(i, tuple):

{metaflow-2.13.5 → metaflow-2.13.7}/metaflow/user_configs/config_options.py

@@ -169,7 +169,7 @@ class ConfigInput:
                     "Please contact support."
                 )
             cls.loaded_configs = all_configs
-        return cls.loaded_configs.get(config_name, None)
+        return cls.loaded_configs[config_name]
 
     def process_configs(
         self,
@@ -326,6 +326,8 @@ class ConfigInput:
         for name, val in merged_configs.items():
             if val is None:
                 missing_configs.add(name)
+                to_return[name] = None
+                flow_cls._flow_state[_FlowState.CONFIGS][name] = None
                 continue
             if val.startswith(_CONVERTED_DEFAULT_NO_FILE):
                 no_default_file.append(name)
@@ -339,15 +341,16 @@ class ConfigInput:
                 val = val[len(_DEFAULT_PREFIX) :]
             if val.startswith("kv."):
                 # This means to load it from a file
-                read_value = self.get_config(val[3:])
-                if read_value is None:
+                try:
+                    read_value = self.get_config(val[3:])
+                except KeyError as e:
                     exc = click.UsageError(
                         "Could not find configuration '%s' in INFO file" % val
                     )
                     if click_obj:
                         click_obj.delayed_config_exception = exc
                         return None
-                    raise exc
+                    raise exc from e
                 flow_cls._flow_state[_FlowState.CONFIGS][name] = read_value
                 to_return[name] = ConfigValue(read_value)
             else:

{metaflow-2.13.5 → metaflow-2.13.7}/metaflow/user_configs/config_parameters.py

@@ -290,17 +290,17 @@ class Config(Parameter, collections.abc.Mapping):
     default : Union[str, Callable[[ParameterContext], str], optional, default None
         Default path from where to read this configuration. A function implies that the
         value will be computed using that function.
-        You can only specify default or default_value.
+        You can only specify default or default_value, not both.
     default_value : Union[str, Dict[str, Any], Callable[[ParameterContext, Union[str, Dict[str, Any]]], Any], optional, default None
         Default value for the parameter. A function
         implies that the value will be computed using that function.
-        You can only specify default or default_value.
+        You can only specify default or default_value, not both.
     help : str, optional, default None
         Help text to show in `run --help`.
     required : bool, optional, default None
-        Require that the user specified a value for the configuration. Note that if
-        a default is provided, the required flag is ignored. A value of None is
-        equivalent to False.
+        Require that the user specifies a value for the configuration. Note that if
+        a default or default_value is provided, the required flag is ignored.
+        A value of None is equivalent to False.
     parser : Union[str, Callable[[str], Dict[Any, Any]]], optional, default None
         If a callable, it is a function that can parse the configuration string
         into an arbitrarily nested dictionary. If a string, the string should refer to
@@ -330,13 +330,13 @@ class Config(Parameter, collections.abc.Mapping):
         **kwargs: Dict[str, str]
     ):
 
-        if default and default_value:
+        if default is not None and default_value is not None:
             raise MetaflowException(
                 "For config '%s', you can only specify default or default_value, not both"
                 % name
             )
         self._default_is_file = default is not None
-        kwargs["default"] = default or default_value
+        kwargs["default"] = default if default is not None else default_value
         super(Config, self).__init__(
             name, required=required, help=help, type=str, **kwargs
         )

metaflow-2.13.7/metaflow/version.py

@@ -0,0 +1 @@
+metaflow_version = "2.13.7"

{metaflow-2.13.5 → metaflow-2.13.7/metaflow.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: metaflow
-Version: 2.13.5
+Version: 2.13.7
 Summary: Metaflow: More Data Science, Less Engineering
 Author: Metaflow Developers
 Author-email: help@metaflow.org
@@ -26,7 +26,7 @@ License-File: LICENSE
 Requires-Dist: requests
 Requires-Dist: boto3
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.13.5; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.13.7; extra == "stubs"
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier

{metaflow-2.13.5 → metaflow-2.13.7}/metaflow.egg-info/requires.txt

@@ -2,4 +2,4 @@ requests
 boto3
 
 [stubs]
-metaflow-stubs==2.13.5
+metaflow-stubs==2.13.7

metaflow-2.13.5/metaflow/version.py

@@ -1 +0,0 @@
-metaflow_version = "2.13.5"