metaflow 2.18.5__py2.py3-none-any.whl → 2.18.7__py2.py3-none-any.whl

metaflow/plugins/events_decorator.py

@@ -70,7 +70,26 @@ class TriggerDecorator(FlowDecorator):
         "options": {},
     }
 
-    def process_event_name(self, event):
+    def process_event(self, event):
+        """
+        Process a single event and return a dictionary if static trigger and a function
+        if deploy-time trigger.
+
+        Parameters
+        ----------
+        event : Union[str, Dict[str, Any], Callable]
+            Event to process
+
+        Returns
+        -------
+        Union[Dict[str, Union[str, Callable]], Callable]
+            Processed event
+
+        Raises
+        ------
+        MetaflowException
+            If the event is not in the correct format
+        """
         if is_stringish(event):
             return {"name": str(event)}
         elif isinstance(event, dict):
@@ -82,12 +101,26 @@ class TriggerDecorator(FlowDecorator):
                 event["name"], DeployTimeField
             ):
                 event["name"] = DeployTimeField(
-                    "event_name", str, None, event["name"], False
+                    "event_name",
+                    str,
+                    None,
+                    event["name"],
+                    False,
+                    print_representation=str(event["name"]),
                 )
-            event["parameters"] = self.process_parameters(event.get("parameters", {}))
+            event["parameters"] = self.process_parameters(
+                event.get("parameters", {}), event["name"]
+            )
             return event
         elif callable(event) and not isinstance(event, DeployTimeField):
-            return DeployTimeField("event", [str, dict], None, event, False)
+            return DeployTimeField(
+                "event",
+                [str, dict],
+                None,
+                event,
+                False,
+                print_representation=str(event),
+            )
         else:
             raise MetaflowException(
                 "Incorrect format for *event* attribute in *@trigger* decorator. "
@@ -96,47 +129,67 @@ class TriggerDecorator(FlowDecorator):
                 "'parameters': {'alpha': 'beta'}})"
             )
 
-    def process_parameters(self, parameters):
+    def process_parameters(self, parameters, event_name):
+        """
+        Process the parameters for an event and return a dictionary of parameter mappings if
+        parameters was statically defined or a function if deploy-time trigger.
+
+        Parameters
+        ----------
+        Parameters : Union[Dict[str, str], List[Union[str, Tuple[str, str]]], Callable]
+            Parameters to process
+
+        event_name : Union[str, callable]
+            Name of the event
+
+        Returns
+        -------
+        Union[Dict[str, str], Callable]
+            Processed parameters
+
+        Raises
+        ------
+        MetaflowException
+            If the parameters are not in the correct format
+        """
         new_param_values = {}
-        if isinstance(parameters, (list, tuple)):
+        if isinstance(parameters, list):
             for mapping in parameters:
                 if is_stringish(mapping):
+                    # param_name
                     new_param_values[mapping] = mapping
-                elif callable(mapping) and not isinstance(mapping, DeployTimeField):
-                    mapping = DeployTimeField(
-                        "parameter_val", str, None, mapping, False
-                    )
-                    new_param_values[mapping] = mapping
-                elif isinstance(mapping, (list, tuple)) and len(mapping) == 2:
-                    if callable(mapping[0]) and not isinstance(
-                        mapping[0], DeployTimeField
-                    ):
-                        mapping[0] = DeployTimeField(
-                            "parameter_val", str, None, mapping[0], False
-                        )
-                    if callable(mapping[1]) and not isinstance(
-                        mapping[1], DeployTimeField
-                    ):
-                        mapping[1] = DeployTimeField(
-                            "parameter_val", str, None, mapping[1], False
+                elif isinstance(mapping, tuple) and len(mapping) == 2:
+                    # (param_name, field_name)
+                    param_name, field_name = mapping
+                    if not is_stringish(param_name) or not is_stringish(field_name):
+                        raise MetaflowException(
+                            f"The *parameters* attribute for event {event_name} is invalid. "
+                            "It should be a list/tuple of strings and lists/tuples of size 2."
                         )
-                    new_param_values[mapping[0]] = mapping[1]
+                    new_param_values[param_name] = field_name
                 else:
                     raise MetaflowException(
                         "The *parameters* attribute for event is invalid. "
                         "It should be a list/tuple of strings and lists/tuples of size 2"
                     )
-        elif callable(parameters) and not isinstance(parameters, DeployTimeField):
-            return DeployTimeField(
-                "parameters", [list, dict, tuple], None, parameters, False
-            )
         elif isinstance(parameters, dict):
             for key, value in parameters.items():
-                if callable(key) and not isinstance(key, DeployTimeField):
-                    key = DeployTimeField("flow_parameter", str, None, key, False)
-                if callable(value) and not isinstance(value, DeployTimeField):
-                    value = DeployTimeField("signal_parameter", str, None, value, False)
+                if not is_stringish(key) or not is_stringish(value):
+                    raise MetaflowException(
+                        f"The *parameters* attribute for event {event_name} is invalid. "
+                        "It should be a dictionary of string keys and string values."
+                    )
                 new_param_values[key] = value
+        elif callable(parameters) and not isinstance(parameters, DeployTimeField):
+            # func
+            return DeployTimeField(
+                "parameters",
+                [list, dict, tuple],
+                None,
+                parameters,
+                False,
+                print_representation=str(parameters),
+            )
         return new_param_values
 
     def flow_init(
@@ -158,7 +211,7 @@ class TriggerDecorator(FlowDecorator):
             )
         elif self.attributes["event"]:
             event = self.attributes["event"]
-            processed_event = self.process_event_name(event)
+            processed_event = self.process_event(event)
             self.triggers.append(processed_event)
         elif self.attributes["events"]:
             # events attribute supports the following formats -
@@ -169,13 +222,18 @@ class TriggerDecorator(FlowDecorator):
             if isinstance(self.attributes["events"], list):
                 # process every event in events
                 for event in self.attributes["events"]:
-                    processed_event = self.process_event_name(event)
+                    processed_event = self.process_event(event)
                     self.triggers.append(processed_event)
             elif callable(self.attributes["events"]) and not isinstance(
                 self.attributes["events"], DeployTimeField
             ):
                 trig = DeployTimeField(
-                    "events", list, None, self.attributes["events"], False
+                    "events",
+                    list,
+                    None,
+                    self.attributes["events"],
+                    False,
+                    print_representation=str(self.attributes["events"]),
                 )
                 self.triggers.append(trig)
             else:
@@ -208,101 +266,40 @@ class TriggerDecorator(FlowDecorator):
 
     def format_deploytime_value(self):
         new_triggers = []
+
+        # First pass to evaluate DeployTimeFields
         for trigger in self.triggers:
             # Case where trigger is a function that returns a list of events
             # Need to do this bc we need to iterate over list later
             if isinstance(trigger, DeployTimeField):
                 evaluated_trigger = deploy_time_eval(trigger)
-                if isinstance(evaluated_trigger, dict):
-                    trigger = evaluated_trigger
-                elif isinstance(evaluated_trigger, str):
-                    trigger = {"name": evaluated_trigger}
                 if isinstance(evaluated_trigger, list):
-                    for trig in evaluated_trigger:
-                        if is_stringish(trig):
-                            new_triggers.append({"name": trig})
-                        else:  # dict or another deploytimefield
-                            new_triggers.append(trig)
+                    for event in evaluated_trigger:
+                        new_triggers.append(self.process_event(event))
                 else:
-                    new_triggers.append(trigger)
+                    new_triggers.append(self.process_event(evaluated_trigger))
             else:
                 new_triggers.append(trigger)
 
+        # Second pass to evaluate names
+        for trigger in new_triggers:
+            name = trigger.get("name")
+            if isinstance(name, DeployTimeField):
+                trigger["name"] = deploy_time_eval(name)
+                if not is_stringish(trigger["name"]):
+                    raise MetaflowException(
+                        f"The *name* attribute for event {trigger} is not a valid string"
+                    )
+
+        # third pass to evaluate parameters
+        for trigger in new_triggers:
+            parameters = trigger.get("parameters", {})
+            if isinstance(parameters, DeployTimeField):
+                parameters_eval = deploy_time_eval(parameters)
+                parameters = self.process_parameters(parameters_eval, trigger["name"])
+                trigger["parameters"] = parameters
+
         self.triggers = new_triggers
-        for trigger in self.triggers:
-            old_trigger = trigger
-            trigger_params = trigger.get("parameters", {})
-            # Case where param is a function (can return list or dict)
-            if isinstance(trigger_params, DeployTimeField):
-                trigger_params = deploy_time_eval(trigger_params)
-            # If params is a list of strings, convert to dict with same key and value
-            if isinstance(trigger_params, (list, tuple)):
-                new_trigger_params = {}
-                for mapping in trigger_params:
-                    if is_stringish(mapping) or callable(mapping):
-                        new_trigger_params[mapping] = mapping
-                    elif callable(mapping) and not isinstance(mapping, DeployTimeField):
-                        mapping = DeployTimeField(
-                            "parameter_val", str, None, mapping, False
-                        )
-                        new_trigger_params[mapping] = mapping
-                    elif isinstance(mapping, (list, tuple)) and len(mapping) == 2:
-                        if callable(mapping[0]) and not isinstance(
-                            mapping[0], DeployTimeField
-                        ):
-                            mapping[0] = DeployTimeField(
-                                "parameter_val",
-                                str,
-                                None,
-                                mapping[1],
-                                False,
-                            )
-                        if callable(mapping[1]) and not isinstance(
-                            mapping[1], DeployTimeField
-                        ):
-                            mapping[1] = DeployTimeField(
-                                "parameter_val",
-                                str,
-                                None,
-                                mapping[1],
-                                False,
-                            )
-
-                        new_trigger_params[mapping[0]] = mapping[1]
-                    else:
-                        raise MetaflowException(
-                            "The *parameters* attribute for event '%s' is invalid. "
-                            "It should be a list/tuple of strings and lists/tuples "
-                            "of size 2" % self.attributes["event"]["name"]
-                        )
-                trigger_params = new_trigger_params
-            trigger["parameters"] = trigger_params
-
-            trigger_name = trigger.get("name")
-            # Case where just the name is a function (always a str)
-            if isinstance(trigger_name, DeployTimeField):
-                trigger_name = deploy_time_eval(trigger_name)
-                trigger["name"] = trigger_name
-
-            # Third layer
-            # {name:, parameters:[func, ..., ...]}
-            # {name:, parameters:{func : func2}}
-            for trigger in self.triggers:
-                old_trigger = trigger
-                trigger_params = trigger.get("parameters", {})
-                new_trigger_params = {}
-                for key, value in trigger_params.items():
-                    if isinstance(value, DeployTimeField) and key is value:
-                        evaluated_param = deploy_time_eval(value)
-                        new_trigger_params[evaluated_param] = evaluated_param
-                    elif isinstance(value, DeployTimeField):
-                        new_trigger_params[key] = deploy_time_eval(value)
-                    elif isinstance(key, DeployTimeField):
-                        new_trigger_params[deploy_time_eval(key)] = value
-                    else:
-                        new_trigger_params[key] = value
-                trigger["parameters"] = new_trigger_params
-            self.triggers[self.triggers.index(old_trigger)] = trigger
 
 
 class TriggerOnFinishDecorator(FlowDecorator):
@@ -402,7 +399,14 @@ class TriggerOnFinishDecorator(FlowDecorator):
             if callable(flow) and not isinstance(
                 self.attributes["flow"], DeployTimeField
             ):
-                trig = DeployTimeField("fq_name", [str, dict], None, flow, False)
+                trig = DeployTimeField(
+                    "fq_name",
+                    [str, dict],
+                    None,
+                    flow,
+                    False,
+                    print_representation=str(flow),
+                )
                 self.triggers.append(trig)
             else:
                 self.triggers.extend(self._parse_static_triggers([flow]))
@@ -411,7 +415,9 @@ class TriggerOnFinishDecorator(FlowDecorator):
             #     1. flows=['FooFlow', 'BarFlow']
             flows = self.attributes["flows"]
             if callable(flows) and not isinstance(flows, DeployTimeField):
-                trig = DeployTimeField("flows", list, None, flows, False)
+                trig = DeployTimeField(
+                    "flows", list, None, flows, False, print_representation=str(flows)
+                )
                 self.triggers.append(trig)
             elif isinstance(flows, list):
                 self.triggers.extend(self._parse_static_triggers(flows))

metaflow/plugins/pypi/pip.py

@@ -60,6 +60,15 @@ class Pip(object):
         else:
             self.logger = lambda *args, **kwargs: None  # No-op logger if not provided
 
+    def _get_resolved_python_version(self, prefix):
+        try:
+            result = self.micromamba._call(["list", "--prefix", prefix, "--json"])
+            for package in result:
+                if package.get("name") == "python":
+                    return package["version"]
+        except Exception:
+            return None
+
     def solve(self, id_, packages, python, platform):
         prefix = self.micromamba.path_to_environment(id_)
         if prefix is None:
@@ -67,13 +76,19 @@ class Pip(object):
             msg += "for id {id}".format(id=id_)
             raise PipException(msg)
 
+        resolved_python = self._get_resolved_python_version(prefix)
+        if not resolved_python:
+            raise PipException(
+                "Could not determine Python version from conda environment"
+            )
+
         debug.conda_exec("Solving packages for PyPI environment %s" % id_)
         with tempfile.TemporaryDirectory() as tmp_dir:
             report = "{tmp_dir}/report.json".format(tmp_dir=tmp_dir)
             implementations, platforms, abis = zip(
                 *[
                     (tag.interpreter, tag.platform, tag.abi)
-                    for tag in pip_tags(python, platform)
+                    for tag in pip_tags(resolved_python, platform)
                 ]
             )
             custom_index_url, extra_index_urls = self.indices(prefix)
@@ -142,6 +157,13 @@ class Pip(object):
 
     def download(self, id_, packages, python, platform):
         prefix = self.micromamba.path_to_environment(id_)
+
+        resolved_python = self._get_resolved_python_version(prefix)
+        if not resolved_python:
+            raise PipException(
+                "Could not determine Python version from conda environment"
+            )
+
         metadata_file = METADATA_FILE.format(prefix=prefix)
         # download packages only if they haven't ever been downloaded before
         if os.path.isfile(metadata_file):
@@ -192,7 +214,11 @@ class Pip(object):
                     if os.path.isfile(os.path.join(path, f)) and f.endswith(".whl")
                 ]
                 if (
-                    len(set(pip_tags(python, platform)).intersection(wheel_tags(wheel)))
+                    len(
+                        set(pip_tags(resolved_python, platform)).intersection(
+                            wheel_tags(wheel)
+                        )
+                    )
                     == 0
                 ):
                     raise PipException(
@@ -211,7 +237,7 @@ class Pip(object):
         implementations, platforms, abis = zip(
             *[
                 (tag.interpreter, tag.platform, tag.abi)
-                for tag in pip_tags(python, platform)
+                for tag in pip_tags(resolved_python, platform)
             ]
         )
 

metaflow/version.py

@@ -1 +1 @@
-metaflow_version = "2.18.5"
+metaflow_version = "2.18.7"

{metaflow-2.18.5.dist-info → metaflow-2.18.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: metaflow
-Version: 2.18.5
+Version: 2.18.7
 Summary: Metaflow: More AI and ML, Less Engineering
 Author: Metaflow Developers
 Author-email: help@metaflow.org
@@ -26,7 +26,7 @@ License-File: LICENSE
 Requires-Dist: requests
 Requires-Dist: boto3
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.18.5; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.18.7; extra == "stubs"
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier

{metaflow-2.18.5.dist-info → metaflow-2.18.7.dist-info}/RECORD

@@ -36,7 +36,7 @@ metaflow/tuple_util.py,sha256=_G5YIEhuugwJ_f6rrZoelMFak3DqAR2tt_5CapS1XTY,830
 metaflow/unbounded_foreach.py,sha256=p184WMbrMJ3xKYHwewj27ZhRUsSj_kw1jlye5gA9xJk,387
 metaflow/util.py,sha256=g2SOU_CRzJLgDM_UGF9QDMANMAIHAsDRXE6S76_YzsY,14594
 metaflow/vendor.py,sha256=A82CGHfStZGDP5pQ5XzRjFkbN1ZC-vFmghXIrzMDDNg,5868
-metaflow/version.py,sha256=j-5Xjp6k9jr89uo7zqXR9auyDCCWozyQgBHgn9_8I8M,28
+metaflow/version.py,sha256=AIRLGaVLM-FX3-8i8T2EI3rpQbi7la4g-ypfTc6tIww,28
 metaflow/_vendor/__init__.py,sha256=y_CiwUD3l4eAKvTVDZeqgVujMy31cAM1qjAB-HfI-9s,353
 metaflow/_vendor/typing_extensions.py,sha256=q9zxWa6p6CzF1zZvSkygSlklduHf_b3K7MCxGz7MJRc,134519
 metaflow/_vendor/zipp.py,sha256=ajztOH-9I7KA_4wqDYygtHa6xUBVZgFpmZ8FE74HHHI,8425
@@ -203,7 +203,7 @@ metaflow/plugins/catch_decorator.py,sha256=vorlDA6MLB2yHSsEuBoNzAZbrJ6Vknj1qJO9v
 metaflow/plugins/debug_logger.py,sha256=mcF5HYzJ0NQmqCMjyVUk3iAP-heroHRIiVWQC6Ha2-I,879
 metaflow/plugins/debug_monitor.py,sha256=Md5X_sDOSssN9pt2D8YcaIjTK5JaQD55UAYTcF6xYF0,1099
 metaflow/plugins/environment_decorator.py,sha256=6m9j2B77d-Ja_l_9CTJ__0O6aB2a8Qt_lAZu6UjAcUA,587
-metaflow/plugins/events_decorator.py,sha256=T_YSK-DlgZhd3ge9PlpTRNaMi15GK0tKZMZl1NdV9DQ,24403
+metaflow/plugins/events_decorator.py,sha256=FULT_Iuue9KsLyNhHOL46uw8cdT-Viq8rmcgEHVRlF0,22704
 metaflow/plugins/logs_cli.py,sha256=77W5UNagU2mOKSMMvrQxQmBLRzvmjK-c8dWxd-Ygbqs,11410
 metaflow/plugins/package_cli.py,sha256=GOxKJs9Wt0x9VtcTJ2EG_wj6pWnlS0XBFAyHpCzzuvs,2000
 metaflow/plugins/parallel_decorator.py,sha256=wtR_3eRIP3eV7fBIm15oouRjmHBFZ9OklxdaNvttLEQ,9702
@@ -361,7 +361,7 @@ metaflow/plugins/pypi/conda_decorator.py,sha256=fXG9EvImP4Eqle_Trhb3tQhs40xTc4cx
 metaflow/plugins/pypi/conda_environment.py,sha256=j6QaIWmP50K2veN6Sywa04S2hfByR1YMqy7OyZpwvYs,24925
 metaflow/plugins/pypi/micromamba.py,sha256=UltfY8NmLphfZ-AbpaMFIdxIeOXLdTYDrMrabvPrYVU,17352
 metaflow/plugins/pypi/parsers.py,sha256=gpOOG2Ph95wI73MWCAi7XjpK0gYhv5k5YIGBs73QPuE,8556
-metaflow/plugins/pypi/pip.py,sha256=WhPyA18RoVT40sqbZRJdCEij4euL9PZwLfm1SrAKqmU,14333
+metaflow/plugins/pypi/pip.py,sha256=N5C3tjuWG2pr4VGZR8YgnRc0HwfRL7LZmOnxxA9ZmEc,15239
 metaflow/plugins/pypi/pypi_decorator.py,sha256=sBxmDH3WIWVfJDsSauiMy58fevOJuJeLSuS6MnIpw44,7312
 metaflow/plugins/pypi/pypi_environment.py,sha256=FYMg8kF3lXqcLfRYWD83a9zpVjcoo_TARqMGZ763rRk,230
 metaflow/plugins/pypi/utils.py,sha256=W8OhDrhz7YIE-2MSSN5Rj7AmESwsN5YDmdnro152J4w,3551
@@ -428,12 +428,12 @@ metaflow/user_decorators/mutable_flow.py,sha256=EywKTN3cnXPQF_s62wQaC4a4aH14j8oe
 metaflow/user_decorators/mutable_step.py,sha256=-BY0UDXf_RCAEnC5JlLzEXGdiw1KD9oSrSxS_SWaB9Y,16791
 metaflow/user_decorators/user_flow_decorator.py,sha256=2yDwZq9QGv9W-7kEuKwa8o4ZkTvuHJ5ESz7VVrGViAI,9890
 metaflow/user_decorators/user_step_decorator.py,sha256=4558NR8RJtN22OyTwCXO80bAMhMTaRGMoX12b1GMcPc,27232
-metaflow-2.18.5.data/data/share/metaflow/devtools/Makefile,sha256=TT4TCq8ALSfqYyGqDPocN5oPcZe2FqoCZxmGO1LmyCc,13760
-metaflow-2.18.5.data/data/share/metaflow/devtools/Tiltfile,sha256=Ty5p6AD3MwJAcAnOGv4yMz8fExAsnNQ11r8whK6uzzw,21381
-metaflow-2.18.5.data/data/share/metaflow/devtools/pick_services.sh,sha256=DCnrMXwtApfx3B4S-YiZESMyAFHbXa3VuNL0MxPLyiE,2196
-metaflow-2.18.5.dist-info/licenses/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
-metaflow-2.18.5.dist-info/METADATA,sha256=sjfM1I825FwjvAscfoOhfUei4-SO9W_kWlZBXgbJGKc,6740
-metaflow-2.18.5.dist-info/WHEEL,sha256=JNWh1Fm1UdwIQV075glCn4MVuCRs0sotJIq-J6rbxCU,109
-metaflow-2.18.5.dist-info/entry_points.txt,sha256=RvEq8VFlgGe_FfqGOZi0D7ze1hLD0pAtXeNyGfzc_Yc,103
-metaflow-2.18.5.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
-metaflow-2.18.5.dist-info/RECORD,,
+metaflow-2.18.7.data/data/share/metaflow/devtools/Makefile,sha256=TT4TCq8ALSfqYyGqDPocN5oPcZe2FqoCZxmGO1LmyCc,13760
+metaflow-2.18.7.data/data/share/metaflow/devtools/Tiltfile,sha256=Ty5p6AD3MwJAcAnOGv4yMz8fExAsnNQ11r8whK6uzzw,21381
+metaflow-2.18.7.data/data/share/metaflow/devtools/pick_services.sh,sha256=DCnrMXwtApfx3B4S-YiZESMyAFHbXa3VuNL0MxPLyiE,2196
+metaflow-2.18.7.dist-info/licenses/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
+metaflow-2.18.7.dist-info/METADATA,sha256=rjd-ZidDUaQyWvbyExLrzZmadNpHPsqEiOkg-TRTlZ4,6740
+metaflow-2.18.7.dist-info/WHEEL,sha256=JNWh1Fm1UdwIQV075glCn4MVuCRs0sotJIq-J6rbxCU,109
+metaflow-2.18.7.dist-info/entry_points.txt,sha256=RvEq8VFlgGe_FfqGOZi0D7ze1hLD0pAtXeNyGfzc_Yc,103
+metaflow-2.18.7.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
+metaflow-2.18.7.dist-info/RECORD,,