metaflow 2.16.0__tar.gz → 2.16.2__tar.gz

{metaflow-2.16.0/metaflow.egg-info → metaflow-2.16.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: metaflow
-Version: 2.16.0
+Version: 2.16.2
 Summary: Metaflow: More AI and ML, Less Engineering
 Author: Metaflow Developers
 Author-email: help@metaflow.org
@@ -26,7 +26,7 @@ License-File: LICENSE
 Requires-Dist: requests
 Requires-Dist: boto3
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.16.0; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.16.2; extra == "stubs"
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier

{metaflow-2.16.0 → metaflow-2.16.2}/metaflow/cli.py

@@ -447,8 +447,8 @@ def start(
         # be raised. For resume, since we ignore those options, we ignore the error.
         raise ctx.obj.delayed_config_exception
 
-    # Init all values in the config decorators and then process them
-    for decorator in ctx.obj.flow._flow_state.get(_FlowState.CONFIG_DECORATORS, []):
+    # Init all values in the flow mutators and then process them
+    for decorator in ctx.obj.flow._flow_state.get(_FlowState.FLOW_MUTATORS, []):
         decorator.external_init()
 
     new_cls = ctx.obj.flow._process_config_decorators(config_options)
@@ -562,6 +562,9 @@ def start(
             ctx.obj.logger,
         )
 
+        # Check the graph again (mutators may have changed it)
+        ctx.obj.graph = ctx.obj.flow._graph
+
         # TODO (savin): Enable lazy instantiation of package
         ctx.obj.package = None
 

{metaflow-2.16.0 → metaflow-2.16.2}/metaflow/cli_components/run_cmds.py

@@ -54,6 +54,8 @@ def before_run(obj, tags, decospecs):
     decorators._init_step_decorators(
         obj.flow, obj.graph, obj.environment, obj.flow_datastore, obj.logger
     )
+    # Re-read graph since it may have been modified by mutators
+    obj.graph = obj.flow._graph
 
     obj.metadata.add_sticky_tags(tags=tags)
 

{metaflow-2.16.0 → metaflow-2.16.2}/metaflow/decorators.py

@@ -715,13 +715,16 @@ def _init_flow_decorators(
 
 
 def _init_step_decorators(flow, graph, environment, flow_datastore, logger):
+    # NOTE: We don't need graph but keeping it for backwards compatibility with
+    # extensions that use it directly. We will remove it at some point.
+
     # We call the mutate method for both the flow and step mutators.
     cls = flow.__class__
     # Run all the decorators. We first run the flow-level decorators
     # and then the step level ones to maintain a consistent order with how
     # other decorators are run.
 
-    for deco in cls._flow_state.get(_FlowState.CONFIG_DECORATORS, []):
+    for deco in cls._flow_state.get(_FlowState.FLOW_MUTATORS, []):
         if isinstance(deco, FlowMutator):
             inserted_by_value = [deco.decorator_name] + (deco.inserted_by or [])
             mutable_flow = MutableFlow(
@@ -777,6 +780,9 @@ def _init_step_decorators(flow, graph, environment, flow_datastore, logger):
             # We remove all mention of the custom step decorator
             setattr(cls, step.name, step)
 
+    cls._init_graph()
+    graph = flow._graph
+
     for step in flow:
         for deco in step.decorators:
             deco.step_init(

{metaflow-2.16.0 → metaflow-2.16.2}/metaflow/flowspec.py

@@ -78,7 +78,7 @@ class ParallelUBF(UnboundedForeachInput):
 
 class _FlowState(Enum):
     CONFIGS = 1
-    CONFIG_DECORATORS = 2
+    FLOW_MUTATORS = 2
     CACHED_PARAMETERS = 3
     SET_CONFIG_PARAMETERS = (
         4  # These are Parameters that now have a ConfigValue (converted)
@@ -135,12 +135,17 @@ class FlowSpecMeta(type):
                         raise DuplicateFlowDecoratorException(deco_name)
                     cls._flow_decorators.setdefault(deco_name, []).extend(deco)
 
-                # Take care of configs and config decorators
-                base_configs = base._flow_state.get(_FlowState.CONFIG_DECORATORS)
+                # Take care of configs and flow mutators
+                base_configs = base._flow_state.get(_FlowState.CONFIGS)
                 if base_configs:
-                    cls._flow_state.setdefault(_FlowState.CONFIG_DECORATORS, []).extend(
+                    cls._flow_state.setdefault(_FlowState.CONFIGS, {}).update(
                         base_configs
                     )
+                base_mutators = base._flow_state.get(_FlowState.FLOW_MUTATORS)
+                if base_mutators:
+                    cls._flow_state.setdefault(_FlowState.FLOW_MUTATORS, []).extend(
+                        base_mutators
+                    )
 
         cls._init_graph()
 
@@ -245,7 +250,7 @@ class FlowSpec(metaclass=FlowSpecMeta):
 
         # Fast path for no user configurations
         if not process_configs or (
-            not cls._flow_state.get(_FlowState.CONFIG_DECORATORS)
+            not cls._flow_state.get(_FlowState.FLOW_MUTATORS)
             and all(len(step.config_decorators) == 0 for step in cls._steps)
         ):
             # Process parameters to allow them to also use config values easily
@@ -284,7 +289,7 @@ class FlowSpec(metaclass=FlowSpecMeta):
         # and then the step level ones to maintain a consistent order with how
         # other decorators are run.
 
-        for deco in cls._flow_state.get(_FlowState.CONFIG_DECORATORS, []):
+        for deco in cls._flow_state.get(_FlowState.FLOW_MUTATORS, []):
             if isinstance(deco, FlowMutator):
                 inserted_by_value = [deco.decorator_name] + (deco.inserted_by or [])
                 mutable_flow = MutableFlow(
@@ -423,7 +428,7 @@ class FlowSpec(metaclass=FlowSpecMeta):
                     "statically_defined": deco.statically_defined,
                     "inserted_by": deco.inserted_by,
                 }
-                for deco in self._flow_state.get(_FlowState.CONFIG_DECORATORS, [])
+                for deco in self._flow_state.get(_FlowState.FLOW_MUTATORS, [])
             ],
             "extensions": extension_info(),
         }

{metaflow-2.16.0 → metaflow-2.16.2}/metaflow/packaging_sys/v1.py

@@ -41,7 +41,11 @@ class MetaflowCodeContentV1(MetaflowCodeContentV1Base):
         # We try to find the modules we need to package. We will first look at all modules
         # and apply the criteria to them. Then we will use the most parent module that
         # fits the criteria as the module to package
-        modules = filter(lambda x: criteria(x[1]), sys.modules.items())
+
+        # Make a copy since sys.modules could be modified while we load other
+        # modules. See https://github.com/Netflix/metaflow/issues/2489
+        all_modules = dict(sys.modules)
+        modules = filter(lambda x: criteria(x[1]), all_modules.items())
         # Ensure that we see the parent modules first
         modules = sorted(modules, key=lambda x: x[0])
         if modules:

{metaflow-2.16.0 → metaflow-2.16.2}/metaflow/plugins/airflow/airflow_cli.py

@@ -289,7 +289,7 @@ def make_flow(
     decorators._init_step_decorators(
         obj.flow, obj.graph, obj.environment, obj.flow_datastore, obj.logger
     )
-
+    obj.graph = obj.flow._graph
     # Save the code package in the flow datastore so that both user code and
     # metaflow package can be retrieved during workflow execution.
     obj.package = MetaflowPackage(

{metaflow-2.16.0 → metaflow-2.16.2}/metaflow/plugins/argo/argo_workflows.py

@@ -2125,8 +2125,17 @@ class ArgoWorkflows(object):
                     .node_selectors(resources.get("node_selector"))
                     # Set tolerations
                     .tolerations(resources.get("tolerations"))
-                    # Set image pull secrets
-                    .image_pull_secrets(resources.get("image_pull_secrets"))
+                    # Set image pull secrets if present. We need to use pod_spec_patch due to Argo not supporting this on a template level.
+                    .pod_spec_patch(
+                        {
+                            "imagePullSecrets": [
+                                {"name": secret}
+                                for secret in resources["image_pull_secrets"]
+                            ]
+                        }
+                        if resources["image_pull_secrets"]
+                        else None
+                    )
                     # Set container
                     .container(
                         # TODO: Unify the logic with kubernetes.py
@@ -3825,6 +3834,14 @@ class Template(object):
             )
         return self
 
+    def pod_spec_patch(self, pod_spec_patch=None):
+        if pod_spec_patch is None:
+            return self
+
+        self.payload["podSpecPatch"] = json.dumps(pod_spec_patch)
+
+        return self
+
     def node_selectors(self, node_selectors):
         if "nodeSelector" not in self.payload:
             self.payload["nodeSelector"] = {}
@@ -3836,10 +3853,6 @@ class Template(object):
         self.payload["tolerations"] = tolerations
         return self
 
-    def image_pull_secrets(self, image_pull_secrets):
-        self.payload["image_pull_secrets"] = image_pull_secrets
-        return self
-
     def to_json(self):
         return self.payload
 

{metaflow-2.16.0 → metaflow-2.16.2}/metaflow/plugins/argo/argo_workflows_cli.py

@@ -515,6 +515,7 @@ def make_flow(
     decorators._init_step_decorators(
         obj.flow, obj.graph, obj.environment, obj.flow_datastore, obj.logger
     )
+    obj.graph = obj.flow._graph
 
     # Save the code package in the flow datastore so that both user code and
     # metaflow package can be retrieved during workflow execution.

{metaflow-2.16.0 → metaflow-2.16.2}/metaflow/plugins/aws/step_functions/step_functions_cli.py

@@ -330,6 +330,7 @@ def make_flow(
     decorators._init_step_decorators(
         obj.flow, obj.graph, obj.environment, obj.flow_datastore, obj.logger
     )
+    obj.graph = obj.flow._graph
 
     obj.package = MetaflowPackage(
         obj.flow,

{metaflow-2.16.0 → metaflow-2.16.2}/metaflow/plugins/metadata_providers/service.py

@@ -591,7 +591,7 @@ class ServiceMetadataProvider(MetadataProvider):
             else:
                 if resp.status_code < 300:
                     return resp.headers.get("METADATA_SERVICE_VERSION", None)
-                elif resp.status_code != 503:
+                elif resp.status_code not in (503, 500):
                     raise ServiceException(
                         "Metadata request (%s) failed"
                         " (code %s): %s" % (url, resp.status_code, resp.text),

{metaflow-2.16.0 → metaflow-2.16.2}/metaflow/plugins/uv/bootstrap.py

@@ -5,7 +5,7 @@ import sys
 import time
 
 from metaflow.util import which
-from metaflow.info_file import read_info_file
+from metaflow.meta_files import read_info_file
 from metaflow.metaflow_config import get_pinned_conda_libs
 from metaflow.packaging_sys import MetaflowCodeContent, ContentType
 from urllib.request import Request, urlopen

{metaflow-2.16.0 → metaflow-2.16.2}/metaflow/task.py

@@ -106,14 +106,15 @@ class MetaflowTask(object):
         except Exception as ex:
             raised_exception = ex
 
-        if do_next:
+        if do_next or raised_exception:
             # If we are skipping the step, or executed a wrapped function,
             # we need to set the transition variables
             # properly. We call the next function as needed
-            graph_node = self.flow._graph[step_function.name]
+            # We also do this in case we want to gobble the exception.
+            graph_node = self.flow._graph[orig_step_func.name]
             out_funcs = [getattr(self.flow, f) for f in graph_node.out_funcs]
             if out_funcs:
-                if isinstance(do_next, bool):
+                if isinstance(do_next, bool) or raised_exception:
                     # We need to extract things from the self.next. This is not possible
                     # in the case where there was a num_parallel.
                     if graph_node.parallel_foreach:
@@ -137,7 +138,7 @@ class MetaflowTask(object):
         # We back out of the stack of generators
         for w in reversed(wrappers_stack):
             raised_exception = w.post_step(
-                step_function.name, self.flow, raised_exception
+                orig_step_func.name, self.flow, raised_exception
             )
         if raised_exception:
             # We have an exception that we need to propagate

{metaflow-2.16.0 → metaflow-2.16.2}/metaflow/user_decorators/mutable_step.py

@@ -368,7 +368,6 @@ class MutableStep:
         if issubclass(canonical_deco_type, UserStepDecoratorBase):
             for attr in ["config_decorators", "wrappers"]:
                 new_deco_list = []
-
                 for deco in getattr(self._my_step, attr):
                     if deco.decorator_name == canonical_deco_type.decorator_name:
                         if do_all:
@@ -396,6 +395,7 @@ class MutableStep:
 
         if did_remove:
             return True
+        new_deco_list = []
         for deco in self._my_step.decorators:
             if deco.name == deco_name:
                 if do_all:

{metaflow-2.16.0 → metaflow-2.16.2}/metaflow/user_decorators/user_flow_decorator.py

@@ -190,7 +190,7 @@ class FlowMutator(metaclass=FlowMutatorMeta):
     ) -> "metaflow.flowspec.FlowSpecMeta":
         from ..flowspec import _FlowState
 
-        flow_spec._flow_state.setdefault(_FlowState.CONFIG_DECORATORS, []).append(self)
+        flow_spec._flow_state.setdefault(_FlowState.FLOW_MUTATORS, []).append(self)
         self._flow_cls = flow_spec
         return flow_spec
 
@@ -225,6 +225,7 @@ class FlowMutator(metaclass=FlowMutatorMeta):
                 raise MetaflowException(
                     "%s is used with arguments but does not implement init" % self
                 )
+        if "init" in self.__class__.__dict__:
             self.init(*self._args, **self._kwargs)
 
     def pre_mutate(

{metaflow-2.16.0 → metaflow-2.16.2}/metaflow/user_decorators/user_step_decorator.py

@@ -393,7 +393,7 @@ class UserStepDecoratorBase(metaclass=UserStepDecoratorMeta):
                 raise MetaflowException(
                     "%s is used with arguments but does not implement init" % self
                 )
-
+        if "init" in self.__class__.__dict__:
             self.init(*self._args, **self._kwargs)
 
 

metaflow-2.16.2/metaflow/version.py

@@ -0,0 +1 @@
+metaflow_version = "2.16.2"

{metaflow-2.16.0 → metaflow-2.16.2/metaflow.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: metaflow
-Version: 2.16.0
+Version: 2.16.2
 Summary: Metaflow: More AI and ML, Less Engineering
 Author: Metaflow Developers
 Author-email: help@metaflow.org
@@ -26,7 +26,7 @@ License-File: LICENSE
 Requires-Dist: requests
 Requires-Dist: boto3
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.16.0; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.16.2; extra == "stubs"
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier

{metaflow-2.16.0 → metaflow-2.16.2}/metaflow.egg-info/requires.txt

@@ -2,4 +2,4 @@ requests
 boto3
 
 [stubs]
-metaflow-stubs==2.16.0
+metaflow-stubs==2.16.2

metaflow-2.16.0/metaflow/version.py

@@ -1 +0,0 @@
-metaflow_version = "2.16.0"