metaflow 2.12.39__tar.gz → 2.13.1__tar.gz

{metaflow-2.12.39/metaflow.egg-info → metaflow-2.13.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: metaflow
-Version: 2.12.39
+Version: 2.13.1
 Summary: Metaflow: More Data Science, Less Engineering
 Author: Metaflow Developers
 Author-email: help@metaflow.org
@@ -26,7 +26,7 @@ License-File: LICENSE
 Requires-Dist: requests
 Requires-Dist: boto3
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.12.39; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.13.1; extra == "stubs"
 
 ![Metaflow_Logo_Horizontal_FullColor_Ribbon_Dark_RGB](https://user-images.githubusercontent.com/763451/89453116-96a57e00-d713-11ea-9fa6-82b29d4d6eff.png)
 

{metaflow-2.12.39 → metaflow-2.13.1}/metaflow/__init__.py

@@ -103,7 +103,7 @@ from .flowspec import FlowSpec
 
 from .parameters import Parameter, JSONTypeClass, JSONType
 
-from .user_configs.config_parameters import Config, config_expr
+from .user_configs.config_parameters import Config, ConfigValue, config_expr
 from .user_configs.config_decorators import CustomFlowDecorator, CustomStepDecorator
 
 # data layer

{metaflow-2.12.39 → metaflow-2.13.1}/metaflow/cli.py

@@ -10,7 +10,8 @@ from metaflow._vendor import click
 from . import decorators, lint, metaflow_version, parameters, plugins
 from .cli_args import cli_args
 from .cli_components.utils import LazyGroup, LazyPluginCommandCollection
-from .datastore import FlowDataStore
+from .datastore import FlowDataStore, TaskDataStoreSet
+from .debug import debug
 from .exception import CommandException, MetaflowException
 from .flowspec import _FlowState
 from .graph import FlowGraph
@@ -35,7 +36,7 @@ from .plugins import (
 )
 from .pylint_wrapper import PyLint
 from .R import metaflow_r_version, use_r
-from .util import resolve_identity
+from .util import get_latest_run_id, resolve_identity
 from .user_configs.config_options import LocalFileInput, config_options
 from .user_configs.config_parameters import ConfigValue
 
@@ -169,7 +170,8 @@ def check(obj, warnings=False):
 @click.pass_obj
 def show(obj):
     echo_always("\n%s" % obj.graph.doc)
-    for _, node in sorted((n.func_lineno, n) for n in obj.graph):
+    for node_name in obj.graph.sorted_nodes:
+        node = obj.graph[node_name]
         echo_always("\nStep *%s*" % node.name, err=False)
         echo_always(node.doc if node.doc else "?", indent=True, err=False)
         if node.type != "end":
@@ -328,8 +330,8 @@ def start(
     event_logger=None,
     monitor=None,
     local_config_file=None,
-    config_file_options=None,
-    config_value_options=None,
+    config_file=None,
+    config_value=None,
     **deco_options
 ):
     if quiet:
@@ -346,6 +348,33 @@ def start(
     echo(" executing *%s*" % ctx.obj.flow.name, fg="magenta", nl=False)
     echo(" for *%s*" % resolve_identity(), fg="magenta")
 
+    # Setup the context
+    cli_args._set_top_kwargs(ctx.params)
+    ctx.obj.echo = echo
+    ctx.obj.echo_always = echo_always
+    ctx.obj.is_quiet = quiet
+    ctx.obj.logger = logger
+    ctx.obj.pylint = pylint
+    ctx.obj.check = functools.partial(_check, echo)
+    ctx.obj.top_cli = cli
+    ctx.obj.package_suffixes = package_suffixes.split(",")
+
+    ctx.obj.datastore_impl = [d for d in DATASTORES if d.TYPE == datastore][0]
+
+    if datastore_root is None:
+        datastore_root = ctx.obj.datastore_impl.get_datastore_root_from_config(
+            ctx.obj.echo
+        )
+    if datastore_root is None:
+        raise CommandException(
+            "Could not find the location of the datastore -- did you correctly set the "
+            "METAFLOW_DATASTORE_SYSROOT_%s environment variable?" % datastore.upper()
+        )
+
+    ctx.obj.datastore_impl.datastore_root = datastore_root
+
+    FlowDataStore.default_storage_impl = ctx.obj.datastore_impl
+
     # At this point, we are able to resolve the user-configuration options so we can
     # process all those decorators that the user added that will modify the flow based
     # on those configurations. It is important to do this as early as possible since it
@@ -354,19 +383,76 @@ def start(
     # When we process the options, the first one processed will return None and the
     # second one processed will return the actual options. The order of processing
     # depends on what (and in what order) the user specifies on the command line.
-    config_options = config_file_options or config_value_options
-    ctx.obj.flow = ctx.obj.flow._process_config_decorators(config_options)
+    config_options = config_file or config_value
+
+    if (
+        hasattr(ctx, "saved_args")
+        and ctx.saved_args
+        and ctx.saved_args[0] == "resume"
+        and getattr(ctx.obj, "has_config_options", False)
+    ):
+        # In the case of resume, we actually need to load the configurations
+        # from the resumed run to process them. This can be slightly onerous so check
+        # if we need to in the first place
+        if getattr(ctx.obj, "has_cl_config_options", False):
+            raise click.UsageError(
+                "Cannot specify --config-file or --config-value with 'resume'"
+            )
+        # We now load the config artifacts from the original run id
+        run_id = None
+        try:
+            idx = ctx.saved_args.index("--origin-run-id")
+        except ValueError:
+            idx = -1
+        if idx >= 0:
+            run_id = ctx.saved_args[idx + 1]
+        else:
+            run_id = get_latest_run_id(ctx.obj.echo, ctx.obj.flow.name)
+        if run_id is None:
+            raise CommandException(
+                "A previous run id was not found. Specify --origin-run-id."
+            )
+        # We get the name of the parameters we need to load from the datastore -- these
+        # are accessed using the *variable* name and not necessarily the *parameter* name
+        config_var_names = []
+        config_param_names = []
+        for name, param in ctx.obj.flow._get_parameters():
+            if not param.IS_CONFIG_PARAMETER:
+                continue
+            config_var_names.append(name)
+            config_param_names.append(param.name)
+
+        # We just need a task datastore that will be thrown away -- we do this so
+        # we don't have to create the logger, monitor, etc.
+        debug.userconf_exec("Loading config parameters from run %s" % run_id)
+        for d in TaskDataStoreSet(
+            FlowDataStore(ctx.obj.flow.name),
+            run_id,
+            steps=["_parameters"],
+            prefetch_data_artifacts=config_var_names,
+        ):
+            param_ds = d
+
+        # We can now set the the CONFIGS value in the flow properly. This will overwrite
+        # anything that may have been passed in by default and we will use exactly what
+        # the original flow had. Note that these are accessed through the parameter name
+        ctx.obj.flow._flow_state[_FlowState.CONFIGS].clear()
+        d = ctx.obj.flow._flow_state[_FlowState.CONFIGS]
+        for param_name, var_name in zip(config_param_names, config_var_names):
+            val = param_ds[var_name]
+            debug.userconf_exec("Loaded config %s as: %s" % (param_name, val))
+            d[param_name] = val
+
+    elif getattr(ctx.obj, "delayed_config_exception", None):
+        # If we are not doing a resume, any exception we had parsing configs needs to
+        # be raised. For resume, since we ignore those options, we ignore the error.
+        raise ctx.obj.delayed_config_exception
+
+    new_cls = ctx.obj.flow._process_config_decorators(config_options)
+    if new_cls:
+        ctx.obj.flow = new_cls(use_cli=False)
 
-    cli_args._set_top_kwargs(ctx.params)
-    ctx.obj.echo = echo
-    ctx.obj.echo_always = echo_always
-    ctx.obj.is_quiet = quiet
     ctx.obj.graph = ctx.obj.flow._graph
-    ctx.obj.logger = logger
-    ctx.obj.pylint = pylint
-    ctx.obj.check = functools.partial(_check, echo)
-    ctx.obj.top_cli = cli
-    ctx.obj.package_suffixes = package_suffixes.split(",")
 
     ctx.obj.environment = [
         e for e in ENVIRONMENTS + [MetaflowEnvironment] if e.TYPE == environment
@@ -389,21 +475,6 @@ def start(
         ctx.obj.environment, ctx.obj.flow, ctx.obj.event_logger, ctx.obj.monitor
     )
 
-    ctx.obj.datastore_impl = [d for d in DATASTORES if d.TYPE == datastore][0]
-
-    if datastore_root is None:
-        datastore_root = ctx.obj.datastore_impl.get_datastore_root_from_config(
-            ctx.obj.echo
-        )
-    if datastore_root is None:
-        raise CommandException(
-            "Could not find the location of the datastore -- did you correctly set the "
-            "METAFLOW_DATASTORE_SYSROOT_%s environment variable?" % datastore.upper()
-        )
-
-    ctx.obj.datastore_impl.datastore_root = datastore_root
-
-    FlowDataStore.default_storage_impl = ctx.obj.datastore_impl
     ctx.obj.flow_datastore = FlowDataStore(
         ctx.obj.flow.name,
         ctx.obj.environment,
@@ -463,7 +534,8 @@ def start(
             decorators._attach_decorators(ctx.obj.flow, all_decospecs)
             decorators._init(ctx.obj.flow)
             # Regenerate graph if we attached more decorators
-            ctx.obj.graph = FlowGraph(ctx.obj.flow.__class__)
+            ctx.obj.flow.__class__._init_attrs()
+            ctx.obj.graph = ctx.obj.flow._graph
 
         decorators._init_step_decorators(
             ctx.obj.flow,
@@ -519,10 +591,13 @@ def _check(echo, graph, flow, environment, pylint=True, warnings=False, **kwargs
 
 def print_metaflow_exception(ex):
     echo_always(ex.headline, indent=True, nl=False, bold=True)
-    if ex.line_no is None:
-        echo_always(":")
-    else:
-        echo_always(" on line %d:" % ex.line_no, bold=True)
+    location = ""
+    if ex.source_file is not None:
+        location += " in file %s" % ex.source_file
+    if ex.line_no is not None:
+        location += " on line %d" % ex.line_no
+    location += ":"
+    echo_always(location, bold=True)
     echo_always(ex.message, indent=True, bold=False, padding_bottom=True)
 
 

{metaflow-2.12.39 → metaflow-2.13.1}/metaflow/cli_args.py

@@ -72,10 +72,10 @@ class CLIArgs(object):
             # keyword in Python, so we call it 'decospecs' in click args
             if k == "decospecs":
                 k = "with"
-            if k in ("config_file_options", "config_value_options"):
+            if k in ("config_file", "config_value"):
                 # Special handling here since we gather them all in one option but actually
                 # need to send them one at a time using --config-value <name> kv.<name>.
-                # Note it can be either config_file_options or config_value_options depending
+                # Note it can be either config_file or config_value depending
                 # on click processing order.
                 for config_name in v.keys():
                     yield "--config-value"

{metaflow-2.12.39 → metaflow-2.13.1}/metaflow/cli_components/run_cmds.py

@@ -41,7 +41,9 @@ def before_run(obj, tags, decospecs):
     if all_decospecs:
         decorators._attach_decorators(obj.flow, all_decospecs)
         decorators._init(obj.flow)
-        obj.graph = FlowGraph(obj.flow.__class__)
+        # Regenerate graph if we attached more decorators
+        obj.flow.__class__._init_attrs()
+        obj.graph = obj.flow._graph
 
     obj.check(obj.graph, obj.flow, obj.environment, pylint=obj.pylint)
     # obj.environment.init_environment(obj.logger)

{metaflow-2.12.39 → metaflow-2.13.1}/metaflow/datastore/flow_datastore.py

@@ -13,7 +13,7 @@ class FlowDataStore(object):
     def __init__(
         self,
         flow_name,
-        environment,
+        environment=None,
         metadata=None,
         event_logger=None,
         monitor=None,
@@ -31,7 +31,7 @@ class FlowDataStore(object):
         ----------
         flow_name : str
             The name of the flow
-        environment : MetaflowEnvironment
+        environment : MetaflowEnvironment, optional
             Environment this datastore is operating in
         metadata : MetadataProvider, optional
             The metadata provider to use and update if needed, by default None

{metaflow-2.12.39 → metaflow-2.13.1}/metaflow/exception.py

@@ -43,13 +43,19 @@ class MetaflowExceptionWrapper(Exception):
 class MetaflowException(Exception):
     headline = "Flow failed"
 
-    def __init__(self, msg="", lineno=None):
+    def __init__(self, msg="", lineno=None, source_file=None):
         self.message = msg
         self.line_no = lineno
+        self.source_file = source_file
         super(MetaflowException, self).__init__()
 
     def __str__(self):
-        prefix = "line %d: " % self.line_no if self.line_no else ""
+        prefix = ""
+        if self.source_file:
+            prefix = "%s:" % self.source_file
+        if self.line_no:
+            prefix = "line %d:" % self.line_no
+        prefix = "%s: " % prefix if prefix else ""
         return "%s%s" % (prefix, self.message)
 
 

{metaflow-2.12.39 → metaflow-2.13.1}/metaflow/flowspec.py

@@ -24,6 +24,7 @@ from .extension_support import extension_info
 from .graph import FlowGraph
 from .unbounded_foreach import UnboundedForeachInput
 from .user_configs.config_decorators import (
+    ConfigValue,
     CustomFlowDecorator,
     CustomStepDecorator,
     MutableFlow,
@@ -81,8 +82,10 @@ class _FlowState(Enum):
 
 
 class FlowSpecMeta(type):
-    def __new__(cls, name, bases, dct):
-        f = super().__new__(cls, name, bases, dct)
+    def __init__(cls, name, bases, attrs):
+        super().__init__(name, bases, attrs)
+        if name == "FlowSpec":
+            return
         # We store some state in the flow class itself. This is primarily used to
         # attach global state to a flow. It is *not* an actual global because of
         # Runner/NBRunner. This is also created here in the meta class to avoid it being
@@ -90,12 +93,18 @@ class FlowSpecMeta(type):
 
         # We should move _flow_decorators into this structure as well but keeping it
         # out to limit the changes for now.
-        f._flow_decorators = {}
+        cls._flow_decorators = {}
 
         # Keys are _FlowState enum values
-        f._flow_state = {}
+        cls._flow_state = {}
+
+        cls._init_attrs()
 
-        return f
+    def _init_attrs(cls):
+        # Graph and steps are specific to the class -- store here so we can access
+        # in class method _process_config_decorators
+        cls._graph = FlowGraph(cls)
+        cls._steps = [getattr(cls, node.name) for node in cls._graph]
 
 
 class FlowSpec(metaclass=FlowSpecMeta):
@@ -145,9 +154,6 @@ class FlowSpec(metaclass=FlowSpecMeta):
         self._transition = None
         self._cached_input = {}
 
-        self._graph = FlowGraph(self.__class__)
-        self._steps = [getattr(self, node.name) for node in self._graph]
-
         if use_cli:
             with parameters.flow_context(self.__class__) as _:
                 from . import cli
@@ -171,9 +177,10 @@ class FlowSpec(metaclass=FlowSpecMeta):
             fname = fname[:-1]
         return os.path.basename(fname)
 
-    def _check_parameters(self, config_parameters=False):
+    @classmethod
+    def _check_parameters(cls, config_parameters=False):
         seen = set()
-        for _, param in self._get_parameters():
+        for _, param in cls._get_parameters():
             if param.IS_CONFIG_PARAMETER != config_parameters:
                 continue
             norm = param.name.lower()
@@ -185,17 +192,17 @@ class FlowSpec(metaclass=FlowSpecMeta):
                 )
             seen.add(norm)
 
-    def _process_config_decorators(self, config_options):
-        current_cls = self.__class__
+    @classmethod
+    def _process_config_decorators(cls, config_options, ignore_errors=False):
 
         # Fast path for no user configurations
-        if not self._flow_state.get(_FlowState.CONFIG_DECORATORS):
+        if not cls._flow_state.get(_FlowState.CONFIG_DECORATORS):
             # Process parameters to allow them to also use config values easily
-            for var, param in self._get_parameters():
+            for var, param in cls._get_parameters():
                 if param.IS_CONFIG_PARAMETER:
                     continue
-                param.init()
-            return self
+                param.init(ignore_errors)
+            return None
 
         debug.userconf_exec("Processing mutating step/flow decorators")
         # We need to convert all the user configurations from DelayedEvaluationParameters
@@ -204,8 +211,8 @@ class FlowSpec(metaclass=FlowSpecMeta):
         # We then reset them to be proper configs so they can be re-evaluated in
         # _set_constants
         to_reset_configs = []
-        self._check_parameters(config_parameters=True)
-        for var, param in self._get_parameters():
+        cls._check_parameters(config_parameters=True)
+        for var, param in cls._get_parameters():
             if not param.IS_CONFIG_PARAMETER:
                 continue
             # Note that a config with no default and not required will be None
@@ -217,38 +224,36 @@ class FlowSpec(metaclass=FlowSpecMeta):
             param._store_value(val)
             to_reset_configs.append((var, param))
             debug.userconf_exec("Setting config %s to %s" % (var, str(val)))
-            setattr(current_cls, var, val)
+            setattr(cls, var, val)
 
         # Run all the decorators. Step decorators are directly in the step and
         # we will run those first and *then* we run all the flow level decorators
-        for step in self._steps:
+        for step in cls._steps:
             for deco in step.config_decorators:
                 if isinstance(deco, CustomStepDecorator):
                     debug.userconf_exec(
                         "Evaluating step level decorator %s for %s"
                         % (deco.__class__.__name__, step.name)
                     )
-                    deco.evaluate(MutableStep(current_cls, step))
+                    deco.evaluate(MutableStep(cls, step))
                 else:
                     raise MetaflowInternalError(
                         "A non CustomFlowDecorator found in step custom decorators"
                     )
             if step.config_decorators:
                 # We remove all mention of the custom step decorator
-                setattr(current_cls, step.name, step)
+                setattr(cls, step.name, step)
 
-        mutable_flow = MutableFlow(current_cls)
-        for deco in self._flow_state[_FlowState.CONFIG_DECORATORS]:
+        mutable_flow = MutableFlow(cls)
+        for deco in cls._flow_state[_FlowState.CONFIG_DECORATORS]:
             if isinstance(deco, CustomFlowDecorator):
                 # Sanity check to make sure we are applying the decorator to the right
                 # class
-                if not deco._flow_cls == current_cls and not issubclass(
-                    current_cls, deco._flow_cls
-                ):
+                if not deco._flow_cls == cls and not issubclass(cls, deco._flow_cls):
                     raise MetaflowInternalError(
                         "CustomFlowDecorator registered on the wrong flow -- "
                         "expected %s but got %s"
-                        % (deco._flow_cls.__name__, current_cls.__name__)
+                        % (deco._flow_cls.__name__, cls.__name__)
                     )
                 debug.userconf_exec(
                     "Evaluating flow level decorator %s" % deco.__class__.__name__
@@ -256,15 +261,15 @@ class FlowSpec(metaclass=FlowSpecMeta):
                 deco.evaluate(mutable_flow)
                 # We reset cached_parameters on the very off chance that the user added
                 # more configurations based on the configuration
-                if _FlowState.CACHED_PARAMETERS in current_cls._flow_state:
-                    del current_cls._flow_state[_FlowState.CACHED_PARAMETERS]
+                if _FlowState.CACHED_PARAMETERS in cls._flow_state:
+                    del cls._flow_state[_FlowState.CACHED_PARAMETERS]
             else:
                 raise MetaflowInternalError(
                     "A non CustomFlowDecorator found in flow custom decorators"
                 )
 
         # Process parameters to allow them to also use config values easily
-        for var, param in self._get_parameters():
+        for var, param in cls._get_parameters():
             if param.IS_CONFIG_PARAMETER:
                 continue
             param.init()
@@ -272,15 +277,18 @@ class FlowSpec(metaclass=FlowSpecMeta):
         # TODO: This means that users can't override configs directly. Not sure if this
         # is a pattern we want to support
         for var, param in to_reset_configs:
-            setattr(current_cls, var, param)
+            setattr(cls, var, param)
 
         # Reset cached parameters again since we added back the config parameters
-        if _FlowState.CACHED_PARAMETERS in current_cls._flow_state:
-            del current_cls._flow_state[_FlowState.CACHED_PARAMETERS]
+        if _FlowState.CACHED_PARAMETERS in cls._flow_state:
+            del cls._flow_state[_FlowState.CACHED_PARAMETERS]
 
         # Set the current flow class we are in (the one we just created)
-        parameters.replace_flow_context(current_cls)
-        return current_cls(use_cli=False)
+        parameters.replace_flow_context(cls)
+
+        # Re-calculate class level attributes after modifying the class
+        cls._init_attrs()
+        return cls
 
     def _set_constants(self, graph, kwargs, config_options):
         from metaflow.decorators import (
@@ -309,6 +317,10 @@ class FlowSpec(metaclass=FlowSpecMeta):
             if isinstance(val, DelayedEvaluationParameter):
                 val = val()
             val = val.split(param.separator) if val and param.separator else val
+            if isinstance(val, ConfigValue):
+                # We store config values as dict so they are accessible with older
+                # metaflow clients. It also makes it easier to access.
+                val = val.to_dict()
             setattr(self, var, val)
             parameters_info.append({"name": var, "type": param.__class__.__name__})
 

{metaflow-2.12.39 → metaflow-2.13.1}/metaflow/graph.py

@@ -45,9 +45,12 @@ def deindent_docstring(doc):
 
 
 class DAGNode(object):
-    def __init__(self, func_ast, decos, doc):
+    def __init__(self, func_ast, decos, doc, source_file, lineno):
         self.name = func_ast.name
-        self.func_lineno = func_ast.lineno
+        self.source_file = source_file
+        # lineno is the start line of decorators in source_file
+        # func_ast.lineno is lines from decorators start to def of function
+        self.func_lineno = lineno + func_ast.lineno - 1
         self.decorators = decos
         self.doc = deindent_docstring(doc)
         self.parallel_step = any(getattr(deco, "IS_PARALLEL", False) for deco in decos)
@@ -62,7 +65,7 @@ class DAGNode(object):
         self.foreach_param = None
         self.num_parallel = 0
         self.parallel_foreach = False
-        self._parse(func_ast)
+        self._parse(func_ast, lineno)
 
         # these attributes are populated by _traverse_graph
         self.in_funcs = set()
@@ -74,7 +77,7 @@ class DAGNode(object):
     def _expr_str(self, expr):
         return "%s.%s" % (expr.value.id, expr.attr)
 
-    def _parse(self, func_ast):
+    def _parse(self, func_ast, lineno):
         self.num_args = len(func_ast.args.args)
         tail = func_ast.body[-1]
 
@@ -94,7 +97,7 @@ class DAGNode(object):
 
             self.has_tail_next = True
             self.invalid_tail_next = True
-            self.tail_next_lineno = tail.lineno
+            self.tail_next_lineno = lineno + tail.lineno - 1
             self.out_funcs = [e.attr for e in tail.value.args]
 
             keywords = dict(
@@ -131,7 +134,7 @@ class DAGNode(object):
             return
 
     def __str__(self):
-        return """*[{0.name} {0.type} (line {0.func_lineno})]*
+        return """*[{0.name} {0.type} ({0.source_file} line {0.func_lineno})]*
     in_funcs={in_funcs}
     out_funcs={out_funcs}
     split_parents={parents}
@@ -156,18 +159,6 @@ class DAGNode(object):
         )
 
 
-class StepVisitor(ast.NodeVisitor):
-    def __init__(self, nodes, flow):
-        self.nodes = nodes
-        self.flow = flow
-        super(StepVisitor, self).__init__()
-
-    def visit_FunctionDef(self, node):
-        func = getattr(self.flow, node.name)
-        if hasattr(func, "is_step"):
-            self.nodes[node.name] = DAGNode(node, func.decorators, func.__doc__)
-
-
 class FlowGraph(object):
     def __init__(self, flow):
         self.name = flow.__name__
@@ -179,13 +170,20 @@ class FlowGraph(object):
         self._postprocess()
 
     def _create_nodes(self, flow):
-        module = __import__(flow.__module__)
-        tree = ast.parse(inspect.getsource(module)).body
-        root = [n for n in tree if isinstance(n, ast.ClassDef) and n.name == self.name][
-            0
-        ]
         nodes = {}
-        StepVisitor(nodes, flow).visit(root)
+        for element in dir(flow):
+            func = getattr(flow, element)
+            if callable(func) and hasattr(func, "is_step"):
+                source_file = inspect.getsourcefile(func)
+                source_lines, lineno = inspect.getsourcelines(func)
+                # This also works for code (strips out leading whitspace based on
+                # first line)
+                source_code = deindent_docstring("".join(source_lines))
+                function_ast = ast.parse(source_code).body[0]
+                node = DAGNode(
+                    function_ast, func.decorators, func.__doc__, source_file, lineno
+                )
+                nodes[element] = node
         return nodes
 
     def _postprocess(self):
@@ -201,6 +199,10 @@ class FlowGraph(object):
 
     def _traverse_graph(self):
         def traverse(node, seen, split_parents):
+            try:
+                self.sorted_nodes.remove(node.name)
+            except ValueError:
+                pass
             self.sorted_nodes.append(node.name)
             if node.type in ("split", "foreach"):
                 node.split_parents = split_parents
@@ -240,9 +242,7 @@ class FlowGraph(object):
         return iter(self.nodes.values())
 
     def __str__(self):
-        return "\n".join(
-            str(n) for _, n in sorted((n.func_lineno, n) for n in self.nodes.values())
-        )
+        return "\n".join(str(self[n]) for n in self.sorted_nodes)
 
     def output_dot(self):
         def edge_specs():
@@ -286,6 +286,7 @@ class FlowGraph(object):
                 "name": name,
                 "type": node_to_type(node),
                 "line": node.func_lineno,
+                "source_file": node.source_file,
                 "doc": node.doc,
                 "decorators": [
                     {

{metaflow-2.12.39 → metaflow-2.13.1}/metaflow/includefile.py

@@ -290,8 +290,8 @@ class IncludeFile(Parameter):
             **kwargs,
         )
 
-    def init(self):
-        super(IncludeFile, self).init()
+    def init(self, ignore_errors=False):
+        super(IncludeFile, self).init(ignore_errors)
 
         # This will use the values set explicitly in the args if present, else will
         # use and remove from kwargs else will use True/utf-8