ob-metaflow 2.12.30.2__py2.py3-none-any.whl → 2.13.6.1__py2.py3-none-any.whl

metaflow/runner/click_api.py

@@ -9,6 +9,7 @@ if sys.version_info < (3, 7):
     )
 
 import datetime
+import functools
 import importlib
 import inspect
 import itertools
@@ -17,6 +18,7 @@ import json
 from collections import OrderedDict
 from typing import Any, Callable, Dict, List, Optional
 from typing import OrderedDict as TOrderedDict
+from typing import Tuple as TTuple
 from typing import Union
 
 from metaflow import FlowSpec, Parameter
@@ -37,7 +39,16 @@ from metaflow._vendor.typeguard import TypeCheckError, check_type
 from metaflow.decorators import add_decorator_options
 from metaflow.exception import MetaflowException
 from metaflow.includefile import FilePathClass
+from metaflow.metaflow_config import CLICK_API_PROCESS_CONFIG
 from metaflow.parameters import JSONTypeClass, flow_context
+from metaflow.user_configs.config_options import (
+    ConfigValue,
+    ConvertDictOrStr,
+    ConvertPath,
+    LocalFileInput,
+    MultipleTuple,
+    config_options_with_config_input,
+)
 
 # Define a recursive type alias for JSON
 JSON = Union[Dict[str, "JSON"], List["JSON"], str, int, float, bool, None]
@@ -55,6 +66,8 @@ click_to_python_types = {
     File: str,
     JSONTypeClass: JSON,
     FilePathClass: str,
+    LocalFileInput: str,
+    MultipleTuple: TTuple[str, Union[JSON, ConfigValue]],
 }
 
 
@@ -65,7 +78,7 @@ def _method_sanity_check(
     defaults: TOrderedDict[str, Any],
     **kwargs
 ) -> Dict[str, Any]:
-    method_params = {"args": {}, "options": {}}
+    method_params = {"args": {}, "options": {}, "defaults": defaults}
 
     possible_params = OrderedDict()
     possible_params.update(possible_arg_params)
@@ -87,10 +100,26 @@ def _method_sanity_check(
                 % (supplied_k, annotations[supplied_k], defaults[supplied_k])
             )
 
-        # because Click expects stringified JSON..
-        supplied_v = (
-            json.dumps(supplied_v) if annotations[supplied_k] == JSON else supplied_v
-        )
+        # Clean up values to make them into what click expects
+        if annotations[supplied_k] == JSON:
+            # JSON should be a string (json dumps)
+            supplied_v = json.dumps(supplied_v)
+        elif supplied_k == "config_value":
+            # Special handling of config value because we need to go look in the tuple
+            new_list = []
+            for cfg_name, cfg_value in supplied_v:
+                if isinstance(cfg_value, ConfigValue):
+                    # ConfigValue should be JSONified and converted to a string
+                    new_list.append((cfg_name, json.dumps(cfg_value.to_dict())))
+                elif isinstance(cfg_value, dict):
+                    # ConfigValue passed as a dictionary
+                    new_list.append((cfg_name, json.dumps(cfg_value)))
+                else:
+                    raise TypeError(
+                        "Invalid type for a config-value, expected a ConfigValue or "
+                        "dict but got '%s'" % type(cfg_value)
+                    )
+            supplied_v = new_list
 
         if supplied_k in possible_arg_params:
             cli_name = possible_arg_params[supplied_k].opts[0].strip("-")
@@ -124,6 +153,37 @@ def _method_sanity_check(
     return method_params
 
 
+def _lazy_load_command(
+    cli_collection: click.Group,
+    flow_parameters: Union[str, List[Parameter]],
+    _self,
+    name: str,
+):
+
+    # Context is not used in get_command so we can pass None. Since we pin click,
+    # this won't change from under us.
+
+    if isinstance(flow_parameters, str):
+        # Resolve flow_parameters -- for start, this is a function which we
+        # need to call to figure out the actual parameters (may be changed by configs)
+        flow_parameters = getattr(_self, flow_parameters)()
+    cmd_obj = cli_collection.get_command(None, name)
+    if cmd_obj:
+        if isinstance(cmd_obj, click.Group):
+            # TODO: possibly check for fake groups with cmd_obj.name in ["cli", "main"]
+            result = functools.partial(extract_group(cmd_obj, flow_parameters), _self)
+        elif isinstance(cmd_obj, click.Command):
+            result = functools.partial(extract_command(cmd_obj, flow_parameters), _self)
+        else:
+            raise RuntimeError(
+                "Cannot handle %s of type %s" % (cmd_obj.name, type(cmd_obj))
+            )
+        setattr(_self, name, result)
+        return result
+    else:
+        raise AttributeError()
+
+
 def get_annotation(param: Union[click.Argument, click.Option]):
     py_type = click_to_python_types[type(param.type)]
     if not param.required:
@@ -154,34 +214,57 @@ loaded_modules = {}
 def extract_flow_class_from_file(flow_file: str) -> FlowSpec:
     if not os.path.exists(flow_file):
         raise FileNotFoundError("Flow file not present at '%s'" % flow_file)
-    # Check if the module has already been loaded
-    if flow_file in loaded_modules:
-        module = loaded_modules[flow_file]
-    else:
-        # Load the module if it's not already loaded
-        spec = importlib.util.spec_from_file_location("module", flow_file)
-        module = importlib.util.module_from_spec(spec)
-        spec.loader.exec_module(module)
-        # Cache the loaded module
-        loaded_modules[flow_file] = module
-    classes = inspect.getmembers(module, inspect.isclass)
-
-    flow_cls = None
-    for _, kls in classes:
-        if kls != FlowSpec and issubclass(kls, FlowSpec):
-            if flow_cls is not None:
-                raise MetaflowException(
-                    "Multiple FlowSpec classes found in %s" % flow_file
-                )
-            flow_cls = kls
-
-    return flow_cls
+
+    flow_dir = os.path.dirname(os.path.abspath(flow_file))
+    path_was_added = False
+
+    # Only add to path if it's not already there
+    if flow_dir not in sys.path:
+        sys.path.insert(0, flow_dir)
+        path_was_added = True
+
+    try:
+        # Check if the module has already been loaded
+        if flow_file in loaded_modules:
+            module = loaded_modules[flow_file]
+        else:
+            # Load the module if it's not already loaded
+            spec = importlib.util.spec_from_file_location("module", flow_file)
+            module = importlib.util.module_from_spec(spec)
+            spec.loader.exec_module(module)
+            # Cache the loaded module
+            loaded_modules[flow_file] = module
+        classes = inspect.getmembers(module, inspect.isclass)
+
+        flow_cls = None
+        for _, kls in classes:
+            if kls != FlowSpec and issubclass(kls, FlowSpec):
+                if flow_cls is not None:
+                    raise MetaflowException(
+                        "Multiple FlowSpec classes found in %s" % flow_file
+                    )
+                flow_cls = kls
+
+        if flow_cls is None:
+            raise MetaflowException("No FlowSpec class found in %s" % flow_file)
+        return flow_cls
+    finally:
+        # Only remove from path if we added it
+        if path_was_added:
+            try:
+                sys.path.remove(flow_dir)
+            except ValueError:
+                # User's code might have removed it already
+                pass
 
 
 class MetaflowAPI(object):
-    def __init__(self, parent=None, **kwargs):
+    def __init__(self, parent=None, flow_cls=None, config_input=None, **kwargs):
         self._parent = parent
         self._chain = [{self._API_NAME: kwargs}]
+        self._flow_cls = flow_cls
+        self._config_input = config_input
+        self._cached_computed_parameters = None
 
     @property
     def parent(self):
@@ -200,23 +283,30 @@ class MetaflowAPI(object):
     @classmethod
     def from_cli(cls, flow_file: str, cli_collection: Callable) -> Callable:
         flow_cls = extract_flow_class_from_file(flow_file)
-        flow_parameters = [p for _, p in flow_cls._get_parameters()]
+
         with flow_context(flow_cls) as _:
-            add_decorator_options(cli_collection)
-
-        class_dict = {"__module__": "metaflow", "_API_NAME": flow_file}
-        command_groups = cli_collection.sources
-        for each_group in command_groups:
-            for _, cmd_obj in each_group.commands.items():
-                if isinstance(cmd_obj, click.Group):
-                    # TODO: possibly check for fake groups with cmd_obj.name in ["cli", "main"]
-                    class_dict[cmd_obj.name] = extract_group(cmd_obj, flow_parameters)
-                elif isinstance(cmd_obj, click.Command):
-                    class_dict[cmd_obj.name] = extract_command(cmd_obj, flow_parameters)
-                else:
-                    raise RuntimeError(
-                        "Cannot handle %s of type %s" % (cmd_obj.name, type(cmd_obj))
-                    )
+            cli_collection, config_input = config_options_with_config_input(
+                cli_collection
+            )
+            cli_collection = add_decorator_options(cli_collection)
+
+        def getattr_wrapper(_self, name):
+            # Functools.partial do not automatically bind self (no __get__)
+            with flow_context(flow_cls) as _:
+                # We also wrap this in the proper flow context because since commands
+                # are loaded lazily, we need the proper flow context to compute things
+                # like parameters. If we do not do this, the outer flow's context will
+                # be used.
+                return _self._internal_getattr(_self, name)
+
+        class_dict = {
+            "__module__": "metaflow",
+            "_API_NAME": flow_file,
+            "_internal_getattr": functools.partial(
+                _lazy_load_command, cli_collection, "_compute_flow_parameters"
+            ),
+            "__getattr__": getattr_wrapper,
+        }
 
         to_return = type(flow_file, (MetaflowAPI,), class_dict)
         to_return.__name__ = flow_file
@@ -237,11 +327,16 @@ class MetaflowAPI(object):
                 defaults,
                 **kwargs,
             )
-            return to_return(parent=None, **method_params)
+            return to_return(
+                parent=None,
+                flow_cls=flow_cls,
+                config_input=config_input,
+                **method_params,
+            )
 
         m = _method
-        m.__name__ = cmd_obj.name
-        m.__doc__ = getattr(cmd_obj, "help", None)
+        m.__name__ = cli_collection.name
+        m.__doc__ = getattr(cli_collection, "help", None)
         m.__signature__ = inspect.signature(_method).replace(
             parameters=params_sigs.values()
         )
@@ -278,8 +373,12 @@ class MetaflowAPI(object):
                 for k, v in options.items():
                     if isinstance(v, list):
                         for i in v:
-                            components.append("--%s" % k)
-                            components.append(str(i))
+                            if isinstance(i, tuple):
+                                components.append("--%s" % k)
+                                components.extend(map(str, i))
+                            else:
+                                components.append("--%s" % k)
+                                components.append(str(i))
                     else:
                         components.append("--%s" % k)
                         if v != "flag":
@@ -287,6 +386,97 @@ class MetaflowAPI(object):
 
         return components
 
+    def _compute_flow_parameters(self):
+        if (
+            self._flow_cls is None
+            or self._config_input is None
+            or self._parent is not None
+        ):
+            raise RuntimeError(
+                "Computing flow-level parameters for a non start API. "
+                "Please report to the Metaflow team."
+            )
+
+        if self._cached_computed_parameters is not None:
+            return self._cached_computed_parameters
+        self._cached_computed_parameters = []
+
+        config_options = None
+        if CLICK_API_PROCESS_CONFIG:
+            with flow_context(self._flow_cls) as _:
+                # We are going to resolve the configs first and then get the parameters.
+                # Note that configs may update/add parameters so the order is important
+                # Since part of the processing of configs happens by click, we need to
+                # "fake" it.
+
+                # Extract any config options as well as datastore and quiet options
+                method_params = self._chain[0][self._API_NAME]
+                opts = method_params["options"]
+                defaults = method_params["defaults"]
+
+                ds = opts.get("datastore", defaults["datastore"])
+                quiet = opts.get("quiet", defaults["quiet"])
+                is_default = False
+                config_file = opts.get("config-file")
+                if config_file is None:
+                    is_default = True
+                    config_file = defaults.get("config_file")
+
+                if config_file:
+                    config_file = map(
+                        lambda x: (x[0], ConvertPath.convert_value(x[1], is_default)),
+                        config_file,
+                    )
+
+                is_default = False
+                config_value = opts.get("config-value")
+                if config_value is None:
+                    is_default = True
+                    config_value = defaults.get("config_value")
+
+                if config_value:
+                    config_value = map(
+                        lambda x: (
+                            x[0],
+                            ConvertDictOrStr.convert_value(x[1], is_default),
+                        ),
+                        config_value,
+                    )
+
+                if (config_file is None) ^ (config_value is None):
+                    # If we have one, we should have the other
+                    raise MetaflowException(
+                        "Options were not properly set -- this is an internal error."
+                    )
+
+                if config_file:
+                    # Process both configurations; the second one will return all the merged
+                    # configuration options properly processed.
+                    self._config_input.process_configs(
+                        self._flow_cls.__name__, "config_file", config_file, quiet, ds
+                    )
+                    config_options = self._config_input.process_configs(
+                        self._flow_cls.__name__, "config_value", config_value, quiet, ds
+                    )
+
+        # At this point, we are like in start() in cli.py -- we obtained the
+        # properly processed config_options which we can now use to process
+        # the config decorators (including CustomStep/FlowDecorators)
+        # Note that if CLICK_API_PROCESS_CONFIG is False, we still do this because
+        # it will init all parameters (config_options will be None)
+        # We ignore any errors if we don't check the configs in the click API.
+        new_cls = self._flow_cls._process_config_decorators(
+            config_options, ignore_errors=not CLICK_API_PROCESS_CONFIG
+        )
+        if new_cls:
+            self._flow_cls = new_cls
+
+        for _, param in self._flow_cls._get_parameters():
+            if param.IS_CONFIG_PARAMETER:
+                continue
+            self._cached_computed_parameters.append(param)
+        return self._cached_computed_parameters
+
 
 def extract_all_params(cmd_obj: Union[click.Command, click.Group]):
     arg_params_sigs = OrderedDict()
@@ -351,7 +541,7 @@ def extract_group(cmd_obj: click.Group, flow_parameters: List[Parameter]) -> Cal
         method_params = _method_sanity_check(
             possible_arg_params, possible_opt_params, annotations, defaults, **kwargs
         )
-        return resulting_class(parent=_self, **method_params)
+        return resulting_class(parent=_self, flow_cls=None, **method_params)
 
     m = _method
     m.__name__ = cmd_obj.name

metaflow/runner/deployer.py

@@ -64,7 +64,7 @@ class Deployer(metaclass=DeployerMeta):
         The directory to run the subprocess in; if not specified, the current
         directory is used.
     file_read_timeout : int, default 3600
-        The timeout until which we try to read the deployer attribute file.
+        The timeout until which we try to read the deployer attribute file (in seconds).
     **kwargs : Any
         Additional arguments that you would pass to `python myflow.py` before
         the deployment command.

metaflow/runner/deployer_impl.py

@@ -2,12 +2,11 @@ import importlib
 import json
 import os
 import sys
-import tempfile
 
 from typing import Any, ClassVar, Dict, Optional, TYPE_CHECKING, Type
 
 from .subprocess_manager import SubprocessManager
-from .utils import get_lower_level_group, handle_timeout
+from .utils import get_lower_level_group, handle_timeout, temporary_fifo
 
 if TYPE_CHECKING:
     import metaflow.runner.deployer
@@ -38,7 +37,7 @@ class DeployerImpl(object):
         The directory to run the subprocess in; if not specified, the current
         directory is used.
     file_read_timeout : int, default 3600
-        The timeout until which we try to read the deployer attribute file.
+        The timeout until which we try to read the deployer attribute file (in seconds).
     **kwargs : Any
         Additional arguments that you would pass to `python myflow.py` before
         the deployment command.
@@ -62,8 +61,13 @@ class DeployerImpl(object):
                 "of DeployerImpl."
             )
 
+        from metaflow.parameters import flow_context
+
         if "metaflow.cli" in sys.modules:
-            importlib.reload(sys.modules["metaflow.cli"])
+            # Reload the CLI with an "empty" flow -- this will remove any configuration
+            # options. They are re-added in from_cli (called below).
+            with flow_context(None) as _:
+                importlib.reload(sys.modules["metaflow.cli"])
         from metaflow.cli import start
         from metaflow.runner.click_api import MetaflowAPI
 
@@ -121,14 +125,11 @@ class DeployerImpl(object):
     def _create(
         self, create_class: Type["metaflow.runner.deployer.DeployedFlow"], **kwargs
     ) -> "metaflow.runner.deployer.DeployedFlow":
-        with tempfile.TemporaryDirectory() as temp_dir:
-            tfp_runner_attribute = tempfile.NamedTemporaryFile(
-                dir=temp_dir, delete=False
-            )
+        with temporary_fifo() as (attribute_file_path, attribute_file_fd):
             # every subclass needs to have `self.deployer_kwargs`
             command = get_lower_level_group(
                 self.api, self.top_level_kwargs, self.TYPE, self.deployer_kwargs
-            ).create(deployer_attribute_file=tfp_runner_attribute.name, **kwargs)
+            ).create(deployer_attribute_file=attribute_file_path, **kwargs)
 
             pid = self.spm.run_command(
                 [sys.executable, *command],
@@ -139,7 +140,7 @@ class DeployerImpl(object):
 
             command_obj = self.spm.get(pid)
             content = handle_timeout(
-                tfp_runner_attribute, command_obj, self.file_read_timeout
+                attribute_file_fd, command_obj, self.file_read_timeout
             )
             content = json.loads(content)
             self.name = content.get("name")
@@ -148,7 +149,7 @@ class DeployerImpl(object):
             # Additional info is used to pass additional deployer specific information.
             # It is used in non-OSS deployers (extensions).
             self.additional_info = content.get("additional_info", {})
-
+            command_obj.sync_wait()
             if command_obj.process.returncode == 0:
                 return create_class(deployer=self)
 

metaflow/runner/metaflow_runner.py

@@ -2,13 +2,18 @@ import importlib
 import os
 import sys
 import json
-import tempfile
 
 from typing import Dict, Iterator, Optional, Tuple
 
 from metaflow import Run
 
-from .utils import handle_timeout
+from metaflow.plugins import get_runner_cli
+
+from .utils import (
+    temporary_fifo,
+    handle_timeout,
+    async_handle_timeout,
+)
 from .subprocess_manager import CommandManager, SubprocessManager
 
 
@@ -184,7 +189,27 @@ class ExecutingRun(object):
             yield position, line
 
 
-class Runner(object):
+class RunnerMeta(type):
+    def __new__(mcs, name, bases, dct):
+        cls = super().__new__(mcs, name, bases, dct)
+
+        def _injected_method(subcommand_name, runner_subcommand):
+            def f(self, *args, **kwargs):
+                return runner_subcommand(self, *args, **kwargs)
+
+            f.__doc__ = runner_subcommand.__doc__ or ""
+            f.__name__ = subcommand_name
+
+            return f
+
+        for runner_subcommand in get_runner_cli():
+            method_name = runner_subcommand.name.replace("-", "_")
+            setattr(cls, method_name, _injected_method(method_name, runner_subcommand))
+
+        return cls
+
+
+class Runner(metaclass=RunnerMeta):
     """
     Metaflow's Runner API that presents a programmatic interface
     to run flows and perform other operations either synchronously or asynchronously.
@@ -218,7 +243,7 @@ class Runner(object):
         The directory to run the subprocess in; if not specified, the current
         directory is used.
     file_read_timeout : int, default 3600
-        The timeout until which we try to read the runner attribute file.
+        The timeout until which we try to read the runner attribute file (in seconds).
     **kwargs : Any
         Additional arguments that you would pass to `python myflow.py` before
         the `run` command.
@@ -242,8 +267,13 @@ class Runner(object):
         # This ability is made possible by the statement:
         # 'from .metaflow_runner import Runner' in '__init__.py'
 
+        from metaflow.parameters import flow_context
+
         if "metaflow.cli" in sys.modules:
-            importlib.reload(sys.modules["metaflow.cli"])
+            # Reload the CLI with an "empty" flow -- this will remove any configuration
+            # options. They are re-added in from_cli (called below).
+            with flow_context(None) as _:
+                importlib.reload(sys.modules["metaflow.cli"])
         from metaflow.cli import start
         from metaflow.runner.click_api import MetaflowAPI
 
@@ -267,9 +297,25 @@ class Runner(object):
     async def __aenter__(self) -> "Runner":
         return self
 
-    def __get_executing_run(self, tfp_runner_attribute, command_obj):
-        content = handle_timeout(
-            tfp_runner_attribute, command_obj, self.file_read_timeout
+    def __get_executing_run(self, attribute_file_fd, command_obj):
+        content = handle_timeout(attribute_file_fd, command_obj, self.file_read_timeout)
+
+        command_obj.sync_wait()
+
+        content = json.loads(content)
+        pathspec = "%s/%s" % (content.get("flow_name"), content.get("run_id"))
+
+        # Set the correct metadata from the runner_attribute file corresponding to this run.
+        metadata_for_flow = content.get("metadata")
+
+        run_object = Run(
+            pathspec, _namespace_check=False, _current_metadata=metadata_for_flow
+        )
+        return ExecutingRun(self, command_obj, run_object)
+
+    async def __async_get_executing_run(self, attribute_file_fd, command_obj):
+        content = await async_handle_timeout(
+            attribute_file_fd, command_obj, self.file_read_timeout
         )
         content = json.loads(content)
         pathspec = "%s/%s" % (content.get("flow_name"), content.get("run_id"))
@@ -298,12 +344,9 @@ class Runner(object):
         ExecutingRun
             ExecutingRun containing the results of the run.
         """
-        with tempfile.TemporaryDirectory() as temp_dir:
-            tfp_runner_attribute = tempfile.NamedTemporaryFile(
-                dir=temp_dir, delete=False
-            )
+        with temporary_fifo() as (attribute_file_path, attribute_file_fd):
             command = self.api(**self.top_level_kwargs).run(
-                runner_attribute_file=tfp_runner_attribute.name, **kwargs
+                runner_attribute_file=attribute_file_path, **kwargs
             )
 
             pid = self.spm.run_command(
@@ -314,9 +357,9 @@ class Runner(object):
             )
             command_obj = self.spm.get(pid)
 
-            return self.__get_executing_run(tfp_runner_attribute, command_obj)
+            return self.__get_executing_run(attribute_file_fd, command_obj)
 
-    def resume(self, **kwargs):
+    def resume(self, **kwargs) -> ExecutingRun:
         """
         Blocking resume execution of the run.
         This method will wait until the resumed run has completed execution.
@@ -332,12 +375,9 @@ class Runner(object):
         ExecutingRun
             ExecutingRun containing the results of the resumed run.
         """
-        with tempfile.TemporaryDirectory() as temp_dir:
-            tfp_runner_attribute = tempfile.NamedTemporaryFile(
-                dir=temp_dir, delete=False
-            )
+        with temporary_fifo() as (attribute_file_path, attribute_file_fd):
             command = self.api(**self.top_level_kwargs).resume(
-                runner_attribute_file=tfp_runner_attribute.name, **kwargs
+                runner_attribute_file=attribute_file_path, **kwargs
             )
 
             pid = self.spm.run_command(
@@ -348,7 +388,7 @@ class Runner(object):
             )
             command_obj = self.spm.get(pid)
 
-            return self.__get_executing_run(tfp_runner_attribute, command_obj)
+            return self.__get_executing_run(attribute_file_fd, command_obj)
 
     async def async_run(self, **kwargs) -> ExecutingRun:
         """
@@ -368,12 +408,9 @@ class Runner(object):
         ExecutingRun
             ExecutingRun representing the run that was started.
         """
-        with tempfile.TemporaryDirectory() as temp_dir:
-            tfp_runner_attribute = tempfile.NamedTemporaryFile(
-                dir=temp_dir, delete=False
-            )
+        with temporary_fifo() as (attribute_file_path, attribute_file_fd):
             command = self.api(**self.top_level_kwargs).run(
-                runner_attribute_file=tfp_runner_attribute.name, **kwargs
+                runner_attribute_file=attribute_file_path, **kwargs
             )
 
             pid = await self.spm.async_run_command(
@@ -383,9 +420,9 @@ class Runner(object):
             )
             command_obj = self.spm.get(pid)
 
-            return self.__get_executing_run(tfp_runner_attribute, command_obj)
+            return await self.__async_get_executing_run(attribute_file_fd, command_obj)
 
-    async def async_resume(self, **kwargs):
+    async def async_resume(self, **kwargs) -> ExecutingRun:
         """
         Non-blocking resume execution of the run.
         This method will return as soon as the resume has launched.
@@ -403,12 +440,9 @@ class Runner(object):
         ExecutingRun
             ExecutingRun representing the resumed run that was started.
         """
-        with tempfile.TemporaryDirectory() as temp_dir:
-            tfp_runner_attribute = tempfile.NamedTemporaryFile(
-                dir=temp_dir, delete=False
-            )
+        with temporary_fifo() as (attribute_file_path, attribute_file_fd):
             command = self.api(**self.top_level_kwargs).resume(
-                runner_attribute_file=tfp_runner_attribute.name, **kwargs
+                runner_attribute_file=attribute_file_path, **kwargs
             )
 
             pid = await self.spm.async_run_command(
@@ -418,7 +452,7 @@ class Runner(object):
             )
             command_obj = self.spm.get(pid)
 
-            return self.__get_executing_run(tfp_runner_attribute, command_obj)
+            return await self.__async_get_executing_run(attribute_file_fd, command_obj)
 
     def __exit__(self, exc_type, exc_value, traceback):
         self.spm.cleanup()

metaflow/runner/nbdeploy.py

@@ -46,6 +46,8 @@ class NBDeployer(object):
     base_dir : str, optional, default None
         The directory to run the subprocess in; if not specified, the current
         working directory is used.
+    file_read_timeout : int, default 3600
+        The timeout until which we try to read the deployer attribute file (in seconds).
     **kwargs : Any
         Additional arguments that you would pass to `python myflow.py` i.e. options
         listed in `python myflow.py --help`