ob-metaflow 2.12.36.3__py2.py3-none-any.whl → 2.12.39.1__py2.py3-none-any.whl

metaflow/plugins/pypi/conda_decorator.py

@@ -50,20 +50,26 @@ class CondaStepDecorator(StepDecorator):
     # conda channels, users can specify channel::package as the package name.
 
     def __init__(self, attributes=None, statically_defined=False):
-        self._user_defined_attributes = (
-            attributes.copy() if attributes is not None else {}
+        self._attributes_with_user_values = (
+            set(attributes.keys()) if attributes is not None else set()
         )
+
         super(CondaStepDecorator, self).__init__(attributes, statically_defined)
 
+    def init(self):
+        super(CondaStepDecorator, self).init()
+
         # Support legacy 'libraries=' attribute for the decorator.
         self.attributes["packages"] = {
             **self.attributes["libraries"],
             **self.attributes["packages"],
         }
         del self.attributes["libraries"]
+        if self.attributes["packages"]:
+            self._attributes_with_user_values.add("packages")
 
     def is_attribute_user_defined(self, name):
-        return name in self._user_defined_attributes
+        return name in self._attributes_with_user_values
 
     def step_init(self, flow, graph, step, decos, environment, flow_datastore, logger):
         # The init_environment hook for Environment creates the relevant virtual
@@ -83,10 +89,10 @@ class CondaStepDecorator(StepDecorator):
                 **super_attributes["packages"],
                 **self.attributes["packages"],
             }
-            self._user_defined_attributes = {
-                **self._user_defined_attributes,
-                **conda_base._user_defined_attributes,
-            }
+            self._attributes_with_user_values.update(
+                conda_base._attributes_with_user_values
+            )
+
             self.attributes["python"] = (
                 self.attributes["python"] or super_attributes["python"]
             )
@@ -333,11 +339,15 @@ class CondaFlowDecorator(FlowDecorator):
     }
 
     def __init__(self, attributes=None, statically_defined=False):
-        self._user_defined_attributes = (
-            attributes.copy() if attributes is not None else {}
+        self._attributes_with_user_values = (
+            set(attributes.keys()) if attributes is not None else set()
         )
+
         super(CondaFlowDecorator, self).__init__(attributes, statically_defined)
 
+    def init(self):
+        super(CondaFlowDecorator, self).init()
+
         # Support legacy 'libraries=' attribute for the decorator.
         self.attributes["packages"] = {
             **self.attributes["libraries"],
@@ -348,7 +358,7 @@ class CondaFlowDecorator(FlowDecorator):
             self.attributes["python"] = str(self.attributes["python"])
 
     def is_attribute_user_defined(self, name):
-        return name in self._user_defined_attributes
+        return name in self._attributes_with_user_values
 
     def flow_init(
         self, flow, graph, environment, flow_datastore, metadata, logger, echo, options

metaflow/plugins/pypi/conda_environment.py

@@ -5,10 +5,11 @@ import functools
 import io
 import json
 import os
-import sys
 import tarfile
+import threading
 import time
-from concurrent.futures import ThreadPoolExecutor
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from functools import wraps
 from hashlib import sha256
 from io import BufferedIOBase, BytesIO
 from itertools import chain
@@ -50,7 +51,6 @@ class CondaEnvironment(MetaflowEnvironment):
 
     def validate_environment(self, logger, datastore_type):
         self.datastore_type = datastore_type
-        self.logger = logger
 
         # Avoiding circular imports.
         from metaflow.plugins import DATASTORES
@@ -62,8 +62,21 @@ class CondaEnvironment(MetaflowEnvironment):
         from .micromamba import Micromamba
         from .pip import Pip
 
-        micromamba = Micromamba()
-        self.solvers = {"conda": micromamba, "pypi": Pip(micromamba)}
+        print_lock = threading.Lock()
+
+        def make_thread_safe(func):
+            @wraps(func)
+            def wrapper(*args, **kwargs):
+                with print_lock:
+                    return func(*args, **kwargs)
+
+            return wrapper
+
+        self.logger = make_thread_safe(logger)
+
+        # TODO: Wire up logging
+        micromamba = Micromamba(self.logger)
+        self.solvers = {"conda": micromamba, "pypi": Pip(micromamba, self.logger)}
 
     def init_environment(self, echo, only_steps=None):
         # The implementation optimizes for latency to ensure as many operations can
@@ -150,6 +163,9 @@ class CondaEnvironment(MetaflowEnvironment):
                 (
                     package["path"],
                     # Lazily fetch package from the interweb if needed.
+                    # TODO: Depending on the len_hint, the package might be downloaded from
+                    #       the interweb prematurely. save_bytes needs to be adjusted to handle
+                    #       this scenario.
                     LazyOpen(
                         package["local_path"],
                         "rb",
@@ -166,22 +182,60 @@ class CondaEnvironment(MetaflowEnvironment):
                 if id_ in dirty:
                     self.write_to_environment_manifest([id_, platform, type_], packages)
 
-        # First resolve environments through Conda, before PyPI.
+        storage = None
+        if self.datastore_type not in ["local"]:
+            # Initialize storage for caching if using a remote datastore
+            storage = self.datastore(_datastore_packageroot(self.datastore, echo))
+
         self.logger("Bootstrapping virtual environment(s) ...")
-        for solver in ["conda", "pypi"]:
-            with ThreadPoolExecutor() as executor:
-                results = list(
-                    executor.map(lambda x: solve(*x, solver), environments(solver))
-                )
-            _ = list(map(lambda x: self.solvers[solver].download(*x), results))
-            with ThreadPoolExecutor() as executor:
-                _ = list(
-                    executor.map(lambda x: self.solvers[solver].create(*x), results)
-                )
-            if self.datastore_type not in ["local"]:
-                # Cache packages only when a remote datastore is in play.
-                storage = self.datastore(_datastore_packageroot(self.datastore, echo))
-                cache(storage, results, solver)
+        # Sequence of operations:
+        #  1. Start all conda solves in parallel
+        #  2. Download conda packages sequentially
+        #  3. Create and cache conda environments in parallel
+        #  4. Start PyPI solves in parallel after each conda environment is created
+        #  5. Download PyPI packages sequentially
+        #  6. Create and cache PyPI environments in parallel
+
+        with ThreadPoolExecutor() as executor:
+            # Start all conda solves in parallel
+            conda_futures = [
+                executor.submit(lambda x: solve(*x, "conda"), env)
+                for env in environments("conda")
+            ]
+
+            pypi_envs = {env[0]: env for env in environments("pypi")}
+            pypi_futures = []
+
+            # Process conda results sequentially for downloads
+            for future in as_completed(conda_futures):
+                result = future.result()
+                # Sequential conda download
+                self.solvers["conda"].download(*result)
+                # Parallel conda create and cache
+                create_future = executor.submit(self.solvers["conda"].create, *result)
+                if storage:
+                    executor.submit(cache, storage, [result], "conda")
+
+                # Queue PyPI solve to start after conda create
+                if result[0] in pypi_envs:
+
+                    def pypi_solve(env):
+                        create_future.result()  # Wait for conda create
+                        return solve(*env, "pypi")
+
+                    pypi_futures.append(
+                        executor.submit(pypi_solve, pypi_envs[result[0]])
+                    )
+
+            # Process PyPI results sequentially for downloads
+            for solve_future in pypi_futures:
+                result = solve_future.result()
+                # Sequential PyPI download
+                self.solvers["pypi"].download(*result)
+                # Parallel PyPI create and cache
+                executor.submit(self.solvers["pypi"].create, *result)
+                if storage:
+                    executor.submit(cache, storage, [result], "pypi")
         self.logger("Virtual environment(s) bootstrapped!")
 
     def executable(self, step_name, default=None):
@@ -385,7 +439,8 @@ class CondaEnvironment(MetaflowEnvironment):
                 'DISABLE_TRACING=True python -m metaflow.plugins.pypi.bootstrap "%s" %s "%s" linux-64'
                 % (self.flow.name, id_, self.datastore_type),
                 "echo 'Environment bootstrapped.'",
-                "export PATH=$PATH:$(pwd)/micromamba",
+                # To avoid having to install micromamba in the PATH in micromamba.py, we add it to the PATH here.
+                "export PATH=$PATH:$(pwd)/micromamba/bin",
             ]
         else:
             # for @conda/@pypi(disabled=True).

metaflow/plugins/pypi/micromamba.py

@@ -1,7 +1,9 @@
+import functools
 import json
 import os
 import subprocess
 import tempfile
+import time
 
 from metaflow.exception import MetaflowException
 from metaflow.util import which
@@ -19,8 +21,11 @@ class MicromambaException(MetaflowException):
         super(MicromambaException, self).__init__(msg)
 
 
+GLIBC_VERSION = os.environ.get("CONDA_OVERRIDE_GLIBC", "2.38")
+
+
 class Micromamba(object):
-    def __init__(self):
+    def __init__(self, logger=None):
         # micromamba is a tiny version of the mamba package manager and comes with
         # metaflow specific performance enhancements.
 
@@ -33,6 +38,12 @@ class Micromamba(object):
             os.path.expanduser(_home),
             "micromamba",
         )
+
+        if logger:
+            self.logger = logger
+        else:
+            self.logger = lambda *args, **kwargs: None  # No-op logger if not provided
+
         self.bin = (
             which(os.environ.get("METAFLOW_PATH_TO_MICROMAMBA") or "micromamba")
             or which("./micromamba")  # to support remote execution
@@ -70,6 +81,9 @@ class Micromamba(object):
                 "MAMBA_ADD_PIP_AS_PYTHON_DEPENDENCY": "true",
                 "CONDA_SUBDIR": platform,
                 # "CONDA_UNSATISFIABLE_HINTS_CHECK_DEPTH": "0" # https://github.com/conda/conda/issues/9862
+                # Add a default glibc version for linux-64 environments (ignored for other platforms)
+                # TODO: Make the version configurable
+                "CONDA_OVERRIDE_GLIBC": GLIBC_VERSION,
             }
             cmd = [
                 "create",
@@ -78,6 +92,7 @@ class Micromamba(object):
                 "--dry-run",
                 "--no-extra-safety-checks",
                 "--repodata-ttl=86400",
+                "--safety-checks=disabled",
                 "--retry-clean-cache",
                 "--prefix=%s/prefix" % tmp_dir,
             ]
@@ -91,10 +106,11 @@ class Micromamba(object):
                 cmd.append("python==%s" % python)
             # TODO: Ensure a human readable message is returned when the environment
             #       can't be resolved for any and all reasons.
-            return [
+            solved_packages = [
                 {k: v for k, v in item.items() if k in ["url"]}
                 for item in self._call(cmd, env)["actions"]["LINK"]
             ]
+            return solved_packages
 
     def download(self, id_, packages, python, platform):
         # Unfortunately all the packages need to be catalogued in package cache
@@ -103,8 +119,6 @@ class Micromamba(object):
         # Micromamba is painfully slow in determining if many packages are infact
         # already cached. As a perf heuristic, we check if the environment already
         # exists to short circuit package downloads.
-        if self.path_to_environment(id_, platform):
-            return
 
         prefix = "{env_dirs}/{keyword}/{platform}/{id}".format(
             env_dirs=self.info()["envs_dirs"][0],
@@ -113,13 +127,18 @@ class Micromamba(object):
             id=id_,
         )
 
-        # Another forced perf heuristic to skip cross-platform downloads.
+        # cheap check
         if os.path.exists(f"{prefix}/fake.done"):
             return
 
+        # somewhat expensive check
+        if self.path_to_environment(id_, platform):
+            return
+
         with tempfile.TemporaryDirectory() as tmp_dir:
             env = {
                 "CONDA_SUBDIR": platform,
+                "CONDA_OVERRIDE_GLIBC": GLIBC_VERSION,
             }
             cmd = [
                 "create",
@@ -159,6 +178,7 @@ class Micromamba(object):
             # use hardlinks when possible, otherwise copy files
             # disabled for now since it adds to environment creation latencies
             "CONDA_ALLOW_SOFTLINKS": "0",
+            "CONDA_OVERRIDE_GLIBC": GLIBC_VERSION,
         }
         cmd = [
             "create",
@@ -174,6 +194,7 @@ class Micromamba(object):
             cmd.append("{url}".format(**package))
         self._call(cmd, env)
 
+    @functools.lru_cache(maxsize=None)
     def info(self):
         return self._call(["config", "list", "-a"])
 
@@ -198,18 +219,24 @@ class Micromamba(object):
         }
         directories = self.info()["pkgs_dirs"]
         # search all package caches for packages
-        metadata = {
-            url: os.path.join(d, file)
+
+        file_to_path = {}
+        for d in directories:
+            if os.path.isdir(d):
+                try:
+                    with os.scandir(d) as entries:
+                        for entry in entries:
+                            if entry.is_file():
+                                # Prefer the first occurrence if the file exists in multiple directories
+                                file_to_path.setdefault(entry.name, entry.path)
+                except OSError:
+                    continue
+        ret = {
+            # set package tarball local paths to None if package tarballs are missing
+            url: file_to_path.get(file)
             for url, file in packages_to_filenames.items()
-            for d in directories
-            if os.path.isdir(d)
-            and file in os.listdir(d)
-            and os.path.isfile(os.path.join(d, file))
         }
-        # set package tarball local paths to None if package tarballs are missing
-        for url in packages_to_filenames:
-            metadata.setdefault(url, None)
-        return metadata
+        return ret
 
     def interpreter(self, id_):
         return os.path.join(self.path_to_environment(id_), "bin/python")

metaflow/plugins/pypi/pip.py

@@ -4,6 +4,7 @@ import re
 import shutil
 import subprocess
 import tempfile
+import time
 from concurrent.futures import ThreadPoolExecutor
 from itertools import chain, product
 from urllib.parse import unquote
@@ -50,10 +51,14 @@ INSTALLATION_MARKER = "{prefix}/.pip/id"
 
 
 class Pip(object):
-    def __init__(self, micromamba=None):
+    def __init__(self, micromamba=None, logger=None):
         # pip is assumed to be installed inside a conda environment managed by
         # micromamba. pip commands are executed using `micromamba run --prefix`
-        self.micromamba = micromamba or Micromamba()
+        self.micromamba = micromamba or Micromamba(logger)
+        if logger:
+            self.logger = logger
+        else:
+            self.logger = lambda *args, **kwargs: None  # No-op logger if not provided
 
     def solve(self, id_, packages, python, platform):
         prefix = self.micromamba.path_to_environment(id_)
@@ -123,7 +128,7 @@ class Pip(object):
                         **res,
                         subdir_str=(
                             "#subdirectory=%s" % subdirectory if subdirectory else ""
-                        )
+                        ),
                     )
                     # used to deduplicate the storage location in case wheel does not
                     # build with enough unique identifiers.

metaflow/plugins/pypi/pypi_decorator.py

@@ -25,9 +25,10 @@ class PyPIStepDecorator(StepDecorator):
     defaults = {"packages": {}, "python": None, "disabled": None}  # wheels
 
     def __init__(self, attributes=None, statically_defined=False):
-        self._user_defined_attributes = (
-            attributes.copy() if attributes is not None else {}
+        self._attributes_with_user_values = (
+            set(attributes.keys()) if attributes is not None else set()
         )
+
         super().__init__(attributes, statically_defined)
 
     def step_init(self, flow, graph, step, decos, environment, flow_datastore, logger):
@@ -42,10 +43,9 @@ class PyPIStepDecorator(StepDecorator):
         if "pypi_base" in self.flow._flow_decorators:
             pypi_base = self.flow._flow_decorators["pypi_base"][0]
             super_attributes = pypi_base.attributes
-            self._user_defined_attributes = {
-                **self._user_defined_attributes,
-                **pypi_base._user_defined_attributes,
-            }
+            self._attributes_with_user_values.update(
+                pypi_base._attributes_with_user_values
+            )
             self.attributes["packages"] = {
                 **super_attributes["packages"],
                 **self.attributes["packages"],
@@ -106,7 +106,7 @@ class PyPIStepDecorator(StepDecorator):
         environment.set_local_root(LocalStorage.get_datastore_root_from_config(logger))
 
     def is_attribute_user_defined(self, name):
-        return name in self._user_defined_attributes
+        return name in self._attributes_with_user_values
 
 
 class PyPIFlowDecorator(FlowDecorator):
@@ -129,9 +129,10 @@ class PyPIFlowDecorator(FlowDecorator):
     defaults = {"packages": {}, "python": None, "disabled": None}
 
     def __init__(self, attributes=None, statically_defined=False):
-        self._user_defined_attributes = (
-            attributes.copy() if attributes is not None else {}
+        self._attributes_with_user_values = (
+            set(attributes.keys()) if attributes is not None else set()
         )
+
         super().__init__(attributes, statically_defined)
 
     def flow_init(
@@ -140,6 +141,7 @@ class PyPIFlowDecorator(FlowDecorator):
         from metaflow import decorators
 
         decorators._attach_decorators(flow, ["pypi"])
+        decorators._init(flow)
 
         # @pypi uses a conda environment to create a virtual environment.
         # The conda environment can be created through micromamba.

metaflow/plugins/timeout_decorator.py

@@ -37,8 +37,8 @@ class TimeoutDecorator(StepDecorator):
     name = "timeout"
     defaults = {"seconds": 0, "minutes": 0, "hours": 0}
 
-    def __init__(self, *args, **kwargs):
-        super(TimeoutDecorator, self).__init__(*args, **kwargs)
+    def init(self):
+        super().init()
         # Initialize secs in __init__ so other decorators could safely use this
         # value without worrying about decorator order.
         # Convert values in attributes to type:int since they can be type:str

metaflow/runner/click_api.py

@@ -9,6 +9,7 @@ if sys.version_info < (3, 7):
     )
 
 import datetime
+import functools
 import importlib
 import inspect
 import itertools
@@ -38,6 +39,7 @@ from metaflow.decorators import add_decorator_options
 from metaflow.exception import MetaflowException
 from metaflow.includefile import FilePathClass
 from metaflow.parameters import JSONTypeClass, flow_context
+from metaflow.user_configs.config_options import LocalFileInput
 
 # Define a recursive type alias for JSON
 JSON = Union[Dict[str, "JSON"], List["JSON"], str, int, float, bool, None]
@@ -55,6 +57,7 @@ click_to_python_types = {
     File: str,
     JSONTypeClass: JSON,
     FilePathClass: str,
+    LocalFileInput: str,
 }
 
 
@@ -124,6 +127,37 @@ def _method_sanity_check(
     return method_params
 
 
+def _lazy_load_command(
+    cli_collection: click.Group,
+    flow_parameters: Union[str, List[Parameter]],
+    _self,
+    name: str,
+):
+
+    # Context is not used in get_command so we can pass None. Since we pin click,
+    # this won't change from under us.
+
+    if isinstance(flow_parameters, str):
+        # Resolve flow_parameters -- for start, this is a function which we
+        # need to call to figure out the actual parameters (may be changed by configs)
+        flow_parameters = getattr(_self, flow_parameters)()
+    cmd_obj = cli_collection.get_command(None, name)
+    if cmd_obj:
+        if isinstance(cmd_obj, click.Group):
+            # TODO: possibly check for fake groups with cmd_obj.name in ["cli", "main"]
+            result = functools.partial(extract_group(cmd_obj, flow_parameters), _self)
+        elif isinstance(cmd_obj, click.Command):
+            result = functools.partial(extract_command(cmd_obj, flow_parameters), _self)
+        else:
+            raise RuntimeError(
+                "Cannot handle %s of type %s" % (cmd_obj.name, type(cmd_obj))
+            )
+        setattr(_self, name, result)
+        return result
+    else:
+        raise AttributeError()
+
+
 def get_annotation(param: Union[click.Argument, click.Option]):
     py_type = click_to_python_types[type(param.type)]
     if not param.required:
@@ -179,9 +213,11 @@ def extract_flow_class_from_file(flow_file: str) -> FlowSpec:
 
 
 class MetaflowAPI(object):
-    def __init__(self, parent=None, **kwargs):
+    def __init__(self, parent=None, flow_cls=None, **kwargs):
         self._parent = parent
         self._chain = [{self._API_NAME: kwargs}]
+        self._flow_cls = flow_cls
+        self._cached_computed_parameters = None
 
     @property
     def parent(self):
@@ -200,23 +236,22 @@ class MetaflowAPI(object):
     @classmethod
     def from_cli(cls, flow_file: str, cli_collection: Callable) -> Callable:
         flow_cls = extract_flow_class_from_file(flow_file)
-        flow_parameters = [p for _, p in flow_cls._get_parameters()]
+
         with flow_context(flow_cls) as _:
             add_decorator_options(cli_collection)
 
-        class_dict = {"__module__": "metaflow", "_API_NAME": flow_file}
-        command_groups = cli_collection.sources
-        for each_group in command_groups:
-            for _, cmd_obj in each_group.commands.items():
-                if isinstance(cmd_obj, click.Group):
-                    # TODO: possibly check for fake groups with cmd_obj.name in ["cli", "main"]
-                    class_dict[cmd_obj.name] = extract_group(cmd_obj, flow_parameters)
-                elif isinstance(cmd_obj, click.Command):
-                    class_dict[cmd_obj.name] = extract_command(cmd_obj, flow_parameters)
-                else:
-                    raise RuntimeError(
-                        "Cannot handle %s of type %s" % (cmd_obj.name, type(cmd_obj))
-                    )
+        def getattr_wrapper(_self, name):
+            # Functools.partial do not automatically bind self (no __get__)
+            return _self._internal_getattr(_self, name)
+
+        class_dict = {
+            "__module__": "metaflow",
+            "_API_NAME": flow_file,
+            "_internal_getattr": functools.partial(
+                _lazy_load_command, cli_collection, "_compute_flow_parameters"
+            ),
+            "__getattr__": getattr_wrapper,
+        }
 
         to_return = type(flow_file, (MetaflowAPI,), class_dict)
         to_return.__name__ = flow_file
@@ -237,11 +272,11 @@ class MetaflowAPI(object):
                 defaults,
                 **kwargs,
             )
-            return to_return(parent=None, **method_params)
+            return to_return(parent=None, flow_cls=flow_cls, **method_params)
 
         m = _method
-        m.__name__ = cmd_obj.name
-        m.__doc__ = getattr(cmd_obj, "help", None)
+        m.__name__ = cli_collection.name
+        m.__doc__ = getattr(cli_collection, "help", None)
         m.__signature__ = inspect.signature(_method).replace(
             parameters=params_sigs.values()
         )
@@ -287,6 +322,25 @@ class MetaflowAPI(object):
 
         return components
 
+    def _compute_flow_parameters(self):
+        if self._flow_cls is None or self._parent is not None:
+            raise RuntimeError(
+                "Computing flow-level parameters for a non start API. "
+                "Please report to the Metaflow team."
+            )
+        # TODO: We need to actually compute the new parameters (based on configs) which
+        # would involve processing the options at least partially. We will do this
+        # before GA but for now making it work for regular parameters
+        if self._cached_computed_parameters is not None:
+            return self._cached_computed_parameters
+        self._cached_computed_parameters = []
+        for _, param in self._flow_cls._get_parameters():
+            if param.IS_CONFIG_PARAMETER:
+                continue
+            param.init()
+            self._cached_computed_parameters.append(param)
+        return self._cached_computed_parameters
+
 
 def extract_all_params(cmd_obj: Union[click.Command, click.Group]):
     arg_params_sigs = OrderedDict()
@@ -351,7 +405,7 @@ def extract_group(cmd_obj: click.Group, flow_parameters: List[Parameter]) -> Cal
         method_params = _method_sanity_check(
             possible_arg_params, possible_opt_params, annotations, defaults, **kwargs
         )
-        return resulting_class(parent=_self, **method_params)
+        return resulting_class(parent=_self, flow_cls=None, **method_params)
 
     m = _method
     m.__name__ = cmd_obj.name

metaflow/runner/deployer.py

@@ -64,7 +64,7 @@ class Deployer(metaclass=DeployerMeta):
         The directory to run the subprocess in; if not specified, the current
         directory is used.
     file_read_timeout : int, default 3600
-        The timeout until which we try to read the deployer attribute file.
+        The timeout until which we try to read the deployer attribute file (in seconds).
     **kwargs : Any
         Additional arguments that you would pass to `python myflow.py` before
         the deployment command.

metaflow/runner/deployer_impl.py

@@ -37,7 +37,7 @@ class DeployerImpl(object):
         The directory to run the subprocess in; if not specified, the current
         directory is used.
     file_read_timeout : int, default 3600
-        The timeout until which we try to read the deployer attribute file.
+        The timeout until which we try to read the deployer attribute file (in seconds).
     **kwargs : Any
         Additional arguments that you would pass to `python myflow.py` before
         the deployment command.
@@ -144,7 +144,7 @@ class DeployerImpl(object):
             # Additional info is used to pass additional deployer specific information.
             # It is used in non-OSS deployers (extensions).
             self.additional_info = content.get("additional_info", {})
-
+            command_obj.sync_wait()
             if command_obj.process.returncode == 0:
                 return create_class(deployer=self)
 

metaflow/runner/metaflow_runner.py

@@ -221,7 +221,7 @@ class Runner(object):
         The directory to run the subprocess in; if not specified, the current
         directory is used.
     file_read_timeout : int, default 3600
-        The timeout until which we try to read the runner attribute file.
+        The timeout until which we try to read the runner attribute file (in seconds).
     **kwargs : Any
         Additional arguments that you would pass to `python myflow.py` before
         the `run` command.
@@ -272,6 +272,9 @@ class Runner(object):
 
     def __get_executing_run(self, attribute_file_fd, command_obj):
         content = handle_timeout(attribute_file_fd, command_obj, self.file_read_timeout)
+
+        command_obj.sync_wait()
+
         content = json.loads(content)
         pathspec = "%s/%s" % (content.get("flow_name"), content.get("run_id"))