metaflow 2.12.22__py2.py3-none-any.whl → 2.12.23__py2.py3-none-any.whl

metaflow/cli.py

@@ -1,5 +1,5 @@
 import inspect
-import os
+import json
 import sys
 import traceback
 from datetime import datetime
@@ -7,6 +7,7 @@ from functools import wraps
 
 import metaflow.tracing as tracing
 from metaflow._vendor import click
+from metaflow.client.core import get_metadata
 
 from . import decorators, lint, metaflow_version, namespace, parameters, plugins
 from .cli_args import cli_args
@@ -698,15 +699,17 @@ def resume(
     runtime.print_workflow_info()
 
     runtime.persist_constants()
-    write_file(
-        runner_attribute_file,
-        "%s@%s:%s"
-        % (
-            obj.metadata.__class__.TYPE,
-            obj.metadata.__class__.INFO,
-            "/".join((obj.flow.name, runtime.run_id)),
-        ),
-    )
+
+    if runner_attribute_file:
+        with open(runner_attribute_file, "w") as f:
+            json.dump(
+                {
+                    "run_id": runtime.run_id,
+                    "flow_name": obj.flow.name,
+                    "metadata": get_metadata(),
+                },
+                f,
+            )
 
     # We may skip clone-only resume if this is not a resume leader,
     # and clone is already complete.
@@ -774,15 +777,17 @@ def run(
     obj.flow._set_constants(obj.graph, kwargs)
     runtime.print_workflow_info()
     runtime.persist_constants()
-    write_file(
-        runner_attribute_file,
-        "%s@%s:%s"
-        % (
-            obj.metadata.__class__.TYPE,
-            obj.metadata.__class__.INFO,
-            "/".join((obj.flow.name, runtime.run_id)),
-        ),
-    )
+
+    if runner_attribute_file:
+        with open(runner_attribute_file, "w") as f:
+            json.dump(
+                {
+                    "run_id": runtime.run_id,
+                    "flow_name": obj.flow.name,
+                    "metadata": get_metadata(),
+                },
+                f,
+            )
     runtime.execute()
 
 

metaflow/client/core.py

@@ -2358,7 +2358,7 @@ class Metaflow(object):
         Flow
             Flow with the given name.
         """
-        return Flow(id)
+        return Flow(name)
 
 
 _CLASSES["flow"] = Flow

metaflow/cmd/develop/stub_generator.py

@@ -115,6 +115,11 @@ class StubGenerator:
         :type members_from_other_modules: List[str]
         """
 
+        # Let metaflow know we are in stubgen mode. This is sometimes useful to skip
+        # some processing like loading libraries, etc. It is used in Metaflow extensions
+        # so do not remove even if you do not see a use for it directly in the code.
+        os.environ["METAFLOW_STUBGEN"] = "1"
+
         self._write_generated_for = include_generated_for
         self._pending_modules = ["metaflow"]  # type: List[str]
         self._pending_modules.extend(get_aliased_modules())
@@ -398,6 +403,18 @@ class StubGenerator:
         name_with_module = self._get_element_name_with_module(clazz.__class__)
         buff.write("metaclass=" + name_with_module + "):\n")
 
+        # Add class docstring
+        if clazz.__doc__:
+            buff.write('%s"""\n' % TAB)
+            my_doc = cast(str, deindent_docstring(clazz.__doc__))
+            init_blank = True
+            for line in my_doc.split("\n"):
+                if init_blank and len(line.strip()) == 0:
+                    continue
+                init_blank = False
+                buff.write("%s%s\n" % (TAB, line.rstrip()))
+            buff.write('%s"""\n' % TAB)
+
         # For NamedTuple, we have __annotations__ but no __init__. In that case,
         # we are going to "create" a __init__ function with the annotations
         # to show what the class takes.

metaflow/cmd/develop/stubs.py

@@ -170,7 +170,7 @@ def install(ctx: Any, force: bool):
                     "Metaflow stubs are already installed and valid -- use --force to reinstall"
                 )
             return
-    mf_version, _ = get_mf_version()
+    mf_version, _ = get_mf_version(True)
     with tempfile.TemporaryDirectory() as tmp_dir:
         with open(os.path.join(tmp_dir, "setup.py"), "w") as f:
             f.write(
@@ -261,10 +261,10 @@ def split_version(vers: str) -> Tuple[str, Optional[str]]:
     return vers_split[0], vers_split[1]
 
 
-def get_mf_version() -> Tuple[str, Optional[str]]:
+def get_mf_version(public: bool = False) -> Tuple[str, Optional[str]]:
     from metaflow.metaflow_version import get_version
 
-    return split_version(get_version())
+    return split_version(get_version(public))
 
 
 def get_stubs_version(stubs_root_path: Optional[str]) -> Tuple[str, Optional[str]]:

metaflow/metaflow_version.py

@@ -195,11 +195,14 @@ def get_version(public=False):
         if ext_version is None:
             ext_version = getattr(extension_module, "__version__", "<unk>")
         # Update the package information about reported version for the extension
-        update_package_info(
-            package_name=pkg_name,
-            extension_name=ext_name,
-            package_version=ext_version,
-        )
+        # (only for the full info which is called at least once -- if we update more
+        # it will error out since we can only update_package_info once)
+        if not public:
+            update_package_info(
+                package_name=pkg_name,
+                extension_name=ext_name,
+                package_version=ext_version,
+            )
         ext_versions.append("%s(%s)" % (ext_name, ext_version))
 
     # We now have all the information about extensions so we can form the final string

metaflow/plugins/argo/argo_client.py

@@ -295,6 +295,8 @@ class ArgoClient(object):
                 "suspend": schedule is None,
                 "schedule": schedule,
                 "timezone": timezone,
+                "failedJobsHistoryLimit": 10000,  # default is unfortunately 1
+                "successfulJobsHistoryLimit": 10000,  # default is unfortunately 3
                 "workflowSpec": {"workflowTemplateRef": {"name": name}},
             },
         }

metaflow/plugins/argo/argo_workflows.py

@@ -1490,7 +1490,11 @@ class ArgoWorkflows(object):
                         # {{foo.bar['param_name']}}.
                         # https://argoproj.github.io/argo-events/tutorials/02-parameterization/
                         # http://masterminds.github.io/sprig/strings.html
-                        "--%s={{workflow.parameters.%s}}"
+                        (
+                            "--%s='{{workflow.parameters.%s}}'"
+                            if parameter["type"] == "JSON"
+                            else "--%s={{workflow.parameters.%s}}"
+                        )
                         % (parameter["name"], parameter["name"])
                         for parameter in self.parameters.values()
                     ]
@@ -2627,50 +2631,54 @@ class ArgoWorkflows(object):
         )
         from kubernetes import client as kubernetes_sdk
 
-        return DaemonTemplate("heartbeat-daemon").container(
-            to_camelcase(
-                kubernetes_sdk.V1Container(
-                    name="main",
-                    # TODO: Make the image configurable
-                    image=resources["image"],
-                    command=cmds,
-                    env=[
-                        kubernetes_sdk.V1EnvVar(name=k, value=str(v))
-                        for k, v in env.items()
-                    ],
-                    env_from=[
-                        kubernetes_sdk.V1EnvFromSource(
-                            secret_ref=kubernetes_sdk.V1SecretEnvSource(
-                                name=str(k),
-                                # optional=True
+        return (
+            DaemonTemplate("heartbeat-daemon")
+            .service_account_name(resources["service_account"])
+            .container(
+                to_camelcase(
+                    kubernetes_sdk.V1Container(
+                        name="main",
+                        # TODO: Make the image configurable
+                        image=resources["image"],
+                        command=cmds,
+                        env=[
+                            kubernetes_sdk.V1EnvVar(name=k, value=str(v))
+                            for k, v in env.items()
+                        ],
+                        env_from=[
+                            kubernetes_sdk.V1EnvFromSource(
+                                secret_ref=kubernetes_sdk.V1SecretEnvSource(
+                                    name=str(k),
+                                    # optional=True
+                                )
                             )
-                        )
-                        for k in list(
-                            []
-                            if not resources.get("secrets")
-                            else (
-                                [resources.get("secrets")]
-                                if isinstance(resources.get("secrets"), str)
-                                else resources.get("secrets")
+                            for k in list(
+                                []
+                                if not resources.get("secrets")
+                                else (
+                                    [resources.get("secrets")]
+                                    if isinstance(resources.get("secrets"), str)
+                                    else resources.get("secrets")
+                                )
                             )
-                        )
-                        + KUBERNETES_SECRETS.split(",")
-                        + ARGO_WORKFLOWS_KUBERNETES_SECRETS.split(",")
-                        if k
-                    ],
-                    resources=kubernetes_sdk.V1ResourceRequirements(
-                        # NOTE: base resources for this are kept to a minimum to save on running costs.
-                        # This has an adverse effect on startup time for the daemon, which can be completely
-                        # alleviated by using a base image that has the required dependencies pre-installed
-                        requests={
-                            "cpu": "200m",
-                            "memory": "100Mi",
-                        },
-                        limits={
-                            "cpu": "200m",
-                            "memory": "100Mi",
-                        },
-                    ),
+                            + KUBERNETES_SECRETS.split(",")
+                            + ARGO_WORKFLOWS_KUBERNETES_SECRETS.split(",")
+                            if k
+                        ],
+                        resources=kubernetes_sdk.V1ResourceRequirements(
+                            # NOTE: base resources for this are kept to a minimum to save on running costs.
+                            # This has an adverse effect on startup time for the daemon, which can be completely
+                            # alleviated by using a base image that has the required dependencies pre-installed
+                            requests={
+                                "cpu": "200m",
+                                "memory": "100Mi",
+                            },
+                            limits={
+                                "cpu": "200m",
+                                "memory": "100Mi",
+                            },
+                        ),
+                    )
                 )
             )
         )
@@ -3292,6 +3300,10 @@ class DaemonTemplate(object):
         self.payload["container"] = container
         return self
 
+    def service_account_name(self, service_account_name):
+        self.payload["serviceAccountName"] = service_account_name
+        return self
+
     def to_json(self):
         return self.payload
 

metaflow/plugins/kubernetes/kubernetes_client.py

@@ -121,7 +121,10 @@ class KubernetesClient(object):
         job_api = self._client.BatchV1Api()
         pods = self._find_active_pods(flow_name, run_id, user)
 
+        active_pods = False
+
         def _kill_pod(pod):
+            active_pods = True
             echo("Killing Kubernetes pod %s\n" % pod.metadata.name)
             try:
                 stream(
@@ -155,7 +158,10 @@ class KubernetesClient(object):
                     echo("failed to kill pod %s - %s" % (pod.metadata.name, str(e)))
 
         with ThreadPoolExecutor() as executor:
-            executor.map(_kill_pod, list(pods))
+            executor.map(_kill_pod, pods)
+
+        if not active_pods:
+            echo("No active Kubernetes pods found for run *%s*" % run_id)
 
     def jobset(self, **kwargs):
         return KubernetesJobSet(self, **kwargs)

metaflow/plugins/kubernetes/kubernetes_decorator.py

@@ -543,7 +543,11 @@ class KubernetesDecorator(StepDecorator):
 
 # TODO: Unify this method with the multi-node setup in @batch
 def _setup_multinode_environment():
-    # FIXME: what about MF_MASTER_PORT
+    # TODO [FIXME SOON]
+    # Even if Kubernetes may deploy control pods before worker pods, there is always a
+    # possibility that the worker pods may start before the control. In the case that this happens,
+    # the worker pods will not be able to resolve the control pod's IP address and this will cause
+    # the worker pods to fail. This function should account for this in the near future.
     import socket
 
     try:

metaflow/plugins/kubernetes/kubernetes_jobsets.py

@@ -866,7 +866,13 @@ class KubernetesJobSet(object):
             spec=dict(
                 replicatedJobs=[self.control.dump(), self.worker.dump()],
                 suspend=False,
-                startupPolicy=None,
+                startupPolicy=dict(
+                    # We explicitly set an InOrder Startup policy so that
+                    # we can ensure that the control pod starts before the worker pods.
+                    # This is required so that when worker pods try to access the control's IP
+                    # we are able to resolve the control's IP address.
+                    startupPolicyOrder="InOrder"
+                ),
                 successPolicy=None,
                 # The Failure Policy helps setting the number of retries for the jobset.
                 # but we don't rely on it and instead rely on either the local scheduler

metaflow/plugins/pypi/bootstrap.py

@@ -89,7 +89,7 @@ if __name__ == "__main__":
         # TODO: micromamba installation can be pawned off to micromamba.py
         f"""set -e;
         if ! command -v micromamba >/dev/null 2>&1; then
-            mkdir micromamba;
+            mkdir -p micromamba;
             python -c "import requests, bz2, sys; data = requests.get('https://micro.mamba.pm/api/micromamba/{architecture}/1.5.7').content; sys.stdout.buffer.write(bz2.decompress(data))" | tar -xv -C $(pwd)/micromamba bin/micromamba --strip-components 1;
             export PATH=$PATH:$(pwd)/micromamba;
             if ! command -v micromamba >/dev/null 2>&1; then

metaflow/plugins/pypi/micromamba.py

@@ -253,7 +253,33 @@ class Micromamba(object):
             try:
                 output = json.loads(e.output)
                 err = []
+                v_pkgs = ["__cuda", "__glibc"]
                 for error in output.get("solver_problems", []):
+                    # raise a specific error message for virtual package related errors
+                    match = next((p for p in v_pkgs if p in error), None)
+                    if match is not None:
+                        vpkg_name = match[2:]
+                        # try to strip version from error msg which are of the format:
+                        # nothing provides <__vpkg> >=2.17,<3.0.a0 needed by <pkg_name>
+                        try:
+                            vpkg_version = error[
+                                len("nothing provides %s " % match) : error.index(
+                                    " needed by"
+                                )
+                            ]
+                        except ValueError:
+                            vpkg_version = None
+                        raise MicromambaException(
+                            "Please set the environment variable CONDA_OVERRIDE_{var} to a specific version{version} of {name}.\n"
+                            "Here is an example of supplying environment variables through the command line -\n\n"
+                            "CONDA_OVERRIDE_{var}=<{name}-version> python flow.py <args>".format(
+                                var=vpkg_name.upper(),
+                                version=(
+                                    "" if not vpkg_version else (" (%s)" % vpkg_version)
+                                ),
+                                name=vpkg_name,
+                            ),
+                        )
                     err.append(error)
                 raise MicromambaException(
                     msg.format(

metaflow/runner/deployer.py

@@ -6,56 +6,11 @@ import importlib
 import functools
 import tempfile
 
-from subprocess import CalledProcessError
 from typing import Optional, Dict, ClassVar
 
 from metaflow.exception import MetaflowNotFound
-from metaflow.runner.subprocess_manager import CommandManager, SubprocessManager
-from metaflow.runner.utils import read_from_file_when_ready
-
-
-def handle_timeout(
-    tfp_runner_attribute, command_obj: CommandManager, file_read_timeout: int
-):
-    """
-    Handle the timeout for a running subprocess command that reads a file
-    and raises an error with appropriate logs if a TimeoutError occurs.
-
-    Parameters
-    ----------
-    tfp_runner_attribute : NamedTemporaryFile
-        Temporary file that stores runner attribute data.
-    command_obj : CommandManager
-        Command manager object that encapsulates the running command details.
-    file_read_timeout : int
-        Timeout for reading the file.
-
-    Returns
-    -------
-    str
-        Content read from the temporary file.
-
-    Raises
-    ------
-    RuntimeError
-        If a TimeoutError occurs, it raises a RuntimeError with the command's
-        stdout and stderr logs.
-    """
-    try:
-        content = read_from_file_when_ready(
-            tfp_runner_attribute.name, command_obj, timeout=file_read_timeout
-        )
-        return content
-    except (CalledProcessError, TimeoutError) as e:
-        stdout_log = open(command_obj.log_files["stdout"]).read()
-        stderr_log = open(command_obj.log_files["stderr"]).read()
-        command = " ".join(command_obj.command)
-        error_message = "Error executing: '%s':\n" % command
-        if stdout_log.strip():
-            error_message += "\nStdout:\n%s\n" % stdout_log
-        if stderr_log.strip():
-            error_message += "\nStderr:\n%s\n" % stderr_log
-        raise RuntimeError(error_message) from e
+from metaflow.runner.subprocess_manager import SubprocessManager
+from metaflow.runner.utils import handle_timeout
 
 
 def get_lower_level_group(
@@ -209,7 +164,7 @@ class TriggeredRun(object):
             elif callable(v):
                 setattr(self, k, functools.partial(v, self))
             else:
-                setattr(self.__class__, k, property(fget=lambda _, v=v: v))
+                setattr(self, k, v)
 
     def wait_for_run(self, timeout=None):
         """
@@ -287,7 +242,7 @@ class DeployedFlow(object):
             elif callable(v):
                 setattr(self, k, functools.partial(v, self))
             else:
-                setattr(self.__class__, k, property(fget=lambda _, v=v: v))
+                setattr(self, k, v)
 
 
 class DeployerImpl(object):

metaflow/runner/metaflow_runner.py

@@ -1,14 +1,14 @@
 import importlib
 import os
 import sys
+import json
 import tempfile
 
-from subprocess import CalledProcessError
 from typing import Dict, Iterator, Optional, Tuple
 
 from metaflow import Run, metadata
 
-from .utils import clear_and_set_os_environ, read_from_file_when_ready
+from .utils import handle_timeout, clear_and_set_os_environ
 from .subprocess_manager import CommandManager, SubprocessManager
 
 
@@ -102,16 +102,19 @@ class ExecutingRun(object):
         for executing the run.
 
         The return value is one of the following strings:
+        - `timeout` indicates that the run timed out.
         - `running` indicates a currently executing run.
         - `failed` indicates a failed run.
-        - `successful` a successful run.
+        - `successful` indicates a successful run.
 
         Returns
         -------
         str
             The current status of the run.
         """
-        if self.command_obj.process.returncode is None:
+        if self.command_obj.timeout:
+            return "timeout"
+        elif self.command_obj.process.returncode is None:
             return "running"
         elif self.command_obj.process.returncode != 0:
             return "failed"
@@ -271,28 +274,22 @@ class Runner(object):
 
         # It is thus necessary to set them to correct values before we return
         # the Run object.
-        try:
-            # Set the environment variables to what they were before the run executed.
-            clear_and_set_os_environ(self.old_env)
 
-            # Set the correct metadata from the runner_attribute file corresponding to this run.
-            content = read_from_file_when_ready(
-                tfp_runner_attribute.name, command_obj, timeout=self.file_read_timeout
-            )
-            metadata_for_flow, pathspec = content.rsplit(":", maxsplit=1)
-            metadata(metadata_for_flow)
-            run_object = Run(pathspec, _namespace_check=False)
-            return ExecutingRun(self, command_obj, run_object)
-        except (CalledProcessError, TimeoutError) as e:
-            stdout_log = open(command_obj.log_files["stdout"]).read()
-            stderr_log = open(command_obj.log_files["stderr"]).read()
-            command = " ".join(command_obj.command)
-            error_message = "Error executing: '%s':\n" % command
-            if stdout_log.strip():
-                error_message += "\nStdout:\n%s\n" % stdout_log
-            if stderr_log.strip():
-                error_message += "\nStderr:\n%s\n" % stderr_log
-            raise RuntimeError(error_message) from e
+        content = handle_timeout(
+            tfp_runner_attribute, command_obj, self.file_read_timeout
+        )
+        content = json.loads(content)
+        pathspec = "%s/%s" % (content.get("flow_name"), content.get("run_id"))
+
+        # Set the environment variables to what they were before the run executed.
+        clear_and_set_os_environ(self.old_env)
+
+        # Set the correct metadata from the runner_attribute file corresponding to this run.
+        metadata_for_flow = content.get("metadata")
+        metadata(metadata_for_flow)
+
+        run_object = Run(pathspec, _namespace_check=False)
+        return ExecutingRun(self, command_obj, run_object)
 
     def run(self, **kwargs) -> ExecutingRun:
         """

metaflow/runner/subprocess_manager.py

@@ -42,6 +42,19 @@ class SubprocessManager(object):
     def __init__(self):
         self.commands: Dict[int, CommandManager] = {}
 
+        try:
+            loop = asyncio.get_running_loop()
+            loop.add_signal_handler(
+                signal.SIGINT,
+                lambda: self._handle_sigint(signum=signal.SIGINT, frame=None),
+            )
+        except RuntimeError:
+            signal.signal(signal.SIGINT, self._handle_sigint)
+
+    def _handle_sigint(self, signum, frame):
+        for each_command in self.commands.values():
+            each_command.kill(termination_timeout=2)
+
     async def __aenter__(self) -> "SubprocessManager":
         return self
 
@@ -83,6 +96,7 @@ class SubprocessManager(object):
         command_obj = CommandManager(command, env, cwd)
         pid = command_obj.run(show_output=show_output)
         self.commands[pid] = command_obj
+        command_obj.sync_wait()
         return pid
 
     async def async_run_command(
@@ -169,11 +183,12 @@ class CommandManager(object):
         self.cwd = cwd if cwd is not None else os.getcwd()
 
         self.process = None
+        self.stdout_thread = None
+        self.stderr_thread = None
         self.run_called: bool = False
+        self.timeout: bool = False
         self.log_files: Dict[str, str] = {}
 
-        signal.signal(signal.SIGINT, self._handle_sigint)
-
     async def __aenter__(self) -> "CommandManager":
         return self
 
@@ -214,13 +229,22 @@ class CommandManager(object):
                 else:
                     await asyncio.wait_for(self.emit_logs(stream), timeout)
             except asyncio.TimeoutError:
+                self.timeout = True
                 command_string = " ".join(self.command)
-                await self.kill()
+                self.kill(termination_timeout=2)
                 print(
                     "Timeout: The process (PID %d; command: '%s') did not complete "
                     "within %s seconds." % (self.process.pid, command_string, timeout)
                 )
 
+    def sync_wait(self):
+        if not self.run_called:
+            raise RuntimeError("No command run yet to wait for...")
+
+        self.process.wait()
+        self.stdout_thread.join()
+        self.stderr_thread.join()
+
     def run(self, show_output: bool = False):
         """
         Run the subprocess synchronously. This can only be called once.
@@ -265,22 +289,17 @@ class CommandManager(object):
 
                 self.run_called = True
 
-                stdout_thread = threading.Thread(
+                self.stdout_thread = threading.Thread(
                     target=stream_to_stdout_and_file,
                     args=(self.process.stdout, stdout_logfile),
                 )
-                stderr_thread = threading.Thread(
+                self.stderr_thread = threading.Thread(
                     target=stream_to_stdout_and_file,
                     args=(self.process.stderr, stderr_logfile),
                 )
 
-                stdout_thread.start()
-                stderr_thread.start()
-
-                self.process.wait()
-
-                stdout_thread.join()
-                stderr_thread.join()
+                self.stdout_thread.start()
+                self.stderr_thread.start()
 
                 return self.process.pid
             except Exception as e:
@@ -441,13 +460,13 @@ class CommandManager(object):
         if self.run_called:
             shutil.rmtree(self.temp_dir, ignore_errors=True)
 
-    async def kill(self, termination_timeout: float = 5):
+    def kill(self, termination_timeout: float = 2):
         """
         Kill the subprocess and its descendants.
 
         Parameters
         ----------
-        termination_timeout : float, default 5
+        termination_timeout : float, default 2
             The time to wait after sending a SIGTERM to the process and its descendants
             before sending a SIGKILL.
         """
@@ -457,9 +476,6 @@ class CommandManager(object):
         else:
             print("No process to kill.")
 
-    def _handle_sigint(self, signum, frame):
-        asyncio.create_task(self.kill())
-
 
 async def main():
     flow_file = "../try.py"

metaflow/runner/utils.py

@@ -1,6 +1,7 @@
 import os
 import ast
 import time
+import asyncio
 
 from subprocess import CalledProcessError
 from typing import Dict, TYPE_CHECKING
@@ -40,6 +41,13 @@ def clear_and_set_os_environ(env: Dict):
     os.environ.update(env)
 
 
+def check_process_status(command_obj: "CommandManager"):
+    if isinstance(command_obj.process, asyncio.subprocess.Process):
+        return command_obj.process.returncode is not None
+    else:
+        return command_obj.process.poll() is not None
+
+
 def read_from_file_when_ready(
     file_path: str, command_obj: "CommandManager", timeout: float = 5
 ):
@@ -47,7 +55,7 @@ def read_from_file_when_ready(
     with open(file_path, "r", encoding="utf-8") as file_pointer:
         content = file_pointer.read()
         while not content:
-            if command_obj.process.poll() is not None:
+            if check_process_status(command_obj):
                 # Check to make sure the file hasn't been read yet to avoid a race
                 # where the file is written between the end of this while loop and the
                 # poll call above.
@@ -64,3 +72,47 @@ def read_from_file_when_ready(
             time.sleep(0.1)
             content = file_pointer.read()
         return content
+
+
+def handle_timeout(
+    tfp_runner_attribute, command_obj: "CommandManager", file_read_timeout: int
+):
+    """
+    Handle the timeout for a running subprocess command that reads a file
+    and raises an error with appropriate logs if a TimeoutError occurs.
+
+    Parameters
+    ----------
+    tfp_runner_attribute : NamedTemporaryFile
+        Temporary file that stores runner attribute data.
+    command_obj : CommandManager
+        Command manager object that encapsulates the running command details.
+    file_read_timeout : int
+        Timeout for reading the file.
+
+    Returns
+    -------
+    str
+        Content read from the temporary file.
+
+    Raises
+    ------
+    RuntimeError
+        If a TimeoutError occurs, it raises a RuntimeError with the command's
+        stdout and stderr logs.
+    """
+    try:
+        content = read_from_file_when_ready(
+            tfp_runner_attribute.name, command_obj, timeout=file_read_timeout
+        )
+        return content
+    except (CalledProcessError, TimeoutError) as e:
+        stdout_log = open(command_obj.log_files["stdout"]).read()
+        stderr_log = open(command_obj.log_files["stderr"]).read()
+        command = " ".join(command_obj.command)
+        error_message = "Error executing: '%s':\n" % command
+        if stdout_log.strip():
+            error_message += "\nStdout:\n%s\n" % stdout_log
+        if stderr_log.strip():
+            error_message += "\nStderr:\n%s\n" % stderr_log
+        raise RuntimeError(error_message) from e

metaflow/version.py

@@ -1 +1 @@
-metaflow_version = "2.12.22"
+metaflow_version = "2.12.23"

{metaflow-2.12.22.dist-info → metaflow-2.12.23.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: metaflow
-Version: 2.12.22
+Version: 2.12.23
 Summary: Metaflow: More Data Science, Less Engineering
 Author: Metaflow Developers
 Author-email: help@metaflow.org
@@ -26,7 +26,7 @@ License-File: LICENSE
 Requires-Dist: requests
 Requires-Dist: boto3
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.12.22; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.12.23; extra == "stubs"
 
 ![Metaflow_Logo_Horizontal_FullColor_Ribbon_Dark_RGB](https://user-images.githubusercontent.com/763451/89453116-96a57e00-d713-11ea-9fa6-82b29d4d6eff.png)
 

{metaflow-2.12.22.dist-info → metaflow-2.12.23.dist-info}/RECORD

@@ -1,7 +1,7 @@
 metaflow/R.py,sha256=CqVfIatvmjciuICNnoyyNGrwE7Va9iXfLdFbQa52hwA,3958
 metaflow/__init__.py,sha256=9pnkRH00bbtzaQDSxEspANaB3r6YDC37_EjX7GxtJug,5641
 metaflow/cards.py,sha256=tP1_RrtmqdFh741pqE4t98S7SA0MtGRlGvRICRZF1Mg,426
-metaflow/cli.py,sha256=fa6dx6F2PKtq0oeMxRM2efA9LyE108jGZQvYCtiUS94,34274
+metaflow/cli.py,sha256=vz8flftUkmRBdjHHwWREsFecfNqlFF0YoAKSzexE30w,34494
 metaflow/cli_args.py,sha256=lcgBGNTvfaiPxiUnejAe60Upt9swG6lRy1_3OqbU6MY,2616
 metaflow/clone_util.py,sha256=XfUX0vssu_hPlyZfhFl1AOnKkLqvt33Qp8xNrmdocGg,2057
 metaflow/cmd_with_io.py,sha256=kl53HkAIyv0ecpItv08wZYczv7u3msD1VCcciqigqf0,588
@@ -21,7 +21,7 @@ metaflow/metaflow_config_funcs.py,sha256=5GlvoafV6SxykwfL8D12WXSfwjBN_NsyuKE_Q3g
 metaflow/metaflow_current.py,sha256=pC-EMnAsnvBLvLd61W6MvfiCKcboryeui9f6r8z_sg8,7161
 metaflow/metaflow_environment.py,sha256=rojFyGdyY56sN1HaEb1-0XX53Q3XPNnl0SaH-8xXZ8w,7987
 metaflow/metaflow_profile.py,sha256=jKPEW-hmAQO-htSxb9hXaeloLacAh41A35rMZH6G8pA,418
-metaflow/metaflow_version.py,sha256=lOsrhMfHdKXBPS-673sZb48dK2FJhr34rIwLqMFeG9I,7291
+metaflow/metaflow_version.py,sha256=duhIzfKZtcxMVMs2uiBqBvUarSHJqyWDwMhaBOQd_g0,7491
 metaflow/monitor.py,sha256=T0NMaBPvXynlJAO_avKtk8OIIRMyEuMAyF8bIp79aZU,5323
 metaflow/multicore_utils.py,sha256=vdTNgczVLODifscUbbveJbuSDOl3Y9pAxhr7sqYiNf4,4760
 metaflow/package.py,sha256=QutDP6WzjwGk1UCKXqBfXa9F10Q--FlRr0J7fwlple0,7399
@@ -36,7 +36,7 @@ metaflow/tuple_util.py,sha256=_G5YIEhuugwJ_f6rrZoelMFak3DqAR2tt_5CapS1XTY,830
 metaflow/unbounded_foreach.py,sha256=p184WMbrMJ3xKYHwewj27ZhRUsSj_kw1jlye5gA9xJk,387
 metaflow/util.py,sha256=olAvJK3y1it_k99MhLulTaAJo7OFVt5rnrD-ulIFLCU,13616
 metaflow/vendor.py,sha256=FchtA9tH22JM-eEtJ2c9FpUdMn8sSb1VHuQS56EcdZk,5139
-metaflow/version.py,sha256=rnbTumj_lXNCrZgYvwtVWNwWj0XG3M0qQhmivZyGWG4,29
+metaflow/version.py,sha256=4Kj4wLDHQlSoABrTJYqXZSqf1PE_WoOTlHFEMNeYCyI,29
 metaflow/_vendor/__init__.py,sha256=y_CiwUD3l4eAKvTVDZeqgVujMy31cAM1qjAB-HfI-9s,353
 metaflow/_vendor/typing_extensions.py,sha256=0nUs5p1A_UrZigrAVBoOEM6TxU37zzPDUtiij1ZwpNc,110417
 metaflow/_vendor/zipp.py,sha256=ajztOH-9I7KA_4wqDYygtHa6xUBVZgFpmZ8FE74HHHI,8425
@@ -111,7 +111,7 @@ metaflow/_vendor/v3_6/importlib_metadata/_meta.py,sha256=_F48Hu_jFxkfKWz5wcYS8vO
 metaflow/_vendor/v3_6/importlib_metadata/_text.py,sha256=HCsFksZpJLeTP3NEk_ngrAeXVRRtTrtyh9eOABoRP4A,2166
 metaflow/_vendor/v3_6/importlib_metadata/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 metaflow/client/__init__.py,sha256=1GtQB4Y_CBkzaxg32L1syNQSlfj762wmLrfrDxGi1b8,226
-metaflow/client/core.py,sha256=jnFTP_XLTLJCHRqaxGKYDn1ZwB_v6a1AAkULlLmJGOk,74171
+metaflow/client/core.py,sha256=vDRmLhoRXOfFiIplY2Xp3Go5lnn-CKeJdPqtOjWIX4Y,74173
 metaflow/client/filecache.py,sha256=Wy0yhhCqC1JZgebqi7z52GCwXYnkAqMZHTtxThvwBgM,15229
 metaflow/cmd/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
 metaflow/cmd/configure_cmd.py,sha256=o-DKnUf2FBo_HiMVyoyzQaGBSMtpbEPEdFTQZ0hkU-k,33396
@@ -119,8 +119,8 @@ metaflow/cmd/main_cli.py,sha256=E546zT_jYQKysmjwfpEgzZd5QMsyirs28M2s0OPU93E,2966
 metaflow/cmd/tutorials_cmd.py,sha256=8FdlKkicTOhCIDKcBR5b0Oz6giDvS-EMY3o9skIrRqw,5156
 metaflow/cmd/util.py,sha256=jS_0rUjOnGGzPT65fzRLdGjrYAOOLA4jU2S0HJLV0oc,406
 metaflow/cmd/develop/__init__.py,sha256=p1Sy8yU1MEKSrH5ttOWOZvNcI1qYu6J6jghdTHwPgOw,689
-metaflow/cmd/develop/stub_generator.py,sha256=-FddHkXy6_DTnZnVACd68KAJjVLJWX04Kt9-jZS2b94,47286
-metaflow/cmd/develop/stubs.py,sha256=uKL3PfczjSDt5kA2-PiFqxWT7ShQGhGpwzXaDpiPVIE,11854
+metaflow/cmd/develop/stub_generator.py,sha256=_P_80CRFxyYjoMFynwg0IhAiexL9Wh2WqsnagiaVYVw,48050
+metaflow/cmd/develop/stubs.py,sha256=JX2qNZDvG0upvPueAcLhoR_zyLtRranZMwY05tLdpRQ,11884
 metaflow/datastore/__init__.py,sha256=VxP6ddJt3rwiCkpiSfAhyVkUCOe1pgZZsytVEJzFmSQ,155
 metaflow/datastore/content_addressed_store.py,sha256=6T7tNqL29kpmecyMLHF35RhoSBOb-OZcExnsB65AvnI,7641
 metaflow/datastore/datastore_set.py,sha256=R5pwnxg1DD8kBY9vElvd2eMknrvwTyiSwvQs67_z9bc,2361
@@ -173,9 +173,9 @@ metaflow/plugins/airflow/sensors/base_sensor.py,sha256=s-OQBfPWZ_T3wn96Ua59CCEj1
 metaflow/plugins/airflow/sensors/external_task_sensor.py,sha256=zhYlrZnXT20KW8-fVk0fCNtTyNiKJB5PMVASacu30r0,6034
 metaflow/plugins/airflow/sensors/s3_sensor.py,sha256=iDReG-7FKnumrtQg-HY6cCUAAqNA90nARrjjjEEk_x4,3275
 metaflow/plugins/argo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-metaflow/plugins/argo/argo_client.py,sha256=MKKhMCbWOPzf6z5zQQiyDRHHkAXcO7ipboDZDqAAvOk,15849
+metaflow/plugins/argo/argo_client.py,sha256=KTUpP0DmnmNsMp4tbdNyKX_zOdTFRVpUkrf7Vv79d-o,16011
 metaflow/plugins/argo/argo_events.py,sha256=_C1KWztVqgi3zuH57pInaE9OzABc2NnncC-zdwOMZ-w,5909
-metaflow/plugins/argo/argo_workflows.py,sha256=W8lrMrtJQUCvJbzNyGcT3UVc1CJyMfGw1mrIm7U5ec4,172389
+metaflow/plugins/argo/argo_workflows.py,sha256=a-ew9ZwBq9nw9W9fCTLIcOlhnuKZZBTbBcyyofkjqWk,173003
 metaflow/plugins/argo/argo_workflows_cli.py,sha256=E_PyhOtxuS2F8DwhBANsRZCMxpeZ5rfID8eksfSOPm8,37231
 metaflow/plugins/argo/argo_workflows_decorator.py,sha256=yprszMdbE3rBTcEA9VR0IEnPjTprUauZBc4SBb-Q7sA,7878
 metaflow/plugins/argo/argo_workflows_deployer.py,sha256=wSSZtThn_VPvE_Wu6NB1L0Q86LmBJh9g009v_lpvBPM,8125
@@ -282,18 +282,18 @@ metaflow/plugins/kubernetes/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMp
 metaflow/plugins/kubernetes/kube_utils.py,sha256=fYDlvqi8jYPsWijDwT6Z2qhQswyFqv7tiwtic_I80Vg,749
 metaflow/plugins/kubernetes/kubernetes.py,sha256=bKBqgZXnIDkoa4xKtKoV6InPtYQy4CujfvcbQ3Pvsbc,31305
 metaflow/plugins/kubernetes/kubernetes_cli.py,sha256=sFZ9Zrjef85vCO0MGpUF-em8Pw3dePFb3hbX3PtAH4I,13463
-metaflow/plugins/kubernetes/kubernetes_client.py,sha256=LuKX9rejRySIajnfmDoNkQ6HrOMjwIMbHAOT7wgF9i8,6362
-metaflow/plugins/kubernetes/kubernetes_decorator.py,sha256=MDW7AJozpFavrFEZAzFlxbF0vIxU0KWRpbvAtjcRIi4,25671
+metaflow/plugins/kubernetes/kubernetes_client.py,sha256=y_CswSO_OdDW8eC56lwKwC69JqIfo9tYVw0njXc_sj8,6519
+metaflow/plugins/kubernetes/kubernetes_decorator.py,sha256=xz2tEIapYWMd9rRiOe8qcYvjRIKT5piWnq-twdySpD8,26031
 metaflow/plugins/kubernetes/kubernetes_job.py,sha256=Cfkee8LbXC17jSXWoeNdomQRvF_8YSeXNg1gvxm6E_M,31806
-metaflow/plugins/kubernetes/kubernetes_jobsets.py,sha256=sCVuy-PRN60s5HWpnRvFmnljS41k70sBoPO3nOtl2ZE,40800
+metaflow/plugins/kubernetes/kubernetes_jobsets.py,sha256=wb0sK1OxW7pRbKdj6bWB4JsskXDsoKKqjyUWo4N9Y6E,41196
 metaflow/plugins/metadata/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
 metaflow/plugins/metadata/local.py,sha256=YhLJC5zjVJrvQFIyQ92ZBByiUmhCC762RUX7ITX12O8,22428
 metaflow/plugins/metadata/service.py,sha256=ihq5F7KQZlxvYwzH_-jyP2aWN_I96i2vp92j_d697s8,20204
 metaflow/plugins/pypi/__init__.py,sha256=0YFZpXvX7HCkyBFglatual7XGifdA1RwC3U4kcizyak,1037
-metaflow/plugins/pypi/bootstrap.py,sha256=LSRjpDhtpvYqCC0lDPIWvZsrHW7PXx626NDzdYjFeVM,5224
+metaflow/plugins/pypi/bootstrap.py,sha256=FI-itExqIz7DUzLnnkGwoB60rFBviygpIFThUtqk_4E,5227
 metaflow/plugins/pypi/conda_decorator.py,sha256=fPeXxvmg51oSFTnlguNlcWUIdXHA9OuMnp9ElaxQPFo,15695
 metaflow/plugins/pypi/conda_environment.py,sha256=--q-8lypKupCdGsASpqABNpNqRxtQi6UCDgq8iHDFe4,19476
-metaflow/plugins/pypi/micromamba.py,sha256=67FiIZZz0Kig9EcN7bZLObsE6Z1MFyo4Dp93fd3Grcc,12178
+metaflow/plugins/pypi/micromamba.py,sha256=HQIxsixkLjqs0ukWGTlATNu5DrbisReOr39Qd21_GZo,13737
 metaflow/plugins/pypi/pip.py,sha256=7B06mPOs5MvY33xbzPVYZlBr1iKMYaN-n8uulL9zSVg,13649
 metaflow/plugins/pypi/pypi_decorator.py,sha256=rDMbHl7r81Ye7-TuIlKAVJ_CDnfjl9jV44ZPws-UsTY,7229
 metaflow/plugins/pypi/pypi_environment.py,sha256=FYMg8kF3lXqcLfRYWD83a9zpVjcoo_TARqMGZ763rRk,230
@@ -303,12 +303,12 @@ metaflow/plugins/secrets/inline_secrets_provider.py,sha256=EChmoBGA1i7qM3jtYwPpL
 metaflow/plugins/secrets/secrets_decorator.py,sha256=s-sFzPWOjahhpr5fMj-ZEaHkDYAPTO0isYXGvaUwlG8,11273
 metaflow/runner/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 metaflow/runner/click_api.py,sha256=Qfg4BOz5K2LaXTYBsi1y4zTfNIsGGHBVF3UkorX_-o8,13878
-metaflow/runner/deployer.py,sha256=ddYv2UFE7lOFD3EPg4REzMdeLqg5WQhAXkdgMD5bUY8,13920
-metaflow/runner/metaflow_runner.py,sha256=zRlnM2j5QLBh9OboPUuaQ4_WpwFaqrCedKBOpjjtxDw,15449
+metaflow/runner/deployer.py,sha256=xNqxFIezWz3AcVqR4jeL-JnInGtefwEYMbXttxgB_I8,12276
+metaflow/runner/metaflow_runner.py,sha256=I7Ao0GHHfP55nUCE8g6CpTPGjuWgXedSc9EZX-tIE2c,15001
 metaflow/runner/nbdeploy.py,sha256=fP1s_5MeiDyT_igP82pB5EUqX9rOy2s06Hyc-OUbOvQ,4115
 metaflow/runner/nbrun.py,sha256=lmvhzMCz7iC9LSPGRijifW1wMXxa4RW_jVmpdjQi22E,7261
-metaflow/runner/subprocess_manager.py,sha256=FJh2C6wJlNnvwE00wyK6zxjf8R49PRRHeVcnPvlO4-0,19839
-metaflow/runner/utils.py,sha256=jFlQf9fVvNRd4c6VKv0KwQ8kez_re7kC2VCRgHVMSsM,2051
+metaflow/runner/subprocess_manager.py,sha256=jC_PIYIeAp_G__lf6WHZF3Lxzpp-WAQleMrRZq9j7nc,20467
+metaflow/runner/utils.py,sha256=aQ6WiNz9b-pqWWE14PdcAqti7_Zh_MIPlEA8zXJ6tXo,3807
 metaflow/sidecar/__init__.py,sha256=1mmNpmQ5puZCpRmmYlCOeieZ4108Su9XQ4_EqF1FGOU,131
 metaflow/sidecar/sidecar.py,sha256=EspKXvPPNiyRToaUZ51PS5TT_PzrBNAurn_wbFnmGr0,1334
 metaflow/sidecar/sidecar_messages.py,sha256=zPsCoYgDIcDkkvdC9MEpJTJ3y6TSGm2JWkRc4vxjbFA,1071
@@ -345,9 +345,9 @@ metaflow/tutorials/07-worldview/README.md,sha256=5vQTrFqulJ7rWN6r20dhot9lI2sVj9W
 metaflow/tutorials/07-worldview/worldview.ipynb,sha256=ztPZPI9BXxvW1QdS2Tfe7LBuVzvFvv0AToDnsDJhLdE,2237
 metaflow/tutorials/08-autopilot/README.md,sha256=GnePFp_q76jPs991lMUqfIIh5zSorIeWznyiUxzeUVE,1039
 metaflow/tutorials/08-autopilot/autopilot.ipynb,sha256=DQoJlILV7Mq9vfPBGW-QV_kNhWPjS5n6SJLqePjFYLY,3191
-metaflow-2.12.22.dist-info/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
-metaflow-2.12.22.dist-info/METADATA,sha256=XI-AUDFfekPRYdOciqofewvcuPCSZx7a5lZd58nU2Rw,5906
-metaflow-2.12.22.dist-info/WHEEL,sha256=AHX6tWk3qWuce7vKLrj7lnulVHEdWoltgauo8bgCXgU,109
-metaflow-2.12.22.dist-info/entry_points.txt,sha256=IKwTN1T3I5eJL3uo_vnkyxVffcgnRdFbKwlghZfn27k,57
-metaflow-2.12.22.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
-metaflow-2.12.22.dist-info/RECORD,,
+metaflow-2.12.23.dist-info/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
+metaflow-2.12.23.dist-info/METADATA,sha256=VWGnCbg946pb6ZQOT9i0GtBrF2dvLDawKzulk-CY2zI,5906
+metaflow-2.12.23.dist-info/WHEEL,sha256=AHX6tWk3qWuce7vKLrj7lnulVHEdWoltgauo8bgCXgU,109
+metaflow-2.12.23.dist-info/entry_points.txt,sha256=IKwTN1T3I5eJL3uo_vnkyxVffcgnRdFbKwlghZfn27k,57
+metaflow-2.12.23.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
+metaflow-2.12.23.dist-info/RECORD,,