metaflow 2.12.32__py2.py3-none-any.whl → 2.12.34__py2.py3-none-any.whl

metaflow/flowspec.py

@@ -38,6 +38,7 @@ INTERNAL_ARTIFACTS_SET = set(
         "_unbounded_foreach",
         "_control_mapper_tasks",
         "_control_task_is_mapper_zero",
+        "_parallel_ubf_iter",
     ]
 )
 

metaflow/multicore_utils.py

@@ -6,7 +6,18 @@ from tempfile import NamedTemporaryFile
 import time
 import metaflow.tracing as tracing
 
-from typing import Any, Callable, Iterable, Iterator, List, Optional
+from typing import (
+    Any,
+    Callable,
+    Iterable,
+    Iterator,
+    List,
+    Optional,
+    NoReturn,
+    Tuple,
+    TypeVar,
+    Union,
+)
 
 try:
     # Python 2
@@ -30,7 +41,13 @@ class MulticoreException(Exception):
     pass
 
 
-def _spawn(func, arg, dir):
+_A = TypeVar("_A")
+_R = TypeVar("_R")
+
+
+def _spawn(
+    func: Callable[[_A], _R], arg: _A, dir: Optional[str]
+) -> Union[Tuple[int, str], NoReturn]:
     with NamedTemporaryFile(prefix="parallel_map_", dir=dir, delete=False) as tmpfile:
         output_file = tmpfile.name
 
@@ -63,11 +80,11 @@ def _spawn(func, arg, dir):
 
 
 def parallel_imap_unordered(
-    func: Callable[[Any], Any],
-    iterable: Iterable[Any],
+    func: Callable[[_A], _R],
+    iterable: Iterable[_A],
     max_parallel: Optional[int] = None,
     dir: Optional[str] = None,
-) -> Iterator[Any]:
+) -> Iterator[_R]:
     """
     Parallelizes execution of a function using multiprocessing. The result
     order is not guaranteed.
@@ -79,9 +96,9 @@ def parallel_imap_unordered(
     iterable : Iterable[Any]
         Iterable over arguments to pass to fun
     max_parallel int, optional, default None
-        Maximum parallelism. If not specified, uses the number of CPUs
+        Maximum parallelism. If not specified, it uses the number of CPUs
     dir : str, optional, default None
-        If specified, directory where temporary files are created
+        If specified, it's the directory where temporary files are created
 
     Yields
     ------
@@ -121,14 +138,14 @@ def parallel_imap_unordered(
 
 
 def parallel_map(
-    func: Callable[[Any], Any],
-    iterable: Iterable[Any],
+    func: Callable[[_A], _R],
+    iterable: Iterable[_A],
     max_parallel: Optional[int] = None,
     dir: Optional[str] = None,
-) -> List[Any]:
+) -> List[_R]:
     """
     Parallelizes execution of a function using multiprocessing. The result
-    order is that of the arguments in `iterable`
+    order is that of the arguments in `iterable`.
 
     Parameters
     ----------
@@ -137,9 +154,9 @@ def parallel_map(
     iterable : Iterable[Any]
         Iterable over arguments to pass to fun
     max_parallel int, optional, default None
-        Maximum parallelism. If not specified, uses the number of CPUs
+        Maximum parallelism. If not specified, it uses the number of CPUs
     dir : str, optional, default None
-        If specified, directory where temporary files are created
+        If specified, it's the directory where temporary files are created
 
     Returns
     -------
@@ -155,4 +172,4 @@ def parallel_map(
     res = parallel_imap_unordered(
         wrapper, enumerate(iterable), max_parallel=max_parallel, dir=dir
     )
-    return [r for idx, r in sorted(res)]
+    return [r for _, r in sorted(res)]

metaflow/plugins/argo/argo_workflows_deployer_objects.py

@@ -10,7 +10,7 @@ from metaflow.metaflow_config import KUBERNETES_NAMESPACE
 from metaflow.plugins.argo.argo_workflows import ArgoWorkflows
 from metaflow.runner.deployer import Deployer, DeployedFlow, TriggeredRun
 
-from metaflow.runner.utils import get_lower_level_group, handle_timeout
+from metaflow.runner.utils import get_lower_level_group, handle_timeout, temporary_fifo
 
 
 def generate_fake_flow_file_contents(
@@ -341,18 +341,14 @@ class ArgoWorkflowsDeployedFlow(DeployedFlow):
         Exception
             If there is an error during the trigger process.
         """
-        with tempfile.TemporaryDirectory() as temp_dir:
-            tfp_runner_attribute = tempfile.NamedTemporaryFile(
-                dir=temp_dir, delete=False
-            )
-
+        with temporary_fifo() as (attribute_file_path, attribute_file_fd):
             # every subclass needs to have `self.deployer_kwargs`
             command = get_lower_level_group(
                 self.deployer.api,
                 self.deployer.top_level_kwargs,
                 self.deployer.TYPE,
                 self.deployer.deployer_kwargs,
-            ).trigger(deployer_attribute_file=tfp_runner_attribute.name, **kwargs)
+            ).trigger(deployer_attribute_file=attribute_file_path, **kwargs)
 
             pid = self.deployer.spm.run_command(
                 [sys.executable, *command],
@@ -363,7 +359,7 @@ class ArgoWorkflowsDeployedFlow(DeployedFlow):
 
             command_obj = self.deployer.spm.get(pid)
             content = handle_timeout(
-                tfp_runner_attribute, command_obj, self.deployer.file_read_timeout
+                attribute_file_fd, command_obj, self.deployer.file_read_timeout
             )
 
             if command_obj.process.returncode == 0:

metaflow/plugins/aws/step_functions/step_functions_deployer_objects.py

@@ -6,7 +6,7 @@ from typing import ClassVar, Optional, List
 from metaflow.plugins.aws.step_functions.step_functions import StepFunctions
 from metaflow.runner.deployer import DeployedFlow, TriggeredRun
 
-from metaflow.runner.utils import get_lower_level_group, handle_timeout
+from metaflow.runner.utils import get_lower_level_group, handle_timeout, temporary_fifo
 
 
 class StepFunctionsTriggeredRun(TriggeredRun):
@@ -196,18 +196,14 @@ class StepFunctionsDeployedFlow(DeployedFlow):
         Exception
             If there is an error during the trigger process.
         """
-        with tempfile.TemporaryDirectory() as temp_dir:
-            tfp_runner_attribute = tempfile.NamedTemporaryFile(
-                dir=temp_dir, delete=False
-            )
-
+        with temporary_fifo() as (attribute_file_path, attribute_file_fd):
             # every subclass needs to have `self.deployer_kwargs`
             command = get_lower_level_group(
                 self.deployer.api,
                 self.deployer.top_level_kwargs,
                 self.deployer.TYPE,
                 self.deployer.deployer_kwargs,
-            ).trigger(deployer_attribute_file=tfp_runner_attribute.name, **kwargs)
+            ).trigger(deployer_attribute_file=attribute_file_path, **kwargs)
 
             pid = self.deployer.spm.run_command(
                 [sys.executable, *command],
@@ -218,7 +214,7 @@ class StepFunctionsDeployedFlow(DeployedFlow):
 
             command_obj = self.deployer.spm.get(pid)
             content = handle_timeout(
-                tfp_runner_attribute, command_obj, self.deployer.file_read_timeout
+                attribute_file_fd, command_obj, self.deployer.file_read_timeout
             )
 
             if command_obj.process.returncode == 0:

metaflow/plugins/datatools/s3/s3.py

@@ -600,7 +600,9 @@ class S3(object):
         # returned are Unicode.
         key = getattr(key_value, "key", key_value)
         if self._s3root is None:
-            parsed = urlparse(to_unicode(key))
+            # NOTE: S3 allows fragments as part of object names, e.g. /dataset #1/data.txt
+            # Without allow_fragments=False the parsed.path for an object name with fragments is incomplete.
+            parsed = urlparse(to_unicode(key), allow_fragments=False)
             if parsed.scheme == "s3" and parsed.path:
                 return key
             else:
@@ -765,7 +767,9 @@ class S3(object):
         """
 
         url = self._url(key)
-        src = urlparse(url)
+        # NOTE: S3 allows fragments as part of object names, e.g. /dataset #1/data.txt
+        # Without allow_fragments=False the parsed src.path for an object name with fragments is incomplete.
+        src = urlparse(url, allow_fragments=False)
 
         def _info(s3, tmp):
             resp = s3.head_object(Bucket=src.netloc, Key=src.path.lstrip('/"'))
@@ -891,7 +895,9 @@ class S3(object):
         DOWNLOAD_MAX_CHUNK = 2 * 1024 * 1024 * 1024 - 1
 
         url, r = self._url_and_range(key)
-        src = urlparse(url)
+        # NOTE: S3 allows fragments as part of object names, e.g. /dataset #1/data.txt
+        # Without allow_fragments=False the parsed src.path for an object name with fragments is incomplete.
+        src = urlparse(url, allow_fragments=False)
 
         def _download(s3, tmp):
             if r:
@@ -1173,7 +1179,9 @@ class S3(object):
         blob.close = lambda: None
 
         url = self._url(key)
-        src = urlparse(url)
+        # NOTE: S3 allows fragments as part of object names, e.g. /dataset #1/data.txt
+        # Without allow_fragments=False the parsed src.path for an object name with fragments is incomplete.
+        src = urlparse(url, allow_fragments=False)
         extra_args = None
         if content_type or metadata or self._encryption:
             extra_args = {}

metaflow/plugins/events_decorator.py

@@ -170,7 +170,7 @@ class TriggerDecorator(FlowDecorator):
                 # process every event in events
                 for event in self.attributes["events"]:
                     processed_event = self.process_event_name(event)
-                    self.triggers.append("processed event", processed_event)
+                    self.triggers.append(processed_event)
             elif callable(self.attributes["events"]) and not isinstance(
                 self.attributes["events"], DeployTimeField
             ):

metaflow/runner/deployer_impl.py

@@ -2,12 +2,11 @@ import importlib
 import json
 import os
 import sys
-import tempfile
 
 from typing import Any, ClassVar, Dict, Optional, TYPE_CHECKING, Type
 
 from .subprocess_manager import SubprocessManager
-from .utils import get_lower_level_group, handle_timeout
+from .utils import get_lower_level_group, handle_timeout, temporary_fifo
 
 if TYPE_CHECKING:
     import metaflow.runner.deployer
@@ -121,14 +120,11 @@ class DeployerImpl(object):
     def _create(
         self, create_class: Type["metaflow.runner.deployer.DeployedFlow"], **kwargs
     ) -> "metaflow.runner.deployer.DeployedFlow":
-        with tempfile.TemporaryDirectory() as temp_dir:
-            tfp_runner_attribute = tempfile.NamedTemporaryFile(
-                dir=temp_dir, delete=False
-            )
+        with temporary_fifo() as (attribute_file_path, attribute_file_fd):
             # every subclass needs to have `self.deployer_kwargs`
             command = get_lower_level_group(
                 self.api, self.top_level_kwargs, self.TYPE, self.deployer_kwargs
-            ).create(deployer_attribute_file=tfp_runner_attribute.name, **kwargs)
+            ).create(deployer_attribute_file=attribute_file_path, **kwargs)
 
             pid = self.spm.run_command(
                 [sys.executable, *command],
@@ -139,7 +135,7 @@ class DeployerImpl(object):
 
             command_obj = self.spm.get(pid)
             content = handle_timeout(
-                tfp_runner_attribute, command_obj, self.file_read_timeout
+                attribute_file_fd, command_obj, self.file_read_timeout
             )
             content = json.loads(content)
             self.name = content.get("name")

metaflow/runner/metaflow_runner.py

@@ -2,13 +2,16 @@ import importlib
 import os
 import sys
 import json
-import tempfile
 
 from typing import Dict, Iterator, Optional, Tuple
 
 from metaflow import Run
 
-from .utils import handle_timeout
+from .utils import (
+    temporary_fifo,
+    handle_timeout,
+    async_handle_timeout,
+)
 from .subprocess_manager import CommandManager, SubprocessManager
 
 
@@ -267,9 +270,22 @@ class Runner(object):
     async def __aenter__(self) -> "Runner":
         return self
 
-    def __get_executing_run(self, tfp_runner_attribute, command_obj):
-        content = handle_timeout(
-            tfp_runner_attribute, command_obj, self.file_read_timeout
+    def __get_executing_run(self, attribute_file_fd, command_obj):
+        content = handle_timeout(attribute_file_fd, command_obj, self.file_read_timeout)
+        content = json.loads(content)
+        pathspec = "%s/%s" % (content.get("flow_name"), content.get("run_id"))
+
+        # Set the correct metadata from the runner_attribute file corresponding to this run.
+        metadata_for_flow = content.get("metadata")
+
+        run_object = Run(
+            pathspec, _namespace_check=False, _current_metadata=metadata_for_flow
+        )
+        return ExecutingRun(self, command_obj, run_object)
+
+    async def __async_get_executing_run(self, attribute_file_fd, command_obj):
+        content = await async_handle_timeout(
+            attribute_file_fd, command_obj, self.file_read_timeout
         )
         content = json.loads(content)
         pathspec = "%s/%s" % (content.get("flow_name"), content.get("run_id"))
@@ -298,12 +314,9 @@ class Runner(object):
         ExecutingRun
             ExecutingRun containing the results of the run.
         """
-        with tempfile.TemporaryDirectory() as temp_dir:
-            tfp_runner_attribute = tempfile.NamedTemporaryFile(
-                dir=temp_dir, delete=False
-            )
+        with temporary_fifo() as (attribute_file_path, attribute_file_fd):
             command = self.api(**self.top_level_kwargs).run(
-                runner_attribute_file=tfp_runner_attribute.name, **kwargs
+                runner_attribute_file=attribute_file_path, **kwargs
             )
 
             pid = self.spm.run_command(
@@ -314,7 +327,7 @@ class Runner(object):
             )
             command_obj = self.spm.get(pid)
 
-            return self.__get_executing_run(tfp_runner_attribute, command_obj)
+            return self.__get_executing_run(attribute_file_fd, command_obj)
 
     def resume(self, **kwargs):
         """
@@ -332,12 +345,9 @@ class Runner(object):
         ExecutingRun
             ExecutingRun containing the results of the resumed run.
         """
-        with tempfile.TemporaryDirectory() as temp_dir:
-            tfp_runner_attribute = tempfile.NamedTemporaryFile(
-                dir=temp_dir, delete=False
-            )
+        with temporary_fifo() as (attribute_file_path, attribute_file_fd):
             command = self.api(**self.top_level_kwargs).resume(
-                runner_attribute_file=tfp_runner_attribute.name, **kwargs
+                runner_attribute_file=attribute_file_path, **kwargs
             )
 
             pid = self.spm.run_command(
@@ -348,7 +358,7 @@ class Runner(object):
             )
             command_obj = self.spm.get(pid)
 
-            return self.__get_executing_run(tfp_runner_attribute, command_obj)
+            return self.__get_executing_run(attribute_file_fd, command_obj)
 
     async def async_run(self, **kwargs) -> ExecutingRun:
         """
@@ -368,12 +378,9 @@ class Runner(object):
         ExecutingRun
             ExecutingRun representing the run that was started.
         """
-        with tempfile.TemporaryDirectory() as temp_dir:
-            tfp_runner_attribute = tempfile.NamedTemporaryFile(
-                dir=temp_dir, delete=False
-            )
+        with temporary_fifo() as (attribute_file_path, attribute_file_fd):
             command = self.api(**self.top_level_kwargs).run(
-                runner_attribute_file=tfp_runner_attribute.name, **kwargs
+                runner_attribute_file=attribute_file_path, **kwargs
             )
 
             pid = await self.spm.async_run_command(
@@ -383,7 +390,7 @@ class Runner(object):
             )
             command_obj = self.spm.get(pid)
 
-            return self.__get_executing_run(tfp_runner_attribute, command_obj)
+            return await self.__async_get_executing_run(attribute_file_fd, command_obj)
 
     async def async_resume(self, **kwargs):
         """
@@ -403,12 +410,9 @@ class Runner(object):
         ExecutingRun
             ExecutingRun representing the resumed run that was started.
         """
-        with tempfile.TemporaryDirectory() as temp_dir:
-            tfp_runner_attribute = tempfile.NamedTemporaryFile(
-                dir=temp_dir, delete=False
-            )
+        with temporary_fifo() as (attribute_file_path, attribute_file_fd):
             command = self.api(**self.top_level_kwargs).resume(
-                runner_attribute_file=tfp_runner_attribute.name, **kwargs
+                runner_attribute_file=attribute_file_path, **kwargs
             )
 
             pid = await self.spm.async_run_command(
@@ -418,7 +422,7 @@ class Runner(object):
             )
             command_obj = self.spm.get(pid)
 
-            return self.__get_executing_run(tfp_runner_attribute, command_obj)
+            return await self.__async_get_executing_run(attribute_file_fd, command_obj)
 
     def __exit__(self, exc_type, exc_value, traceback):
         self.spm.cleanup()

metaflow/runner/subprocess_manager.py

@@ -9,26 +9,61 @@ import tempfile
 import threading
 from typing import Callable, Dict, Iterator, List, Optional, Tuple
 
+from .utils import check_process_exited
 
-def kill_process_and_descendants(pid, termination_timeout):
+
+def kill_processes_and_descendants(pids: List[str], termination_timeout: float):
     # TODO: there's a race condition that new descendants might
     # spawn b/w the invocations of 'pkill' and 'kill'.
     # Needs to be fixed in future.
     try:
-        subprocess.check_call(["pkill", "-TERM", "-P", str(pid)])
-        subprocess.check_call(["kill", "-TERM", str(pid)])
+        subprocess.check_call(["pkill", "-TERM", "-P", *pids])
+        subprocess.check_call(["kill", "-TERM", *pids])
     except subprocess.CalledProcessError:
         pass
 
     time.sleep(termination_timeout)
 
     try:
-        subprocess.check_call(["pkill", "-KILL", "-P", str(pid)])
-        subprocess.check_call(["kill", "-KILL", str(pid)])
+        subprocess.check_call(["pkill", "-KILL", "-P", *pids])
+        subprocess.check_call(["kill", "-KILL", *pids])
     except subprocess.CalledProcessError:
         pass
 
 
+async def async_kill_processes_and_descendants(
+    pids: List[str], termination_timeout: float
+):
+    # TODO: there's a race condition that new descendants might
+    # spawn b/w the invocations of 'pkill' and 'kill'.
+    # Needs to be fixed in future.
+    try:
+        sub_term = await asyncio.create_subprocess_exec("pkill", "-TERM", "-P", *pids)
+        await sub_term.wait()
+    except Exception:
+        pass
+
+    try:
+        main_term = await asyncio.create_subprocess_exec("kill", "-TERM", *pids)
+        await main_term.wait()
+    except Exception:
+        pass
+
+    await asyncio.sleep(termination_timeout)
+
+    try:
+        sub_kill = await asyncio.create_subprocess_exec("pkill", "-KILL", "-P", *pids)
+        await sub_kill.wait()
+    except Exception:
+        pass
+
+    try:
+        main_kill = await asyncio.create_subprocess_exec("kill", "-KILL", *pids)
+        await main_kill.wait()
+    except Exception:
+        pass
+
+
 class LogReadTimeoutError(Exception):
     """Exception raised when reading logs times out."""
 
@@ -46,14 +81,28 @@ class SubprocessManager(object):
             loop = asyncio.get_running_loop()
             loop.add_signal_handler(
                 signal.SIGINT,
-                lambda: self._handle_sigint(signum=signal.SIGINT, frame=None),
+                lambda: asyncio.create_task(self._async_handle_sigint()),
             )
         except RuntimeError:
             signal.signal(signal.SIGINT, self._handle_sigint)
 
+    async def _async_handle_sigint(self):
+        pids = [
+            str(command.process.pid)
+            for command in self.commands.values()
+            if command.process and not check_process_exited(command)
+        ]
+        if pids:
+            await async_kill_processes_and_descendants(pids, termination_timeout=2)
+
     def _handle_sigint(self, signum, frame):
-        for each_command in self.commands.values():
-            each_command.kill(termination_timeout=2)
+        pids = [
+            str(command.process.pid)
+            for command in self.commands.values()
+            if command.process and not check_process_exited(command)
+        ]
+        if pids:
+            kill_processes_and_descendants(pids, termination_timeout=2)
 
     async def __aenter__(self) -> "SubprocessManager":
         return self
@@ -472,7 +521,7 @@ class CommandManager(object):
         """
 
         if self.process is not None:
-            kill_process_and_descendants(self.process.pid, termination_timeout)
+            kill_processes_and_descendants([str(self.process.pid)], termination_timeout)
         else:
             print("No process to kill.")
 

metaflow/runner/utils.py

@@ -2,9 +2,11 @@ import os
 import ast
 import time
 import asyncio
-
+import tempfile
+import select
+from contextlib import contextmanager
 from subprocess import CalledProcessError
-from typing import Any, Dict, TYPE_CHECKING
+from typing import Any, Dict, TYPE_CHECKING, ContextManager, Tuple
 
 if TYPE_CHECKING:
     import tempfile
@@ -39,45 +41,194 @@ def format_flowfile(cell):
     return "\n".join(lines)
 
 
-def check_process_status(
+def check_process_exited(
     command_obj: "metaflow.runner.subprocess_manager.CommandManager",
-):
+) -> bool:
     if isinstance(command_obj.process, asyncio.subprocess.Process):
         return command_obj.process.returncode is not None
     else:
         return command_obj.process.poll() is not None
 
 
-def read_from_file_when_ready(
-    file_path: str,
+@contextmanager
+def temporary_fifo() -> ContextManager[Tuple[str, int]]:
+    """
+    Create and open the read side of a temporary FIFO in a non-blocking mode.
+
+    Returns
+    -------
+    str
+        Path to the temporary FIFO.
+    int
+        File descriptor of the temporary FIFO.
+    """
+    with tempfile.TemporaryDirectory() as temp_dir:
+        path = os.path.join(temp_dir, "fifo")
+        os.mkfifo(path)
+        # Blocks until the write side is opened unless in non-blocking mode
+        fd = os.open(path, os.O_RDONLY | os.O_NONBLOCK)
+        try:
+            yield path, fd
+        finally:
+            os.close(fd)
+
+
+def read_from_fifo_when_ready(
+    fifo_fd: int,
+    command_obj: "metaflow.runner.subprocess_manager.CommandManager",
+    encoding: str = "utf-8",
+    timeout: int = 3600,
+) -> str:
+    """
+    Read the content from the FIFO file descriptor when it is ready.
+
+    Parameters
+    ----------
+    fifo_fd : int
+        File descriptor of the FIFO.
+    command_obj : CommandManager
+        Command manager object that handles the write side of the FIFO.
+    encoding : str, optional
+        Encoding to use while reading the file, by default "utf-8".
+    timeout : int, optional
+        Timeout for reading the file in milliseconds, by default 3600.
+
+    Returns
+    -------
+    str
+        Content read from the FIFO.
+
+    Raises
+    ------
+    TimeoutError
+        If no event occurs on the FIFO within the timeout.
+    CalledProcessError
+        If the process managed by `command_obj` has exited without writing any
+        content to the FIFO.
+    """
+    content = bytearray()
+
+    poll = select.poll()
+    poll.register(fifo_fd, select.POLLIN)
+
+    while True:
+        poll_begin = time.time()
+        poll.poll(timeout)
+        timeout -= 1000 * (time.time() - poll_begin)
+
+        if timeout <= 0:
+            raise TimeoutError("Timeout while waiting for the file content")
+
+        try:
+            data = os.read(fifo_fd, 128)
+            while data:
+                content += data
+                data = os.read(fifo_fd, 128)
+
+            # Read from a non-blocking closed FIFO returns an empty byte array
+            break
+
+        except BlockingIOError:
+            # FIFO is open but no data is available yet
+            continue
+
+    if not content and check_process_exited(command_obj):
+        raise CalledProcessError(command_obj.process.returncode, command_obj.command)
+
+    return content.decode(encoding)
+
+
+async def async_read_from_fifo_when_ready(
+    fifo_fd: int,
+    command_obj: "metaflow.runner.subprocess_manager.CommandManager",
+    encoding: str = "utf-8",
+    timeout: int = 3600,
+) -> str:
+    """
+    Read the content from the FIFO file descriptor when it is ready.
+
+    Parameters
+    ----------
+    fifo_fd : int
+        File descriptor of the FIFO.
+    command_obj : CommandManager
+        Command manager object that handles the write side of the FIFO.
+    encoding : str, optional
+        Encoding to use while reading the file, by default "utf-8".
+    timeout : int, optional
+        Timeout for reading the file in milliseconds, by default 3600.
+
+    Returns
+    -------
+    str
+        Content read from the FIFO.
+
+    Raises
+    ------
+    TimeoutError
+        If no event occurs on the FIFO within the timeout.
+    CalledProcessError
+        If the process managed by `command_obj` has exited without writing any
+        content to the FIFO.
+    """
+    return await asyncio.to_thread(
+        read_from_fifo_when_ready, fifo_fd, command_obj, encoding, timeout
+    )
+
+
+def make_process_error_message(
     command_obj: "metaflow.runner.subprocess_manager.CommandManager",
-    timeout: float = 5,
 ):
-    start_time = time.time()
-    with open(file_path, "r", encoding="utf-8") as file_pointer:
-        content = file_pointer.read()
-        while not content:
-            if check_process_status(command_obj):
-                # Check to make sure the file hasn't been read yet to avoid a race
-                # where the file is written between the end of this while loop and the
-                # poll call above.
-                content = file_pointer.read()
-                if content:
-                    break
-                raise CalledProcessError(
-                    command_obj.process.returncode, command_obj.command
-                )
-            if time.time() - start_time > timeout:
-                raise TimeoutError(
-                    "Timeout while waiting for file content from '%s'" % file_path
-                )
-            time.sleep(0.1)
-            content = file_pointer.read()
-        return content
+    stdout_log = open(command_obj.log_files["stdout"], encoding="utf-8").read()
+    stderr_log = open(command_obj.log_files["stderr"], encoding="utf-8").read()
+    command = " ".join(command_obj.command)
+    error_message = "Error executing: '%s':\n" % command
+    if stdout_log.strip():
+        error_message += "\nStdout:\n%s\n" % stdout_log
+    if stderr_log.strip():
+        error_message += "\nStderr:\n%s\n" % stderr_log
+    return error_message
 
 
 def handle_timeout(
-    tfp_runner_attribute: "tempfile._TemporaryFileWrapper[str]",
+    attribute_file_fd: int,
+    command_obj: "metaflow.runner.subprocess_manager.CommandManager",
+    file_read_timeout: int,
+):
+    """
+    Handle the timeout for a running subprocess command that reads a file
+    and raises an error with appropriate logs if a TimeoutError occurs.
+
+    Parameters
+    ----------
+    attribute_file_fd : int
+        File descriptor belonging to the FIFO containing the attribute data.
+    command_obj : CommandManager
+        Command manager object that encapsulates the running command details.
+    file_read_timeout : int
+        Timeout for reading the file.
+
+    Returns
+    -------
+    str
+        Content read from the temporary file.
+
+    Raises
+    ------
+    RuntimeError
+        If a TimeoutError occurs, it raises a RuntimeError with the command's
+        stdout and stderr logs.
+    """
+    try:
+        return read_from_fifo_when_ready(
+            attribute_file_fd, command_obj=command_obj, timeout=file_read_timeout
+        )
+    except (CalledProcessError, TimeoutError) as e:
+        raise RuntimeError(make_process_error_message(command_obj)) from e
+
+
+async def async_handle_timeout(
+    attribute_file_fd: "int",
     command_obj: "metaflow.runner.subprocess_manager.CommandManager",
     file_read_timeout: int,
 ):
@@ -87,8 +238,8 @@ def handle_timeout(
 
     Parameters
     ----------
-    tfp_runner_attribute : NamedTemporaryFile
-        Temporary file that stores runner attribute data.
+    attribute_file_fd : int
+        File descriptor belonging to the FIFO containing the attribute data.
     command_obj : CommandManager
         Command manager object that encapsulates the running command details.
     file_read_timeout : int
@@ -106,20 +257,11 @@ def handle_timeout(
         stdout and stderr logs.
     """
     try:
-        content = read_from_file_when_ready(
-            tfp_runner_attribute.name, command_obj, timeout=file_read_timeout
+        return await async_read_from_fifo_when_ready(
+            attribute_file_fd, command_obj=command_obj, timeout=file_read_timeout
         )
-        return content
     except (CalledProcessError, TimeoutError) as e:
-        stdout_log = open(command_obj.log_files["stdout"], encoding="utf-8").read()
-        stderr_log = open(command_obj.log_files["stderr"], encoding="utf-8").read()
-        command = " ".join(command_obj.command)
-        error_message = "Error executing: '%s':\n" % command
-        if stdout_log.strip():
-            error_message += "\nStdout:\n%s\n" % stdout_log
-        if stderr_log.strip():
-            error_message += "\nStderr:\n%s\n" % stderr_log
-        raise RuntimeError(error_message) from e
+        raise RuntimeError(make_process_error_message(command_obj)) from e
 
 
 def get_lower_level_group(

metaflow/util.py

@@ -436,12 +436,17 @@ def to_pod(value):
         Value to convert to POD format. The value can be a string, number, list,
         dictionary, or a nested structure of these types.
     """
+    # Prevent circular imports
+    from metaflow.parameters import DeployTimeField
+
     if isinstance(value, (str, int, float)):
         return value
     if isinstance(value, dict):
         return {to_pod(k): to_pod(v) for k, v in value.items()}
     if isinstance(value, (list, set, tuple)):
         return [to_pod(v) for v in value]
+    if isinstance(value, DeployTimeField):
+        return value.print_representation
     return str(value)
 
 

metaflow/version.py

@@ -1 +1 @@
-metaflow_version = "2.12.32"
+metaflow_version = "2.12.34"

{metaflow-2.12.32.dist-info → metaflow-2.12.34.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: metaflow
-Version: 2.12.32
+Version: 2.12.34
 Summary: Metaflow: More Data Science, Less Engineering
 Author: Metaflow Developers
 Author-email: help@metaflow.org
@@ -26,7 +26,7 @@ License-File: LICENSE
 Requires-Dist: requests
 Requires-Dist: boto3
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.12.32; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.12.34; extra == "stubs"
 
 ![Metaflow_Logo_Horizontal_FullColor_Ribbon_Dark_RGB](https://user-images.githubusercontent.com/763451/89453116-96a57e00-d713-11ea-9fa6-82b29d4d6eff.png)
 

{metaflow-2.12.32.dist-info → metaflow-2.12.34.dist-info}/RECORD

@@ -10,7 +10,7 @@ metaflow/decorators.py,sha256=hbJwRlZuLbl6t7fOsTiH7Sux4Lx6zgEEpldIXoM5TQc,21540
 metaflow/event_logger.py,sha256=joTVRqZPL87nvah4ZOwtqWX8NeraM_CXKXXGVpKGD8o,780
 metaflow/events.py,sha256=ahjzkSbSnRCK9RZ-9vTfUviz_6gMvSO9DGkJ86X80-k,5300
 metaflow/exception.py,sha256=KC1LHJQzzYkWib0DeQ4l_A2r8VaudywsSqIQuq1RDZU,4954
-metaflow/flowspec.py,sha256=v8jH21Teq_RPcfaIB-yyya8xv09wl4IHU-Ya0K7UPYI,27268
+metaflow/flowspec.py,sha256=VEeaItArFwCJqRNq0e2i9gZrBSZAaq0XwmN_8dTcY1M,27298
 metaflow/graph.py,sha256=HFJ7V_bPSht_NHIm8BejrSqOX2fyBQpVOczRCliRw08,11975
 metaflow/includefile.py,sha256=rDJnxF0U7vD3cz9hhPkKlW_KS3ToaXnlOjhjNZ__Rx4,19628
 metaflow/info_file.py,sha256=wtf2_F0M6dgiUu74AFImM8lfy5RrUw5Yj7Rgs2swKRY,686
@@ -23,7 +23,7 @@ metaflow/metaflow_environment.py,sha256=rojFyGdyY56sN1HaEb1-0XX53Q3XPNnl0SaH-8xX
 metaflow/metaflow_profile.py,sha256=jKPEW-hmAQO-htSxb9hXaeloLacAh41A35rMZH6G8pA,418
 metaflow/metaflow_version.py,sha256=duhIzfKZtcxMVMs2uiBqBvUarSHJqyWDwMhaBOQd_g0,7491
 metaflow/monitor.py,sha256=T0NMaBPvXynlJAO_avKtk8OIIRMyEuMAyF8bIp79aZU,5323
-metaflow/multicore_utils.py,sha256=vdTNgczVLODifscUbbveJbuSDOl3Y9pAxhr7sqYiNf4,4760
+metaflow/multicore_utils.py,sha256=yEo5T6Gemn4_vl8b6IOz7fsTUYtEyqa3AaKZgJY96Wc,4974
 metaflow/package.py,sha256=QutDP6WzjwGk1UCKXqBfXa9F10Q--FlRr0J7fwlple0,7399
 metaflow/parameters.py,sha256=pzjG0ssuVHPyYQqWE86dS3yYChEqbT90rOUcRD4wNog,16079
 metaflow/procpoll.py,sha256=U2tE4iK_Mwj2WDyVTx_Uglh6xZ-jixQOo4wrM9OOhxg,2859
@@ -34,9 +34,9 @@ metaflow/tagging_util.py,sha256=ctyf0Q1gBi0RyZX6J0e9DQGNkNHblV_CITfy66axXB4,2346
 metaflow/task.py,sha256=xVVLWy8NH16OlLu2VoOb1OfiFzcOVVCdQldlmb1Zb_w,29691
 metaflow/tuple_util.py,sha256=_G5YIEhuugwJ_f6rrZoelMFak3DqAR2tt_5CapS1XTY,830
 metaflow/unbounded_foreach.py,sha256=p184WMbrMJ3xKYHwewj27ZhRUsSj_kw1jlye5gA9xJk,387
-metaflow/util.py,sha256=olAvJK3y1it_k99MhLulTaAJo7OFVt5rnrD-ulIFLCU,13616
+metaflow/util.py,sha256=w7oylILPaNAjtM8MR8dfUazTVBArV_CKPpqGs4HnowM,13785
 metaflow/vendor.py,sha256=FchtA9tH22JM-eEtJ2c9FpUdMn8sSb1VHuQS56EcdZk,5139
-metaflow/version.py,sha256=UlmCI1Z4dlLrwJowsJ1ORnkNpYz3GOen97kVpkivC5Q,29
+metaflow/version.py,sha256=1QFMy_kk8f34f8BuD9G4-ZA0vcxYQo1O9WhpDuY8MXQ,29
 metaflow/_vendor/__init__.py,sha256=y_CiwUD3l4eAKvTVDZeqgVujMy31cAM1qjAB-HfI-9s,353
 metaflow/_vendor/typing_extensions.py,sha256=0nUs5p1A_UrZigrAVBoOEM6TxU37zzPDUtiij1ZwpNc,110417
 metaflow/_vendor/zipp.py,sha256=ajztOH-9I7KA_4wqDYygtHa6xUBVZgFpmZ8FE74HHHI,8425
@@ -148,7 +148,7 @@ metaflow/plugins/catch_decorator.py,sha256=UOM2taN_OL2RPpuJhwEOA9ZALm0-hHD0XS2Hn
 metaflow/plugins/debug_logger.py,sha256=mcF5HYzJ0NQmqCMjyVUk3iAP-heroHRIiVWQC6Ha2-I,879
 metaflow/plugins/debug_monitor.py,sha256=Md5X_sDOSssN9pt2D8YcaIjTK5JaQD55UAYTcF6xYF0,1099
 metaflow/plugins/environment_decorator.py,sha256=6m9j2B77d-Ja_l_9CTJ__0O6aB2a8Qt_lAZu6UjAcUA,587
-metaflow/plugins/events_decorator.py,sha256=oULWTJw426MA9bdfMS-7q72LOQ1rI4oc_fp8JJd0P9o,26475
+metaflow/plugins/events_decorator.py,sha256=8YSapp_sT3UzNrb6cYBJ19-wmX_CKow6OOJN8kNVnpg,26456
 metaflow/plugins/logs_cli.py,sha256=77W5UNagU2mOKSMMvrQxQmBLRzvmjK-c8dWxd-Ygbqs,11410
 metaflow/plugins/package_cli.py,sha256=-J6D4cupHfWSZ4GEFo2yy9Je9oL3owRWm5pEJwaiqd4,1649
 metaflow/plugins/parallel_decorator.py,sha256=GIjZZVTqkvtnMuGE8RNtObX6CAJavZTxttqRujGmnGs,8973
@@ -179,7 +179,7 @@ metaflow/plugins/argo/argo_workflows.py,sha256=d-d_xvjUeb5t11kaHZ98yIrGAqYg__Fhx
 metaflow/plugins/argo/argo_workflows_cli.py,sha256=NdLwzfBcTsR72qLycZBesR4Pwv48o3Z_v6OfYrZuVEY,36721
 metaflow/plugins/argo/argo_workflows_decorator.py,sha256=QdM1rK9gM-lDhyZldK8WqvFqJDvfJ7i3JPR5Uzaq2as,7887
 metaflow/plugins/argo/argo_workflows_deployer.py,sha256=6kHxEnYXJwzNCM9swI8-0AckxtPWqwhZLerYkX8fxUM,4444
-metaflow/plugins/argo/argo_workflows_deployer_objects.py,sha256=jOlvn7dp2Qa_Y1lsOY_sta7i78qQVirunzgIH0O4w2E,12414
+metaflow/plugins/argo/argo_workflows_deployer_objects.py,sha256=GJ1Jsm5KHYaBbJjKRz82Cwhi_PN1XnMiSmL1C0LNLYQ,12318
 metaflow/plugins/argo/capture_error.py,sha256=Ys9dscGrTpW-ZCirLBU0gD9qBM0BjxyxGlUMKcwewQc,1852
 metaflow/plugins/argo/generate_input_paths.py,sha256=loYsI6RFX9LlFsHb7Fe-mzlTTtRdySoOu7sYDy-uXK0,881
 metaflow/plugins/argo/jobset_input_paths.py,sha256=_JhZWngA6p9Q_O2fx3pdzKI0WE-HPRHz_zFvY2pHPTQ,525
@@ -204,7 +204,7 @@ metaflow/plugins/aws/step_functions/step_functions_cli.py,sha256=Ar9ed2nlZ_M0J3H
 metaflow/plugins/aws/step_functions/step_functions_client.py,sha256=DKpNwAIWElvWjFANs5Ku3rgzjxFoqAD6k-EF8Xhkg3Q,4754
 metaflow/plugins/aws/step_functions/step_functions_decorator.py,sha256=LoZC5BuQLqyFtfE-sGla26l2xXlCKN9aSvIlzPKV134,3800
 metaflow/plugins/aws/step_functions/step_functions_deployer.py,sha256=JKYtDhKivtXUWPklprZFzkqezh14loGDmk8mNk6QtpI,3714
-metaflow/plugins/aws/step_functions/step_functions_deployer_objects.py,sha256=6frhj4JkJxH04Wocg-053ppD3xmDjd_3wIwSVfjeMrw,7315
+metaflow/plugins/aws/step_functions/step_functions_deployer_objects.py,sha256=EbBd0m1GL5E7hIP2sBFgyRXo2XTf6qMZdxLA8VjXGX8,7219
 metaflow/plugins/azure/__init__.py,sha256=GuuhTVC-zSdyAf79a1wiERMq0Zts7fwVT7t9fAf234A,100
 metaflow/plugins/azure/azure_credential.py,sha256=JmdGEbVzgxy8ucqnQDdTTI_atyMX9WSZUw3qYOo7RhE,2174
 metaflow/plugins/azure/azure_exceptions.py,sha256=NnbwpUC23bc61HZjJmeXztY0tBNn_Y_VpIpDDuYWIZ0,433
@@ -247,7 +247,7 @@ metaflow/plugins/datastores/s3_storage.py,sha256=CZdNqaKtxDXQbEg2YHyphph3hWcLIE5
 metaflow/plugins/datatools/__init__.py,sha256=ge4L16OBQLy2J_MMvoHg3lMfdm-MluQgRWoyZ5GCRnk,1267
 metaflow/plugins/datatools/local.py,sha256=FJvMOBcjdyhSPHmdLocBSiIT0rmKkKBmsaclxH75x08,4233
 metaflow/plugins/datatools/s3/__init__.py,sha256=14tr9fPjN3ULW5IOfKHeG7Uhjmgm7LMtQHfz1SFv-h8,248
-metaflow/plugins/datatools/s3/s3.py,sha256=-gkOKx4Ed-0u7eL2rOdbik98PJPltxDgfVPml0J9Ym4,66157
+metaflow/plugins/datatools/s3/s3.py,sha256=dSUEf3v_BVyvYXlehWy04xcBtNDhKKR5Gnn7oXwagaw,67037
 metaflow/plugins/datatools/s3/s3op.py,sha256=h7l-OLQz4RLpyzsB-k903BYCDF9CM2_O9jLl-FQyMis,43454
 metaflow/plugins/datatools/s3/s3tail.py,sha256=boQjQGQMI-bvTqcMP2y7uSlSYLcvWOy7J3ZUaF78NAA,2597
 metaflow/plugins/datatools/s3/s3util.py,sha256=FgRgaVmEq7-i2dV7q8XK5w5PfFt-xJjZa8WrK8IJfdI,3769
@@ -306,12 +306,12 @@ metaflow/plugins/secrets/secrets_decorator.py,sha256=s-sFzPWOjahhpr5fMj-ZEaHkDYA
 metaflow/runner/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 metaflow/runner/click_api.py,sha256=r7StpxnKkTwzs7_NG-estHVcztE-ey7wM6QSdYu6UCY,13943
 metaflow/runner/deployer.py,sha256=goCCDq9mRJIrH5-9HU6E6J55pYoju--CtM1zADTzHvY,8967
-metaflow/runner/deployer_impl.py,sha256=gJK30oujnRKw23jAXG5h3JLLkEUBB3GdgKmJHZxq8Mk,5736
-metaflow/runner/metaflow_runner.py,sha256=qq6EF9lFcX5dn93P5S8v-SDddhpFWfjrbGMAEHncNYc,14592
+metaflow/runner/deployer_impl.py,sha256=QjMoTM1IviRK-dZ7WB3vGExpaScOCESlHxSR9ZoAkgA,5625
+metaflow/runner/metaflow_runner.py,sha256=PrAEJhWqjbGuu4IsnxCSLLf8o4AqqRpeTM7rCMbOAvg,14827
 metaflow/runner/nbdeploy.py,sha256=UYo3Yfw1wlBFnUHjs_RngnWcWj9CDdMYaMqt9wZx0rk,4035
 metaflow/runner/nbrun.py,sha256=otWTsCkrVSdxlaxXNrRc9oyjdgQR8mHHhL8NAO0B5iE,7286
-metaflow/runner/subprocess_manager.py,sha256=jC_PIYIeAp_G__lf6WHZF3Lxzpp-WAQleMrRZq9j7nc,20467
-metaflow/runner/utils.py,sha256=WnEoJ4KJTOWzhAevH5atUcOBLMJE5wWq27E_4nWTqzQ,5035
+metaflow/runner/subprocess_manager.py,sha256=lUD59T663Ar4ZcsUOk8VehaV1pRIg-QoWT_s_dFOVGI,21953
+metaflow/runner/utils.py,sha256=JTWBFxQ2dru-dL2tI-8dIIVvQn_S3asGU9Ma6eeN7rc,8818
 metaflow/sidecar/__init__.py,sha256=1mmNpmQ5puZCpRmmYlCOeieZ4108Su9XQ4_EqF1FGOU,131
 metaflow/sidecar/sidecar.py,sha256=EspKXvPPNiyRToaUZ51PS5TT_PzrBNAurn_wbFnmGr0,1334
 metaflow/sidecar/sidecar_messages.py,sha256=zPsCoYgDIcDkkvdC9MEpJTJ3y6TSGm2JWkRc4vxjbFA,1071
@@ -348,9 +348,9 @@ metaflow/tutorials/07-worldview/README.md,sha256=5vQTrFqulJ7rWN6r20dhot9lI2sVj9W
 metaflow/tutorials/07-worldview/worldview.ipynb,sha256=ztPZPI9BXxvW1QdS2Tfe7LBuVzvFvv0AToDnsDJhLdE,2237
 metaflow/tutorials/08-autopilot/README.md,sha256=GnePFp_q76jPs991lMUqfIIh5zSorIeWznyiUxzeUVE,1039
 metaflow/tutorials/08-autopilot/autopilot.ipynb,sha256=DQoJlILV7Mq9vfPBGW-QV_kNhWPjS5n6SJLqePjFYLY,3191
-metaflow-2.12.32.dist-info/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
-metaflow-2.12.32.dist-info/METADATA,sha256=_gQS-g4TESv6MPnBVuDHjf9qwcWyfTcDbEY-XcUaFAM,5907
-metaflow-2.12.32.dist-info/WHEEL,sha256=pxeNX5JdtCe58PUSYP9upmc7jdRPgvT0Gm9kb1SHlVw,109
-metaflow-2.12.32.dist-info/entry_points.txt,sha256=IKwTN1T3I5eJL3uo_vnkyxVffcgnRdFbKwlghZfn27k,57
-metaflow-2.12.32.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
-metaflow-2.12.32.dist-info/RECORD,,
+metaflow-2.12.34.dist-info/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
+metaflow-2.12.34.dist-info/METADATA,sha256=nyJPMX2oVYbAC9lY9KuNPMngDomft-j5DJoeHGAnJuY,5907
+metaflow-2.12.34.dist-info/WHEEL,sha256=pxeNX5JdtCe58PUSYP9upmc7jdRPgvT0Gm9kb1SHlVw,109
+metaflow-2.12.34.dist-info/entry_points.txt,sha256=IKwTN1T3I5eJL3uo_vnkyxVffcgnRdFbKwlghZfn27k,57
+metaflow-2.12.34.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
+metaflow-2.12.34.dist-info/RECORD,,