parsl 2023.12.4__py3-none-any.whl → 2023.12.18__py3-none-any.whl

parsl/dataflow/dflow.py

@@ -854,10 +854,13 @@ class DataFlowKernel:
                 try:
                     new_args.extend([dep.result()])
                 except Exception as e:
-                    if hasattr(dep, 'task_record'):
-                        tid = dep.task_record['id']
+                    # If this Future is associated with a task inside this DFK,
+                    # then refer to the task ID.
+                    # Otherwise make a repr of the Future object.
+                    if hasattr(dep, 'task_record') and dep.task_record['dfk'] == self:
+                        tid = "task " + repr(dep.task_record['id'])
                     else:
-                        tid = None
+                        tid = repr(dep)
                     dep_failures.extend([(e, tid)])
             else:
                 new_args.extend([dep])

parsl/dataflow/errors.py

@@ -36,7 +36,9 @@ class DependencyError(DataFlowException):
        in a dependency.
 
     Args:
-         - dependent_exceptions_tids: List of dependency task IDs which failed
+         - dependent_exceptions_tids: List of exceptions and identifiers for
+           dependencies which failed. The identifier might be a task ID or
+           the repr of a non-DFK Future.
          - task_id: Task ID of the task that failed because of the dependency error
     """
 
@@ -45,8 +47,8 @@ class DependencyError(DataFlowException):
         self.task_id = task_id
 
     def __str__(self) -> str:
-        dep_tids = [tid for (exception, tid) in self.dependent_exceptions_tids]
-        return "Dependency failure for task {} with failed dependencies from tasks {}".format(self.task_id, dep_tids)
+        deps = ", ".join(tid for _exc, tid in self.dependent_exceptions_tids)
+        return f"Dependency failure for task {self.task_id} with failed dependencies from {deps}"
 
 
 class JoinError(DataFlowException):

parsl/executors/high_throughput/process_worker_pool.py

@@ -17,6 +17,8 @@ import math
 import json
 import psutil
 import multiprocessing
+from multiprocessing.managers import DictProxy
+from multiprocessing.sharedctypes import Synchronized
 
 from parsl.process_loggers import wrap_with_logs
 
@@ -24,10 +26,7 @@ from parsl.version import VERSION as PARSL_VERSION
 from parsl.app.errors import RemoteExceptionWrapper
 from parsl.executors.high_throughput.errors import WorkerLost
 from parsl.executors.high_throughput.probe import probe_addresses
-from parsl.multiprocessing import ForkProcess as mpProcess
-
-from parsl.multiprocessing import SizedQueue as mpQueue
-
+from parsl.multiprocessing import SpawnContext
 from parsl.serialize import unpack_apply_message, serialize
 
 HEARTBEAT_CODE = (2 ** 32) - 1
@@ -49,22 +48,22 @@ class Manager:
                 |                          |                IPC-Qeueues
 
     """
-    def __init__(self,
-                 addresses="127.0.0.1",
-                 address_probe_timeout=30,
-                 task_port="50097",
-                 result_port="50098",
-                 cores_per_worker=1,
-                 mem_per_worker=None,
-                 max_workers=float('inf'),
-                 prefetch_capacity=0,
-                 uid=None,
-                 block_id=None,
-                 heartbeat_threshold=120,
-                 heartbeat_period=30,
-                 poll_period=10,
-                 cpu_affinity=False,
-                 available_accelerators: Sequence[str] = ()):
+    def __init__(self, *,
+                 addresses,
+                 address_probe_timeout,
+                 task_port,
+                 result_port,
+                 cores_per_worker,
+                 mem_per_worker,
+                 max_workers,
+                 prefetch_capacity,
+                 uid,
+                 block_id,
+                 heartbeat_threshold,
+                 heartbeat_period,
+                 poll_period,
+                 cpu_affinity,
+                 available_accelerators: Sequence[str]):
         """
         Parameters
         ----------
@@ -73,7 +72,7 @@ class Manager:
 
         address_probe_timeout : int
              Timeout in seconds for the address probe to detect viable addresses
-             to the interchange. Default : 30s
+             to the interchange.
 
         uid : str
              string unique identifier
@@ -83,43 +82,41 @@ class Manager:
 
         cores_per_worker : float
              cores to be assigned to each worker. Oversubscription is possible
-             by setting cores_per_worker < 1.0. Default=1
+             by setting cores_per_worker < 1.0.
 
         mem_per_worker : float
              GB of memory required per worker. If this option is specified, the node manager
              will check the available memory at startup and limit the number of workers such that
              the there's sufficient memory for each worker. If set to None, memory on node is not
              considered in the determination of workers to be launched on node by the manager.
-             Default: None
 
         max_workers : int
              caps the maximum number of workers that can be launched.
-             default: infinity
 
         prefetch_capacity : int
              Number of tasks that could be prefetched over available worker capacity.
              When there are a few tasks (<100) or when tasks are long running, this option should
-             be set to 0 for better load balancing. Default is 0.
+             be set to 0 for better load balancing.
 
         heartbeat_threshold : int
              Seconds since the last message from the interchange after which the
-             interchange is assumed to be un-available, and the manager initiates shutdown. Default:120s
+             interchange is assumed to be un-available, and the manager initiates shutdown.
 
              Number of seconds since the last message from the interchange after which the worker
-             assumes that the interchange is lost and the manager shuts down. Default:120
+             assumes that the interchange is lost and the manager shuts down.
 
         heartbeat_period : int
              Number of seconds after which a heartbeat message is sent to the interchange, and workers
              are checked for liveness.
 
         poll_period : int
-             Timeout period used by the manager in milliseconds. Default: 10ms
+             Timeout period used by the manager in milliseconds.
 
         cpu_affinity : str
              Whether or how each worker should force its affinity to different CPUs
 
         available_accelerators: list of str
-            List of accelerators available to the workers. Default: Empty list
+            List of accelerators available to the workers.
 
         """
 
@@ -160,7 +157,7 @@ class Manager:
         if os.environ.get('PARSL_CORES'):
             cores_on_node = int(os.environ['PARSL_CORES'])
         else:
-            cores_on_node = multiprocessing.cpu_count()
+            cores_on_node = SpawnContext.cpu_count()
 
         if os.environ.get('PARSL_MEMORY_GB'):
             available_mem_on_node = float(os.environ['PARSL_MEMORY_GB'])
@@ -175,13 +172,16 @@ class Manager:
         if mem_per_worker and mem_per_worker > 0:
             mem_slots = math.floor(available_mem_on_node / mem_per_worker)
 
-        self.worker_count = min(max_workers,
-                                mem_slots,
-                                math.floor(cores_on_node / cores_per_worker))
+        self.worker_count: int = min(max_workers,
+                                     mem_slots,
+                                     math.floor(cores_on_node / cores_per_worker))
 
-        self.pending_task_queue = mpQueue()
-        self.pending_result_queue = mpQueue()
-        self.ready_worker_queue = mpQueue()
+        self._mp_manager = SpawnContext.Manager()  # Starts a server process
+
+        self.monitoring_queue = self._mp_manager.Queue()
+        self.pending_task_queue = SpawnContext.Queue()
+        self.pending_result_queue = SpawnContext.Queue()
+        self.ready_worker_count = SpawnContext.Value("i", 0)
 
         self.max_queue_size = self.prefetch_capacity + self.worker_count
 
@@ -252,10 +252,13 @@ class Manager:
         poll_timer = self.poll_period
 
         while not kill_event.is_set():
-            ready_worker_count = self.ready_worker_queue.qsize()
-            pending_task_count = self.pending_task_queue.qsize()
+            try:
+                pending_task_count = self.pending_task_queue.qsize()
+            except NotImplementedError:
+                # Ref: https://github.com/python/cpython/blob/6d5e0dc0e330f4009e8dc3d1642e46b129788877/Lib/multiprocessing/queues.py#L125
+                pending_task_count = f"pending task count is not available on {platform.system()}"
 
-            logger.debug("ready workers: {}, pending tasks: {}".format(ready_worker_count,
+            logger.debug("ready workers: {}, pending tasks: {}".format(self.ready_worker_count.value,
                                                                        pending_task_count))
 
             if time.time() > last_beat + self.heartbeat_period:
@@ -381,6 +384,36 @@ class Manager:
 
         logger.critical("Exiting")
 
+    @wrap_with_logs
+    def handle_monitoring_messages(self, kill_event: threading.Event):
+        """Transfer messages from the managed monitoring queue to the result queue.
+
+        We separate the queues so that the result queue does not rely on a manager
+        process, which adds overhead that causes slower queue operations but enables
+        use across processes started in fork and spawn contexts.
+
+        We transfer the messages to the result queue to reuse the ZMQ connection between
+        the manager and the interchange.
+        """
+        logger.debug("Starting monitoring handler thread")
+
+        poll_period_s = max(10, self.poll_period) / 1000    # Must be at least 10 ms
+
+        while not kill_event.is_set():
+            try:
+                logger.debug("Starting monitor_queue.get()")
+                msg = self.monitoring_queue.get(block=True, timeout=poll_period_s)
+            except queue.Empty:
+                logger.debug("monitoring_queue.get() has timed out")
+            except Exception as e:
+                logger.exception(f"Got an exception: {e}")
+            else:
+                logger.debug("Got a monitoring message")
+                self.pending_result_queue.put(msg)
+                logger.debug("Put monitoring message on pending_result_queue")
+
+        logger.critical("Exiting")
+
     def start(self):
         """ Start the worker processes.
 
@@ -388,7 +421,7 @@ class Manager:
         """
         start = time.time()
         self._kill_event = threading.Event()
-        self._tasks_in_progress = multiprocessing.Manager().dict()
+        self._tasks_in_progress = self._mp_manager.dict()
 
         self.procs = {}
         for worker_id in range(self.worker_count):
@@ -406,9 +439,14 @@ class Manager:
         self._worker_watchdog_thread = threading.Thread(target=self.worker_watchdog,
                                                         args=(self._kill_event,),
                                                         name="worker-watchdog")
+        self._monitoring_handler_thread = threading.Thread(target=self.handle_monitoring_messages,
+                                                           args=(self._kill_event,),
+                                                           name="Monitoring-Handler")
+
         self._task_puller_thread.start()
         self._result_pusher_thread.start()
         self._worker_watchdog_thread.start()
+        self._monitoring_handler_thread.start()
 
         logger.info("Loop start")
 
@@ -420,6 +458,7 @@ class Manager:
         self._task_puller_thread.join()
         self._result_pusher_thread.join()
         self._worker_watchdog_thread.join()
+        self._monitoring_handler_thread.join()
         for proc_id in self.procs:
             self.procs[proc_id].terminate()
             logger.critical("Terminating worker {}: is_alive()={}".format(self.procs[proc_id],
@@ -435,7 +474,7 @@ class Manager:
         return
 
     def _start_worker(self, worker_id: int):
-        p = mpProcess(
+        p = SpawnContext.Process(
             target=worker,
             args=(
                 worker_id,
@@ -443,10 +482,16 @@ class Manager:
                 self.worker_count,
                 self.pending_task_queue,
                 self.pending_result_queue,
-                self.ready_worker_queue,
+                self.monitoring_queue,
+                self.ready_worker_count,
                 self._tasks_in_progress,
                 self.cpu_affinity,
                 self.available_accelerators[worker_id] if self.accelerators_available else None,
+                self.block_id,
+                self.heartbeat_period,
+                os.getpid(),
+                args.logdir,
+                args.debug,
             ),
             name="HTEX-Worker-{}".format(worker_id),
         )
@@ -484,7 +529,23 @@ def execute_task(bufs):
 
 
 @wrap_with_logs(target="worker_log")
-def worker(worker_id, pool_id, pool_size, task_queue, result_queue, worker_queue, tasks_in_progress, cpu_affinity, accelerator: Optional[str]):
+def worker(
+    worker_id: int,
+    pool_id: str,
+    pool_size: int,
+    task_queue: multiprocessing.Queue,
+    result_queue: multiprocessing.Queue,
+    monitoring_queue: queue.Queue,
+    ready_worker_count: Synchronized,
+    tasks_in_progress: DictProxy,
+    cpu_affinity: str,
+    accelerator: Optional[str],
+    block_id: str,
+    task_queue_timeout: int,
+    manager_pid: int,
+    logdir: str,
+    debug: bool,
+):
     """
 
     Put request token into queue
@@ -496,23 +557,22 @@ def worker(worker_id, pool_id, pool_size, task_queue, result_queue, worker_queue
     # override the global logger inherited from the __main__ process (which
     # usually logs to manager.log) with one specific to this worker.
     global logger
-    logger = start_file_logger('{}/block-{}/{}/worker_{}.log'.format(args.logdir, args.block_id, pool_id, worker_id),
+    logger = start_file_logger('{}/block-{}/{}/worker_{}.log'.format(logdir, block_id, pool_id, worker_id),
                                worker_id,
                                name="worker_log",
-                               level=logging.DEBUG if args.debug else logging.INFO)
+                               level=logging.DEBUG if debug else logging.INFO)
 
     # Store worker ID as an environment variable
     os.environ['PARSL_WORKER_RANK'] = str(worker_id)
     os.environ['PARSL_WORKER_COUNT'] = str(pool_size)
     os.environ['PARSL_WORKER_POOL_ID'] = str(pool_id)
-    os.environ['PARSL_WORKER_BLOCK_ID'] = str(args.block_id)
+    os.environ['PARSL_WORKER_BLOCK_ID'] = str(block_id)
 
-    # share the result queue with monitoring code so it too can send results down that channel
     import parsl.executors.high_throughput.monitoring_info as mi
-    mi.result_queue = result_queue
+    mi.result_queue = monitoring_queue
 
     logger.info('Worker {} started'.format(worker_id))
-    if args.debug:
+    if debug:
         logger.debug("Debug logging enabled")
 
     # If desired, set process affinity
@@ -553,20 +613,37 @@ def worker(worker_id, pool_id, pool_size, task_queue, result_queue, worker_queue
 
         logger.info(f'Pinned worker to accelerator: {accelerator}')
 
-    while True:
-        worker_queue.put(worker_id)
+    def manager_is_alive():
+        try:
+            # This does not kill the process, but instead raises
+            # an exception if the process doesn't exist
+            os.kill(manager_pid, 0)
+        except OSError:
+            logger.critical(f"Manager ({manager_pid}) died; worker {worker_id} shutting down")
+            return False
+        else:
+            return True
+
+    worker_enqueued = False
+    while manager_is_alive():
+        if not worker_enqueued:
+            with ready_worker_count.get_lock():
+                ready_worker_count.value += 1
+            worker_enqueued = True
+
+        try:
+            # The worker will receive {'task_id':<tid>, 'buffer':<buf>}
+            req = task_queue.get(timeout=task_queue_timeout)
+        except queue.Empty:
+            continue
 
-        # The worker will receive {'task_id':<tid>, 'buffer':<buf>}
-        req = task_queue.get()
         tasks_in_progress[worker_id] = req
         tid = req['task_id']
         logger.info("Received executor task {}".format(tid))
 
-        try:
-            worker_queue.get()
-        except queue.Empty:
-            logger.warning("Worker ID: {} failed to remove itself from ready_worker_queue".format(worker_id))
-            pass
+        with ready_worker_count.get_lock():
+            ready_worker_count.value -= 1
+        worker_enqueued = False
 
         try:
             result = execute_task(req['buffer'])
@@ -653,6 +730,7 @@ if __name__ == "__main__":
     parser.add_argument("-r", "--result_port", required=True,
                         help="REQUIRED: Result port for posting results to the interchange")
     parser.add_argument("--cpu-affinity", type=str, choices=["none", "block", "alternating", "block-reverse"],
+                        required=True,
                         help="Whether/how workers should control CPU affinity.")
     parser.add_argument("--available-accelerators", type=str, nargs="*",
                         help="Names of available accelerators")

parsl/executors/high_throughput/zmq_pipes.py

@@ -46,7 +46,6 @@ class CommandClient:
         """ This function needs to be fast at the same time aware of the possibility of
         ZMQ pipes overflowing.
 
-        The timeout increases slowly if contention is detected on ZMQ pipes.
         We could set copy=False and get slightly better latency but this results
         in ZMQ sockets reaching a broken state once there are ~10k tasks in flight.
         This issue can be magnified if each the serialized buffer itself is larger.
@@ -55,8 +54,11 @@ class CommandClient:
         with self._lock:
             for _ in range(max_retries):
                 try:
+                    logger.debug("Sending command client command")
                     self.zmq_socket.send_pyobj(message, copy=True)
+                    logger.debug("Waiting for command client response")
                     reply = self.zmq_socket.recv_pyobj()
+                    logger.debug("Received command client response")
                 except zmq.ZMQError:
                     logger.exception("Potential ZMQ REQ-REP deadlock caught")
                     logger.info("Trying to reestablish context")
@@ -115,7 +117,9 @@ class TasksOutgoing:
             socks = dict(self.poller.poll(timeout=timeout_ms))
             if self.zmq_socket in socks and socks[self.zmq_socket] == zmq.POLLOUT:
                 # The copy option adds latency but reduces the risk of ZMQ overflow
+                logger.debug("Sending TasksOutgoing message")
                 self.zmq_socket.send_pyobj(message, copy=True)
+                logger.debug("Sent TasksOutgoing message")
                 return
             else:
                 timeout_ms *= 2
@@ -149,7 +153,10 @@ class ResultsIncoming:
                                                               max_port=port_range[1])
 
     def get(self):
-        return self.results_receiver.recv_multipart()
+        logger.debug("Waiting for ResultsIncoming message")
+        m = self.results_receiver.recv_multipart()
+        logger.debug("Received ResultsIncoming message")
+        return m
 
     def close(self):
         self.results_receiver.close()

parsl/multiprocessing.py

@@ -1,4 +1,4 @@
-"""Helpers for cross-plaform multiprocessing support.
+"""Helpers for cross-platform multiprocessing support.
 """
 
 import logging
@@ -10,9 +10,12 @@ from typing import Callable, Type
 
 logger = logging.getLogger(__name__)
 
+ForkContext = multiprocessing.get_context("fork")
+SpawnContext = multiprocessing.get_context("spawn")
+
 # maybe ForkProcess should be: Callable[..., Process] so as to make
 # it clear that it returns a Process always to the type checker?
-ForkProcess: Type = multiprocessing.get_context('fork').Process
+ForkProcess: Type = ForkContext.Process
 
 
 class MacSafeQueue(multiprocessing.queues.Queue):

parsl/providers/local/local.py

@@ -28,8 +28,9 @@ class LocalProvider(ExecutionProvider, RepresentationMixin):
         Ratio of provisioned task slots to active tasks. A parallelism value of 1 represents aggressive
         scaling where as many resources as possible are used; parallelism close to 0 represents
         the opposite situation in which as few resources as possible (i.e., min_blocks) are used.
-    move_files : Optional[Bool]: should files be moved? by default, Parsl will try to figure
-        this out itself (= None). If True, then will always move. If False, will never move.
+    move_files : Optional[Bool]
+        Should files be moved? By default, Parsl will try to figure this out itself (= None).
+        If True, then will always move. If False, will never move.
     worker_init : str
         Command to be run before starting a worker, such as 'module load Anaconda; source activate env'.
     """

parsl/providers/slurm/slurm.py

@@ -47,6 +47,10 @@ class SlurmProvider(ClusterProvider, RepresentationMixin):
     account : str
         Slurm account to which to charge resources used by the job. If unspecified or ``None``, the job will use the
         user's default account.
+    qos : str
+        Slurm queue to place job in. If unspecified or ``None``, no queue slurm directive will be specified.
+    constraint : str
+        Slurm job constraint, often used to choose cpu or gpu type. If unspecified or ``None``, no constraint slurm directive will be added.
     channel : Channel
         Channel for accessing this provider. Possible channels include
         :class:`~parsl.channels.LocalChannel` (the default),
@@ -92,6 +96,8 @@ class SlurmProvider(ClusterProvider, RepresentationMixin):
     def __init__(self,
                  partition: Optional[str] = None,
                  account: Optional[str] = None,
+                 qos: Optional[str] = None,
+                 constraint: Optional[str] = None,
                  channel: Channel = LocalChannel(),
                  nodes_per_block: int = 1,
                  cores_per_node: Optional[int] = None,
@@ -126,6 +132,8 @@ class SlurmProvider(ClusterProvider, RepresentationMixin):
         self.exclusive = exclusive
         self.move_files = move_files
         self.account = account
+        self.qos = qos
+        self.constraint = constraint
         self.scheduler_options = scheduler_options + '\n'
         if exclusive:
             self.scheduler_options += "#SBATCH --exclusive\n"
@@ -133,6 +141,11 @@ class SlurmProvider(ClusterProvider, RepresentationMixin):
             self.scheduler_options += "#SBATCH --partition={}\n".format(partition)
         if account:
             self.scheduler_options += "#SBATCH --account={}\n".format(account)
+        if qos:
+            self.scheduler_options += "#SBATCH --qos={}\n".format(qos)
+        if constraint:
+            self.scheduler_options += "#SBATCH --constraint={}\n".format(constraint)
+
         self.regex_job_id = regex_job_id
         self.worker_init = worker_init + '\n'
 

parsl/tests/test_error_handling/test_htex_manager_failure.py

@@ -0,0 +1,52 @@
+import os
+import signal
+import time
+
+import pytest
+
+import parsl
+from parsl.app.app import python_app
+from parsl.tests.configs.htex_local import fresh_config
+
+
+@pytest.fixture(autouse=True, scope="function")
+def load_config():
+    config = fresh_config()
+    config.executors[0].poll_period = 1
+    config.executors[0].max_workers = 1
+    config.executors[0].heartbeat_period = 1
+
+    parsl.load(config)
+    yield
+
+    parsl.dfk().cleanup()
+    parsl.clear()
+
+
+@python_app
+def get_worker_pid():
+    import os
+    return os.getpid()
+
+
+@python_app
+def kill_manager(sig: int):
+    import os
+    os.kill(os.getppid(), sig)
+
+
+@pytest.mark.local
+@pytest.mark.parametrize("sig", [signal.SIGTERM, signal.SIGKILL])
+def test_htex_manager_failure_worker_shutdown(sig: int):
+    """Ensure that HTEX workers shut down when the Manager process dies."""
+    worker_pid = get_worker_pid().result()
+
+    kill_manager(sig)
+
+    with pytest.raises(OSError):
+        end = time.monotonic() + 5
+        while time.monotonic() < end:
+            # Raises an exception if the process
+            # does not exist
+            os.kill(worker_pid, 0)
+            time.sleep(.1)

parsl/tests/test_python_apps/test_dep_standard_futures.py

@@ -33,4 +33,12 @@ def test_future_fail_dependency():
 
     plain_fut.set_exception(ValueError("Plain failure"))
 
-    assert isinstance(parsl_fut.exception(), DependencyError)
+    ex = parsl_fut.exception()
+
+    # check that what we got is a dependency error...
+    assert isinstance(ex, DependencyError)
+
+    # and that the dependency error string mentions the dependency
+    # Future, plain_fut, somewhere in its str
+
+    assert repr(plain_fut) in str(ex)

parsl/tests/test_python_apps/test_depfail_propagation.py

@@ -21,6 +21,10 @@ def test_depfail_once():
     assert not isinstance(f1.exception(), DependencyError)
     assert isinstance(f2.exception(), DependencyError)
 
+    # check that the task ID of the failing task is mentioned
+    # in the DependencyError message
+    assert ("task " + str(f1.task_record['id'])) in str(f2.exception())
+
 
 def test_depfail_chain():
     """Test that dependency failures chain"""

parsl/version.py

@@ -3,4 +3,4 @@
 Year.Month.Day[alpha/beta/..]
 Alphas will be numbered like this -> 2024.12.10a0
 """
-VERSION = '2023.12.04'
+VERSION = '2023.12.18'

{parsl-2023.12.4.data → parsl-2023.12.18.data}/scripts/process_worker_pool.py

@@ -17,6 +17,8 @@ import math
 import json
 import psutil
 import multiprocessing
+from multiprocessing.managers import DictProxy
+from multiprocessing.sharedctypes import Synchronized
 
 from parsl.process_loggers import wrap_with_logs
 
@@ -24,10 +26,7 @@ from parsl.version import VERSION as PARSL_VERSION
 from parsl.app.errors import RemoteExceptionWrapper
 from parsl.executors.high_throughput.errors import WorkerLost
 from parsl.executors.high_throughput.probe import probe_addresses
-from parsl.multiprocessing import ForkProcess as mpProcess
-
-from parsl.multiprocessing import SizedQueue as mpQueue
-
+from parsl.multiprocessing import SpawnContext
 from parsl.serialize import unpack_apply_message, serialize
 
 HEARTBEAT_CODE = (2 ** 32) - 1
@@ -49,22 +48,22 @@ class Manager:
                 |                          |                IPC-Qeueues
 
     """
-    def __init__(self,
-                 addresses="127.0.0.1",
-                 address_probe_timeout=30,
-                 task_port="50097",
-                 result_port="50098",
-                 cores_per_worker=1,
-                 mem_per_worker=None,
-                 max_workers=float('inf'),
-                 prefetch_capacity=0,
-                 uid=None,
-                 block_id=None,
-                 heartbeat_threshold=120,
-                 heartbeat_period=30,
-                 poll_period=10,
-                 cpu_affinity=False,
-                 available_accelerators: Sequence[str] = ()):
+    def __init__(self, *,
+                 addresses,
+                 address_probe_timeout,
+                 task_port,
+                 result_port,
+                 cores_per_worker,
+                 mem_per_worker,
+                 max_workers,
+                 prefetch_capacity,
+                 uid,
+                 block_id,
+                 heartbeat_threshold,
+                 heartbeat_period,
+                 poll_period,
+                 cpu_affinity,
+                 available_accelerators: Sequence[str]):
         """
         Parameters
         ----------
@@ -73,7 +72,7 @@ class Manager:
 
         address_probe_timeout : int
              Timeout in seconds for the address probe to detect viable addresses
-             to the interchange. Default : 30s
+             to the interchange.
 
         uid : str
              string unique identifier
@@ -83,43 +82,41 @@ class Manager:
 
         cores_per_worker : float
              cores to be assigned to each worker. Oversubscription is possible
-             by setting cores_per_worker < 1.0. Default=1
+             by setting cores_per_worker < 1.0.
 
         mem_per_worker : float
              GB of memory required per worker. If this option is specified, the node manager
              will check the available memory at startup and limit the number of workers such that
              the there's sufficient memory for each worker. If set to None, memory on node is not
              considered in the determination of workers to be launched on node by the manager.
-             Default: None
 
         max_workers : int
              caps the maximum number of workers that can be launched.
-             default: infinity
 
         prefetch_capacity : int
              Number of tasks that could be prefetched over available worker capacity.
              When there are a few tasks (<100) or when tasks are long running, this option should
-             be set to 0 for better load balancing. Default is 0.
+             be set to 0 for better load balancing.
 
         heartbeat_threshold : int
              Seconds since the last message from the interchange after which the
-             interchange is assumed to be un-available, and the manager initiates shutdown. Default:120s
+             interchange is assumed to be un-available, and the manager initiates shutdown.
 
              Number of seconds since the last message from the interchange after which the worker
-             assumes that the interchange is lost and the manager shuts down. Default:120
+             assumes that the interchange is lost and the manager shuts down.
 
         heartbeat_period : int
              Number of seconds after which a heartbeat message is sent to the interchange, and workers
              are checked for liveness.
 
         poll_period : int
-             Timeout period used by the manager in milliseconds. Default: 10ms
+             Timeout period used by the manager in milliseconds.
 
         cpu_affinity : str
              Whether or how each worker should force its affinity to different CPUs
 
         available_accelerators: list of str
-            List of accelerators available to the workers. Default: Empty list
+            List of accelerators available to the workers.
 
         """
 
@@ -160,7 +157,7 @@ class Manager:
         if os.environ.get('PARSL_CORES'):
             cores_on_node = int(os.environ['PARSL_CORES'])
         else:
-            cores_on_node = multiprocessing.cpu_count()
+            cores_on_node = SpawnContext.cpu_count()
 
         if os.environ.get('PARSL_MEMORY_GB'):
             available_mem_on_node = float(os.environ['PARSL_MEMORY_GB'])
@@ -175,13 +172,16 @@ class Manager:
         if mem_per_worker and mem_per_worker > 0:
             mem_slots = math.floor(available_mem_on_node / mem_per_worker)
 
-        self.worker_count = min(max_workers,
-                                mem_slots,
-                                math.floor(cores_on_node / cores_per_worker))
+        self.worker_count: int = min(max_workers,
+                                     mem_slots,
+                                     math.floor(cores_on_node / cores_per_worker))
 
-        self.pending_task_queue = mpQueue()
-        self.pending_result_queue = mpQueue()
-        self.ready_worker_queue = mpQueue()
+        self._mp_manager = SpawnContext.Manager()  # Starts a server process
+
+        self.monitoring_queue = self._mp_manager.Queue()
+        self.pending_task_queue = SpawnContext.Queue()
+        self.pending_result_queue = SpawnContext.Queue()
+        self.ready_worker_count = SpawnContext.Value("i", 0)
 
         self.max_queue_size = self.prefetch_capacity + self.worker_count
 
@@ -252,10 +252,13 @@ class Manager:
         poll_timer = self.poll_period
 
         while not kill_event.is_set():
-            ready_worker_count = self.ready_worker_queue.qsize()
-            pending_task_count = self.pending_task_queue.qsize()
+            try:
+                pending_task_count = self.pending_task_queue.qsize()
+            except NotImplementedError:
+                # Ref: https://github.com/python/cpython/blob/6d5e0dc0e330f4009e8dc3d1642e46b129788877/Lib/multiprocessing/queues.py#L125
+                pending_task_count = f"pending task count is not available on {platform.system()}"
 
-            logger.debug("ready workers: {}, pending tasks: {}".format(ready_worker_count,
+            logger.debug("ready workers: {}, pending tasks: {}".format(self.ready_worker_count.value,
                                                                        pending_task_count))
 
             if time.time() > last_beat + self.heartbeat_period:
@@ -381,6 +384,36 @@ class Manager:
 
         logger.critical("Exiting")
 
+    @wrap_with_logs
+    def handle_monitoring_messages(self, kill_event: threading.Event):
+        """Transfer messages from the managed monitoring queue to the result queue.
+
+        We separate the queues so that the result queue does not rely on a manager
+        process, which adds overhead that causes slower queue operations but enables
+        use across processes started in fork and spawn contexts.
+
+        We transfer the messages to the result queue to reuse the ZMQ connection between
+        the manager and the interchange.
+        """
+        logger.debug("Starting monitoring handler thread")
+
+        poll_period_s = max(10, self.poll_period) / 1000    # Must be at least 10 ms
+
+        while not kill_event.is_set():
+            try:
+                logger.debug("Starting monitor_queue.get()")
+                msg = self.monitoring_queue.get(block=True, timeout=poll_period_s)
+            except queue.Empty:
+                logger.debug("monitoring_queue.get() has timed out")
+            except Exception as e:
+                logger.exception(f"Got an exception: {e}")
+            else:
+                logger.debug("Got a monitoring message")
+                self.pending_result_queue.put(msg)
+                logger.debug("Put monitoring message on pending_result_queue")
+
+        logger.critical("Exiting")
+
     def start(self):
         """ Start the worker processes.
 
@@ -388,7 +421,7 @@ class Manager:
         """
         start = time.time()
         self._kill_event = threading.Event()
-        self._tasks_in_progress = multiprocessing.Manager().dict()
+        self._tasks_in_progress = self._mp_manager.dict()
 
         self.procs = {}
         for worker_id in range(self.worker_count):
@@ -406,9 +439,14 @@ class Manager:
         self._worker_watchdog_thread = threading.Thread(target=self.worker_watchdog,
                                                         args=(self._kill_event,),
                                                         name="worker-watchdog")
+        self._monitoring_handler_thread = threading.Thread(target=self.handle_monitoring_messages,
+                                                           args=(self._kill_event,),
+                                                           name="Monitoring-Handler")
+
         self._task_puller_thread.start()
         self._result_pusher_thread.start()
         self._worker_watchdog_thread.start()
+        self._monitoring_handler_thread.start()
 
         logger.info("Loop start")
 
@@ -420,6 +458,7 @@ class Manager:
         self._task_puller_thread.join()
         self._result_pusher_thread.join()
         self._worker_watchdog_thread.join()
+        self._monitoring_handler_thread.join()
         for proc_id in self.procs:
             self.procs[proc_id].terminate()
             logger.critical("Terminating worker {}: is_alive()={}".format(self.procs[proc_id],
@@ -435,7 +474,7 @@ class Manager:
         return
 
     def _start_worker(self, worker_id: int):
-        p = mpProcess(
+        p = SpawnContext.Process(
             target=worker,
             args=(
                 worker_id,
@@ -443,10 +482,16 @@ class Manager:
                 self.worker_count,
                 self.pending_task_queue,
                 self.pending_result_queue,
-                self.ready_worker_queue,
+                self.monitoring_queue,
+                self.ready_worker_count,
                 self._tasks_in_progress,
                 self.cpu_affinity,
                 self.available_accelerators[worker_id] if self.accelerators_available else None,
+                self.block_id,
+                self.heartbeat_period,
+                os.getpid(),
+                args.logdir,
+                args.debug,
             ),
             name="HTEX-Worker-{}".format(worker_id),
         )
@@ -484,7 +529,23 @@ def execute_task(bufs):
 
 
 @wrap_with_logs(target="worker_log")
-def worker(worker_id, pool_id, pool_size, task_queue, result_queue, worker_queue, tasks_in_progress, cpu_affinity, accelerator: Optional[str]):
+def worker(
+    worker_id: int,
+    pool_id: str,
+    pool_size: int,
+    task_queue: multiprocessing.Queue,
+    result_queue: multiprocessing.Queue,
+    monitoring_queue: queue.Queue,
+    ready_worker_count: Synchronized,
+    tasks_in_progress: DictProxy,
+    cpu_affinity: str,
+    accelerator: Optional[str],
+    block_id: str,
+    task_queue_timeout: int,
+    manager_pid: int,
+    logdir: str,
+    debug: bool,
+):
     """
 
     Put request token into queue
@@ -496,23 +557,22 @@ def worker(worker_id, pool_id, pool_size, task_queue, result_queue, worker_queue
     # override the global logger inherited from the __main__ process (which
     # usually logs to manager.log) with one specific to this worker.
     global logger
-    logger = start_file_logger('{}/block-{}/{}/worker_{}.log'.format(args.logdir, args.block_id, pool_id, worker_id),
+    logger = start_file_logger('{}/block-{}/{}/worker_{}.log'.format(logdir, block_id, pool_id, worker_id),
                                worker_id,
                                name="worker_log",
-                               level=logging.DEBUG if args.debug else logging.INFO)
+                               level=logging.DEBUG if debug else logging.INFO)
 
     # Store worker ID as an environment variable
     os.environ['PARSL_WORKER_RANK'] = str(worker_id)
     os.environ['PARSL_WORKER_COUNT'] = str(pool_size)
     os.environ['PARSL_WORKER_POOL_ID'] = str(pool_id)
-    os.environ['PARSL_WORKER_BLOCK_ID'] = str(args.block_id)
+    os.environ['PARSL_WORKER_BLOCK_ID'] = str(block_id)
 
-    # share the result queue with monitoring code so it too can send results down that channel
     import parsl.executors.high_throughput.monitoring_info as mi
-    mi.result_queue = result_queue
+    mi.result_queue = monitoring_queue
 
     logger.info('Worker {} started'.format(worker_id))
-    if args.debug:
+    if debug:
         logger.debug("Debug logging enabled")
 
     # If desired, set process affinity
@@ -553,20 +613,37 @@ def worker(worker_id, pool_id, pool_size, task_queue, result_queue, worker_queue
 
         logger.info(f'Pinned worker to accelerator: {accelerator}')
 
-    while True:
-        worker_queue.put(worker_id)
+    def manager_is_alive():
+        try:
+            # This does not kill the process, but instead raises
+            # an exception if the process doesn't exist
+            os.kill(manager_pid, 0)
+        except OSError:
+            logger.critical(f"Manager ({manager_pid}) died; worker {worker_id} shutting down")
+            return False
+        else:
+            return True
+
+    worker_enqueued = False
+    while manager_is_alive():
+        if not worker_enqueued:
+            with ready_worker_count.get_lock():
+                ready_worker_count.value += 1
+            worker_enqueued = True
+
+        try:
+            # The worker will receive {'task_id':<tid>, 'buffer':<buf>}
+            req = task_queue.get(timeout=task_queue_timeout)
+        except queue.Empty:
+            continue
 
-        # The worker will receive {'task_id':<tid>, 'buffer':<buf>}
-        req = task_queue.get()
         tasks_in_progress[worker_id] = req
         tid = req['task_id']
         logger.info("Received executor task {}".format(tid))
 
-        try:
-            worker_queue.get()
-        except queue.Empty:
-            logger.warning("Worker ID: {} failed to remove itself from ready_worker_queue".format(worker_id))
-            pass
+        with ready_worker_count.get_lock():
+            ready_worker_count.value -= 1
+        worker_enqueued = False
 
         try:
             result = execute_task(req['buffer'])
@@ -653,6 +730,7 @@ if __name__ == "__main__":
     parser.add_argument("-r", "--result_port", required=True,
                         help="REQUIRED: Result port for posting results to the interchange")
     parser.add_argument("--cpu-affinity", type=str, choices=["none", "block", "alternating", "block-reverse"],
+                        required=True,
                         help="Whether/how workers should control CPU affinity.")
     parser.add_argument("--available-accelerators", type=str, nargs="*",
                         help="Names of available accelerators")

{parsl-2023.12.4.dist-info → parsl-2023.12.18.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: parsl
-Version: 2023.12.4
+Version: 2023.12.18
 Summary: Simple data dependent workflows in Python
 Home-page: https://github.com/Parsl/parsl
-Download-URL: https://github.com/Parsl/parsl/archive/2023.12.04.tar.gz
+Download-URL: https://github.com/Parsl/parsl/archive/2023.12.18.tar.gz
 Author: The Parsl Team
 Author-email: parsl@googlegroups.com
 License: Apache 2.0

{parsl-2023.12.4.dist-info → parsl-2023.12.18.dist-info}/RECORD

@@ -3,11 +3,11 @@ parsl/addresses.py,sha256=L4RjQ-jGY9RfT-hBpsGw1uCzWaIdrEKxcPWV-TkGJes,4767
 parsl/config.py,sha256=ysUWBfm9bygayHHdItaJbP4oozkHJJmVQVnWCt5igjE,6808
 parsl/errors.py,sha256=SzINzQFZDBDbj9l-DPQznD0TbGkNhHIRAPkcBCogf_A,1019
 parsl/log_utils.py,sha256=AGem-dhQs5TYUyJg6GKkRuHxAw8FHhYlWB_0s7_ROw4,3175
-parsl/multiprocessing.py,sha256=e6WKPkqjC8TK0BesuVKUQPtipUtP0a-jM0iOzY_bagw,1904
+parsl/multiprocessing.py,sha256=w3t1pFkHo4oZpznc2KF6Ff-Jj8MvXqvjm-hoiRqZDDQ,1984
 parsl/process_loggers.py,sha256=1G3Rfrh5wuZNo2X03grG4kTYPGOxz7hHCyG6L_A3b0A,1137
 parsl/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 parsl/utils.py,sha256=_flbNpTu6IXHbzIyE5JkUbOBIK4poc1R1bjBtwJUVdo,11622
-parsl/version.py,sha256=f6Ei3SwZ3QtwHwPBUkqRduc-oCtfvLSKvo0s-j5PC3g,131
+parsl/version.py,sha256=TdZ44_l5ykDC_eoJiyMUVahV--8rnh2rNzC-MtW2rL4,131
 parsl/app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 parsl/app/app.py,sha256=wAHchJetgnicT1pn0NJKDeDX0lV3vDFlG8cQd_Ciax4,8522
 parsl/app/bash.py,sha256=bx9x1XFwkOTpZZD3CPwnVL9SyNRDjbUGtOnuGLvxN_8,5396
@@ -62,8 +62,8 @@ parsl/data_provider/http.py,sha256=nDHTW7XmJqAukWJjPRQjyhUXt8r6GsQ36mX9mv_wOig,2
 parsl/data_provider/rsync.py,sha256=2-ZxqrT-hBj39x082NusJaBqsGW4Jd2qCW6JkVPpEl0,4254
 parsl/data_provider/staging.py,sha256=l-mAXFburs3BWPjkSmiQKuAgJpsxCG62yATPDbrafYI,4523
 parsl/dataflow/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-parsl/dataflow/dflow.py,sha256=875obETSEog7WyovHDxR9k07zr_nifhbn67H5zmHxf8,63658
-parsl/dataflow/errors.py,sha256=JfjoqqqSg1WvTash5uwUuJt1ZDvxw-aAMenO-N3zoas,2084
+parsl/dataflow/dflow.py,sha256=uuXY9pURFDBpL0w52J0DWGCOtorTQ5wFy5V0WwHS9L8,63909
+parsl/dataflow/errors.py,sha256=w2vOt_ymzG2dOqJUO4IDcmTlrCIHlMZL8nBVyVq0O_8,2176
 parsl/dataflow/futures.py,sha256=aVfEUTzp4-EdunDAtNcqVQf8l_A7ArDi2c82KZMwxfY,5256
 parsl/dataflow/memoization.py,sha256=AsJO6c6cRp2ac6H8uGn2USlEi78_nX3QWvpxYt4XdYE,9583
 parsl/dataflow/rundirs.py,sha256=XKmBZpBEIsGACBhYOkbbs2e5edC0pQegJcSlk4FWeag,1154
@@ -85,8 +85,8 @@ parsl/executors/high_throughput/interchange.py,sha256=tX_EvQf7WkSKMJG-TNmA-WADjh
 parsl/executors/high_throughput/manager_record.py,sha256=T8-JVMfDJU6SJfzJRooD0mO8AHGMXlcn3PBOM0m_vng,366
 parsl/executors/high_throughput/monitoring_info.py,sha256=3gQpwQjjNDEBz0cQqJZB6hRiwLiWwXs83zkQDmbOwxY,297
 parsl/executors/high_throughput/probe.py,sha256=lvnuf-vBv57tHvFh-J51F9sDYBES7jCgs6KYgWvmKRs,2749
-parsl/executors/high_throughput/process_worker_pool.py,sha256=KaGaIE0RfJmJPld26CFivD58DYmCGSYIJk5g9uZKYYQ,31185
-parsl/executors/high_throughput/zmq_pipes.py,sha256=3-UrPu4DlXYb6JufjBcENspLd31Qk5URDaZP6IyC6SM,5720
+parsl/executors/high_throughput/process_worker_pool.py,sha256=l0l5F3mpJ60idMCN-d1AbdaogmOtO5eO3uGWogspNXg,34070
+parsl/executors/high_throughput/zmq_pipes.py,sha256=88VJz9QejOCQ_yyhaO5C1uQuDYZTovYEcnKn15WxHSU,6103
 parsl/executors/radical/__init__.py,sha256=CKbtV2numw5QvgIBq1htMUrt9TqDCIC2zifyf2svTNU,186
 parsl/executors/radical/executor.py,sha256=ZYycq58jXlBlhmIO1355JCK1xIJHkspiy62NN1XiMYQ,20729
 parsl/executors/radical/rpex_master.py,sha256=nMGxYWw3r-8_vZVnEwfB5eCfdTqXkeQDP5yvU0jXgc8,1368
@@ -174,7 +174,7 @@ parsl/providers/kubernetes/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJ
 parsl/providers/kubernetes/kube.py,sha256=uOr-sPgp73r1JFNc6wYhGhNGCvqkI8xBZznuJvfIfyk,12819
 parsl/providers/kubernetes/template.py,sha256=VsRz6cmNaII-y4OdMT6sCwzQy95SJX6NMB0hmmFBhX4,50
 parsl/providers/local/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-parsl/providers/local/local.py,sha256=eH6l03i3e-To4A0YFgWsE-xXS3EKeDyQynvWUGJE1kQ,11362
+parsl/providers/local/local.py,sha256=4X6Ds7PUuwFgkc6ZuzkEyDWTzjchb9eNzAgrwYfUKAs,11369
 parsl/providers/lsf/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 parsl/providers/lsf/lsf.py,sha256=AECVpjl_CTreE-APFQSjMVVIb3HheG6zculJn-zYtdM,11502
 parsl/providers/lsf/template.py,sha256=leQ_TpXv7ePMzbHfLaWvqMR0VORxlp-hjX5JxtkcwwU,269
@@ -182,7 +182,7 @@ parsl/providers/pbspro/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3
 parsl/providers/pbspro/pbspro.py,sha256=zXsb45LhgCkLEwwKXjdjsm2jv884j1fXHJ2hky9auD0,7789
 parsl/providers/pbspro/template.py,sha256=ozMbrx0HNsLnSoWbkZhy-55yJoTX5gpdRrDuVn6TFWA,369
 parsl/providers/slurm/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-parsl/providers/slurm/slurm.py,sha256=uQQryft2_ynjQkOsoaMHOqZdu3K1fTsL6LcfVJ6R-6o,11694
+parsl/providers/slurm/slurm.py,sha256=qHTNI5crS90PzdcoDu_lzDrGrYNss8yY_1XuWU5S2bc,12330
 parsl/providers/slurm/template.py,sha256=cc-3l5wnThEWfqzpniPgi3FP6934Ni05UZ9r0A1RA8s,369
 parsl/providers/torque/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 parsl/providers/torque/template.py,sha256=4qfc2gmlEhRCAD7erFDOs4prJQ43I8s4E8DSUSVQx3A,358
@@ -326,6 +326,7 @@ parsl/tests/test_docs/test_workflow4.py,sha256=PfOVDx5v_NtwDvg-ccC3A3SVM-SF0Pcyb
 parsl/tests/test_error_handling/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 parsl/tests/test_error_handling/test_fail.py,sha256=xx4TGWfL7le4cQ9nvnUkrlmKQJkskhD0l_3W1xwZSEI,282
 parsl/tests/test_error_handling/test_htex_basic.py,sha256=VRP_-Ro2SYp8TqfjpG_zCBJOZWuVFFCr3E0WKN_blg8,455
+parsl/tests/test_error_handling/test_htex_manager_failure.py,sha256=5YsCS1z7wOfUcFCD7uzR7t_rD3x5toZnoaCKVrHaMb0,1152
 parsl/tests/test_error_handling/test_htex_missing_worker.py,sha256=Tux0Xla719eup7RdWj8LmxNH-CTscMN0NM4CPuPP1ng,967
 parsl/tests/test_error_handling/test_htex_worker_failure.py,sha256=KO3ZegC8C6tY62XI1-uiS4w4gEYoRZZOEUnALEdBU8c,594
 parsl/tests/test_error_handling/test_python_walltime.py,sha256=rdmGZHIkuann2Njt3i62odKJ0FaODGr7-L96rOXNVYg,950
@@ -350,9 +351,9 @@ parsl/tests/test_providers/test_local_provider.py,sha256=G6Fuko22SvAtD7xhfQv8k_8
 parsl/tests/test_python_apps/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 parsl/tests/test_python_apps/test_arg_input_types.py,sha256=JXpfHiu8lr9BN6u1OzqFvGwBhxzsGTPMewHx6Wdo-HI,670
 parsl/tests/test_python_apps/test_basic.py,sha256=lFqh4ugePbp_FRiHGUXxzV34iS7l8C5UkxTHuLcpnYs,855
-parsl/tests/test_python_apps/test_dep_standard_futures.py,sha256=L3mUPmcrpC6QhZgHQbmnZIPE5JRmAA0JSMEmRQo_oOA,639
+parsl/tests/test_python_apps/test_dep_standard_futures.py,sha256=BloeaYBci0jS5al2d8Eqe3OfZ1tvolA5ZflOBQPR9Wo,859
 parsl/tests/test_python_apps/test_dependencies.py,sha256=IRiTI_lPoWBSFSFnaBlE6Bv08PKEaf-qj5dfqO2RjT0,272
-parsl/tests/test_python_apps/test_depfail_propagation.py,sha256=Jukzl6P5Be4I7VpZw2SoD2cwY_Yd0oU_THKl8wswx7k,1316
+parsl/tests/test_python_apps/test_depfail_propagation.py,sha256=3q3HlVWrOixFtXWBvR_ypKtbdAHAJcKndXQ5drwrBQU,1488
 parsl/tests/test_python_apps/test_fail.py,sha256=0Gld8LS6NB0Io1bU82vVR73twkuL5nW0ifKbIUcsJcw,1671
 parsl/tests/test_python_apps/test_fibonacci_iterative.py,sha256=ly2s5HuB9R53Z2FM_zy0WWdOk01iVhgcwSpQyK6ErIY,573
 parsl/tests/test_python_apps/test_fibonacci_recursive.py,sha256=q7LMFcu_pJSNPdz8iY0UiRoIweEWIBGwMjQffHWAuDc,592
@@ -412,12 +413,12 @@ parsl/tests/test_threads/test_configs.py,sha256=QA9YjIMAtZ2jmkfOWqBzEfzQQcFVCDiz
 parsl/tests/test_threads/test_lazy_errors.py,sha256=nGhYfCMHFZYSy6YJ4gnAmiLl9SfYs0WVnuvj8DXQ9bw,560
 parsl/usage_tracking/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 parsl/usage_tracking/usage.py,sha256=TEuAIm_U_G2ojZxvd0bbVa6gZlU61_mVRa2yJC9mGiI,7555
-parsl-2023.12.4.data/scripts/exec_parsl_function.py,sha256=NtWNeBvRqksej38eRPw8zPBJ1CeW6vgaitve0tfz_qc,7801
-parsl-2023.12.4.data/scripts/parsl_coprocess.py,sha256=kzX_1RI3V2KMKs6L-il4I1qkLNVodDKFXN_1FHB9fmM,6031
-parsl-2023.12.4.data/scripts/process_worker_pool.py,sha256=nSHtxNcJMeliSah83zSZRDxMwSxQBl55sPV5nfnGnXc,31171
-parsl-2023.12.4.dist-info/LICENSE,sha256=tAkwu8-AdEyGxGoSvJ2gVmQdcicWw3j1ZZueVV74M-E,11357
-parsl-2023.12.4.dist-info/METADATA,sha256=Rv935KrQ9z1QJOgdhx5a5rxCFJYnJFTMmLQvobUazvI,3817
-parsl-2023.12.4.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-parsl-2023.12.4.dist-info/entry_points.txt,sha256=XqnsWDYoEcLbsMcpnYGKLEnSBmaIe1YoM5YsBdJG2tI,176
-parsl-2023.12.4.dist-info/top_level.txt,sha256=PIheYoUFQtF2icLsgOykgU-Cjuwr2Oi6On2jo5RYgRM,6
-parsl-2023.12.4.dist-info/RECORD,,
+parsl-2023.12.18.data/scripts/exec_parsl_function.py,sha256=NtWNeBvRqksej38eRPw8zPBJ1CeW6vgaitve0tfz_qc,7801
+parsl-2023.12.18.data/scripts/parsl_coprocess.py,sha256=kzX_1RI3V2KMKs6L-il4I1qkLNVodDKFXN_1FHB9fmM,6031
+parsl-2023.12.18.data/scripts/process_worker_pool.py,sha256=ytz3F8ZYeBr8tFqSRv2O9eZGdsID7oZRulBmmQmZaV8,34056
+parsl-2023.12.18.dist-info/LICENSE,sha256=tAkwu8-AdEyGxGoSvJ2gVmQdcicWw3j1ZZueVV74M-E,11357
+parsl-2023.12.18.dist-info/METADATA,sha256=hmDBTXvoQYLYqsFRJi8HWQYLk9bwf7MCNEtljOy_tOY,3818
+parsl-2023.12.18.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+parsl-2023.12.18.dist-info/entry_points.txt,sha256=XqnsWDYoEcLbsMcpnYGKLEnSBmaIe1YoM5YsBdJG2tI,176
+parsl-2023.12.18.dist-info/top_level.txt,sha256=PIheYoUFQtF2icLsgOykgU-Cjuwr2Oi6On2jo5RYgRM,6
+parsl-2023.12.18.dist-info/RECORD,,