parsl 2025.3.3__py3-none-any.whl → 2025.3.10__py3-none-any.whl

parsl/executors/high_throughput/executor.py

@@ -536,7 +536,7 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin, UsageIn
                               "interchange_address": self.address,
                               "worker_ports": self.worker_ports,
                               "worker_port_range": self.worker_port_range,
-                              "hub_address": self.hub_address,
+                              "hub_address": self.loopback_address,
                               "hub_zmq_port": self.hub_zmq_port,
                               "logdir": self.logdir,
                               "heartbeat_threshold": self.heartbeat_threshold,

parsl/executors/high_throughput/mpi_resource_management.py

@@ -4,6 +4,7 @@ import os
 import pickle
 import queue
 import subprocess
+from dataclasses import dataclass, field
 from enum import Enum
 from typing import Dict, List, Optional
 
@@ -69,6 +70,14 @@ class MPINodesUnavailable(Exception):
         return f"MPINodesUnavailable(requested={self.requested} available={self.available})"
 
 
+@dataclass(order=True)
+class PrioritizedTask:
+    # Comparing dict will fail since they are unhashable
+    # This dataclass limits comparison to the priority field
+    priority: int
+    task: Dict = field(compare=False)
+
+
 class TaskScheduler:
     """Default TaskScheduler that does no taskscheduling
 
@@ -111,7 +120,7 @@ class MPITaskScheduler(TaskScheduler):
         super().__init__(pending_task_q, pending_result_q)
         self.scheduler = identify_scheduler()
         # PriorityQueue is threadsafe
-        self._backlog_queue: queue.PriorityQueue = queue.PriorityQueue()
+        self._backlog_queue: queue.PriorityQueue[PrioritizedTask] = queue.PriorityQueue()
         self._map_tasks_to_nodes: Dict[str, List[str]] = {}
         self.available_nodes = get_nodes_in_batchjob(self.scheduler)
         self._free_node_counter = SpawnContext.Value("i", len(self.available_nodes))
@@ -169,7 +178,7 @@ class MPITaskScheduler(TaskScheduler):
                 allocated_nodes = self._get_nodes(nodes_needed)
             except MPINodesUnavailable:
                 logger.info(f"Not enough resources, placing task {tid} into backlog")
-                self._backlog_queue.put((nodes_needed, task_package))
+                self._backlog_queue.put(PrioritizedTask(nodes_needed, task_package))
                 return
             else:
                 resource_spec["MPI_NODELIST"] = ",".join(allocated_nodes)
@@ -183,8 +192,8 @@ class MPITaskScheduler(TaskScheduler):
     def _schedule_backlog_tasks(self):
         """Attempt to schedule backlogged tasks"""
         try:
-            _nodes_requested, task_package = self._backlog_queue.get(block=False)
-            self.put_task(task_package)
+            prioritized_task = self._backlog_queue.get(block=False)
+            self.put_task(prioritized_task.task)
         except queue.Empty:
             return
         else:

parsl/monitoring/monitoring.py

@@ -3,23 +3,21 @@ from __future__ import annotations
 import logging
 import multiprocessing.synchronize as ms
 import os
-import pickle
 import queue
-import time
 from multiprocessing import Event
 from multiprocessing.queues import Queue
-from typing import TYPE_CHECKING, Literal, Optional, Tuple, Union, cast
+from typing import TYPE_CHECKING, Literal, Optional, Tuple, Union
 
 import typeguard
 
-from parsl.log_utils import set_file_logger
 from parsl.monitoring.errors import MonitoringHubStartError
+from parsl.monitoring.radios.filesystem_router import filesystem_router_starter
 from parsl.monitoring.radios.multiprocessing import MultiprocessingQueueRadioSender
-from parsl.monitoring.router import router_starter
+from parsl.monitoring.radios.udp_router import udp_router_starter
+from parsl.monitoring.radios.zmq_router import zmq_router_starter
 from parsl.monitoring.types import TaggedMonitoringMessage
 from parsl.multiprocessing import ForkProcess, SizedQueue
-from parsl.process_loggers import wrap_with_logs
-from parsl.utils import RepresentationMixin, setproctitle
+from parsl.utils import RepresentationMixin
 
 _db_manager_excepts: Optional[Exception]
 
@@ -121,11 +119,14 @@ class MonitoringHub(RepresentationMixin):
         # in the future, Queue will allow runtime subscripts.
 
         if TYPE_CHECKING:
-            comm_q: Queue[Union[Tuple[int, int], str]]
+            zmq_comm_q: Queue[Union[int, str]]
+            udp_comm_q: Queue[Union[int, str]]
         else:
-            comm_q: Queue
+            zmq_comm_q: Queue
+            udp_comm_q: Queue
 
-        comm_q = SizedQueue(maxsize=10)
+        zmq_comm_q = SizedQueue(maxsize=10)
+        udp_comm_q = SizedQueue(maxsize=10)
 
         self.exception_q: Queue[Tuple[str, str]]
         self.exception_q = SizedQueue(maxsize=10)
@@ -136,21 +137,35 @@ class MonitoringHub(RepresentationMixin):
         self.router_exit_event: ms.Event
         self.router_exit_event = Event()
 
-        self.router_proc = ForkProcess(target=router_starter,
-                                       kwargs={"comm_q": comm_q,
-                                               "exception_q": self.exception_q,
-                                               "resource_msgs": self.resource_msgs,
-                                               "exit_event": self.router_exit_event,
-                                               "hub_address": self.hub_address,
-                                               "udp_port": self.hub_port,
-                                               "zmq_port_range": self.hub_port_range,
-                                               "run_dir": dfk_run_dir,
-                                               "logging_level": logging.DEBUG if self.monitoring_debug else logging.INFO,
-                                               },
-                                       name="Monitoring-Router-Process",
-                                       daemon=True,
-                                       )
-        self.router_proc.start()
+        self.zmq_router_proc = ForkProcess(target=zmq_router_starter,
+                                           kwargs={"comm_q": zmq_comm_q,
+                                                   "exception_q": self.exception_q,
+                                                   "resource_msgs": self.resource_msgs,
+                                                   "exit_event": self.router_exit_event,
+                                                   "hub_address": self.hub_address,
+                                                   "zmq_port_range": self.hub_port_range,
+                                                   "run_dir": dfk_run_dir,
+                                                   "logging_level": logging.DEBUG if self.monitoring_debug else logging.INFO,
+                                                   },
+                                           name="Monitoring-ZMQ-Router-Process",
+                                           daemon=True,
+                                           )
+        self.zmq_router_proc.start()
+
+        self.udp_router_proc = ForkProcess(target=udp_router_starter,
+                                           kwargs={"comm_q": udp_comm_q,
+                                                   "exception_q": self.exception_q,
+                                                   "resource_msgs": self.resource_msgs,
+                                                   "exit_event": self.router_exit_event,
+                                                   "hub_address": self.hub_address,
+                                                   "udp_port": self.hub_port,
+                                                   "run_dir": dfk_run_dir,
+                                                   "logging_level": logging.DEBUG if self.monitoring_debug else logging.INFO,
+                                                   },
+                                           name="Monitoring-UDP-Router-Process",
+                                           daemon=True,
+                                           )
+        self.udp_router_proc.start()
 
         self.dbm_proc = ForkProcess(target=dbm_starter,
                                     args=(self.exception_q, self.resource_msgs,),
@@ -162,9 +177,10 @@ class MonitoringHub(RepresentationMixin):
                                     daemon=True,
                                     )
         self.dbm_proc.start()
-        logger.info("Started the router process %s and DBM process %s", self.router_proc.pid, self.dbm_proc.pid)
+        logger.info("Started ZMQ router process %s, UDP router process %s and DBM process %s",
+                    self.zmq_router_proc.pid, self.udp_router_proc.pid, self.dbm_proc.pid)
 
-        self.filesystem_proc = ForkProcess(target=filesystem_receiver,
+        self.filesystem_proc = ForkProcess(target=filesystem_router_starter,
                                            args=(self.resource_msgs, dfk_run_dir),
                                            name="Monitoring-Filesystem-Process",
                                            daemon=True
@@ -175,25 +191,36 @@ class MonitoringHub(RepresentationMixin):
         self.radio = MultiprocessingQueueRadioSender(self.resource_msgs)
 
         try:
-            comm_q_result = comm_q.get(block=True, timeout=120)
-            comm_q.close()
-            comm_q.join_thread()
+            zmq_comm_q_result = zmq_comm_q.get(block=True, timeout=120)
+            zmq_comm_q.close()
+            zmq_comm_q.join_thread()
         except queue.Empty:
-            logger.error("Hub has not completed initialization in 120s. Aborting")
+            logger.error("Monitoring ZMQ Router has not reported port in 120s. Aborting")
             raise MonitoringHubStartError()
 
-        if isinstance(comm_q_result, str):
-            logger.error("MonitoringRouter sent an error message: %s", comm_q_result)
-            raise RuntimeError(f"MonitoringRouter failed to start: {comm_q_result}")
+        if isinstance(zmq_comm_q_result, str):
+            logger.error("MonitoringRouter sent an error message: %s", zmq_comm_q_result)
+            raise RuntimeError(f"MonitoringRouter failed to start: {zmq_comm_q_result}")
+
+        self.hub_zmq_port = zmq_comm_q_result
+
+        try:
+            udp_comm_q_result = udp_comm_q.get(block=True, timeout=120)
+            udp_comm_q.close()
+            udp_comm_q.join_thread()
+        except queue.Empty:
+            logger.error("Monitoring UDP router has not reported port in 120s. Aborting")
+            raise MonitoringHubStartError()
 
-        udp_port, zmq_port = comm_q_result
+        if isinstance(udp_comm_q_result, str):
+            logger.error("MonitoringRouter sent an error message: %s", udp_comm_q_result)
+            raise RuntimeError(f"MonitoringRouter failed to start: {udp_comm_q_result}")
 
+        udp_port = udp_comm_q_result
         self.monitoring_hub_url = "udp://{}:{}".format(self.hub_address, udp_port)
 
         logger.info("Monitoring Hub initialized")
 
-        self.hub_zmq_port = zmq_port
-
     def send(self, message: TaggedMonitoringMessage) -> None:
         logger.debug("Sending message type %s", message[0])
         self.radio.send(message)
@@ -216,14 +243,21 @@ class MonitoringHub(RepresentationMixin):
                         exception_msg[0],
                         exception_msg[1]
                     )
-                self.router_proc.terminate()
+                self.zmq_router_proc.terminate()
+                self.udp_router_proc.terminate()
                 self.dbm_proc.terminate()
                 self.filesystem_proc.terminate()
             logger.info("Setting router termination event")
             self.router_exit_event.set()
-            logger.info("Waiting for router to terminate")
-            self.router_proc.join()
-            self.router_proc.close()
+
+            logger.info("Waiting for ZMQ router to terminate")
+            self.zmq_router_proc.join()
+            self.zmq_router_proc.close()
+
+            logger.info("Waiting for UDP router to terminate")
+            self.udp_router_proc.join()
+            self.udp_router_proc.close()
+
             logger.debug("Finished waiting for router termination")
             if len(exception_msgs) == 0:
                 logger.debug("Sending STOP to DBM")
@@ -248,41 +282,3 @@ class MonitoringHub(RepresentationMixin):
             self.resource_msgs.close()
             self.resource_msgs.join_thread()
             logger.info("Closed monitoring multiprocessing queues")
-
-
-@wrap_with_logs
-def filesystem_receiver(q: Queue[TaggedMonitoringMessage], run_dir: str) -> None:
-    logger = set_file_logger(f"{run_dir}/monitoring_filesystem_radio.log",
-                             name="monitoring_filesystem_radio",
-                             level=logging.INFO)
-
-    logger.info("Starting filesystem radio receiver")
-    setproctitle("parsl: monitoring filesystem receiver")
-    base_path = f"{run_dir}/monitor-fs-radio/"
-    tmp_dir = f"{base_path}/tmp/"
-    new_dir = f"{base_path}/new/"
-    logger.debug("Creating new and tmp paths under %s", base_path)
-
-    target_radio = MultiprocessingQueueRadioSender(q)
-
-    os.makedirs(tmp_dir, exist_ok=True)
-    os.makedirs(new_dir, exist_ok=True)
-
-    while True:  # this loop will end on process termination
-        logger.debug("Start filesystem radio receiver loop")
-
-        # iterate over files in new_dir
-        for filename in os.listdir(new_dir):
-            try:
-                logger.info("Processing filesystem radio file %s", filename)
-                full_path_filename = f"{new_dir}/{filename}"
-                with open(full_path_filename, "rb") as f:
-                    message = pickle.load(f)
-                logger.debug("Message received is: %s", message)
-                assert isinstance(message, tuple)
-                target_radio.send(cast(TaggedMonitoringMessage, message))
-                os.remove(full_path_filename)
-            except Exception:
-                logger.exception("Exception processing %s - probably will be retried next iteration", filename)
-
-        time.sleep(1)  # whats a good time for this poll?

parsl/monitoring/radios/filesystem_router.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+
+import logging
+import os
+import pickle
+import time
+from multiprocessing.queues import Queue
+from typing import cast
+
+from parsl.log_utils import set_file_logger
+from parsl.monitoring.radios.multiprocessing import MultiprocessingQueueRadioSender
+from parsl.monitoring.types import TaggedMonitoringMessage
+from parsl.process_loggers import wrap_with_logs
+from parsl.utils import setproctitle
+
+
+@wrap_with_logs
+def filesystem_router_starter(q: Queue[TaggedMonitoringMessage], run_dir: str) -> None:
+    logger = set_file_logger(f"{run_dir}/monitoring_filesystem_radio.log",
+                             name="monitoring_filesystem_radio",
+                             level=logging.INFO)
+
+    logger.info("Starting filesystem radio receiver")
+    setproctitle("parsl: monitoring filesystem receiver")
+    base_path = f"{run_dir}/monitor-fs-radio/"
+    tmp_dir = f"{base_path}/tmp/"
+    new_dir = f"{base_path}/new/"
+    logger.debug("Creating new and tmp paths under %s", base_path)
+
+    target_radio = MultiprocessingQueueRadioSender(q)
+
+    os.makedirs(tmp_dir, exist_ok=True)
+    os.makedirs(new_dir, exist_ok=True)
+
+    while True:  # this loop will end on process termination
+        logger.debug("Start filesystem radio receiver loop")
+
+        # iterate over files in new_dir
+        for filename in os.listdir(new_dir):
+            try:
+                logger.info("Processing filesystem radio file %s", filename)
+                full_path_filename = f"{new_dir}/{filename}"
+                with open(full_path_filename, "rb") as f:
+                    message = pickle.load(f)
+                logger.debug("Message received is: %s", message)
+                assert isinstance(message, tuple)
+                target_radio.send(cast(TaggedMonitoringMessage, message))
+                os.remove(full_path_filename)
+            except Exception:
+                logger.exception("Exception processing %s - probably will be retried next iteration", filename)
+
+        time.sleep(1)  # whats a good time for this poll?

parsl/monitoring/{router.py → radios/udp_router.py}

@@ -5,17 +5,14 @@ import multiprocessing.queues as mpq
 import os
 import pickle
 import socket
-import threading
 import time
 from multiprocessing.synchronize import Event
-from typing import Optional, Tuple
+from typing import Optional
 
 import typeguard
-import zmq
 
 from parsl.log_utils import set_file_logger
 from parsl.monitoring.radios.multiprocessing import MultiprocessingQueueRadioSender
-from parsl.monitoring.types import TaggedMonitoringMessage
 from parsl.process_loggers import wrap_with_logs
 from parsl.utils import setproctitle
 
@@ -28,7 +25,6 @@ class MonitoringRouter:
                  *,
                  hub_address: str,
                  udp_port: Optional[int] = None,
-                 zmq_port_range: Tuple[int, int] = (55050, 56000),
 
                  monitoring_hub_address: str = "127.0.0.1",
                  run_dir: str = ".",
@@ -45,9 +41,6 @@ class MonitoringRouter:
              The ip address at which the workers will be able to reach the Hub.
         udp_port : int
              The specific port at which workers will be able to reach the Hub via UDP. Default: None
-        zmq_port_range : tuple(int, int)
-             The MonitoringHub picks ports at random from the range which will be used by Hub.
-             Default: (55050, 56000)
         run_dir : str
              Parsl log directory paths. Logs and temp files go here. Default: '.'
         logging_level : int
@@ -60,7 +53,7 @@ class MonitoringRouter:
             An event that the main Parsl process will set to signal that the monitoring router should shut down.
         """
         os.makedirs(run_dir, exist_ok=True)
-        self.logger = set_file_logger(f"{run_dir}/monitoring_router.log",
+        self.logger = set_file_logger(f"{run_dir}/monitoring_udp_router.log",
                                       name="monitoring_router",
                                       level=logging_level)
         self.logger.debug("Monitoring router starting")
@@ -88,37 +81,12 @@ class MonitoringRouter:
         self.udp_sock.settimeout(self.loop_freq / 1000)
         self.logger.info("Initialized the UDP socket on 0.0.0.0:{}".format(self.udp_port))
 
-        self._context = zmq.Context()
-        self.zmq_receiver_channel = self._context.socket(zmq.DEALER)
-        self.zmq_receiver_channel.setsockopt(zmq.LINGER, 0)
-        self.zmq_receiver_channel.set_hwm(0)
-        self.zmq_receiver_channel.RCVTIMEO = int(self.loop_freq)  # in milliseconds
-        self.logger.debug("hub_address: {}. zmq_port_range {}".format(hub_address, zmq_port_range))
-        self.zmq_receiver_port = self.zmq_receiver_channel.bind_to_random_port("tcp://*",
-                                                                               min_port=zmq_port_range[0],
-                                                                               max_port=zmq_port_range[1])
-
         self.target_radio = MultiprocessingQueueRadioSender(resource_msgs)
         self.exit_event = exit_event
 
     @wrap_with_logs(target="monitoring_router")
     def start(self) -> None:
-        self.logger.info("Starting UDP listener thread")
-        udp_radio_receiver_thread = threading.Thread(target=self.start_udp_listener, daemon=True)
-        udp_radio_receiver_thread.start()
-
-        self.logger.info("Starting ZMQ listener thread")
-        zmq_radio_receiver_thread = threading.Thread(target=self.start_zmq_listener, daemon=True)
-        zmq_radio_receiver_thread.start()
-
-        self.logger.info("Joining on ZMQ listener thread")
-        zmq_radio_receiver_thread.join()
-        self.logger.info("Joining on UDP listener thread")
-        udp_radio_receiver_thread.join()
-        self.logger.info("Joined on both ZMQ and UDP listener threads")
-
-    @wrap_with_logs(target="monitoring_router")
-    def start_udp_listener(self) -> None:
+        self.logger.info("Starting UDP listener")
         try:
             while not self.exit_event.is_set():
                 try:
@@ -145,55 +113,24 @@ class MonitoringRouter:
         finally:
             self.logger.info("UDP listener finished")
 
-    @wrap_with_logs(target="monitoring_router")
-    def start_zmq_listener(self) -> None:
-        try:
-            while not self.exit_event.is_set():
-                try:
-                    dfk_loop_start = time.time()
-                    while time.time() - dfk_loop_start < 1.0:  # TODO make configurable
-                        # note that nothing checks that msg really is of the annotated type
-                        msg: TaggedMonitoringMessage
-                        msg = self.zmq_receiver_channel.recv_pyobj()
-
-                        assert isinstance(msg, tuple), "ZMQ Receiver expects only tuples, got {}".format(msg)
-                        assert len(msg) >= 1, "ZMQ Receiver expects tuples of length at least 1, got {}".format(msg)
-                        assert len(msg) == 2, "ZMQ Receiver expects message tuples of exactly length 2, got {}".format(msg)
-
-                        self.target_radio.send(msg)
-                except zmq.Again:
-                    pass
-                except Exception:
-                    # This will catch malformed messages. What happens if the
-                    # channel is broken in such a way that it always raises
-                    # an exception? Looping on this would maybe be the wrong
-                    # thing to do.
-                    self.logger.warning("Failure processing a ZMQ message", exc_info=True)
-
-            self.logger.info("ZMQ listener finishing normally")
-        finally:
-            self.logger.info("ZMQ listener finished")
-
 
 @wrap_with_logs
 @typeguard.typechecked
-def router_starter(*,
-                   comm_q: mpq.Queue,
-                   exception_q: mpq.Queue,
-                   resource_msgs: mpq.Queue,
-                   exit_event: Event,
-
-                   hub_address: str,
-                   udp_port: Optional[int],
-                   zmq_port_range: Tuple[int, int],
-
-                   run_dir: str,
-                   logging_level: int) -> None:
-    setproctitle("parsl: monitoring router")
+def udp_router_starter(*,
+                       comm_q: mpq.Queue,
+                       exception_q: mpq.Queue,
+                       resource_msgs: mpq.Queue,
+                       exit_event: Event,
+
+                       hub_address: str,
+                       udp_port: Optional[int],
+
+                       run_dir: str,
+                       logging_level: int) -> None:
+    setproctitle("parsl: monitoring UDP router")
     try:
         router = MonitoringRouter(hub_address=hub_address,
                                   udp_port=udp_port,
-                                  zmq_port_range=zmq_port_range,
                                   run_dir=run_dir,
                                   logging_level=logging_level,
                                   resource_msgs=resource_msgs,
@@ -202,11 +139,11 @@ def router_starter(*,
         logger.error("MonitoringRouter construction failed.", exc_info=True)
         comm_q.put(f"Monitoring router construction failed: {e}")
     else:
-        comm_q.put((router.udp_port, router.zmq_receiver_port))
+        comm_q.put(router.udp_port)
 
         router.logger.info("Starting MonitoringRouter in router_starter")
         try:
             router.start()
         except Exception as e:
-            router.logger.exception("router.start exception")
+            router.logger.exception("UDP router start exception")
             exception_q.put(('Hub', str(e)))

parsl/monitoring/radios/zmq_router.py

@@ -0,0 +1,138 @@
+from __future__ import annotations
+
+import logging
+import multiprocessing.queues as mpq
+import os
+import time
+from multiprocessing.synchronize import Event
+from typing import Tuple
+
+import typeguard
+import zmq
+
+from parsl.log_utils import set_file_logger
+from parsl.monitoring.radios.multiprocessing import MultiprocessingQueueRadioSender
+from parsl.monitoring.types import TaggedMonitoringMessage
+from parsl.process_loggers import wrap_with_logs
+from parsl.utils import setproctitle
+
+logger = logging.getLogger(__name__)
+
+
+class MonitoringRouter:
+
+    def __init__(self,
+                 *,
+                 hub_address: str,
+                 zmq_port_range: Tuple[int, int] = (55050, 56000),
+
+                 run_dir: str = ".",
+                 logging_level: int = logging.INFO,
+                 resource_msgs: mpq.Queue,
+                 exit_event: Event,
+                 ):
+        """ Initializes a monitoring configuration class.
+
+        Parameters
+        ----------
+        hub_address : str
+             The ip address at which the workers will be able to reach the Hub.
+        zmq_port_range : tuple(int, int)
+             The MonitoringHub picks ports at random from the range which will be used by Hub.
+             Default: (55050, 56000)
+        run_dir : str
+             Parsl log directory paths. Logs and temp files go here. Default: '.'
+        logging_level : int
+             Logging level as defined in the logging module. Default: logging.INFO
+        resource_msgs : multiprocessing.Queue
+            A multiprocessing queue to receive messages to be routed onwards to the database process
+        exit_event : Event
+            An event that the main Parsl process will set to signal that the monitoring router should shut down.
+        """
+        os.makedirs(run_dir, exist_ok=True)
+        self.logger = set_file_logger(f"{run_dir}/monitoring_zmq_router.log",
+                                      name="monitoring_router",
+                                      level=logging_level)
+        self.logger.debug("Monitoring router starting")
+
+        self.hub_address = hub_address
+
+        self.loop_freq = 10.0  # milliseconds
+
+        self._context = zmq.Context()
+        self.zmq_receiver_channel = self._context.socket(zmq.DEALER)
+        self.zmq_receiver_channel.setsockopt(zmq.LINGER, 0)
+        self.zmq_receiver_channel.set_hwm(0)
+        self.zmq_receiver_channel.RCVTIMEO = int(self.loop_freq)  # in milliseconds
+        self.logger.debug("hub_address: {}. zmq_port_range {}".format(hub_address, zmq_port_range))
+        self.zmq_receiver_port = self.zmq_receiver_channel.bind_to_random_port("tcp://*",
+                                                                               min_port=zmq_port_range[0],
+                                                                               max_port=zmq_port_range[1])
+
+        self.target_radio = MultiprocessingQueueRadioSender(resource_msgs)
+        self.exit_event = exit_event
+
+    @wrap_with_logs(target="monitoring_router")
+    def start(self) -> None:
+        self.logger.info("Starting ZMQ listener")
+        try:
+            while not self.exit_event.is_set():
+                try:
+                    dfk_loop_start = time.time()
+                    while time.time() - dfk_loop_start < 1.0:  # TODO make configurable
+                        # note that nothing checks that msg really is of the annotated type
+                        msg: TaggedMonitoringMessage
+                        msg = self.zmq_receiver_channel.recv_pyobj()
+
+                        assert isinstance(msg, tuple), "ZMQ Receiver expects only tuples, got {}".format(msg)
+                        assert len(msg) >= 1, "ZMQ Receiver expects tuples of length at least 1, got {}".format(msg)
+                        assert len(msg) == 2, "ZMQ Receiver expects message tuples of exactly length 2, got {}".format(msg)
+
+                        self.target_radio.send(msg)
+                except zmq.Again:
+                    pass
+                except Exception:
+                    # This will catch malformed messages. What happens if the
+                    # channel is broken in such a way that it always raises
+                    # an exception? Looping on this would maybe be the wrong
+                    # thing to do.
+                    self.logger.warning("Failure processing a ZMQ message", exc_info=True)
+
+            self.logger.info("ZMQ listener finishing normally")
+        finally:
+            self.logger.info("ZMQ listener finished")
+
+
+@wrap_with_logs
+@typeguard.typechecked
+def zmq_router_starter(*,
+                       comm_q: mpq.Queue,
+                       exception_q: mpq.Queue,
+                       resource_msgs: mpq.Queue,
+                       exit_event: Event,
+
+                       hub_address: str,
+                       zmq_port_range: Tuple[int, int],
+
+                       run_dir: str,
+                       logging_level: int) -> None:
+    setproctitle("parsl: monitoring zmq router")
+    try:
+        router = MonitoringRouter(hub_address=hub_address,
+                                  zmq_port_range=zmq_port_range,
+                                  run_dir=run_dir,
+                                  logging_level=logging_level,
+                                  resource_msgs=resource_msgs,
+                                  exit_event=exit_event)
+    except Exception as e:
+        logger.error("MonitoringRouter construction failed.", exc_info=True)
+        comm_q.put(f"Monitoring router construction failed: {e}")
+    else:
+        comm_q.put(router.zmq_receiver_port)
+
+        router.logger.info("Starting MonitoringRouter in router_starter")
+        try:
+            router.start()
+        except Exception as e:
+            router.logger.exception("ZMQ router start exception")
+            exception_q.put(('Hub', str(e)))

parsl/tests/test_mpi_apps/test_mpi_scheduler.py

@@ -161,3 +161,28 @@ def test_MPISched_contention():
     assert task_on_worker_side['task_id'] == 2
     _, _, _, resource_spec = unpack_res_spec_apply_message(task_on_worker_side['buffer'])
     assert len(resource_spec['MPI_NODELIST'].split(',')) == 8
+
+
+@pytest.mark.local
+def test_hashable_backlog_queue():
+    """Run multiple large tasks that to force entry into backlog_queue
+    where queue.PriorityQueue expects hashability/comparability
+    """
+
+    task_q, result_q = SpawnContext.Queue(), SpawnContext.Queue()
+    scheduler = MPITaskScheduler(task_q, result_q)
+
+    assert scheduler.available_nodes
+    assert len(scheduler.available_nodes) == 8
+
+    assert scheduler._free_node_counter.value == 8
+
+    for i in range(3):
+        mock_task_buffer = pack_res_spec_apply_message("func", "args", "kwargs",
+                                                       resource_specification={
+                                                           "num_nodes": 8,
+                                                           "ranks_per_node": 2
+                                                       })
+        task_package = {"task_id": i, "buffer": mock_task_buffer}
+        scheduler.put_task(task_package)
+    assert scheduler._backlog_queue.qsize() == 2, "Expected 2 backlogged tasks"

parsl/tests/test_scaling/test_shutdown_scalein.py

@@ -14,6 +14,9 @@ from parsl.providers import LocalProvider
 # timeout later on.
 BLOCK_COUNT = 3
 
+# the try_assert timeout for the above number of blocks to get started
+PERMITTED_STARTUP_TIME_S = 30
+
 
 class AccumulatingLocalProvider(LocalProvider):
     def __init__(self, *args, **kwargs):
@@ -67,7 +70,7 @@ def test_shutdown_scalein_blocks(tmpd_cwd, try_assert):
 
     with parsl.load(config):
         # this will wait for everything to be scaled out fully
-        try_assert(lambda: len(htex.connected_managers()) == BLOCK_COUNT)
+        try_assert(lambda: len(htex.connected_managers()) == BLOCK_COUNT, timeout_ms=PERMITTED_STARTUP_TIME_S * 1000)
 
     assert len(accumulating_provider.submit_job_ids) == BLOCK_COUNT, f"Exactly {BLOCK_COUNT} blocks should have been launched"
     assert len(accumulating_provider.cancel_job_ids) == BLOCK_COUNT, f"Exactly {BLOCK_COUNT} blocks should have been scaled in"

parsl/tests/test_shutdown/test_kill_monitoring.py

@@ -30,7 +30,7 @@ def test_no_kills():
 
 @pytest.mark.local
 @pytest.mark.parametrize("sig", [signal.SIGINT, signal.SIGTERM, signal.SIGKILL, signal.SIGQUIT])
-@pytest.mark.parametrize("process_attr", ["router_proc", "dbm_proc"])
+@pytest.mark.parametrize("process_attr", ["zmq_router_proc", "udp_router_proc", "dbm_proc", "filesystem_proc"])
 def test_kill_monitoring_helper_process(sig, process_attr, try_assert):
     """This tests that we can kill a monitoring process and still have successful shutdown.
     SIGINT emulates some racy behaviour when ctrl-C is pressed: that

parsl/version.py

@@ -3,4 +3,4 @@
 Year.Month.Day[alpha/beta/..]
 Alphas will be numbered like this -> 2024.12.10a0
 """
-VERSION = '2025.03.03'
+VERSION = '2025.03.10'

{parsl-2025.3.3.dist-info → parsl-2025.3.10.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: parsl
-Version: 2025.3.3
+Version: 2025.3.10
 Summary: Simple data dependent workflows in Python
 Home-page: https://github.com/Parsl/parsl
-Download-URL: https://github.com/Parsl/parsl/archive/2025.03.03.tar.gz
+Download-URL: https://github.com/Parsl/parsl/archive/2025.03.10.tar.gz
 Author: The Parsl Team
 Author-email: parsl@googlegroups.com
 License: Apache 2.0

{parsl-2025.3.3.dist-info → parsl-2025.3.10.dist-info}/RECORD

@@ -8,7 +8,7 @@ parsl/multiprocessing.py,sha256=MyaEcEq-Qf860u7V98u-PZrPNdtzOZL_NW6EhIJnmfQ,1937
 parsl/process_loggers.py,sha256=uQ7Gd0W72Jz7rrcYlOMfLsAEhkRltxXJL2MgdduJjEw,1136
 parsl/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 parsl/utils.py,sha256=codTX6_KLhgeTwNkRzc1lo4bgc1M93eJ-lkqOO98fvk,14331
-parsl/version.py,sha256=JZR2YCezBq1F5cw4-KEsJxZK4DKSK_Po-wBKZDC4T7o,131
+parsl/version.py,sha256=3cSnT_xfCul6H60imXWe7VlUXG29OzzoAFknr7Fc3TQ,131
 parsl/app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 parsl/app/app.py,sha256=0gbM4AH2OtFOLsv07I5nglpElcwMSOi-FzdZZfrk7So,8532
 parsl/app/bash.py,sha256=jm2AvePlCT9DZR7H_4ANDWxatp5dN_22FUlT_gWhZ-g,5528
@@ -73,14 +73,14 @@ parsl/executors/flux/executor.py,sha256=8_xakLUu5zNJAHL0LbeTCFEWqWzRK1eE-3ep4GII
 parsl/executors/flux/flux_instance_manager.py,sha256=5T3Rp7ZM-mlT0Pf0Gxgs5_YmnaPrSF9ec7zvRfLfYJw,2129
 parsl/executors/high_throughput/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 parsl/executors/high_throughput/errors.py,sha256=k2XuvvFdUfNs2foHFnxmS-BToRMfdXpYEa4EF3ELKq4,1554
-parsl/executors/high_throughput/executor.py,sha256=Otf0k9Ia_ZX9mFBs6HOfF4x1LALlwG-i_ga_RsrUBJY,38747
+parsl/executors/high_throughput/executor.py,sha256=esMYMgPHmgD0wPTb0U61vBX96DaPrWj9sQLzpKvB06k,38752
 parsl/executors/high_throughput/interchange.py,sha256=7sKIvxP3a7HSzqEq25ZCpABx-1Q2f585pFDGzUvo7_4,29459
 parsl/executors/high_throughput/manager_record.py,sha256=ZMsqFxvreGLRXAw3N-JnODDa9Qfizw2tMmcBhm4lco4,490
 parsl/executors/high_throughput/manager_selector.py,sha256=UKcUE6v0tO7PDMTThpKSKxVpOpOUilxDL7UbNgpZCxo,2116
 parsl/executors/high_throughput/monitoring_info.py,sha256=HC0drp6nlXQpAop5PTUKNjdXMgtZVvrBL0JzZJebPP4,298
 parsl/executors/high_throughput/mpi_executor.py,sha256=U-aatbLF_Mu1p6lP0HmT7Yn1Swn3cc7hPmDfuUb9TpI,4797
 parsl/executors/high_throughput/mpi_prefix_composer.py,sha256=DmpKugANNa1bdYlqQBLHkrFc15fJpefPPhW9hkAlh1s,4308
-parsl/executors/high_throughput/mpi_resource_management.py,sha256=d3NSt3-isdr7pj-oXg9XGRX9D9VsK5e9zSpp7-nyybc,7854
+parsl/executors/high_throughput/mpi_resource_management.py,sha256=xeJp4h4LysG8KuBLKqy1sgFahL1eqiG7XLpr09VLwy4,8144
 parsl/executors/high_throughput/probe.py,sha256=QOEaliO3x5cB6ltMOZMsZQ-ath9AAuFqXcBzRgWOM60,2754
 parsl/executors/high_throughput/process_worker_pool.py,sha256=YOJvTUMg3eIHr9fYfBWFHRiI1QQ898IGiuXyj5VRQNo,41084
 parsl/executors/high_throughput/zmq_pipes.py,sha256=NUK25IEh0UkxzdqQQyM8tMtuZmjSiTeWu1DzkkAIOhA,8980
@@ -117,19 +117,21 @@ parsl/monitoring/__init__.py,sha256=0ywNz6i0lM1xo_7_BIxhETDGeVd2C_0wwD7qgeaMR4c,
 parsl/monitoring/db_manager.py,sha256=ra5PqmbUstfDx0o_bkBYI8GIUi461-GV3b4A-Q6DVVE,33300
 parsl/monitoring/errors.py,sha256=D6jpYzEzp0d6FmVKGqhvjAxr4ztZfJX2s-aXemH9bBU,148
 parsl/monitoring/message_type.py,sha256=Khn88afNxcOIciKiCK4GLnn90I5BlRTiOL3zK-P07yQ,401
-parsl/monitoring/monitoring.py,sha256=fkBZU4fWp7qBQUcKYtWjd4d-SsFlJUZNMZacFOh0IoA,12687
+parsl/monitoring/monitoring.py,sha256=p79F982lyPsplXeTVxqlvNuB8G1p3PAI8nTMHcZJ5UE,13113
 parsl/monitoring/remote.py,sha256=t0qCTUMCzeJ_JOARFpjqlTNrAWdEb20BxhmZh9X7kEM,13728
-parsl/monitoring/router.py,sha256=GUNdvixMcVGITk5LHEfgtbKBE7DqRtQIO-fkR4Z_VYM,9224
 parsl/monitoring/types.py,sha256=oOCrzv-ab-_rv4pb8o58Sdb8G_RGp1aZriRbdf9zBEk,339
 parsl/monitoring/queries/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 parsl/monitoring/queries/pandas.py,sha256=0Z2r0rjTKCemf0eaDkF1irvVHn5g7KC5SYETvQPRxwU,2232
 parsl/monitoring/radios/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 parsl/monitoring/radios/base.py,sha256=Ep5kHf07Sm-ApMBJVudRhoWRyuiu0udjO4NvEir5LEk,291
 parsl/monitoring/radios/filesystem.py,sha256=ioZ3jOKX5Qf0DYRtWmpCEorfuMVbS58OMS_QV7DOFOs,1765
+parsl/monitoring/radios/filesystem_router.py,sha256=ZxPImntYHw-3arvCTMYOC67kUgkyk7I7XLDrCXnvkBw,2055
 parsl/monitoring/radios/htex.py,sha256=qBu4O5NYnSETHX0ptdwxSpqa2Pp3Z_V6a6lb3TbjKm4,1643
 parsl/monitoring/radios/multiprocessing.py,sha256=fsfaaoMDp6VJv1DSAl-P0R2ofO6jp13byx6NsPItV3Y,655
 parsl/monitoring/radios/udp.py,sha256=bTpt7JYp-5hyBBLzgiLj1_BlSTn28UVp39OYgVGLXCw,1613
+parsl/monitoring/radios/udp_router.py,sha256=Dtat4lVNz4cpnzZmXTjo5VA1Xcri5VTSNNpyepSjIVE,5868
 parsl/monitoring/radios/zmq.py,sha256=fhoHp9ylhf-D3eTJb2aSHRsuic8-FJ_oRNGnniGkCAI,592
+parsl/monitoring/radios/zmq_router.py,sha256=oKfMg_dc3UxJcSzDe1ZqkGJYQcOa4somvyGPzwOqQuA,5860
 parsl/monitoring/visualization/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 parsl/monitoring/visualization/app.py,sha256=xMeRlAnzl5lHddAOdSBcqY3D5lmOYw3Z3Z2_YyoVwnw,1425
 parsl/monitoring/visualization/models.py,sha256=C7CcF6w6PhtrdvDX9VgDH-aSrpLfvYU1fJ4-HDUeFVQ,5138
@@ -347,7 +349,7 @@ parsl/tests/test_mpi_apps/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJW
 parsl/tests/test_mpi_apps/test_bad_mpi_config.py,sha256=QKvEUSrHIBrvqu2fRj1MAqxsYxDfcrdQ7dzWdOZejuU,1320
 parsl/tests/test_mpi_apps/test_mpi_mode_enabled.py,sha256=_fpiaDq9yEUuBxTiuxLFsBt5r1oX9S-3S-YL5yRB13E,5423
 parsl/tests/test_mpi_apps/test_mpi_prefix.py,sha256=yJslZvYK3JeL9UgxMwF9DDPR9QD4zJLGVjubD0F-utc,1950
-parsl/tests/test_mpi_apps/test_mpi_scheduler.py,sha256=YdV8A-m67DHk9wxgNpj69wwGEKrFGL20KAC1TzLke3c,6332
+parsl/tests/test_mpi_apps/test_mpi_scheduler.py,sha256=LPvk5wywYANQNCoQ8muwOLEznnZqwler4jJglinAT9I,7370
 parsl/tests/test_mpi_apps/test_mpiex.py,sha256=mlFdHK3A1B6NsEhxTQQX8lhs9qVza36FMG99vNrBRW4,2021
 parsl/tests/test_mpi_apps/test_resource_spec.py,sha256=5k6HM2jtb6sa7jetpI-Tl1nPQiN33VLaM7YT10c307E,3756
 parsl/tests/test_providers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -409,7 +411,7 @@ parsl/tests/test_scaling/test_regression_3696_oscillation.py,sha256=7Xc3vgocXXUb
 parsl/tests/test_scaling/test_scale_down.py,sha256=vHJOMRUriW6xPtaY8GTKYXd5P0WJkQV6Q1IPui05aLU,2736
 parsl/tests/test_scaling/test_scale_down_htex_auto_scale.py,sha256=EnVNllKO2AGKkGa6927cLrzvvG6mpNQeFDzVktv6x08,4521
 parsl/tests/test_scaling/test_scale_down_htex_unregistered.py,sha256=OrdnYmd58n7UfkANPJ7mzha4WSCPdbgJRX1O1Zdu0tI,1954
-parsl/tests/test_scaling/test_shutdown_scalein.py,sha256=QMlby0g4SgRUqFYZy-d80a23L8FmYl_dwse67E86oVs,2325
+parsl/tests/test_scaling/test_shutdown_scalein.py,sha256=sr40of5DwxeyQI97MDZxFqJILZSXZJb9Dv7qTf2gql8,2471
 parsl/tests/test_scaling/test_worker_interchange_bad_messages_3262.py,sha256=GaXmRli1srTal-JQmCGDTP4BAwAKI_daXMmrjULsZkY,2788
 parsl/tests/test_serialization/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 parsl/tests/test_serialization/test_2555_caching_deserializer.py,sha256=jEXJvbriaLVI7frV5t-iJRKYyeQ7a9_-t3X9lhhBWQo,767
@@ -420,7 +422,7 @@ parsl/tests/test_serialization/test_pack_resource_spec.py,sha256=-Vtyh8KyezZw8e7
 parsl/tests/test_serialization/test_proxystore_configured.py,sha256=lGWOSEWul16enDWhW-s7CK0d3eMDzm1324Fmj0cZMVU,2293
 parsl/tests/test_serialization/test_proxystore_impl.py,sha256=uGd45sfPm9rJhzqKV0rI3lqdSOAUddQf-diEpcJAlcY,1228
 parsl/tests/test_shutdown/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-parsl/tests/test_shutdown/test_kill_monitoring.py,sha256=S9CnCziBk3sQMKgccqvNUEBHanf1hWMK1SLc2aF8uWs,1906
+parsl/tests/test_shutdown/test_kill_monitoring.py,sha256=BycTDLwxhHbbV68Qkgrmn8UUzSr55SvbNvydp35UCTM,1948
 parsl/tests/test_staging/__init__.py,sha256=WZl9EHSkfYiSoE3Gbulcq2ifmn7IFGUkasJIobL5T5A,208
 parsl/tests/test_staging/staging_provider.py,sha256=6FDpImkWOLgysqM68NbCAoXZciZokI8dmBWRAxnggBk,3242
 parsl/tests/test_staging/test_1316.py,sha256=eS0e2BDM2vmPNF60aDr35wcuGgDPfXjTjRV6kyBZOQc,2652
@@ -455,13 +457,13 @@ parsl/usage_tracking/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hS
 parsl/usage_tracking/api.py,sha256=iaCY58Dc5J4UM7_dJzEEs871P1p1HdxBMtNGyVdzc9g,1821
 parsl/usage_tracking/levels.py,sha256=xbfzYEsd55KiZJ-mzNgPebvOH4rRHum04hROzEf41tU,291
 parsl/usage_tracking/usage.py,sha256=f9k6QcpbQxkGyP5WTC9PVyv0CA05s9NDpRe5wwRdBTM,9163
-parsl-2025.3.3.data/scripts/exec_parsl_function.py,sha256=YXKVVIa4zXmOtz-0Ca4E_5nQfN_3S2bh2tB75uZZB4w,7774
-parsl-2025.3.3.data/scripts/interchange.py,sha256=17MrOc7-FXxKBWTwkzIbUoa8fvvDfPelfjByd3ZD2Wk,29446
-parsl-2025.3.3.data/scripts/parsl_coprocess.py,sha256=zrVjEqQvFOHxsLufPi00xzMONagjVwLZbavPM7bbjK4,5722
-parsl-2025.3.3.data/scripts/process_worker_pool.py,sha256=BbVJ1PS7ZW2grz0iAPPV0BgJyRMyQ7bbXSzLzWCBkyU,41070
-parsl-2025.3.3.dist-info/LICENSE,sha256=tAkwu8-AdEyGxGoSvJ2gVmQdcicWw3j1ZZueVV74M-E,11357
-parsl-2025.3.3.dist-info/METADATA,sha256=FiDSNMMf3JHayeZGotfxx9jG2XcVNwbF50cPkEtLBc8,4026
-parsl-2025.3.3.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-parsl-2025.3.3.dist-info/entry_points.txt,sha256=XqnsWDYoEcLbsMcpnYGKLEnSBmaIe1YoM5YsBdJG2tI,176
-parsl-2025.3.3.dist-info/top_level.txt,sha256=PIheYoUFQtF2icLsgOykgU-Cjuwr2Oi6On2jo5RYgRM,6
-parsl-2025.3.3.dist-info/RECORD,,
+parsl-2025.3.10.data/scripts/exec_parsl_function.py,sha256=YXKVVIa4zXmOtz-0Ca4E_5nQfN_3S2bh2tB75uZZB4w,7774
+parsl-2025.3.10.data/scripts/interchange.py,sha256=17MrOc7-FXxKBWTwkzIbUoa8fvvDfPelfjByd3ZD2Wk,29446
+parsl-2025.3.10.data/scripts/parsl_coprocess.py,sha256=zrVjEqQvFOHxsLufPi00xzMONagjVwLZbavPM7bbjK4,5722
+parsl-2025.3.10.data/scripts/process_worker_pool.py,sha256=BbVJ1PS7ZW2grz0iAPPV0BgJyRMyQ7bbXSzLzWCBkyU,41070
+parsl-2025.3.10.dist-info/LICENSE,sha256=tAkwu8-AdEyGxGoSvJ2gVmQdcicWw3j1ZZueVV74M-E,11357
+parsl-2025.3.10.dist-info/METADATA,sha256=rEtmq9LYtfBbXFR2JuX9DmmPQyDshdmj0GuakTMQeSM,4027
+parsl-2025.3.10.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+parsl-2025.3.10.dist-info/entry_points.txt,sha256=XqnsWDYoEcLbsMcpnYGKLEnSBmaIe1YoM5YsBdJG2tI,176
+parsl-2025.3.10.dist-info/top_level.txt,sha256=PIheYoUFQtF2icLsgOykgU-Cjuwr2Oi6On2jo5RYgRM,6
+parsl-2025.3.10.dist-info/RECORD,,