parsl 2025.3.3__py3-none-any.whl → 2025.3.17__py3-none-any.whl

parsl/monitoring/monitoring.py

@@ -3,23 +3,22 @@ from __future__ import annotations
 import logging
 import multiprocessing.synchronize as ms
 import os
-import pickle
 import queue
-import time
 from multiprocessing import Event
+from multiprocessing.context import ForkProcess as ForkProcessType
 from multiprocessing.queues import Queue
-from typing import TYPE_CHECKING, Literal, Optional, Tuple, Union, cast
+from typing import TYPE_CHECKING, Optional, Tuple, Union
 
 import typeguard
 
-from parsl.log_utils import set_file_logger
 from parsl.monitoring.errors import MonitoringHubStartError
+from parsl.monitoring.radios.filesystem_router import filesystem_router_starter
 from parsl.monitoring.radios.multiprocessing import MultiprocessingQueueRadioSender
-from parsl.monitoring.router import router_starter
+from parsl.monitoring.radios.udp_router import udp_router_starter
+from parsl.monitoring.radios.zmq_router import zmq_router_starter
 from parsl.monitoring.types import TaggedMonitoringMessage
 from parsl.multiprocessing import ForkProcess, SizedQueue
-from parsl.process_loggers import wrap_with_logs
-from parsl.utils import RepresentationMixin, setproctitle
+from parsl.utils import RepresentationMixin
 
 _db_manager_excepts: Optional[Exception]
 
@@ -121,51 +120,68 @@ class MonitoringHub(RepresentationMixin):
         # in the future, Queue will allow runtime subscripts.
 
         if TYPE_CHECKING:
-            comm_q: Queue[Union[Tuple[int, int], str]]
+            zmq_comm_q: Queue[Union[int, str]]
+            udp_comm_q: Queue[Union[int, str]]
         else:
-            comm_q: Queue
+            zmq_comm_q: Queue
+            udp_comm_q: Queue
 
-        comm_q = SizedQueue(maxsize=10)
+        zmq_comm_q = SizedQueue(maxsize=10)
+        udp_comm_q = SizedQueue(maxsize=10)
 
-        self.exception_q: Queue[Tuple[str, str]]
-        self.exception_q = SizedQueue(maxsize=10)
-
-        self.resource_msgs: Queue[Union[TaggedMonitoringMessage, Literal["STOP"]]]
+        self.resource_msgs: Queue[TaggedMonitoringMessage]
         self.resource_msgs = SizedQueue()
 
         self.router_exit_event: ms.Event
         self.router_exit_event = Event()
 
-        self.router_proc = ForkProcess(target=router_starter,
-                                       kwargs={"comm_q": comm_q,
-                                               "exception_q": self.exception_q,
-                                               "resource_msgs": self.resource_msgs,
-                                               "exit_event": self.router_exit_event,
-                                               "hub_address": self.hub_address,
-                                               "udp_port": self.hub_port,
-                                               "zmq_port_range": self.hub_port_range,
-                                               "run_dir": dfk_run_dir,
-                                               "logging_level": logging.DEBUG if self.monitoring_debug else logging.INFO,
-                                               },
-                                       name="Monitoring-Router-Process",
-                                       daemon=True,
-                                       )
-        self.router_proc.start()
+        self.zmq_router_proc = ForkProcess(target=zmq_router_starter,
+                                           kwargs={"comm_q": zmq_comm_q,
+                                                   "resource_msgs": self.resource_msgs,
+                                                   "exit_event": self.router_exit_event,
+                                                   "hub_address": self.hub_address,
+                                                   "zmq_port_range": self.hub_port_range,
+                                                   "run_dir": dfk_run_dir,
+                                                   "logging_level": logging.DEBUG if self.monitoring_debug else logging.INFO,
+                                                   },
+                                           name="Monitoring-ZMQ-Router-Process",
+                                           daemon=True,
+                                           )
+        self.zmq_router_proc.start()
+
+        self.udp_router_proc = ForkProcess(target=udp_router_starter,
+                                           kwargs={"comm_q": udp_comm_q,
+                                                   "resource_msgs": self.resource_msgs,
+                                                   "exit_event": self.router_exit_event,
+                                                   "hub_address": self.hub_address,
+                                                   "udp_port": self.hub_port,
+                                                   "run_dir": dfk_run_dir,
+                                                   "logging_level": logging.DEBUG if self.monitoring_debug else logging.INFO,
+                                                   },
+                                           name="Monitoring-UDP-Router-Process",
+                                           daemon=True,
+                                           )
+        self.udp_router_proc.start()
+
+        self.dbm_exit_event: ms.Event
+        self.dbm_exit_event = Event()
 
         self.dbm_proc = ForkProcess(target=dbm_starter,
-                                    args=(self.exception_q, self.resource_msgs,),
+                                    args=(self.resource_msgs,),
                                     kwargs={"run_dir": dfk_run_dir,
                                             "logging_level": logging.DEBUG if self.monitoring_debug else logging.INFO,
                                             "db_url": self.logging_endpoint,
+                                            "exit_event": self.dbm_exit_event,
                                             },
                                     name="Monitoring-DBM-Process",
                                     daemon=True,
                                     )
         self.dbm_proc.start()
-        logger.info("Started the router process %s and DBM process %s", self.router_proc.pid, self.dbm_proc.pid)
+        logger.info("Started ZMQ router process %s, UDP router process %s and DBM process %s",
+                    self.zmq_router_proc.pid, self.udp_router_proc.pid, self.dbm_proc.pid)
 
-        self.filesystem_proc = ForkProcess(target=filesystem_receiver,
-                                           args=(self.resource_msgs, dfk_run_dir),
+        self.filesystem_proc = ForkProcess(target=filesystem_router_starter,
+                                           args=(self.resource_msgs, dfk_run_dir, self.router_exit_event),
                                            name="Monitoring-Filesystem-Process",
                                            daemon=True
                                            )
@@ -175,114 +191,98 @@ class MonitoringHub(RepresentationMixin):
         self.radio = MultiprocessingQueueRadioSender(self.resource_msgs)
 
         try:
-            comm_q_result = comm_q.get(block=True, timeout=120)
-            comm_q.close()
-            comm_q.join_thread()
+            zmq_comm_q_result = zmq_comm_q.get(block=True, timeout=120)
+            zmq_comm_q.close()
+            zmq_comm_q.join_thread()
         except queue.Empty:
-            logger.error("Hub has not completed initialization in 120s. Aborting")
+            logger.error("Monitoring ZMQ Router has not reported port in 120s. Aborting")
             raise MonitoringHubStartError()
 
-        if isinstance(comm_q_result, str):
-            logger.error("MonitoringRouter sent an error message: %s", comm_q_result)
-            raise RuntimeError(f"MonitoringRouter failed to start: {comm_q_result}")
+        if isinstance(zmq_comm_q_result, str):
+            logger.error("MonitoringRouter sent an error message: %s", zmq_comm_q_result)
+            raise RuntimeError(f"MonitoringRouter failed to start: {zmq_comm_q_result}")
 
-        udp_port, zmq_port = comm_q_result
+        self.hub_zmq_port = zmq_comm_q_result
 
+        try:
+            udp_comm_q_result = udp_comm_q.get(block=True, timeout=120)
+            udp_comm_q.close()
+            udp_comm_q.join_thread()
+        except queue.Empty:
+            logger.error("Monitoring UDP router has not reported port in 120s. Aborting")
+            raise MonitoringHubStartError()
+
+        if isinstance(udp_comm_q_result, str):
+            logger.error("MonitoringRouter sent an error message: %s", udp_comm_q_result)
+            raise RuntimeError(f"MonitoringRouter failed to start: {udp_comm_q_result}")
+
+        udp_port = udp_comm_q_result
         self.monitoring_hub_url = "udp://{}:{}".format(self.hub_address, udp_port)
 
         logger.info("Monitoring Hub initialized")
 
-        self.hub_zmq_port = zmq_port
-
     def send(self, message: TaggedMonitoringMessage) -> None:
         logger.debug("Sending message type %s", message[0])
         self.radio.send(message)
 
     def close(self) -> None:
         logger.info("Terminating Monitoring Hub")
-        exception_msgs = []
-        while True:
-            try:
-                exception_msgs.append(self.exception_q.get(block=False))
-                logger.error("There was a queued exception (Either router or DBM process got exception much earlier?)")
-            except queue.Empty:
-                break
         if self.monitoring_hub_active:
             self.monitoring_hub_active = False
-            if exception_msgs:
-                for exception_msg in exception_msgs:
-                    logger.error(
-                        "%s process delivered an exception: %s. Terminating all monitoring processes immediately.",
-                        exception_msg[0],
-                        exception_msg[1]
-                    )
-                self.router_proc.terminate()
-                self.dbm_proc.terminate()
-                self.filesystem_proc.terminate()
             logger.info("Setting router termination event")
             self.router_exit_event.set()
-            logger.info("Waiting for router to terminate")
-            self.router_proc.join()
-            self.router_proc.close()
+
+            logger.info("Waiting for ZMQ router to terminate")
+            join_terminate_close_proc(self.zmq_router_proc)
+
+            logger.info("Waiting for UDP router to terminate")
+            join_terminate_close_proc(self.udp_router_proc)
+
             logger.debug("Finished waiting for router termination")
-            if len(exception_msgs) == 0:
-                logger.debug("Sending STOP to DBM")
-                self.resource_msgs.put("STOP")
-            else:
-                logger.debug("Not sending STOP to DBM, because there were DBM exceptions")
             logger.debug("Waiting for DB termination")
-            self.dbm_proc.join()
-            self.dbm_proc.close()
+            self.dbm_exit_event.set()
+            join_terminate_close_proc(self.dbm_proc)
             logger.debug("Finished waiting for DBM termination")
 
-            # should this be message based? it probably doesn't need to be if
-            # we believe we've received all messages
             logger.info("Terminating filesystem radio receiver process")
-            self.filesystem_proc.terminate()
-            self.filesystem_proc.join()
-            self.filesystem_proc.close()
+            join_terminate_close_proc(self.filesystem_proc)
 
             logger.info("Closing monitoring multiprocessing queues")
-            self.exception_q.close()
-            self.exception_q.join_thread()
             self.resource_msgs.close()
             self.resource_msgs.join_thread()
             logger.info("Closed monitoring multiprocessing queues")
 
 
-@wrap_with_logs
-def filesystem_receiver(q: Queue[TaggedMonitoringMessage], run_dir: str) -> None:
-    logger = set_file_logger(f"{run_dir}/monitoring_filesystem_radio.log",
-                             name="monitoring_filesystem_radio",
-                             level=logging.INFO)
-
-    logger.info("Starting filesystem radio receiver")
-    setproctitle("parsl: monitoring filesystem receiver")
-    base_path = f"{run_dir}/monitor-fs-radio/"
-    tmp_dir = f"{base_path}/tmp/"
-    new_dir = f"{base_path}/new/"
-    logger.debug("Creating new and tmp paths under %s", base_path)
-
-    target_radio = MultiprocessingQueueRadioSender(q)
-
-    os.makedirs(tmp_dir, exist_ok=True)
-    os.makedirs(new_dir, exist_ok=True)
-
-    while True:  # this loop will end on process termination
-        logger.debug("Start filesystem radio receiver loop")
-
-        # iterate over files in new_dir
-        for filename in os.listdir(new_dir):
-            try:
-                logger.info("Processing filesystem radio file %s", filename)
-                full_path_filename = f"{new_dir}/{filename}"
-                with open(full_path_filename, "rb") as f:
-                    message = pickle.load(f)
-                logger.debug("Message received is: %s", message)
-                assert isinstance(message, tuple)
-                target_radio.send(cast(TaggedMonitoringMessage, message))
-                os.remove(full_path_filename)
-            except Exception:
-                logger.exception("Exception processing %s - probably will be retried next iteration", filename)
-
-        time.sleep(1)  # whats a good time for this poll?
+def join_terminate_close_proc(process: ForkProcessType, *, timeout: int = 30) -> None:
+    """Increasingly aggressively terminate a process.
+
+    This function assumes that the process is likely to exit before
+    the join timeout, driven by some other means, such as the
+    MonitoringHub router_exit_event. If the process does not exit, then
+    first terminate() and then kill() will be used to end the process.
+
+    In the case of a very mis-behaving process, this function might take
+    up to 3*timeout to exhaust all termination methods and return.
+    """
+    logger.debug("Joining process")
+    process.join(timeout)
+
+    # run a sequence of increasingly aggressive steps to shut down the process.
+    if process.is_alive():
+        logger.error("Process did not join. Terminating.")
+        process.terminate()
+        process.join(timeout)
+        if process.is_alive():
+            logger.error("Process did not join after terminate. Killing.")
+            process.kill()
+            process.join(timeout)
+            # This kill should not be caught by any signal handlers so it is
+            # unlikely that this join will timeout. If it does, there isn't
+            # anything further to do except log an error in the next if-block.
+
+    if process.is_alive():
+        logger.error("Process failed to end")
+        # don't call close if the process hasn't ended:
+        # process.close() doesn't work on a running process.
+    else:
+        process.close()

parsl/monitoring/radios/filesystem_router.py

@@ -0,0 +1,54 @@
+from __future__ import annotations
+
+import logging
+import os
+import pickle
+import time
+from multiprocessing.queues import Queue
+from multiprocessing.synchronize import Event
+from typing import cast
+
+from parsl.log_utils import set_file_logger
+from parsl.monitoring.radios.multiprocessing import MultiprocessingQueueRadioSender
+from parsl.monitoring.types import TaggedMonitoringMessage
+from parsl.process_loggers import wrap_with_logs
+from parsl.utils import setproctitle
+
+
+@wrap_with_logs
+def filesystem_router_starter(q: Queue[TaggedMonitoringMessage], run_dir: str, exit_event: Event) -> None:
+    logger = set_file_logger(f"{run_dir}/monitoring_filesystem_radio.log",
+                             name="monitoring_filesystem_radio",
+                             level=logging.INFO)
+
+    logger.info("Starting filesystem radio receiver")
+    setproctitle("parsl: monitoring filesystem receiver")
+    base_path = f"{run_dir}/monitor-fs-radio/"
+    tmp_dir = f"{base_path}/tmp/"
+    new_dir = f"{base_path}/new/"
+    logger.debug("Creating new and tmp paths under %s", base_path)
+
+    target_radio = MultiprocessingQueueRadioSender(q)
+
+    os.makedirs(tmp_dir, exist_ok=True)
+    os.makedirs(new_dir, exist_ok=True)
+
+    while not exit_event.is_set():
+        logger.debug("Start filesystem radio receiver loop")
+
+        # iterate over files in new_dir
+        for filename in os.listdir(new_dir):
+            try:
+                logger.info("Processing filesystem radio file %s", filename)
+                full_path_filename = f"{new_dir}/{filename}"
+                with open(full_path_filename, "rb") as f:
+                    message = pickle.load(f)
+                logger.debug("Message received is: %s", message)
+                assert isinstance(message, tuple)
+                target_radio.send(cast(TaggedMonitoringMessage, message))
+                os.remove(full_path_filename)
+            except Exception:
+                logger.exception("Exception processing %s - probably will be retried next iteration", filename)
+
+        time.sleep(1)  # whats a good time for this poll?
+    logger.info("Ending filesystem radio receiver")

parsl/monitoring/{router.py → radios/udp_router.py}

@@ -5,17 +5,14 @@ import multiprocessing.queues as mpq
 import os
 import pickle
 import socket
-import threading
 import time
 from multiprocessing.synchronize import Event
-from typing import Optional, Tuple
+from typing import Optional
 
 import typeguard
-import zmq
 
 from parsl.log_utils import set_file_logger
 from parsl.monitoring.radios.multiprocessing import MultiprocessingQueueRadioSender
-from parsl.monitoring.types import TaggedMonitoringMessage
 from parsl.process_loggers import wrap_with_logs
 from parsl.utils import setproctitle
 
@@ -28,7 +25,6 @@ class MonitoringRouter:
                  *,
                  hub_address: str,
                  udp_port: Optional[int] = None,
-                 zmq_port_range: Tuple[int, int] = (55050, 56000),
 
                  monitoring_hub_address: str = "127.0.0.1",
                  run_dir: str = ".",
@@ -45,9 +41,6 @@ class MonitoringRouter:
              The ip address at which the workers will be able to reach the Hub.
         udp_port : int
              The specific port at which workers will be able to reach the Hub via UDP. Default: None
-        zmq_port_range : tuple(int, int)
-             The MonitoringHub picks ports at random from the range which will be used by Hub.
-             Default: (55050, 56000)
         run_dir : str
              Parsl log directory paths. Logs and temp files go here. Default: '.'
         logging_level : int
@@ -60,7 +53,7 @@ class MonitoringRouter:
             An event that the main Parsl process will set to signal that the monitoring router should shut down.
         """
         os.makedirs(run_dir, exist_ok=True)
-        self.logger = set_file_logger(f"{run_dir}/monitoring_router.log",
+        self.logger = set_file_logger(f"{run_dir}/monitoring_udp_router.log",
                                       name="monitoring_router",
                                       level=logging_level)
         self.logger.debug("Monitoring router starting")
@@ -88,37 +81,12 @@ class MonitoringRouter:
         self.udp_sock.settimeout(self.loop_freq / 1000)
         self.logger.info("Initialized the UDP socket on 0.0.0.0:{}".format(self.udp_port))
 
-        self._context = zmq.Context()
-        self.zmq_receiver_channel = self._context.socket(zmq.DEALER)
-        self.zmq_receiver_channel.setsockopt(zmq.LINGER, 0)
-        self.zmq_receiver_channel.set_hwm(0)
-        self.zmq_receiver_channel.RCVTIMEO = int(self.loop_freq)  # in milliseconds
-        self.logger.debug("hub_address: {}. zmq_port_range {}".format(hub_address, zmq_port_range))
-        self.zmq_receiver_port = self.zmq_receiver_channel.bind_to_random_port("tcp://*",
-                                                                               min_port=zmq_port_range[0],
-                                                                               max_port=zmq_port_range[1])
-
         self.target_radio = MultiprocessingQueueRadioSender(resource_msgs)
         self.exit_event = exit_event
 
     @wrap_with_logs(target="monitoring_router")
     def start(self) -> None:
-        self.logger.info("Starting UDP listener thread")
-        udp_radio_receiver_thread = threading.Thread(target=self.start_udp_listener, daemon=True)
-        udp_radio_receiver_thread.start()
-
-        self.logger.info("Starting ZMQ listener thread")
-        zmq_radio_receiver_thread = threading.Thread(target=self.start_zmq_listener, daemon=True)
-        zmq_radio_receiver_thread.start()
-
-        self.logger.info("Joining on ZMQ listener thread")
-        zmq_radio_receiver_thread.join()
-        self.logger.info("Joining on UDP listener thread")
-        udp_radio_receiver_thread.join()
-        self.logger.info("Joined on both ZMQ and UDP listener threads")
-
-    @wrap_with_logs(target="monitoring_router")
-    def start_udp_listener(self) -> None:
+        self.logger.info("Starting UDP listener")
         try:
             while not self.exit_event.is_set():
                 try:
@@ -145,55 +113,23 @@ class MonitoringRouter:
         finally:
             self.logger.info("UDP listener finished")
 
-    @wrap_with_logs(target="monitoring_router")
-    def start_zmq_listener(self) -> None:
-        try:
-            while not self.exit_event.is_set():
-                try:
-                    dfk_loop_start = time.time()
-                    while time.time() - dfk_loop_start < 1.0:  # TODO make configurable
-                        # note that nothing checks that msg really is of the annotated type
-                        msg: TaggedMonitoringMessage
-                        msg = self.zmq_receiver_channel.recv_pyobj()
-
-                        assert isinstance(msg, tuple), "ZMQ Receiver expects only tuples, got {}".format(msg)
-                        assert len(msg) >= 1, "ZMQ Receiver expects tuples of length at least 1, got {}".format(msg)
-                        assert len(msg) == 2, "ZMQ Receiver expects message tuples of exactly length 2, got {}".format(msg)
-
-                        self.target_radio.send(msg)
-                except zmq.Again:
-                    pass
-                except Exception:
-                    # This will catch malformed messages. What happens if the
-                    # channel is broken in such a way that it always raises
-                    # an exception? Looping on this would maybe be the wrong
-                    # thing to do.
-                    self.logger.warning("Failure processing a ZMQ message", exc_info=True)
-
-            self.logger.info("ZMQ listener finishing normally")
-        finally:
-            self.logger.info("ZMQ listener finished")
-
 
 @wrap_with_logs
 @typeguard.typechecked
-def router_starter(*,
-                   comm_q: mpq.Queue,
-                   exception_q: mpq.Queue,
-                   resource_msgs: mpq.Queue,
-                   exit_event: Event,
-
-                   hub_address: str,
-                   udp_port: Optional[int],
-                   zmq_port_range: Tuple[int, int],
-
-                   run_dir: str,
-                   logging_level: int) -> None:
-    setproctitle("parsl: monitoring router")
+def udp_router_starter(*,
+                       comm_q: mpq.Queue,
+                       resource_msgs: mpq.Queue,
+                       exit_event: Event,
+
+                       hub_address: str,
+                       udp_port: Optional[int],
+
+                       run_dir: str,
+                       logging_level: int) -> None:
+    setproctitle("parsl: monitoring UDP router")
     try:
         router = MonitoringRouter(hub_address=hub_address,
                                   udp_port=udp_port,
-                                  zmq_port_range=zmq_port_range,
                                   run_dir=run_dir,
                                   logging_level=logging_level,
                                   resource_msgs=resource_msgs,
@@ -202,11 +138,10 @@ def router_starter(*,
         logger.error("MonitoringRouter construction failed.", exc_info=True)
         comm_q.put(f"Monitoring router construction failed: {e}")
     else:
-        comm_q.put((router.udp_port, router.zmq_receiver_port))
+        comm_q.put(router.udp_port)
 
         router.logger.info("Starting MonitoringRouter in router_starter")
         try:
             router.start()
-        except Exception as e:
-            router.logger.exception("router.start exception")
-            exception_q.put(('Hub', str(e)))
+        except Exception:
+            router.logger.exception("UDP router start exception")

parsl/monitoring/radios/zmq_router.py

@@ -0,0 +1,131 @@
+from __future__ import annotations
+
+import logging
+import multiprocessing.queues as mpq
+import os
+import time
+from multiprocessing.synchronize import Event
+from typing import Tuple
+
+import typeguard
+import zmq
+
+from parsl.log_utils import set_file_logger
+from parsl.monitoring.radios.multiprocessing import MultiprocessingQueueRadioSender
+from parsl.monitoring.types import TaggedMonitoringMessage
+from parsl.process_loggers import wrap_with_logs
+from parsl.utils import setproctitle
+
+logger = logging.getLogger(__name__)
+
+
+class MonitoringRouter:
+
+    def __init__(self,
+                 *,
+                 hub_address: str,
+                 zmq_port_range: Tuple[int, int] = (55050, 56000),
+
+                 run_dir: str = ".",
+                 logging_level: int = logging.INFO,
+                 resource_msgs: mpq.Queue,
+                 exit_event: Event,
+                 ):
+        """ Initializes a monitoring configuration class.
+
+        Parameters
+        ----------
+        hub_address : str
+             The ip address at which the workers will be able to reach the Hub.
+        zmq_port_range : tuple(int, int)
+             The MonitoringHub picks ports at random from the range which will be used by Hub.
+             Default: (55050, 56000)
+        run_dir : str
+             Parsl log directory paths. Logs and temp files go here. Default: '.'
+        logging_level : int
+             Logging level as defined in the logging module. Default: logging.INFO
+        resource_msgs : multiprocessing.Queue
+            A multiprocessing queue to receive messages to be routed onwards to the database process
+        exit_event : Event
+            An event that the main Parsl process will set to signal that the monitoring router should shut down.
+        """
+        os.makedirs(run_dir, exist_ok=True)
+        self.logger = set_file_logger(f"{run_dir}/monitoring_zmq_router.log",
+                                      name="monitoring_router",
+                                      level=logging_level)
+        self.logger.debug("Monitoring router starting")
+
+        self.hub_address = hub_address
+
+        self.loop_freq = 10.0  # milliseconds
+
+        self._context = zmq.Context()
+        self.zmq_receiver_channel = self._context.socket(zmq.DEALER)
+        self.zmq_receiver_channel.setsockopt(zmq.LINGER, 0)
+        self.zmq_receiver_channel.set_hwm(0)
+        self.zmq_receiver_channel.RCVTIMEO = int(self.loop_freq)  # in milliseconds
+        self.logger.debug("hub_address: {}. zmq_port_range {}".format(hub_address, zmq_port_range))
+        self.zmq_receiver_port = self.zmq_receiver_channel.bind_to_random_port("tcp://*",
+                                                                               min_port=zmq_port_range[0],
+                                                                               max_port=zmq_port_range[1])
+
+        self.target_radio = MultiprocessingQueueRadioSender(resource_msgs)
+        self.exit_event = exit_event
+
+    @wrap_with_logs(target="monitoring_router")
+    def start(self) -> None:
+        self.logger.info("Starting ZMQ listener")
+        try:
+            while not self.exit_event.is_set():
+                try:
+                    dfk_loop_start = time.time()
+                    while time.time() - dfk_loop_start < 1.0:  # TODO make configurable
+                        # note that nothing checks that msg really is of the annotated type
+                        msg: TaggedMonitoringMessage
+                        msg = self.zmq_receiver_channel.recv_pyobj()
+
+                        assert isinstance(msg, tuple), "ZMQ Receiver expects only tuples, got {}".format(msg)
+                        assert len(msg) >= 1, "ZMQ Receiver expects tuples of length at least 1, got {}".format(msg)
+                        assert len(msg) == 2, "ZMQ Receiver expects message tuples of exactly length 2, got {}".format(msg)
+
+                        self.target_radio.send(msg)
+                except zmq.Again:
+                    pass
+                except Exception:
+                    # This will catch malformed messages. What happens if the
+                    # channel is broken in such a way that it always raises
+                    # an exception? Looping on this would maybe be the wrong
+                    # thing to do.
+                    self.logger.warning("Failure processing a ZMQ message", exc_info=True)
+
+            self.logger.info("ZMQ listener finishing normally")
+        finally:
+            self.logger.info("ZMQ listener finished")
+
+
+@wrap_with_logs
+@typeguard.typechecked
+def zmq_router_starter(*,
+                       comm_q: mpq.Queue,
+                       resource_msgs: mpq.Queue,
+                       exit_event: Event,
+
+                       hub_address: str,
+                       zmq_port_range: Tuple[int, int],
+
+                       run_dir: str,
+                       logging_level: int) -> None:
+    setproctitle("parsl: monitoring zmq router")
+    try:
+        router = MonitoringRouter(hub_address=hub_address,
+                                  zmq_port_range=zmq_port_range,
+                                  run_dir=run_dir,
+                                  logging_level=logging_level,
+                                  resource_msgs=resource_msgs,
+                                  exit_event=exit_event)
+    except Exception as e:
+        logger.error("MonitoringRouter construction failed.", exc_info=True)
+        comm_q.put(f"Monitoring router construction failed: {e}")
+    else:
+        comm_q.put(router.zmq_receiver_port)
+        router.start()