parsl 2025.6.16__py3-none-any.whl → 2025.6.30__py3-none-any.whl

parsl/monitoring/monitoring.py

@@ -3,16 +3,12 @@ from __future__ import annotations
 import logging
 import multiprocessing.synchronize as ms
 import os
-import queue
 import warnings
 from multiprocessing.queues import Queue
-from typing import TYPE_CHECKING, Any, Optional, Union
+from typing import Any, Optional, Union
 
 import typeguard
 
-from parsl.monitoring.errors import MonitoringHubStartError
-from parsl.monitoring.radios.filesystem_router import filesystem_router_starter
-from parsl.monitoring.radios.udp_router import udp_router_starter
 from parsl.monitoring.types import TaggedMonitoringMessage
 from parsl.multiprocessing import (
     SizedQueue,
@@ -38,9 +34,9 @@ logger = logging.getLogger(__name__)
 @typeguard.typechecked
 class MonitoringHub(RepresentationMixin):
     def __init__(self,
-                 hub_address: str,
-                 hub_port: Optional[int] = None,
-                 hub_port_range: Any = None,
+                 hub_address: Any = None,  # unused, so no type enforcement
+                 hub_port_range: Any = None,  # unused, so no type enforcement
+                 hub_port: Any = None,  # unused, so no type enforcement
 
                  workflow_name: Optional[str] = None,
                  workflow_version: Optional[str] = None,
@@ -51,16 +47,14 @@ class MonitoringHub(RepresentationMixin):
         """
         Parameters
         ----------
-        hub_address : str
-             The ip address at which the workers will be able to reach the Hub.
-        hub_port : int
-             The UDP port to which workers will be able to deliver messages to
-             the monitoring router.
-             Note that despite the similar name, this is not related to
-             hub_port_range.
-             Default: None
+        hub_address : unused
+        hub_port : unused
+             Unused, but probably retained until 2026-06-01 to give deprecation warning.
+             These two values previously configured UDP parameters when UDP was used
+             for monitoring messages from workers. These are now configured on the
+             relevant UDPRadio.
         hub_port_range : unused
-             Unused, but retained until 2025-09-14 to avoid configuration errors.
+             Unused, but probably retained until 2026-06-01 to give deprecation warning.
              This value previously configured one ZMQ channel inside the
              HighThroughputExecutor. That ZMQ channel is now configured by the
              interchange_port_range parameter of HighThroughputExecutor.
@@ -88,15 +82,27 @@ class MonitoringHub(RepresentationMixin):
         if _db_manager_excepts:
             raise _db_manager_excepts
 
+        # The following three parameters need to exist as attributes to be
+        # output by RepresentationMixin.
+        if hub_address is not None:
+            message = "Instead of MonitoringHub.hub_address, specify UDPRadio(address=...)"
+            warnings.warn(message, DeprecationWarning)
+            logger.warning(message)
+
         self.hub_address = hub_address
+
+        if hub_port is not None:
+            message = "Instead of MonitoringHub.hub_port, specify UDPRadio(port=...)"
+            warnings.warn(message, DeprecationWarning)
+            logger.warning(message)
+
         self.hub_port = hub_port
 
         if hub_port_range is not None:
             message = "Instead of MonitoringHub.hub_port_range, Use HighThroughputExecutor.interchange_port_range"
             warnings.warn(message, DeprecationWarning)
             logger.warning(message)
-        # This is used by RepresentationMixin so needs to exist as an attribute
-        # even though now it is otherwise unused.
+
         self.hub_port_range = hub_port_range
 
         self.logging_endpoint = logging_endpoint
@@ -119,40 +125,9 @@ class MonitoringHub(RepresentationMixin):
 
         self.monitoring_hub_active = True
 
-        # This annotation is incompatible with typeguard 4.x instrumentation
-        # of local variables: Queue is not subscriptable at runtime, as far
-        # as typeguard is concerned. The more general Queue annotation works,
-        # but does not restrict the contents of the Queue. Using TYPE_CHECKING
-        # here allows the stricter definition to be seen by mypy, and the
-        # simpler definition to be seen by typeguard. Hopefully at some point
-        # in the future, Queue will allow runtime subscripts.
-
-        if TYPE_CHECKING:
-            udp_comm_q: Queue[Union[int, str]]
-        else:
-            udp_comm_q: Queue
-
-        udp_comm_q = SizedQueue(maxsize=10)
-
         self.resource_msgs: Queue[TaggedMonitoringMessage]
         self.resource_msgs = SizedQueue()
 
-        self.router_exit_event: ms.Event
-        self.router_exit_event = SpawnEvent()
-
-        self.udp_router_proc = SpawnProcess(target=udp_router_starter,
-                                            kwargs={"comm_q": udp_comm_q,
-                                                    "resource_msgs": self.resource_msgs,
-                                                    "exit_event": self.router_exit_event,
-                                                    "udp_port": self.hub_port,
-                                                    "run_dir": dfk_run_dir,
-                                                    "logging_level": logging.DEBUG if self.monitoring_debug else logging.INFO,
-                                                    },
-                                            name="Monitoring-UDP-Router-Process",
-                                            daemon=True,
-                                            )
-        self.udp_router_proc.start()
-
         self.dbm_exit_event: ms.Event
         self.dbm_exit_event = SpawnEvent()
 
@@ -167,55 +142,18 @@ class MonitoringHub(RepresentationMixin):
                                      daemon=True,
                                      )
         self.dbm_proc.start()
-        logger.info("Started UDP router process %s and DBM process %s",
-                    self.udp_router_proc.pid, self.dbm_proc.pid)
-
-        self.filesystem_proc = SpawnProcess(target=filesystem_router_starter,
-                                            kwargs={"q": self.resource_msgs,
-                                                    "run_dir": dfk_run_dir,
-                                                    "exit_event": self.router_exit_event},
-                                            name="Monitoring-Filesystem-Process",
-                                            daemon=True
-                                            )
-        self.filesystem_proc.start()
-        logger.info("Started filesystem radio receiver process %s", self.filesystem_proc.pid)
-
-        try:
-            udp_comm_q_result = udp_comm_q.get(block=True, timeout=120)
-            udp_comm_q.close()
-            udp_comm_q.join_thread()
-        except queue.Empty:
-            logger.error("Monitoring UDP router has not reported port in 120s. Aborting")
-            raise MonitoringHubStartError()
-
-        if isinstance(udp_comm_q_result, str):
-            logger.error("MonitoringRouter sent an error message: %s", udp_comm_q_result)
-            raise RuntimeError(f"MonitoringRouter failed to start: {udp_comm_q_result}")
-
-        udp_port = udp_comm_q_result
-        self.monitoring_hub_url = "udp://{}:{}".format(self.hub_address, udp_port)
-
+        logger.info("Started DBM process %s", self.dbm_proc.pid)
         logger.info("Monitoring Hub initialized")
 
     def close(self) -> None:
         logger.info("Terminating Monitoring Hub")
         if self.monitoring_hub_active:
             self.monitoring_hub_active = False
-            logger.info("Setting router termination event")
-            self.router_exit_event.set()
-
-            logger.info("Waiting for UDP router to terminate")
-            join_terminate_close_proc(self.udp_router_proc)
-
-            logger.debug("Finished waiting for router termination")
             logger.debug("Waiting for DB termination")
             self.dbm_exit_event.set()
             join_terminate_close_proc(self.dbm_proc)
             logger.debug("Finished waiting for DBM termination")
 
-            logger.info("Terminating filesystem radio receiver process")
-            join_terminate_close_proc(self.filesystem_proc)
-
             logger.info("Closing monitoring multiprocessing queues")
             self.resource_msgs.close()
             self.resource_msgs.join_thread()

parsl/monitoring/radios/base.py

@@ -1,10 +1,71 @@
-import logging
 from abc import ABCMeta, abstractmethod
+from multiprocessing.queues import Queue
 
-logger = logging.getLogger(__name__)
+
+class MonitoringRadioReceiver(metaclass=ABCMeta):
+    @abstractmethod
+    def shutdown(self) -> None:
+        pass
 
 
 class MonitoringRadioSender(metaclass=ABCMeta):
     @abstractmethod
     def send(self, message: object) -> None:
         pass
+
+
+class RadioConfig(metaclass=ABCMeta):
+    """Base class for radio plugin configuration.
+
+    This provides the configuration for a particular way of sending monitoring
+    messages from a source of monitoring messages into the submit side
+    monitoring database.
+
+    This uses staged initialization like lots of Parsl configuration, but in
+    a slightly different form.
+
+    A RadioConfig object must be pickleable, because it will be sent to remote
+    workers to configure senders. The MonitoringRadioSender and
+    MonitoringRadioReceiver objects do not need to be pickleable (and often
+    will not be - for example, when they hold references to other processes).
+
+    The RadioConfig object will be used by Parsl in this sequence:
+
+    * A user creates a RadioConfig object from the appropriate subclass for
+      radio mechanism they want to use, and specifies it as part of their
+      executor configuration.
+
+    Methods on the RadioConfig will then be invoked by Parsl like this:
+
+     * one create_receiver call, on the submit side
+        - this call can modify the state of radioconfig to contain information
+          about how a sender can connect back to the receiver. for example,
+          after binding to a particular port, can store that port so that the
+          sender knows which port to connect to.
+
+     * Possibly many serializations to get the RadioConfig to remote workers
+
+     * Many (0 or more) create_sender calls, possibly on remote workers, to
+       create the sending side of the radio (MonitoringRadioSender instances)
+
+     * Those senders are used to send messages
+
+     * At executor shutdown, the receiver is shut down.
+
+    This object cannot be re-used across parsl configurations - like many other
+    pieces of parsl config it is single use in that respect.
+    """
+
+    @abstractmethod
+    def create_receiver(self, *, run_dir: str, resource_msgs: Queue) -> MonitoringRadioReceiver:
+        """Create a receiver for this RadioConfig, and update this RadioConfig
+        with enough context to create senders.
+        """
+        pass
+
+    @abstractmethod
+    def create_sender(self) -> MonitoringRadioSender:
+        """Create a sender to connect to the receiver created by an
+        earlier call to create_receiver.
+        """
+        pass

parsl/monitoring/radios/filesystem.py

@@ -2,13 +2,19 @@ import logging
 import os
 import pickle
 import uuid
+from multiprocessing.queues import Queue
 
-from parsl.monitoring.radios.base import MonitoringRadioSender
+from parsl.monitoring.radios.base import (
+    MonitoringRadioReceiver,
+    MonitoringRadioSender,
+    RadioConfig,
+)
+from parsl.monitoring.radios.filesystem_router import FilesystemRadioReceiver
 
 logger = logging.getLogger(__name__)
 
 
-class FilesystemRadioSender(MonitoringRadioSender):
+class FilesystemRadio(RadioConfig):
     """A MonitoringRadioSender that sends messages over a shared filesystem.
 
     The messsage directory structure is based on maildir,
@@ -26,8 +32,17 @@ class FilesystemRadioSender(MonitoringRadioSender):
     the UDP radio, but should be much more reliable.
     """
 
-    def __init__(self, *, monitoring_url: str, timeout: int = 10, run_dir: str):
-        logger.info("filesystem based monitoring channel initializing")
+    def create_sender(self) -> MonitoringRadioSender:
+        return FilesystemRadioSender(run_dir=self.run_dir)
+
+    def create_receiver(self, *, run_dir: str, resource_msgs: Queue) -> MonitoringRadioReceiver:
+        self.run_dir = run_dir
+        return FilesystemRadioReceiver(resource_msgs, run_dir)
+
+
+class FilesystemRadioSender(MonitoringRadioSender):
+    def __init__(self, *, run_dir: str):
+        logger.info("filesystem based monitoring radio initializing")
         self.base_path = f"{run_dir}/monitor-fs-radio/"
         self.tmp_path = f"{self.base_path}/tmp"
         self.new_path = f"{self.base_path}/new"

parsl/monitoring/radios/filesystem_router.py

@@ -9,17 +9,20 @@ from multiprocessing.synchronize import Event
 from typing import cast
 
 from parsl.log_utils import set_file_logger
+from parsl.monitoring.radios.base import MonitoringRadioReceiver
 from parsl.monitoring.radios.multiprocessing import MultiprocessingQueueRadioSender
 from parsl.monitoring.types import TaggedMonitoringMessage
+from parsl.multiprocessing import SpawnEvent, SpawnProcess, join_terminate_close_proc
 from parsl.process_loggers import wrap_with_logs
 from parsl.utils import setproctitle
 
+logger = logging.getLogger(__name__)
+
 
 @wrap_with_logs
 def filesystem_router_starter(*, q: Queue[TaggedMonitoringMessage], run_dir: str, exit_event: Event) -> None:
-    logger = set_file_logger(f"{run_dir}/monitoring_filesystem_radio.log",
-                             name="monitoring_filesystem_radio",
-                             level=logging.INFO)
+    set_file_logger(f"{run_dir}/monitoring_filesystem_radio.log",
+                    level=logging.INFO)
 
     logger.info("Starting filesystem radio receiver")
     setproctitle("parsl: monitoring filesystem receiver")
@@ -52,3 +55,19 @@ def filesystem_router_starter(*, q: Queue[TaggedMonitoringMessage], run_dir: str
 
         time.sleep(1)  # whats a good time for this poll?
     logger.info("Ending filesystem radio receiver")
+
+
+class FilesystemRadioReceiver(MonitoringRadioReceiver):
+    def __init__(self, resource_msgs: Queue, run_dir: str) -> None:
+        self.exit_event = SpawnEvent()
+        self.process = SpawnProcess(target=filesystem_router_starter,
+                                    kwargs={"q": resource_msgs, "run_dir": run_dir, "exit_event": self.exit_event},
+                                    name="Monitoring-Filesystem-Process",
+                                    daemon=True
+                                    )
+        self.process.start()
+        logger.info("Started filesystem radio receiver process %s", self.process.pid)
+
+    def shutdown(self) -> None:
+        self.exit_event.set()
+        join_terminate_close_proc(self.process)

parsl/monitoring/radios/htex.py

@@ -1,24 +1,29 @@
 import logging
 import pickle
+from multiprocessing.queues import Queue
 
-from parsl.monitoring.radios.base import MonitoringRadioSender
+from parsl.monitoring.radios.base import (
+    MonitoringRadioReceiver,
+    MonitoringRadioSender,
+    RadioConfig,
+)
 
 logger = logging.getLogger(__name__)
 
 
-class HTEXRadioSender(MonitoringRadioSender):
+class HTEXRadio(RadioConfig):
+    def create_sender(self) -> MonitoringRadioSender:
+        return HTEXRadioSender()
 
-    def __init__(self, monitoring_url: str, timeout: int = 10):
-        """
-        Parameters
-        ----------
+    def create_receiver(self, *, run_dir: str, resource_msgs: Queue) -> MonitoringRadioReceiver:
+        return HTEXRadioReceiver()
 
-        monitoring_url : str
-            URL of the form <scheme>://<IP>:<PORT>
-        timeout : int
-            timeout, default=10s
-        """
-        logger.info("htex-based monitoring channel initialising")
+
+class HTEXRadioSender(MonitoringRadioSender):
+
+    def __init__(self) -> None:
+        # there is nothing to initialize
+        pass
 
     def send(self, message: object) -> None:
         """ Sends a message to the UDP receiver
@@ -54,4 +59,8 @@ class HTEXRadioSender(MonitoringRadioSender):
         else:
             logger.error("result_queue is uninitialized - cannot put monitoring message")
 
-        return
+
+class HTEXRadioReceiver(MonitoringRadioReceiver):
+    def shutdown(self) -> None:
+        # there is nothing to shut down
+        pass

parsl/monitoring/radios/multiprocessing.py

@@ -1,6 +1,10 @@
-from multiprocessing.queues import Queue
+from multiprocessing import Queue
 
-from parsl.monitoring.radios.base import MonitoringRadioSender
+from parsl.monitoring.radios.base import (
+    MonitoringRadioReceiver,
+    MonitoringRadioSender,
+    RadioConfig,
+)
 
 
 class MultiprocessingQueueRadioSender(MonitoringRadioSender):
@@ -15,3 +19,19 @@ class MultiprocessingQueueRadioSender(MonitoringRadioSender):
 
     def send(self, message: object) -> None:
         self.queue.put(message)
+
+
+class MultiprocessingQueueRadio(RadioConfig):
+    def create_sender(self) -> MonitoringRadioSender:
+        return MultiprocessingQueueRadioSender(self._queue)
+
+    def create_receiver(self, *, run_dir: str, resource_msgs: Queue) -> MonitoringRadioReceiver:
+        # This object is only for use with an in-process thread-pool so it
+        # is fine to store a reference to the message queue directly.
+        self._queue = resource_msgs
+        return MultiprocessingQueueRadioReceiver()
+
+
+class MultiprocessingQueueRadioReceiver(MonitoringRadioReceiver):
+    def shutdown(self) -> None:
+        pass

parsl/monitoring/radios/udp.py

@@ -1,29 +1,63 @@
+import hashlib
+import hmac
 import logging
 import pickle
+import secrets
 import socket
+from multiprocessing.queues import Queue
+from typing import Optional
 
-from parsl.monitoring.radios.base import MonitoringRadioSender
+from parsl.monitoring.radios.base import (
+    MonitoringRadioReceiver,
+    MonitoringRadioSender,
+    RadioConfig,
+)
+from parsl.monitoring.radios.udp_router import start_udp_receiver
 
+logger = logging.getLogger(__name__)
 
-class UDPRadioSender(MonitoringRadioSender):
 
-    def __init__(self, monitoring_url: str, timeout: int = 10):
-        """
-        Parameters
-        ----------
+class UDPRadio(RadioConfig):
+    def __init__(self, *, port: Optional[int] = None, atexit_timeout: int = 3, address: str, debug: bool = False, hmac_digest: str = 'sha512'):
+        self.port = port
+        self.atexit_timeout = atexit_timeout
+        self.address = address
+        self.debug = debug
+        self.hmac_digest = hmac_digest
 
-        monitoring_url : str
-            URL of the form <scheme>://<IP>:<PORT>
-        timeout : int
-            timeout, default=10s
-        """
-        self.monitoring_url = monitoring_url
+    def create_sender(self) -> MonitoringRadioSender:
+        assert self.port is not None, "self.port should have been initialized by create_receiver"
+        return UDPRadioSender(self.address, self.port, self.hmac_key, self.hmac_digest)
+
+    def create_receiver(self, run_dir: str, resource_msgs: Queue) -> MonitoringRadioReceiver:
+        # RFC 2104 section 2 recommends that the key length be at
+        # least as long as the hash output (64 bytes in the case of SHA512).
+        # RFC 2014 section 3 talks about periodic key refreshing. This key is
+        # not refreshed inside a workflow run, but each separate workflow run
+        # uses a new key.
+        keysize = hashlib.new(self.hmac_digest).digest_size
+        self.hmac_key = secrets.token_bytes(keysize)
+
+        udp_receiver = start_udp_receiver(logdir=run_dir,
+                                          monitoring_messages=resource_msgs,
+                                          port=self.port,
+                                          debug=self.debug,
+                                          atexit_timeout=self.atexit_timeout,
+                                          hmac_key=self.hmac_key,
+                                          hmac_digest=self.hmac_digest
+                                          )
+        self.port = udp_receiver.port
+        return udp_receiver
+
+
+class UDPRadioSender(MonitoringRadioSender):
+
+    def __init__(self, address: str, port: int, hmac_key: bytes, hmac_digest: str, *, timeout: int = 10) -> None:
         self.sock_timeout = timeout
-        try:
-            self.scheme, self.ip, port = (x.strip('/') for x in monitoring_url.split(':'))
-            self.port = int(port)
-        except Exception:
-            raise Exception("Failed to parse monitoring url: {}".format(monitoring_url))
+        self.address = address
+        self.port = port
+        self.hmac_key = hmac_key
+        self.hmac_digest = hmac_digest
 
         self.sock = socket.socket(socket.AF_INET,
                                   socket.SOCK_DGRAM,
@@ -42,15 +76,19 @@ class UDPRadioSender(MonitoringRadioSender):
         Returns:
             None
         """
+        logger.info("Starting UDP radio message send")
         try:
-            buffer = pickle.dumps(message)
+            data = pickle.dumps(message)
+            origin_hmac = hmac.digest(self.hmac_key, data, self.hmac_digest)
+            buffer = origin_hmac + data
         except Exception:
             logging.exception("Exception during pickling", exc_info=True)
             return
 
         try:
-            self.sock.sendto(buffer, (self.ip, self.port))
+            self.sock.sendto(buffer, (self.address, self.port))
         except socket.timeout:
             logging.error("Could not send message within timeout limit")
             return
+        logger.info("Normal ending for UDP radio message send")
         return