parsl 2025.6.23__py3-none-any.whl → 2025.6.30__py3-none-any.whl

parsl/monitoring/radios/base.py

@@ -1,10 +1,71 @@
-import logging
 from abc import ABCMeta, abstractmethod
+from multiprocessing.queues import Queue
 
-logger = logging.getLogger(__name__)
+
+class MonitoringRadioReceiver(metaclass=ABCMeta):
+    @abstractmethod
+    def shutdown(self) -> None:
+        pass
 
 
 class MonitoringRadioSender(metaclass=ABCMeta):
     @abstractmethod
     def send(self, message: object) -> None:
         pass
+
+
+class RadioConfig(metaclass=ABCMeta):
+    """Base class for radio plugin configuration.
+
+    This provides the configuration for a particular way of sending monitoring
+    messages from a source of monitoring messages into the submit side
+    monitoring database.
+
+    This uses staged initialization like lots of Parsl configuration, but in
+    a slightly different form.
+
+    A RadioConfig object must be pickleable, because it will be sent to remote
+    workers to configure senders. The MonitoringRadioSender and
+    MonitoringRadioReceiver objects do not need to be pickleable (and often
+    will not be - for example, when they hold references to other processes).
+
+    The RadioConfig object will be used by Parsl in this sequence:
+
+    * A user creates a RadioConfig object from the appropriate subclass for
+      radio mechanism they want to use, and specifies it as part of their
+      executor configuration.
+
+    Methods on the RadioConfig will then be invoked by Parsl like this:
+
+     * one create_receiver call, on the submit side
+        - this call can modify the state of radioconfig to contain information
+          about how a sender can connect back to the receiver. for example,
+          after binding to a particular port, can store that port so that the
+          sender knows which port to connect to.
+
+     * Possibly many serializations to get the RadioConfig to remote workers
+
+     * Many (0 or more) create_sender calls, possibly on remote workers, to
+       create the sending side of the radio (MonitoringRadioSender instances)
+
+     * Those senders are used to send messages
+
+     * At executor shutdown, the receiver is shut down.
+
+    This object cannot be re-used across parsl configurations - like many other
+    pieces of parsl config it is single use in that respect.
+    """
+
+    @abstractmethod
+    def create_receiver(self, *, run_dir: str, resource_msgs: Queue) -> MonitoringRadioReceiver:
+        """Create a receiver for this RadioConfig, and update this RadioConfig
+        with enough context to create senders.
+        """
+        pass
+
+    @abstractmethod
+    def create_sender(self) -> MonitoringRadioSender:
+        """Create a sender to connect to the receiver created by an
+        earlier call to create_receiver.
+        """
+        pass

parsl/monitoring/radios/filesystem.py

@@ -2,13 +2,19 @@ import logging
 import os
 import pickle
 import uuid
+from multiprocessing.queues import Queue
 
-from parsl.monitoring.radios.base import MonitoringRadioSender
+from parsl.monitoring.radios.base import (
+    MonitoringRadioReceiver,
+    MonitoringRadioSender,
+    RadioConfig,
+)
+from parsl.monitoring.radios.filesystem_router import FilesystemRadioReceiver
 
 logger = logging.getLogger(__name__)
 
 
-class FilesystemRadioSender(MonitoringRadioSender):
+class FilesystemRadio(RadioConfig):
     """A MonitoringRadioSender that sends messages over a shared filesystem.
 
     The messsage directory structure is based on maildir,
@@ -26,7 +32,16 @@ class FilesystemRadioSender(MonitoringRadioSender):
     the UDP radio, but should be much more reliable.
     """
 
-    def __init__(self, *, monitoring_url: str, timeout: int = 10, run_dir: str):
+    def create_sender(self) -> MonitoringRadioSender:
+        return FilesystemRadioSender(run_dir=self.run_dir)
+
+    def create_receiver(self, *, run_dir: str, resource_msgs: Queue) -> MonitoringRadioReceiver:
+        self.run_dir = run_dir
+        return FilesystemRadioReceiver(resource_msgs, run_dir)
+
+
+class FilesystemRadioSender(MonitoringRadioSender):
+    def __init__(self, *, run_dir: str):
         logger.info("filesystem based monitoring radio initializing")
         self.base_path = f"{run_dir}/monitor-fs-radio/"
         self.tmp_path = f"{self.base_path}/tmp"

parsl/monitoring/radios/filesystem_router.py

@@ -4,15 +4,15 @@ import logging
 import os
 import pickle
 import time
-from multiprocessing.context import SpawnProcess
 from multiprocessing.queues import Queue
 from multiprocessing.synchronize import Event
 from typing import cast
 
 from parsl.log_utils import set_file_logger
+from parsl.monitoring.radios.base import MonitoringRadioReceiver
 from parsl.monitoring.radios.multiprocessing import MultiprocessingQueueRadioSender
 from parsl.monitoring.types import TaggedMonitoringMessage
-from parsl.multiprocessing import SpawnEvent, join_terminate_close_proc
+from parsl.multiprocessing import SpawnEvent, SpawnProcess, join_terminate_close_proc
 from parsl.process_loggers import wrap_with_logs
 from parsl.utils import setproctitle
 
@@ -57,30 +57,17 @@ def filesystem_router_starter(*, q: Queue[TaggedMonitoringMessage], run_dir: str
     logger.info("Ending filesystem radio receiver")
 
 
-class FilesystemRadioReceiver():
-    def __init__(self, *, process: SpawnProcess, exit_event: Event) -> None:
-        self.process = process
-        self.exit_event = exit_event
+class FilesystemRadioReceiver(MonitoringRadioReceiver):
+    def __init__(self, resource_msgs: Queue, run_dir: str) -> None:
+        self.exit_event = SpawnEvent()
+        self.process = SpawnProcess(target=filesystem_router_starter,
+                                    kwargs={"q": resource_msgs, "run_dir": run_dir, "exit_event": self.exit_event},
+                                    name="Monitoring-Filesystem-Process",
+                                    daemon=True
+                                    )
+        self.process.start()
+        logger.info("Started filesystem radio receiver process %s", self.process.pid)
 
-    def close(self) -> None:
+    def shutdown(self) -> None:
         self.exit_event.set()
         join_terminate_close_proc(self.process)
-
-
-def start_filesystem_receiver(*,
-                              monitoring_messages: Queue,
-                              logdir: str,
-                              debug: bool) -> FilesystemRadioReceiver:
-
-    router_exit_event = SpawnEvent()
-
-    filesystem_proc = SpawnProcess(target=filesystem_router_starter,
-                                   kwargs={"q": monitoring_messages,
-                                           "run_dir": logdir,
-                                           "exit_event": router_exit_event},
-                                   name="Monitoring-Filesystem-Process",
-                                   daemon=True
-                                   )
-    filesystem_proc.start()
-
-    return FilesystemRadioReceiver(process=filesystem_proc, exit_event=router_exit_event)

parsl/monitoring/radios/htex.py

@@ -1,24 +1,29 @@
 import logging
 import pickle
+from multiprocessing.queues import Queue
 
-from parsl.monitoring.radios.base import MonitoringRadioSender
+from parsl.monitoring.radios.base import (
+    MonitoringRadioReceiver,
+    MonitoringRadioSender,
+    RadioConfig,
+)
 
 logger = logging.getLogger(__name__)
 
 
-class HTEXRadioSender(MonitoringRadioSender):
+class HTEXRadio(RadioConfig):
+    def create_sender(self) -> MonitoringRadioSender:
+        return HTEXRadioSender()
 
-    def __init__(self, monitoring_url: str, timeout: int = 10):
-        """
-        Parameters
-        ----------
+    def create_receiver(self, *, run_dir: str, resource_msgs: Queue) -> MonitoringRadioReceiver:
+        return HTEXRadioReceiver()
 
-        monitoring_url : str
-            URL of the form <scheme>://<IP>:<PORT>
-        timeout : int
-            timeout, default=10s
-        """
-        logger.info("htex-based monitoring radio initialising")
+
+class HTEXRadioSender(MonitoringRadioSender):
+
+    def __init__(self) -> None:
+        # there is nothing to initialize
+        pass
 
     def send(self, message: object) -> None:
         """ Sends a message to the UDP receiver
@@ -54,4 +59,8 @@ class HTEXRadioSender(MonitoringRadioSender):
         else:
             logger.error("result_queue is uninitialized - cannot put monitoring message")
 
-        return
+
+class HTEXRadioReceiver(MonitoringRadioReceiver):
+    def shutdown(self) -> None:
+        # there is nothing to shut down
+        pass

parsl/monitoring/radios/multiprocessing.py

@@ -1,6 +1,10 @@
-from multiprocessing.queues import Queue
+from multiprocessing import Queue
 
-from parsl.monitoring.radios.base import MonitoringRadioSender
+from parsl.monitoring.radios.base import (
+    MonitoringRadioReceiver,
+    MonitoringRadioSender,
+    RadioConfig,
+)
 
 
 class MultiprocessingQueueRadioSender(MonitoringRadioSender):
@@ -15,3 +19,19 @@ class MultiprocessingQueueRadioSender(MonitoringRadioSender):
 
     def send(self, message: object) -> None:
         self.queue.put(message)
+
+
+class MultiprocessingQueueRadio(RadioConfig):
+    def create_sender(self) -> MonitoringRadioSender:
+        return MultiprocessingQueueRadioSender(self._queue)
+
+    def create_receiver(self, *, run_dir: str, resource_msgs: Queue) -> MonitoringRadioReceiver:
+        # This object is only for use with an in-process thread-pool so it
+        # is fine to store a reference to the message queue directly.
+        self._queue = resource_msgs
+        return MultiprocessingQueueRadioReceiver()
+
+
+class MultiprocessingQueueRadioReceiver(MonitoringRadioReceiver):
+    def shutdown(self) -> None:
+        pass

parsl/monitoring/radios/udp.py

@@ -1,29 +1,63 @@
+import hashlib
+import hmac
 import logging
 import pickle
+import secrets
 import socket
+from multiprocessing.queues import Queue
+from typing import Optional
 
-from parsl.monitoring.radios.base import MonitoringRadioSender
+from parsl.monitoring.radios.base import (
+    MonitoringRadioReceiver,
+    MonitoringRadioSender,
+    RadioConfig,
+)
+from parsl.monitoring.radios.udp_router import start_udp_receiver
 
+logger = logging.getLogger(__name__)
 
-class UDPRadioSender(MonitoringRadioSender):
 
-    def __init__(self, monitoring_url: str, timeout: int = 10):
-        """
-        Parameters
-        ----------
+class UDPRadio(RadioConfig):
+    def __init__(self, *, port: Optional[int] = None, atexit_timeout: int = 3, address: str, debug: bool = False, hmac_digest: str = 'sha512'):
+        self.port = port
+        self.atexit_timeout = atexit_timeout
+        self.address = address
+        self.debug = debug
+        self.hmac_digest = hmac_digest
 
-        monitoring_url : str
-            URL of the form <scheme>://<IP>:<PORT>
-        timeout : int
-            timeout, default=10s
-        """
-        self.monitoring_url = monitoring_url
+    def create_sender(self) -> MonitoringRadioSender:
+        assert self.port is not None, "self.port should have been initialized by create_receiver"
+        return UDPRadioSender(self.address, self.port, self.hmac_key, self.hmac_digest)
+
+    def create_receiver(self, run_dir: str, resource_msgs: Queue) -> MonitoringRadioReceiver:
+        # RFC 2104 section 2 recommends that the key length be at
+        # least as long as the hash output (64 bytes in the case of SHA512).
+        # RFC 2014 section 3 talks about periodic key refreshing. This key is
+        # not refreshed inside a workflow run, but each separate workflow run
+        # uses a new key.
+        keysize = hashlib.new(self.hmac_digest).digest_size
+        self.hmac_key = secrets.token_bytes(keysize)
+
+        udp_receiver = start_udp_receiver(logdir=run_dir,
+                                          monitoring_messages=resource_msgs,
+                                          port=self.port,
+                                          debug=self.debug,
+                                          atexit_timeout=self.atexit_timeout,
+                                          hmac_key=self.hmac_key,
+                                          hmac_digest=self.hmac_digest
+                                          )
+        self.port = udp_receiver.port
+        return udp_receiver
+
+
+class UDPRadioSender(MonitoringRadioSender):
+
+    def __init__(self, address: str, port: int, hmac_key: bytes, hmac_digest: str, *, timeout: int = 10) -> None:
         self.sock_timeout = timeout
-        try:
-            self.scheme, self.ip, port = (x.strip('/') for x in monitoring_url.split(':'))
-            self.port = int(port)
-        except Exception:
-            raise Exception("Failed to parse monitoring url: {}".format(monitoring_url))
+        self.address = address
+        self.port = port
+        self.hmac_key = hmac_key
+        self.hmac_digest = hmac_digest
 
         self.sock = socket.socket(socket.AF_INET,
                                   socket.SOCK_DGRAM,
@@ -42,15 +76,19 @@ class UDPRadioSender(MonitoringRadioSender):
         Returns:
             None
         """
+        logger.info("Starting UDP radio message send")
         try:
-            buffer = pickle.dumps(message)
+            data = pickle.dumps(message)
+            origin_hmac = hmac.digest(self.hmac_key, data, self.hmac_digest)
+            buffer = origin_hmac + data
         except Exception:
             logging.exception("Exception during pickling", exc_info=True)
             return
 
         try:
-            self.sock.sendto(buffer, (self.ip, self.port))
+            self.sock.sendto(buffer, (self.address, self.port))
         except socket.timeout:
             logging.error("Could not send message within timeout limit")
             return
+        logger.info("Normal ending for UDP radio message send")
         return

parsl/monitoring/radios/udp_router.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import hmac
 import logging
 import multiprocessing.queues as mpq
 import os
@@ -17,6 +18,7 @@ import typeguard
 
 from parsl.log_utils import set_file_logger
 from parsl.monitoring.errors import MonitoringRouterStartError
+from parsl.monitoring.radios.base import MonitoringRadioReceiver
 from parsl.monitoring.radios.multiprocessing import MultiprocessingQueueRadioSender
 from parsl.multiprocessing import (
     SizedQueue,
@@ -37,9 +39,11 @@ class MonitoringRouter:
                  udp_port: Optional[int] = None,
                  run_dir: str = ".",
                  logging_level: int = logging.INFO,
-                 atexit_timeout: int = 3,   # in seconds
+                 atexit_timeout: int,   # in seconds
                  resource_msgs: mpq.Queue,
                  exit_event: Event,
+                 hmac_key: bytes,
+                 hmac_digest: str,
                  ):
         """ Initializes a monitoring configuration class.
 
@@ -65,6 +69,9 @@ class MonitoringRouter:
 
         self.atexit_timeout = atexit_timeout
 
+        self.hmac_key = hmac_key
+        self.hmac_digest = hmac_digest
+
         self.loop_freq = 10.0  # milliseconds
 
         # Initialize the UDP socket
@@ -94,10 +101,7 @@ class MonitoringRouter:
         try:
             while not self.exit_event.is_set():
                 try:
-                    data, addr = self.udp_sock.recvfrom(2048)
-                    resource_msg = pickle.loads(data)
-                    logger.debug("Got UDP Message from {}: {}".format(addr, resource_msg))
-                    self.target_radio.send(resource_msg)
+                    self.process_message()
                 except socket.timeout:
                     pass
 
@@ -105,10 +109,7 @@ class MonitoringRouter:
             last_msg_received_time = time.time()
             while time.time() - last_msg_received_time < self.atexit_timeout:
                 try:
-                    data, addr = self.udp_sock.recvfrom(2048)
-                    msg = pickle.loads(data)
-                    logger.debug("Got UDP Message from {}: {}".format(addr, msg))
-                    self.target_radio.send(msg)
+                    self.process_message()
                     last_msg_received_time = time.time()
                 except socket.timeout:
                     pass
@@ -117,6 +118,28 @@ class MonitoringRouter:
         finally:
             logger.info("UDP listener finished")
 
+    def process_message(self) -> None:
+        hmdata, addr = self.udp_sock.recvfrom(2048)
+        h = hmac.HMAC(key=self.hmac_key, digestmod=self.hmac_digest)
+        origin_hmac = hmdata[0:h.digest_size]
+        h.update(hmdata[h.digest_size:])
+        data = hmdata[h.digest_size:]
+
+        # Check hmac before pickle load.
+        # If data is wrong, do not log it because it is suspect,
+        # but it should be safe to log the addr, at error level.
+
+        recomputed_hmac = h.digest()
+
+        if not hmac.compare_digest(origin_hmac, recomputed_hmac):
+            logger.error("HMAC does not match on received message")
+            # No exception, because this can be arbitrary network noise
+            # that shouldn't break the receiver.
+        else:
+            resource_msg = pickle.loads(data)
+            logger.debug("Got UDP Message from {}: {}".format(addr, resource_msg))
+            self.target_radio.send(resource_msg)
+
 
 @wrap_with_logs
 @typeguard.typechecked
@@ -126,16 +149,21 @@ def udp_router_starter(*,
                        exit_event: Event,
 
                        udp_port: Optional[int],
-
+                       hmac_key: bytes,
                        run_dir: str,
-                       logging_level: int) -> None:
+                       logging_level: int,
+                       atexit_timeout: int,
+                       hmac_digest: str) -> None:
     setproctitle("parsl: monitoring UDP router")
     try:
         router = MonitoringRouter(udp_port=udp_port,
                                   run_dir=run_dir,
                                   logging_level=logging_level,
                                   resource_msgs=resource_msgs,
-                                  exit_event=exit_event)
+                                  exit_event=exit_event,
+                                  atexit_timeout=atexit_timeout,
+                                  hmac_key=hmac_key,
+                                  hmac_digest=hmac_digest)
     except Exception as e:
         logger.error("MonitoringRouter construction failed.", exc_info=True)
         comm_q.put(f"Monitoring router construction failed: {e}")
@@ -149,13 +177,13 @@ def udp_router_starter(*,
             logger.exception("UDP router start exception")
 
 
-class UDPRadioReceiver():
+class UDPRadioReceiver(MonitoringRadioReceiver):
     def __init__(self, *, process: SpawnProcessType, exit_event: EventType, port: int) -> None:
         self.process = process
         self.exit_event = exit_event
         self.port = port
 
-    def close(self) -> None:
+    def shutdown(self) -> None:
         self.exit_event.set()
         join_terminate_close_proc(self.process)
 
@@ -164,7 +192,10 @@ def start_udp_receiver(*,
                        monitoring_messages: Queue,
                        port: Optional[int],
                        logdir: str,
-                       debug: bool) -> UDPRadioReceiver:
+                       debug: bool,
+                       atexit_timeout: int,
+                       hmac_key: bytes,
+                       hmac_digest: str) -> UDPRadioReceiver:
 
     udp_comm_q: Queue[Union[int, str]]
     udp_comm_q = SizedQueue(maxsize=10)
@@ -178,6 +209,9 @@ def start_udp_receiver(*,
                                        "udp_port": port,
                                        "run_dir": logdir,
                                        "logging_level": logging.DEBUG if debug else logging.INFO,
+                                       "atexit_timeout": atexit_timeout,
+                                       "hmac_key": hmac_key,
+                                       "hmac_digest": hmac_digest,
                                        },
                                name="Monitoring-UDP-Router-Process",
                                daemon=True,

parsl/monitoring/remote.py

@@ -7,10 +7,7 @@ from multiprocessing import Event
 from typing import Any, Callable, Dict, List, Sequence, Tuple
 
 from parsl.monitoring.message_type import MessageType
-from parsl.monitoring.radios.base import MonitoringRadioSender
-from parsl.monitoring.radios.filesystem import FilesystemRadioSender
-from parsl.monitoring.radios.htex import HTEXRadioSender
-from parsl.monitoring.radios.udp import UDPRadioSender
+from parsl.monitoring.radios.base import RadioConfig
 from parsl.multiprocessing import ForkProcess
 from parsl.process_loggers import wrap_with_logs
 
@@ -23,11 +20,10 @@ def monitor_wrapper(*,
                     kwargs: Dict,     # per invocation
                     x_try_id: int,    # per invocation
                     x_task_id: int,   # per invocation
-                    monitoring_hub_url: str,   # per workflow
+                    radio_config: RadioConfig,   # per executor
                     run_id: str,      # per workflow
                     logging_level: int,  # per workflow
                     sleep_dur: float,  # per workflow
-                    radio_mode: str,   # per executor
                     monitor_resources: bool,  # per workflow
                     run_dir: str) -> Tuple[Callable, Sequence, Dict]:
     """Wrap the Parsl app with a function that will call the monitor function and point it at the correct pid when the task begins.
@@ -41,9 +37,8 @@ def monitor_wrapper(*,
         # Send first message to monitoring router
         send_first_message(try_id,
                            task_id,
-                           monitoring_hub_url,
+                           radio_config,
                            run_id,
-                           radio_mode,
                            run_dir)
 
         if monitor_resources and sleep_dur > 0:
@@ -52,9 +47,8 @@ def monitor_wrapper(*,
                              args=(os.getpid(),
                                    try_id,
                                    task_id,
-                                   monitoring_hub_url,
+                                   radio_config,
                                    run_id,
-                                   radio_mode,
                                    logging_level,
                                    sleep_dur,
                                    run_dir,
@@ -87,9 +81,9 @@ def monitor_wrapper(*,
 
             send_last_message(try_id,
                               task_id,
-                              monitoring_hub_url,
+                              radio_config,
                               run_id,
-                              radio_mode, run_dir)
+                              run_dir)
 
     new_kwargs = kwargs.copy()
     new_kwargs['_parsl_monitoring_task_id'] = x_task_id
@@ -98,47 +92,33 @@ def monitor_wrapper(*,
     return (wrapped, args, new_kwargs)
 
 
-def get_radio(radio_mode: str, monitoring_hub_url: str, task_id: int, run_dir: str) -> MonitoringRadioSender:
-    radio: MonitoringRadioSender
-    if radio_mode == "udp":
-        radio = UDPRadioSender(monitoring_hub_url)
-    elif radio_mode == "htex":
-        radio = HTEXRadioSender(monitoring_hub_url)
-    elif radio_mode == "filesystem":
-        radio = FilesystemRadioSender(monitoring_url=monitoring_hub_url,
-                                      run_dir=run_dir)
-    else:
-        raise RuntimeError(f"Unknown radio mode: {radio_mode}")
-    return radio
-
-
 @wrap_with_logs
 def send_first_message(try_id: int,
                        task_id: int,
-                       monitoring_hub_url: str,
-                       run_id: str, radio_mode: str, run_dir: str) -> None:
-    send_first_last_message(try_id, task_id, monitoring_hub_url, run_id,
-                            radio_mode, run_dir, False)
+                       radio_config: RadioConfig,
+                       run_id: str, run_dir: str) -> None:
+    send_first_last_message(try_id, task_id, radio_config, run_id,
+                            run_dir, False)
 
 
 @wrap_with_logs
 def send_last_message(try_id: int,
                       task_id: int,
-                      monitoring_hub_url: str,
-                      run_id: str, radio_mode: str, run_dir: str) -> None:
-    send_first_last_message(try_id, task_id, monitoring_hub_url, run_id,
-                            radio_mode, run_dir, True)
+                      radio_config: RadioConfig,
+                      run_id: str, run_dir: str) -> None:
+    send_first_last_message(try_id, task_id, radio_config, run_id,
+                            run_dir, True)
 
 
 def send_first_last_message(try_id: int,
                             task_id: int,
-                            monitoring_hub_url: str,
-                            run_id: str, radio_mode: str, run_dir: str,
+                            radio_config: RadioConfig,
+                            run_id: str, run_dir: str,
                             is_last: bool) -> None:
     import os
     import platform
 
-    radio = get_radio(radio_mode, monitoring_hub_url, task_id, run_dir)
+    radio = radio_config.create_sender()
 
     msg = (MessageType.RESOURCE_INFO,
            {'run_id': run_id,
@@ -158,9 +138,8 @@ def send_first_last_message(try_id: int,
 def monitor(pid: int,
             try_id: int,
             task_id: int,
-            monitoring_hub_url: str,
+            radio_config: RadioConfig,
             run_id: str,
-            radio_mode: str,
             logging_level: int,
             sleep_dur: float,
             run_dir: str,
@@ -184,7 +163,7 @@ def monitor(pid: int,
 
     setproctitle("parsl: task resource monitor")
 
-    radio = get_radio(radio_mode, monitoring_hub_url, task_id, run_dir)
+    radio = radio_config.create_sender()
 
     logging.debug("start of monitor")