parsl 2025.6.16__py3-none-any.whl → 2025.6.30__py3-none-any.whl

parsl/executors/high_throughput/process_worker_pool.py

@@ -1,7 +1,6 @@
 #!/usr/bin/env python3
 
 import argparse
-import json
 import logging
 import math
 import multiprocessing
@@ -66,8 +65,7 @@ class Manager:
     def __init__(self, *,
                  addresses,
                  address_probe_timeout,
-                 task_port,
-                 result_port,
+                 port,
                  cores_per_worker,
                  mem_per_worker,
                  max_workers_per_node,
@@ -157,15 +155,13 @@ class Manager:
         self._start_time = time.time()
 
         try:
-            ix_address = probe_addresses(addresses.split(','), task_port, timeout=address_probe_timeout)
+            ix_address = probe_addresses(addresses.split(','), port, timeout=address_probe_timeout)
             if not ix_address:
                 raise Exception("No viable address found")
             else:
-                logger.info("Connection to Interchange successful on {}".format(ix_address))
-                task_q_url = tcp_url(ix_address, task_port)
-                result_q_url = tcp_url(ix_address, result_port)
-                logger.info("Task url : {}".format(task_q_url))
-                logger.info("Result url : {}".format(result_q_url))
+                logger.info(f"Connection to Interchange successful on {ix_address}")
+                ix_url = tcp_url(ix_address, port)
+                logger.info(f"Interchange url: {ix_url}")
         except Exception:
             logger.exception("Caught exception while trying to determine viable address to interchange")
             print("Failed to find a viable address to connect to interchange. Exiting")
@@ -174,8 +170,7 @@ class Manager:
         self.cert_dir = cert_dir
         self.zmq_context = curvezmq.ClientContext(self.cert_dir)
 
-        self._task_q_url = task_q_url
-        self._result_q_url = result_q_url
+        self._ix_url = ix_url
 
         self.uid = uid
         self.block_id = block_id
@@ -250,37 +245,37 @@ class Manager:
             self.worker_count = min(len(self.available_accelerators), self.worker_count)
         logger.info("Manager will spawn {} workers".format(self.worker_count))
 
-    def create_reg_message(self):
+    def create_reg_message(self) -> dict:
         """ Creates a registration message to identify the worker to the interchange
         """
-        msg = {'type': 'registration',
-               'parsl_v': PARSL_VERSION,
-               'python_v': "{}.{}.{}".format(sys.version_info.major,
-                                             sys.version_info.minor,
-                                             sys.version_info.micro),
-               'packages': {dist.metadata['Name']: dist.version for dist in distributions()},
-               'worker_count': self.worker_count,
-               'uid': self.uid,
-               'block_id': self.block_id,
-               'start_time': self.start_time,
-               'prefetch_capacity': self.prefetch_capacity,
-               'max_capacity': self.worker_count + self.prefetch_capacity,
-               'os': platform.system(),
-               'hostname': platform.node(),
-               'dir': os.getcwd(),
-               'cpu_count': psutil.cpu_count(logical=False),
-               'total_memory': psutil.virtual_memory().total,
-               }
-        b_msg = json.dumps(msg).encode('utf-8')
-        return b_msg
+        return {
+            'type': 'registration',
+            'parsl_v': PARSL_VERSION,
+            'python_v': "{}.{}.{}".format(
+                sys.version_info.major,
+                sys.version_info.minor,
+                sys.version_info.micro
+            ),
+            'packages': {d.metadata['Name']: d.version for d in distributions()},
+            'worker_count': self.worker_count,
+            'uid': self.uid,
+            'block_id': self.block_id,
+            'start_time': self.start_time,
+            'prefetch_capacity': self.prefetch_capacity,
+            'max_capacity': self.worker_count + self.prefetch_capacity,
+            'os': platform.system(),
+            'hostname': platform.node(),
+            'dir': os.getcwd(),
+            'cpu_count': psutil.cpu_count(logical=False),
+            'total_memory': psutil.virtual_memory().total,
+        }
 
     @staticmethod
     def heartbeat_to_incoming(task_incoming: zmq.Socket) -> None:
         """ Send heartbeat to the incoming task queue
         """
-        msg = {'type': 'heartbeat'}
         # don't need to dumps and encode this every time - could do as a global on import?
-        b_msg = json.dumps(msg).encode('utf-8')
+        b_msg = pickle.dumps({'type': 'heartbeat'})
         task_incoming.send(b_msg)
         logger.debug("Sent heartbeat")
 
@@ -289,32 +284,38 @@ class Manager:
         """ Send heartbeat to the incoming task queue
         """
         msg = {'type': 'drain'}
-        b_msg = json.dumps(msg).encode('utf-8')
+        b_msg = pickle.dumps(msg)
         task_incoming.send(b_msg)
         logger.debug("Sent drain")
 
     @wrap_with_logs
-    def pull_tasks(self):
+    def interchange_communicator(self, pair_setup: threading.Event):
         """ Pull tasks from the incoming tasks zmq pipe onto the internal
         pending task queue
         """
         logger.info("starting")
 
+        results_sock = self.zmq_context.socket(zmq.PAIR)
+        results_sock.setsockopt(zmq.LINGER, 0)
+        results_sock.bind("inproc://results")
+        pair_setup.set()
+
         # Linger is set to 0, so that the manager can exit even when there might be
         # messages in the pipe
-        task_incoming = self.zmq_context.socket(zmq.DEALER)
-        task_incoming.setsockopt(zmq.IDENTITY, self.uid.encode('utf-8'))
-        task_incoming.setsockopt(zmq.LINGER, 0)
-        task_incoming.connect(self._task_q_url)
+        ix_sock = self.zmq_context.socket(zmq.DEALER)
+        ix_sock.setsockopt(zmq.IDENTITY, self.uid.encode('utf-8'))
+        ix_sock.setsockopt(zmq.LINGER, 0)
+        ix_sock.connect(self._ix_url)
         logger.info("Manager task pipe connected to interchange")
 
         poller = zmq.Poller()
-        poller.register(task_incoming, zmq.POLLIN)
+        poller.register(results_sock, zmq.POLLIN)
+        poller.register(ix_sock, zmq.POLLIN)
 
         # Send a registration message
         msg = self.create_reg_message()
-        logger.debug("Sending registration message: {}".format(msg))
-        task_incoming.send(msg)
+        logger.debug("Sending registration message: %s", msg)
+        ix_sock.send(pickle.dumps(msg))
         last_beat = time.time()
         last_interchange_contact = time.time()
         task_recv_counter = 0
@@ -335,18 +336,21 @@ class Manager:
                 pending_task_count = self.pending_task_queue.qsize()
             except NotImplementedError:
                 # Ref: https://github.com/python/cpython/blob/6d5e0dc0e330f4009e8dc3d1642e46b129788877/Lib/multiprocessing/queues.py#L125
-                pending_task_count = f"pending task count is not available on {platform.system()}"
+                pending_task_count = f"pending task count is not available on {platform.system()}"  # type: ignore[assignment]
 
-            logger.debug("ready workers: {}, pending tasks: {}".format(self.ready_worker_count.value,
-                                                                       pending_task_count))
+            logger.debug(
+                'ready workers: %d, pending tasks: %d',
+                self.ready_worker_count.value,  # type: ignore[attr-defined]
+                pending_task_count,
+            )
 
             if time.time() >= last_beat + self.heartbeat_period:
-                self.heartbeat_to_incoming(task_incoming)
+                self.heartbeat_to_incoming(ix_sock)
                 last_beat = time.time()
 
             if time.time() > self.drain_time:
                 logger.info("Requesting drain")
-                self.drain_to_incoming(task_incoming)
+                self.drain_to_incoming(ix_sock)
                 # This will start the pool draining...
                 # Drained exit behaviour does not happen here. It will be
                 # driven by the interchange sending a DRAINED_CODE message.
@@ -358,8 +362,8 @@ class Manager:
             poll_duration_s = max(0, next_interesting_event_time - time.time())
             socks = dict(poller.poll(timeout=poll_duration_s * 1000))
 
-            if socks.get(task_incoming) == zmq.POLLIN:
-                _, pkl_msg = task_incoming.recv_multipart()
+            if socks.get(ix_sock) == zmq.POLLIN:
+                pkl_msg = ix_sock.recv()
                 tasks = pickle.loads(pkl_msg)
                 last_interchange_contact = time.time()
 
@@ -377,6 +381,11 @@ class Manager:
                     for task in tasks:
                         self.task_scheduler.put_task(task)
 
+            elif socks.get(results_sock) == zmq.POLLIN:
+                meta_b = pickle.dumps({'type': 'result'})
+                ix_sock.send_multipart([meta_b, results_sock.recv()])
+                logger.debug("Result sent to interchange")
+
             else:
                 logger.debug("No incoming tasks")
 
@@ -387,45 +396,36 @@ class Manager:
                     logger.critical("Exiting")
                     break
 
-        task_incoming.close()
+        ix_sock.close()
         logger.info("Exiting")
 
     @wrap_with_logs
-    def push_results(self):
-        """ Listens on the pending_result_queue and sends out results via zmq
+    def ferry_result(self, may_connect: threading.Event):
+        """ Listens on the pending_result_queue and ferries results to the interchange
+         connected thread
         """
-        logger.debug("Starting result push thread")
+        logger.debug("Begin")
 
         # Linger is set to 0, so that the manager can exit even when there might be
         # messages in the pipe
-        result_outgoing = self.zmq_context.socket(zmq.DEALER)
-        result_outgoing.setsockopt(zmq.IDENTITY, self.uid.encode('utf-8'))
-        result_outgoing.setsockopt(zmq.LINGER, 0)
-        result_outgoing.connect(self._result_q_url)
-        logger.info("Manager result pipe connected to interchange")
+        notify_sock = self.zmq_context.socket(zmq.PAIR)
+        notify_sock.setsockopt(zmq.LINGER, 0)
+        may_connect.wait()
+        notify_sock.connect("inproc://results")
 
         while not self._stop_event.is_set():
-            logger.debug("Starting pending_result_queue get")
             try:
                 r = self.task_scheduler.get_result()
                 if r is None:
                     continue
-                logger.debug("Result received from worker: %s", id(r))
-                result_outgoing.send(r)
-                logger.debug("Result sent to interchange: %s", id(r))
+                logger.debug("Result received from worker")
+                notify_sock.send(r)
             except Exception:
                 logger.exception("Failed to send result to interchange")
 
-        result_outgoing.close()
+        notify_sock.close()
         logger.debug("Exiting")
 
-    @wrap_with_logs
-    def heartbeater(self):
-        while not self._stop_event.wait(self.heartbeat_period):
-            heartbeat_message = f"heartbeat_period={self.heartbeat_period} seconds"
-            logger.info(f"Sending heartbeat via results connection: {heartbeat_message}")
-            self.pending_result_queue.put(pickle.dumps({'type': 'heartbeat'}))
-
     def worker_watchdog(self, procs: dict[int, SpawnProcess]):
         """Keeps workers alive."""
         logger.debug("Starting worker watchdog")
@@ -492,23 +492,26 @@ class Manager:
 
         logger.debug("Workers started")
 
-        thr_task_puller = threading.Thread(target=self.pull_tasks, name="Task-Puller")
-        thr_result_pusher = threading.Thread(
-            target=self.push_results, name="Result-Pusher"
+        pair_setup = threading.Event()
+
+        thr_task_puller = threading.Thread(
+            target=self.interchange_communicator,
+            args=(pair_setup,),
+            name="Interchange-Communicator",
         )
+        thr_result_ferry = threading.Thread(
+            target=self.ferry_result, args=(pair_setup,), name="Result-Shovel")
         thr_worker_watchdog = threading.Thread(
             target=self.worker_watchdog, args=(procs,), name="worker-watchdog"
         )
         thr_monitoring_handler = threading.Thread(
             target=self.handle_monitoring_messages, name="Monitoring-Handler"
         )
-        thr_heartbeater = threading.Thread(target=self.heartbeater, name="Heartbeater")
 
         thr_task_puller.start()
-        thr_result_pusher.start()
+        thr_result_ferry.start()
         thr_worker_watchdog.start()
         thr_monitoring_handler.start()
-        thr_heartbeater.start()
 
         logger.info("Manager threads started")
 
@@ -520,11 +523,10 @@ class Manager:
         self.monitoring_queue.put(None)
         self.pending_result_queue.put(None)
 
-        thr_heartbeater.join()
-        thr_task_puller.join()
-        thr_result_pusher.join()
-        thr_worker_watchdog.join()
         thr_monitoring_handler.join()
+        thr_worker_watchdog.join()
+        thr_result_ferry.join()
+        thr_task_puller.join()
 
         for worker_id in procs:
             p = procs[worker_id]
@@ -862,10 +864,10 @@ def get_arg_parser() -> argparse.ArgumentParser:
         help="GB of memory assigned to each worker process. Default=0, no assignment",
     )
     parser.add_argument(
-        "-t",
-        "--task_port",
+        "-P",
+        "--port",
         required=True,
-        help="Task port for receiving tasks from the interchange",
+        help="Port for communication with the interchange",
     )
     parser.add_argument(
         "--max_workers_per_node",
@@ -901,12 +903,6 @@ def get_arg_parser() -> argparse.ArgumentParser:
     parser.add_argument(
         "--poll", default=10, help="Poll period used in milliseconds"
     )
-    parser.add_argument(
-        "-r",
-        "--result_port",
-        required=True,
-        help="Result port for posting results to the interchange",
-    )
     parser.add_argument(
         "--cpu-affinity",
         type=strategyorlist,
@@ -953,8 +949,7 @@ if __name__ == "__main__":
         f"\n  Block ID: {args.block_id}"
         f"\n  cores_per_worker: {args.cores_per_worker}"
         f"\n  mem_per_worker: {args.mem_per_worker}"
-        f"\n  task_port: {args.task_port}"
-        f"\n  result_port: {args.result_port}"
+        f"\n  Interchange port: {args.port}"
         f"\n  addresses: {args.addresses}"
         f"\n  max_workers_per_node: {args.max_workers_per_node}"
         f"\n  poll_period: {args.poll}"
@@ -969,8 +964,7 @@ if __name__ == "__main__":
         f"\n  mpi_launcher: {args.mpi_launcher}"
     )
     try:
-        manager = Manager(task_port=args.task_port,
-                          result_port=args.result_port,
+        manager = Manager(port=args.port,
                           addresses=args.addresses,
                           address_probe_timeout=int(args.address_probe_timeout),
                           uid=args.uid,

parsl/executors/radical/executor.py

@@ -601,6 +601,9 @@ class RadicalPilotExecutor(ParslExecutor, RepresentationMixin):
             self._bulk_thread.join()
 
         self.session.close(download=True)
+
+        super().shutdown()
+
         logger.info("RadicalPilotExecutor is terminated.")
 
         return True

parsl/executors/taskvine/executor.py

@@ -40,6 +40,8 @@ from parsl.executors.taskvine.factory_config import TaskVineFactoryConfig
 from parsl.executors.taskvine.manager import _taskvine_submit_wait
 from parsl.executors.taskvine.manager_config import TaskVineManagerConfig
 from parsl.executors.taskvine.utils import ParslFileToVine, ParslTaskToVine
+from parsl.monitoring.radios.base import RadioConfig
+from parsl.monitoring.radios.filesystem import FilesystemRadio
 from parsl.multiprocessing import SpawnContext
 from parsl.process_loggers import wrap_with_logs
 from parsl.providers import CondorProvider, LocalProvider
@@ -98,8 +100,6 @@ class TaskVineExecutor(BlockProviderExecutor, putils.RepresentationMixin):
             Default is None.
     """
 
-    radio_mode = "filesystem"
-
     @typeguard.typechecked
     def __init__(self,
                  label: str = "TaskVineExecutor",
@@ -108,7 +108,8 @@ class TaskVineExecutor(BlockProviderExecutor, putils.RepresentationMixin):
                  manager_config: TaskVineManagerConfig = TaskVineManagerConfig(),
                  factory_config: TaskVineFactoryConfig = TaskVineFactoryConfig(),
                  provider: Optional[ExecutionProvider] = LocalProvider(init_blocks=1),
-                 storage_access: Optional[List[Staging]] = None):
+                 storage_access: Optional[List[Staging]] = None,
+                 remote_monitoring_radio: Optional[RadioConfig] = None):
 
         # Set worker launch option for this executor
         if worker_launch_method == 'factory' or worker_launch_method == 'manual':
@@ -134,6 +135,11 @@ class TaskVineExecutor(BlockProviderExecutor, putils.RepresentationMixin):
         self.factory_config = factory_config
         self.storage_access = storage_access
 
+        if remote_monitoring_radio is not None:
+            self.remote_monitoring_radio = remote_monitoring_radio
+        else:
+            self.remote_monitoring_radio = FilesystemRadio()
+
         # Queue to send ready tasks from TaskVine executor process to TaskVine manager process
         self._ready_task_queue: multiprocessing.Queue = SpawnContext.Queue()
 
@@ -601,6 +607,8 @@ class TaskVineExecutor(BlockProviderExecutor, putils.RepresentationMixin):
         self._finished_task_queue.close()
         self._finished_task_queue.join_thread()
 
+        super().shutdown()
+
         logger.debug("TaskVine shutdown completed")
 
     @wrap_with_logs

parsl/executors/taskvine/manager.py

@@ -6,6 +6,7 @@ import shutil
 import subprocess
 import uuid
 
+import parsl
 from parsl.executors.taskvine import exec_parsl_function
 from parsl.executors.taskvine.utils import VineTaskToParsl, run_parsl_function
 from parsl.process_loggers import wrap_with_logs
@@ -255,7 +256,8 @@ def _taskvine_submit_wait(ready_task_queue=None,
                                                                      run_parsl_function,
                                                                      poncho_env=poncho_env_path,
                                                                      init_command=manager_config.init_command,
-                                                                     add_env=add_env)
+                                                                     add_env=add_env,
+                                                                     hoisting_modules=[parsl.serialize, run_parsl_function])
 
                     # Configure the library if provided
                     if manager_config.library_config:

parsl/executors/threads.py

@@ -7,6 +7,8 @@ import typeguard
 from parsl.data_provider.staging import Staging
 from parsl.executors.base import ParslExecutor
 from parsl.executors.errors import InvalidResourceSpecification
+from parsl.monitoring.radios.base import RadioConfig
+from parsl.monitoring.radios.multiprocessing import MultiprocessingQueueRadio
 from parsl.utils import RepresentationMixin
 
 logger = logging.getLogger(__name__)
@@ -28,7 +30,7 @@ class ThreadPoolExecutor(ParslExecutor, RepresentationMixin):
     @typeguard.typechecked
     def __init__(self, label: str = 'threads', max_threads: Optional[int] = 2,
                  thread_name_prefix: str = '', storage_access: Optional[List[Staging]] = None,
-                 working_dir: Optional[str] = None):
+                 working_dir: Optional[str] = None, remote_monitoring_radio: Optional[RadioConfig] = None):
         ParslExecutor.__init__(self)
         self.label = label
         self.max_threads = max_threads
@@ -40,6 +42,11 @@ class ThreadPoolExecutor(ParslExecutor, RepresentationMixin):
         self.storage_access = storage_access
         self.working_dir = working_dir
 
+        if remote_monitoring_radio is not None:
+            self.remote_monitoring_radio = remote_monitoring_radio
+        else:
+            self.remote_monitoring_radio = MultiprocessingQueueRadio()
+
     def start(self):
         self.executor = cf.ThreadPoolExecutor(max_workers=self.max_threads,
                                               thread_name_prefix=self.thread_name_prefix)
@@ -73,9 +80,18 @@ class ThreadPoolExecutor(ParslExecutor, RepresentationMixin):
         """
         logger.debug("Shutting down executor, which involves waiting for running tasks to complete")
         self.executor.shutdown(wait=block)
+
+        super().shutdown()
+
         logger.debug("Done with executor shutdown")
 
     def monitor_resources(self):
-        """Resource monitoring sometimes deadlocks when using threads, so this function
-        returns false to disable it."""
+        """Resource monitoring does not make sense when using the
+        ThreadPoolExecutor, as there is no per-task process tree: all tasks
+        run inside the same single submitting process.
+
+        In addition, the use of fork-based multiprocessing in the remote
+        wrapper in parsl/monitoring/remote.py was especially prone to deadlock
+        with this executor.
+        """
         return False

parsl/executors/workqueue/executor.py

@@ -31,6 +31,8 @@ from parsl.errors import OptionalModuleMissing
 from parsl.executors.errors import ExecutorError, InvalidResourceSpecification
 from parsl.executors.status_handling import BlockProviderExecutor
 from parsl.executors.workqueue import exec_parsl_function
+from parsl.monitoring.radios.base import RadioConfig
+from parsl.monitoring.radios.filesystem import FilesystemRadio
 from parsl.multiprocessing import SpawnContext, SpawnProcess
 from parsl.process_loggers import wrap_with_logs
 from parsl.providers import CondorProvider, LocalProvider
@@ -227,8 +229,6 @@ class WorkQueueExecutor(BlockProviderExecutor, putils.RepresentationMixin):
             specifiation for each task).
     """
 
-    radio_mode = "filesystem"
-
     @typeguard.typechecked
     def __init__(self,
                  label: str = "WorkQueueExecutor",
@@ -255,7 +255,8 @@ class WorkQueueExecutor(BlockProviderExecutor, putils.RepresentationMixin):
                  worker_executable: str = 'work_queue_worker',
                  function_dir: Optional[str] = None,
                  coprocess: bool = False,
-                 scaling_cores_per_worker: int = 1):
+                 scaling_cores_per_worker: int = 1,
+                 remote_monitoring_radio: Optional[RadioConfig] = None):
         BlockProviderExecutor.__init__(self, provider=provider,
                                        block_error_handler=True)
         if not _work_queue_enabled:
@@ -308,6 +309,11 @@ class WorkQueueExecutor(BlockProviderExecutor, putils.RepresentationMixin):
         if self.init_command != "":
             self.launch_cmd = self.init_command + "; " + self.launch_cmd
 
+        if remote_monitoring_radio is not None:
+            self.remote_monitoring_radio = remote_monitoring_radio
+        else:
+            self.remote_monitoring_radio = FilesystemRadio()
+
     def _get_launch_command(self, block_id):
         # this executor uses different terminology for worker/launch
         # commands than in htex
@@ -714,6 +720,8 @@ class WorkQueueExecutor(BlockProviderExecutor, putils.RepresentationMixin):
         self.collector_queue.close()
         self.collector_queue.join_thread()
 
+        super().shutdown()
+
         logger.debug("Work Queue shutdown completed")
 
     @wrap_with_logs

parsl/monitoring/errors.py

@@ -1,11 +1,11 @@
 from parsl.errors import ParslError
 
 
-class MonitoringHubStartError(ParslError):
+class MonitoringRouterStartError(ParslError):
     def __str__(self) -> str:
-        return "Hub failed to start"
+        return "Monitoring router failed to start"
 
 
-class MonitoringRouterStartError(ParslError):
+class RadioRequiredError(ParslError):
     def __str__(self) -> str:
-        return "Monitoring router failed to start"
+        return "A radio must be configured for remote task monitoring"