parsl 2025.3.17__py3-none-any.whl → 2025.3.31__py3-none-any.whl

parsl/dataflow/dflow.py

@@ -45,6 +45,7 @@ from parsl.executors.threads import ThreadPoolExecutor
 from parsl.jobs.job_status_poller import JobStatusPoller
 from parsl.monitoring import MonitoringHub
 from parsl.monitoring.message_type import MessageType
+from parsl.monitoring.radios.multiprocessing import MultiprocessingQueueRadioSender
 from parsl.monitoring.remote import monitor_wrapper
 from parsl.process_loggers import wrap_with_logs
 from parsl.usage_tracking.usage import UsageTracker
@@ -110,8 +111,11 @@ class DataFlowKernel:
         self.monitoring: Optional[MonitoringHub]
         self.monitoring = config.monitoring
 
+        self.monitoring_radio = None
+
         if self.monitoring:
             self.monitoring.start(self.run_dir, self.config.run_dir)
+            self.monitoring_radio = MultiprocessingQueueRadioSender(self.monitoring.resource_msgs)
 
         self.time_began = datetime.datetime.now()
         self.time_completed: Optional[datetime.datetime] = None
@@ -156,9 +160,9 @@ class DataFlowKernel:
                 'host': gethostname(),
         }
 
-        if self.monitoring:
-            self.monitoring.send((MessageType.WORKFLOW_INFO,
-                                 workflow_info))
+        if self.monitoring_radio:
+            self.monitoring_radio.send((MessageType.WORKFLOW_INFO,
+                                       workflow_info))
 
         if config.checkpoint_files is not None:
             checkpoint_files = config.checkpoint_files
@@ -231,9 +235,9 @@ class DataFlowKernel:
             raise InternalConsistencyError(f"Exit case for {mode} should be unreachable, validated by typeguard on Config()")
 
     def _send_task_log_info(self, task_record: TaskRecord) -> None:
-        if self.monitoring:
+        if self.monitoring_radio:
             task_log_info = self._create_task_log_info(task_record)
-            self.monitoring.send((MessageType.TASK_INFO, task_log_info))
+            self.monitoring_radio.send((MessageType.TASK_INFO, task_log_info))
 
     def _create_task_log_info(self, task_record: TaskRecord) -> Dict[str, Any]:
         """
@@ -1128,9 +1132,7 @@ class DataFlowKernel:
             executor.run_id = self.run_id
             executor.run_dir = self.run_dir
             if self.monitoring:
-                executor.hub_address = self.monitoring.hub_address
-                executor.hub_zmq_port = self.monitoring.hub_zmq_port
-                executor.submit_monitoring_radio = self.monitoring.radio
+                executor.monitoring_messages = self.monitoring.resource_msgs
             if hasattr(executor, 'provider'):
                 if hasattr(executor.provider, 'script_dir'):
                     executor.provider.script_dir = os.path.join(self.run_dir, 'submit_scripts')
@@ -1217,15 +1219,16 @@ class DataFlowKernel:
         logger.info("Terminated executors")
         self.time_completed = datetime.datetime.now()
 
-        if self.monitoring:
+        if self.monitoring_radio:
             logger.info("Sending final monitoring message")
-            self.monitoring.send((MessageType.WORKFLOW_INFO,
-                                 {'tasks_failed_count': self.task_state_counts[States.failed],
-                                  'tasks_completed_count': self.task_state_counts[States.exec_done],
-                                  "time_began": self.time_began,
-                                  'time_completed': self.time_completed,
-                                  'run_id': self.run_id, 'rundir': self.run_dir}))
+            self.monitoring_radio.send((MessageType.WORKFLOW_INFO,
+                                       {'tasks_failed_count': self.task_state_counts[States.failed],
+                                        'tasks_completed_count': self.task_state_counts[States.exec_done],
+                                        "time_began": self.time_began,
+                                        'time_completed': self.time_completed,
+                                        'run_id': self.run_id, 'rundir': self.run_dir}))
 
+        if self.monitoring:
             logger.info("Terminating monitoring")
             self.monitoring.close()
             logger.info("Terminated monitoring")

parsl/executors/base.py

@@ -1,11 +1,14 @@
+from __future__ import annotations
+
 import os
 from abc import ABCMeta, abstractmethod
 from concurrent.futures import Future
+from multiprocessing.queues import Queue
 from typing import Any, Callable, Dict, Optional
 
 from typing_extensions import Literal, Self
 
-from parsl.monitoring.radios.base import MonitoringRadioSender
+from parsl.monitoring.types import TaggedMonitoringMessage
 
 
 class ParslExecutor(metaclass=ABCMeta):
@@ -42,6 +45,13 @@ class ParslExecutor(metaclass=ABCMeta):
               invariant, not co-variant, and it looks like @typeguard cannot be
               persuaded otherwise. So if you're implementing an executor and want to
               @typeguard the constructor, you'll have to use List[Any] here.
+
+    The DataFlowKernel will set this attribute before calling .start(),
+    if monitoring is enabled:
+
+        monitoring_messages: Optional[Queue[TaggedMonitoringMessage]] - an executor
+            can send messages to the monitoring hub by putting them into
+            this queue.
     """
 
     label: str = "undefined"
@@ -50,15 +60,11 @@ class ParslExecutor(metaclass=ABCMeta):
     def __init__(
         self,
         *,
-        hub_address: Optional[str] = None,
-        hub_zmq_port: Optional[int] = None,
-        submit_monitoring_radio: Optional[MonitoringRadioSender] = None,
+        monitoring_messages: Optional[Queue[TaggedMonitoringMessage]] = None,
         run_dir: str = ".",
         run_id: Optional[str] = None,
     ):
-        self.hub_address = hub_address
-        self.hub_zmq_port = hub_zmq_port
-        self.submit_monitoring_radio = submit_monitoring_radio
+        self.monitoring_messages = monitoring_messages
         self.run_dir = os.path.abspath(run_dir)
         self.run_id = run_id
 
@@ -125,33 +131,3 @@ class ParslExecutor(metaclass=ABCMeta):
     @run_id.setter
     def run_id(self, value: Optional[str]) -> None:
         self._run_id = value
-
-    @property
-    def hub_address(self) -> Optional[str]:
-        """Address to the Hub for monitoring.
-        """
-        return self._hub_address
-
-    @hub_address.setter
-    def hub_address(self, value: Optional[str]) -> None:
-        self._hub_address = value
-
-    @property
-    def hub_zmq_port(self) -> Optional[int]:
-        """Port to the Hub for monitoring.
-        """
-        return self._hub_zmq_port
-
-    @hub_zmq_port.setter
-    def hub_zmq_port(self, value: Optional[int]) -> None:
-        self._hub_zmq_port = value
-
-    @property
-    def submit_monitoring_radio(self) -> Optional[MonitoringRadioSender]:
-        """Local radio for sending monitoring messages
-        """
-        return self._submit_monitoring_radio
-
-    @submit_monitoring_radio.setter
-    def submit_monitoring_radio(self, value: Optional[MonitoringRadioSender]) -> None:
-        self._submit_monitoring_radio = value

parsl/executors/flux/executor.py

@@ -231,6 +231,7 @@ class FluxExecutor(ParslExecutor, RepresentationMixin):
 
     def start(self):
         """Called when DFK starts the executor when the config is loaded."""
+        super().start()
         os.makedirs(self.working_dir, exist_ok=True)
         self._submission_thread.start()
 

parsl/executors/globus_compute.py

@@ -2,10 +2,11 @@ from __future__ import annotations
 
 import copy
 from concurrent.futures import Future
-from typing import Any, Callable, Dict
+from typing import Any, Callable, Dict, List, Optional
 
 import typeguard
 
+from parsl.data_provider.staging import Staging
 from parsl.errors import OptionalModuleMissing
 from parsl.executors.base import ParslExecutor
 from parsl.utils import RepresentationMixin
@@ -40,6 +41,8 @@ class GlobusComputeExecutor(ParslExecutor, RepresentationMixin):
         self,
         executor: Executor,
         label: str = 'GlobusComputeExecutor',
+        storage_access: Optional[List[Staging]] = None,
+        working_dir: Optional[str] = None,
     ):
         """
         Parameters
@@ -52,6 +55,12 @@ class GlobusComputeExecutor(ParslExecutor, RepresentationMixin):
 
         label:
             a label to name the executor
+
+        storage_access:
+            a list of staging providers that will be used for file staging
+
+        working_dir:
+            The working dir to be used for file staging
         """
         if not _globus_compute_enabled:
             raise OptionalModuleMissing(
@@ -64,10 +73,12 @@ class GlobusComputeExecutor(ParslExecutor, RepresentationMixin):
         self.resource_specification = self.executor.resource_specification
         self.user_endpoint_config = self.executor.user_endpoint_config
         self.label = label
+        self.storage_access = storage_access
+        self.working_dir = working_dir
 
     def start(self) -> None:
         """ Start the Globus Compute Executor """
-        pass
+        super().start()
 
     def submit(self, func: Callable, resource_specification: Dict[str, Any], *args: Any, **kwargs: Any) -> Future:
         """ Submit func to globus-compute

parsl/executors/high_throughput/executor.py

@@ -29,6 +29,7 @@ from parsl.executors.high_throughput.manager_selector import (
 )
 from parsl.executors.status_handling import BlockProviderExecutor
 from parsl.jobs.states import TERMINAL_STATES, JobState, JobStatus
+from parsl.monitoring.radios.zmq_router import ZMQRadioReceiver, start_zmq_receiver
 from parsl.process_loggers import wrap_with_logs
 from parsl.providers import LocalProvider
 from parsl.providers.base import ExecutionProvider
@@ -334,6 +335,10 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin, UsageIn
         self._result_queue_thread_exit = threading.Event()
         self._result_queue_thread: Optional[threading.Thread] = None
 
+        self.zmq_monitoring: Optional[ZMQRadioReceiver]
+        self.zmq_monitoring = None
+        self.hub_zmq_port = None
+
     radio_mode = "htex"
     enable_mpi_mode: bool = False
     mpi_launcher: str = "mpiexec"
@@ -407,6 +412,7 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin, UsageIn
     def start(self):
         """Create the Interchange process and connect to it.
         """
+        super().start()
         if self.encrypted and self.cert_dir is None:
             logger.debug("Creating CurveZMQ certificates")
             self.cert_dir = curvezmq.create_certificates(self.logdir)
@@ -427,6 +433,15 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin, UsageIn
             self.loopback_address, self.interchange_port_range, self.cert_dir
         )
 
+        if self.monitoring_messages is not None:
+            self.zmq_monitoring = start_zmq_receiver(monitoring_messages=self.monitoring_messages,
+                                                     loopback_address=self.loopback_address,
+                                                     port_range=self.interchange_port_range,
+                                                     logdir=self.logdir,
+                                                     worker_debug=self.worker_debug,
+                                                     )
+            self.hub_zmq_port = self.zmq_monitoring.port
+
         self._result_queue_thread = None
         self._start_result_queue_thread()
         self._start_local_interchange_process()
@@ -861,6 +876,9 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin, UsageIn
         if self._result_queue_thread:
             self._result_queue_thread.join()
 
+        if self.zmq_monitoring:
+            self.zmq_monitoring.close()
+
         logger.info("Finished HighThroughputExecutor shutdown attempt")
 
     def get_usage_information(self):

parsl/executors/high_throughput/interchange.py

@@ -328,7 +328,7 @@ class Interchange:
             self.process_results_incoming(interesting_managers, monitoring_radio)
             self.expire_bad_managers(interesting_managers, monitoring_radio)
             self.expire_drained_managers(interesting_managers, monitoring_radio)
-            self.process_tasks_to_send(interesting_managers)
+            self.process_tasks_to_send(interesting_managers, monitoring_radio)
 
         self.zmq_context.destroy()
         delta = time.time() - start
@@ -452,7 +452,7 @@ class Interchange:
                 m['active'] = False
                 self._send_monitoring_info(monitoring_radio, m)
 
-    def process_tasks_to_send(self, interesting_managers: Set[bytes]) -> None:
+    def process_tasks_to_send(self, interesting_managers: Set[bytes], monitoring_radio: Optional[MonitoringRadioSender]) -> None:
         # Check if there are tasks that could be sent to managers
 
         logger.debug(
@@ -481,13 +481,14 @@ class Interchange:
                         m['idle_since'] = None
                         logger.debug("Sent tasks: %s to manager %r", tids, manager_id)
                         # recompute real_capacity after sending tasks
-                        real_capacity = m['max_capacity'] - tasks_inflight
+                        real_capacity -= task_count
                         if real_capacity > 0:
                             logger.debug("Manager %r has free capacity %s", manager_id, real_capacity)
                             # ... so keep it in the interesting_managers list
                         else:
                             logger.debug("Manager %r is now saturated", manager_id)
                             interesting_managers.remove(manager_id)
+                    self._send_monitoring_info(monitoring_radio, m)
                 else:
                     interesting_managers.remove(manager_id)
                     # logger.debug("Nothing to send to manager {}".format(manager_id))
@@ -505,13 +506,24 @@ class Interchange:
             else:
                 logger.debug("Got %s result items in batch from manager %r", len(all_messages), manager_id)
 
-                b_messages = []
+                m = self._ready_managers[manager_id]
+                b_messages_to_send = []
 
                 for p_message in all_messages:
                     r = pickle.loads(p_message)
                     if r['type'] == 'result':
                         # process this for task ID and forward to executor
-                        b_messages.append((p_message, r))
+                        logger.debug("Removing task %s from manager record %r", r["task_id"], manager_id)
+                        try:
+                            m['tasks'].remove(r['task_id'])
+                            b_messages_to_send.append(p_message)
+                        except Exception:
+                            logger.exception(
+                                "Ignoring exception removing task_id %s for manager %r with task list %s",
+                                r['task_id'],
+                                manager_id,
+                                m["tasks"]
+                            )
                     elif r['type'] == 'monitoring':
                         # the monitoring code makes the assumption that no
                         # monitoring messages will be received if monitoring
@@ -525,43 +537,21 @@ class Interchange:
                     else:
                         logger.error("Interchange discarding result_queue message of unknown type: %s", r["type"])
 
-                got_result = False
-                m = self._ready_managers[manager_id]
-                for (_, r) in b_messages:
-                    assert 'type' in r, f"Message is missing type entry: {r}"
-                    if r['type'] == 'result':
-                        got_result = True
-                        try:
-                            logger.debug("Removing task %s from manager record %r", r["task_id"], manager_id)
-                            m['tasks'].remove(r['task_id'])
-                        except Exception:
-                            # If we reach here, there's something very wrong.
-                            logger.exception(
-                                "Ignoring exception removing task_id %s for manager %r with task list %s",
-                                r['task_id'],
-                                manager_id,
-                                m["tasks"]
-                            )
-
-                b_messages_to_send = []
-                for (b_message, _) in b_messages:
-                    b_messages_to_send.append(b_message)
-
                 if b_messages_to_send:
                     logger.debug("Sending messages on results_outgoing")
                     self.results_outgoing.send_multipart(b_messages_to_send)
                     logger.debug("Sent messages on results_outgoing")
 
-                logger.debug("Current tasks on manager %r: %s", manager_id, m["tasks"])
-                if len(m['tasks']) == 0 and m['idle_since'] is None:
-                    m['idle_since'] = time.time()
-
-                # A manager is only made interesting here if a result was
-                # received, which means there should be capacity for a new
-                # task now. Heartbeats and monitoring messages do not make a
-                # manager become interesting.
-                if got_result:
+                    # At least one result received, so manager now has idle capacity
                     interesting_managers.add(manager_id)
+
+                    if len(m['tasks']) == 0 and m['idle_since'] is None:
+                        m['idle_since'] = time.time()
+
+                    self._send_monitoring_info(monitoring_radio, m)
+
+                logger.debug("Current tasks on manager %r: %s", manager_id, m["tasks"])
+
             logger.debug("leaving results_incoming section")
 
     def expire_bad_managers(self, interesting_managers: Set[bytes], monitoring_radio: Optional[MonitoringRadioSender]) -> None:

parsl/executors/radical/executor.py

@@ -215,6 +215,7 @@ class RadicalPilotExecutor(ParslExecutor, RepresentationMixin):
         """Create the Pilot component and pass it.
         """
         logger.info("starting RadicalPilotExecutor")
+        super().start()
         logger.info('Parsl: {0}'.format(parsl.__version__))
         logger.info('RADICAL pilot: {0}'.format(rp.version))
         self.session = rp.Session(cfg={'base': self.run_dir},

parsl/executors/status_handling.py

@@ -14,6 +14,7 @@ from parsl.executors.errors import BadStateException, ScalingFailed
 from parsl.jobs.error_handlers import noop_error_handler, simple_error_handler
 from parsl.jobs.states import TERMINAL_STATES, JobState, JobStatus
 from parsl.monitoring.message_type import MessageType
+from parsl.monitoring.radios.multiprocessing import MultiprocessingQueueRadioSender
 from parsl.providers.base import ExecutionProvider
 from parsl.utils import AtomicIDCounter
 
@@ -83,6 +84,13 @@ class BlockProviderExecutor(ParslExecutor):
         # of pending, active and recently terminated blocks
         self._status = {}  # type: Dict[str, JobStatus]
 
+        self.submit_monitoring_radio: Optional[MultiprocessingQueueRadioSender] = None
+
+    def start(self):
+        super().start()
+        if self.monitoring_messages:
+            self.submit_monitoring_radio = MultiprocessingQueueRadioSender(self.monitoring_messages)
+
     def _make_status_dict(self, block_ids: List[str], status_list: List[JobStatus]) -> Dict[str, JobStatus]:
         """Given a list of block ids and a list of corresponding status strings,
         returns a dictionary mapping each block id to the corresponding status
@@ -281,20 +289,20 @@ class BlockProviderExecutor(ParslExecutor):
             logger.debug("Sending block monitoring message: %r", msg)
             self.submit_monitoring_radio.send((MessageType.BLOCK_INFO, msg))
 
-    def create_monitoring_info(self, status: Dict[str, JobStatus]) -> Sequence[object]:
+    def create_monitoring_info(self, status: Dict[str, JobStatus]) -> Sequence[Dict[str, Any]]:
         """Create a monitoring message for each block based on the poll status.
         """
-        msg = []
-        for bid, s in status.items():
-            d: Dict[str, Any] = {}
-            d['run_id'] = self.run_id
-            d['status'] = s.status_name
-            d['timestamp'] = datetime.datetime.now()
-            d['executor_label'] = self.label
-            d['job_id'] = self.blocks_to_job_id.get(bid, None)
-            d['block_id'] = bid
-            msg.append(d)
-        return msg
+        return [
+            {
+                "run_id": self.run_id,
+                "status": s.status_name,
+                "timestamp": datetime.datetime.now(),
+                "executor_label": self.label,
+                "job_id": self.blocks_to_job_id.get(bid, None),
+                "block_id": bid
+            }
+            for bid, s in status.items()
+        ]
 
     def poll_facade(self) -> None:
         now = time.time()

parsl/executors/taskvine/executor.py

@@ -40,6 +40,7 @@ from parsl.executors.taskvine.factory_config import TaskVineFactoryConfig
 from parsl.executors.taskvine.manager import _taskvine_submit_wait
 from parsl.executors.taskvine.manager_config import TaskVineManagerConfig
 from parsl.executors.taskvine.utils import ParslFileToVine, ParslTaskToVine
+from parsl.multiprocessing import SpawnContext
 from parsl.process_loggers import wrap_with_logs
 from parsl.providers import CondorProvider, LocalProvider
 from parsl.providers.base import ExecutionProvider
@@ -134,13 +135,13 @@ class TaskVineExecutor(BlockProviderExecutor, putils.RepresentationMixin):
         self.storage_access = storage_access
 
         # Queue to send ready tasks from TaskVine executor process to TaskVine manager process
-        self._ready_task_queue: multiprocessing.Queue = multiprocessing.Queue()
+        self._ready_task_queue: multiprocessing.Queue = SpawnContext.Queue()
 
         # Queue to send finished tasks from TaskVine manager process to TaskVine executor process
-        self._finished_task_queue: multiprocessing.Queue = multiprocessing.Queue()
+        self._finished_task_queue: multiprocessing.Queue = SpawnContext.Queue()
 
         # Event to signal whether the manager and factory processes should stop running
-        self._should_stop = multiprocessing.Event()
+        self._should_stop = SpawnContext.Event()
 
         # TaskVine manager process
         self._submit_process = None
@@ -239,6 +240,7 @@ class TaskVineExecutor(BlockProviderExecutor, putils.RepresentationMixin):
         retrieve Parsl tasks within the TaskVine system.
         """
 
+        super().start()
         # Synchronize connection and communication settings between the manager and factory
         self.__synchronize_manager_factory_comm_settings()
 
@@ -252,17 +254,17 @@ class TaskVineExecutor(BlockProviderExecutor, putils.RepresentationMixin):
                                  "finished_task_queue": self._finished_task_queue,
                                  "should_stop": self._should_stop,
                                  "manager_config": self.manager_config}
-        self._submit_process = multiprocessing.Process(target=_taskvine_submit_wait,
-                                                       name="TaskVine-Submit-Process",
-                                                       kwargs=submit_process_kwargs)
+        self._submit_process = SpawnContext.Process(target=_taskvine_submit_wait,
+                                                    name="TaskVine-Submit-Process",
+                                                    kwargs=submit_process_kwargs)
 
         # Create a process to run the TaskVine factory if enabled.
         if self.worker_launch_method == 'factory':
             factory_process_kwargs = {"should_stop": self._should_stop,
                                       "factory_config": self.factory_config}
-            self._factory_process = multiprocessing.Process(target=_taskvine_factory,
-                                                            name="TaskVine-Factory-Process",
-                                                            kwargs=factory_process_kwargs)
+            self._factory_process = SpawnContext.Process(target=_taskvine_factory,
+                                                         name="TaskVine-Factory-Process",
+                                                         kwargs=factory_process_kwargs)
 
         # Run thread to collect results and set tasks' futures.
         self._collector_thread = threading.Thread(target=self._collect_taskvine_results,
@@ -621,8 +623,8 @@ class TaskVineExecutor(BlockProviderExecutor, putils.RepresentationMixin):
                 with self._tasks_lock:
                     future = self.tasks.pop(task_report.executor_id)
 
-                logger.debug(f'Updating Future for Parsl Task: {task_report.executor_id}. \
-                               Task {task_report.executor_id} has result_received set to {task_report.result_received}')
+                logger.debug(f'Updating Future for Parsl Task: {task_report.executor_id}. '
+                             f'Task {task_report.executor_id} has result_received set to {task_report.result_received}')
                 if task_report.result_received:
                     try:
                         with open(task_report.result_file, 'rb') as f_in:

parsl/executors/workqueue/executor.py

@@ -31,6 +31,7 @@ from parsl.errors import OptionalModuleMissing
 from parsl.executors.errors import ExecutorError, InvalidResourceSpecification
 from parsl.executors.status_handling import BlockProviderExecutor
 from parsl.executors.workqueue import exec_parsl_function
+from parsl.multiprocessing import SpawnContext, SpawnProcess
 from parsl.process_loggers import wrap_with_logs
 from parsl.providers import CondorProvider, LocalProvider
 from parsl.providers.base import ExecutionProvider
@@ -260,8 +261,8 @@ class WorkQueueExecutor(BlockProviderExecutor, putils.RepresentationMixin):
 
         self.scaling_cores_per_worker = scaling_cores_per_worker
         self.label = label
-        self.task_queue = multiprocessing.Queue()  # type: multiprocessing.Queue
-        self.collector_queue = multiprocessing.Queue()  # type: multiprocessing.Queue
+        self.task_queue: multiprocessing.Queue = SpawnContext.Queue()
+        self.collector_queue: multiprocessing.Queue = SpawnContext.Queue()
         self.address = address
         self.port = port
         self.executor_task_counter = -1
@@ -282,7 +283,7 @@ class WorkQueueExecutor(BlockProviderExecutor, putils.RepresentationMixin):
         self.autolabel_window = autolabel_window
         self.autocategory = autocategory
         self.max_retries = max_retries
-        self.should_stop = multiprocessing.Value(c_bool, False)
+        self.should_stop = SpawnContext.Value(c_bool, False)
         self.cached_envs = {}  # type: Dict[int, str]
         self.worker_options = worker_options
         self.worker_executable = worker_executable
@@ -314,6 +315,7 @@ class WorkQueueExecutor(BlockProviderExecutor, putils.RepresentationMixin):
         """Create submit process and collector thread to create, send, and
         retrieve Parsl tasks within the Work Queue system.
         """
+        super().start()
         self.tasks_lock = threading.Lock()
 
         # Create directories for data and results
@@ -333,7 +335,7 @@ class WorkQueueExecutor(BlockProviderExecutor, putils.RepresentationMixin):
 
         logger.debug("Starting WorkQueueExecutor")
 
-        port_mailbox = multiprocessing.Queue()
+        port_mailbox = SpawnContext.Queue()
 
         # Create a Process to perform WorkQueue submissions
         submit_process_kwargs = {"task_queue": self.task_queue,
@@ -354,9 +356,9 @@ class WorkQueueExecutor(BlockProviderExecutor, putils.RepresentationMixin):
                                  "port_mailbox": port_mailbox,
                                  "coprocess": self.coprocess
                                  }
-        self.submit_process = multiprocessing.Process(target=_work_queue_submit_wait,
-                                                      name="WorkQueue-Submit-Process",
-                                                      kwargs=submit_process_kwargs)
+        self.submit_process = SpawnProcess(target=_work_queue_submit_wait,
+                                           name="WorkQueue-Submit-Process",
+                                           kwargs=submit_process_kwargs)
 
         self.collector_thread = threading.Thread(target=self._collect_work_queue_results,
                                                  name="WorkQueue-collector-thread")

parsl/monitoring/errors.py

@@ -4,3 +4,8 @@ from parsl.errors import ParslError
 class MonitoringHubStartError(ParslError):
     def __str__(self) -> str:
         return "Hub failed to start"
+
+
+class MonitoringRouterStartError(ParslError):
+    def __str__(self) -> str:
+        return "Monitoring router failed to start"