parsl 2025.3.3__py3-none-any.whl → 2025.3.17__py3-none-any.whl

parsl/executors/high_throughput/executor.py

@@ -536,7 +536,7 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin, UsageIn
                               "interchange_address": self.address,
                               "worker_ports": self.worker_ports,
                               "worker_port_range": self.worker_port_range,
-                              "hub_address": self.hub_address,
+                              "hub_address": self.loopback_address,
                               "hub_zmq_port": self.hub_zmq_port,
                               "logdir": self.logdir,
                               "heartbeat_threshold": self.heartbeat_threshold,

parsl/executors/high_throughput/mpi_resource_management.py

@@ -4,6 +4,7 @@ import os
 import pickle
 import queue
 import subprocess
+from dataclasses import dataclass, field
 from enum import Enum
 from typing import Dict, List, Optional
 
@@ -69,6 +70,14 @@ class MPINodesUnavailable(Exception):
         return f"MPINodesUnavailable(requested={self.requested} available={self.available})"
 
 
+@dataclass(order=True)
+class PrioritizedTask:
+    # Comparing dict will fail since they are unhashable
+    # This dataclass limits comparison to the priority field
+    priority: int
+    task: Dict = field(compare=False)
+
+
 class TaskScheduler:
     """Default TaskScheduler that does no taskscheduling
 
@@ -111,7 +120,7 @@ class MPITaskScheduler(TaskScheduler):
         super().__init__(pending_task_q, pending_result_q)
         self.scheduler = identify_scheduler()
         # PriorityQueue is threadsafe
-        self._backlog_queue: queue.PriorityQueue = queue.PriorityQueue()
+        self._backlog_queue: queue.PriorityQueue[PrioritizedTask] = queue.PriorityQueue()
         self._map_tasks_to_nodes: Dict[str, List[str]] = {}
         self.available_nodes = get_nodes_in_batchjob(self.scheduler)
         self._free_node_counter = SpawnContext.Value("i", len(self.available_nodes))
@@ -169,7 +178,7 @@ class MPITaskScheduler(TaskScheduler):
                 allocated_nodes = self._get_nodes(nodes_needed)
             except MPINodesUnavailable:
                 logger.info(f"Not enough resources, placing task {tid} into backlog")
-                self._backlog_queue.put((nodes_needed, task_package))
+                self._backlog_queue.put(PrioritizedTask(nodes_needed, task_package))
                 return
             else:
                 resource_spec["MPI_NODELIST"] = ",".join(allocated_nodes)
@@ -183,8 +192,8 @@ class MPITaskScheduler(TaskScheduler):
     def _schedule_backlog_tasks(self):
         """Attempt to schedule backlogged tasks"""
         try:
-            _nodes_requested, task_package = self._backlog_queue.get(block=False)
-            self.put_task(task_package)
+            prioritized_task = self._backlog_queue.get(block=False)
+            self.put_task(prioritized_task.task)
         except queue.Empty:
             return
         else:
@@ -194,6 +203,8 @@ class MPITaskScheduler(TaskScheduler):
     def get_result(self, block: bool = True, timeout: Optional[float] = None):
         """Return result and relinquish provisioned nodes"""
         result_pkl = self.pending_result_q.get(block, timeout)
+        if result_pkl is None:
+            return None
         result_dict = pickle.loads(result_pkl)
         # TODO (wardlt): If the task did not request nodes, it won't be in `self._map_tasks_to_nodes`.
         #  Causes Parsl to hang. See Issue #3427

parsl/executors/high_throughput/process_worker_pool.py

@@ -15,6 +15,7 @@ import threading
 import time
 import uuid
 from importlib.metadata import distributions
+from multiprocessing.context import SpawnProcess
 from multiprocessing.managers import DictProxy
 from multiprocessing.sharedctypes import Synchronized
 from typing import Dict, List, Optional, Sequence
@@ -403,52 +404,34 @@ class Manager:
         result_outgoing.connect(self._result_q_url)
         logger.info("Manager result pipe connected to interchange")
 
-        push_poll_period = max(10, self.poll_period) / 1000    # push_poll_period must be atleast 10 ms
-        logger.debug("push poll period: {}".format(push_poll_period))
-
-        last_beat = time.time()
-        last_result_beat = time.time()
-        items = []
-
         while not self._stop_event.is_set():
+            logger.debug("Starting pending_result_queue get")
             try:
-                logger.debug("Starting pending_result_queue get")
-                r = self.task_scheduler.get_result(block=True, timeout=push_poll_period)
-                logger.debug("Got a result item")
-                items.append(r)
-            except queue.Empty:
-                logger.debug("pending_result_queue get timeout without result item")
-            except Exception as e:
-                logger.exception("Got an exception: {}".format(e))
-
-            if time.time() > last_result_beat + self.heartbeat_period:
-                heartbeat_message = f"last_result_beat={last_result_beat} heartbeat_period={self.heartbeat_period} seconds"
-                logger.info(f"Sending heartbeat via results connection: {heartbeat_message}")
-                last_result_beat = time.time()
-                items.append(pickle.dumps({'type': 'heartbeat'}))
-
-            if len(items) >= self.max_queue_size or time.time() > last_beat + push_poll_period:
-                last_beat = time.time()
-                if items:
-                    logger.debug(f"Result send: Pushing {len(items)} items")
-                    result_outgoing.send_multipart(items)
-                    logger.debug("Result send: Pushed")
-                    items = []
-                else:
-                    logger.debug("Result send: No items to push")
-            else:
-                logger.debug(f"Result send: check condition not met - deferring {len(items)} result items")
+                r = self.task_scheduler.get_result()
+                if r is None:
+                    continue
+                logger.debug("Result received from worker: %s", id(r))
+                result_outgoing.send(r)
+                logger.debug("Result sent to interchange: %s", id(r))
+            except Exception:
+                logger.exception("Failed to send result to interchange")
 
         result_outgoing.close()
-        logger.info("Exiting")
+        logger.debug("Exiting")
 
     @wrap_with_logs
-    def worker_watchdog(self):
+    def heartbeater(self):
+        while not self._stop_event.wait(self.heartbeat_period):
+            heartbeat_message = f"heartbeat_period={self.heartbeat_period} seconds"
+            logger.info(f"Sending heartbeat via results connection: {heartbeat_message}")
+            self.pending_result_queue.put(pickle.dumps({'type': 'heartbeat'}))
+
+    def worker_watchdog(self, procs: dict[int, SpawnProcess]):
         """Keeps workers alive."""
         logger.debug("Starting worker watchdog")
 
         while not self._stop_event.wait(self.heartbeat_period):
-            for worker_id, p in self.procs.items():
+            for worker_id, p in procs.items():
                 if not p.is_alive():
                     logger.error("Worker {} has died".format(worker_id))
                     try:
@@ -466,11 +449,10 @@ class Manager:
                     except KeyError:
                         logger.info("Worker {} was not busy when it died".format(worker_id))
 
-                    p = self._start_worker(worker_id)
-                    self.procs[worker_id] = p
+                    procs[worker_id] = self._start_worker(worker_id)
                     logger.info("Worker {} has been restarted".format(worker_id))
 
-        logger.critical("Exiting")
+        logger.debug("Exiting")
 
     @wrap_with_logs
     def handle_monitoring_messages(self):
@@ -485,32 +467,28 @@ class Manager:
         """
         logger.debug("Starting monitoring handler thread")
 
-        poll_period_s = max(10, self.poll_period) / 1000    # Must be at least 10 ms
-
         while not self._stop_event.is_set():
             try:
                 logger.debug("Starting monitor_queue.get()")
-                msg = self.monitoring_queue.get(block=True, timeout=poll_period_s)
-            except queue.Empty:
-                logger.debug("monitoring_queue.get() has timed out")
-            except Exception as e:
-                logger.exception(f"Got an exception: {e}")
-            else:
+                msg = self.monitoring_queue.get(block=True)
+                if msg is None:
+                    continue
                 logger.debug("Got a monitoring message")
                 self.pending_result_queue.put(msg)
                 logger.debug("Put monitoring message on pending_result_queue")
+            except Exception:
+                logger.exception("Failed to forward monitoring message")
 
-        logger.critical("Exiting")
+        logger.debug("Exiting")
 
     def start(self):
         """ Start the worker processes.
 
         TODO: Move task receiving to a thread
         """
-        self.procs = {}
+        procs: dict[int, SpawnProcess] = {}
         for worker_id in range(self.worker_count):
-            p = self._start_worker(worker_id)
-            self.procs[worker_id] = p
+            procs[worker_id] = self._start_worker(worker_id)
 
         logger.debug("Workers started")
 
@@ -519,40 +497,69 @@ class Manager:
             target=self.push_results, name="Result-Pusher"
         )
         thr_worker_watchdog = threading.Thread(
-            target=self.worker_watchdog, name="worker-watchdog"
+            target=self.worker_watchdog, args=(procs,), name="worker-watchdog"
         )
         thr_monitoring_handler = threading.Thread(
             target=self.handle_monitoring_messages, name="Monitoring-Handler"
         )
+        thr_heartbeater = threading.Thread(target=self.heartbeater, name="Heartbeater")
 
         thr_task_puller.start()
         thr_result_pusher.start()
         thr_worker_watchdog.start()
         thr_monitoring_handler.start()
+        thr_heartbeater.start()
 
         logger.info("Manager threads started")
 
         # This might need a multiprocessing event to signal back.
         self._stop_event.wait()
-        logger.critical("Received kill event, terminating worker processes")
+        logger.info("Stop event set; terminating worker processes")
+
+        # Invite blocking threads to quit
+        self.monitoring_queue.put(None)
+        self.pending_result_queue.put(None)
 
+        thr_heartbeater.join()
         thr_task_puller.join()
         thr_result_pusher.join()
         thr_worker_watchdog.join()
         thr_monitoring_handler.join()
-        for proc_id in self.procs:
-            self.procs[proc_id].terminate()
-            logger.critical("Terminating worker {}: is_alive()={}".format(self.procs[proc_id],
-                                                                          self.procs[proc_id].is_alive()))
-            self.procs[proc_id].join()
-            logger.debug("Worker {} joined successfully".format(self.procs[proc_id]))
+
+        for worker_id in procs:
+            p = procs[worker_id]
+            proc_info = f"(PID: {p.pid}, Worker ID: {worker_id})"
+            logger.debug(f"Signaling worker {p.name} (TERM). {proc_info}")
+            p.terminate()
 
         self.zmq_context.term()
+
+        # give processes 1 second to gracefully shut themselves down, based on the
+        # SIGTERM (.terminate()) just sent; after then, we pull the plug.
+        force_child_shutdown_at = time.monotonic() + 1
+        while procs:
+            worker_id, p = procs.popitem()
+            timeout = max(force_child_shutdown_at - time.monotonic(), 0.000001)
+            p.join(timeout=timeout)
+            proc_info = f"(PID: {p.pid}, Worker ID: {worker_id})"
+            if p.exitcode is not None:
+                logger.debug(
+                    "Worker joined successfully.  %s (exitcode: %s)", proc_info, p.exitcode
+                )
+
+            else:
+                logger.warning(
+                    f"Worker {p.name} ({worker_id}) failed to terminate in a timely"
+                    f" manner; sending KILL signal to process. {proc_info}"
+                )
+                p.kill()
+                p.join()
+            p.close()
+
         delta = time.time() - self._start_time
         logger.info("process_worker_pool ran for {} seconds".format(delta))
-        return
 
-    def _start_worker(self, worker_id: int):
+    def _start_worker(self, worker_id: int) -> SpawnProcess:
         p = SpawnContext.Process(
             target=worker,
             args=(
@@ -939,27 +946,27 @@ if __name__ == "__main__":
     )
     logger.info(
         f"\n  Python version: {sys.version}"
-        f"  Debug logging: {args.debug}"
-        f"  Certificates dir: {args.cert_dir}"
-        f"  Log dir: {args.logdir}"
-        f"  Manager ID: {args.uid}"
-        f"  Block ID: {args.block_id}"
-        f"  cores_per_worker: {args.cores_per_worker}"
-        f"  mem_per_worker: {args.mem_per_worker}"
-        f"  task_port: {args.task_port}"
-        f"  result_port: {args.result_port}"
-        f"  addresses: {args.addresses}"
-        f"  max_workers_per_node: {args.max_workers_per_node}"
-        f"  poll_period: {args.poll}"
-        f"  address_probe_timeout: {args.address_probe_timeout}"
-        f"  Prefetch capacity: {args.prefetch_capacity}"
-        f"  Heartbeat threshold: {args.hb_threshold}"
-        f"  Heartbeat period: {args.hb_period}"
-        f"  Drain period: {args.drain_period}"
-        f"  CPU affinity: {args.cpu_affinity}"
-        f"  Accelerators: {' '.join(args.available_accelerators)}"
-        f"  enable_mpi_mode: {args.enable_mpi_mode}"
-        f"  mpi_launcher: {args.mpi_launcher}"
+        f"\n  Debug logging: {args.debug}"
+        f"\n  Certificates dir: {args.cert_dir}"
+        f"\n  Log dir: {args.logdir}"
+        f"\n  Manager ID: {args.uid}"
+        f"\n  Block ID: {args.block_id}"
+        f"\n  cores_per_worker: {args.cores_per_worker}"
+        f"\n  mem_per_worker: {args.mem_per_worker}"
+        f"\n  task_port: {args.task_port}"
+        f"\n  result_port: {args.result_port}"
+        f"\n  addresses: {args.addresses}"
+        f"\n  max_workers_per_node: {args.max_workers_per_node}"
+        f"\n  poll_period: {args.poll}"
+        f"\n  address_probe_timeout: {args.address_probe_timeout}"
+        f"\n  Prefetch capacity: {args.prefetch_capacity}"
+        f"\n  Heartbeat threshold: {args.hb_threshold}"
+        f"\n  Heartbeat period: {args.hb_period}"
+        f"\n  Drain period: {args.drain_period}"
+        f"\n  CPU affinity: {args.cpu_affinity}"
+        f"\n  Accelerators: {' '.join(args.available_accelerators)}"
+        f"\n  enable_mpi_mode: {args.enable_mpi_mode}"
+        f"\n  mpi_launcher: {args.mpi_launcher}"
     )
     try:
         manager = Manager(task_port=args.task_port,

parsl/monitoring/db_manager.py

@@ -1,6 +1,7 @@
 import datetime
 import logging
 import multiprocessing.queues as mpq
+import multiprocessing.synchronize as mpe
 import os
 import queue
 import threading
@@ -278,11 +279,13 @@ class Database:
 
 class DatabaseManager:
     def __init__(self,
+                 *,
                  db_url: str = 'sqlite:///runinfo/monitoring.db',
                  run_dir: str = '.',
                  logging_level: int = logging.INFO,
                  batching_interval: float = 1,
                  batching_threshold: float = 99999,
+                 exit_event: mpe.Event
                  ):
 
         self.workflow_end = False
@@ -307,6 +310,8 @@ class DatabaseManager:
         self.pending_block_queue: queue.Queue[MonitoringMessage] = queue.Queue()
         self.pending_resource_queue: queue.Queue[MonitoringMessage] = queue.Queue()
 
+        self.external_exit_event = exit_event
+
     def start(self,
               resource_queue: mpq.Queue) -> None:
 
@@ -555,15 +560,16 @@ class DatabaseManager:
         while not kill_event.is_set() or logs_queue.qsize() != 0:
             logger.debug("Checking STOP conditions: kill event: %s, queue has entries: %s",
                          kill_event.is_set(), logs_queue.qsize() != 0)
+
+            if self.external_exit_event.is_set():
+                self.close()
+
             try:
                 x = logs_queue.get(timeout=0.1)
             except queue.Empty:
                 continue
             else:
-                if x == 'STOP':
-                    self.close()
-                else:
-                    self._dispatch_to_internal(x)
+                self._dispatch_to_internal(x)
 
     def _dispatch_to_internal(self, x: Tuple) -> None:
         assert isinstance(x, tuple)
@@ -678,11 +684,11 @@ class DatabaseManager:
 
 @wrap_with_logs(target="database_manager")
 @typeguard.typechecked
-def dbm_starter(exception_q: mpq.Queue,
-                resource_msgs: mpq.Queue,
+def dbm_starter(resource_msgs: mpq.Queue,
                 db_url: str,
                 run_dir: str,
-                logging_level: int) -> None:
+                logging_level: int,
+                exit_event: mpe.Event) -> None:
     """Start the database manager process
 
     The DFK should start this function. The args, kwargs match that of the monitoring config
@@ -693,16 +699,16 @@ def dbm_starter(exception_q: mpq.Queue,
     try:
         dbm = DatabaseManager(db_url=db_url,
                               run_dir=run_dir,
-                              logging_level=logging_level)
+                              logging_level=logging_level,
+                              exit_event=exit_event)
         logger.info("Starting dbm in dbm starter")
         dbm.start(resource_msgs)
     except KeyboardInterrupt:
         logger.exception("KeyboardInterrupt signal caught")
         dbm.close()
         raise
-    except Exception as e:
+    except Exception:
         logger.exception("dbm.start exception")
-        exception_q.put(("DBM", str(e)))
         dbm.close()
 
     logger.info("End of dbm_starter")