parsl 2024.5.20__py3-none-any.whl → 2024.6.3__py3-none-any.whl

parsl/executors/high_throughput/executor.py

@@ -1,44 +1,39 @@
+import logging
+import math
+import pickle
+import threading
 import typing
+import warnings
 from collections import defaultdict
 from concurrent.futures import Future
-import typeguard
-import logging
-import threading
-import queue
-import pickle
 from dataclasses import dataclass
-from multiprocessing import Process, Queue
-from typing import Dict, Sequence
-from typing import List, Optional, Tuple, Union, Callable
-import math
-import warnings
+from multiprocessing import Process
+from typing import Callable, Dict, List, Optional, Sequence, Tuple, Union
+
+import typeguard
 
 import parsl.launchers
-from parsl.usage_tracking.api import UsageInformation
-from parsl.serialize import pack_res_spec_apply_message, deserialize
-from parsl.serialize.errors import SerializationError, DeserializationError
+from parsl import curvezmq
+from parsl.addresses import get_all_addresses
 from parsl.app.errors import RemoteExceptionWrapper
-from parsl.jobs.states import JobStatus, JobState, TERMINAL_STATES
-from parsl.executors.high_throughput import zmq_pipes
-from parsl.executors.high_throughput import interchange
-from parsl.executors.errors import (
-    BadMessage, ScalingFailed,
-)
+from parsl.data_provider.staging import Staging
+from parsl.executors.errors import BadMessage, ScalingFailed
+from parsl.executors.high_throughput import interchange, zmq_pipes
+from parsl.executors.high_throughput.errors import CommandClientTimeoutError
 from parsl.executors.high_throughput.mpi_prefix_composer import (
     VALID_LAUNCHERS,
-    validate_resource_spec
+    validate_resource_spec,
 )
-
-from parsl import curvezmq
 from parsl.executors.status_handling import BlockProviderExecutor
-from parsl.providers.base import ExecutionProvider
-from parsl.data_provider.staging import Staging
-from parsl.addresses import get_all_addresses
-from parsl.process_loggers import wrap_with_logs
-
+from parsl.jobs.states import TERMINAL_STATES, JobState, JobStatus
 from parsl.multiprocessing import ForkProcess
-from parsl.utils import RepresentationMixin
+from parsl.process_loggers import wrap_with_logs
 from parsl.providers import LocalProvider
+from parsl.providers.base import ExecutionProvider
+from parsl.serialize import deserialize, pack_res_spec_apply_message
+from parsl.serialize.errors import DeserializationError, SerializationError
+from parsl.usage_tracking.api import UsageInformation
+from parsl.utils import RepresentationMixin
 
 logger = logging.getLogger(__name__)
 
@@ -415,13 +410,13 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin, UsageIn
             )
 
         self.outgoing_q = zmq_pipes.TasksOutgoing(
-            curvezmq.ClientContext(self.cert_dir), "127.0.0.1", self.interchange_port_range
+            "127.0.0.1", self.interchange_port_range, self.cert_dir
         )
         self.incoming_q = zmq_pipes.ResultsIncoming(
-            curvezmq.ClientContext(self.cert_dir), "127.0.0.1", self.interchange_port_range
+            "127.0.0.1", self.interchange_port_range, self.cert_dir
         )
         self.command_client = zmq_pipes.CommandClient(
-            curvezmq.ClientContext(self.cert_dir), "127.0.0.1", self.interchange_port_range
+            "127.0.0.1", self.interchange_port_range, self.cert_dir
         )
 
         self._queue_management_thread = None
@@ -531,9 +526,7 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin, UsageIn
         Starts the interchange process locally and uses an internal command queue to
         get the worker task and result ports that the interchange has bound to.
         """
-        comm_q = Queue(maxsize=10)
         self.interchange_proc = ForkProcess(target=interchange.starter,
-                                            args=(comm_q,),
                                             kwargs={"client_ports": (self.outgoing_q.port,
                                                                      self.incoming_q.port,
                                                                      self.command_client.port),
@@ -552,9 +545,10 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin, UsageIn
                                             name="HTEX-Interchange"
                                             )
         self.interchange_proc.start()
+
         try:
-            (self.worker_task_port, self.worker_result_port) = comm_q.get(block=True, timeout=120)
-        except queue.Empty:
+            (self.worker_task_port, self.worker_result_port) = self.command_client.run("WORKER_PORTS", timeout_s=120)
+        except CommandClientTimeoutError:
             logger.error("Interchange has not completed initialization in 120s. Aborting")
             raise Exception("Interchange failed to start")
 
@@ -645,7 +639,8 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin, UsageIn
         Returns:
               Future
         """
-        validate_resource_spec(resource_specification)
+
+        validate_resource_spec(resource_specification, self.enable_mpi_mode)
 
         if self.bad_state_is_set:
             raise self.executor_exception

parsl/executors/high_throughput/interchange.py

@@ -1,31 +1,28 @@
 #!/usr/bin/env python
-import multiprocessing
-import zmq
+import datetime
+import json
+import logging
 import os
-import sys
+import pickle
 import platform
+import queue
 import random
-import time
-import datetime
-import pickle
 import signal
-import logging
-import queue
+import sys
 import threading
-import json
+import time
+from typing import Any, Dict, List, NoReturn, Optional, Sequence, Set, Tuple, cast
 
-from typing import cast, Any, Dict, NoReturn, Sequence, Set, Optional, Tuple, List
+import zmq
 
 from parsl import curvezmq
-from parsl.utils import setproctitle
-from parsl.version import VERSION as PARSL_VERSION
-from parsl.serialize import serialize as serialize_object
-
 from parsl.app.errors import RemoteExceptionWrapper
 from parsl.executors.high_throughput.manager_record import ManagerRecord
 from parsl.monitoring.message_type import MessageType
 from parsl.process_loggers import wrap_with_logs
-
+from parsl.serialize import serialize as serialize_object
+from parsl.utils import setproctitle
+from parsl.version import VERSION as PARSL_VERSION
 
 PKL_HEARTBEAT_CODE = pickle.dumps((2 ** 32) - 1)
 PKL_DRAINED_CODE = pickle.dumps((2 ** 32) - 2)
@@ -328,6 +325,9 @@ class Interchange:
 
                     reply = None
 
+                elif command_req == "WORKER_PORTS":
+                    reply = (self.worker_task_port, self.worker_result_port)
+
                 else:
                     logger.error(f"Received unknown command: {command_req}")
                     reply = None
@@ -672,7 +672,7 @@ def start_file_logger(filename: str, level: int = logging.DEBUG, format_string:
 
 
 @wrap_with_logs(target="interchange")
-def starter(comm_q: multiprocessing.Queue, *args: Any, **kwargs: Any) -> None:
+def starter(*args: Any, **kwargs: Any) -> None:
     """Start the interchange process
 
     The executor is expected to call this function. The args, kwargs match that of the Interchange.__init__
@@ -680,6 +680,4 @@ def starter(comm_q: multiprocessing.Queue, *args: Any, **kwargs: Any) -> None:
     setproctitle("parsl: HTEX interchange")
     # logger = multiprocessing.get_logger()
     ic = Interchange(*args, **kwargs)
-    comm_q.put((ic.worker_task_port,
-                ic.worker_result_port))
     ic.start()

parsl/executors/high_throughput/manager_record.py

@@ -1,5 +1,6 @@
 from datetime import datetime
 from typing import Any, List, Optional
+
 from typing_extensions import TypedDict
 
 

parsl/executors/high_throughput/monitoring_info.py

@@ -3,6 +3,7 @@
 # then be acquired by any other code running in
 # a worker context - specifically the monitoring
 # wrapper code.
-from typing import Optional
 from queue import Queue
+from typing import Optional
+
 result_queue: Optional[Queue] = None

parsl/executors/high_throughput/mpi_executor.py

@@ -1,10 +1,13 @@
 """A simplified interface for HTEx when running in MPI mode"""
-from typing import Optional, Tuple, List, Union, Callable, Dict
+from typing import Callable, Dict, List, Optional, Tuple, Union
 
 import typeguard
 
 from parsl.data_provider.staging import Staging
-from parsl.executors.high_throughput.executor import HighThroughputExecutor, GENERAL_HTEX_PARAM_DOCS
+from parsl.executors.high_throughput.executor import (
+    GENERAL_HTEX_PARAM_DOCS,
+    HighThroughputExecutor,
+)
 from parsl.executors.status_handling import BlockProviderExecutor
 from parsl.jobs.states import JobStatus
 from parsl.providers import LocalProvider
@@ -20,7 +23,7 @@ class MPIExecutor(HighThroughputExecutor):
     to spawn multi-node tasks.
 
     Specify the maximum number of multi-node tasks to run at once using ``max_workers_per_block``.
-    The maximum number should be smaller than the ``nodes_per_block`` in the Provider.
+    The value should be less than or equal to the ``nodes_per_block`` in the Provider.
 
     Parameters
     ----------

parsl/executors/high_throughput/mpi_prefix_composer.py

@@ -1,5 +1,5 @@
 import logging
-from typing import Dict, List, Tuple, Set
+from typing import Dict, List, Set, Tuple
 
 logger = logging.getLogger(__name__)
 
@@ -8,8 +8,18 @@ VALID_LAUNCHERS = ('srun',
                    'mpiexec')
 
 
+class MissingResourceSpecification(Exception):
+    """Exception raised when input is  not supplied a resource specification"""
+
+    def __init__(self, reason: str):
+        self.reason = reason
+
+    def __str__(self):
+        return f"Missing resource specification: {self.reason}"
+
+
 class InvalidResourceSpecification(Exception):
-    """Exception raised when Invalid keys are supplied via resource specification"""
+    """Exception raised when Invalid input is supplied via resource specification"""
 
     def __init__(self, invalid_keys: Set[str]):
         self.invalid_keys = invalid_keys
@@ -18,13 +28,19 @@ class InvalidResourceSpecification(Exception):
         return f"Invalid resource specification options supplied: {self.invalid_keys}"
 
 
-def validate_resource_spec(resource_spec: Dict[str, str]):
+def validate_resource_spec(resource_spec: Dict[str, str], is_mpi_enabled: bool):
     """Basic validation of keys in the resource_spec
 
     Raises: InvalidResourceSpecification if the resource_spec
         is invalid (e.g, contains invalid keys)
     """
     user_keys = set(resource_spec.keys())
+
+    # empty resource_spec when mpi_mode is set causes parsl to hang
+    # ref issue #3427
+    if is_mpi_enabled and len(user_keys) == 0:
+        raise MissingResourceSpecification('MPI mode requires optional parsl_resource_specification keyword argument to be configured')
+
     legal_keys = set(("ranks_per_node",
                       "num_nodes",
                       "num_ranks",

parsl/executors/high_throughput/mpi_resource_management.py

@@ -8,8 +8,7 @@ from enum import Enum
 from typing import Dict, List
 
 from parsl.multiprocessing import SpawnContext
-from parsl.serialize import (pack_res_spec_apply_message,
-                             unpack_res_spec_apply_message)
+from parsl.serialize import pack_res_spec_apply_message, unpack_res_spec_apply_message
 
 logger = logging.getLogger(__name__)
 

parsl/executors/high_throughput/probe.py

@@ -1,11 +1,13 @@
-import zmq
 import argparse
-import uuid
-import time
 import logging
-from parsl.addresses import get_all_addresses
+import time
+import uuid
+
+import zmq
 from zmq.utils.monitor import recv_monitor_message
 
+from parsl.addresses import get_all_addresses
+
 logger = logging.getLogger(__name__)
 
 

parsl/executors/high_throughput/process_worker_pool.py

@@ -1,39 +1,41 @@
 #!/usr/bin/env python3
 
 import argparse
+import json
 import logging
+import math
+import multiprocessing
 import os
-import sys
+import pickle
 import platform
+import queue
+import sys
 import threading
-import pickle
 import time
-import queue
 import uuid
-from typing import Sequence, Optional, Dict, List
-
-import zmq
-import math
-import json
-import psutil
-import multiprocessing
 from multiprocessing.managers import DictProxy
 from multiprocessing.sharedctypes import Synchronized
+from typing import Dict, List, Optional, Sequence
+
+import psutil
+import zmq
 
 from parsl import curvezmq
-from parsl.process_loggers import wrap_with_logs
-from parsl.version import VERSION as PARSL_VERSION
 from parsl.app.errors import RemoteExceptionWrapper
 from parsl.executors.high_throughput.errors import WorkerLost
-from parsl.executors.high_throughput.probe import probe_addresses
-from parsl.multiprocessing import SpawnContext
-from parsl.serialize import unpack_res_spec_apply_message, serialize
+from parsl.executors.high_throughput.mpi_prefix_composer import (
+    VALID_LAUNCHERS,
+    compose_all,
+)
 from parsl.executors.high_throughput.mpi_resource_management import (
+    MPITaskScheduler,
     TaskScheduler,
-    MPITaskScheduler
 )
-
-from parsl.executors.high_throughput.mpi_prefix_composer import compose_all, VALID_LAUNCHERS
+from parsl.executors.high_throughput.probe import probe_addresses
+from parsl.multiprocessing import SpawnContext
+from parsl.process_loggers import wrap_with_logs
+from parsl.serialize import serialize, unpack_res_spec_apply_message
+from parsl.version import VERSION as PARSL_VERSION
 
 HEARTBEAT_CODE = (2 ** 32) - 1
 DRAINED_CODE = (2 ** 32) - 2
@@ -677,7 +679,8 @@ def worker(
     # If desired, set process affinity
     if cpu_affinity != "none":
         # Count the number of cores per worker
-        avail_cores = sorted(os.sched_getaffinity(0))  # Get the available threads
+        # OSX does not implement os.sched_getaffinity
+        avail_cores = sorted(os.sched_getaffinity(0))  # type: ignore[attr-defined, unused-ignore]
         cores_per_worker = len(avail_cores) // pool_size
         assert cores_per_worker > 0, "Affinity does not work if there are more workers than cores"
 
@@ -717,7 +720,15 @@ def worker(
         os.environ["KMP_AFFINITY"] = f"explicit,proclist=[{proc_list}]"  # For Intel OpenMP
 
         # Set the affinity for this worker
-        os.sched_setaffinity(0, my_cores)
+        # OSX does not implement os.sched_setaffinity so type checking
+        # is ignored here in two ways:
+        # On a platform without sched_setaffinity, that attribute will not
+        # be defined, so ignore[attr-defined] will tell mypy to ignore this
+        # incorrect-for-OS X attribute access.
+        # On a platform with sched_setaffinity, that type: ignore message
+        # will be redundant, and ignore[unused-ignore] tells mypy to ignore
+        # that this ignore is unneeded.
+        os.sched_setaffinity(0, my_cores)  # type: ignore[attr-defined, unused-ignore]
         logger.info("Set worker CPU affinity to {}".format(my_cores))
 
     # If desired, pin to accelerator

parsl/executors/high_throughput/zmq_pipes.py

@@ -1,10 +1,18 @@
 #!/usr/bin/env python3
 
-import zmq
 import logging
 import threading
+import time
+from typing import Optional
+
+import zmq
 
 from parsl import curvezmq
+from parsl.errors import InternalConsistencyError
+from parsl.executors.high_throughput.errors import (
+    CommandClientBadError,
+    CommandClientTimeoutError,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -12,25 +20,29 @@ logger = logging.getLogger(__name__)
 class CommandClient:
     """ CommandClient
     """
-    def __init__(self, zmq_context: curvezmq.ClientContext, ip_address, port_range):
+    def __init__(self, ip_address, port_range, cert_dir: Optional[str] = None):
         """
         Parameters
         ----------
 
-        zmq_context: curvezmq.ClientContext
-            CurveZMQ client context used to create secure sockets
         ip_address: str
            IP address of the client (where Parsl runs)
+
         port_range: tuple(int, int)
            Port range for the comms between client and interchange
 
+        cert_dir: str | None
+            Path to the certificate directory. Setting this to None will disable encryption.
+            default: None
+
         """
-        self.zmq_context = zmq_context
+        self.zmq_context = curvezmq.ClientContext(cert_dir)
         self.ip_address = ip_address
         self.port_range = port_range
         self.port = None
         self.create_socket_and_bind()
         self._lock = threading.Lock()
+        self.ok = True
 
     def create_socket_and_bind(self):
         """ Creates socket and binds to a port.
@@ -46,7 +58,7 @@ class CommandClient:
         else:
             self.zmq_socket.bind("tcp://{}:{}".format(self.ip_address, self.port))
 
-    def run(self, message, max_retries=3):
+    def run(self, message, max_retries=3, timeout_s=None):
         """ This function needs to be fast at the same time aware of the possibility of
         ZMQ pipes overflowing.
 
@@ -54,13 +66,43 @@ class CommandClient:
         in ZMQ sockets reaching a broken state once there are ~10k tasks in flight.
         This issue can be magnified if each the serialized buffer itself is larger.
         """
+        if not self.ok:
+            raise CommandClientBadError()
+
+        start_time_s = time.monotonic()
+
         reply = '__PARSL_ZMQ_PIPES_MAGIC__'
         with self._lock:
             for _ in range(max_retries):
                 try:
                     logger.debug("Sending command client command")
+
+                    if timeout_s is not None:
+                        remaining_time_s = start_time_s + timeout_s - time.monotonic()
+                        poll_result = self.zmq_socket.poll(timeout=remaining_time_s * 1000, flags=zmq.POLLOUT)
+                        if poll_result == zmq.POLLOUT:
+                            pass  # this is OK, so continue
+                        elif poll_result == 0:
+                            raise CommandClientTimeoutError("Waiting for command channel to be ready for a command")
+                        else:
+                            raise InternalConsistencyError(f"ZMQ poll returned unexpected value: {poll_result}")
+
                     self.zmq_socket.send_pyobj(message, copy=True)
-                    logger.debug("Waiting for command client response")
+
+                    if timeout_s is not None:
+                        logger.debug("Polling for command client response or timeout")
+                        remaining_time_s = start_time_s + timeout_s - time.monotonic()
+                        poll_result = self.zmq_socket.poll(timeout=remaining_time_s * 1000, flags=zmq.POLLIN)
+                        if poll_result == zmq.POLLIN:
+                            pass  # this is OK, so continue
+                        elif poll_result == 0:
+                            logger.error("Command timed-out - command client is now bad forever")
+                            self.ok = False
+                            raise CommandClientTimeoutError("Waiting for a reply from command channel")
+                        else:
+                            raise InternalConsistencyError(f"ZMQ poll returned unexpected value: {poll_result}")
+
+                    logger.debug("Receiving command client response")
                     reply = self.zmq_socket.recv_pyobj()
                     logger.debug("Received command client response")
                 except zmq.ZMQError:
@@ -85,20 +127,23 @@ class CommandClient:
 class TasksOutgoing:
     """ Outgoing task queue from the executor to the Interchange
     """
-    def __init__(self, zmq_context: curvezmq.ClientContext, ip_address, port_range):
+    def __init__(self, ip_address, port_range, cert_dir: Optional[str] = None):
         """
         Parameters
         ----------
 
-        zmq_context: curvezmq.ClientContext
-            CurveZMQ client context used to create secure sockets
         ip_address: str
            IP address of the client (where Parsl runs)
+
         port_range: tuple(int, int)
            Port range for the comms between client and interchange
 
+        cert_dir: str | None
+            Path to the certificate directory. Setting this to None will disable encryption.
+            default: None
+
         """
-        self.zmq_context = zmq_context
+        self.zmq_context = curvezmq.ClientContext(cert_dir)
         self.zmq_socket = self.zmq_context.socket(zmq.DEALER)
         self.zmq_socket.set_hwm(0)
         self.port = self.zmq_socket.bind_to_random_port("tcp://{}".format(ip_address),
@@ -138,20 +183,23 @@ class ResultsIncoming:
     """ Incoming results queue from the Interchange to the executor
     """
 
-    def __init__(self, zmq_context: curvezmq.ClientContext, ip_address, port_range):
+    def __init__(self, ip_address, port_range, cert_dir: Optional[str] = None):
         """
         Parameters
         ----------
 
-        zmq_context: curvezmq.ClientContext
-            CurveZMQ client context used to create secure sockets
         ip_address: str
            IP address of the client (where Parsl runs)
+
         port_range: tuple(int, int)
            Port range for the comms between client and interchange
 
+        cert_dir: str | None
+            Path to the certificate directory. Setting this to None will disable encryption.
+            default: None
+
         """
-        self.zmq_context = zmq_context
+        self.zmq_context = curvezmq.ClientContext(cert_dir)
         self.results_receiver = self.zmq_context.socket(zmq.DEALER)
         self.results_receiver.set_hwm(0)
         self.port = self.results_receiver.bind_to_random_port("tcp://{}".format(ip_address),

parsl/executors/radical/executor.py

@@ -1,30 +1,30 @@
 """RadicalPilotExecutor builds on the RADICAL-Pilot/Parsl
 """
+import inspect
+import logging
 import os
-import sys
-import time
-import parsl
 import queue
-import logging
-import inspect
-import requests
-import typeguard
+import sys
 import threading as mt
-
+import time
+from concurrent.futures import Future
 from functools import partial
-from typing import Optional, Dict
 from pathlib import Path, PosixPath
-from concurrent.futures import Future
+from typing import Dict, Optional
+
+import requests
+import typeguard
 
+import parsl
+from parsl.app.errors import BashExitFailure, RemoteExceptionWrapper
 from parsl.app.python import timeout
-from .rpex_resources import ResourceConfig
 from parsl.data_provider.files import File
-from parsl.utils import RepresentationMixin
-from parsl.app.errors import BashExitFailure
 from parsl.executors.base import ParslExecutor
-from parsl.app.errors import RemoteExceptionWrapper
 from parsl.serialize import deserialize, pack_res_spec_apply_message
-from parsl.serialize.errors import SerializationError, DeserializationError
+from parsl.serialize.errors import DeserializationError, SerializationError
+from parsl.utils import RepresentationMixin
+
+from .rpex_resources import ResourceConfig
 
 try:
     import radical.pilot as rp

parsl/executors/radical/rpex_master.py

@@ -2,9 +2,8 @@
 
 import sys
 
-import radical.utils as ru
 import radical.pilot as rp
-
+import radical.utils as ru
 
 # ------------------------------------------------------------------------------
 #

parsl/executors/radical/rpex_resources.py

@@ -1,17 +1,12 @@
-import sys
 import json
-
+import sys
 from typing import List
 
-_setup_paths: List[str]
+_setup_paths: List[str] = []
 try:
     import radical.pilot as rp
-    import radical.utils as ru
 except ImportError:
-    _setup_paths = []
-else:
-    _setup_paths = [rp.sdist_path,
-                    ru.sdist_path]
+    pass
 
 
 MPI = "mpi"
@@ -77,7 +72,7 @@ class ResourceConfig:
 
     pilot_env_setup : list
         List of setup commands/packages for the pilot environment.
-        Default setup includes "parsl", rp.sdist_path, and ru.sdist_path.
+        Default is an empty list.
 
     python_v : str
         The Python version to be used in the pilot environment.

parsl/executors/radical/rpex_worker.py

@@ -1,10 +1,11 @@
 import sys
+
 import radical.pilot as rp
 
 import parsl.app.errors as pe
 from parsl.app.bash import remote_side_bash_executor
-from parsl.serialize import unpack_res_spec_apply_message, serialize
 from parsl.executors.high_throughput.process_worker_pool import execute_task
+from parsl.serialize import serialize, unpack_res_spec_apply_message
 
 
 class ParslWorker: