parsl 2024.5.27__py3-none-any.whl → 2024.6.10__py3-none-any.whl

parsl/executors/high_throughput/interchange.py

@@ -1,31 +1,28 @@
 #!/usr/bin/env python
-import multiprocessing
-import zmq
+import datetime
+import json
+import logging
 import os
-import sys
+import pickle
 import platform
+import queue
 import random
-import time
-import datetime
-import pickle
 import signal
-import logging
-import queue
+import sys
 import threading
-import json
+import time
+from typing import Any, Dict, List, NoReturn, Optional, Sequence, Set, Tuple, cast
 
-from typing import cast, Any, Dict, NoReturn, Sequence, Set, Optional, Tuple, List
+import zmq
 
 from parsl import curvezmq
-from parsl.utils import setproctitle
-from parsl.version import VERSION as PARSL_VERSION
-from parsl.serialize import serialize as serialize_object
-
 from parsl.app.errors import RemoteExceptionWrapper
 from parsl.executors.high_throughput.manager_record import ManagerRecord
 from parsl.monitoring.message_type import MessageType
 from parsl.process_loggers import wrap_with_logs
-
+from parsl.serialize import serialize as serialize_object
+from parsl.utils import setproctitle
+from parsl.version import VERSION as PARSL_VERSION
 
 PKL_HEARTBEAT_CODE = pickle.dumps((2 ** 32) - 1)
 PKL_DRAINED_CODE = pickle.dumps((2 ** 32) - 2)
@@ -68,18 +65,19 @@ class Interchange:
     3. Detect workers that have failed using heartbeats
     """
     def __init__(self,
-                 client_address: str = "127.0.0.1",
-                 interchange_address: Optional[str] = None,
-                 client_ports: Tuple[int, int, int] = (50055, 50056, 50057),
-                 worker_ports: Optional[Tuple[int, int]] = None,
-                 worker_port_range: Tuple[int, int] = (54000, 55000),
-                 hub_address: Optional[str] = None,
-                 hub_zmq_port: Optional[int] = None,
-                 heartbeat_threshold: int = 60,
-                 logdir: str = ".",
-                 logging_level: int = logging.INFO,
-                 poll_period: int = 10,
-                 cert_dir: Optional[str] = None,
+                 *,
+                 client_address: str,
+                 interchange_address: Optional[str],
+                 client_ports: Tuple[int, int, int],
+                 worker_ports: Optional[Tuple[int, int]],
+                 worker_port_range: Tuple[int, int],
+                 hub_address: Optional[str],
+                 hub_zmq_port: Optional[int],
+                 heartbeat_threshold: int,
+                 logdir: str,
+                 logging_level: int,
+                 poll_period: int,
+                 cert_dir: Optional[str],
                  ) -> None:
         """
         Parameters
@@ -95,34 +93,34 @@ class Interchange:
              The ports at which the client can be reached
 
         worker_ports : tuple(int, int)
-             The specific two ports at which workers will connect to the Interchange. Default: None
+             The specific two ports at which workers will connect to the Interchange.
 
         worker_port_range : tuple(int, int)
              The interchange picks ports at random from the range which will be used by workers.
-             This is overridden when the worker_ports option is set. Default: (54000, 55000)
+             This is overridden when the worker_ports option is set.
 
         hub_address : str
              The IP address at which the interchange can send info about managers to when monitoring is enabled.
-             Default: None (meaning monitoring disabled)
+             When None, monitoring is disabled.
 
         hub_zmq_port : str
              The port at which the interchange can send info about managers to when monitoring is enabled.
-             Default: None (meaning monitoring disabled)
+             When None, monitoring is disabled.
 
         heartbeat_threshold : int
              Number of seconds since the last heartbeat after which worker is considered lost.
 
         logdir : str
-             Parsl log directory paths. Logs and temp files go here. Default: '.'
+             Parsl log directory paths. Logs and temp files go here.
 
         logging_level : int
-             Logging level as defined in the logging module. Default: logging.INFO
+             Logging level as defined in the logging module.
 
         poll_period : int
-             The main thread polling period, in milliseconds. Default: 10ms
+             The main thread polling period, in milliseconds.
 
         cert_dir : str | None
-            Path to the certificate directory. Default: None
+            Path to the certificate directory.
         """
         self.cert_dir = cert_dir
         self.logdir = logdir
@@ -328,6 +326,9 @@ class Interchange:
 
                     reply = None
 
+                elif command_req == "WORKER_PORTS":
+                    reply = (self.worker_task_port, self.worker_result_port)
+
                 else:
                     logger.error(f"Received unknown command: {command_req}")
                     reply = None
@@ -672,7 +673,7 @@ def start_file_logger(filename: str, level: int = logging.DEBUG, format_string:
 
 
 @wrap_with_logs(target="interchange")
-def starter(comm_q: multiprocessing.Queue, *args: Any, **kwargs: Any) -> None:
+def starter(*args: Any, **kwargs: Any) -> None:
     """Start the interchange process
 
     The executor is expected to call this function. The args, kwargs match that of the Interchange.__init__
@@ -680,6 +681,4 @@ def starter(comm_q: multiprocessing.Queue, *args: Any, **kwargs: Any) -> None:
     setproctitle("parsl: HTEX interchange")
     # logger = multiprocessing.get_logger()
     ic = Interchange(*args, **kwargs)
-    comm_q.put((ic.worker_task_port,
-                ic.worker_result_port))
     ic.start()

parsl/executors/high_throughput/manager_record.py

@@ -1,5 +1,6 @@
 from datetime import datetime
 from typing import Any, List, Optional
+
 from typing_extensions import TypedDict
 
 

parsl/executors/high_throughput/monitoring_info.py

@@ -3,6 +3,7 @@
 # then be acquired by any other code running in
 # a worker context - specifically the monitoring
 # wrapper code.
-from typing import Optional
 from queue import Queue
+from typing import Optional
+
 result_queue: Optional[Queue] = None

parsl/executors/high_throughput/mpi_executor.py

@@ -1,10 +1,13 @@
 """A simplified interface for HTEx when running in MPI mode"""
-from typing import Optional, Tuple, List, Union, Callable, Dict
+from typing import Callable, Dict, List, Optional, Tuple, Union
 
 import typeguard
 
 from parsl.data_provider.staging import Staging
-from parsl.executors.high_throughput.executor import HighThroughputExecutor, GENERAL_HTEX_PARAM_DOCS
+from parsl.executors.high_throughput.executor import (
+    GENERAL_HTEX_PARAM_DOCS,
+    HighThroughputExecutor,
+)
 from parsl.executors.status_handling import BlockProviderExecutor
 from parsl.jobs.states import JobStatus
 from parsl.providers import LocalProvider

parsl/executors/high_throughput/mpi_prefix_composer.py

@@ -1,5 +1,5 @@
 import logging
-from typing import Dict, List, Tuple, Set
+from typing import Dict, List, Set, Tuple
 
 logger = logging.getLogger(__name__)
 

parsl/executors/high_throughput/mpi_resource_management.py

@@ -8,8 +8,7 @@ from enum import Enum
 from typing import Dict, List
 
 from parsl.multiprocessing import SpawnContext
-from parsl.serialize import (pack_res_spec_apply_message,
-                             unpack_res_spec_apply_message)
+from parsl.serialize import pack_res_spec_apply_message, unpack_res_spec_apply_message
 
 logger = logging.getLogger(__name__)
 

parsl/executors/high_throughput/probe.py

@@ -1,11 +1,13 @@
-import zmq
 import argparse
-import uuid
-import time
 import logging
-from parsl.addresses import get_all_addresses
+import time
+import uuid
+
+import zmq
 from zmq.utils.monitor import recv_monitor_message
 
+from parsl.addresses import get_all_addresses
+
 logger = logging.getLogger(__name__)
 
 

parsl/executors/high_throughput/process_worker_pool.py

@@ -1,39 +1,41 @@
 #!/usr/bin/env python3
 
 import argparse
+import json
 import logging
+import math
+import multiprocessing
 import os
-import sys
+import pickle
 import platform
+import queue
+import sys
 import threading
-import pickle
 import time
-import queue
 import uuid
-from typing import Sequence, Optional, Dict, List
-
-import zmq
-import math
-import json
-import psutil
-import multiprocessing
 from multiprocessing.managers import DictProxy
 from multiprocessing.sharedctypes import Synchronized
+from typing import Dict, List, Optional, Sequence
+
+import psutil
+import zmq
 
 from parsl import curvezmq
-from parsl.process_loggers import wrap_with_logs
-from parsl.version import VERSION as PARSL_VERSION
 from parsl.app.errors import RemoteExceptionWrapper
 from parsl.executors.high_throughput.errors import WorkerLost
-from parsl.executors.high_throughput.probe import probe_addresses
-from parsl.multiprocessing import SpawnContext
-from parsl.serialize import unpack_res_spec_apply_message, serialize
+from parsl.executors.high_throughput.mpi_prefix_composer import (
+    VALID_LAUNCHERS,
+    compose_all,
+)
 from parsl.executors.high_throughput.mpi_resource_management import (
+    MPITaskScheduler,
     TaskScheduler,
-    MPITaskScheduler
 )
-
-from parsl.executors.high_throughput.mpi_prefix_composer import compose_all, VALID_LAUNCHERS
+from parsl.executors.high_throughput.probe import probe_addresses
+from parsl.multiprocessing import SpawnContext
+from parsl.process_loggers import wrap_with_logs
+from parsl.serialize import serialize, unpack_res_spec_apply_message
+from parsl.version import VERSION as PARSL_VERSION
 
 HEARTBEAT_CODE = (2 ** 32) - 1
 DRAINED_CODE = (2 ** 32) - 2
@@ -677,7 +679,8 @@ def worker(
     # If desired, set process affinity
     if cpu_affinity != "none":
         # Count the number of cores per worker
-        avail_cores = sorted(os.sched_getaffinity(0))  # Get the available threads
+        # OSX does not implement os.sched_getaffinity
+        avail_cores = sorted(os.sched_getaffinity(0))  # type: ignore[attr-defined, unused-ignore]
         cores_per_worker = len(avail_cores) // pool_size
         assert cores_per_worker > 0, "Affinity does not work if there are more workers than cores"
 
@@ -717,7 +720,15 @@ def worker(
         os.environ["KMP_AFFINITY"] = f"explicit,proclist=[{proc_list}]"  # For Intel OpenMP
 
         # Set the affinity for this worker
-        os.sched_setaffinity(0, my_cores)
+        # OSX does not implement os.sched_setaffinity so type checking
+        # is ignored here in two ways:
+        # On a platform without sched_setaffinity, that attribute will not
+        # be defined, so ignore[attr-defined] will tell mypy to ignore this
+        # incorrect-for-OS X attribute access.
+        # On a platform with sched_setaffinity, that type: ignore message
+        # will be redundant, and ignore[unused-ignore] tells mypy to ignore
+        # that this ignore is unneeded.
+        os.sched_setaffinity(0, my_cores)  # type: ignore[attr-defined, unused-ignore]
         logger.info("Set worker CPU affinity to {}".format(my_cores))
 
     # If desired, pin to accelerator

parsl/executors/high_throughput/zmq_pipes.py

@@ -1,13 +1,18 @@
 #!/usr/bin/env python3
 
-import zmq
 import logging
 import threading
 import time
+from typing import Optional
+
+import zmq
 
 from parsl import curvezmq
 from parsl.errors import InternalConsistencyError
-from parsl.executors.high_throughput.errors import CommandClientBadError, CommandClientTimeoutError
+from parsl.executors.high_throughput.errors import (
+    CommandClientBadError,
+    CommandClientTimeoutError,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -15,20 +20,23 @@ logger = logging.getLogger(__name__)
 class CommandClient:
     """ CommandClient
     """
-    def __init__(self, zmq_context: curvezmq.ClientContext, ip_address, port_range):
+    def __init__(self, ip_address, port_range, cert_dir: Optional[str] = None):
         """
         Parameters
         ----------
 
-        zmq_context: curvezmq.ClientContext
-            CurveZMQ client context used to create secure sockets
         ip_address: str
            IP address of the client (where Parsl runs)
+
         port_range: tuple(int, int)
            Port range for the comms between client and interchange
 
+        cert_dir: str | None
+            Path to the certificate directory. Setting this to None will disable encryption.
+            default: None
+
         """
-        self.zmq_context = zmq_context
+        self.zmq_context = curvezmq.ClientContext(cert_dir)
         self.ip_address = ip_address
         self.port_range = port_range
         self.port = None
@@ -119,20 +127,23 @@ class CommandClient:
 class TasksOutgoing:
     """ Outgoing task queue from the executor to the Interchange
     """
-    def __init__(self, zmq_context: curvezmq.ClientContext, ip_address, port_range):
+    def __init__(self, ip_address, port_range, cert_dir: Optional[str] = None):
         """
         Parameters
         ----------
 
-        zmq_context: curvezmq.ClientContext
-            CurveZMQ client context used to create secure sockets
         ip_address: str
            IP address of the client (where Parsl runs)
+
         port_range: tuple(int, int)
            Port range for the comms between client and interchange
 
+        cert_dir: str | None
+            Path to the certificate directory. Setting this to None will disable encryption.
+            default: None
+
         """
-        self.zmq_context = zmq_context
+        self.zmq_context = curvezmq.ClientContext(cert_dir)
         self.zmq_socket = self.zmq_context.socket(zmq.DEALER)
         self.zmq_socket.set_hwm(0)
         self.port = self.zmq_socket.bind_to_random_port("tcp://{}".format(ip_address),
@@ -172,20 +183,23 @@ class ResultsIncoming:
     """ Incoming results queue from the Interchange to the executor
     """
 
-    def __init__(self, zmq_context: curvezmq.ClientContext, ip_address, port_range):
+    def __init__(self, ip_address, port_range, cert_dir: Optional[str] = None):
         """
         Parameters
         ----------
 
-        zmq_context: curvezmq.ClientContext
-            CurveZMQ client context used to create secure sockets
         ip_address: str
            IP address of the client (where Parsl runs)
+
         port_range: tuple(int, int)
            Port range for the comms between client and interchange
 
+        cert_dir: str | None
+            Path to the certificate directory. Setting this to None will disable encryption.
+            default: None
+
         """
-        self.zmq_context = zmq_context
+        self.zmq_context = curvezmq.ClientContext(cert_dir)
         self.results_receiver = self.zmq_context.socket(zmq.DEALER)
         self.results_receiver.set_hwm(0)
         self.port = self.results_receiver.bind_to_random_port("tcp://{}".format(ip_address),

parsl/executors/radical/executor.py

@@ -1,30 +1,30 @@
 """RadicalPilotExecutor builds on the RADICAL-Pilot/Parsl
 """
+import inspect
+import logging
 import os
-import sys
-import time
-import parsl
 import queue
-import logging
-import inspect
-import requests
-import typeguard
+import sys
 import threading as mt
-
+import time
+from concurrent.futures import Future
 from functools import partial
-from typing import Optional, Dict
 from pathlib import Path, PosixPath
-from concurrent.futures import Future
+from typing import Dict, Optional
+
+import requests
+import typeguard
 
+import parsl
+from parsl.app.errors import BashExitFailure, RemoteExceptionWrapper
 from parsl.app.python import timeout
-from .rpex_resources import ResourceConfig
 from parsl.data_provider.files import File
-from parsl.utils import RepresentationMixin
-from parsl.app.errors import BashExitFailure
 from parsl.executors.base import ParslExecutor
-from parsl.app.errors import RemoteExceptionWrapper
 from parsl.serialize import deserialize, pack_res_spec_apply_message
-from parsl.serialize.errors import SerializationError, DeserializationError
+from parsl.serialize.errors import DeserializationError, SerializationError
+from parsl.utils import RepresentationMixin
+
+from .rpex_resources import ResourceConfig
 
 try:
     import radical.pilot as rp

parsl/executors/radical/rpex_master.py

@@ -2,9 +2,8 @@
 
 import sys
 
-import radical.utils as ru
 import radical.pilot as rp
-
+import radical.utils as ru
 
 # ------------------------------------------------------------------------------
 #

parsl/executors/radical/rpex_resources.py

@@ -1,6 +1,5 @@
-import sys
 import json
-
+import sys
 from typing import List
 
 _setup_paths: List[str] = []

parsl/executors/radical/rpex_worker.py

@@ -1,10 +1,11 @@
 import sys
+
 import radical.pilot as rp
 
 import parsl.app.errors as pe
 from parsl.app.bash import remote_side_bash_executor
-from parsl.serialize import unpack_res_spec_apply_message, serialize
 from parsl.executors.high_throughput.process_worker_pool import execute_task
+from parsl.serialize import serialize, unpack_res_spec_apply_message
 
 
 class ParslWorker:

parsl/executors/status_handling.py

@@ -1,17 +1,18 @@
 from __future__ import annotations
+
 import datetime
 import logging
 import threading
 import time
-from itertools import compress
 from abc import abstractmethod, abstractproperty
 from concurrent.futures import Future
-from typing import List, Any, Dict, Optional, Sequence, Tuple, Union, Callable
+from itertools import compress
+from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, Union
 
 from parsl.executors.base import ParslExecutor
 from parsl.executors.errors import BadStateException, ScalingFailed
-from parsl.jobs.states import JobStatus, JobState
-from parsl.jobs.error_handlers import simple_error_handler, noop_error_handler
+from parsl.jobs.error_handlers import noop_error_handler, simple_error_handler
+from parsl.jobs.states import JobState, JobStatus
 from parsl.monitoring.message_type import MessageType
 from parsl.providers.base import ExecutionProvider
 from parsl.utils import AtomicIDCounter

parsl/executors/taskvine/__init__.py

@@ -1,5 +1,5 @@
 from parsl.executors.taskvine.executor import TaskVineExecutor
-from parsl.executors.taskvine.manager_config import TaskVineManagerConfig
 from parsl.executors.taskvine.factory_config import TaskVineFactoryConfig
+from parsl.executors.taskvine.manager_config import TaskVineManagerConfig
 
 __all__ = ['TaskVineExecutor', 'TaskVineManagerConfig', 'TaskVineFactoryConfig']

parsl/executors/taskvine/errors.py

@@ -1,5 +1,5 @@
-from parsl.errors import ParslError
 from parsl.app.errors import AppException
+from parsl.errors import ParslError
 
 
 class TaskVineTaskFailure(AppException):

parsl/executors/taskvine/exec_parsl_function.py

@@ -1,10 +1,10 @@
-import traceback
 import sys
+import traceback
 
 from parsl.app.errors import RemoteExceptionWrapper
 from parsl.data_provider.files import File
-from parsl.utils import get_std_fname_mode
 from parsl.serialize import deserialize, serialize
+from parsl.utils import get_std_fname_mode
 
 # This scripts executes a parsl function which is pickled in 4 files:
 #

parsl/executors/taskvine/executor.py

@@ -3,46 +3,45 @@ Cooperative Computing Lab (CCL) at Notre Dame to provide a fault-tolerant,
 high-throughput system for delegating Parsl tasks to thousands of remote machines
 """
 
-# Import Python built-in libraries
-import threading
-import multiprocessing
-import logging
-import tempfile
 import hashlib
-import subprocess
+import inspect
+import itertools
+import logging
+import multiprocessing
 import os
 import queue
-import inspect
 import shutil
-import itertools
+import subprocess
+import tempfile
+
+# Import Python built-in libraries
+import threading
 import uuid
 from concurrent.futures import Future
-from typing import List, Optional, Union, Literal
+from typing import List, Literal, Optional, Union
+
+# Import other libraries
+import typeguard
 
 # Import Parsl constructs
 import parsl.utils as putils
-from parsl.data_provider.staging import Staging
-from parsl.serialize import serialize, deserialize
+from parsl.addresses import get_any_address
 from parsl.data_provider.files import File
+from parsl.data_provider.staging import Staging
 from parsl.errors import OptionalModuleMissing
-from parsl.providers.base import ExecutionProvider
-from parsl.providers import LocalProvider, CondorProvider
-from parsl.process_loggers import wrap_with_logs
-from parsl.addresses import get_any_address
 from parsl.executors.errors import ExecutorError
 from parsl.executors.status_handling import BlockProviderExecutor
 from parsl.executors.taskvine import exec_parsl_function
-from parsl.executors.taskvine.manager_config import TaskVineManagerConfig
+from parsl.executors.taskvine.errors import TaskVineManagerFailure, TaskVineTaskFailure
+from parsl.executors.taskvine.factory import _taskvine_factory
 from parsl.executors.taskvine.factory_config import TaskVineFactoryConfig
-from parsl.executors.taskvine.errors import TaskVineTaskFailure
-from parsl.executors.taskvine.errors import TaskVineManagerFailure
-from parsl.executors.taskvine.utils import ParslTaskToVine
-from parsl.executors.taskvine.utils import ParslFileToVine
 from parsl.executors.taskvine.manager import _taskvine_submit_wait
-from parsl.executors.taskvine.factory import _taskvine_factory
-
-# Import other libraries
-import typeguard
+from parsl.executors.taskvine.manager_config import TaskVineManagerConfig
+from parsl.executors.taskvine.utils import ParslFileToVine, ParslTaskToVine
+from parsl.process_loggers import wrap_with_logs
+from parsl.providers import CondorProvider, LocalProvider
+from parsl.providers.base import ExecutionProvider
+from parsl.serialize import deserialize, serialize
 
 logger = logging.getLogger(__name__)
 

parsl/executors/taskvine/factory.py

@@ -1,7 +1,7 @@
 import logging
 
-from parsl.process_loggers import wrap_with_logs
 from parsl.executors.taskvine.errors import TaskVineFactoryFailure
+from parsl.process_loggers import wrap_with_logs
 
 # This try except clause prevents import errors
 # when TaskVine is not used in Parsl.

parsl/executors/taskvine/manager.py

@@ -1,25 +1,23 @@
-import logging
 import hashlib
-import subprocess
+import logging
 import os
 import queue
 import shutil
+import subprocess
 import uuid
 
-from parsl.utils import setproctitle
-from parsl.process_loggers import wrap_with_logs
 from parsl.executors.taskvine import exec_parsl_function
-from parsl.executors.taskvine.utils import VineTaskToParsl
-from parsl.executors.taskvine.utils import run_parsl_function
+from parsl.executors.taskvine.utils import VineTaskToParsl, run_parsl_function
+from parsl.process_loggers import wrap_with_logs
+from parsl.utils import setproctitle
 
 try:
-    from ndcctools.taskvine import cvine
-    from ndcctools.taskvine import Manager
-    from ndcctools.taskvine import Task
-    from ndcctools.taskvine import FunctionCall
-    from ndcctools.taskvine.cvine import VINE_ALLOCATION_MODE_MAX_THROUGHPUT
-    from ndcctools.taskvine.cvine import VINE_ALLOCATION_MODE_EXHAUSTIVE_BUCKETING
-    from ndcctools.taskvine.cvine import VINE_ALLOCATION_MODE_MAX
+    from ndcctools.taskvine import FunctionCall, Manager, Task, cvine
+    from ndcctools.taskvine.cvine import (
+        VINE_ALLOCATION_MODE_EXHAUSTIVE_BUCKETING,
+        VINE_ALLOCATION_MODE_MAX,
+        VINE_ALLOCATION_MODE_MAX_THROUGHPUT,
+    )
 except ImportError:
     _taskvine_enabled = False
 else: