parsl 2024.5.27__py3-none-any.whl → 2024.6.3__py3-none-any.whl

parsl/data_provider/zip.py

@@ -1,16 +1,15 @@
-import filelock
 import logging
 import os
-import parsl
 import zipfile
-
 from typing import Tuple
 
-from parsl.data_provider.staging import Staging
+import filelock
+
+import parsl
 from parsl.data_provider.files import File
+from parsl.data_provider.staging import Staging
 from parsl.errors import ParslError
 
-
 logger = logging.getLogger(__name__)
 
 

parsl/dataflow/dflow.py

@@ -1,23 +1,25 @@
 from __future__ import annotations
+
 import atexit
+import datetime
+import inspect
 import logging
 import os
 import pathlib
 import pickle
 import random
-import time
-import typeguard
-import inspect
-import threading
 import sys
-import datetime
+import threading
+import time
+from concurrent.futures import Future
+from functools import partial
 from getpass import getuser
-from typeguard import typechecked
+from socket import gethostname
 from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, Union
 from uuid import uuid4
-from socket import gethostname
-from concurrent.futures import Future
-from functools import partial
+
+import typeguard
+from typeguard import typechecked
 
 import parsl
 from parsl.app.errors import RemoteExceptionWrapper
@@ -31,21 +33,24 @@ from parsl.dataflow.errors import BadCheckpoint, DependencyError, JoinError
 from parsl.dataflow.futures import AppFuture
 from parsl.dataflow.memoization import Memoizer
 from parsl.dataflow.rundirs import make_rundir
-from parsl.dataflow.states import States, FINAL_STATES, FINAL_FAILURE_STATES
+from parsl.dataflow.states import FINAL_FAILURE_STATES, FINAL_STATES, States
 from parsl.dataflow.taskrecord import TaskRecord
-from parsl.errors import ConfigurationError, InternalConsistencyError, NoDataFlowKernelError
-from parsl.jobs.job_status_poller import JobStatusPoller
-from parsl.usage_tracking.usage import UsageTracker
+from parsl.errors import (
+    ConfigurationError,
+    InternalConsistencyError,
+    NoDataFlowKernelError,
+)
 from parsl.executors.base import ParslExecutor
 from parsl.executors.status_handling import BlockProviderExecutor
 from parsl.executors.threads import ThreadPoolExecutor
+from parsl.jobs.job_status_poller import JobStatusPoller
 from parsl.monitoring import MonitoringHub
+from parsl.monitoring.message_type import MessageType
 from parsl.monitoring.remote import monitor_wrapper
 from parsl.process_loggers import wrap_with_logs
 from parsl.providers.base import ExecutionProvider
-from parsl.utils import get_version, get_std_fname_mode, get_all_checkpoints, Timer
-
-from parsl.monitoring.message_type import MessageType
+from parsl.usage_tracking.usage import UsageTracker
+from parsl.utils import Timer, get_all_checkpoints, get_std_fname_mode, get_version
 
 logger = logging.getLogger(__name__)
 

parsl/dataflow/errors.py

@@ -1,6 +1,7 @@
-from parsl.errors import ParslError
 from typing import Optional, Sequence, Tuple
 
+from parsl.errors import ParslError
+
 
 class DataFlowException(ParslError):
     """Base class for all exceptions.

parsl/dataflow/futures.py

@@ -1,12 +1,11 @@
 from __future__ import annotations
 
-from concurrent.futures import Future
 import logging
 import threading
+from concurrent.futures import Future
 from typing import Any, Optional, Sequence, Union
 
 import parsl.app.app as app
-
 from parsl.app.futures import DataFuture
 from parsl.dataflow.taskrecord import TaskRecord
 

parsl/dataflow/memoization.py

@@ -1,18 +1,18 @@
 from __future__ import annotations
+
 import hashlib
-from functools import lru_cache, singledispatch
 import logging
 import pickle
-from parsl.dataflow.taskrecord import TaskRecord
+from functools import lru_cache, singledispatch
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
 
-from typing import Dict, Any, List, Optional, TYPE_CHECKING
+from parsl.dataflow.taskrecord import TaskRecord
 
 if TYPE_CHECKING:
     from parsl import DataFlowKernel  # import loop at runtime - needed for typechecking - TODO turn into "if typing:"
 
-from concurrent.futures import Future
-
 import types
+from concurrent.futures import Future
 
 logger = logging.getLogger(__name__)
 

parsl/dataflow/rundirs.py

@@ -1,6 +1,6 @@
+import logging
 import os
 from glob import glob
-import logging
 
 logger = logging.getLogger(__name__)
 

parsl/dataflow/taskrecord.py

@@ -1,19 +1,18 @@
 from __future__ import annotations
 
-import threading
 import datetime
-from typing_extensions import TypedDict
+import threading
 from concurrent.futures import Future
 
-
 # only for type checking:
-from typing import Any, Callable, Dict, Optional, List, Sequence, TYPE_CHECKING, Union
+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Sequence, Union
+
+from typing_extensions import TypedDict
 
 if TYPE_CHECKING:
     from parsl.dataflow.futures import AppFuture
 
 import parsl.dataflow.dflow as dflow
-
 from parsl.dataflow.states import States
 
 

parsl/executors/__init__.py

@@ -1,8 +1,8 @@
-from parsl.executors.threads import ThreadPoolExecutor
-from parsl.executors.workqueue.executor import WorkQueueExecutor
+from parsl.executors.flux.executor import FluxExecutor
 from parsl.executors.high_throughput.executor import HighThroughputExecutor
 from parsl.executors.high_throughput.mpi_executor import MPIExecutor
-from parsl.executors.flux.executor import FluxExecutor
+from parsl.executors.threads import ThreadPoolExecutor
+from parsl.executors.workqueue.executor import WorkQueueExecutor
 
 __all__ = ['ThreadPoolExecutor',
            'HighThroughputExecutor',

parsl/executors/base.py

@@ -2,6 +2,7 @@ import os
 from abc import ABCMeta, abstractmethod
 from concurrent.futures import Future
 from typing import Any, Callable, Dict, Optional
+
 from typing_extensions import Literal, Self
 
 from parsl.monitoring.radios import MonitoringRadio

parsl/executors/flux/execute_parsl_task.py

@@ -1,12 +1,12 @@
 """Script for executing tasks inside of Flux jobs."""
 
 import argparse
-import os
 import logging
+import os
 
+from parsl.executors.flux import TaskResult
 from parsl.executors.high_throughput.process_worker_pool import execute_task
 from parsl.serialize import serialize
-from parsl.executors.flux import TaskResult
 
 
 def main():

parsl/executors/flux/executor.py

@@ -1,33 +1,32 @@
 """Defines the FluxExecutor class."""
 
+import collections
 import concurrent.futures as cf
 import functools
+import itertools
 import os
+import queue
+import shutil
 import sys
-import uuid
 import threading
-import itertools
-import shutil
-import queue
-from socket import gethostname
-import collections
-from collections.abc import Mapping, Callable
-from typing import Optional, Any, Dict
+import uuid
 import weakref
+from collections.abc import Callable, Mapping
+from socket import gethostname
+from typing import Any, Dict, Optional
 
 import zmq
 
-from parsl.utils import RepresentationMixin
+from parsl.app.errors import AppException
 from parsl.executors.base import ParslExecutor
+from parsl.executors.errors import ScalingFailed
 from parsl.executors.flux.execute_parsl_task import __file__ as _WORKER_PATH
 from parsl.executors.flux.flux_instance_manager import __file__ as _MANAGER_PATH
-from parsl.executors.errors import ScalingFailed
 from parsl.providers import LocalProvider
 from parsl.providers.base import ExecutionProvider
 from parsl.serialize import deserialize, pack_res_spec_apply_message
 from parsl.serialize.errors import SerializationError
-from parsl.app.errors import AppException
-
+from parsl.utils import RepresentationMixin
 
 _WORKER_PATH = os.path.realpath(_WORKER_PATH)
 _MANAGER_PATH = os.path.realpath(_MANAGER_PATH)

parsl/executors/flux/flux_instance_manager.py

@@ -1,10 +1,10 @@
 """Script meant to be the initial program of a Flux instance."""
 
 import argparse
+import logging
 import os
 from os.path import dirname
-import logging
-from socket import gethostname, gethostbyname
+from socket import gethostbyname, gethostname
 
 import zmq
 
@@ -16,8 +16,8 @@ def main():
     encapsulating Flux instance.
     """
     # flux imports only available when launched under Flux instance
-    import flux.job
     import flux
+    import flux.job
 
     logging.basicConfig(
         level=logging.DEBUG, format="%(asctime)s [%(levelname)s] %(message)s"

parsl/executors/high_throughput/executor.py

@@ -1,44 +1,39 @@
+import logging
+import math
+import pickle
+import threading
 import typing
+import warnings
 from collections import defaultdict
 from concurrent.futures import Future
-import typeguard
-import logging
-import threading
-import queue
-import pickle
 from dataclasses import dataclass
-from multiprocessing import Process, Queue
-from typing import Dict, Sequence
-from typing import List, Optional, Tuple, Union, Callable
-import math
-import warnings
+from multiprocessing import Process
+from typing import Callable, Dict, List, Optional, Sequence, Tuple, Union
+
+import typeguard
 
 import parsl.launchers
-from parsl.usage_tracking.api import UsageInformation
-from parsl.serialize import pack_res_spec_apply_message, deserialize
-from parsl.serialize.errors import SerializationError, DeserializationError
+from parsl import curvezmq
+from parsl.addresses import get_all_addresses
 from parsl.app.errors import RemoteExceptionWrapper
-from parsl.jobs.states import JobStatus, JobState, TERMINAL_STATES
-from parsl.executors.high_throughput import zmq_pipes
-from parsl.executors.high_throughput import interchange
-from parsl.executors.errors import (
-    BadMessage, ScalingFailed,
-)
+from parsl.data_provider.staging import Staging
+from parsl.executors.errors import BadMessage, ScalingFailed
+from parsl.executors.high_throughput import interchange, zmq_pipes
+from parsl.executors.high_throughput.errors import CommandClientTimeoutError
 from parsl.executors.high_throughput.mpi_prefix_composer import (
     VALID_LAUNCHERS,
-    validate_resource_spec
+    validate_resource_spec,
 )
-
-from parsl import curvezmq
 from parsl.executors.status_handling import BlockProviderExecutor
-from parsl.providers.base import ExecutionProvider
-from parsl.data_provider.staging import Staging
-from parsl.addresses import get_all_addresses
-from parsl.process_loggers import wrap_with_logs
-
+from parsl.jobs.states import TERMINAL_STATES, JobState, JobStatus
 from parsl.multiprocessing import ForkProcess
-from parsl.utils import RepresentationMixin
+from parsl.process_loggers import wrap_with_logs
 from parsl.providers import LocalProvider
+from parsl.providers.base import ExecutionProvider
+from parsl.serialize import deserialize, pack_res_spec_apply_message
+from parsl.serialize.errors import DeserializationError, SerializationError
+from parsl.usage_tracking.api import UsageInformation
+from parsl.utils import RepresentationMixin
 
 logger = logging.getLogger(__name__)
 
@@ -415,13 +410,13 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin, UsageIn
             )
 
         self.outgoing_q = zmq_pipes.TasksOutgoing(
-            curvezmq.ClientContext(self.cert_dir), "127.0.0.1", self.interchange_port_range
+            "127.0.0.1", self.interchange_port_range, self.cert_dir
         )
         self.incoming_q = zmq_pipes.ResultsIncoming(
-            curvezmq.ClientContext(self.cert_dir), "127.0.0.1", self.interchange_port_range
+            "127.0.0.1", self.interchange_port_range, self.cert_dir
         )
         self.command_client = zmq_pipes.CommandClient(
-            curvezmq.ClientContext(self.cert_dir), "127.0.0.1", self.interchange_port_range
+            "127.0.0.1", self.interchange_port_range, self.cert_dir
         )
 
         self._queue_management_thread = None
@@ -531,9 +526,7 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin, UsageIn
         Starts the interchange process locally and uses an internal command queue to
         get the worker task and result ports that the interchange has bound to.
         """
-        comm_q = Queue(maxsize=10)
         self.interchange_proc = ForkProcess(target=interchange.starter,
-                                            args=(comm_q,),
                                             kwargs={"client_ports": (self.outgoing_q.port,
                                                                      self.incoming_q.port,
                                                                      self.command_client.port),
@@ -552,9 +545,10 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin, UsageIn
                                             name="HTEX-Interchange"
                                             )
         self.interchange_proc.start()
+
         try:
-            (self.worker_task_port, self.worker_result_port) = comm_q.get(block=True, timeout=120)
-        except queue.Empty:
+            (self.worker_task_port, self.worker_result_port) = self.command_client.run("WORKER_PORTS", timeout_s=120)
+        except CommandClientTimeoutError:
             logger.error("Interchange has not completed initialization in 120s. Aborting")
             raise Exception("Interchange failed to start")
 

parsl/executors/high_throughput/interchange.py

@@ -1,31 +1,28 @@
 #!/usr/bin/env python
-import multiprocessing
-import zmq
+import datetime
+import json
+import logging
 import os
-import sys
+import pickle
 import platform
+import queue
 import random
-import time
-import datetime
-import pickle
 import signal
-import logging
-import queue
+import sys
 import threading
-import json
+import time
+from typing import Any, Dict, List, NoReturn, Optional, Sequence, Set, Tuple, cast
 
-from typing import cast, Any, Dict, NoReturn, Sequence, Set, Optional, Tuple, List
+import zmq
 
 from parsl import curvezmq
-from parsl.utils import setproctitle
-from parsl.version import VERSION as PARSL_VERSION
-from parsl.serialize import serialize as serialize_object
-
 from parsl.app.errors import RemoteExceptionWrapper
 from parsl.executors.high_throughput.manager_record import ManagerRecord
 from parsl.monitoring.message_type import MessageType
 from parsl.process_loggers import wrap_with_logs
-
+from parsl.serialize import serialize as serialize_object
+from parsl.utils import setproctitle
+from parsl.version import VERSION as PARSL_VERSION
 
 PKL_HEARTBEAT_CODE = pickle.dumps((2 ** 32) - 1)
 PKL_DRAINED_CODE = pickle.dumps((2 ** 32) - 2)
@@ -328,6 +325,9 @@ class Interchange:
 
                     reply = None
 
+                elif command_req == "WORKER_PORTS":
+                    reply = (self.worker_task_port, self.worker_result_port)
+
                 else:
                     logger.error(f"Received unknown command: {command_req}")
                     reply = None
@@ -672,7 +672,7 @@ def start_file_logger(filename: str, level: int = logging.DEBUG, format_string:
 
 
 @wrap_with_logs(target="interchange")
-def starter(comm_q: multiprocessing.Queue, *args: Any, **kwargs: Any) -> None:
+def starter(*args: Any, **kwargs: Any) -> None:
     """Start the interchange process
 
     The executor is expected to call this function. The args, kwargs match that of the Interchange.__init__
@@ -680,6 +680,4 @@ def starter(comm_q: multiprocessing.Queue, *args: Any, **kwargs: Any) -> None:
     setproctitle("parsl: HTEX interchange")
     # logger = multiprocessing.get_logger()
     ic = Interchange(*args, **kwargs)
-    comm_q.put((ic.worker_task_port,
-                ic.worker_result_port))
     ic.start()

parsl/executors/high_throughput/manager_record.py

@@ -1,5 +1,6 @@
 from datetime import datetime
 from typing import Any, List, Optional
+
 from typing_extensions import TypedDict
 
 

parsl/executors/high_throughput/monitoring_info.py

@@ -3,6 +3,7 @@
 # then be acquired by any other code running in
 # a worker context - specifically the monitoring
 # wrapper code.
-from typing import Optional
 from queue import Queue
+from typing import Optional
+
 result_queue: Optional[Queue] = None

parsl/executors/high_throughput/mpi_executor.py

@@ -1,10 +1,13 @@
 """A simplified interface for HTEx when running in MPI mode"""
-from typing import Optional, Tuple, List, Union, Callable, Dict
+from typing import Callable, Dict, List, Optional, Tuple, Union
 
 import typeguard
 
 from parsl.data_provider.staging import Staging
-from parsl.executors.high_throughput.executor import HighThroughputExecutor, GENERAL_HTEX_PARAM_DOCS
+from parsl.executors.high_throughput.executor import (
+    GENERAL_HTEX_PARAM_DOCS,
+    HighThroughputExecutor,
+)
 from parsl.executors.status_handling import BlockProviderExecutor
 from parsl.jobs.states import JobStatus
 from parsl.providers import LocalProvider

parsl/executors/high_throughput/mpi_prefix_composer.py

@@ -1,5 +1,5 @@
 import logging
-from typing import Dict, List, Tuple, Set
+from typing import Dict, List, Set, Tuple
 
 logger = logging.getLogger(__name__)
 

parsl/executors/high_throughput/mpi_resource_management.py

@@ -8,8 +8,7 @@ from enum import Enum
 from typing import Dict, List
 
 from parsl.multiprocessing import SpawnContext
-from parsl.serialize import (pack_res_spec_apply_message,
-                             unpack_res_spec_apply_message)
+from parsl.serialize import pack_res_spec_apply_message, unpack_res_spec_apply_message
 
 logger = logging.getLogger(__name__)
 

parsl/executors/high_throughput/probe.py

@@ -1,11 +1,13 @@
-import zmq
 import argparse
-import uuid
-import time
 import logging
-from parsl.addresses import get_all_addresses
+import time
+import uuid
+
+import zmq
 from zmq.utils.monitor import recv_monitor_message
 
+from parsl.addresses import get_all_addresses
+
 logger = logging.getLogger(__name__)
 
 

parsl/executors/high_throughput/process_worker_pool.py

@@ -1,39 +1,41 @@
 #!/usr/bin/env python3
 
 import argparse
+import json
 import logging
+import math
+import multiprocessing
 import os
-import sys
+import pickle
 import platform
+import queue
+import sys
 import threading
-import pickle
 import time
-import queue
 import uuid
-from typing import Sequence, Optional, Dict, List
-
-import zmq
-import math
-import json
-import psutil
-import multiprocessing
 from multiprocessing.managers import DictProxy
 from multiprocessing.sharedctypes import Synchronized
+from typing import Dict, List, Optional, Sequence
+
+import psutil
+import zmq
 
 from parsl import curvezmq
-from parsl.process_loggers import wrap_with_logs
-from parsl.version import VERSION as PARSL_VERSION
 from parsl.app.errors import RemoteExceptionWrapper
 from parsl.executors.high_throughput.errors import WorkerLost
-from parsl.executors.high_throughput.probe import probe_addresses
-from parsl.multiprocessing import SpawnContext
-from parsl.serialize import unpack_res_spec_apply_message, serialize
+from parsl.executors.high_throughput.mpi_prefix_composer import (
+    VALID_LAUNCHERS,
+    compose_all,
+)
 from parsl.executors.high_throughput.mpi_resource_management import (
+    MPITaskScheduler,
     TaskScheduler,
-    MPITaskScheduler
 )
-
-from parsl.executors.high_throughput.mpi_prefix_composer import compose_all, VALID_LAUNCHERS
+from parsl.executors.high_throughput.probe import probe_addresses
+from parsl.multiprocessing import SpawnContext
+from parsl.process_loggers import wrap_with_logs
+from parsl.serialize import serialize, unpack_res_spec_apply_message
+from parsl.version import VERSION as PARSL_VERSION
 
 HEARTBEAT_CODE = (2 ** 32) - 1
 DRAINED_CODE = (2 ** 32) - 2
@@ -677,7 +679,8 @@ def worker(
     # If desired, set process affinity
     if cpu_affinity != "none":
         # Count the number of cores per worker
-        avail_cores = sorted(os.sched_getaffinity(0))  # Get the available threads
+        # OSX does not implement os.sched_getaffinity
+        avail_cores = sorted(os.sched_getaffinity(0))  # type: ignore[attr-defined, unused-ignore]
         cores_per_worker = len(avail_cores) // pool_size
         assert cores_per_worker > 0, "Affinity does not work if there are more workers than cores"
 
@@ -717,7 +720,15 @@ def worker(
         os.environ["KMP_AFFINITY"] = f"explicit,proclist=[{proc_list}]"  # For Intel OpenMP
 
         # Set the affinity for this worker
-        os.sched_setaffinity(0, my_cores)
+        # OSX does not implement os.sched_setaffinity so type checking
+        # is ignored here in two ways:
+        # On a platform without sched_setaffinity, that attribute will not
+        # be defined, so ignore[attr-defined] will tell mypy to ignore this
+        # incorrect-for-OS X attribute access.
+        # On a platform with sched_setaffinity, that type: ignore message
+        # will be redundant, and ignore[unused-ignore] tells mypy to ignore
+        # that this ignore is unneeded.
+        os.sched_setaffinity(0, my_cores)  # type: ignore[attr-defined, unused-ignore]
         logger.info("Set worker CPU affinity to {}".format(my_cores))
 
     # If desired, pin to accelerator