parsl 2024.5.27__py3-none-any.whl → 2024.6.3__py3-none-any.whl

parsl/executors/high_throughput/zmq_pipes.py

@@ -1,13 +1,18 @@
 #!/usr/bin/env python3
 
-import zmq
 import logging
 import threading
 import time
+from typing import Optional
+
+import zmq
 
 from parsl import curvezmq
 from parsl.errors import InternalConsistencyError
-from parsl.executors.high_throughput.errors import CommandClientBadError, CommandClientTimeoutError
+from parsl.executors.high_throughput.errors import (
+    CommandClientBadError,
+    CommandClientTimeoutError,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -15,20 +20,23 @@ logger = logging.getLogger(__name__)
 class CommandClient:
     """ CommandClient
     """
-    def __init__(self, zmq_context: curvezmq.ClientContext, ip_address, port_range):
+    def __init__(self, ip_address, port_range, cert_dir: Optional[str] = None):
         """
         Parameters
         ----------
 
-        zmq_context: curvezmq.ClientContext
-            CurveZMQ client context used to create secure sockets
         ip_address: str
            IP address of the client (where Parsl runs)
+
         port_range: tuple(int, int)
            Port range for the comms between client and interchange
 
+        cert_dir: str | None
+            Path to the certificate directory. Setting this to None will disable encryption.
+            default: None
+
         """
-        self.zmq_context = zmq_context
+        self.zmq_context = curvezmq.ClientContext(cert_dir)
         self.ip_address = ip_address
         self.port_range = port_range
         self.port = None
@@ -119,20 +127,23 @@ class CommandClient:
 class TasksOutgoing:
     """ Outgoing task queue from the executor to the Interchange
     """
-    def __init__(self, zmq_context: curvezmq.ClientContext, ip_address, port_range):
+    def __init__(self, ip_address, port_range, cert_dir: Optional[str] = None):
         """
         Parameters
         ----------
 
-        zmq_context: curvezmq.ClientContext
-            CurveZMQ client context used to create secure sockets
         ip_address: str
            IP address of the client (where Parsl runs)
+
         port_range: tuple(int, int)
            Port range for the comms between client and interchange
 
+        cert_dir: str | None
+            Path to the certificate directory. Setting this to None will disable encryption.
+            default: None
+
         """
-        self.zmq_context = zmq_context
+        self.zmq_context = curvezmq.ClientContext(cert_dir)
         self.zmq_socket = self.zmq_context.socket(zmq.DEALER)
         self.zmq_socket.set_hwm(0)
         self.port = self.zmq_socket.bind_to_random_port("tcp://{}".format(ip_address),
@@ -172,20 +183,23 @@ class ResultsIncoming:
     """ Incoming results queue from the Interchange to the executor
     """
 
-    def __init__(self, zmq_context: curvezmq.ClientContext, ip_address, port_range):
+    def __init__(self, ip_address, port_range, cert_dir: Optional[str] = None):
         """
         Parameters
         ----------
 
-        zmq_context: curvezmq.ClientContext
-            CurveZMQ client context used to create secure sockets
         ip_address: str
            IP address of the client (where Parsl runs)
+
         port_range: tuple(int, int)
            Port range for the comms between client and interchange
 
+        cert_dir: str | None
+            Path to the certificate directory. Setting this to None will disable encryption.
+            default: None
+
         """
-        self.zmq_context = zmq_context
+        self.zmq_context = curvezmq.ClientContext(cert_dir)
         self.results_receiver = self.zmq_context.socket(zmq.DEALER)
         self.results_receiver.set_hwm(0)
         self.port = self.results_receiver.bind_to_random_port("tcp://{}".format(ip_address),

parsl/executors/radical/executor.py

@@ -1,30 +1,30 @@
 """RadicalPilotExecutor builds on the RADICAL-Pilot/Parsl
 """
+import inspect
+import logging
 import os
-import sys
-import time
-import parsl
 import queue
-import logging
-import inspect
-import requests
-import typeguard
+import sys
 import threading as mt
-
+import time
+from concurrent.futures import Future
 from functools import partial
-from typing import Optional, Dict
 from pathlib import Path, PosixPath
-from concurrent.futures import Future
+from typing import Dict, Optional
+
+import requests
+import typeguard
 
+import parsl
+from parsl.app.errors import BashExitFailure, RemoteExceptionWrapper
 from parsl.app.python import timeout
-from .rpex_resources import ResourceConfig
 from parsl.data_provider.files import File
-from parsl.utils import RepresentationMixin
-from parsl.app.errors import BashExitFailure
 from parsl.executors.base import ParslExecutor
-from parsl.app.errors import RemoteExceptionWrapper
 from parsl.serialize import deserialize, pack_res_spec_apply_message
-from parsl.serialize.errors import SerializationError, DeserializationError
+from parsl.serialize.errors import DeserializationError, SerializationError
+from parsl.utils import RepresentationMixin
+
+from .rpex_resources import ResourceConfig
 
 try:
     import radical.pilot as rp

parsl/executors/radical/rpex_master.py

@@ -2,9 +2,8 @@
 
 import sys
 
-import radical.utils as ru
 import radical.pilot as rp
-
+import radical.utils as ru
 
 # ------------------------------------------------------------------------------
 #

parsl/executors/radical/rpex_resources.py

@@ -1,6 +1,5 @@
-import sys
 import json
-
+import sys
 from typing import List
 
 _setup_paths: List[str] = []

parsl/executors/radical/rpex_worker.py

@@ -1,10 +1,11 @@
 import sys
+
 import radical.pilot as rp
 
 import parsl.app.errors as pe
 from parsl.app.bash import remote_side_bash_executor
-from parsl.serialize import unpack_res_spec_apply_message, serialize
 from parsl.executors.high_throughput.process_worker_pool import execute_task
+from parsl.serialize import serialize, unpack_res_spec_apply_message
 
 
 class ParslWorker:

parsl/executors/status_handling.py

@@ -1,17 +1,18 @@
 from __future__ import annotations
+
 import datetime
 import logging
 import threading
 import time
-from itertools import compress
 from abc import abstractmethod, abstractproperty
 from concurrent.futures import Future
-from typing import List, Any, Dict, Optional, Sequence, Tuple, Union, Callable
+from itertools import compress
+from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, Union
 
 from parsl.executors.base import ParslExecutor
 from parsl.executors.errors import BadStateException, ScalingFailed
-from parsl.jobs.states import JobStatus, JobState
-from parsl.jobs.error_handlers import simple_error_handler, noop_error_handler
+from parsl.jobs.error_handlers import noop_error_handler, simple_error_handler
+from parsl.jobs.states import JobState, JobStatus
 from parsl.monitoring.message_type import MessageType
 from parsl.providers.base import ExecutionProvider
 from parsl.utils import AtomicIDCounter

parsl/executors/taskvine/__init__.py

@@ -1,5 +1,5 @@
 from parsl.executors.taskvine.executor import TaskVineExecutor
-from parsl.executors.taskvine.manager_config import TaskVineManagerConfig
 from parsl.executors.taskvine.factory_config import TaskVineFactoryConfig
+from parsl.executors.taskvine.manager_config import TaskVineManagerConfig
 
 __all__ = ['TaskVineExecutor', 'TaskVineManagerConfig', 'TaskVineFactoryConfig']

parsl/executors/taskvine/errors.py

@@ -1,5 +1,5 @@
-from parsl.errors import ParslError
 from parsl.app.errors import AppException
+from parsl.errors import ParslError
 
 
 class TaskVineTaskFailure(AppException):

parsl/executors/taskvine/exec_parsl_function.py

@@ -1,10 +1,10 @@
-import traceback
 import sys
+import traceback
 
 from parsl.app.errors import RemoteExceptionWrapper
 from parsl.data_provider.files import File
-from parsl.utils import get_std_fname_mode
 from parsl.serialize import deserialize, serialize
+from parsl.utils import get_std_fname_mode
 
 # This scripts executes a parsl function which is pickled in 4 files:
 #

parsl/executors/taskvine/executor.py

@@ -3,46 +3,45 @@ Cooperative Computing Lab (CCL) at Notre Dame to provide a fault-tolerant,
 high-throughput system for delegating Parsl tasks to thousands of remote machines
 """
 
-# Import Python built-in libraries
-import threading
-import multiprocessing
-import logging
-import tempfile
 import hashlib
-import subprocess
+import inspect
+import itertools
+import logging
+import multiprocessing
 import os
 import queue
-import inspect
 import shutil
-import itertools
+import subprocess
+import tempfile
+
+# Import Python built-in libraries
+import threading
 import uuid
 from concurrent.futures import Future
-from typing import List, Optional, Union, Literal
+from typing import List, Literal, Optional, Union
+
+# Import other libraries
+import typeguard
 
 # Import Parsl constructs
 import parsl.utils as putils
-from parsl.data_provider.staging import Staging
-from parsl.serialize import serialize, deserialize
+from parsl.addresses import get_any_address
 from parsl.data_provider.files import File
+from parsl.data_provider.staging import Staging
 from parsl.errors import OptionalModuleMissing
-from parsl.providers.base import ExecutionProvider
-from parsl.providers import LocalProvider, CondorProvider
-from parsl.process_loggers import wrap_with_logs
-from parsl.addresses import get_any_address
 from parsl.executors.errors import ExecutorError
 from parsl.executors.status_handling import BlockProviderExecutor
 from parsl.executors.taskvine import exec_parsl_function
-from parsl.executors.taskvine.manager_config import TaskVineManagerConfig
+from parsl.executors.taskvine.errors import TaskVineManagerFailure, TaskVineTaskFailure
+from parsl.executors.taskvine.factory import _taskvine_factory
 from parsl.executors.taskvine.factory_config import TaskVineFactoryConfig
-from parsl.executors.taskvine.errors import TaskVineTaskFailure
-from parsl.executors.taskvine.errors import TaskVineManagerFailure
-from parsl.executors.taskvine.utils import ParslTaskToVine
-from parsl.executors.taskvine.utils import ParslFileToVine
 from parsl.executors.taskvine.manager import _taskvine_submit_wait
-from parsl.executors.taskvine.factory import _taskvine_factory
-
-# Import other libraries
-import typeguard
+from parsl.executors.taskvine.manager_config import TaskVineManagerConfig
+from parsl.executors.taskvine.utils import ParslFileToVine, ParslTaskToVine
+from parsl.process_loggers import wrap_with_logs
+from parsl.providers import CondorProvider, LocalProvider
+from parsl.providers.base import ExecutionProvider
+from parsl.serialize import deserialize, serialize
 
 logger = logging.getLogger(__name__)
 

parsl/executors/taskvine/factory.py

@@ -1,7 +1,7 @@
 import logging
 
-from parsl.process_loggers import wrap_with_logs
 from parsl.executors.taskvine.errors import TaskVineFactoryFailure
+from parsl.process_loggers import wrap_with_logs
 
 # This try except clause prevents import errors
 # when TaskVine is not used in Parsl.

parsl/executors/taskvine/manager.py

@@ -1,25 +1,23 @@
-import logging
 import hashlib
-import subprocess
+import logging
 import os
 import queue
 import shutil
+import subprocess
 import uuid
 
-from parsl.utils import setproctitle
-from parsl.process_loggers import wrap_with_logs
 from parsl.executors.taskvine import exec_parsl_function
-from parsl.executors.taskvine.utils import VineTaskToParsl
-from parsl.executors.taskvine.utils import run_parsl_function
+from parsl.executors.taskvine.utils import VineTaskToParsl, run_parsl_function
+from parsl.process_loggers import wrap_with_logs
+from parsl.utils import setproctitle
 
 try:
-    from ndcctools.taskvine import cvine
-    from ndcctools.taskvine import Manager
-    from ndcctools.taskvine import Task
-    from ndcctools.taskvine import FunctionCall
-    from ndcctools.taskvine.cvine import VINE_ALLOCATION_MODE_MAX_THROUGHPUT
-    from ndcctools.taskvine.cvine import VINE_ALLOCATION_MODE_EXHAUSTIVE_BUCKETING
-    from ndcctools.taskvine.cvine import VINE_ALLOCATION_MODE_MAX
+    from ndcctools.taskvine import FunctionCall, Manager, Task, cvine
+    from ndcctools.taskvine.cvine import (
+        VINE_ALLOCATION_MODE_EXHAUSTIVE_BUCKETING,
+        VINE_ALLOCATION_MODE_MAX,
+        VINE_ALLOCATION_MODE_MAX_THROUGHPUT,
+    )
 except ImportError:
     _taskvine_enabled = False
 else:

parsl/executors/threads.py

@@ -1,14 +1,13 @@
-import logging
-import typeguard
 import concurrent.futures as cf
-
+import logging
 from typing import List, Optional
 
+import typeguard
+
 from parsl.data_provider.staging import Staging
 from parsl.executors.base import ParslExecutor
-from parsl.utils import RepresentationMixin
 from parsl.executors.errors import UnsupportedFeatureError
-
+from parsl.utils import RepresentationMixin
 
 logger = logging.getLogger(__name__)
 

parsl/executors/workqueue/errors.py

@@ -1,7 +1,7 @@
 from typing import Optional
 
-from parsl.errors import ParslError
 from parsl.app.errors import AppException
+from parsl.errors import ParslError
 
 
 class WorkQueueTaskFailure(AppException):

parsl/executors/workqueue/exec_parsl_function.py

@@ -1,10 +1,11 @@
+import pickle
+import sys
+import traceback
+
 from parsl.app.errors import RemoteExceptionWrapper
 from parsl.data_provider.files import File
-from parsl.utils import get_std_fname_mode
-import traceback
-import sys
-import pickle
 from parsl.serialize import serialize
+from parsl.utils import get_std_fname_mode
 
 # This scripts executes a parsl function which is pickled in a file:
 #

parsl/executors/workqueue/executor.py

@@ -3,50 +3,49 @@ Cooperative Computing Lab (CCL) at Notre Dame to provide a fault-tolerant,
 high-throughput system for delegating Parsl tasks to thousands of remote machines
 """
 
-import threading
-import multiprocessing
-import logging
-from concurrent.futures import Future
-from ctypes import c_bool
-
-import tempfile
 import hashlib
-import subprocess
+import inspect
+import itertools
+import logging
+import multiprocessing
 import os
-import socket
-import time
 import pickle
 import queue
-import inspect
 import shutil
-import itertools
+import socket
+import subprocess
+import tempfile
+import threading
+import time
+from collections import namedtuple
+from concurrent.futures import Future
+from ctypes import c_bool
+from typing import Dict, List, Optional, Set, Union
+
+import typeguard
 
-from parsl.serialize import pack_apply_message, deserialize
 import parsl.utils as putils
-from parsl.executors.errors import ExecutorError
 from parsl.data_provider.files import File
+from parsl.data_provider.staging import Staging
 from parsl.errors import OptionalModuleMissing
+from parsl.executors.errors import ExecutorError
 from parsl.executors.status_handling import BlockProviderExecutor
-from parsl.providers.base import ExecutionProvider
-from parsl.providers import LocalProvider, CondorProvider
 from parsl.executors.workqueue import exec_parsl_function
 from parsl.process_loggers import wrap_with_logs
+from parsl.providers import CondorProvider, LocalProvider
+from parsl.providers.base import ExecutionProvider
+from parsl.serialize import deserialize, pack_apply_message
 from parsl.utils import setproctitle
 
-import typeguard
-from typing import Dict, List, Optional, Set, Union
-from parsl.data_provider.staging import Staging
-
-from .errors import WorkQueueTaskFailure
-from .errors import WorkQueueFailure
-
-from collections import namedtuple
+from .errors import WorkQueueFailure, WorkQueueTaskFailure
 
 try:
     import work_queue as wq
-    from work_queue import WorkQueue
-    from work_queue import WORK_QUEUE_DEFAULT_PORT
-    from work_queue import WORK_QUEUE_ALLOCATION_MODE_MAX_THROUGHPUT
+    from work_queue import (
+        WORK_QUEUE_ALLOCATION_MODE_MAX_THROUGHPUT,
+        WORK_QUEUE_DEFAULT_PORT,
+        WorkQueue,
+    )
 except ImportError:
     _work_queue_enabled = False
     WORK_QUEUE_DEFAULT_PORT = 0

parsl/executors/workqueue/parsl_coprocess.py

@@ -1,8 +1,8 @@
 #! /usr/bin/env python3
 
-import socket
 import json
 import os
+import socket
 import sys
 
 # If enabled, coprocess will print to stdout

parsl/jobs/error_handlers.py

@@ -3,8 +3,8 @@ from __future__ import annotations
 from typing import Dict, Tuple
 
 import parsl.executors.status_handling as status_handling
-from parsl.jobs.states import JobStatus, JobState
 from parsl.jobs.errors import TooManyJobFailuresError
+from parsl.jobs.states import JobState, JobStatus
 
 
 def noop_error_handler(executor: status_handling.BlockProviderExecutor, status: Dict[str, JobStatus], threshold: int = 3) -> None:

parsl/jobs/job_status_poller.py

@@ -1,13 +1,10 @@
 import logging
-from typing import List, Sequence, Optional, Union
+from typing import List, Optional, Sequence, Union
 
-from parsl.jobs.strategy import Strategy
 from parsl.executors.status_handling import BlockProviderExecutor
-
-
+from parsl.jobs.strategy import Strategy
 from parsl.utils import Timer
 
-
 logger = logging.getLogger(__name__)
 
 

parsl/jobs/states.py

@@ -1,6 +1,6 @@
+import logging
 import os
 from enum import IntEnum
-import logging
 from typing import Optional
 
 logger = logging.getLogger(__name__)

parsl/jobs/strategy.py

@@ -1,7 +1,8 @@
 from __future__ import annotations
+
 import logging
-import time
 import math
+import time
 import warnings
 from typing import Dict, List, Optional, Sequence, TypedDict
 
@@ -11,7 +12,6 @@ from parsl.executors.status_handling import BlockProviderExecutor
 from parsl.jobs.states import JobState
 from parsl.process_loggers import wrap_with_logs
 
-
 logger = logging.getLogger(__name__)
 
 

parsl/launchers/__init__.py

@@ -1,6 +1,15 @@
-from parsl.launchers.launchers import SimpleLauncher, SingleNodeLauncher, \
-    SrunLauncher, AprunLauncher, SrunMPILauncher, WrappedLauncher, \
-    GnuParallelLauncher, MpiExecLauncher, MpiRunLauncher, JsrunLauncher
+from parsl.launchers.launchers import (
+    AprunLauncher,
+    GnuParallelLauncher,
+    JsrunLauncher,
+    MpiExecLauncher,
+    MpiRunLauncher,
+    SimpleLauncher,
+    SingleNodeLauncher,
+    SrunLauncher,
+    SrunMPILauncher,
+    WrappedLauncher,
+)
 
 __all__ = ['SimpleLauncher',
            'WrappedLauncher',

parsl/launchers/errors.py

@@ -1,5 +1,5 @@
-from parsl.providers.errors import ExecutionProviderException
 from parsl.launchers.base import Launcher
+from parsl.providers.errors import ExecutionProviderException
 
 
 class BadLauncher(ExecutionProviderException, TypeError):

parsl/log_utils.py

@@ -12,10 +12,9 @@ provided for logging:
 """
 import io
 import logging
-import typeguard
-
 from typing import Optional
 
+import typeguard
 
 DEFAULT_FORMAT = (
     "%(created)f %(asctime)s %(processName)s-%(process)d "

parsl/monitoring/db_manager.py

@@ -1,15 +1,14 @@
+import datetime
 import logging
-import threading
-import queue
 import os
+import queue
+import threading
 import time
-import datetime
-
 from typing import Any, Dict, List, Optional, Set, Tuple, TypeVar, cast
 
-from parsl.log_utils import set_file_logger
 from parsl.dataflow.states import States
 from parsl.errors import OptionalModuleMissing
+from parsl.log_utils import set_file_logger
 from parsl.monitoring.message_type import MessageType
 from parsl.monitoring.types import MonitoringMessage, TaggedMonitoringMessage
 from parsl.process_loggers import wrap_with_logs
@@ -21,11 +20,18 @@ X = TypeVar('X')
 
 try:
     import sqlalchemy as sa
-    from sqlalchemy import Column, Text, Float, Boolean, BigInteger, Integer, DateTime, PrimaryKeyConstraint, Table
-    from sqlalchemy.orm import Mapper
-    from sqlalchemy.orm import mapperlib
-    from sqlalchemy.orm import sessionmaker
-    from sqlalchemy.orm import declarative_base
+    from sqlalchemy import (
+        BigInteger,
+        Boolean,
+        Column,
+        DateTime,
+        Float,
+        Integer,
+        PrimaryKeyConstraint,
+        Table,
+        Text,
+    )
+    from sqlalchemy.orm import Mapper, declarative_base, mapperlib, sessionmaker
 except ImportError:
     _sqlalchemy_enabled = False
 else:

parsl/monitoring/monitoring.py

@@ -1,29 +1,25 @@
 from __future__ import annotations
 
-import os
-import time
 import logging
 import multiprocessing.synchronize as ms
-import typeguard
-
+import os
 import queue
-
-from parsl.multiprocessing import ForkProcess, SizedQueue
-from multiprocessing import Process
-from multiprocessing import Event
+import time
+from multiprocessing import Event, Process
 from multiprocessing.queues import Queue
-from parsl.log_utils import set_file_logger
-from parsl.utils import RepresentationMixin
-from parsl.process_loggers import wrap_with_logs
-from parsl.utils import setproctitle
+from typing import TYPE_CHECKING, Any, Optional, Tuple, Union, cast
 
-from parsl.serialize import deserialize
+import typeguard
 
+from parsl.log_utils import set_file_logger
+from parsl.monitoring.message_type import MessageType
 from parsl.monitoring.radios import MultiprocessingQueueRadio
 from parsl.monitoring.router import router_starter
-from parsl.monitoring.message_type import MessageType
 from parsl.monitoring.types import AddressedMonitoringMessage
-from typing import cast, Any, Optional, Tuple, Union, TYPE_CHECKING
+from parsl.multiprocessing import ForkProcess, SizedQueue
+from parsl.process_loggers import wrap_with_logs
+from parsl.serialize import deserialize
+from parsl.utils import RepresentationMixin, setproctitle
 
 _db_manager_excepts: Optional[Exception]