parsl 2024.3.18__py3-none-any.whl → 2025.1.13__py3-none-any.whl

parsl/executors/status_handling.py

@@ -1,15 +1,19 @@
 from __future__ import annotations
+
+import datetime
 import logging
 import threading
-from itertools import compress
+import time
 from abc import abstractmethod, abstractproperty
 from concurrent.futures import Future
-from typing import List, Any, Dict, Optional, Tuple, Union, Callable
+from itertools import compress
+from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, Union
 
 from parsl.executors.base import ParslExecutor
 from parsl.executors.errors import BadStateException, ScalingFailed
-from parsl.jobs.states import JobStatus, JobState
-from parsl.jobs.error_handlers import simple_error_handler, noop_error_handler
+from parsl.jobs.error_handlers import noop_error_handler, simple_error_handler
+from parsl.jobs.states import TERMINAL_STATES, JobState, JobStatus
+from parsl.monitoring.message_type import MessageType
 from parsl.providers.base import ExecutionProvider
 from parsl.utils import AtomicIDCounter
 
@@ -40,9 +44,6 @@ class BlockProviderExecutor(ParslExecutor):
     invoking scale_out, but it will not initialize the blocks requested by
     any init_blocks parameter. Subclasses must implement that behaviour
     themselves.
-
-    BENC: TODO: block error handling: maybe I want this more user pluggable?
-    I'm not sure of use cases for switchability at the moment beyond "yes or no"
     """
     def __init__(self, *,
                  provider: Optional[ExecutionProvider],
@@ -58,18 +59,29 @@ class BlockProviderExecutor(ParslExecutor):
         else:
             self.block_error_handler = block_error_handler
 
-        # errors can happen during the submit call to the provider; this is used
-        # to keep track of such errors so that they can be handled in one place
-        # together with errors reported by status()
-        self._simulated_status: Dict[Any, JobStatus] = {}
         self._executor_bad_state = threading.Event()
         self._executor_exception: Optional[Exception] = None
 
         self._block_id_counter = AtomicIDCounter()
 
         self._tasks = {}  # type: Dict[object, Future]
-        self.blocks = {}  # type: Dict[str, str]
-        self.block_mapping = {}  # type: Dict[str, str]
+
+        self._last_poll_time = 0.0
+
+        # these four structures track, in loosely coordinated fashion, the
+        # existence of blocks and jobs and how to map between their
+        # identifiers.
+        self.blocks_to_job_id = {}  # type: Dict[str, str]
+        self.job_ids_to_block = {}  # type: Dict[str, str]
+
+        # errors can happen during the submit call to the provider; this is used
+        # to keep track of such errors so that they can be handled in one place
+        # together with errors reported by status()
+        self._simulated_status: Dict[str, JobStatus] = {}
+
+        # this stores an approximation (sometimes delayed) of the latest status
+        # of pending, active and recently terminated blocks
+        self._status = {}  # type: Dict[str, JobStatus]
 
     def _make_status_dict(self, block_ids: List[str], status_list: List[JobStatus]) -> Dict[str, JobStatus]:
         """Given a list of block ids and a list of corresponding status strings,
@@ -102,15 +114,6 @@ class BlockProviderExecutor(ParslExecutor):
         else:
             return self._provider.status_polling_interval
 
-    def _fail_job_async(self, block_id: Any, message: str):
-        """Marks a job that has failed to start but would not otherwise be included in status()
-        as failed and report it in status()
-        """
-        if block_id is None:
-            block_id = str(self._block_id_counter.get_id())
-            logger.info(f"Allocated block ID {block_id} for simulated failure")
-        self._simulated_status[block_id] = JobStatus(JobState.FAILED, message)
-
     @abstractproperty
     def outstanding(self) -> int:
         """This should return the number of tasks that the executor has been given to run (waiting to run, and running now)"""
@@ -118,20 +121,6 @@ class BlockProviderExecutor(ParslExecutor):
         raise NotImplementedError("Classes inheriting from BlockProviderExecutor must implement "
                                   "outstanding()")
 
-    def status(self) -> Dict[str, JobStatus]:
-        """Return the status of all jobs/blocks currently known to this executor.
-
-        :return: a dictionary mapping block ids (in string) to job status
-        """
-        if self._provider:
-            block_ids, job_ids = self._get_block_and_job_ids()
-            status = self._make_status_dict(block_ids, self._provider.status(job_ids))
-        else:
-            status = {}
-        status.update(self._simulated_status)
-
-        return status
-
     def set_bad_state_and_fail_all(self, exception: Exception):
         """Allows external error handlers to mark this executor as irrecoverably bad and cause
         all tasks submitted to it now and in the future to fail. The executor is responsible
@@ -178,46 +167,82 @@ class BlockProviderExecutor(ParslExecutor):
     def provider(self):
         return self._provider
 
-    def _filter_scale_in_ids(self, to_kill, killed):
+    def _filter_scale_in_ids(self, to_kill: Sequence[Any], killed: Sequence[bool]) -> Sequence[Any]:
         """ Filter out job id's that were not killed
         """
         assert len(to_kill) == len(killed)
+
+        if False in killed:
+            killed_job_ids = [jid for jid, k in zip(to_kill, killed) if k]
+            not_killed_job_ids = [jid for jid, k in zip(to_kill, killed) if not k]
+            logger.warning("Some jobs were not killed successfully: "
+                           f"killed jobs: {killed_job_ids}, "
+                           f"not-killed jobs: {not_killed_job_ids}")
+
         # Filters first iterable by bool values in second
         return list(compress(to_kill, killed))
 
-    def scale_out(self, blocks: int = 1) -> List[str]:
+    def scale_out_facade(self, n: int) -> List[str]:
         """Scales out the number of blocks by "blocks"
         """
         if not self.provider:
             raise ScalingFailed(self, "No execution provider available")
         block_ids = []
-        logger.info(f"Scaling out by {blocks} blocks")
-        for _ in range(blocks):
+        monitoring_status_changes = {}
+        logger.info(f"Scaling out by {n} blocks")
+        for _ in range(n):
             block_id = str(self._block_id_counter.get_id())
             logger.info(f"Allocated block ID {block_id}")
             try:
                 job_id = self._launch_block(block_id)
-                self.blocks[block_id] = job_id
-                self.block_mapping[job_id] = block_id
+
+                pending_status = JobStatus(JobState.PENDING)
+
+                self.blocks_to_job_id[block_id] = job_id
+                self.job_ids_to_block[job_id] = block_id
+                self._status[block_id] = pending_status
+
+                monitoring_status_changes[block_id] = pending_status
                 block_ids.append(block_id)
+
             except Exception as ex:
-                self._fail_job_async(block_id,
-                                     "Failed to start block {}: {}".format(block_id, ex))
+                failed_status = JobStatus(JobState.FAILED, "Failed to start block {}: {}".format(block_id, ex))
+                self._simulated_status[block_id] = failed_status
+                self._status[block_id] = failed_status
+
+        self.send_monitoring_info(monitoring_status_changes)
         return block_ids
 
-    @abstractmethod
     def scale_in(self, blocks: int) -> List[str]:
         """Scale in method.
 
         Cause the executor to reduce the number of blocks by count.
 
-        We should have the scale in method simply take resource object
-        which will have the scaling methods, scale_in itself should be a coroutine, since
-        scaling tasks can be slow.
+        The default implementation will kill blocks without regard to their
+        status or whether they are executing tasks. Executors with more
+        nuanced scaling strategies might overload this method to work with
+        that strategy - see the HighThroughputExecutor for an example of that.
 
         :return: A list of block ids corresponding to the blocks that were removed.
         """
-        pass
+
+        active_blocks = [block_id for block_id, status in self._status.items()
+                         if status.state not in TERMINAL_STATES]
+
+        block_ids_to_kill = active_blocks[:blocks]
+
+        job_ids_to_kill = [self.blocks_to_job_id[block] for block in block_ids_to_kill]
+
+        # Cancel the blocks provisioned
+        if self.provider:
+            logger.info(f"Scaling in jobs: {job_ids_to_kill}")
+            r = self.provider.cancel(job_ids_to_kill)
+            job_ids = self._filter_scale_in_ids(job_ids_to_kill, r)
+            block_ids_killed = [self.job_ids_to_block[job_id] for job_id in job_ids]
+            return block_ids_killed
+        else:
+            logger.error("No execution provider available to scale in")
+            return []
 
     def _launch_block(self, block_id: str) -> Any:
         launch_cmd = self._get_launch_command(block_id)
@@ -239,12 +264,93 @@ class BlockProviderExecutor(ParslExecutor):
         # Not using self.blocks.keys() and self.blocks.values() simultaneously
         # The dictionary may be changed during invoking this function
         # As scale_in and scale_out are invoked in multiple threads
-        block_ids = list(self.blocks.keys())
+        block_ids = list(self.blocks_to_job_id.keys())
         job_ids = []  # types: List[Any]
         for bid in block_ids:
-            job_ids.append(self.blocks[bid])
+            job_ids.append(self.blocks_to_job_id[bid])
         return block_ids, job_ids
 
     @abstractproperty
     def workers_per_node(self) -> Union[int, float]:
         pass
+
+    def send_monitoring_info(self, status: Dict) -> None:
+        # Send monitoring info for HTEX when monitoring enabled
+        if self.submit_monitoring_radio:
+            msg = self.create_monitoring_info(status)
+            logger.debug("Sending block monitoring message: %r", msg)
+            self.submit_monitoring_radio.send((MessageType.BLOCK_INFO, msg))
+
+    def create_monitoring_info(self, status: Dict[str, JobStatus]) -> Sequence[object]:
+        """Create a monitoring message for each block based on the poll status.
+        """
+        msg = []
+        for bid, s in status.items():
+            d: Dict[str, Any] = {}
+            d['run_id'] = self.run_id
+            d['status'] = s.status_name
+            d['timestamp'] = datetime.datetime.now()
+            d['executor_label'] = self.label
+            d['job_id'] = self.blocks_to_job_id.get(bid, None)
+            d['block_id'] = bid
+            msg.append(d)
+        return msg
+
+    def poll_facade(self) -> None:
+        now = time.time()
+        if now >= self._last_poll_time + self.status_polling_interval:
+            previous_status = self._status
+            self._status = self.status()
+            self._last_poll_time = now
+            delta_status = {}
+            for block_id in self._status:
+                if block_id not in previous_status \
+                   or previous_status[block_id].state != self._status[block_id].state:
+                    delta_status[block_id] = self._status[block_id]
+
+            if delta_status:
+                self.send_monitoring_info(delta_status)
+
+    def status(self) -> Dict[str, JobStatus]:
+        """Return the status of all jobs/blocks currently known to this executor.
+
+        :return: a dictionary mapping block ids (in string) to job status
+        """
+        if self._provider:
+            block_ids, job_ids = self._get_block_and_job_ids()
+            status = self._make_status_dict(block_ids, self._provider.status(job_ids))
+        else:
+            status = {}
+        status.update(self._simulated_status)
+
+        return status
+
+    @property
+    def status_facade(self) -> Dict[str, JobStatus]:
+        """Return the status of all jobs/blocks of the executor of this poller.
+
+        :return: a dictionary mapping block ids (in string) to job status
+        """
+        return self._status
+
+    def scale_in_facade(self, n: int, max_idletime: Optional[float] = None) -> List[str]:
+
+        if max_idletime is None:
+            block_ids = self.scale_in(n)
+        else:
+            # This is a HighThroughputExecutor-specific interface violation.
+            # This code hopes, through pan-codebase reasoning, that this
+            # scale_in method really does come from HighThroughputExecutor,
+            # and so does have an extra max_idletime parameter not present
+            # in the executor interface.
+            block_ids = self.scale_in(n, max_idletime=max_idletime)  # type: ignore[call-arg]
+        if block_ids is not None:
+            new_status = {}
+            for block_id in block_ids:
+                logger.debug("Marking block %s as SCALED_IN", block_id)
+                s = JobStatus(JobState.SCALED_IN)
+                new_status[block_id] = s
+                self._status[block_id] = s
+                self._simulated_status[block_id] = s
+            self.send_monitoring_info(new_status)
+        return block_ids

parsl/executors/taskvine/__init__.py

@@ -1,5 +1,5 @@
 from parsl.executors.taskvine.executor import TaskVineExecutor
-from parsl.executors.taskvine.manager_config import TaskVineManagerConfig
 from parsl.executors.taskvine.factory_config import TaskVineFactoryConfig
+from parsl.executors.taskvine.manager_config import TaskVineManagerConfig
 
 __all__ = ['TaskVineExecutor', 'TaskVineManagerConfig', 'TaskVineFactoryConfig']

parsl/executors/taskvine/errors.py

@@ -1,5 +1,5 @@
-from parsl.errors import ParslError
 from parsl.app.errors import AppException
+from parsl.errors import ParslError
 
 
 class TaskVineTaskFailure(AppException):

parsl/executors/taskvine/exec_parsl_function.py

@@ -1,10 +1,10 @@
-import traceback
 import sys
+import traceback
 
 from parsl.app.errors import RemoteExceptionWrapper
 from parsl.data_provider.files import File
-from parsl.utils import get_std_fname_mode
 from parsl.serialize import deserialize, serialize
+from parsl.utils import get_std_fname_mode
 
 # This scripts executes a parsl function which is pickled in 4 files:
 #

parsl/executors/taskvine/executor.py

@@ -3,46 +3,47 @@ Cooperative Computing Lab (CCL) at Notre Dame to provide a fault-tolerant,
 high-throughput system for delegating Parsl tasks to thousands of remote machines
 """
 
-# Import Python built-in libraries
-import threading
-import multiprocessing
-import logging
-import tempfile
+import getpass
 import hashlib
-import subprocess
+import inspect
+import itertools
+import logging
+import multiprocessing
 import os
 import queue
-import inspect
 import shutil
-import itertools
+import subprocess
+import tempfile
+
+# Import Python built-in libraries
+import threading
 import uuid
 from concurrent.futures import Future
-from typing import List, Optional, Union, Literal
+from datetime import datetime
+from typing import List, Literal, Optional, Union
+
+# Import other libraries
+import typeguard
 
 # Import Parsl constructs
 import parsl.utils as putils
-from parsl.data_provider.staging import Staging
-from parsl.serialize import serialize, deserialize
+from parsl.addresses import get_any_address
 from parsl.data_provider.files import File
+from parsl.data_provider.staging import Staging
 from parsl.errors import OptionalModuleMissing
-from parsl.providers.base import ExecutionProvider
-from parsl.providers import LocalProvider, CondorProvider
-from parsl.process_loggers import wrap_with_logs
-from parsl.addresses import get_any_address
 from parsl.executors.errors import ExecutorError
 from parsl.executors.status_handling import BlockProviderExecutor
 from parsl.executors.taskvine import exec_parsl_function
-from parsl.executors.taskvine.manager_config import TaskVineManagerConfig
+from parsl.executors.taskvine.errors import TaskVineManagerFailure, TaskVineTaskFailure
+from parsl.executors.taskvine.factory import _taskvine_factory
 from parsl.executors.taskvine.factory_config import TaskVineFactoryConfig
-from parsl.executors.taskvine.errors import TaskVineTaskFailure
-from parsl.executors.taskvine.errors import TaskVineManagerFailure
-from parsl.executors.taskvine.utils import ParslTaskToVine
-from parsl.executors.taskvine.utils import ParslFileToVine
 from parsl.executors.taskvine.manager import _taskvine_submit_wait
-from parsl.executors.taskvine.factory import _taskvine_factory
-
-# Import other libraries
-import typeguard
+from parsl.executors.taskvine.manager_config import TaskVineManagerConfig
+from parsl.executors.taskvine.utils import ParslFileToVine, ParslTaskToVine
+from parsl.process_loggers import wrap_with_logs
+from parsl.providers import CondorProvider, LocalProvider
+from parsl.providers.base import ExecutionProvider
+from parsl.serialize import deserialize, serialize
 
 logger = logging.getLogger(__name__)
 
@@ -171,7 +172,7 @@ class TaskVineExecutor(BlockProviderExecutor, putils.RepresentationMixin):
         # Path to directory that holds all tasks' data and results.
         self._function_data_dir = ""
 
-        # helper scripts to prepare package tarballs for Parsl apps
+        # Helper scripts to prepare package tarballs for Parsl apps
         self._package_analyze_script = shutil.which("poncho_package_analyze")
         self._package_create_script = shutil.which("poncho_package_create")
         if self._package_analyze_script is None or self._package_create_script is None:
@@ -216,9 +217,9 @@ class TaskVineExecutor(BlockProviderExecutor, putils.RepresentationMixin):
 
         # Create directories for data and results
         log_dir = os.path.join(run_dir, self.label)
-        self._function_data_dir = os.path.join(run_dir, self.label, "function_data")
         os.makedirs(log_dir)
-        os.makedirs(self._function_data_dir)
+        tmp_prefix = f'{self.label}-{getpass.getuser()}-{datetime.now().strftime("%Y%m%d%H%M%S%f")}-'
+        self._function_data_dir = tempfile.TemporaryDirectory(prefix=tmp_prefix)
 
         # put TaskVine logs outside of a Parsl run as TaskVine caches between runs while
         # Parsl does not.
@@ -228,7 +229,7 @@ class TaskVineExecutor(BlockProviderExecutor, putils.RepresentationMixin):
 
         # factory logs go with manager logs regardless
         self.factory_config.scratch_dir = self.manager_config.vine_log_dir
-        logger.debug(f"Function data directory: {self._function_data_dir}, log directory: {log_dir}")
+        logger.debug(f"Function data directory: {self._function_data_dir.name}, log directory: {log_dir}")
         logger.debug(
             f"TaskVine manager log directory: {self.manager_config.vine_log_dir}, "
             f"factory log directory: {self.factory_config.scratch_dir}")
@@ -294,7 +295,7 @@ class TaskVineExecutor(BlockProviderExecutor, putils.RepresentationMixin):
             'map': Pickled file with a dict between local parsl names, and remote taskvine names.
         """
         task_dir = "{:04d}".format(executor_task_id)
-        return os.path.join(self._function_data_dir, task_dir, *path_components)
+        return os.path.join(self._function_data_dir.name, task_dir, *path_components)
 
     def submit(self, func, resource_specification, *args, **kwargs):
         """Processes the Parsl app by its arguments and submits the function
@@ -564,13 +565,6 @@ class TaskVineExecutor(BlockProviderExecutor, putils.RepresentationMixin):
         self._worker_command = self._construct_worker_command()
         self._patch_providers()
 
-        if hasattr(self.provider, 'init_blocks'):
-            try:
-                self.scale_out(blocks=self.provider.init_blocks)
-            except Exception as e:
-                logger.error("Initial block scaling out failed: {}".format(e))
-                raise e
-
     @property
     def outstanding(self) -> int:
         """Count the number of outstanding tasks."""
@@ -581,19 +575,6 @@ class TaskVineExecutor(BlockProviderExecutor, putils.RepresentationMixin):
     def workers_per_node(self) -> Union[int, float]:
         return 1
 
-    def scale_in(self, count):
-        """Scale in method. Cancel a given number of blocks
-        """
-        # Obtain list of blocks to kill
-        to_kill = list(self.blocks.keys())[:count]
-        kill_ids = [self.blocks[block] for block in to_kill]
-
-        # Cancel the blocks provisioned
-        if self.provider:
-            self.provider.cancel(kill_ids)
-        else:
-            logger.error("No execution provider available to scale")
-
     def shutdown(self, *args, **kwargs):
         """Shutdown the executor. Sets flag to cancel the submit process and
         collector thread, which shuts down the TaskVine system submission.
@@ -601,20 +582,22 @@ class TaskVineExecutor(BlockProviderExecutor, putils.RepresentationMixin):
         logger.debug("TaskVine shutdown started")
         self._should_stop.set()
 
-        # Remove the workers that are still going
-        kill_ids = [self.blocks[block] for block in self.blocks.keys()]
-        if self.provider:
-            logger.debug("Cancelling blocks")
-            self.provider.cancel(kill_ids)
-
         # Join all processes before exiting
         logger.debug("Joining on submit process")
         self._submit_process.join()
+        self._submit_process.close()
         logger.debug("Joining on collector thread")
         self._collector_thread.join()
         if self.worker_launch_method == 'factory':
             logger.debug("Joining on factory process")
             self._factory_process.join()
+            self._factory_process.close()
+
+        # Shutdown multiprocessing queues
+        self._ready_task_queue.close()
+        self._ready_task_queue.join_thread()
+        self._finished_task_queue.close()
+        self._finished_task_queue.join_thread()
 
         logger.debug("TaskVine shutdown completed")
 

parsl/executors/taskvine/factory.py

@@ -1,7 +1,7 @@
 import logging
 
-from parsl.process_loggers import wrap_with_logs
 from parsl.executors.taskvine.errors import TaskVineFactoryFailure
+from parsl.process_loggers import wrap_with_logs
 
 # This try except clause prevents import errors
 # when TaskVine is not used in Parsl.

parsl/executors/taskvine/factory_config.py

@@ -32,7 +32,7 @@ class TaskVineFactoryConfig:
 
     worker_options: Optional[str]
         Additional options to pass to workers. Run
-        `vine_worker --help` for more details.
+        ``vine_worker --help`` for more details.
         Default is None.
 
     worker_executable: str

parsl/executors/taskvine/manager.py

@@ -1,25 +1,23 @@
-import logging
 import hashlib
-import subprocess
+import logging
 import os
 import queue
 import shutil
+import subprocess
 import uuid
 
-from parsl.utils import setproctitle
-from parsl.process_loggers import wrap_with_logs
 from parsl.executors.taskvine import exec_parsl_function
-from parsl.executors.taskvine.utils import VineTaskToParsl
-from parsl.executors.taskvine.utils import run_parsl_function
+from parsl.executors.taskvine.utils import VineTaskToParsl, run_parsl_function
+from parsl.process_loggers import wrap_with_logs
+from parsl.utils import setproctitle
 
 try:
-    from ndcctools.taskvine import cvine
-    from ndcctools.taskvine import Manager
-    from ndcctools.taskvine import Task
-    from ndcctools.taskvine import FunctionCall
-    from ndcctools.taskvine.cvine import VINE_ALLOCATION_MODE_MAX_THROUGHPUT
-    from ndcctools.taskvine.cvine import VINE_ALLOCATION_MODE_EXHAUSTIVE_BUCKETING
-    from ndcctools.taskvine.cvine import VINE_ALLOCATION_MODE_MAX
+    from ndcctools.taskvine import FunctionCall, Manager, Task, cvine
+    from ndcctools.taskvine.cvine import (
+        VINE_ALLOCATION_MODE_EXHAUSTIVE_BUCKETING,
+        VINE_ALLOCATION_MODE_MAX,
+        VINE_ALLOCATION_MODE_MAX_THROUGHPUT,
+    )
 except ImportError:
     _taskvine_enabled = False
 else:
@@ -46,11 +44,17 @@ def _set_manager_attributes(m, config):
     # Enable peer transfer feature between workers if specified
     if config.enable_peer_transfers:
         m.enable_peer_transfers()
+    else:
+        m.disable_peer_transfers()
 
     # Set catalog report to parsl if project name exists
     if m.name:
         m.set_property("framework", "parsl")
 
+    if config.tune_parameters is not None:
+        for k, v in config.tune_parameters.items():
+            m.tune(k, v)
+
 
 def _prepare_environment_serverless(manager_config, env_cache_dir, poncho_create_script):
     # Return path to a packaged poncho environment

parsl/executors/taskvine/manager_config.py

@@ -24,7 +24,7 @@ class TaskVineManagerConfig:
 
     address: Optional[str]
         Address of the local machine.
-        If None, socket.gethostname() will be used to determine the address.
+        If None, :py:func:`parsl.addresses.get_any_address` will be used to determine the address.
 
     project_name: Optional[str]
         If given, TaskVine will periodically report its status and performance
@@ -56,7 +56,7 @@ class TaskVineManagerConfig:
         environment name is given, TaskVine will package the conda
         environment in a tarball and send it along with tasks to be
         executed in a replicated conda environment.
-        If a tarball of packages (*.tar.gz) is given, TaskVine
+        If a tarball of packages (``*.tar.gz``) is given, TaskVine
         skips the packaging step and sends the tarball along with
         tasks to be executed in a replicated conda environment.
 
@@ -156,6 +156,10 @@ class TaskVineManagerConfig:
         Directory to store TaskVine logging facilities.
         Default is None, in which all TaskVine logs will be contained
         in the Parsl logging directory.
+
+    tune_parameters: Optional[dict]
+        Extended vine_tune parameters, expressed in a dictionary
+        by { 'tune-parameter' : value }.
     """
 
     # Connection and communication settings
@@ -181,6 +185,7 @@ class TaskVineManagerConfig:
     autocategory: bool = True
     enable_peer_transfers: bool = True
     wait_for_workers: Optional[int] = None
+    tune_parameters: Optional[dict] = None
 
     # Logging settings
     vine_log_dir: Optional[str] = None

parsl/executors/threads.py

@@ -1,14 +1,13 @@
-import logging
-import typeguard
 import concurrent.futures as cf
-
+import logging
 from typing import List, Optional
 
+import typeguard
+
 from parsl.data_provider.staging import Staging
 from parsl.executors.base import ParslExecutor
+from parsl.executors.errors import InvalidResourceSpecification
 from parsl.utils import RepresentationMixin
-from parsl.executors.errors import UnsupportedFeatureError
-
 
 logger = logging.getLogger(__name__)
 
@@ -55,7 +54,8 @@ class ThreadPoolExecutor(ParslExecutor, RepresentationMixin):
         if resource_specification:
             logger.error("Ignoring the resource specification. "
                          "Parsl resource specification is not supported in ThreadPool Executor.")
-            raise UnsupportedFeatureError('resource specification', 'ThreadPool Executor', None)
+            raise InvalidResourceSpecification(set(resource_specification.keys()),
+                                               "Parsl resource specification is not supported in ThreadPool Executor.")
 
         return self.executor.submit(func, *args, **kwargs)
 

parsl/executors/workqueue/errors.py

@@ -1,7 +1,7 @@
 from typing import Optional
 
-from parsl.errors import ParslError
 from parsl.app.errors import AppException
+from parsl.errors import ParslError
 
 
 class WorkQueueTaskFailure(AppException):

parsl/executors/workqueue/exec_parsl_function.py

@@ -1,10 +1,11 @@
+import pickle
+import sys
+import traceback
+
 from parsl.app.errors import RemoteExceptionWrapper
 from parsl.data_provider.files import File
-from parsl.utils import get_std_fname_mode
-import traceback
-import sys
-import pickle
 from parsl.serialize import serialize
+from parsl.utils import get_std_fname_mode
 
 # This scripts executes a parsl function which is pickled in a file:
 #
@@ -93,7 +94,7 @@ def unpack_source_code_function(function_info, user_namespace):
 
 def unpack_byte_code_function(function_info, user_namespace):
     from parsl.serialize import unpack_apply_message
-    func, args, kwargs = unpack_apply_message(function_info["byte code"], user_namespace, copy=False)
+    func, args, kwargs = unpack_apply_message(function_info["byte code"])
     return (func, 'parsl_function_name', args, kwargs)