parsl 2024.2.12__py3-none-any.whl → 2024.2.26__py3-none-any.whl

parsl/executors/high_throughput/process_worker_pool.py

@@ -10,7 +10,7 @@ import pickle
 import time
 import queue
 import uuid
-from typing import Sequence, Optional
+from typing import Sequence, Optional, Dict, List
 
 import zmq
 import math
@@ -27,7 +27,13 @@ from parsl.app.errors import RemoteExceptionWrapper
 from parsl.executors.high_throughput.errors import WorkerLost
 from parsl.executors.high_throughput.probe import probe_addresses
 from parsl.multiprocessing import SpawnContext
-from parsl.serialize import unpack_apply_message, serialize
+from parsl.serialize import unpack_res_spec_apply_message, serialize
+from parsl.executors.high_throughput.mpi_resource_management import (
+    TaskScheduler,
+    MPITaskScheduler
+)
+
+from parsl.executors.high_throughput.mpi_prefix_composer import compose_all, VALID_LAUNCHERS
 
 HEARTBEAT_CODE = (2 ** 32) - 1
 
@@ -64,6 +70,8 @@ class Manager:
                  heartbeat_period,
                  poll_period,
                  cpu_affinity,
+                 enable_mpi_mode: bool = False,
+                 mpi_launcher: str = "mpiexec",
                  available_accelerators: Sequence[str],
                  cert_dir: Optional[str]):
         """
@@ -120,6 +128,14 @@ class Manager:
         available_accelerators: list of str
             List of accelerators available to the workers.
 
+        enable_mpi_mode: bool
+            When set to true, the manager assumes ownership of the batch job and each worker
+            claims a subset of nodes from a shared pool to execute multi-node mpi tasks. Node
+            info is made available to workers via env vars.
+
+        mpi_launcher: str
+            Set to one of the supported MPI launchers: ("srun", "aprun", "mpiexec")
+
         cert_dir : str | None
             Path to the certificate directory.
         """
@@ -159,6 +175,9 @@ class Manager:
         self.uid = uid
         self.block_id = block_id
 
+        self.enable_mpi_mode = enable_mpi_mode
+        self.mpi_launcher = mpi_launcher
+
         if os.environ.get('PARSL_CORES'):
             cores_on_node = int(os.environ['PARSL_CORES'])
         else:
@@ -186,6 +205,17 @@ class Manager:
         self.monitoring_queue = self._mp_manager.Queue()
         self.pending_task_queue = SpawnContext.Queue()
         self.pending_result_queue = SpawnContext.Queue()
+        self.task_scheduler: TaskScheduler
+        if self.enable_mpi_mode:
+            self.task_scheduler = MPITaskScheduler(
+                self.pending_task_queue,
+                self.pending_result_queue,
+            )
+        else:
+            self.task_scheduler = TaskScheduler(
+                self.pending_task_queue,
+                self.pending_result_queue
+            )
         self.ready_worker_count = SpawnContext.Value("i", 0)
 
         self.max_queue_size = self.prefetch_capacity + self.worker_count
@@ -286,9 +316,7 @@ class Manager:
                     logger.debug("Got executor tasks: {}, cumulative count of tasks: {}".format([t['task_id'] for t in tasks], task_recv_counter))
 
                     for task in tasks:
-                        self.pending_task_queue.put(task)
-                        # logger.debug("Ready tasks: {}".format(
-                        #    [i['task_id'] for i in self.pending_task_queue]))
+                        self.task_scheduler.put_task(task)
 
             else:
                 logger.debug("No incoming tasks")
@@ -327,7 +355,7 @@ class Manager:
         while not kill_event.is_set():
             try:
                 logger.debug("Starting pending_result_queue get")
-                r = self.pending_result_queue.get(block=True, timeout=push_poll_period)
+                r = self.task_scheduler.get_result(block=True, timeout=push_poll_period)
                 logger.debug("Got a result item")
                 items.append(r)
             except queue.Empty:
@@ -497,6 +525,7 @@ class Manager:
                 os.getpid(),
                 args.logdir,
                 args.debug,
+                self.mpi_launcher,
             ),
             name="HTEX-Worker-{}".format(worker_id),
         )
@@ -504,7 +533,13 @@ class Manager:
         return p
 
 
-def execute_task(bufs):
+def update_resource_spec_env_vars(mpi_launcher: str, resource_spec: Dict, node_info: List[str]) -> None:
+    prefix_table = compose_all(mpi_launcher, resource_spec=resource_spec, node_hostnames=node_info)
+    for key in prefix_table:
+        os.environ[key] = prefix_table[key]
+
+
+def execute_task(bufs, mpi_launcher: Optional[str] = None):
     """Deserialize the buffer and execute the task.
 
     Returns the result or throws exception.
@@ -512,8 +547,20 @@ def execute_task(bufs):
     user_ns = locals()
     user_ns.update({'__builtins__': __builtins__})
 
-    f, args, kwargs = unpack_apply_message(bufs, user_ns, copy=False)
+    f, args, kwargs, resource_spec = unpack_res_spec_apply_message(bufs, user_ns, copy=False)
+
+    for varname in resource_spec:
+        envname = "PARSL_" + str(varname).upper()
+        os.environ[envname] = str(resource_spec[varname])
 
+    if resource_spec.get("MPI_NODELIST"):
+        worker_id = os.environ['PARSL_WORKER_RANK']
+        nodes_for_task = resource_spec["MPI_NODELIST"].split(',')
+        logger.info(f"Launching task on provisioned nodes: {nodes_for_task}")
+        assert mpi_launcher
+        update_resource_spec_env_vars(mpi_launcher,
+                                      resource_spec=resource_spec,
+                                      node_info=nodes_for_task)
     # We might need to look into callability of the function from itself
     # since we change it's name in the new namespace
     prefix = "parsl_"
@@ -550,6 +597,7 @@ def worker(
     manager_pid: int,
     logdir: str,
     debug: bool,
+    mpi_launcher: str,
 ):
     """
 
@@ -668,7 +716,7 @@ def worker(
         worker_enqueued = False
 
         try:
-            result = execute_task(req['buffer'])
+            result = execute_task(req['buffer'], mpi_launcher=mpi_launcher)
             serialized_result = serialize(result, buffer_threshold=1000000)
         except Exception as e:
             logger.info('Caught an exception: {}'.format(e))
@@ -768,6 +816,10 @@ if __name__ == "__main__":
                         help="Whether/how workers should control CPU affinity.")
     parser.add_argument("--available-accelerators", type=str, nargs="*",
                         help="Names of available accelerators")
+    parser.add_argument("--enable_mpi_mode", action='store_true',
+                        help="Enable MPI mode")
+    parser.add_argument("--mpi-launcher", type=str, choices=VALID_LAUNCHERS,
+                        help="MPI launcher to use iff enable_mpi_mode=true")
 
     args = parser.parse_args()
 
@@ -797,6 +849,8 @@ if __name__ == "__main__":
         logger.info("Heartbeat period: {}".format(args.hb_period))
         logger.info("CPU affinity: {}".format(args.cpu_affinity))
         logger.info("Accelerators: {}".format(" ".join(args.available_accelerators)))
+        logger.info("enable_mpi_mode: {}".format(args.enable_mpi_mode))
+        logger.info("mpi_launcher: {}".format(args.mpi_launcher))
 
         manager = Manager(task_port=args.task_port,
                           result_port=args.result_port,
@@ -812,6 +866,8 @@ if __name__ == "__main__":
                           heartbeat_period=int(args.hb_period),
                           poll_period=int(args.poll),
                           cpu_affinity=args.cpu_affinity,
+                          enable_mpi_mode=args.enable_mpi_mode,
+                          mpi_launcher=args.mpi_launcher,
                           available_accelerators=args.available_accelerators,
                           cert_dir=None if args.cert_dir == "None" else args.cert_dir)
         manager.start()

parsl/executors/radical/executor.py

@@ -23,7 +23,7 @@ from parsl.utils import RepresentationMixin
 from parsl.app.errors import BashExitFailure
 from parsl.executors.base import ParslExecutor
 from parsl.app.errors import RemoteExceptionWrapper
-from parsl.serialize import pack_apply_message, deserialize
+from parsl.serialize import deserialize, pack_res_spec_apply_message
 from parsl.serialize.errors import SerializationError, DeserializationError
 
 try:
@@ -400,8 +400,11 @@ class RadicalPilotExecutor(ParslExecutor, RepresentationMixin):
 
     def _pack_and_apply_message(self, func, args, kwargs):
         try:
-            buffer = pack_apply_message(func, args, kwargs,
-                                        buffer_threshold=1024 * 1024)
+            buffer = pack_res_spec_apply_message(func,
+                                                 args,
+                                                 kwargs,
+                                                 resource_specification={},
+                                                 buffer_threshold=1024 * 1024)
             task_func = rp.utils.serialize_bson(buffer)
         except TypeError:
             raise SerializationError(func.__name__)

parsl/executors/radical/rpex_worker.py

@@ -3,7 +3,7 @@ import radical.pilot as rp
 
 import parsl.app.errors as pe
 from parsl.app.bash import remote_side_bash_executor
-from parsl.serialize import unpack_apply_message, serialize
+from parsl.serialize import unpack_res_spec_apply_message, serialize
 from parsl.executors.high_throughput.process_worker_pool import execute_task
 
 
@@ -32,7 +32,7 @@ class ParslWorker:
 
         try:
             buffer = rp.utils.deserialize_bson(task['description']['executable'])
-            func, args, kwargs = unpack_apply_message(buffer, {}, copy=False)
+            func, args, kwargs, _resource_spec = unpack_res_spec_apply_message(buffer, {}, copy=False)
             ret = remote_side_bash_executor(func, *args, **kwargs)
             exc = (None, None)
             val = None

parsl/jobs/states.py

@@ -47,7 +47,7 @@ class JobState(IntEnum):
     """
 
     def __str__(self) -> str:
-        return self.__class__.__name__ + "." + self.name
+        return f"{self.__class__.__name__}.{self.name}"
 
 
 TERMINAL_STATES = [JobState.CANCELLED, JobState.COMPLETED, JobState.FAILED,
@@ -84,16 +84,16 @@ class JobStatus:
 
     def __repr__(self) -> str:
         if self.message is not None:
-            extra = f"state={self.state} message={self.message}".format(self.state, self.message)
+            extra = f"state={self.state} message={self.message}"
         else:
-            extra = f"state={self.state}".format(self.state)
+            extra = f"state={self.state}"
         return f"<{type(self).__module__}.{type(self).__qualname__} object at {hex(id(self))}, {extra}>"
 
     def __str__(self) -> str:
         if self.message is not None:
-            return "{} ({})".format(self.state, self.message)
+            return f"{self.state} ({self.message})"
         else:
-            return "{}".format(self.state)
+            return f"{self.state}"
 
     @property
     def stdout(self) -> Optional[str]:

parsl/monitoring/db_manager.py

@@ -444,7 +444,8 @@ class DatabaseManager:
                                               'run_id', 'task_id',
                                               'task_fail_count',
                                               'task_fail_cost',
-                                              'task_hashsum'],
+                                              'task_hashsum',
+                                              'task_inputs'],
                                      messages=task_info_update_messages)
                     logger.debug("Inserting {} task_info_all_messages into status table".format(len(task_info_all_messages)))
 

parsl/monitoring/monitoring.py

@@ -84,7 +84,7 @@ class MonitoringHub(RepresentationMixin):
 
                  workflow_name: Optional[str] = None,
                  workflow_version: Optional[str] = None,
-                 logging_endpoint: str = 'sqlite:///runinfo/monitoring.db',
+                 logging_endpoint: Optional[str] = None,
                  logdir: Optional[str] = None,
                  monitoring_debug: bool = False,
                  resource_monitoring_enabled: bool = True,
@@ -118,7 +118,7 @@ class MonitoringHub(RepresentationMixin):
         logging_endpoint : str
              The database connection url for monitoring to log the information.
              These URLs follow RFC-1738, and can include username, password, hostname, database name.
-             Default: 'sqlite:///monitoring.db'
+             Default: sqlite, in the configured run_dir.
         logdir : str
              Parsl log directory paths. Logs and temp files go here. Default: '.'
         monitoring_debug : Bool
@@ -162,11 +162,14 @@ class MonitoringHub(RepresentationMixin):
         self.resource_monitoring_enabled = resource_monitoring_enabled
         self.resource_monitoring_interval = resource_monitoring_interval
 
-    def start(self, run_id: str, run_dir: str) -> int:
+    def start(self, run_id: str, dfk_run_dir: str, config_run_dir: Union[str, os.PathLike]) -> int:
 
         if self.logdir is None:
             self.logdir = "."
 
+        if self.logging_endpoint is None:
+            self.logging_endpoint = f"sqlite:///{os.fspath(config_run_dir)}/monitoring.db"
+
         os.makedirs(self.logdir, exist_ok=True)
 
         # Initialize the ZMQ pipe to the Parsl Client
@@ -231,7 +234,7 @@ class MonitoringHub(RepresentationMixin):
         self.logger.info("Started the router process {} and DBM process {}".format(self.router_proc.pid, self.dbm_proc.pid))
 
         self.filesystem_proc = Process(target=filesystem_receiver,
-                                       args=(self.logdir, self.resource_msgs, run_dir),
+                                       args=(self.logdir, self.resource_msgs, dfk_run_dir),
                                        name="Monitoring-Filesystem-Process",
                                        daemon=True
                                        )

parsl/multiprocessing.py

@@ -5,17 +5,16 @@ import logging
 import multiprocessing
 import multiprocessing.queues
 import platform
+from multiprocessing.context import ForkProcess as ForkProcessType
 
-from typing import Callable, Type
+from typing import Callable
 
 logger = logging.getLogger(__name__)
 
 ForkContext = multiprocessing.get_context("fork")
 SpawnContext = multiprocessing.get_context("spawn")
 
-# maybe ForkProcess should be: Callable[..., Process] so as to make
-# it clear that it returns a Process always to the type checker?
-ForkProcess: Type = ForkContext.Process
+ForkProcess: Callable[..., ForkProcessType] = ForkContext.Process
 
 
 class MacSafeQueue(multiprocessing.queues.Queue):

parsl/providers/cobalt/cobalt.py

@@ -1,6 +1,7 @@
 import logging
 import os
 import time
+import warnings
 
 from parsl.providers.errors import ScaleOutFailed
 from parsl.channels import LocalChannel
@@ -24,6 +25,8 @@ translate_table = {
 class CobaltProvider(ClusterProvider, RepresentationMixin):
     """ Cobalt Execution Provider
 
+    WARNING: CobaltProvider is deprecated and will be removed by 2024.04
+
     This provider uses cobalt to submit (qsub), obtain the status of (qstat), and cancel (qdel)
     jobs. Theo script to be used is created from a template file in this
     same module.
@@ -86,6 +89,9 @@ class CobaltProvider(ClusterProvider, RepresentationMixin):
         self.queue = queue
         self.scheduler_options = scheduler_options
         self.worker_init = worker_init
+        warnings.warn("CobaltProvider is deprecated; This will be removed after 2024-04",
+                      DeprecationWarning,
+                      stacklevel=2)
 
     def _status(self):
         """Returns the status list for a list of job_ids

parsl/providers/pbspro/pbspro.py

@@ -119,13 +119,17 @@ class PBSProProvider(TorqueProvider):
 
                 job_state = job.get('job_state', JobState.UNKNOWN)
                 state = translate_table.get(job_state, JobState.UNKNOWN)
-                self.resources[job_id]['status'] = JobStatus(state)
+                self.resources[job_id]['status'] = JobStatus(state,
+                                                             stdout_path=self.resources[job_id]['job_stdout_path'],
+                                                             stderr_path=self.resources[job_id]['job_stderr_path'])
                 jobs_missing.remove(job_id)
 
         # squeue does not report on jobs that are not running. So we are filling in the
         # blanks for missing jobs, we might lose some information about why the jobs failed.
         for missing_job in jobs_missing:
-            self.resources[missing_job]['status'] = JobStatus(JobState.COMPLETED)
+            self.resources[missing_job]['status'] = JobStatus(JobState.COMPLETED,
+                                                              stdout_path=self.resources[missing_job]['job_stdout_path'],
+                                                              stderr_path=self.resources[missing_job]['job_stderr_path'])
 
     def submit(self, command, tasks_per_node, job_name="parsl"):
         """Submits the command job.
@@ -149,7 +153,11 @@ class PBSProProvider(TorqueProvider):
 
         job_name = "{0}.{1}".format(job_name, time.time())
 
-        script_path = os.path.abspath("{0}/{1}.submit".format(self.script_dir, job_name))
+        assert self.script_dir, "Expected script_dir to be set"
+        script_path = os.path.join(self.script_dir, job_name)
+        script_path = os.path.abspath(script_path)
+        job_stdout_path = script_path + ".stdout"
+        job_stderr_path = script_path + ".stderr"
 
         logger.debug("Requesting {} nodes_per_block, {} tasks_per_node".format(
             self.nodes_per_block, tasks_per_node)
@@ -163,6 +171,8 @@ class PBSProProvider(TorqueProvider):
         job_config["scheduler_options"] = self.scheduler_options
         job_config["worker_init"] = self.worker_init
         job_config["user_script"] = command
+        job_config["job_stdout_path"] = job_stdout_path
+        job_config["job_stderr_path"] = job_stderr_path
 
         # Add a colon to select_options if one isn't included
         if self.select_options and not self.select_options.startswith(":"):
@@ -194,7 +204,11 @@ class PBSProProvider(TorqueProvider):
             for line in stdout.split('\n'):
                 if line.strip():
                     job_id = line.strip()
-                    self.resources[job_id] = {'job_id': job_id, 'status': JobStatus(JobState.PENDING)}
+                    self.resources[job_id] = {'job_id': job_id,
+                                              'status': JobStatus(JobState.PENDING),
+                                              'job_stdout_path': job_stdout_path,
+                                              'job_stderr_path': job_stderr_path,
+                                              }
         else:
             message = "Command '{}' failed with return code {}".format(launch_cmd, retcode)
             if (stdout is not None) and (stderr is not None):

parsl/providers/pbspro/template.py

@@ -5,8 +5,8 @@ template_string = '''#!/bin/bash
 #PBS -m n
 #PBS -l walltime=$walltime
 #PBS -l select=${nodes_per_block}:ncpus=${ncpus}${select_options}
-#PBS -o ${submit_script_dir}/${jobname}.submit.stdout
-#PBS -e ${submit_script_dir}/${jobname}.submit.stderr
+#PBS -o ${job_stdout_path}
+#PBS -e ${job_stderr_path}
 ${scheduler_options}
 
 ${worker_init}

parsl/providers/slurm/slurm.py

@@ -188,14 +188,18 @@ class SlurmProvider(ClusterProvider, RepresentationMixin):
                 logger.warning(f"Slurm status {slurm_state} is not recognized")
             status = translate_table.get(slurm_state, JobState.UNKNOWN)
             logger.debug("Updating job {} with slurm status {} to parsl state {!s}".format(job_id, slurm_state, status))
-            self.resources[job_id]['status'] = JobStatus(status)
+            self.resources[job_id]['status'] = JobStatus(status,
+                                                         stdout_path=self.resources[job_id]['job_stdout_path'],
+                                                         stderr_path=self.resources[job_id]['job_stderr_path'])
             jobs_missing.remove(job_id)
 
         # squeue does not report on jobs that are not running. So we are filling in the
         # blanks for missing jobs, we might lose some information about why the jobs failed.
         for missing_job in jobs_missing:
             logger.debug("Updating missing job {} to completed status".format(missing_job))
-            self.resources[missing_job]['status'] = JobStatus(JobState.COMPLETED)
+            self.resources[missing_job]['status'] = JobStatus(JobState.COMPLETED,
+                                                              stdout_path=self.resources[missing_job]['job_stdout_path'],
+                                                              stderr_path=self.resources[missing_job]['job_stderr_path'])
 
     def submit(self, command: str, tasks_per_node: int, job_name="parsl.slurm") -> str:
         """Submit the command as a slurm job.
@@ -226,8 +230,11 @@ class SlurmProvider(ClusterProvider, RepresentationMixin):
 
         job_name = "{0}.{1}".format(job_name, time.time())
 
-        script_path = "{0}/{1}.submit".format(self.script_dir, job_name)
+        assert self.script_dir, "Expected script_dir to be set"
+        script_path = os.path.join(self.script_dir, job_name)
         script_path = os.path.abspath(script_path)
+        job_stdout_path = script_path + ".stdout"
+        job_stderr_path = script_path + ".stderr"
 
         logger.debug("Requesting one block with {} nodes".format(self.nodes_per_block))
 
@@ -239,6 +246,8 @@ class SlurmProvider(ClusterProvider, RepresentationMixin):
         job_config["scheduler_options"] = scheduler_options
         job_config["worker_init"] = worker_init
         job_config["user_script"] = command
+        job_config["job_stdout_path"] = job_stdout_path
+        job_config["job_stderr_path"] = job_stderr_path
 
         # Wrap the command
         job_config["user_script"] = self.launcher(command,
@@ -262,7 +271,11 @@ class SlurmProvider(ClusterProvider, RepresentationMixin):
                 match = re.match(self.regex_job_id, line)
                 if match:
                     job_id = match.group("id")
-                    self.resources[job_id] = {'job_id': job_id, 'status': JobStatus(JobState.PENDING)}
+                    self.resources[job_id] = {'job_id': job_id,
+                                              'status': JobStatus(JobState.PENDING),
+                                              'job_stdout_path': job_stdout_path,
+                                              'job_stderr_path': job_stderr_path,
+                                              }
                     return job_id
             else:
                 logger.error("Could not read job ID from submit command standard output.")

parsl/providers/slurm/template.py

@@ -1,8 +1,8 @@
 template_string = '''#!/bin/bash
 
 #SBATCH --job-name=${jobname}
-#SBATCH --output=${submit_script_dir}/${jobname}.submit.stdout
-#SBATCH --error=${submit_script_dir}/${jobname}.submit.stderr
+#SBATCH --output=${job_stdout_path}
+#SBATCH --error=${job_stderr_path}
 #SBATCH --nodes=${nodes}
 #SBATCH --time=${walltime}
 #SBATCH --ntasks-per-node=${tasks_per_node}

parsl/serialize/__init__.py

@@ -1,6 +1,11 @@
-from parsl.serialize.facade import serialize, deserialize, pack_apply_message, unpack_apply_message
+from parsl.serialize.facade import (serialize, deserialize, pack_apply_message,
+                                    unpack_apply_message, unpack_res_spec_apply_message,
+                                    pack_res_spec_apply_message)
 
 __all__ = ['serialize',
            'deserialize',
            'pack_apply_message',
-           'unpack_apply_message']
+           'unpack_apply_message',
+           'unpack_res_spec_apply_message',
+           'pack_res_spec_apply_message'
+           ]

parsl/serialize/facade.py

@@ -62,13 +62,44 @@ def pack_apply_message(func: Any, args: Any, kwargs: Any, buffer_threshold: int
     return packed_buffer
 
 
+def pack_res_spec_apply_message(func: Any, args: Any, kwargs: Any, resource_specification: Any, buffer_threshold: int = int(128 * 1e6)) -> bytes:
+    """Serialize and pack function, parameters, and resource_specification
+
+    Parameters
+    ----------
+
+    func: Function
+        A function to ship
+
+    args: Tuple/list of objects
+        positional parameters as a list
+
+    kwargs: Dict
+        Dict containing named parameters
+
+    resource_specification: Dict
+        Dict containing application resource specification
+
+    buffer_threshold: int
+        Limits buffer to specified size in bytes. Exceeding this limit would give you
+        a warning in the log. Default is 128MB.
+    """
+    return pack_apply_message(func, args, (kwargs, resource_specification), buffer_threshold=buffer_threshold)
+
+
 def unpack_apply_message(packed_buffer: bytes, user_ns: Any = None, copy: Any = False) -> List[Any]:
     """ Unpack and deserialize function and parameters
-
     """
     return [deserialize(buf) for buf in unpack_buffers(packed_buffer)]
 
 
+def unpack_res_spec_apply_message(packed_buffer: bytes, user_ns: Any = None, copy: Any = False) -> List[Any]:
+    """ Unpack and deserialize function, parameters, and resource_specification
+    """
+    func, args, (kwargs, resource_spec) = unpack_apply_message(packed_buffer, user_ns=user_ns, copy=copy)
+    return [func, args, kwargs, resource_spec]
+
+
 def serialize(obj: Any, buffer_threshold: int = int(1e6)) -> bytes:
     """ Try available serialization methods one at a time
 

parsl/tests/test_error_handling/test_resource_spec.py

@@ -2,6 +2,8 @@ import parsl
 from parsl.app.app import python_app
 from parsl.executors.errors import UnsupportedFeatureError, ExecutorError
 from parsl.executors import WorkQueueExecutor
+from parsl.executors.high_throughput.mpi_prefix_composer import InvalidResourceSpecification
+from parsl.executors.high_throughput.executor import HighThroughputExecutor
 
 
 @python_app
@@ -22,6 +24,8 @@ def test_resource(n=2):
     fut = double(n, parsl_resource_specification=spec)
     try:
         fut.result()
+    except InvalidResourceSpecification:
+        assert isinstance(executor, HighThroughputExecutor)
     except UnsupportedFeatureError:
         assert not isinstance(executor, WorkQueueExecutor)
     except Exception as e:
@@ -33,6 +37,8 @@ def test_resource(n=2):
     fut = double(n, parsl_resource_specification=spec)
     try:
         fut.result()
+    except InvalidResourceSpecification:
+        assert isinstance(executor, HighThroughputExecutor)
     except UnsupportedFeatureError:
         assert not isinstance(executor, WorkQueueExecutor)
     except Exception as e: