parsl 2024.3.25__py3-none-any.whl → 2024.4.8__py3-none-any.whl

parsl/dataflow/dflow.py

@@ -34,12 +34,12 @@ from parsl.dataflow.states import States, FINAL_STATES, FINAL_FAILURE_STATES
 from parsl.dataflow.taskrecord import TaskRecord
 from parsl.errors import ConfigurationError, InternalConsistencyError, NoDataFlowKernelError
 from parsl.jobs.job_status_poller import JobStatusPoller
-from parsl.jobs.states import JobStatus, JobState
 from parsl.usage_tracking.usage import UsageTracker
 from parsl.executors.base import ParslExecutor
 from parsl.executors.status_handling import BlockProviderExecutor
 from parsl.executors.threads import ThreadPoolExecutor
 from parsl.monitoring import MonitoringHub
+from parsl.monitoring.remote import monitor_wrapper
 from parsl.process_loggers import wrap_with_logs
 from parsl.providers.base import ExecutionProvider
 from parsl.utils import get_version, get_std_fname_mode, get_all_checkpoints, Timer
@@ -207,7 +207,7 @@ class DataFlowKernel:
         atexit.register(self.atexit_cleanup)
 
     def __enter__(self):
-        pass
+        return self
 
     def __exit__(self, exc_type, exc_value, traceback):
         logger.debug("Exiting the context manager, calling cleanup for DFK")
@@ -714,14 +714,18 @@ class DataFlowKernel:
 
         if self.monitoring is not None and self.monitoring.resource_monitoring_enabled:
             wrapper_logging_level = logging.DEBUG if self.monitoring.monitoring_debug else logging.INFO
-            (function, args, kwargs) = self.monitoring.monitor_wrapper(function, args, kwargs, try_id, task_id,
-                                                                       self.monitoring.monitoring_hub_url,
-                                                                       self.run_id,
-                                                                       wrapper_logging_level,
-                                                                       self.monitoring.resource_monitoring_interval,
-                                                                       executor.radio_mode,
-                                                                       executor.monitor_resources(),
-                                                                       self.run_dir)
+            (function, args, kwargs) = monitor_wrapper(f=function,
+                                                       args=args,
+                                                       kwargs=kwargs,
+                                                       x_try_id=try_id,
+                                                       x_task_id=task_id,
+                                                       monitoring_hub_url=self.monitoring.monitoring_hub_url,
+                                                       run_id=self.run_id,
+                                                       logging_level=wrapper_logging_level,
+                                                       sleep_dur=self.monitoring.resource_monitoring_interval,
+                                                       radio_mode=executor.radio_mode,
+                                                       monitor_resources=executor.monitor_resources(),
+                                                       run_dir=self.run_dir)
 
         with self.submitter_lock:
             exec_fu = executor.submit(function, task_record['resource_specification'], *args, **kwargs)
@@ -1141,14 +1145,7 @@ class DataFlowKernel:
                         self._create_remote_dirs_over_channel(executor.provider, executor.provider.channel)
 
             self.executors[executor.label] = executor
-            block_ids = executor.start()
-            if self.monitoring and block_ids:
-                new_status = {}
-                for bid in block_ids:
-                    new_status[bid] = JobStatus(JobState.PENDING)
-                msg = executor.create_monitoring_info(new_status)
-                logger.debug("Sending monitoring message {} to hub from DFK".format(msg))
-                self.monitoring.send(MessageType.BLOCK_INFO, msg)
+            executor.start()
         block_executors = [e for e in executors if isinstance(e, BlockProviderExecutor)]
         self.job_status_poller.add_executors(block_executors)
 
@@ -1221,24 +1218,9 @@ class DataFlowKernel:
         self.job_status_poller.close()
         logger.info("Terminated job status poller")
 
-        logger.info("Scaling in and shutting down executors")
+        logger.info("Shutting down executors")
 
         for executor in self.executors.values():
-            if isinstance(executor, BlockProviderExecutor):
-                if not executor.bad_state_is_set:
-                    logger.info(f"Scaling in executor {executor.label}")
-                    if executor.provider:
-                        job_ids = executor.provider.resources.keys()
-                        block_ids = executor.scale_in(len(job_ids))
-                        if self.monitoring and block_ids:
-                            new_status = {}
-                            for bid in block_ids:
-                                new_status[bid] = JobStatus(JobState.CANCELLED)
-                            msg = executor.create_monitoring_info(new_status)
-                            logger.debug("Sending message {} to hub from DFK".format(msg))
-                            self.monitoring.send(MessageType.BLOCK_INFO, msg)
-                else:  # and bad_state_is_set
-                    logger.warning(f"Not shutting down executor {executor.label} because it is in bad state")
             logger.info(f"Shutting down executor {executor.label}")
             executor.shutdown()
             logger.info(f"Shut down executor {executor.label}")

parsl/executors/base.py

@@ -53,7 +53,7 @@ class ParslExecutor(metaclass=ABCMeta):
         return False
 
     @abstractmethod
-    def start(self) -> Optional[List[str]]:
+    def start(self) -> None:
         """Start the executor.
 
         Any spin-up operations (for example: starting thread pools) should be performed here.

parsl/executors/high_throughput/executor.py

@@ -1,4 +1,5 @@
 import typing
+from collections import defaultdict
 from concurrent.futures import Future
 import typeguard
 import logging
@@ -400,16 +401,6 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin):
 
         logger.debug("Starting HighThroughputExecutor with provider:\n%s", self.provider)
 
-        # TODO: why is this a provider property?
-        block_ids = []
-        if hasattr(self.provider, 'init_blocks'):
-            try:
-                block_ids = self.scale_out(blocks=self.provider.init_blocks)
-            except Exception as e:
-                logger.error("Scaling out failed: {}".format(e))
-                raise e
-        return block_ids
-
     def start(self):
         """Create the Interchange process and connect to it.
         """
@@ -439,8 +430,7 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin):
 
         logger.debug("Created management thread: {}".format(self._queue_management_thread))
 
-        block_ids = self.initialize_scaling()
-        return block_ids
+        self.initialize_scaling()
 
     @wrap_with_logs
     def _queue_management_worker(self):
@@ -698,7 +688,7 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin):
             d['status'] = s.status_name
             d['timestamp'] = datetime.datetime.now()
             d['executor_label'] = self.label
-            d['job_id'] = self.blocks.get(bid, None)
+            d['job_id'] = self.blocks_to_job_id.get(bid, None)
             d['block_id'] = bid
             msg.append(d)
         return msg
@@ -741,13 +731,11 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin):
             idle: float  # shortest idle time of any manager in this block
 
         managers = self.connected_managers()
-        block_info: Dict[str, BlockInfo] = {}
+        block_info: Dict[str, BlockInfo] = defaultdict(lambda: BlockInfo(tasks=0, idle=float('inf')))
         for manager in managers:
             if not manager['active']:
                 continue
             b_id = manager['block_id']
-            if b_id not in block_info:
-                block_info[b_id] = BlockInfo(tasks=0, idle=float('inf'))
             block_info[b_id].tasks += manager['tasks']
             block_info[b_id].idle = min(block_info[b_id].idle, manager['idle_duration'])
 
@@ -779,14 +767,14 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin):
 
         # Now kill via provider
         # Potential issue with multiple threads trying to remove the same blocks
-        to_kill = [self.blocks[bid] for bid in block_ids_to_kill if bid in self.blocks]
+        to_kill = [self.blocks_to_job_id[bid] for bid in block_ids_to_kill if bid in self.blocks_to_job_id]
 
         r = self.provider.cancel(to_kill)
         job_ids = self._filter_scale_in_ids(to_kill, r)
 
-        # to_kill block_ids are fetched from self.blocks
-        # If a block_id is in self.block, it must exist in self.block_mapping
-        block_ids_killed = [self.block_mapping[jid] for jid in job_ids]
+        # to_kill block_ids are fetched from self.blocks_to_job_id
+        # If a block_id is in self.blocks_to_job_id, it must exist in self.job_ids_to_block
+        block_ids_killed = [self.job_ids_to_block[jid] for jid in job_ids]
 
         return block_ids_killed
 

parsl/executors/high_throughput/process_worker_pool.py

@@ -335,14 +335,17 @@ class Manager:
                 self.heartbeat_to_incoming()
                 last_beat = time.time()
 
-            if self.drain_time and time.time() > self.drain_time:
+            if time.time() > self.drain_time:
                 logger.info("Requesting drain")
                 self.drain_to_incoming()
-                self.drain_time = None
                 # This will start the pool draining...
                 # Drained exit behaviour does not happen here. It will be
                 # driven by the interchange sending a DRAINED_CODE message.
 
+                # now set drain time to the far future so we don't send a drain
+                # message every iteration.
+                self.drain_time = float('inf')
+
             poll_duration_s = max(0, next_interesting_event_time - time.time())
             socks = dict(poller.poll(timeout=poll_duration_s * 1000))
 

parsl/executors/status_handling.py

@@ -68,8 +68,8 @@ class BlockProviderExecutor(ParslExecutor):
         self._block_id_counter = AtomicIDCounter()
 
         self._tasks = {}  # type: Dict[object, Future]
-        self.blocks = {}  # type: Dict[str, str]
-        self.block_mapping = {}  # type: Dict[str, str]
+        self.blocks_to_job_id = {}  # type: Dict[str, str]
+        self.job_ids_to_block = {}  # type: Dict[str, str]
 
     def _make_status_dict(self, block_ids: List[str], status_list: List[JobStatus]) -> Dict[str, JobStatus]:
         """Given a list of block ids and a list of corresponding status strings,
@@ -102,12 +102,6 @@ class BlockProviderExecutor(ParslExecutor):
         else:
             return self._provider.status_polling_interval
 
-    def _fail_job_async(self, block_id: str, message: str):
-        """Marks a job that has failed to start but would not otherwise be included in status()
-        as failed and report it in status()
-        """
-        self._simulated_status[block_id] = JobStatus(JobState.FAILED, message)
-
     @abstractproperty
     def outstanding(self) -> int:
         """This should return the number of tasks that the executor has been given to run (waiting to run, and running now)"""
@@ -194,12 +188,11 @@ class BlockProviderExecutor(ParslExecutor):
             logger.info(f"Allocated block ID {block_id}")
             try:
                 job_id = self._launch_block(block_id)
-                self.blocks[block_id] = job_id
-                self.block_mapping[job_id] = block_id
+                self.blocks_to_job_id[block_id] = job_id
+                self.job_ids_to_block[job_id] = block_id
                 block_ids.append(block_id)
             except Exception as ex:
-                self._fail_job_async(block_id,
-                                     "Failed to start block {}: {}".format(block_id, ex))
+                self._simulated_status[block_id] = JobStatus(JobState.FAILED, "Failed to start block {}: {}".format(block_id, ex))
         return block_ids
 
     @abstractmethod
@@ -232,10 +225,10 @@ class BlockProviderExecutor(ParslExecutor):
         # Not using self.blocks.keys() and self.blocks.values() simultaneously
         # The dictionary may be changed during invoking this function
         # As scale_in and scale_out are invoked in multiple threads
-        block_ids = list(self.blocks.keys())
+        block_ids = list(self.blocks_to_job_id.keys())
         job_ids = []  # types: List[Any]
         for bid in block_ids:
-            job_ids.append(self.blocks[bid])
+            job_ids.append(self.blocks_to_job_id[bid])
         return block_ids, job_ids
 
     @abstractproperty

parsl/executors/taskvine/executor.py

@@ -186,7 +186,13 @@ class TaskVineExecutor(BlockProviderExecutor, putils.RepresentationMixin):
         # Attribute indicating whether this executor was started to shut it down properly.
         # This safeguards cases where an object of this executor is created but
         # the executor never starts, so it shouldn't be shutdowned.
-        self._started = False
+        self._is_started = False
+
+        # Attribute indicating whether this executor was shutdown before.
+        # This safeguards cases where this object is automatically shut down (e.g.,
+        # via atexit) and the user also explicitly calls shut down. While this is
+        # permitted, the effect of an executor shutdown should happen only once.
+        self._is_shutdown = False
 
     def atexit_cleanup(self):
         # Calls this executor's shutdown method upon Python exiting the process.
@@ -252,7 +258,7 @@ class TaskVineExecutor(BlockProviderExecutor, putils.RepresentationMixin):
         """
 
         # Mark this executor object as started
-        self._started = True
+        self._is_started = True
 
         # Synchronize connection and communication settings between the manager and factory
         self.__synchronize_manager_factory_comm_settings()
@@ -580,13 +586,6 @@ class TaskVineExecutor(BlockProviderExecutor, putils.RepresentationMixin):
         self._worker_command = self._construct_worker_command()
         self._patch_providers()
 
-        if hasattr(self.provider, 'init_blocks'):
-            try:
-                self.scale_out(blocks=self.provider.init_blocks)
-            except Exception as e:
-                logger.error("Initial block scaling out failed: {}".format(e))
-                raise e
-
     @property
     def outstanding(self) -> int:
         """Count the number of outstanding tasks."""
@@ -601,8 +600,8 @@ class TaskVineExecutor(BlockProviderExecutor, putils.RepresentationMixin):
         """Scale in method. Cancel a given number of blocks
         """
         # Obtain list of blocks to kill
-        to_kill = list(self.blocks.keys())[:count]
-        kill_ids = [self.blocks[block] for block in to_kill]
+        to_kill = list(self.blocks_to_job_id.keys())[:count]
+        kill_ids = [self.blocks_to_job_id[block] for block in to_kill]
 
         # Cancel the blocks provisioned
         if self.provider:
@@ -614,15 +613,19 @@ class TaskVineExecutor(BlockProviderExecutor, putils.RepresentationMixin):
         """Shutdown the executor. Sets flag to cancel the submit process and
         collector thread, which shuts down the TaskVine system submission.
         """
-        if not self._started:
+        if not self._is_started:
             # Don't shutdown if the executor never starts.
             return
 
+        if self._is_shutdown:
+            # Don't shutdown this executor again.
+            return
+
         logger.debug("TaskVine shutdown started")
         self._should_stop.set()
 
         # Remove the workers that are still going
-        kill_ids = [self.blocks[block] for block in self.blocks.keys()]
+        kill_ids = [self.blocks_to_job_id[block] for block in self.blocks_to_job_id.keys()]
         if self.provider:
             logger.debug("Cancelling blocks")
             self.provider.cancel(kill_ids)
@@ -636,6 +639,7 @@ class TaskVineExecutor(BlockProviderExecutor, putils.RepresentationMixin):
             logger.debug("Joining on factory process")
             self._factory_process.join()
 
+        self._is_shutdown = True
         logger.debug("TaskVine shutdown completed")
 
     @wrap_with_logs

parsl/executors/workqueue/executor.py

@@ -255,7 +255,6 @@ class WorkQueueExecutor(BlockProviderExecutor, putils.RepresentationMixin):
         self.label = label
         self.task_queue = multiprocessing.Queue()  # type: multiprocessing.Queue
         self.collector_queue = multiprocessing.Queue()  # type: multiprocessing.Queue
-        self.blocks = {}  # type: Dict[str, str]
         self.address = address
         self.port = port
         self.executor_task_counter = -1
@@ -305,7 +304,13 @@ class WorkQueueExecutor(BlockProviderExecutor, putils.RepresentationMixin):
         # Attribute indicating whether this executor was started to shut it down properly.
         # This safeguards cases where an object of this executor is created but
         # the executor never starts, so it shouldn't be shutdowned.
-        self.started = False
+        self.is_started = False
+
+        # Attribute indicating whether this executor was shutdown before.
+        # This safeguards cases where this object is automatically shut down (e.g.,
+        # via atexit) and the user also explicitly calls shut down. While this is
+        # permitted, the effect of an executor shutdown should happen only once.
+        self.is_shutdown = False
 
     def atexit_cleanup(self):
         # Calls this executor's shutdown method upon Python exiting the process.
@@ -321,7 +326,7 @@ class WorkQueueExecutor(BlockProviderExecutor, putils.RepresentationMixin):
         retrieve Parsl tasks within the Work Queue system.
         """
         # Mark this executor object as started
-        self.started = True
+        self.is_started = True
         self.tasks_lock = threading.Lock()
 
         # Create directories for data and results
@@ -669,13 +674,6 @@ class WorkQueueExecutor(BlockProviderExecutor, putils.RepresentationMixin):
         self.worker_command = self._construct_worker_command()
         self._patch_providers()
 
-        if hasattr(self.provider, 'init_blocks'):
-            try:
-                self.scale_out(blocks=self.provider.init_blocks)
-            except Exception as e:
-                logger.error("Initial block scaling out failed: {}".format(e))
-                raise e
-
     @property
     def outstanding(self) -> int:
         """Count the number of outstanding tasks. This is inefficiently
@@ -697,8 +695,8 @@ class WorkQueueExecutor(BlockProviderExecutor, putils.RepresentationMixin):
         """Scale in method.
         """
         # Obtain list of blocks to kill
-        to_kill = list(self.blocks.keys())[:count]
-        kill_ids = [self.blocks[block] for block in to_kill]
+        to_kill = list(self.blocks_to_job_id.keys())[:count]
+        kill_ids = [self.blocks_to_job_id[block] for block in to_kill]
 
         # Cancel the blocks provisioned
         if self.provider:
@@ -710,15 +708,19 @@ class WorkQueueExecutor(BlockProviderExecutor, putils.RepresentationMixin):
         """Shutdown the executor. Sets flag to cancel the submit process and
         collector thread, which shuts down the Work Queue system submission.
         """
-        if not self.started:
+        if not self.is_started:
             # Don't shutdown if the executor never starts.
             return
 
+        if self.is_shutdown:
+            # Don't shutdown this executor again.
+            return
+
         logger.debug("Work Queue shutdown started")
         self.should_stop.value = True
 
         # Remove the workers that are still going
-        kill_ids = [self.blocks[block] for block in self.blocks.keys()]
+        kill_ids = [self.blocks_to_job_id[block] for block in self.blocks_to_job_id.keys()]
         if self.provider:
             logger.debug("Cancelling blocks")
             self.provider.cancel(kill_ids)
@@ -728,6 +730,7 @@ class WorkQueueExecutor(BlockProviderExecutor, putils.RepresentationMixin):
         logger.debug("Joining on collector thread")
         self.collector_thread.join()
 
+        self.is_shutdown = True
         logger.debug("Work Queue shutdown completed")
 
     @wrap_with_logs

parsl/jobs/job_status_poller.py

@@ -16,20 +16,19 @@ from parsl.utils import Timer
 logger = logging.getLogger(__name__)
 
 
-class PollItem:
+class PolledExecutorFacade:
     def __init__(self, executor: BlockProviderExecutor, dfk: Optional["parsl.dataflow.dflow.DataFlowKernel"] = None):
         self._executor = executor
-        self._dfk = dfk
         self._interval = executor.status_polling_interval
         self._last_poll_time = 0.0
         self._status = {}  # type: Dict[str, JobStatus]
 
         # Create a ZMQ channel to send poll status to monitoring
         self.monitoring_enabled = False
-        if self._dfk and self._dfk.monitoring is not None:
+        if dfk and dfk.monitoring is not None:
             self.monitoring_enabled = True
-            hub_address = self._dfk.hub_address
-            hub_port = self._dfk.hub_zmq_port
+            hub_address = dfk.hub_address
+            hub_port = dfk.hub_zmq_port
             context = zmq.Context()
             self.hub_channel = context.socket(zmq.DEALER)
             self.hub_channel.set_hwm(0)
@@ -109,7 +108,7 @@ class JobStatusPoller(Timer):
     def __init__(self, *, strategy: Optional[str], max_idletime: float,
                  strategy_period: Union[float, int],
                  dfk: Optional["parsl.dataflow.dflow.DataFlowKernel"] = None) -> None:
-        self._poll_items = []  # type: List[PollItem]
+        self._executor_facades = []  # type: List[PolledExecutorFacade]
         self.dfk = dfk
         self._strategy = Strategy(strategy=strategy,
                                   max_idletime=max_idletime)
@@ -117,21 +116,37 @@ class JobStatusPoller(Timer):
 
     def poll(self) -> None:
         self._update_state()
-        self._run_error_handlers(self._poll_items)
-        self._strategy.strategize(self._poll_items)
+        self._run_error_handlers(self._executor_facades)
+        self._strategy.strategize(self._executor_facades)
 
-    def _run_error_handlers(self, status: List[PollItem]) -> None:
+    def _run_error_handlers(self, status: List[PolledExecutorFacade]) -> None:
         for es in status:
             es.executor.handle_errors(es.status)
 
     def _update_state(self) -> None:
         now = time.time()
-        for item in self._poll_items:
+        for item in self._executor_facades:
             item.poll(now)
 
     def add_executors(self, executors: Sequence[BlockProviderExecutor]) -> None:
         for executor in executors:
             if executor.status_polling_interval > 0:
                 logger.debug("Adding executor {}".format(executor.label))
-                self._poll_items.append(PollItem(executor, self.dfk))
+                self._executor_facades.append(PolledExecutorFacade(executor, self.dfk))
         self._strategy.add_executors(executors)
+
+    def close(self):
+        super().close()
+        for ef in self._executor_facades:
+            if not ef.executor.bad_state_is_set:
+                logger.info(f"Scaling in executor {ef.executor.label}")
+
+                # this code needs to be at least as many blocks as need
+                # cancelling, but it is safe to be more, as the scaling
+                # code will cope with being asked to cancel more blocks
+                # than exist.
+                block_count = len(ef.status)
+                ef.scale_in(block_count)
+
+            else:  # and bad_state_is_set
+                logger.warning(f"Not scaling in executor {ef.executor.label} because it is in bad state")

parsl/jobs/strategy.py

@@ -26,6 +26,10 @@ class ExecutorState(TypedDict):
     If the executor is not idle, then None.
     """
 
+    first: bool
+    """True if this executor has not yet had a strategy poll.
+    """
+
 
 class Strategy:
     """Scaling strategy.
@@ -129,8 +133,8 @@ class Strategy:
         self.executors = {}
         self.max_idletime = max_idletime
 
-        self.strategies = {None: self._strategy_noop,
-                           'none': self._strategy_noop,
+        self.strategies = {None: self._strategy_init_only,
+                           'none': self._strategy_init_only,
                            'simple': self._strategy_simple,
                            'htex_auto_scale': self._strategy_htex_auto_scale}
 
@@ -144,17 +148,24 @@ class Strategy:
 
     def add_executors(self, executors: Sequence[ParslExecutor]) -> None:
         for executor in executors:
-            self.executors[executor.label] = {'idle_since': None}
+            self.executors[executor.label] = {'idle_since': None, 'first': True}
 
-    def _strategy_noop(self, status: List[jsp.PollItem]) -> None:
-        """Do nothing.
+    def _strategy_init_only(self, executor_facades: List[jsp.PolledExecutorFacade]) -> None:
+        """Scale up to init_blocks at the start, then nothing more.
         """
-        logger.debug("strategy_noop: doing nothing")
+        for ef in executor_facades:
+            executor = ef.executor
+            if self.executors[executor.label]['first']:
+                logger.debug(f"strategy_init_only: scaling out {executor.provider.init_blocks} initial blocks for {executor.label}")
+                ef.scale_out(executor.provider.init_blocks)
+                self.executors[executor.label]['first'] = False
+            else:
+                logger.debug("strategy_init_only: doing nothing")
 
-    def _strategy_simple(self, status_list: List[jsp.PollItem]) -> None:
-        self._general_strategy(status_list, strategy_type='simple')
+    def _strategy_simple(self, executor_facades: List[jsp.PolledExecutorFacade]) -> None:
+        self._general_strategy(executor_facades, strategy_type='simple')
 
-    def _strategy_htex_auto_scale(self, status_list: List[jsp.PollItem]) -> None:
+    def _strategy_htex_auto_scale(self, executor_facades: List[jsp.PolledExecutorFacade]) -> None:
         """HTEX specific auto scaling strategy
 
         This strategy works only for HTEX. This strategy will scale out by
@@ -169,24 +180,30 @@ class Strategy:
         expected to scale in effectively only when # of workers, or tasks executing
         per block is close to 1.
         """
-        self._general_strategy(status_list, strategy_type='htex')
+        self._general_strategy(executor_facades, strategy_type='htex')
 
     @wrap_with_logs
-    def _general_strategy(self, status_list, *, strategy_type):
-        logger.debug(f"general strategy starting with strategy_type {strategy_type} for {len(status_list)} executors")
+    def _general_strategy(self, executor_facades, *, strategy_type):
+        logger.debug(f"general strategy starting with strategy_type {strategy_type} for {len(executor_facades)} executors")
 
-        for exec_status in status_list:
-            executor = exec_status.executor
+        for ef in executor_facades:
+            executor = ef.executor
             label = executor.label
             if not isinstance(executor, BlockProviderExecutor):
                 logger.debug(f"Not strategizing for executor {label} because scaling not enabled")
                 continue
             logger.debug(f"Strategizing for executor {label}")
 
+            if self.executors[label]['first']:
+                executor = ef.executor
+                logger.debug(f"Scaling out {executor.provider.init_blocks} initial blocks for {label}")
+                ef.scale_out(executor.provider.init_blocks)
+                self.executors[label]['first'] = False
+
             # Tasks that are either pending completion
             active_tasks = executor.outstanding
 
-            status = exec_status.status
+            status = ef.status
 
             # FIXME we need to handle case where provider does not define these
             # FIXME probably more of this logic should be moved to the provider
@@ -242,7 +259,7 @@ class Strategy:
                         # We have resources idle for the max duration,
                         # we have to scale_in now.
                         logger.debug(f"Idle time has reached {self.max_idletime}s for executor {label}; scaling in")
-                        exec_status.scale_in(active_blocks - min_blocks)
+                        ef.scale_in(active_blocks - min_blocks)
 
                     else:
                         logger.debug(
@@ -265,7 +282,7 @@ class Strategy:
                     excess_blocks = math.ceil(float(excess_slots) / (tasks_per_node * nodes_per_block))
                     excess_blocks = min(excess_blocks, max_blocks - active_blocks)
                     logger.debug(f"Requesting {excess_blocks} more blocks")
-                    exec_status.scale_out(excess_blocks)
+                    ef.scale_out(excess_blocks)
 
             elif active_slots == 0 and active_tasks > 0:
                 logger.debug("Strategy case 4a: No active slots but some active tasks - could scale out by a single block")
@@ -274,7 +291,7 @@ class Strategy:
                 if active_blocks < max_blocks:
                     logger.debug("Requesting single block")
 
-                    exec_status.scale_out(1)
+                    ef.scale_out(1)
                 else:
                     logger.debug("Not requesting single block, because at maxblocks already")
 
@@ -290,7 +307,7 @@ class Strategy:
                             excess_blocks = math.ceil(float(excess_slots) / (tasks_per_node * nodes_per_block))
                             excess_blocks = min(excess_blocks, active_blocks - min_blocks)
                             logger.debug(f"Requesting scaling in by {excess_blocks} blocks with idle time {self.max_idletime}s")
-                            exec_status.scale_in(excess_blocks, max_idletime=self.max_idletime)
+                            ef.scale_in(excess_blocks, max_idletime=self.max_idletime)
                     else:
                         logger.error("This strategy does not support scaling in except for HighThroughputExecutor - taking no action")
                 else:

parsl/monitoring/monitoring.py

@@ -8,8 +8,6 @@ import zmq
 
 import queue
 
-import parsl.monitoring.remote
-
 from parsl.multiprocessing import ForkProcess, SizedQueue
 from multiprocessing import Process
 from multiprocessing.queues import Queue
@@ -23,7 +21,7 @@ from parsl.serialize import deserialize
 from parsl.monitoring.router import router_starter
 from parsl.monitoring.message_type import MessageType
 from parsl.monitoring.types import AddressedMonitoringMessage
-from typing import cast, Any, Callable, Dict, Optional, Sequence, Tuple, Union, TYPE_CHECKING
+from typing import cast, Any, Optional, Tuple, Union, TYPE_CHECKING
 
 _db_manager_excepts: Optional[Exception]
 
@@ -269,23 +267,6 @@ class MonitoringHub(RepresentationMixin):
             self.filesystem_proc.terminate()
             self.filesystem_proc.join()
 
-    @staticmethod
-    def monitor_wrapper(f: Any,
-                        args: Sequence,
-                        kwargs: Dict,
-                        try_id: int,
-                        task_id: int,
-                        monitoring_hub_url: str,
-                        run_id: str,
-                        logging_level: int,
-                        sleep_dur: float,
-                        radio_mode: str,
-                        monitor_resources: bool,
-                        run_dir: str) -> Tuple[Callable, Sequence, Dict]:
-        return parsl.monitoring.remote.monitor_wrapper(f, args, kwargs, try_id, task_id, monitoring_hub_url,
-                                                       run_id, logging_level, sleep_dur, radio_mode,
-                                                       monitor_resources, run_dir)
-
 
 @wrap_with_logs
 def filesystem_receiver(logdir: str, q: "queue.Queue[AddressedMonitoringMessage]", run_dir: str) -> None:

parsl/monitoring/remote.py

@@ -15,7 +15,8 @@ from typing import Any, Callable, Dict, List, Sequence, Tuple
 logger = logging.getLogger(__name__)
 
 
-def monitor_wrapper(f: Any,           # per app
+def monitor_wrapper(*,
+                    f: Any,           # per app
                     args: Sequence,   # per invocation
                     kwargs: Dict,     # per invocation
                     x_try_id: int,    # per invocation

parsl/tests/site_tests/test_provider.py

@@ -58,7 +58,7 @@ def test_provider():
     logger.info("Job in terminal state")
 
     _, current_jobs = executor._get_block_and_job_ids()
-    # PR 1952 stoped removing scale_in blocks from self.blocks
+    # PR 1952 stoped removing scale_in blocks from self.blocks_to_job_id
     # A new PR will handle removing blocks from self.block
     # this includes failed/completed/canceled blocks
     assert len(current_jobs) == 1, "Expected current_jobs == 1"

parsl/tests/test_htex/test_disconnected_blocks.py

@@ -26,7 +26,6 @@ def local_config():
                 ),
             )
         ],
-        run_dir="/tmp/test_htex",
         max_idletime=0.5,
         strategy='htex_auto_scale',
     )

parsl/tests/test_htex/test_drain.py

@@ -35,6 +35,7 @@ def local_config():
             )
         ],
         strategy='none',
+        strategy_period=0.1
     )
 
 

parsl/tests/test_monitoring/test_htex_init_blocks_vs_monitoring.py

@@ -59,24 +59,27 @@ def test_row_counts(tmpd_cwd, strategy):
     from sqlalchemy import text
 
     db_url = f"sqlite:///{tmpd_cwd}/monitoring.db"
-    parsl.load(fresh_config(tmpd_cwd, strategy, db_url))
+    with parsl.load(fresh_config(tmpd_cwd, strategy, db_url)):
+        dfk = parsl.dfk()
+        run_id = dfk.run_id
 
-    this_app().result()
+        this_app().result()
 
-    parsl.dfk().cleanup()
     parsl.clear()
 
     engine = sqlalchemy.create_engine(db_url)
     with engine.begin() as connection:
 
-        result = connection.execute(text("SELECT COUNT(DISTINCT block_id) FROM block"))
+        binds = {"run_id": run_id}
+
+        result = connection.execute(text("SELECT COUNT(DISTINCT block_id) FROM block WHERE run_id = :run_id"), binds)
         (c, ) = result.first()
         assert c == 1, "We should see a single block in this database"
 
-        result = connection.execute(text("SELECT COUNT(*) FROM block WHERE block_id = 0 AND status = 'PENDING'"))
+        result = connection.execute(text("SELECT COUNT(*) FROM block WHERE block_id = 0 AND status = 'PENDING' AND run_id = :run_id"), binds)
         (c, ) = result.first()
         assert c == 1, "There should be a single pending status"
 
-        result = connection.execute(text("SELECT COUNT(*) FROM block WHERE block_id = 0 AND status = 'CANCELLED'"))
+        result = connection.execute(text("SELECT COUNT(*) FROM block WHERE block_id = 0 AND status = 'CANCELLED' AND run_id = :run_id"), binds)
         (c, ) = result.first()
         assert c == 1, "There should be a single cancelled status"

parsl/tests/test_python_apps/test_context_manager.py

@@ -2,6 +2,7 @@ import parsl
 from parsl.tests.configs.local_threads import fresh_config
 import pytest
 from parsl.errors import NoDataFlowKernelError
+from parsl.dataflow.dflow import DataFlowKernel
 
 
 @parsl.python_app
@@ -25,9 +26,8 @@ def local_teardown():
 @pytest.mark.local
 def test_within_context_manger():
     config = fresh_config()
-    with parsl.load(config=config):
-        py_future = square(2)
-        assert py_future.result() == 4
+    with parsl.load(config=config) as dfk:
+        assert isinstance(dfk, DataFlowKernel)
 
         bash_future = foo(1)
         assert bash_future.result() == 0

parsl/tests/test_scaling/test_shutdown_scalein.py

@@ -0,0 +1,78 @@
+import threading
+
+import pytest
+
+import parsl
+from parsl.channels import LocalChannel
+from parsl.config import Config
+from parsl.executors import HighThroughputExecutor
+from parsl.launchers import SimpleLauncher
+from parsl.providers import LocalProvider
+
+import random
+
+# we need some blocks, but it doesn't matter too much how many, as long
+# as they can all start up and get registered within the try_assert
+# timeout later on.
+BLOCK_COUNT = 3
+
+
+class AccumulatingLocalProvider(LocalProvider):
+    def __init__(self, *args, **kwargs):
+        # Use a list for submitted job IDs because if there are multiple
+        # submissions returning the same job ID, this test should count
+        # those...
+        self.submit_job_ids = []
+
+        # ... but there's no requirement, I think, that cancel must be called
+        # only once per job id. What matters here is that each job ID is
+        # cancelled at least once.
+        self.cancel_job_ids = set()
+
+        super().__init__(*args, **kwargs)
+
+    def submit(self, *args, **kwargs):
+        job_id = super().submit(*args, **kwargs)
+        self.submit_job_ids.append(job_id)
+        return job_id
+
+    def cancel(self, job_ids):
+        self.cancel_job_ids.update(job_ids)
+        return super().cancel(job_ids)
+
+
+@pytest.mark.local
+def test_shutdown_scalein_blocks(tmpd_cwd, try_assert):
+    """
+    This test scales up several blocks, and then checks that they are all
+    scaled in at DFK shutdown.
+    """
+    accumulating_provider = AccumulatingLocalProvider(
+        channel=LocalChannel(),
+        init_blocks=BLOCK_COUNT,
+        min_blocks=0,
+        max_blocks=0,
+        launcher=SimpleLauncher(),
+    )
+
+    htex = HighThroughputExecutor(
+               label="htex_local",
+               cores_per_worker=1,
+               provider=accumulating_provider
+           )
+
+    config = Config(
+        executors=[htex],
+        strategy='none',
+        strategy_period=0.1,
+        run_dir=str(tmpd_cwd)
+    )
+
+    with parsl.load(config):
+        # this will wait for everything to be scaled out fully
+        try_assert(lambda: len(htex.connected_managers()) == BLOCK_COUNT)
+
+    parsl.clear()
+
+    assert len(accumulating_provider.submit_job_ids) == BLOCK_COUNT, f"Exactly {BLOCK_COUNT} blocks should have been launched"
+    assert len(accumulating_provider.cancel_job_ids) == BLOCK_COUNT, f"Exactly {BLOCK_COUNT} blocks should have been scaled in"

parsl/version.py

@@ -3,4 +3,4 @@
 Year.Month.Day[alpha/beta/..]
 Alphas will be numbered like this -> 2024.12.10a0
 """
-VERSION = '2024.03.25'
+VERSION = '2024.04.08'

{parsl-2024.3.25.data → parsl-2024.4.8.data}/scripts/process_worker_pool.py

@@ -335,14 +335,17 @@ class Manager:
                 self.heartbeat_to_incoming()
                 last_beat = time.time()
 
-            if self.drain_time and time.time() > self.drain_time:
+            if time.time() > self.drain_time:
                 logger.info("Requesting drain")
                 self.drain_to_incoming()
-                self.drain_time = None
                 # This will start the pool draining...
                 # Drained exit behaviour does not happen here. It will be
                 # driven by the interchange sending a DRAINED_CODE message.
 
+                # now set drain time to the far future so we don't send a drain
+                # message every iteration.
+                self.drain_time = float('inf')
+
             poll_duration_s = max(0, next_interesting_event_time - time.time())
             socks = dict(poller.poll(timeout=poll_duration_s * 1000))
 

{parsl-2024.3.25.dist-info → parsl-2024.4.8.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: parsl
-Version: 2024.3.25
+Version: 2024.4.8
 Summary: Simple data dependent workflows in Python
 Home-page: https://github.com/Parsl/parsl
-Download-URL: https://github.com/Parsl/parsl/archive/2024.03.25.tar.gz
+Download-URL: https://github.com/Parsl/parsl/archive/2024.04.08.tar.gz
 Author: The Parsl Team
 Author-email: parsl@googlegroups.com
 License: Apache 2.0

{parsl-2024.3.25.dist-info → parsl-2024.4.8.dist-info}/RECORD

@@ -8,7 +8,7 @@ parsl/multiprocessing.py,sha256=hakfdg-sgxEjwloZeDrt6EhzwdzecvjJhkPHHxh8lII,1938
 parsl/process_loggers.py,sha256=1G3Rfrh5wuZNo2X03grG4kTYPGOxz7hHCyG6L_A3b0A,1137
 parsl/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 parsl/utils.py,sha256=A3WDMGaNB4ajVx_jCuc-74W6PFy4zswJy-pLE7u8Dz0,10979
-parsl/version.py,sha256=xw1wZ0QmQ9UaGCS5xiCsTOYpLaH5Ht4qp-xYq_4FbaE,131
+parsl/version.py,sha256=ahe277o-2uHeGXWS6lLdiHiTpTRqrLfBk6sTJtXZeNQ,131
 parsl/app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 parsl/app/app.py,sha256=wAHchJetgnicT1pn0NJKDeDX0lV3vDFlG8cQd_Ciax4,8522
 parsl/app/bash.py,sha256=bx9x1XFwkOTpZZD3CPwnVL9SyNRDjbUGtOnuGLvxN_8,5396
@@ -60,7 +60,7 @@ parsl/data_provider/http.py,sha256=nDHTW7XmJqAukWJjPRQjyhUXt8r6GsQ36mX9mv_wOig,2
 parsl/data_provider/rsync.py,sha256=2-ZxqrT-hBj39x082NusJaBqsGW4Jd2qCW6JkVPpEl0,4254
 parsl/data_provider/staging.py,sha256=l-mAXFburs3BWPjkSmiQKuAgJpsxCG62yATPDbrafYI,4523
 parsl/dataflow/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-parsl/dataflow/dflow.py,sha256=38bZ0cFnCKqlbyf-XYxyA8PscsypQNOzd_8bIUWT0wM,64461
+parsl/dataflow/dflow.py,sha256=FRsenqapR1uRR_6YU8bjqsENyTDMF7I9PftzLopOVzY,63254
 parsl/dataflow/errors.py,sha256=w2vOt_ymzG2dOqJUO4IDcmTlrCIHlMZL8nBVyVq0O_8,2176
 parsl/dataflow/futures.py,sha256=aVfEUTzp4-EdunDAtNcqVQf8l_A7ArDi2c82KZMwxfY,5256
 parsl/dataflow/memoization.py,sha256=AsJO6c6cRp2ac6H8uGn2USlEi78_nX3QWvpxYt4XdYE,9583
@@ -68,9 +68,9 @@ parsl/dataflow/rundirs.py,sha256=XKmBZpBEIsGACBhYOkbbs2e5edC0pQegJcSlk4FWeag,115
 parsl/dataflow/states.py,sha256=hV6mfv-y4A6xrujeQglcomnfEs7y3Xm2g6JFwC6dvgQ,2612
 parsl/dataflow/taskrecord.py,sha256=bzIBmlDTsRrELtB9PUQwxTWcwrCd8aMsUAzvijle1eo,3114
 parsl/executors/__init__.py,sha256=J50N97Nm9YRjz6K0oNXDxUYIsDjL43_tp3LVb2w7n-M,381
-parsl/executors/base.py,sha256=CNWddQ7eP_Kqd2THv4bj5Dg1Jgb3dbI3z3aznsRP6dc,4574
+parsl/executors/base.py,sha256=AFX7AlMbOoXaImrttO74vhNWhbJwu41JFS5EaWPl8fg,4559
 parsl/executors/errors.py,sha256=xVswxgi7vmJcUMCeYDAPK8sQT2kHFFROVoOr0dnmcWE,2098
-parsl/executors/status_handling.py,sha256=DP7Wu2BjoMPcJ8bdYmwALHvC5Fd1y8-g9sLj_cKoGOQ,10574
+parsl/executors/status_handling.py,sha256=8G0QBkcB271bR_UMTUh9lHE3EguMKQNdcQ8BrSR8Cwg,10322
 parsl/executors/threads.py,sha256=bMU3JFghm17Lpcua13pr3NgQhkUDDc2mqvF2yJBrVNQ,3353
 parsl/executors/flux/__init__.py,sha256=P9grTTeRPXfqXurFhlSS7XhmE6tTbnCnyQ1f9b-oYHE,136
 parsl/executors/flux/execute_parsl_task.py,sha256=yUG_WjZLcX8LrgPl26mpEBWZhQMlVNbRLGu08yIjdf4,1553
@@ -78,14 +78,14 @@ parsl/executors/flux/executor.py,sha256=0omXRPvykdW5VZb8mwgBJjxVk4H6G8xoL5D_R9pu
 parsl/executors/flux/flux_instance_manager.py,sha256=tTEOATClm9SwdgLeBRWPC6D55iNDuh0YxqJOw3c3eQ4,2036
 parsl/executors/high_throughput/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 parsl/executors/high_throughput/errors.py,sha256=vl69wLuVOplbKxHI9WphEGBExHWkTn5n8T9QhBXuNH0,380
-parsl/executors/high_throughput/executor.py,sha256=mSxDizbw79zmnW4yMDuRflZfGwZKciWmZ8XW9rAy4gI,37591
+parsl/executors/high_throughput/executor.py,sha256=W9515Ggt1iLAz1Xs0HYB54cxbfsHj2beW2Pdt2alKws,37225
 parsl/executors/high_throughput/interchange.py,sha256=Rt6HyFvQYFuqUJ1ytXmUFTDIK9wOBm4l96IHoL6OFRc,31491
 parsl/executors/high_throughput/manager_record.py,sha256=w5EwzVqPtsLOyOW8jP44U3uaogt8H--tkwp7FNyKN_o,385
 parsl/executors/high_throughput/monitoring_info.py,sha256=3gQpwQjjNDEBz0cQqJZB6hRiwLiWwXs83zkQDmbOwxY,297
 parsl/executors/high_throughput/mpi_prefix_composer.py,sha256=GPSejuNL407gvzw9f7dTWArTLn6heTi-erJjgcM-_8Y,4273
 parsl/executors/high_throughput/mpi_resource_management.py,sha256=geLYmp2teKYgTnzATAR_JPtjAa0ysu6pHpXs90vwkds,7975
 parsl/executors/high_throughput/probe.py,sha256=lvnuf-vBv57tHvFh-J51F9sDYBES7jCgs6KYgWvmKRs,2749
-parsl/executors/high_throughput/process_worker_pool.py,sha256=hUKno44b3hzPQHKnF91K_BQxusDoapT8K8y-2E0DlDo,41109
+parsl/executors/high_throughput/process_worker_pool.py,sha256=PjZ2rFieJUF_sVf2GCsRnsf3X3exYGX1qftA2Zs3kHc,41221
 parsl/executors/high_throughput/zmq_pipes.py,sha256=TEIr1PcBDVbchBukzPaEsku2lbIIFCYYjeUq5zw_VBA,6514
 parsl/executors/radical/__init__.py,sha256=CKbtV2numw5QvgIBq1htMUrt9TqDCIC2zifyf2svTNU,186
 parsl/executors/radical/executor.py,sha256=5r9WZkOr0clg79zm35E7nC7zNv0DpbyM8iTC2B6d4N0,21024
@@ -95,7 +95,7 @@ parsl/executors/radical/rpex_worker.py,sha256=1M1df-hzFdmZMWbRZlUzIX7uAWMKJ_SkxL
 parsl/executors/taskvine/__init__.py,sha256=sWIJdvSLgQKul9dlSjIkNat7yBDgU3SrBF3X2yhT86E,293
 parsl/executors/taskvine/errors.py,sha256=MNS_NjpvHjwevQXOjqjSEBFroqEWi-LT1ZEVZ2C5Dx0,652
 parsl/executors/taskvine/exec_parsl_function.py,sha256=oUAKbPWwpbzWwQ47bZQlVDxS8txhnhPsonMf3AOEMGQ,7085
-parsl/executors/taskvine/executor.py,sha256=YAPnZZV31R_H1A4mILNIiDQVVvzO6G1wUo0HIjmcw7g,32264
+parsl/executors/taskvine/executor.py,sha256=gpVJg0MXl4D6YzEklhrhRXnbEQxRJaPBvGNGrro0qKI,32531
 parsl/executors/taskvine/factory.py,sha256=sHhfGv7xRFrWkQclzRXuFEAHuSXhsZu2lR5LJ81aucA,2638
 parsl/executors/taskvine/factory_config.py,sha256=AbE2fN2snrF5ITYrrS4DnGn2XkJHUFr_17DYHDHIwq0,3693
 parsl/executors/taskvine/manager.py,sha256=VxVN2L5zFVPNfSAJrGgq87MRJKpcxf-BHdO5QWxB4TU,25822
@@ -104,15 +104,15 @@ parsl/executors/taskvine/utils.py,sha256=iSrIogeiauL3UNy_9tiZp1cBSNn6fIJkMYQRVi1
 parsl/executors/workqueue/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 parsl/executors/workqueue/errors.py,sha256=ghB93Ptb_QbOAvgLe7siV_snRRkU_T-cFHv3AR6Ziwo,541
 parsl/executors/workqueue/exec_parsl_function.py,sha256=NtWNeBvRqksej38eRPw8zPBJ1CeW6vgaitve0tfz_qc,7801
-parsl/executors/workqueue/executor.py,sha256=QnHNdj7aVVYOzK-jmo0YqKMqW2__XmmruHHilqGUVy0,49823
+parsl/executors/workqueue/executor.py,sha256=GG5F0PCu6gLvJjWqa7nxqHxIyRa_Iqt25NESDd7MExA,50038
 parsl/executors/workqueue/parsl_coprocess.py,sha256=kEFGC-A97c_gweUPvrc9EEGume7vUpkJLJlyAb87xtQ,5737
 parsl/executors/workqueue/parsl_coprocess_stub.py,sha256=_bJmpPIgL42qM6bVzeEKt1Mn1trSP41rtJguXxPGfHI,735
 parsl/jobs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 parsl/jobs/error_handlers.py,sha256=WcWZUA7KyE1ocX5zrMf_EwqOob8Jb7uHMjD3nlb_BUo,2319
 parsl/jobs/errors.py,sha256=cpSQXCrlKtuHsQf7usjF-lX8XsDkFnE5kWpmFjiN6OU,178
-parsl/jobs/job_status_poller.py,sha256=hLFyT9tnvYrIev72bg0jZUhabYR9GJ0o15M5l9TR3Qo,5422
+parsl/jobs/job_status_poller.py,sha256=4aWn_IO1A7NHxzbFT2Mm9Cvk0V1f2mzNxCEXJBQDtS8,6143
 parsl/jobs/states.py,sha256=rPBoAEEudKngWFijlwvXXhAagDs_9DCXvQP9rwzVgCM,4855
-parsl/jobs/strategy.py,sha256=a-W3vxEHHCfe521LMfSoZLpJjdTtwCfTgdn1ChxzUuI,12959
+parsl/jobs/strategy.py,sha256=-rumnV3-p_SSGR9532nh-KZK5P2yxEZs2FhCqxvIIPo,13860
 parsl/launchers/__init__.py,sha256=k8zAB3IBP-brfqXUptKwGkvsIRaXjAJZNBJa2XVtY1A,546
 parsl/launchers/base.py,sha256=CblcvPTJiu-MNLWaRtFe29SZQ0BpTOlaY8CGcHdlHIE,538
 parsl/launchers/errors.py,sha256=v5i460H_rovzukSccQetxQBVtd92jLQz-NbuDe2TdGI,467
@@ -120,9 +120,9 @@ parsl/launchers/launchers.py,sha256=VB--fiVv_IQne3DydTMSdGUY0o0g69puAs-Hd3mJ2vo,
 parsl/monitoring/__init__.py,sha256=0ywNz6i0lM1xo_7_BIxhETDGeVd2C_0wwD7qgeaMR4c,83
 parsl/monitoring/db_manager.py,sha256=hdmmXSTXp8Wwhr7vLpQalD_ahRl3SNxKYVsplnThRk8,37021
 parsl/monitoring/message_type.py,sha256=Khn88afNxcOIciKiCK4GLnn90I5BlRTiOL3zK-P07yQ,401
-parsl/monitoring/monitoring.py,sha256=ZkwbIKGG7Zx8Nxj8TEaTT_FrmjwJvmv1jNlEmaUZYAM,14313
+parsl/monitoring/monitoring.py,sha256=5R3-T4vtxedwQnde5aK6MVssKvjf_VU17S0gcft6oAc,13422
 parsl/monitoring/radios.py,sha256=T2_6QuUjC-dd_7qMnIk6WHQead1iWz7m_P6ZC4QAqdA,5265
-parsl/monitoring/remote.py,sha256=OcIgudujtPO_DsY-YV36x92skeiNdGt-6aEOqaCU8T0,13900
+parsl/monitoring/remote.py,sha256=0wqANMcuvq3dpja3agdbOzD72n5oUYp7PcNKyLCC35E,13923
 parsl/monitoring/router.py,sha256=Y_PJjffS23HwfTJClhg5W4gUXnkAI_3crjjZMoyzxVA,9592
 parsl/monitoring/types.py,sha256=SO6Fjjbb83sv_MtbutoxGssiWh6oXKkEEsD4EvwOnZ4,629
 parsl/monitoring/queries/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -279,7 +279,7 @@ parsl/tests/scaling_tests/wqex_condor.py,sha256=hMo1hK8aj-L36vj0PoByQvL9YQTVrenL
 parsl/tests/scaling_tests/wqex_local.py,sha256=C-eYESKhi4V4XZuHLO0lgP5rovEj8LNYGJOHLpUDdOM,545
 parsl/tests/site_tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 parsl/tests/site_tests/site_config_selector.py,sha256=hk8SO0QMLFk9Ef2QxSa4JTnXJxpUks1mLOeDhaquyqI,1980
-parsl/tests/site_tests/test_provider.py,sha256=5rtzDNj7LWCuWdic6ifEQIfqErYOhL-MPRuaehaMXEQ,2684
+parsl/tests/site_tests/test_provider.py,sha256=mRhfOnmsNi5wh3v8XFY4OIyMZgf2321zyt7KLiaI_YQ,2694
 parsl/tests/site_tests/test_site.py,sha256=gk23zU9BAZgctkSRAxUTolZKo6abVi45vrEavwpRCcs,1954
 parsl/tests/sites/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 parsl/tests/sites/test_affinity.py,sha256=4bym6aNynLCpPAjlDqFYugQbVHsvJlzBu20TCWYaX1M,1582
@@ -313,11 +313,6 @@ parsl/tests/test_checkpointing/test_regression_232.py,sha256=AsI6AJ0DcFaefAbEY9q
 parsl/tests/test_checkpointing/test_regression_233.py,sha256=jii7BKuygK6KMIGtg4IeBjix7Z28cYhv57rE9ixoXMU,1774
 parsl/tests/test_checkpointing/test_regression_239.py,sha256=P5kmf1LOo_qHtArkBLMhdvNbSPtURDU5u2tI8SXZTb0,2441
 parsl/tests/test_checkpointing/test_task_exit.py,sha256=3-ldQhX7YVEAowWK2TiZ6nrQQ7ktfWr-qaCShtjJZK8,1721
-parsl/tests/test_data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-parsl/tests/test_data/test_file.py,sha256=Dqen1RJ-uLfzX8xIyJN2Qw3oVy4cFkQQVh6KC72xFDA,950
-parsl/tests/test_data/test_file_apps.py,sha256=zTwLAf4R-lFLoqeyz9ZfFeVTs9PL9dmpKjeZEVG7C2s,1540
-parsl/tests/test_data/test_file_staging.py,sha256=PTBZhTQJsNtUi38uUZOdIb8yw18-qxMoY9GFodzPYuE,674
-parsl/tests/test_data/test_output_chain_filenames.py,sha256=9Mxfl9oU_x1ZSP8JSxT_t4WFCfDTprLjSeFNMm4vVxA,894
 parsl/tests/test_docs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 parsl/tests/test_docs/test_from_slides.py,sha256=0qJHAsSN3eqn4LAFTyCAq1rIUOotBzyQg7d_rJfBoes,653
 parsl/tests/test_docs/test_kwargs.py,sha256=-rMtAtarg2FOdxMuDLsZY5Crn_jmSwtelMwRNEtTlVk,925
@@ -340,8 +335,8 @@ parsl/tests/test_htex/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3h
 parsl/tests/test_htex/test_basic.py,sha256=GIOF3cr6A87QDVMxeN0LrvJnXO2Nap0c-TH462OaBK4,464
 parsl/tests/test_htex/test_connected_blocks.py,sha256=0628oJ_0_aVsda5xuFwG3_3q8ZiEAM-sfIOINkUHQrk,1639
 parsl/tests/test_htex/test_cpu_affinity_explicit.py,sha256=tv12ojw4DdymlVBjVNnrFX7_mhwix2jnBLkRbKOQRao,1422
-parsl/tests/test_htex/test_disconnected_blocks.py,sha256=HQhtX757t4CdWbtu-VT3MttLHQowGyHPOp9vX0TN_a4,1890
-parsl/tests/test_htex/test_drain.py,sha256=bnbQfoEQi9EIlDJsDMp7adR45gsAScTNPoOZ2vc12HY,2260
+parsl/tests/test_htex/test_disconnected_blocks.py,sha256=iga7wmhGACwUN6gkEFPw1dLodj6SzNZpevgSHNYSyjI,1856
+parsl/tests/test_htex/test_drain.py,sha256=BvPQIo0xx-z191eVR2rG51x22yzqD-6dLSH7bCAUhOg,2288
 parsl/tests/test_htex/test_htex.py,sha256=4dXtcthZQvgEDtMc00g6Pw7FnqNWB_0j8fuJqHKO-IE,3896
 parsl/tests/test_htex/test_manager_failure.py,sha256=gemQopZoDEoZLOvep5JZkY6tQlZoko8Z0Kmpj1-Gbws,1161
 parsl/tests/test_htex/test_missing_worker.py,sha256=oiDN3ylsf-72jmX-Y5OWA2kQWpbVbvmoSLnu2vnyZeY,976
@@ -353,7 +348,7 @@ parsl/tests/test_monitoring/test_app_names.py,sha256=4Ziggxv0JLP0UGAd5jjXdivUdZQ
 parsl/tests/test_monitoring/test_basic.py,sha256=uXWx2O2Y2gfSO4e8zTjyj5bucKHG9OVzMxQNnq9abeY,2776
 parsl/tests/test_monitoring/test_db_locks.py,sha256=PGoRmvqA6AYPXTPHOZPLH38Z4D6EEgSb6ZgNfZtwIGk,2910
 parsl/tests/test_monitoring/test_fuzz_zmq.py,sha256=CpTRF2z2xpshlHHTNiNIIJMOx8bxSmSyAwbMYcOkgBk,3121
-parsl/tests/test_monitoring/test_htex_init_blocks_vs_monitoring.py,sha256=aFxQWL0p3kmVWHgLjYTx0MdY3JMkVPQSae3pmRn5T3s,2605
+parsl/tests/test_monitoring/test_htex_init_blocks_vs_monitoring.py,sha256=Lfa6ENZWrExRsZcISMdF_G4VjswzSb0wlRSQFoZXkyQ,2765
 parsl/tests/test_monitoring/test_incomplete_futures.py,sha256=9lJhkWlVB8gCCTkFjObzoh1uCL1pRmU6gFgEzLCztnY,2021
 parsl/tests/test_monitoring/test_memoization_representation.py,sha256=tErT7zseSMaQ5eNmK3hH90J6OZKuAaFQG50OXK6Jy9s,2660
 parsl/tests/test_monitoring/test_viz_colouring.py,sha256=k8SiELxPtnGYZ4r02VQt46RC61fGDVC4nmY768snX1U,591
@@ -374,7 +369,7 @@ parsl/tests/test_providers/test_submiterror_deprecation.py,sha256=ZutVj_0VJ7M-5U
 parsl/tests/test_python_apps/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 parsl/tests/test_python_apps/test_arg_input_types.py,sha256=JXpfHiu8lr9BN6u1OzqFvGwBhxzsGTPMewHx6Wdo-HI,670
 parsl/tests/test_python_apps/test_basic.py,sha256=lFqh4ugePbp_FRiHGUXxzV34iS7l8C5UkxTHuLcpnYs,855
-parsl/tests/test_python_apps/test_context_manager.py,sha256=ajG8Vy0gxXzMO_rAm7yO_WO6XJCjEqUgmdRHdPBLCY0,844
+parsl/tests/test_python_apps/test_context_manager.py,sha256=iMBjOFmqEO2rDGPJwDySa1kWz-dRnZZNEWDPipS_Xi4,877
 parsl/tests/test_python_apps/test_dep_standard_futures.py,sha256=BloeaYBci0jS5al2d8Eqe3OfZ1tvolA5ZflOBQPR9Wo,859
 parsl/tests/test_python_apps/test_dependencies.py,sha256=IRiTI_lPoWBSFSFnaBlE6Bv08PKEaf-qj5dfqO2RjT0,272
 parsl/tests/test_python_apps/test_depfail_propagation.py,sha256=3q3HlVWrOixFtXWBvR_ypKtbdAHAJcKndXQ5drwrBQU,1488
@@ -417,6 +412,7 @@ parsl/tests/test_scaling/test_block_error_handler.py,sha256=VFKs_jq7yd7bpdfYva3S
 parsl/tests/test_scaling/test_regression_1621.py,sha256=iRu3GFsg2l9J61AVZKWLc6zJcvI2JYD0WvtTYDSv22I,1770
 parsl/tests/test_scaling/test_scale_down.py,sha256=T8NVmoIebdpSjrNJCdgDHumpz9eKLkJrpeW7Kwi8cBg,2821
 parsl/tests/test_scaling/test_scale_down_htex_auto_scale.py,sha256=1vP2a8qygnxuUji7B3kJOUgwjmmIC1fDPhDdqzs5YFA,4597
+parsl/tests/test_scaling/test_shutdown_scalein.py,sha256=8QYnU67Ezx7Il9edR-Wrwzxp3xE3E3ocXfrs4P1eCFQ,2417
 parsl/tests/test_serialization/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 parsl/tests/test_serialization/test_2555_caching_deserializer.py,sha256=J8__b4djA5tErd8FUSXGkGcdXlW2KHbBWRbCTAnV08Q,767
 parsl/tests/test_serialization/test_basic.py,sha256=51KshqIk2RNr7S2iSkl5tZo40CJBb0h6uby8YPgOGlg,543
@@ -432,6 +428,10 @@ parsl/tests/test_staging/test_1316.py,sha256=pj1QbmOJSRES1R4Ov380MmVe6xXvPUXh4FB
 parsl/tests/test_staging/test_docs_1.py,sha256=SIGIYo9w2vwkQ-i9Io38sYYj8ns7uFrD1uziR_0Ae2w,628
 parsl/tests/test_staging/test_docs_2.py,sha256=zy6P6aanR27_U6ASDrB0YyG8udyRvA8r2HRDX5RcslU,463
 parsl/tests/test_staging/test_elaborate_noop_file.py,sha256=d694K2jKhyBM0bIY9j3w_huVjTU2CVFPgIRfYFpIQQM,2466
+parsl/tests/test_staging/test_file.py,sha256=Dqen1RJ-uLfzX8xIyJN2Qw3oVy4cFkQQVh6KC72xFDA,950
+parsl/tests/test_staging/test_file_apps.py,sha256=zTwLAf4R-lFLoqeyz9ZfFeVTs9PL9dmpKjeZEVG7C2s,1540
+parsl/tests/test_staging/test_file_staging.py,sha256=PTBZhTQJsNtUi38uUZOdIb8yw18-qxMoY9GFodzPYuE,674
+parsl/tests/test_staging/test_output_chain_filenames.py,sha256=9Mxfl9oU_x1ZSP8JSxT_t4WFCfDTprLjSeFNMm4vVxA,894
 parsl/tests/test_staging/test_staging_ftp.py,sha256=EkRoTcQ00FZGh8lDVYBdKb-pQ-ybW2Sx5vqGltoMGJ4,778
 parsl/tests/test_staging/test_staging_ftp_in_task.py,sha256=kR2XrGvbvVFDpHg53NnjO04kqEksTJjQAMQwYqBdb2M,884
 parsl/tests/test_staging/test_staging_globus.py,sha256=ds8nDH5dNbI10FV_GxMHyVaY6GPnuPPzkX9IiqROLF0,2339
@@ -443,12 +443,12 @@ parsl/tests/test_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3
 parsl/tests/test_utils/test_representation_mixin.py,sha256=kUZeIDwA2rlbJ3-beGzLLwf3dOplTMCrWJN87etHcyY,1633
 parsl/usage_tracking/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 parsl/usage_tracking/usage.py,sha256=pSADeogWqvkYI_n2pojv4IWDEFAQ3KwXNx6LDTohMHQ,6684
-parsl-2024.3.25.data/scripts/exec_parsl_function.py,sha256=NtWNeBvRqksej38eRPw8zPBJ1CeW6vgaitve0tfz_qc,7801
-parsl-2024.3.25.data/scripts/parsl_coprocess.py,sha256=Y7Tc-h9WGui-YDe3w_h91w2Sm1JNL1gJ9kAV4PE_gw8,5722
-parsl-2024.3.25.data/scripts/process_worker_pool.py,sha256=iVrw160CpTAVuX9PH-ezU4ebm9C1_U6IMrkcdyTQJ58,41095
-parsl-2024.3.25.dist-info/LICENSE,sha256=tAkwu8-AdEyGxGoSvJ2gVmQdcicWw3j1ZZueVV74M-E,11357
-parsl-2024.3.25.dist-info/METADATA,sha256=9FTVzNNJqpfn1NqveTatneCqEQhMu3Oo9kA-M7fx40k,3974
-parsl-2024.3.25.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-parsl-2024.3.25.dist-info/entry_points.txt,sha256=XqnsWDYoEcLbsMcpnYGKLEnSBmaIe1YoM5YsBdJG2tI,176
-parsl-2024.3.25.dist-info/top_level.txt,sha256=PIheYoUFQtF2icLsgOykgU-Cjuwr2Oi6On2jo5RYgRM,6
-parsl-2024.3.25.dist-info/RECORD,,
+parsl-2024.4.8.data/scripts/exec_parsl_function.py,sha256=NtWNeBvRqksej38eRPw8zPBJ1CeW6vgaitve0tfz_qc,7801
+parsl-2024.4.8.data/scripts/parsl_coprocess.py,sha256=Y7Tc-h9WGui-YDe3w_h91w2Sm1JNL1gJ9kAV4PE_gw8,5722
+parsl-2024.4.8.data/scripts/process_worker_pool.py,sha256=V3K4admJ7QvwR9sN0GH-c6uOTgNU8zVb76q872WtYCo,41207
+parsl-2024.4.8.dist-info/LICENSE,sha256=tAkwu8-AdEyGxGoSvJ2gVmQdcicWw3j1ZZueVV74M-E,11357
+parsl-2024.4.8.dist-info/METADATA,sha256=iCcfSeM8LvJdXuanu5jEEGo_H-wXXz56VWD0iCzUYfg,3973
+parsl-2024.4.8.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+parsl-2024.4.8.dist-info/entry_points.txt,sha256=XqnsWDYoEcLbsMcpnYGKLEnSBmaIe1YoM5YsBdJG2tI,176
+parsl-2024.4.8.dist-info/top_level.txt,sha256=PIheYoUFQtF2icLsgOykgU-Cjuwr2Oi6On2jo5RYgRM,6
+parsl-2024.4.8.dist-info/RECORD,,