parsl 2024.4.1__tar.gz → 2024.4.8__tar.gz

{parsl-2024.4.1/parsl.egg-info → parsl-2024.4.8}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: parsl
-Version: 2024.4.1
+Version: 2024.4.8
 Summary: Simple data dependent workflows in Python
 Home-page: https://github.com/Parsl/parsl
-Download-URL: https://github.com/Parsl/parsl/archive/2024.04.01.tar.gz
+Download-URL: https://github.com/Parsl/parsl/archive/2024.04.08.tar.gz
 Author: The Parsl Team
 Author-email: parsl@googlegroups.com
 License: Apache 2.0

{parsl-2024.4.1 → parsl-2024.4.8}/parsl/dataflow/dflow.py

@@ -714,14 +714,18 @@ class DataFlowKernel:
 
         if self.monitoring is not None and self.monitoring.resource_monitoring_enabled:
             wrapper_logging_level = logging.DEBUG if self.monitoring.monitoring_debug else logging.INFO
-            (function, args, kwargs) = monitor_wrapper(function, args, kwargs, try_id, task_id,
-                                                       self.monitoring.monitoring_hub_url,
-                                                       self.run_id,
-                                                       wrapper_logging_level,
-                                                       self.monitoring.resource_monitoring_interval,
-                                                       executor.radio_mode,
-                                                       executor.monitor_resources(),
-                                                       self.run_dir)
+            (function, args, kwargs) = monitor_wrapper(f=function,
+                                                       args=args,
+                                                       kwargs=kwargs,
+                                                       x_try_id=try_id,
+                                                       x_task_id=task_id,
+                                                       monitoring_hub_url=self.monitoring.monitoring_hub_url,
+                                                       run_id=self.run_id,
+                                                       logging_level=wrapper_logging_level,
+                                                       sleep_dur=self.monitoring.resource_monitoring_interval,
+                                                       radio_mode=executor.radio_mode,
+                                                       monitor_resources=executor.monitor_resources(),
+                                                       run_dir=self.run_dir)
 
         with self.submitter_lock:
             exec_fu = executor.submit(function, task_record['resource_specification'], *args, **kwargs)
@@ -1214,21 +1218,7 @@ class DataFlowKernel:
         self.job_status_poller.close()
         logger.info("Terminated job status poller")
 
-        logger.info("Scaling in and shutting down executors")
-
-        for ef in self.job_status_poller._executor_facades:
-            if not ef.executor.bad_state_is_set:
-                logger.info(f"Scaling in executor {ef.executor.label}")
-
-                # this code needs to be at least as many blocks as need
-                # cancelling, but it is safe to be more, as the scaling
-                # code will cope with being asked to cancel more blocks
-                # than exist.
-                block_count = len(ef.status)
-                ef.scale_in(block_count)
-
-            else:  # and bad_state_is_set
-                logger.warning(f"Not scaling in executor {ef.executor.label} because it is in bad state")
+        logger.info("Shutting down executors")
 
         for executor in self.executors.values():
             logger.info(f"Shutting down executor {executor.label}")

{parsl-2024.4.1 → parsl-2024.4.8}/parsl/executors/status_handling.py

@@ -102,12 +102,6 @@ class BlockProviderExecutor(ParslExecutor):
         else:
             return self._provider.status_polling_interval
 
-    def _fail_job_async(self, block_id: str, message: str):
-        """Marks a job that has failed to start but would not otherwise be included in status()
-        as failed and report it in status()
-        """
-        self._simulated_status[block_id] = JobStatus(JobState.FAILED, message)
-
     @abstractproperty
     def outstanding(self) -> int:
         """This should return the number of tasks that the executor has been given to run (waiting to run, and running now)"""
@@ -198,8 +192,7 @@ class BlockProviderExecutor(ParslExecutor):
                 self.job_ids_to_block[job_id] = block_id
                 block_ids.append(block_id)
             except Exception as ex:
-                self._fail_job_async(block_id,
-                                     "Failed to start block {}: {}".format(block_id, ex))
+                self._simulated_status[block_id] = JobStatus(JobState.FAILED, "Failed to start block {}: {}".format(block_id, ex))
         return block_ids
 
     @abstractmethod

{parsl-2024.4.1 → parsl-2024.4.8}/parsl/jobs/job_status_poller.py

@@ -19,18 +19,16 @@ logger = logging.getLogger(__name__)
 class PolledExecutorFacade:
     def __init__(self, executor: BlockProviderExecutor, dfk: Optional["parsl.dataflow.dflow.DataFlowKernel"] = None):
         self._executor = executor
-        self._dfk = dfk
         self._interval = executor.status_polling_interval
         self._last_poll_time = 0.0
         self._status = {}  # type: Dict[str, JobStatus]
-        self.first = True
 
         # Create a ZMQ channel to send poll status to monitoring
         self.monitoring_enabled = False
-        if self._dfk and self._dfk.monitoring is not None:
+        if dfk and dfk.monitoring is not None:
             self.monitoring_enabled = True
-            hub_address = self._dfk.hub_address
-            hub_port = self._dfk.hub_zmq_port
+            hub_address = dfk.hub_address
+            hub_port = dfk.hub_zmq_port
             context = zmq.Context()
             self.hub_channel = context.socket(zmq.DEALER)
             self.hub_channel.set_hwm(0)
@@ -136,3 +134,19 @@ class JobStatusPoller(Timer):
                 logger.debug("Adding executor {}".format(executor.label))
                 self._executor_facades.append(PolledExecutorFacade(executor, self.dfk))
         self._strategy.add_executors(executors)
+
+    def close(self):
+        super().close()
+        for ef in self._executor_facades:
+            if not ef.executor.bad_state_is_set:
+                logger.info(f"Scaling in executor {ef.executor.label}")
+
+                # this code needs to be at least as many blocks as need
+                # cancelling, but it is safe to be more, as the scaling
+                # code will cope with being asked to cancel more blocks
+                # than exist.
+                block_count = len(ef.status)
+                ef.scale_in(block_count)
+
+            else:  # and bad_state_is_set
+                logger.warning(f"Not scaling in executor {ef.executor.label} because it is in bad state")

{parsl-2024.4.1 → parsl-2024.4.8}/parsl/jobs/strategy.py

@@ -26,6 +26,10 @@ class ExecutorState(TypedDict):
     If the executor is not idle, then None.
     """
 
+    first: bool
+    """True if this executor has not yet had a strategy poll.
+    """
+
 
 class Strategy:
     """Scaling strategy.
@@ -144,17 +148,17 @@ class Strategy:
 
     def add_executors(self, executors: Sequence[ParslExecutor]) -> None:
         for executor in executors:
-            self.executors[executor.label] = {'idle_since': None}
+            self.executors[executor.label] = {'idle_since': None, 'first': True}
 
     def _strategy_init_only(self, executor_facades: List[jsp.PolledExecutorFacade]) -> None:
         """Scale up to init_blocks at the start, then nothing more.
         """
         for ef in executor_facades:
-            if ef.first:
-                executor = ef.executor
+            executor = ef.executor
+            if self.executors[executor.label]['first']:
                 logger.debug(f"strategy_init_only: scaling out {executor.provider.init_blocks} initial blocks for {executor.label}")
                 ef.scale_out(executor.provider.init_blocks)
-                ef.first = False
+                self.executors[executor.label]['first'] = False
             else:
                 logger.debug("strategy_init_only: doing nothing")
 
@@ -190,11 +194,11 @@ class Strategy:
                 continue
             logger.debug(f"Strategizing for executor {label}")
 
-            if ef.first:
+            if self.executors[label]['first']:
                 executor = ef.executor
                 logger.debug(f"Scaling out {executor.provider.init_blocks} initial blocks for {label}")
                 ef.scale_out(executor.provider.init_blocks)
-                ef.first = False
+                self.executors[label]['first'] = False
 
             # Tasks that are either pending completion
             active_tasks = executor.outstanding

{parsl-2024.4.1 → parsl-2024.4.8}/parsl/monitoring/remote.py

@@ -15,7 +15,8 @@ from typing import Any, Callable, Dict, List, Sequence, Tuple
 logger = logging.getLogger(__name__)
 
 
-def monitor_wrapper(f: Any,           # per app
+def monitor_wrapper(*,
+                    f: Any,           # per app
                     args: Sequence,   # per invocation
                     kwargs: Dict,     # per invocation
                     x_try_id: int,    # per invocation

{parsl-2024.4.1 → parsl-2024.4.8}/parsl/version.py

@@ -3,4 +3,4 @@
 Year.Month.Day[alpha/beta/..]
 Alphas will be numbered like this -> 2024.12.10a0
 """
-VERSION = '2024.04.01'
+VERSION = '2024.04.08'

{parsl-2024.4.1 → parsl-2024.4.8/parsl.egg-info}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: parsl
-Version: 2024.4.1
+Version: 2024.4.8
 Summary: Simple data dependent workflows in Python
 Home-page: https://github.com/Parsl/parsl
-Download-URL: https://github.com/Parsl/parsl/archive/2024.04.01.tar.gz
+Download-URL: https://github.com/Parsl/parsl/archive/2024.04.08.tar.gz
 Author: The Parsl Team
 Author-email: parsl@googlegroups.com
 License: Apache 2.0