parsl 2024.4.1__py3-none-any.whl → 2024.4.8__py3-none-any.whl

parsl/dataflow/dflow.py

@@ -714,14 +714,18 @@ class DataFlowKernel:
 
         if self.monitoring is not None and self.monitoring.resource_monitoring_enabled:
             wrapper_logging_level = logging.DEBUG if self.monitoring.monitoring_debug else logging.INFO
-            (function, args, kwargs) = monitor_wrapper(function, args, kwargs, try_id, task_id,
-                                                       self.monitoring.monitoring_hub_url,
-                                                       self.run_id,
-                                                       wrapper_logging_level,
-                                                       self.monitoring.resource_monitoring_interval,
-                                                       executor.radio_mode,
-                                                       executor.monitor_resources(),
-                                                       self.run_dir)
+            (function, args, kwargs) = monitor_wrapper(f=function,
+                                                       args=args,
+                                                       kwargs=kwargs,
+                                                       x_try_id=try_id,
+                                                       x_task_id=task_id,
+                                                       monitoring_hub_url=self.monitoring.monitoring_hub_url,
+                                                       run_id=self.run_id,
+                                                       logging_level=wrapper_logging_level,
+                                                       sleep_dur=self.monitoring.resource_monitoring_interval,
+                                                       radio_mode=executor.radio_mode,
+                                                       monitor_resources=executor.monitor_resources(),
+                                                       run_dir=self.run_dir)
 
         with self.submitter_lock:
             exec_fu = executor.submit(function, task_record['resource_specification'], *args, **kwargs)
@@ -1214,21 +1218,7 @@ class DataFlowKernel:
         self.job_status_poller.close()
         logger.info("Terminated job status poller")
 
-        logger.info("Scaling in and shutting down executors")
-
-        for ef in self.job_status_poller._executor_facades:
-            if not ef.executor.bad_state_is_set:
-                logger.info(f"Scaling in executor {ef.executor.label}")
-
-                # this code needs to be at least as many blocks as need
-                # cancelling, but it is safe to be more, as the scaling
-                # code will cope with being asked to cancel more blocks
-                # than exist.
-                block_count = len(ef.status)
-                ef.scale_in(block_count)
-
-            else:  # and bad_state_is_set
-                logger.warning(f"Not scaling in executor {ef.executor.label} because it is in bad state")
+        logger.info("Shutting down executors")
 
         for executor in self.executors.values():
             logger.info(f"Shutting down executor {executor.label}")

parsl/executors/status_handling.py

@@ -102,12 +102,6 @@ class BlockProviderExecutor(ParslExecutor):
         else:
             return self._provider.status_polling_interval
 
-    def _fail_job_async(self, block_id: str, message: str):
-        """Marks a job that has failed to start but would not otherwise be included in status()
-        as failed and report it in status()
-        """
-        self._simulated_status[block_id] = JobStatus(JobState.FAILED, message)
-
     @abstractproperty
     def outstanding(self) -> int:
         """This should return the number of tasks that the executor has been given to run (waiting to run, and running now)"""
@@ -198,8 +192,7 @@ class BlockProviderExecutor(ParslExecutor):
                 self.job_ids_to_block[job_id] = block_id
                 block_ids.append(block_id)
             except Exception as ex:
-                self._fail_job_async(block_id,
-                                     "Failed to start block {}: {}".format(block_id, ex))
+                self._simulated_status[block_id] = JobStatus(JobState.FAILED, "Failed to start block {}: {}".format(block_id, ex))
         return block_ids
 
     @abstractmethod

parsl/jobs/job_status_poller.py

@@ -19,18 +19,16 @@ logger = logging.getLogger(__name__)
 class PolledExecutorFacade:
     def __init__(self, executor: BlockProviderExecutor, dfk: Optional["parsl.dataflow.dflow.DataFlowKernel"] = None):
         self._executor = executor
-        self._dfk = dfk
         self._interval = executor.status_polling_interval
         self._last_poll_time = 0.0
         self._status = {}  # type: Dict[str, JobStatus]
-        self.first = True
 
         # Create a ZMQ channel to send poll status to monitoring
         self.monitoring_enabled = False
-        if self._dfk and self._dfk.monitoring is not None:
+        if dfk and dfk.monitoring is not None:
             self.monitoring_enabled = True
-            hub_address = self._dfk.hub_address
-            hub_port = self._dfk.hub_zmq_port
+            hub_address = dfk.hub_address
+            hub_port = dfk.hub_zmq_port
             context = zmq.Context()
             self.hub_channel = context.socket(zmq.DEALER)
             self.hub_channel.set_hwm(0)
@@ -136,3 +134,19 @@ class JobStatusPoller(Timer):
                 logger.debug("Adding executor {}".format(executor.label))
                 self._executor_facades.append(PolledExecutorFacade(executor, self.dfk))
         self._strategy.add_executors(executors)
+
+    def close(self):
+        super().close()
+        for ef in self._executor_facades:
+            if not ef.executor.bad_state_is_set:
+                logger.info(f"Scaling in executor {ef.executor.label}")
+
+                # this code needs to be at least as many blocks as need
+                # cancelling, but it is safe to be more, as the scaling
+                # code will cope with being asked to cancel more blocks
+                # than exist.
+                block_count = len(ef.status)
+                ef.scale_in(block_count)
+
+            else:  # and bad_state_is_set
+                logger.warning(f"Not scaling in executor {ef.executor.label} because it is in bad state")

parsl/jobs/strategy.py

@@ -26,6 +26,10 @@ class ExecutorState(TypedDict):
     If the executor is not idle, then None.
     """
 
+    first: bool
+    """True if this executor has not yet had a strategy poll.
+    """
+
 
 class Strategy:
     """Scaling strategy.
@@ -144,17 +148,17 @@ class Strategy:
 
     def add_executors(self, executors: Sequence[ParslExecutor]) -> None:
         for executor in executors:
-            self.executors[executor.label] = {'idle_since': None}
+            self.executors[executor.label] = {'idle_since': None, 'first': True}
 
     def _strategy_init_only(self, executor_facades: List[jsp.PolledExecutorFacade]) -> None:
         """Scale up to init_blocks at the start, then nothing more.
         """
         for ef in executor_facades:
-            if ef.first:
-                executor = ef.executor
+            executor = ef.executor
+            if self.executors[executor.label]['first']:
                 logger.debug(f"strategy_init_only: scaling out {executor.provider.init_blocks} initial blocks for {executor.label}")
                 ef.scale_out(executor.provider.init_blocks)
-                ef.first = False
+                self.executors[executor.label]['first'] = False
             else:
                 logger.debug("strategy_init_only: doing nothing")
 
@@ -190,11 +194,11 @@ class Strategy:
                 continue
             logger.debug(f"Strategizing for executor {label}")
 
-            if ef.first:
+            if self.executors[label]['first']:
                 executor = ef.executor
                 logger.debug(f"Scaling out {executor.provider.init_blocks} initial blocks for {label}")
                 ef.scale_out(executor.provider.init_blocks)
-                ef.first = False
+                self.executors[label]['first'] = False
 
             # Tasks that are either pending completion
             active_tasks = executor.outstanding

parsl/monitoring/remote.py

@@ -15,7 +15,8 @@ from typing import Any, Callable, Dict, List, Sequence, Tuple
 logger = logging.getLogger(__name__)
 
 
-def monitor_wrapper(f: Any,           # per app
+def monitor_wrapper(*,
+                    f: Any,           # per app
                     args: Sequence,   # per invocation
                     kwargs: Dict,     # per invocation
                     x_try_id: int,    # per invocation

parsl/version.py

@@ -3,4 +3,4 @@
 Year.Month.Day[alpha/beta/..]
 Alphas will be numbered like this -> 2024.12.10a0
 """
-VERSION = '2024.04.01'
+VERSION = '2024.04.08'

{parsl-2024.4.1.dist-info → parsl-2024.4.8.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: parsl
-Version: 2024.4.1
+Version: 2024.4.8
 Summary: Simple data dependent workflows in Python
 Home-page: https://github.com/Parsl/parsl
-Download-URL: https://github.com/Parsl/parsl/archive/2024.04.01.tar.gz
+Download-URL: https://github.com/Parsl/parsl/archive/2024.04.08.tar.gz
 Author: The Parsl Team
 Author-email: parsl@googlegroups.com
 License: Apache 2.0

{parsl-2024.4.1.dist-info → parsl-2024.4.8.dist-info}/RECORD

@@ -8,7 +8,7 @@ parsl/multiprocessing.py,sha256=hakfdg-sgxEjwloZeDrt6EhzwdzecvjJhkPHHxh8lII,1938
 parsl/process_loggers.py,sha256=1G3Rfrh5wuZNo2X03grG4kTYPGOxz7hHCyG6L_A3b0A,1137
 parsl/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 parsl/utils.py,sha256=A3WDMGaNB4ajVx_jCuc-74W6PFy4zswJy-pLE7u8Dz0,10979
-parsl/version.py,sha256=2xAmun0db7lx3hTmt30wYML-vGWWFXBqeVr7_i8j3Ac,131
+parsl/version.py,sha256=ahe277o-2uHeGXWS6lLdiHiTpTRqrLfBk6sTJtXZeNQ,131
 parsl/app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 parsl/app/app.py,sha256=wAHchJetgnicT1pn0NJKDeDX0lV3vDFlG8cQd_Ciax4,8522
 parsl/app/bash.py,sha256=bx9x1XFwkOTpZZD3CPwnVL9SyNRDjbUGtOnuGLvxN_8,5396
@@ -60,7 +60,7 @@ parsl/data_provider/http.py,sha256=nDHTW7XmJqAukWJjPRQjyhUXt8r6GsQ36mX9mv_wOig,2
 parsl/data_provider/rsync.py,sha256=2-ZxqrT-hBj39x082NusJaBqsGW4Jd2qCW6JkVPpEl0,4254
 parsl/data_provider/staging.py,sha256=l-mAXFburs3BWPjkSmiQKuAgJpsxCG62yATPDbrafYI,4523
 parsl/dataflow/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-parsl/dataflow/dflow.py,sha256=N72pg1yxgICjOAQVsTN_n3PwTaE2MYfPcc8MUTc8oMA,63593
+parsl/dataflow/dflow.py,sha256=FRsenqapR1uRR_6YU8bjqsENyTDMF7I9PftzLopOVzY,63254
 parsl/dataflow/errors.py,sha256=w2vOt_ymzG2dOqJUO4IDcmTlrCIHlMZL8nBVyVq0O_8,2176
 parsl/dataflow/futures.py,sha256=aVfEUTzp4-EdunDAtNcqVQf8l_A7ArDi2c82KZMwxfY,5256
 parsl/dataflow/memoization.py,sha256=AsJO6c6cRp2ac6H8uGn2USlEi78_nX3QWvpxYt4XdYE,9583
@@ -70,7 +70,7 @@ parsl/dataflow/taskrecord.py,sha256=bzIBmlDTsRrELtB9PUQwxTWcwrCd8aMsUAzvijle1eo,
 parsl/executors/__init__.py,sha256=J50N97Nm9YRjz6K0oNXDxUYIsDjL43_tp3LVb2w7n-M,381
 parsl/executors/base.py,sha256=AFX7AlMbOoXaImrttO74vhNWhbJwu41JFS5EaWPl8fg,4559
 parsl/executors/errors.py,sha256=xVswxgi7vmJcUMCeYDAPK8sQT2kHFFROVoOr0dnmcWE,2098
-parsl/executors/status_handling.py,sha256=RRPDFOCxrQT8_5XTpF5-keml4pttxX1rxztZGaJ1Wzw,10620
+parsl/executors/status_handling.py,sha256=8G0QBkcB271bR_UMTUh9lHE3EguMKQNdcQ8BrSR8Cwg,10322
 parsl/executors/threads.py,sha256=bMU3JFghm17Lpcua13pr3NgQhkUDDc2mqvF2yJBrVNQ,3353
 parsl/executors/flux/__init__.py,sha256=P9grTTeRPXfqXurFhlSS7XhmE6tTbnCnyQ1f9b-oYHE,136
 parsl/executors/flux/execute_parsl_task.py,sha256=yUG_WjZLcX8LrgPl26mpEBWZhQMlVNbRLGu08yIjdf4,1553
@@ -110,9 +110,9 @@ parsl/executors/workqueue/parsl_coprocess_stub.py,sha256=_bJmpPIgL42qM6bVzeEKt1M
 parsl/jobs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 parsl/jobs/error_handlers.py,sha256=WcWZUA7KyE1ocX5zrMf_EwqOob8Jb7uHMjD3nlb_BUo,2319
 parsl/jobs/errors.py,sha256=cpSQXCrlKtuHsQf7usjF-lX8XsDkFnE5kWpmFjiN6OU,178
-parsl/jobs/job_status_poller.py,sha256=owi712XlFEBLNEb-uSXq0hoCWm1Qgbq2cFMScc98w9I,5526
+parsl/jobs/job_status_poller.py,sha256=4aWn_IO1A7NHxzbFT2Mm9Cvk0V1f2mzNxCEXJBQDtS8,6143
 parsl/jobs/states.py,sha256=rPBoAEEudKngWFijlwvXXhAagDs_9DCXvQP9rwzVgCM,4855
-parsl/jobs/strategy.py,sha256=Jgz5H0sFLhAhFxqp6UCTihpDgG-HGp7NGz0ynXOmDSo,13656
+parsl/jobs/strategy.py,sha256=-rumnV3-p_SSGR9532nh-KZK5P2yxEZs2FhCqxvIIPo,13860
 parsl/launchers/__init__.py,sha256=k8zAB3IBP-brfqXUptKwGkvsIRaXjAJZNBJa2XVtY1A,546
 parsl/launchers/base.py,sha256=CblcvPTJiu-MNLWaRtFe29SZQ0BpTOlaY8CGcHdlHIE,538
 parsl/launchers/errors.py,sha256=v5i460H_rovzukSccQetxQBVtd92jLQz-NbuDe2TdGI,467
@@ -122,7 +122,7 @@ parsl/monitoring/db_manager.py,sha256=hdmmXSTXp8Wwhr7vLpQalD_ahRl3SNxKYVsplnThRk
 parsl/monitoring/message_type.py,sha256=Khn88afNxcOIciKiCK4GLnn90I5BlRTiOL3zK-P07yQ,401
 parsl/monitoring/monitoring.py,sha256=5R3-T4vtxedwQnde5aK6MVssKvjf_VU17S0gcft6oAc,13422
 parsl/monitoring/radios.py,sha256=T2_6QuUjC-dd_7qMnIk6WHQead1iWz7m_P6ZC4QAqdA,5265
-parsl/monitoring/remote.py,sha256=OcIgudujtPO_DsY-YV36x92skeiNdGt-6aEOqaCU8T0,13900
+parsl/monitoring/remote.py,sha256=0wqANMcuvq3dpja3agdbOzD72n5oUYp7PcNKyLCC35E,13923
 parsl/monitoring/router.py,sha256=Y_PJjffS23HwfTJClhg5W4gUXnkAI_3crjjZMoyzxVA,9592
 parsl/monitoring/types.py,sha256=SO6Fjjbb83sv_MtbutoxGssiWh6oXKkEEsD4EvwOnZ4,629
 parsl/monitoring/queries/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -443,12 +443,12 @@ parsl/tests/test_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3
 parsl/tests/test_utils/test_representation_mixin.py,sha256=kUZeIDwA2rlbJ3-beGzLLwf3dOplTMCrWJN87etHcyY,1633
 parsl/usage_tracking/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 parsl/usage_tracking/usage.py,sha256=pSADeogWqvkYI_n2pojv4IWDEFAQ3KwXNx6LDTohMHQ,6684
-parsl-2024.4.1.data/scripts/exec_parsl_function.py,sha256=NtWNeBvRqksej38eRPw8zPBJ1CeW6vgaitve0tfz_qc,7801
-parsl-2024.4.1.data/scripts/parsl_coprocess.py,sha256=Y7Tc-h9WGui-YDe3w_h91w2Sm1JNL1gJ9kAV4PE_gw8,5722
-parsl-2024.4.1.data/scripts/process_worker_pool.py,sha256=V3K4admJ7QvwR9sN0GH-c6uOTgNU8zVb76q872WtYCo,41207
-parsl-2024.4.1.dist-info/LICENSE,sha256=tAkwu8-AdEyGxGoSvJ2gVmQdcicWw3j1ZZueVV74M-E,11357
-parsl-2024.4.1.dist-info/METADATA,sha256=kM9BKd2VQ5ij6tV9au1Z1GvCH9MZcB7UWNo_YJVbqTA,3973
-parsl-2024.4.1.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-parsl-2024.4.1.dist-info/entry_points.txt,sha256=XqnsWDYoEcLbsMcpnYGKLEnSBmaIe1YoM5YsBdJG2tI,176
-parsl-2024.4.1.dist-info/top_level.txt,sha256=PIheYoUFQtF2icLsgOykgU-Cjuwr2Oi6On2jo5RYgRM,6
-parsl-2024.4.1.dist-info/RECORD,,
+parsl-2024.4.8.data/scripts/exec_parsl_function.py,sha256=NtWNeBvRqksej38eRPw8zPBJ1CeW6vgaitve0tfz_qc,7801
+parsl-2024.4.8.data/scripts/parsl_coprocess.py,sha256=Y7Tc-h9WGui-YDe3w_h91w2Sm1JNL1gJ9kAV4PE_gw8,5722
+parsl-2024.4.8.data/scripts/process_worker_pool.py,sha256=V3K4admJ7QvwR9sN0GH-c6uOTgNU8zVb76q872WtYCo,41207
+parsl-2024.4.8.dist-info/LICENSE,sha256=tAkwu8-AdEyGxGoSvJ2gVmQdcicWw3j1ZZueVV74M-E,11357
+parsl-2024.4.8.dist-info/METADATA,sha256=iCcfSeM8LvJdXuanu5jEEGo_H-wXXz56VWD0iCzUYfg,3973
+parsl-2024.4.8.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+parsl-2024.4.8.dist-info/entry_points.txt,sha256=XqnsWDYoEcLbsMcpnYGKLEnSBmaIe1YoM5YsBdJG2tI,176
+parsl-2024.4.8.dist-info/top_level.txt,sha256=PIheYoUFQtF2icLsgOykgU-Cjuwr2Oi6On2jo5RYgRM,6
+parsl-2024.4.8.dist-info/RECORD,,