parsl 2024.7.15__py3-none-any.whl → 2024.7.22__py3-none-any.whl

parsl/executors/high_throughput/process_worker_pool.py

@@ -732,17 +732,18 @@ def worker(
         os.sched_setaffinity(0, my_cores)  # type: ignore[attr-defined, unused-ignore]
         logger.info("Set worker CPU affinity to {}".format(my_cores))
 
-    # If CUDA devices, find total number of devices to allow for MPS
-    # See: https://developer.nvidia.com/system-management-interface
-    nvidia_smi_cmd = "nvidia-smi -L > /dev/null && nvidia-smi -L | wc -l"
-    nvidia_smi_ret = subprocess.run(nvidia_smi_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    if nvidia_smi_ret.returncode == 0:
-        num_cuda_devices = int(nvidia_smi_ret.stdout.split()[0])
-    else:
-        num_cuda_devices = None
-
     # If desired, pin to accelerator
     if accelerator is not None:
+
+        # If CUDA devices, find total number of devices to allow for MPS
+        # See: https://developer.nvidia.com/system-management-interface
+        nvidia_smi_cmd = "nvidia-smi -L > /dev/null && nvidia-smi -L | wc -l"
+        nvidia_smi_ret = subprocess.run(nvidia_smi_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        if nvidia_smi_ret.returncode == 0:
+            num_cuda_devices = int(nvidia_smi_ret.stdout.split()[0])
+        else:
+            num_cuda_devices = None
+
         try:
             if num_cuda_devices is not None:
                 procs_per_cuda_device = pool_size // num_cuda_devices

parsl/executors/status_handling.py

@@ -59,20 +59,28 @@ class BlockProviderExecutor(ParslExecutor):
         else:
             self.block_error_handler = block_error_handler
 
-        # errors can happen during the submit call to the provider; this is used
-        # to keep track of such errors so that they can be handled in one place
-        # together with errors reported by status()
-        self._simulated_status: Dict[str, JobStatus] = {}
         self._executor_bad_state = threading.Event()
         self._executor_exception: Optional[Exception] = None
 
         self._block_id_counter = AtomicIDCounter()
 
         self._tasks = {}  # type: Dict[object, Future]
+
+        self._last_poll_time = 0.0
+
+        # these four structures track, in loosely coordinated fashion, the
+        # existence of blocks and jobs and how to map between their
+        # identifiers.
         self.blocks_to_job_id = {}  # type: Dict[str, str]
         self.job_ids_to_block = {}  # type: Dict[str, str]
 
-        self._last_poll_time = 0.0
+        # errors can happen during the submit call to the provider; this is used
+        # to keep track of such errors so that they can be handled in one place
+        # together with errors reported by status()
+        self._simulated_status: Dict[str, JobStatus] = {}
+
+        # this stores an approximation (sometimes delayed) of the latest status
+        # of pending, active and recently terminated blocks
         self._status = {}  # type: Dict[str, JobStatus]
 
     def _make_status_dict(self, block_ids: List[str], status_list: List[JobStatus]) -> Dict[str, JobStatus]:
@@ -113,20 +121,6 @@ class BlockProviderExecutor(ParslExecutor):
         raise NotImplementedError("Classes inheriting from BlockProviderExecutor must implement "
                                   "outstanding()")
 
-    def status(self) -> Dict[str, JobStatus]:
-        """Return the status of all jobs/blocks currently known to this executor.
-
-        :return: a dictionary mapping block ids (in string) to job status
-        """
-        if self._provider:
-            block_ids, job_ids = self._get_block_and_job_ids()
-            status = self._make_status_dict(block_ids, self._provider.status(job_ids))
-        else:
-            status = {}
-        status.update(self._simulated_status)
-
-        return status
-
     def set_bad_state_and_fail_all(self, exception: Exception):
         """Allows external error handlers to mark this executor as irrecoverably bad and cause
         all tasks submitted to it now and in the future to fail. The executor is responsible
@@ -180,7 +174,7 @@ class BlockProviderExecutor(ParslExecutor):
         # Filters first iterable by bool values in second
         return list(compress(to_kill, killed))
 
-    def scale_out(self, blocks: int = 1) -> List[str]:
+    def _scale_out(self, blocks: int = 1) -> List[str]:
         """Scales out the number of blocks by "blocks"
         """
         if not self.provider:
@@ -276,6 +270,20 @@ class BlockProviderExecutor(ParslExecutor):
             if delta_status:
                 self.send_monitoring_info(delta_status)
 
+    def status(self) -> Dict[str, JobStatus]:
+        """Return the status of all jobs/blocks currently known to this executor.
+
+        :return: a dictionary mapping block ids (in string) to job status
+        """
+        if self._provider:
+            block_ids, job_ids = self._get_block_and_job_ids()
+            status = self._make_status_dict(block_ids, self._provider.status(job_ids))
+        else:
+            status = {}
+        status.update(self._simulated_status)
+
+        return status
+
     @property
     def status_facade(self) -> Dict[str, JobStatus]:
         """Return the status of all jobs/blocks of the executor of this poller.
@@ -304,7 +312,7 @@ class BlockProviderExecutor(ParslExecutor):
         return block_ids
 
     def scale_out_facade(self, n: int) -> List[str]:
-        block_ids = self.scale_out(n)
+        block_ids = self._scale_out(n)
         if block_ids is not None:
             new_status = {}
             for block_id in block_ids:

parsl/version.py

@@ -3,4 +3,4 @@
 Year.Month.Day[alpha/beta/..]
 Alphas will be numbered like this -> 2024.12.10a0
 """
-VERSION = '2024.07.15'
+VERSION = '2024.07.22'

{parsl-2024.7.15.data → parsl-2024.7.22.data}/scripts/process_worker_pool.py

@@ -732,17 +732,18 @@ def worker(
         os.sched_setaffinity(0, my_cores)  # type: ignore[attr-defined, unused-ignore]
         logger.info("Set worker CPU affinity to {}".format(my_cores))
 
-    # If CUDA devices, find total number of devices to allow for MPS
-    # See: https://developer.nvidia.com/system-management-interface
-    nvidia_smi_cmd = "nvidia-smi -L > /dev/null && nvidia-smi -L | wc -l"
-    nvidia_smi_ret = subprocess.run(nvidia_smi_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    if nvidia_smi_ret.returncode == 0:
-        num_cuda_devices = int(nvidia_smi_ret.stdout.split()[0])
-    else:
-        num_cuda_devices = None
-
     # If desired, pin to accelerator
     if accelerator is not None:
+
+        # If CUDA devices, find total number of devices to allow for MPS
+        # See: https://developer.nvidia.com/system-management-interface
+        nvidia_smi_cmd = "nvidia-smi -L > /dev/null && nvidia-smi -L | wc -l"
+        nvidia_smi_ret = subprocess.run(nvidia_smi_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        if nvidia_smi_ret.returncode == 0:
+            num_cuda_devices = int(nvidia_smi_ret.stdout.split()[0])
+        else:
+            num_cuda_devices = None
+
         try:
             if num_cuda_devices is not None:
                 procs_per_cuda_device = pool_size // num_cuda_devices

{parsl-2024.7.15.dist-info → parsl-2024.7.22.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: parsl
-Version: 2024.7.15
+Version: 2024.7.22
 Summary: Simple data dependent workflows in Python
 Home-page: https://github.com/Parsl/parsl
-Download-URL: https://github.com/Parsl/parsl/archive/2024.07.15.tar.gz
+Download-URL: https://github.com/Parsl/parsl/archive/2024.07.22.tar.gz
 Author: The Parsl Team
 Author-email: parsl@googlegroups.com
 License: Apache 2.0

{parsl-2024.7.15.dist-info → parsl-2024.7.22.dist-info}/RECORD

@@ -8,7 +8,7 @@ parsl/multiprocessing.py,sha256=MyaEcEq-Qf860u7V98u-PZrPNdtzOZL_NW6EhIJnmfQ,1937
 parsl/process_loggers.py,sha256=uQ7Gd0W72Jz7rrcYlOMfLsAEhkRltxXJL2MgdduJjEw,1136
 parsl/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 parsl/utils.py,sha256=91FjQiTUY383ueAjkBAgE21My9nba6SP2a2SrbB1r1Q,11250
-parsl/version.py,sha256=xBBY22CXKXmBYJqrmCPAgPlHvalhorEzfXaNGRSVeQU,131
+parsl/version.py,sha256=MTzhueu1_EeHa7_SmFzbomAqeyRCXKJ4cimHiEDRYbs,131
 parsl/app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 parsl/app/app.py,sha256=0gbM4AH2OtFOLsv07I5nglpElcwMSOi-FzdZZfrk7So,8532
 parsl/app/bash.py,sha256=jm2AvePlCT9DZR7H_4ANDWxatp5dN_22FUlT_gWhZ-g,5528
@@ -72,7 +72,7 @@ parsl/dataflow/taskrecord.py,sha256=-FuujdZQ1y5GSc-PJ91QKGT-Kp0lrg70MFDoxpbWI1Q,
 parsl/executors/__init__.py,sha256=Cg8e-F2NUaBD8A9crDAXKCSdoBEwQVIdgm4FlXd-wvk,476
 parsl/executors/base.py,sha256=10xMzqVa2vV7muet08Tm1iHBZ4m2jCESPPiRnbwzGUk,5120
 parsl/executors/errors.py,sha256=xVswxgi7vmJcUMCeYDAPK8sQT2kHFFROVoOr0dnmcWE,2098
-parsl/executors/status_handling.py,sha256=o3rBvCd_j87ZkUD9SZlaHjwg-WMPRzg3gkMETt-HNis,13473
+parsl/executors/status_handling.py,sha256=Hwcp8eCJSc_vVXycZX2vPTfikAP1SigtQJEiYbarjLw,13784
 parsl/executors/threads.py,sha256=hJt1LzxphqX4fe_9R9Cf1MU0lepWTU_eJe8O665B0Xo,3352
 parsl/executors/flux/__init__.py,sha256=P9grTTeRPXfqXurFhlSS7XhmE6tTbnCnyQ1f9b-oYHE,136
 parsl/executors/flux/execute_parsl_task.py,sha256=gRN7F4HhdrKQ-bvn4wXrquBzFOp_9WF88hMIeUaRg5I,1553
@@ -88,7 +88,7 @@ parsl/executors/high_throughput/mpi_executor.py,sha256=V07t1GOzFhcwdlZGuYUPqc1Na
 parsl/executors/high_throughput/mpi_prefix_composer.py,sha256=hah_IznfFqk-rzuHWmg6aiF_saiDRrpW-aSo4kH9Nso,4854
 parsl/executors/high_throughput/mpi_resource_management.py,sha256=LFBbJ3BnzTcY_v-jNu30uoIB2Enk4cleN4ygY3dncjY,8194
 parsl/executors/high_throughput/probe.py,sha256=TNpGTXb4_DEeg_h-LHu4zEKi1-hffboxvKcZUl2OZGk,2751
-parsl/executors/high_throughput/process_worker_pool.py,sha256=P1ZqQOyEpfvXxtfsevGpJvPH_PIxso3Mh0u8PyRbwD8,42958
+parsl/executors/high_throughput/process_worker_pool.py,sha256=weEld9iZr669gGmPxJC77ISVop7Y47Lc8TkjEfmnAyk,42991
 parsl/executors/high_throughput/zmq_pipes.py,sha256=tAjQB3aNVMuTXziN3dbJWre46YpXgliD55qMBbhYTLU,8581
 parsl/executors/radical/__init__.py,sha256=CKbtV2numw5QvgIBq1htMUrt9TqDCIC2zifyf2svTNU,186
 parsl/executors/radical/executor.py,sha256=426cMt6d8uJFZ_7Ub1kCslaND4OKtBX5WZdz-0RXjMk,22554
@@ -467,13 +467,13 @@ parsl/usage_tracking/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hS
 parsl/usage_tracking/api.py,sha256=iaCY58Dc5J4UM7_dJzEEs871P1p1HdxBMtNGyVdzc9g,1821
 parsl/usage_tracking/levels.py,sha256=xbfzYEsd55KiZJ-mzNgPebvOH4rRHum04hROzEf41tU,291
 parsl/usage_tracking/usage.py,sha256=qNEJ7nPimqd3Y7OWFLdYmNwJ6XDKlyfV_fTzasxsQw8,8690
-parsl-2024.7.15.data/scripts/exec_parsl_function.py,sha256=RUkJ4JSJAjr7YyRZ58zhMdg8cR5dVV9odUl3AuzNf3k,7802
-parsl-2024.7.15.data/scripts/interchange.py,sha256=n0aOHLX64DEWx-OA4vWrYRVZfmaz8Rc8haNtafbgh4k,30565
-parsl-2024.7.15.data/scripts/parsl_coprocess.py,sha256=zrVjEqQvFOHxsLufPi00xzMONagjVwLZbavPM7bbjK4,5722
-parsl-2024.7.15.data/scripts/process_worker_pool.py,sha256=pfIQ_JzqjviaiTfVI49qw4qy8FBS8AavN_12oL8DyzE,42944
-parsl-2024.7.15.dist-info/LICENSE,sha256=tAkwu8-AdEyGxGoSvJ2gVmQdcicWw3j1ZZueVV74M-E,11357
-parsl-2024.7.15.dist-info/METADATA,sha256=bagqkFFK8EeAICbm5afqQ4--DJWNZ_900VszWxbxsZk,4124
-parsl-2024.7.15.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-parsl-2024.7.15.dist-info/entry_points.txt,sha256=XqnsWDYoEcLbsMcpnYGKLEnSBmaIe1YoM5YsBdJG2tI,176
-parsl-2024.7.15.dist-info/top_level.txt,sha256=PIheYoUFQtF2icLsgOykgU-Cjuwr2Oi6On2jo5RYgRM,6
-parsl-2024.7.15.dist-info/RECORD,,
+parsl-2024.7.22.data/scripts/exec_parsl_function.py,sha256=RUkJ4JSJAjr7YyRZ58zhMdg8cR5dVV9odUl3AuzNf3k,7802
+parsl-2024.7.22.data/scripts/interchange.py,sha256=n0aOHLX64DEWx-OA4vWrYRVZfmaz8Rc8haNtafbgh4k,30565
+parsl-2024.7.22.data/scripts/parsl_coprocess.py,sha256=zrVjEqQvFOHxsLufPi00xzMONagjVwLZbavPM7bbjK4,5722
+parsl-2024.7.22.data/scripts/process_worker_pool.py,sha256=Ar-HLibZxnEVSVanAbOnBFtYdwQ_bSOwXGznoVQIdqI,42977
+parsl-2024.7.22.dist-info/LICENSE,sha256=tAkwu8-AdEyGxGoSvJ2gVmQdcicWw3j1ZZueVV74M-E,11357
+parsl-2024.7.22.dist-info/METADATA,sha256=D24HXi2DjCTXBzgd8QBjrkbk97stTxn18dY1_fv9tYM,4124
+parsl-2024.7.22.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+parsl-2024.7.22.dist-info/entry_points.txt,sha256=XqnsWDYoEcLbsMcpnYGKLEnSBmaIe1YoM5YsBdJG2tI,176
+parsl-2024.7.22.dist-info/top_level.txt,sha256=PIheYoUFQtF2icLsgOykgU-Cjuwr2Oi6On2jo5RYgRM,6
+parsl-2024.7.22.dist-info/RECORD,,