parsl 2023.12.11__py3-none-any.whl → 2023.12.25__py3-none-any.whl

parsl/dataflow/dflow.py

@@ -854,10 +854,13 @@ class DataFlowKernel:
                 try:
                     new_args.extend([dep.result()])
                 except Exception as e:
-                    if hasattr(dep, 'task_record'):
-                        tid = dep.task_record['id']
+                    # If this Future is associated with a task inside this DFK,
+                    # then refer to the task ID.
+                    # Otherwise make a repr of the Future object.
+                    if hasattr(dep, 'task_record') and dep.task_record['dfk'] == self:
+                        tid = "task " + repr(dep.task_record['id'])
                     else:
-                        tid = None
+                        tid = repr(dep)
                     dep_failures.extend([(e, tid)])
             else:
                 new_args.extend([dep])

parsl/dataflow/errors.py

@@ -36,7 +36,9 @@ class DependencyError(DataFlowException):
        in a dependency.
 
     Args:
-         - dependent_exceptions_tids: List of dependency task IDs which failed
+         - dependent_exceptions_tids: List of exceptions and identifiers for
+           dependencies which failed. The identifier might be a task ID or
+           the repr of a non-DFK Future.
          - task_id: Task ID of the task that failed because of the dependency error
     """
 
@@ -45,8 +47,8 @@ class DependencyError(DataFlowException):
         self.task_id = task_id
 
     def __str__(self) -> str:
-        dep_tids = [tid for (exception, tid) in self.dependent_exceptions_tids]
-        return "Dependency failure for task {} with failed dependencies from tasks {}".format(self.task_id, dep_tids)
+        deps = ", ".join(tid for _exc, tid in self.dependent_exceptions_tids)
+        return f"Dependency failure for task {self.task_id} with failed dependencies from {deps}"
 
 
 class JoinError(DataFlowException):

parsl/executors/high_throughput/process_worker_pool.py

@@ -10,7 +10,7 @@ import pickle
 import time
 import queue
 import uuid
-from typing import Sequence, Optional, Union
+from typing import Sequence, Optional
 
 import zmq
 import math
@@ -48,22 +48,22 @@ class Manager:
                 |                          |                IPC-Qeueues
 
     """
-    def __init__(self,
-                 addresses="127.0.0.1",
-                 address_probe_timeout=30,
-                 task_port="50097",
-                 result_port="50098",
-                 cores_per_worker=1,
-                 mem_per_worker=None,
-                 max_workers=float('inf'),
-                 prefetch_capacity=0,
-                 uid=None,
-                 block_id=None,
-                 heartbeat_threshold=120,
-                 heartbeat_period=30,
-                 poll_period=10,
-                 cpu_affinity=False,
-                 available_accelerators: Sequence[str] = ()):
+    def __init__(self, *,
+                 addresses,
+                 address_probe_timeout,
+                 task_port,
+                 result_port,
+                 cores_per_worker,
+                 mem_per_worker,
+                 max_workers,
+                 prefetch_capacity,
+                 uid,
+                 block_id,
+                 heartbeat_threshold,
+                 heartbeat_period,
+                 poll_period,
+                 cpu_affinity,
+                 available_accelerators: Sequence[str]):
         """
         Parameters
         ----------
@@ -72,7 +72,7 @@ class Manager:
 
         address_probe_timeout : int
              Timeout in seconds for the address probe to detect viable addresses
-             to the interchange. Default : 30s
+             to the interchange.
 
         uid : str
              string unique identifier
@@ -82,43 +82,41 @@ class Manager:
 
         cores_per_worker : float
              cores to be assigned to each worker. Oversubscription is possible
-             by setting cores_per_worker < 1.0. Default=1
+             by setting cores_per_worker < 1.0.
 
         mem_per_worker : float
              GB of memory required per worker. If this option is specified, the node manager
              will check the available memory at startup and limit the number of workers such that
              the there's sufficient memory for each worker. If set to None, memory on node is not
              considered in the determination of workers to be launched on node by the manager.
-             Default: None
 
         max_workers : int
              caps the maximum number of workers that can be launched.
-             default: infinity
 
         prefetch_capacity : int
              Number of tasks that could be prefetched over available worker capacity.
              When there are a few tasks (<100) or when tasks are long running, this option should
-             be set to 0 for better load balancing. Default is 0.
+             be set to 0 for better load balancing.
 
         heartbeat_threshold : int
              Seconds since the last message from the interchange after which the
-             interchange is assumed to be un-available, and the manager initiates shutdown. Default:120s
+             interchange is assumed to be un-available, and the manager initiates shutdown.
 
              Number of seconds since the last message from the interchange after which the worker
-             assumes that the interchange is lost and the manager shuts down. Default:120
+             assumes that the interchange is lost and the manager shuts down.
 
         heartbeat_period : int
              Number of seconds after which a heartbeat message is sent to the interchange, and workers
              are checked for liveness.
 
         poll_period : int
-             Timeout period used by the manager in milliseconds. Default: 10ms
+             Timeout period used by the manager in milliseconds.
 
         cpu_affinity : str
              Whether or how each worker should force its affinity to different CPUs
 
         available_accelerators: list of str
-            List of accelerators available to the workers. Default: Empty list
+            List of accelerators available to the workers.
 
         """
 
@@ -540,7 +538,7 @@ def worker(
     monitoring_queue: queue.Queue,
     ready_worker_count: Synchronized,
     tasks_in_progress: DictProxy,
-    cpu_affinity: Union[str, bool],
+    cpu_affinity: str,
     accelerator: Optional[str],
     block_id: str,
     task_queue_timeout: int,
@@ -732,6 +730,7 @@ if __name__ == "__main__":
     parser.add_argument("-r", "--result_port", required=True,
                         help="REQUIRED: Result port for posting results to the interchange")
     parser.add_argument("--cpu-affinity", type=str, choices=["none", "block", "alternating", "block-reverse"],
+                        required=True,
                         help="Whether/how workers should control CPU affinity.")
     parser.add_argument("--available-accelerators", type=str, nargs="*",
                         help="Names of available accelerators")

parsl/providers/slurm/slurm.py

@@ -132,6 +132,8 @@ class SlurmProvider(ClusterProvider, RepresentationMixin):
         self.exclusive = exclusive
         self.move_files = move_files
         self.account = account
+        self.qos = qos
+        self.constraint = constraint
         self.scheduler_options = scheduler_options + '\n'
         if exclusive:
             self.scheduler_options += "#SBATCH --exclusive\n"

parsl/tests/test_python_apps/test_dep_standard_futures.py

@@ -33,4 +33,12 @@ def test_future_fail_dependency():
 
     plain_fut.set_exception(ValueError("Plain failure"))
 
-    assert isinstance(parsl_fut.exception(), DependencyError)
+    ex = parsl_fut.exception()
+
+    # check that what we got is a dependency error...
+    assert isinstance(ex, DependencyError)
+
+    # and that the dependency error string mentions the dependency
+    # Future, plain_fut, somewhere in its str
+
+    assert repr(plain_fut) in str(ex)

parsl/tests/test_python_apps/test_depfail_propagation.py

@@ -21,6 +21,10 @@ def test_depfail_once():
     assert not isinstance(f1.exception(), DependencyError)
     assert isinstance(f2.exception(), DependencyError)
 
+    # check that the task ID of the failing task is mentioned
+    # in the DependencyError message
+    assert ("task " + str(f1.task_record['id'])) in str(f2.exception())
+
 
 def test_depfail_chain():
     """Test that dependency failures chain"""

parsl/version.py

@@ -3,4 +3,4 @@
 Year.Month.Day[alpha/beta/..]
 Alphas will be numbered like this -> 2024.12.10a0
 """
-VERSION = '2023.12.11'
+VERSION = '2023.12.25'

{parsl-2023.12.11.data → parsl-2023.12.25.data}/scripts/process_worker_pool.py

@@ -10,7 +10,7 @@ import pickle
 import time
 import queue
 import uuid
-from typing import Sequence, Optional, Union
+from typing import Sequence, Optional
 
 import zmq
 import math
@@ -48,22 +48,22 @@ class Manager:
                 |                          |                IPC-Qeueues
 
     """
-    def __init__(self,
-                 addresses="127.0.0.1",
-                 address_probe_timeout=30,
-                 task_port="50097",
-                 result_port="50098",
-                 cores_per_worker=1,
-                 mem_per_worker=None,
-                 max_workers=float('inf'),
-                 prefetch_capacity=0,
-                 uid=None,
-                 block_id=None,
-                 heartbeat_threshold=120,
-                 heartbeat_period=30,
-                 poll_period=10,
-                 cpu_affinity=False,
-                 available_accelerators: Sequence[str] = ()):
+    def __init__(self, *,
+                 addresses,
+                 address_probe_timeout,
+                 task_port,
+                 result_port,
+                 cores_per_worker,
+                 mem_per_worker,
+                 max_workers,
+                 prefetch_capacity,
+                 uid,
+                 block_id,
+                 heartbeat_threshold,
+                 heartbeat_period,
+                 poll_period,
+                 cpu_affinity,
+                 available_accelerators: Sequence[str]):
         """
         Parameters
         ----------
@@ -72,7 +72,7 @@ class Manager:
 
         address_probe_timeout : int
              Timeout in seconds for the address probe to detect viable addresses
-             to the interchange. Default : 30s
+             to the interchange.
 
         uid : str
              string unique identifier
@@ -82,43 +82,41 @@ class Manager:
 
         cores_per_worker : float
              cores to be assigned to each worker. Oversubscription is possible
-             by setting cores_per_worker < 1.0. Default=1
+             by setting cores_per_worker < 1.0.
 
         mem_per_worker : float
              GB of memory required per worker. If this option is specified, the node manager
              will check the available memory at startup and limit the number of workers such that
              the there's sufficient memory for each worker. If set to None, memory on node is not
              considered in the determination of workers to be launched on node by the manager.
-             Default: None
 
         max_workers : int
              caps the maximum number of workers that can be launched.
-             default: infinity
 
         prefetch_capacity : int
              Number of tasks that could be prefetched over available worker capacity.
              When there are a few tasks (<100) or when tasks are long running, this option should
-             be set to 0 for better load balancing. Default is 0.
+             be set to 0 for better load balancing.
 
         heartbeat_threshold : int
              Seconds since the last message from the interchange after which the
-             interchange is assumed to be un-available, and the manager initiates shutdown. Default:120s
+             interchange is assumed to be un-available, and the manager initiates shutdown.
 
              Number of seconds since the last message from the interchange after which the worker
-             assumes that the interchange is lost and the manager shuts down. Default:120
+             assumes that the interchange is lost and the manager shuts down.
 
         heartbeat_period : int
              Number of seconds after which a heartbeat message is sent to the interchange, and workers
              are checked for liveness.
 
         poll_period : int
-             Timeout period used by the manager in milliseconds. Default: 10ms
+             Timeout period used by the manager in milliseconds.
 
         cpu_affinity : str
              Whether or how each worker should force its affinity to different CPUs
 
         available_accelerators: list of str
-            List of accelerators available to the workers. Default: Empty list
+            List of accelerators available to the workers.
 
         """
 
@@ -540,7 +538,7 @@ def worker(
     monitoring_queue: queue.Queue,
     ready_worker_count: Synchronized,
     tasks_in_progress: DictProxy,
-    cpu_affinity: Union[str, bool],
+    cpu_affinity: str,
     accelerator: Optional[str],
     block_id: str,
     task_queue_timeout: int,
@@ -732,6 +730,7 @@ if __name__ == "__main__":
     parser.add_argument("-r", "--result_port", required=True,
                         help="REQUIRED: Result port for posting results to the interchange")
     parser.add_argument("--cpu-affinity", type=str, choices=["none", "block", "alternating", "block-reverse"],
+                        required=True,
                         help="Whether/how workers should control CPU affinity.")
     parser.add_argument("--available-accelerators", type=str, nargs="*",
                         help="Names of available accelerators")

{parsl-2023.12.11.dist-info → parsl-2023.12.25.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: parsl
-Version: 2023.12.11
+Version: 2023.12.25
 Summary: Simple data dependent workflows in Python
 Home-page: https://github.com/Parsl/parsl
-Download-URL: https://github.com/Parsl/parsl/archive/2023.12.11.tar.gz
+Download-URL: https://github.com/Parsl/parsl/archive/2023.12.25.tar.gz
 Author: The Parsl Team
 Author-email: parsl@googlegroups.com
 License: Apache 2.0

{parsl-2023.12.11.dist-info → parsl-2023.12.25.dist-info}/RECORD

@@ -7,7 +7,7 @@ parsl/multiprocessing.py,sha256=w3t1pFkHo4oZpznc2KF6Ff-Jj8MvXqvjm-hoiRqZDDQ,1984
 parsl/process_loggers.py,sha256=1G3Rfrh5wuZNo2X03grG4kTYPGOxz7hHCyG6L_A3b0A,1137
 parsl/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 parsl/utils.py,sha256=_flbNpTu6IXHbzIyE5JkUbOBIK4poc1R1bjBtwJUVdo,11622
-parsl/version.py,sha256=dq6nySNPdvDlzzMFyTCed-Ct6nwInHRR7BShO48ZzA0,131
+parsl/version.py,sha256=72YSwmW-MS_RN7eW_oxX5vVjFjXLm9v5J1JePWhtnv4,131
 parsl/app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 parsl/app/app.py,sha256=wAHchJetgnicT1pn0NJKDeDX0lV3vDFlG8cQd_Ciax4,8522
 parsl/app/bash.py,sha256=bx9x1XFwkOTpZZD3CPwnVL9SyNRDjbUGtOnuGLvxN_8,5396
@@ -62,8 +62,8 @@ parsl/data_provider/http.py,sha256=nDHTW7XmJqAukWJjPRQjyhUXt8r6GsQ36mX9mv_wOig,2
 parsl/data_provider/rsync.py,sha256=2-ZxqrT-hBj39x082NusJaBqsGW4Jd2qCW6JkVPpEl0,4254
 parsl/data_provider/staging.py,sha256=l-mAXFburs3BWPjkSmiQKuAgJpsxCG62yATPDbrafYI,4523
 parsl/dataflow/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-parsl/dataflow/dflow.py,sha256=875obETSEog7WyovHDxR9k07zr_nifhbn67H5zmHxf8,63658
-parsl/dataflow/errors.py,sha256=JfjoqqqSg1WvTash5uwUuJt1ZDvxw-aAMenO-N3zoas,2084
+parsl/dataflow/dflow.py,sha256=uuXY9pURFDBpL0w52J0DWGCOtorTQ5wFy5V0WwHS9L8,63909
+parsl/dataflow/errors.py,sha256=w2vOt_ymzG2dOqJUO4IDcmTlrCIHlMZL8nBVyVq0O_8,2176
 parsl/dataflow/futures.py,sha256=aVfEUTzp4-EdunDAtNcqVQf8l_A7ArDi2c82KZMwxfY,5256
 parsl/dataflow/memoization.py,sha256=AsJO6c6cRp2ac6H8uGn2USlEi78_nX3QWvpxYt4XdYE,9583
 parsl/dataflow/rundirs.py,sha256=XKmBZpBEIsGACBhYOkbbs2e5edC0pQegJcSlk4FWeag,1154
@@ -85,7 +85,7 @@ parsl/executors/high_throughput/interchange.py,sha256=tX_EvQf7WkSKMJG-TNmA-WADjh
 parsl/executors/high_throughput/manager_record.py,sha256=T8-JVMfDJU6SJfzJRooD0mO8AHGMXlcn3PBOM0m_vng,366
 parsl/executors/high_throughput/monitoring_info.py,sha256=3gQpwQjjNDEBz0cQqJZB6hRiwLiWwXs83zkQDmbOwxY,297
 parsl/executors/high_throughput/probe.py,sha256=lvnuf-vBv57tHvFh-J51F9sDYBES7jCgs6KYgWvmKRs,2749
-parsl/executors/high_throughput/process_worker_pool.py,sha256=1DZKDkGaO1R2tognVbI6Q35xWN7SbeyfTeG8x1pbM0I,34287
+parsl/executors/high_throughput/process_worker_pool.py,sha256=l0l5F3mpJ60idMCN-d1AbdaogmOtO5eO3uGWogspNXg,34070
 parsl/executors/high_throughput/zmq_pipes.py,sha256=88VJz9QejOCQ_yyhaO5C1uQuDYZTovYEcnKn15WxHSU,6103
 parsl/executors/radical/__init__.py,sha256=CKbtV2numw5QvgIBq1htMUrt9TqDCIC2zifyf2svTNU,186
 parsl/executors/radical/executor.py,sha256=ZYycq58jXlBlhmIO1355JCK1xIJHkspiy62NN1XiMYQ,20729
@@ -182,7 +182,7 @@ parsl/providers/pbspro/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3
 parsl/providers/pbspro/pbspro.py,sha256=zXsb45LhgCkLEwwKXjdjsm2jv884j1fXHJ2hky9auD0,7789
 parsl/providers/pbspro/template.py,sha256=ozMbrx0HNsLnSoWbkZhy-55yJoTX5gpdRrDuVn6TFWA,369
 parsl/providers/slurm/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-parsl/providers/slurm/slurm.py,sha256=lx8wzZSC4bMDgG_ppSXwrnYK93TStdLBYyt1FUvbuhQ,12270
+parsl/providers/slurm/slurm.py,sha256=qHTNI5crS90PzdcoDu_lzDrGrYNss8yY_1XuWU5S2bc,12330
 parsl/providers/slurm/template.py,sha256=cc-3l5wnThEWfqzpniPgi3FP6934Ni05UZ9r0A1RA8s,369
 parsl/providers/torque/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 parsl/providers/torque/template.py,sha256=4qfc2gmlEhRCAD7erFDOs4prJQ43I8s4E8DSUSVQx3A,358
@@ -351,9 +351,9 @@ parsl/tests/test_providers/test_local_provider.py,sha256=G6Fuko22SvAtD7xhfQv8k_8
 parsl/tests/test_python_apps/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 parsl/tests/test_python_apps/test_arg_input_types.py,sha256=JXpfHiu8lr9BN6u1OzqFvGwBhxzsGTPMewHx6Wdo-HI,670
 parsl/tests/test_python_apps/test_basic.py,sha256=lFqh4ugePbp_FRiHGUXxzV34iS7l8C5UkxTHuLcpnYs,855
-parsl/tests/test_python_apps/test_dep_standard_futures.py,sha256=L3mUPmcrpC6QhZgHQbmnZIPE5JRmAA0JSMEmRQo_oOA,639
+parsl/tests/test_python_apps/test_dep_standard_futures.py,sha256=BloeaYBci0jS5al2d8Eqe3OfZ1tvolA5ZflOBQPR9Wo,859
 parsl/tests/test_python_apps/test_dependencies.py,sha256=IRiTI_lPoWBSFSFnaBlE6Bv08PKEaf-qj5dfqO2RjT0,272
-parsl/tests/test_python_apps/test_depfail_propagation.py,sha256=Jukzl6P5Be4I7VpZw2SoD2cwY_Yd0oU_THKl8wswx7k,1316
+parsl/tests/test_python_apps/test_depfail_propagation.py,sha256=3q3HlVWrOixFtXWBvR_ypKtbdAHAJcKndXQ5drwrBQU,1488
 parsl/tests/test_python_apps/test_fail.py,sha256=0Gld8LS6NB0Io1bU82vVR73twkuL5nW0ifKbIUcsJcw,1671
 parsl/tests/test_python_apps/test_fibonacci_iterative.py,sha256=ly2s5HuB9R53Z2FM_zy0WWdOk01iVhgcwSpQyK6ErIY,573
 parsl/tests/test_python_apps/test_fibonacci_recursive.py,sha256=q7LMFcu_pJSNPdz8iY0UiRoIweEWIBGwMjQffHWAuDc,592
@@ -413,12 +413,12 @@ parsl/tests/test_threads/test_configs.py,sha256=QA9YjIMAtZ2jmkfOWqBzEfzQQcFVCDiz
 parsl/tests/test_threads/test_lazy_errors.py,sha256=nGhYfCMHFZYSy6YJ4gnAmiLl9SfYs0WVnuvj8DXQ9bw,560
 parsl/usage_tracking/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 parsl/usage_tracking/usage.py,sha256=TEuAIm_U_G2ojZxvd0bbVa6gZlU61_mVRa2yJC9mGiI,7555
-parsl-2023.12.11.data/scripts/exec_parsl_function.py,sha256=NtWNeBvRqksej38eRPw8zPBJ1CeW6vgaitve0tfz_qc,7801
-parsl-2023.12.11.data/scripts/parsl_coprocess.py,sha256=kzX_1RI3V2KMKs6L-il4I1qkLNVodDKFXN_1FHB9fmM,6031
-parsl-2023.12.11.data/scripts/process_worker_pool.py,sha256=1GAfG53wE2uxq-3TiX2ur4A6JUcZbT-ifyug-FTKbv8,34273
-parsl-2023.12.11.dist-info/LICENSE,sha256=tAkwu8-AdEyGxGoSvJ2gVmQdcicWw3j1ZZueVV74M-E,11357
-parsl-2023.12.11.dist-info/METADATA,sha256=-fGjlELlK1VkSk3_6O8KlISe9ASmoUwZdWENYIhdGYw,3818
-parsl-2023.12.11.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-parsl-2023.12.11.dist-info/entry_points.txt,sha256=XqnsWDYoEcLbsMcpnYGKLEnSBmaIe1YoM5YsBdJG2tI,176
-parsl-2023.12.11.dist-info/top_level.txt,sha256=PIheYoUFQtF2icLsgOykgU-Cjuwr2Oi6On2jo5RYgRM,6
-parsl-2023.12.11.dist-info/RECORD,,
+parsl-2023.12.25.data/scripts/exec_parsl_function.py,sha256=NtWNeBvRqksej38eRPw8zPBJ1CeW6vgaitve0tfz_qc,7801
+parsl-2023.12.25.data/scripts/parsl_coprocess.py,sha256=kzX_1RI3V2KMKs6L-il4I1qkLNVodDKFXN_1FHB9fmM,6031
+parsl-2023.12.25.data/scripts/process_worker_pool.py,sha256=ytz3F8ZYeBr8tFqSRv2O9eZGdsID7oZRulBmmQmZaV8,34056
+parsl-2023.12.25.dist-info/LICENSE,sha256=tAkwu8-AdEyGxGoSvJ2gVmQdcicWw3j1ZZueVV74M-E,11357
+parsl-2023.12.25.dist-info/METADATA,sha256=vri9mnQiuUfyLPY1aPnU1F6V12aehbagmSqm6wEdi1s,3818
+parsl-2023.12.25.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+parsl-2023.12.25.dist-info/entry_points.txt,sha256=XqnsWDYoEcLbsMcpnYGKLEnSBmaIe1YoM5YsBdJG2tI,176
+parsl-2023.12.25.dist-info/top_level.txt,sha256=PIheYoUFQtF2icLsgOykgU-Cjuwr2Oi6On2jo5RYgRM,6
+parsl-2023.12.25.dist-info/RECORD,,