parsl 2025.8.4__py3-none-any.whl → 2025.11.10__py3-none-any.whl

parsl/tests/test_checkpointing/test_python_checkpoint_1.py

@@ -1,4 +1,5 @@
 import os
+from pathlib import Path
 
 import pytest
 
@@ -20,12 +21,14 @@ def uuid_app():
 
 
 @pytest.mark.local
-def test_initial_checkpoint_write():
+def test_initial_checkpoint_write() -> None:
     """1. Launch a few apps and write the checkpoint once a few have completed
     """
     uuid_app().result()
 
-    cpt_dir = parsl.dfk().checkpoint()
+    parsl.dfk().checkpoint()
 
-    cptpath = cpt_dir + '/tasks.pkl'
+    cpt_dir = Path(parsl.dfk().run_dir) / 'checkpoint'
+
+    cptpath = cpt_dir / 'tasks.pkl'
     assert os.path.exists(cptpath), f"Tasks checkpoint missing: {cptpath}"

parsl/tests/test_checkpointing/test_regression_233.py

@@ -5,7 +5,6 @@ from parsl.dataflow.dflow import DataFlowKernel
 
 
 def run_checkpointed(checkpoints):
-    # set_stream_logger()
     from parsl.tests.configs.local_threads_checkpoint_task_exit import config
     config.checkpoint_files = checkpoints
     dfk = DataFlowKernel(config=config)

parsl/tests/test_curvezmq.py

@@ -296,48 +296,6 @@ def test_server_context_destroy(server_ctx: curvezmq.ServerContext, encrypted: b
         assert not server_ctx.auth_thread.pipe
 
 
-@pytest.mark.local
-@pytest.mark.parametrize("encrypted", (True, False), indirect=True)
-def test_client_context_recreate(client_ctx: curvezmq.ClientContext):
-    hidden_ctx = client_ctx._ctx
-    sock = client_ctx.socket(zmq.REQ)
-
-    assert not sock.closed
-    assert not client_ctx.closed
-
-    client_ctx.recreate()
-
-    assert sock.closed
-    assert not client_ctx.closed
-    assert hidden_ctx != client_ctx._ctx
-    assert hidden_ctx.closed
-
-
-@pytest.mark.local
-@pytest.mark.parametrize("encrypted", (True, False), indirect=True)
-def test_server_context_recreate(server_ctx: curvezmq.ServerContext, encrypted: bool):
-    hidden_ctx = server_ctx._ctx
-    sock = server_ctx.socket(zmq.REP)
-
-    assert not sock.closed
-    assert not server_ctx.closed
-    if encrypted:
-        assert server_ctx.auth_thread
-        auth_thread = server_ctx.auth_thread
-        assert auth_thread.pipe
-
-    server_ctx.recreate()
-
-    assert sock.closed
-    assert not server_ctx.closed
-    assert hidden_ctx.closed
-    assert hidden_ctx != server_ctx._ctx
-    if encrypted:
-        assert server_ctx.auth_thread
-        assert auth_thread != server_ctx.auth_thread
-        assert server_ctx.auth_thread.pipe
-
-
 @pytest.mark.local
 @pytest.mark.parametrize("encrypted", (True, False), indirect=True)
 def test_connection(

parsl/tests/test_execute_task.py

@@ -3,7 +3,7 @@ import os
 import pytest
 
 from parsl.executors.execute_task import execute_task
-from parsl.serialize.facade import pack_res_spec_apply_message
+from parsl.serialize.facade import pack_apply_message
 
 
 def addemup(*args: int, name: str = "apples"):
@@ -15,15 +15,6 @@ def addemup(*args: int, name: str = "apples"):
 def test_execute_task():
     args = (1, 2, 3)
     kwargs = {"name": "boots"}
-    buff = pack_res_spec_apply_message(addemup, args, kwargs, {})
+    buff = pack_apply_message(addemup, args, kwargs)
     res = execute_task(buff)
     assert res == addemup(*args, **kwargs)
-
-
-@pytest.mark.local
-def test_execute_task_resource_spec():
-    resource_spec = {"num_nodes": 2, "ranks_per_node": 2, "num_ranks": 4}
-    buff = pack_res_spec_apply_message(addemup, (1, 2), {}, resource_spec)
-    execute_task(buff)
-    for key, val in resource_spec.items():
-        assert os.environ[f"PARSL_{key.upper()}"] == str(val)

parsl/tests/test_htex/test_command_concurrency_regression_1321.py

@@ -0,0 +1,54 @@
+import time
+from threading import Event, Thread
+
+import pytest
+
+import parsl
+from parsl.tests.configs.htex_local import fresh_config as local_config
+
+N_THREADS = 50
+DURATION_S = 10
+
+
+@pytest.mark.local
+def test_concurrency_blast():
+    """Blast interchange command channel from many threads.
+    """
+
+    cc = parsl.dfk().executors['htex_local'].command_client
+
+    threads = []
+
+    ok_so_far = True
+
+    for _ in range(N_THREADS):
+
+        # This event will be set if the thread reaches the end of its body.
+        event = Event()
+
+        thread = Thread(target=blast, args=(cc, event))
+        threads.append((thread, event))
+
+    for thread, event in threads:
+        thread.start()
+
+    for thread, event in threads:
+        thread.join()
+        if not event.is_set():
+            ok_so_far = False
+
+    assert ok_so_far, "at least one thread did not exit normally"
+
+
+def blast(cc, e):
+    target_end = time.monotonic() + DURATION_S
+
+    while time.monotonic() < target_end:
+        cc.run("WORKERS")
+        cc.run("MANGERs_PACKAGES")
+        cc.run("CONNECTED_BLOCKS")
+        cc.run("WORKER_BINDS")
+
+    # If any of the preceeding cc.run calls raises an exception, the thread
+    # will not set its successful completion event.
+    e.set()

parsl/tests/test_htex/test_htex.py

@@ -7,6 +7,7 @@ from unittest import mock
 import pytest
 
 from parsl import HighThroughputExecutor, curvezmq
+from parsl.serialize.facade import pack_apply_message, unpack_apply_message
 
 _MOCK_BASE = "parsl.executors.high_throughput.executor"
 
@@ -19,11 +20,16 @@ def encrypted(request: pytest.FixtureRequest):
 
 
 @pytest.fixture
-def htex(encrypted: bool):
+def htex(encrypted: bool, tmpd_cwd):
     htex = HighThroughputExecutor(encrypted=encrypted)
+    htex.max_workers_per_node = 1
+    htex.run_dir = tmpd_cwd
+    htex.provider.script_dir = tmpd_cwd
 
     yield htex
 
+    if hasattr(htex, "outgoing_q"):
+        htex.scale_in(blocks=1000)
     htex.shutdown()
 
 
@@ -146,3 +152,32 @@ def test_htex_interchange_launch_cmd(cmd: Optional[Sequence[str]]):
     else:
         htex = HighThroughputExecutor()
         assert htex.interchange_launch_cmd == ["interchange.py"]
+
+
+def dyn_exec(buf, *vec_y):
+    f, a, _ = unpack_apply_message(buf)
+    custom_args = [a, vec_y]
+    return f(*custom_args)
+
+
+@pytest.mark.local
+def test_worker_dynamic_import(htex: HighThroughputExecutor):
+    def _dot_prod(vec_x, vec_y):
+        return sum(x * y for x, y in zip(vec_x, vec_y))
+
+    htex.start()
+    htex.scale_out_facade(1)
+
+    num_array = tuple(range(10))
+
+    fn_buf = pack_apply_message(_dot_prod, num_array, {})
+    ctxt = {
+        "task_executor": {
+            "f": f"{dyn_exec.__module__}.{dyn_exec.__name__}",
+            "a": num_array,  # prove "custom" dyn_exec
+        }
+    }
+    val = htex.submit_payload(ctxt, fn_buf).result()
+    exp_val = _dot_prod(num_array, num_array)
+
+    assert val == exp_val

parsl/tests/test_htex/test_interchange_exit_bad_registration.py

@@ -1,4 +1,3 @@
-import json
 import logging
 import os
 import pickle
@@ -50,7 +49,8 @@ def test_exit_with_bad_registration(tmpd_cwd, try_assert):
                           "logging_level": logging.DEBUG,
                           "cert_dir": None,
                           "manager_selector": RandomManagerSelector(),
-                          "run_id": "test"
+                          "run_id": "test",
+                          "_check_python_mismatch": True,
                           }
 
     config_pickle = pickle.dumps(interchange_config)

parsl/tests/test_htex/test_priority_queue.py

@@ -1,3 +1,5 @@
+from random import randint
+
 import pytest
 
 import parsl
@@ -16,7 +18,7 @@ def fake_task(parsl_resource_specification=None):
 
 
 @pytest.mark.local
-def test_priority_queue():
+def test_priority_queue(try_assert):
     provider = LocalProvider(
         init_blocks=0,
         max_blocks=0,
@@ -28,6 +30,7 @@ def test_priority_queue():
         max_workers_per_node=1,
         manager_selector=RandomManagerSelector(),
         provider=provider,
+        worker_debug=True,  # needed to instrospect interchange logs
     )
 
     config = Config(
@@ -40,12 +43,32 @@ def test_priority_queue():
         futures = {}
 
         # Submit tasks with mixed priorities
-        # Priorities: [10, 10, 5, 5, 1, 1] to test fallback behavior
-        for i, priority in enumerate([10, 10, 5, 5, 1, 1]):
+        # Test fallback behavior with a guaranteed-unsorted priorities
+        priorities = [randint(2, 9) for _ in range(randint(1, 10))]
+        priorities.insert(0, 10)
+        priorities.extend((1, 10, 1))
+        for i, priority in enumerate(priorities):
             spec = {'priority': priority}
             futures[(priority, i)] = fake_task(parsl_resource_specification=spec)
 
+        # wait for the interchange to have received all tasks
+        # (which happens asynchronously to the main thread, and is otherwise
+        # a race condition which can cause this test to fail)
+
+        n = len(priorities)
+
+        def interchange_logs_task_count():
+            with open(htex.worker_logdir + "/interchange.log", "r") as f:
+                lines = f.readlines()
+                for line in lines:
+                    if f"Put task {n} onto pending_task_queue" in line:
+                        return True
+            return False
+
+        try_assert(interchange_logs_task_count)
+
         provider.max_blocks = 1
+        htex.scale_out_facade(1)  # don't wait for the JSP to catch up
 
         # Wait for completion
         results = {

parsl/tests/test_htex/test_zmq_binding.py

@@ -29,7 +29,8 @@ def make_interchange(*,
                        logging_level=logging.INFO,
                        manager_selector=RandomManagerSelector(),
                        poll_period=10,
-                       run_id="test_run_id")
+                       run_id="test_run_id",
+                       _check_python_mismatch=True)
 
 
 @pytest.fixture

parsl/tests/test_mpi_apps/test_mpi_scheduler.py

@@ -1,4 +1,3 @@
-import logging
 import os
 import pickle
 from unittest import mock
@@ -10,7 +9,9 @@ from parsl.executors.high_throughput.mpi_resource_management import (
     TaskScheduler,
 )
 from parsl.multiprocessing import SpawnContext
-from parsl.serialize import pack_res_spec_apply_message, unpack_res_spec_apply_message
+from parsl.serialize import pack_apply_message
+
+mock_task_buffer = pack_apply_message("func", "args", "kwargs")
 
 
 @pytest.fixture(autouse=True)
@@ -42,12 +43,8 @@ def test_MPISched_put_task():
     assert len(scheduler.available_nodes) == 8
     assert scheduler._free_node_counter.value == 8
 
-    mock_task_buffer = pack_res_spec_apply_message("func",
-                                                   "args",
-                                                   "kwargs",
-                                                   resource_specification={"num_nodes": 2,
-                                                                           "ranks_per_node": 2})
-    task_package = {"task_id": 1, "buffer": mock_task_buffer}
+    ctxt = {"resource_spec": {"num_nodes": 2, "ranks_per_node": 2}}
+    task_package = {"task_id": 1, "buffer": mock_task_buffer, "context": ctxt}
     scheduler.put_task(task_package)
 
     assert scheduler._free_node_counter.value == 6
@@ -82,21 +79,17 @@ def test_MPISched_roundtrip():
     assert scheduler.available_nodes
     assert len(scheduler.available_nodes) == 8
 
-    for round in range(1, 9):
+    for trip in range(1, 9):
         assert scheduler._free_node_counter.value == 8
 
-        mock_task_buffer = pack_res_spec_apply_message("func",
-                                                       "args",
-                                                       "kwargs",
-                                                       resource_specification={"num_nodes": round,
-                                                                               "ranks_per_node": 2})
-        task_package = {"task_id": round, "buffer": mock_task_buffer}
+        ctxt = {"resource_spec": {"num_nodes": trip, "ranks_per_node": 2}}
+        task_package = {"task_id": trip, "buffer": mock_task_buffer, "context": ctxt}
         scheduler.put_task(task_package)
 
-        assert scheduler._free_node_counter.value == 8 - round
+        assert scheduler._free_node_counter.value == 8 - trip
 
         # Pop in a mock result
-        result_pkl = pickle.dumps({"task_id": round, "type": "result", "buffer": "RESULT BUF"})
+        result_pkl = pickle.dumps({"task_id": trip, "type": "result", "buffer": "RESULT BUF"})
         result_q.put(result_pkl)
 
         got_result = scheduler.get_result(True, 1)
@@ -114,27 +107,15 @@ def test_MPISched_contention():
 
     assert scheduler._free_node_counter.value == 8
 
-    mock_task_buffer = pack_res_spec_apply_message("func",
-                                                   "args",
-                                                   "kwargs",
-                                                   resource_specification={
-                                                       "num_nodes": 8,
-                                                       "ranks_per_node": 2
-                                                   })
-    task_package = {"task_id": 1, "buffer": mock_task_buffer}
+    ctxt_1 = {"resource_spec": {"num_nodes": 8, "ranks_per_node": 2}}
+    task_package = {"task_id": 1, "buffer": mock_task_buffer, "context": ctxt_1}
     scheduler.put_task(task_package)
 
     assert scheduler._free_node_counter.value == 0
     assert scheduler._backlog_queue.empty()
 
-    mock_task_buffer = pack_res_spec_apply_message("func",
-                                                   "args",
-                                                   "kwargs",
-                                                   resource_specification={
-                                                       "num_nodes": 8,
-                                                       "ranks_per_node": 2
-                                                   })
-    task_package = {"task_id": 2, "buffer": mock_task_buffer}
+    ctxt_2 = {"resource_spec": {"num_nodes": 8, "ranks_per_node": 2}}
+    task_package = {"task_id": 2, "buffer": mock_task_buffer, "context": ctxt_2}
     scheduler.put_task(task_package)
 
     # Second task should now be in the backlog_queue
@@ -143,8 +124,7 @@ def test_MPISched_contention():
     # Confirm that the first task is available and has all 8 nodes provisioned
     task_on_worker_side = task_q.get()
     assert task_on_worker_side['task_id'] == 1
-    _, _, _, resource_spec = unpack_res_spec_apply_message(task_on_worker_side['buffer'])
-    assert len(resource_spec['MPI_NODELIST'].split(',')) == 8
+    assert len(ctxt_1["resource_spec"]["MPI_NODELIST"].split(",")) == 8
     assert task_q.empty()  # Confirm that task 2 is not yet scheduled
 
     # Simulate worker returning result and the scheduler picking up result
@@ -159,8 +139,7 @@ def test_MPISched_contention():
     # Pop in a mock result
     task_on_worker_side = task_q.get()
     assert task_on_worker_side['task_id'] == 2
-    _, _, _, resource_spec = unpack_res_spec_apply_message(task_on_worker_side['buffer'])
-    assert len(resource_spec['MPI_NODELIST'].split(',')) == 8
+    assert len(ctxt_2["resource_spec"]["MPI_NODELIST"].split(",")) == 8
 
 
 @pytest.mark.local
@@ -178,11 +157,7 @@ def test_hashable_backlog_queue():
     assert scheduler._free_node_counter.value == 8
 
     for i in range(3):
-        mock_task_buffer = pack_res_spec_apply_message("func", "args", "kwargs",
-                                                       resource_specification={
-                                                           "num_nodes": 8,
-                                                           "ranks_per_node": 2
-                                                       })
-        task_package = {"task_id": i, "buffer": mock_task_buffer}
+        ctxt = {"resource_spec": {"num_nodes": 8, "ranks_per_node": 2}}
+        task_package = {"task_id": i, "buffer": mock_task_buffer, "context": ctxt}
         scheduler.put_task(task_package)
     assert scheduler._backlog_queue.qsize() == 2, "Expected 2 backlogged tasks"

parsl/tests/test_python_apps/test_basic.py

@@ -14,12 +14,6 @@ def import_square(x):
     return math.pow(x, 2)
 
 
-@python_app
-def custom_exception():
-    from globus_sdk import GlobusError
-    raise GlobusError('foobar')
-
-
 def test_simple(n=2):
     x = double(n)
     assert x.result() == n * 2
@@ -38,11 +32,3 @@ def test_parallel_for(n):
 
     for i in d:
         assert d[i].result() == 2 * i
-
-
-def test_custom_exception():
-    from globus_sdk import GlobusError
-
-    x = custom_exception()
-    with pytest.raises(GlobusError):
-        x.result()

parsl/tests/test_python_apps/test_depfail_propagation.py

@@ -1,5 +1,7 @@
+import parsl
 from parsl import python_app
 from parsl.dataflow.errors import DependencyError
+from parsl.dataflow.states import States
 
 
 @python_app
@@ -14,6 +16,7 @@ def depends(parent):
 
 def test_depfail_once():
     """Test the simplest dependency failure case"""
+    start_dep_fail_count = parsl.dfk().task_state_counts[States.dep_fail]
     f1 = fails()
     f2 = depends(f1)
 
@@ -25,9 +28,12 @@ def test_depfail_once():
     # in the DependencyError message
     assert ("task " + str(f1.task_record['id'])) in str(f2.exception())
 
+    assert parsl.dfk().task_state_counts[States.dep_fail] == start_dep_fail_count + 1
+
 
 def test_depfail_chain():
     """Test that dependency failures chain"""
+    start_dep_fail_count = parsl.dfk().task_state_counts[States.dep_fail]
     f1 = fails()
     f2 = depends(f1)
     f3 = depends(f2)
@@ -39,11 +45,13 @@ def test_depfail_chain():
     assert isinstance(f3.exception(), DependencyError)
     assert isinstance(f4.exception(), DependencyError)
 
+    assert parsl.dfk().task_state_counts[States.dep_fail] == start_dep_fail_count + 3
+
 
 def test_depfail_branches():
     """Test that dependency failures propagate in the
     presence of multiple downstream tasks."""
-
+    start_dep_fail_count = parsl.dfk().task_state_counts[States.dep_fail]
     f1 = fails()
     f2 = depends(f1)
     f3 = depends(f1)
@@ -52,3 +60,5 @@ def test_depfail_branches():
     assert not isinstance(f1.exception(), DependencyError)
     assert isinstance(f2.exception(), DependencyError)
     assert isinstance(f3.exception(), DependencyError)
+
+    assert parsl.dfk().task_state_counts[States.dep_fail] == start_dep_fail_count + 2

parsl/tests/test_python_apps/test_exception.py

@@ -0,0 +1,19 @@
+import pytest
+
+from parsl.app.app import python_app
+
+
+class CustomException(Exception):
+    pass
+
+
+@python_app
+def custom_exception():
+    from parsl.tests.test_python_apps.test_exception import CustomException
+    raise CustomException('foobar')
+
+
+def test_custom_exception():
+    x = custom_exception()
+    with pytest.raises(CustomException):
+        x.result()

parsl/tests/test_python_apps/test_garbage_collect.py

@@ -27,10 +27,5 @@ def test_garbage_collect():
 
     evt.set()
     assert x.result() == 10 * 4
-    if parsl.dfk().checkpoint_mode is not None:
-        # We explicit call checkpoint if checkpoint_mode is enabled covering
-        # cases like manual/periodic where checkpointing may be deferred.
-        parsl.dfk().checkpoint()
-
-    time.sleep(0.01)  # Give enough time for task wipes to work
+    time.sleep(0.01)  # Give enough time for task wipes to work - see issue #1279
     assert x.tid not in parsl.dfk().tasks, "Task record should be wiped after task completion"

parsl/tests/test_python_apps/test_memoize_2.py

@@ -4,7 +4,17 @@ import pytest
 
 import parsl
 from parsl.app.app import python_app
-from parsl.tests.configs.local_threads_no_cache import fresh_config as local_config
+from parsl.config import Config
+from parsl.executors.threads import ThreadPoolExecutor
+
+
+def local_config():
+    return Config(
+        executors=[
+            ThreadPoolExecutor(max_threads=4),
+        ],
+        app_cache=False
+    )
 
 
 @python_app

parsl/tests/test_python_apps/test_memoize_exception.py

@@ -0,0 +1,41 @@
+import parsl
+from parsl.app.app import python_app
+
+
+@python_app(cache=True)
+def raise_exception_cache(x, cache=True):
+    raise RuntimeError("exception from raise_exception_cache")
+
+
+@python_app(cache=False)
+def raise_exception_nocache(x, cache=True):
+    raise RuntimeError("exception from raise_exception_nocache")
+
+
+def test_python_memoization(n=2):
+    """Test Python memoization of exceptions, with cache=True"""
+    x = raise_exception_cache(0)
+
+    # wait for x to be done
+    x.exception()
+
+    for i in range(0, n):
+        fut = raise_exception_cache(0)
+
+        # check that we get back the same exception object, rather than
+        # a new one from a second invocation of raise_exception().
+        assert fut.exception() is x.exception(), "Memoized exception should have been memoized"
+
+
+def test_python_no_memoization(n=2):
+    """Test Python non-memoization of exceptions, with cache=False"""
+    x = raise_exception_nocache(0)
+
+    # wait for x to be done
+    x.exception()
+
+    for i in range(0, n):
+        fut = raise_exception_nocache(0)
+
+        # check that we get back a different exception object each time
+        assert fut.exception() is not x.exception(), "Memoized exception should have been memoized"

parsl/tests/test_regression/test_3874.py

@@ -0,0 +1,47 @@
+import shutil
+
+import pytest
+
+import parsl
+from parsl.app.app import python_app
+from parsl.config import Config
+from parsl.executors import HighThroughputExecutor
+
+
+@python_app
+def noop():
+    pass
+
+
+@pytest.mark.local
+def test_regression_3874(tmpd_cwd_session):
+    # HTEX run 1
+
+    rundir_1 = str(tmpd_cwd_session / "1")
+
+    config = Config(executors=[HighThroughputExecutor()], strategy_period=0.5)
+    config.run_dir = rundir_1
+
+    with parsl.load(config):
+        noop().result()
+
+    # It is necessary to delete this rundir to exercise the bug. Otherwise,
+    # the next run will be able to continue looking at this directory - the
+    # bug manifests when it cannot.
+
+    shutil.rmtree(rundir_1)
+
+    # HTEX run 2
+    # In the case of issue 3874, this run hangs (rather than failing) as the
+    # JobStatusPoller fails to collect status of all of its managed tasks
+    # every iteration, without converging towards failure.
+
+    rundir_2 = str(tmpd_cwd_session / "2")
+
+    config = Config(executors=[HighThroughputExecutor()], strategy_period=0.5)
+    config.run_dir = rundir_2
+
+    with parsl.load(config):
+        noop().result()
+
+    shutil.rmtree(rundir_2)

parsl/tests/test_scaling/test_regression_3696_oscillation.py

@@ -51,6 +51,7 @@ def test_htex_strategy_does_not_oscillate(ns):
     executor.outstanding = lambda: n_tasks
     executor.status_facade = statuses
     executor.workers_per_node = n_workers
+    executor.bad_state_is_set = False
 
     provider.parallelism = 1
     provider.init_blocks = 0

parsl/tests/test_staging/test_staging_globus.py

@@ -3,9 +3,9 @@ import pytest
 import parsl
 from parsl.app.app import python_app
 from parsl.data_provider.files import File
-from parsl.tests.configs.local_threads_globus import config, remote_writeable
+from parsl.tests.configs.local_threads_globus import fresh_config, remote_writeable
 
-local_config = config
+local_config = fresh_config
 
 
 @python_app