parsl 2024.3.18__py3-none-any.whl → 2025.1.13__py3-none-any.whl

parsl/tests/test_checkpointing/test_periodic.py

@@ -1,22 +1,12 @@
-import argparse
-import time
-
 import pytest
 
 import parsl
 from parsl.app.app import python_app
-from parsl.tests.configs.local_threads_checkpoint_periodic import config
+from parsl.tests.configs.local_threads_checkpoint_periodic import fresh_config
 
 
 def local_setup():
-    global dfk
-    dfk = parsl.load(config)
-
-
-def local_teardown():
-    # explicit clear without dfk.cleanup here, because the
-    # test does that already
-    parsl.clear()
+    parsl.load(fresh_config())
 
 
 @python_app(cache=True)
@@ -27,40 +17,32 @@ def slow_double(x, sleep_dur=1):
 
 
 def tstamp_to_seconds(line):
-    print("Parsing line: ", line)
     f = line.partition(" ")[0]
     return float(f)
 
 
 @pytest.mark.local
-def test_periodic(n=4):
+def test_periodic():
     """Test checkpointing with task_periodic behavior
     """
-
-    d = {}
-
-    print("Launching : ", n)
-    for i in range(0, n):
-        d[i] = slow_double(i)
-    print("Done launching")
-
-    for i in range(0, n):
-        d[i].result()
-    print("Done sleeping")
-
-    time.sleep(16)
-    dfk.cleanup()
+    h, m, s = map(int, parsl.dfk().config.checkpoint_period.split(":"))
+    assert h == 0, "Verify test setup"
+    assert m == 0, "Verify test setup"
+    assert s > 0, "Verify test setup"
+    sleep_for = s + 1
+    with parsl.dfk():
+        futs = [slow_double(sleep_for) for _ in range(4)]
+        [f.result() for f in futs]
+        run_dir = parsl.dfk().run_dir
 
     # Here we will check if the loglines came back with 5 seconds deltas
-    print("Rundir: ", dfk.run_dir)
-
-    with open("{}/parsl.log".format(dfk.run_dir), 'r') as f:
+    with open("{}/parsl.log".format(run_dir)) as f:
         log_lines = f.readlines()
-        expected_msg = " Done checkpointing"
-        expected_msg2 = " No tasks checkpointed in this pass"
-
-        lines = [line for line in log_lines if expected_msg in line or expected_msg2 in line]
-        assert len(lines) >= 3, "Insufficient checkpoint lines in logfile"
-        deltas = [tstamp_to_seconds(line) for line in lines]
-        assert deltas[1] - deltas[0] < 5.5, "Delta between checkpoints exceeded period"
-        assert deltas[2] - deltas[1] < 5.5, "Delta between checkpoints exceeded period"
+    expected_msg = " Done checkpointing"
+    expected_msg2 = " No tasks checkpointed in this pass"
+
+    lines = [line for line in log_lines if expected_msg in line or expected_msg2 in line]
+    assert len(lines) >= 3, "Insufficient checkpoint lines in logfile"
+    deltas = [tstamp_to_seconds(line) for line in lines]
+    assert deltas[1] - deltas[0] < 5.5, "Delta between checkpoints exceeded period"
+    assert deltas[2] - deltas[1] < 5.5, "Delta between checkpoints exceeded period"

parsl/tests/test_checkpointing/test_python_checkpoint_1.py

@@ -1,4 +1,5 @@
 import os
+
 import pytest
 
 import parsl

parsl/tests/test_checkpointing/test_python_checkpoint_2.py

@@ -1,9 +1,10 @@
 import contextlib
 import os
+
 import pytest
+
 import parsl
 from parsl import python_app
-
 from parsl.tests.configs.local_threads_checkpoint import fresh_config
 
 
@@ -19,7 +20,6 @@ def parsl_configured(run_dir, **kw):
     yield dfk
 
     parsl.dfk().cleanup()
-    parsl.clear()
 
 
 @python_app(cache=True)

parsl/tests/test_checkpointing/test_python_checkpoint_3.py

@@ -14,7 +14,6 @@ def local_setup():
 
 def local_teardown():
     parsl.dfk().cleanup()
-    parsl.clear()
 
 
 @python_app

parsl/tests/test_checkpointing/test_regression_239.py

@@ -2,7 +2,7 @@ import pickle
 
 import pytest
 
-from parsl import python_app, DataFlowKernel
+from parsl import DataFlowKernel, python_app
 from parsl.utils import time_limited_open
 
 

parsl/tests/test_checkpointing/test_task_exit.py

@@ -5,8 +5,8 @@ import pytest
 
 import parsl
 from parsl.app.app import python_app
-from parsl.utils import time_limited_open
 from parsl.tests.configs.local_threads_checkpoint_task_exit import config
+from parsl.utils import time_limited_open
 
 
 def local_setup():
@@ -15,8 +15,7 @@ def local_setup():
 
 
 def local_teardown():
-    parsl.dfk().cleanup
-    parsl.clear()
+    parsl.dfk().cleanup()
 
 
 @python_app(cache=True)

parsl/tests/test_docs/test_from_slides.py

@@ -1,8 +1,10 @@
+import os
+
+import pytest
+
 from parsl.app.app import bash_app, python_app
 from parsl.data_provider.files import File
 
-import os
-
 
 @bash_app
 def echo(message, outputs=[]):
@@ -15,6 +17,7 @@ def cat(inputs=[]):
         return f.readlines()
 
 
+@pytest.mark.staging_required
 def test_slides():
     """Testing code snippet from slides """
 

parsl/tests/test_docs/test_kwargs.py

@@ -1,7 +1,9 @@
 """Functions used to explain kwargs"""
 from pathlib import Path
 
-from parsl import python_app, File
+import pytest
+
+from parsl import File, python_app
 
 
 def test_inputs():
@@ -19,6 +21,7 @@ def test_inputs():
     assert reduce_future.result() == 6
 
 
+@pytest.mark.shared_fs
 def test_outputs(tmpd_cwd):
     @python_app()
     def write_app(message, outputs=()):

parsl/tests/test_docs/test_tutorial_1.py

@@ -1,12 +1,11 @@
 import argparse
-import parsl
 
 import pytest
 
+import parsl
 from parsl.app.app import bash_app
 from parsl.tests.configs.local_threads import config
 
-
 local_config = config
 
 

parsl/tests/test_docs/test_workflow1.py

@@ -1,12 +1,12 @@
 import os
+
 import pytest
-import parsl
 
+import parsl
 from parsl.app.app import bash_app, python_app
 from parsl.data_provider.files import File
 from parsl.tests.configs.local_threads import config
 
-
 # parsl.set_stream_logger()
 
 

parsl/tests/test_docs/test_workflow2.py

@@ -5,7 +5,6 @@ import pytest
 from parsl.app.app import python_app
 from parsl.tests.configs.local_threads import config
 
-
 local_config = config
 
 

parsl/tests/test_error_handling/test_rand_fail.py

@@ -15,8 +15,8 @@ def local_config():
 
 @python_app
 def sleep_fail(sleep_dur, sleep_rand_max, fail_prob, inputs=[]):
-    import time
     import random
+    import time
 
     s = sleep_dur + random.randint(-sleep_rand_max, sleep_rand_max)
 
@@ -144,8 +144,8 @@ def test_deps(numtasks=10):
 
 @python_app
 def sleep_then_fail(sleep_dur=0.1):
-    import time
     import math
+    import time
     time.sleep(sleep_dur)
     math.ceil("Trigger TypeError")
     return 0

parsl/tests/test_error_handling/test_resource_spec.py

@@ -1,9 +1,9 @@
 import parsl
 from parsl.app.app import python_app
-from parsl.executors.errors import UnsupportedFeatureError, ExecutorError
 from parsl.executors import WorkQueueExecutor
-from parsl.executors.high_throughput.mpi_prefix_composer import InvalidResourceSpecification
+from parsl.executors.errors import InvalidResourceSpecification
 from parsl.executors.high_throughput.executor import HighThroughputExecutor
+from parsl.executors.threads import ThreadPoolExecutor
 
 
 @python_app
@@ -25,11 +25,10 @@ def test_resource(n=2):
     try:
         fut.result()
     except InvalidResourceSpecification:
-        assert isinstance(executor, HighThroughputExecutor)
-    except UnsupportedFeatureError:
-        assert not isinstance(executor, WorkQueueExecutor)
-    except Exception as e:
-        assert isinstance(e, ExecutorError)
+        assert (
+            isinstance(executor, HighThroughputExecutor) or
+            isinstance(executor, WorkQueueExecutor) or
+            isinstance(executor, ThreadPoolExecutor))
 
     # Specify resources with wrong types
     # 'cpus' is incorrect, should be 'cores'
@@ -38,8 +37,7 @@ def test_resource(n=2):
     try:
         fut.result()
     except InvalidResourceSpecification:
-        assert isinstance(executor, HighThroughputExecutor)
-    except UnsupportedFeatureError:
-        assert not isinstance(executor, WorkQueueExecutor)
-    except Exception as e:
-        assert isinstance(e, ExecutorError)
+        assert (
+            isinstance(executor, HighThroughputExecutor) or
+            isinstance(executor, WorkQueueExecutor) or
+            isinstance(executor, ThreadPoolExecutor))

parsl/tests/test_error_handling/test_retries.py

@@ -1,8 +1,7 @@
-import argparse
 import os
+
 import pytest
 
-import parsl
 from parsl import bash_app, python_app
 from parsl.tests.configs.local_threads import fresh_config
 
@@ -15,8 +14,8 @@ def local_config():
 
 @python_app
 def sleep_then_fail(inputs=[], sleep_dur=0.1):
-    import time
     import math
+    import time
     time.sleep(sleep_dur)
     math.ceil("Trigger TypeError")
     return 0
@@ -67,8 +66,6 @@ def test_fail_nowait(numtasks=10):
         assert isinstance(
             e, TypeError), "Expected a TypeError, got {}".format(e)
 
-    print("Done")
-
 
 @pytest.mark.local
 def test_fail_delayed(numtasks=10):
@@ -93,19 +90,12 @@ def test_fail_delayed(numtasks=10):
         assert isinstance(
             e, TypeError), "Expected a TypeError, got {}".format(e)
 
-    print("Done")
-
 
 @pytest.mark.local
-def test_retry():
+def test_retry(tmpd_cwd):
     """Test retries via app that succeeds on the Nth retry.
     """
 
-    fname = "retry.out"
-    try:
-        os.remove(fname)
-    except OSError:
-        pass
-    fu = succeed_on_retry(fname)
-
-    fu.result()
+    fpath = tmpd_cwd / "retry.out"
+    sout = str(tmpd_cwd / "stdout")
+    succeed_on_retry(str(fpath), stdout=sout).result()

parsl/tests/test_error_handling/test_retry_handler.py

@@ -1,4 +1,5 @@
 import os
+
 import pytest
 
 import parsl

parsl/tests/test_error_handling/test_retry_handler_failure.py

@@ -1,6 +1,7 @@
-import parsl
 import pytest
 
+import parsl
+
 
 @parsl.python_app
 def always_fails():

parsl/tests/test_error_handling/test_serialization_fail.py

@@ -1,8 +1,8 @@
 import pytest
 
 from parsl import python_app
-from parsl.tests.configs.htex_local import fresh_config
 from parsl.serialize.errors import SerializationError
+from parsl.tests.configs.htex_local import fresh_config
 
 
 def local_config():

parsl/tests/test_error_handling/test_wrap_with_logs.py

@@ -1,4 +1,5 @@
 import logging
+
 import pytest
 
 from parsl.process_loggers import wrap_with_logs

parsl/tests/test_execute_task.py

@@ -0,0 +1,29 @@
+import os
+
+import pytest
+
+from parsl.executors.execute_task import execute_task
+from parsl.serialize.facade import pack_res_spec_apply_message
+
+
+def addemup(*args: int, name: str = "apples"):
+    total = sum(args)
+    return f"{total} {name}"
+
+
+@pytest.mark.local
+def test_execute_task():
+    args = (1, 2, 3)
+    kwargs = {"name": "boots"}
+    buff = pack_res_spec_apply_message(addemup, args, kwargs, {})
+    res = execute_task(buff)
+    assert res == addemup(*args, **kwargs)
+
+
+@pytest.mark.local
+def test_execute_task_resource_spec():
+    resource_spec = {"num_nodes": 2, "ranks_per_node": 2, "num_ranks": 4}
+    buff = pack_res_spec_apply_message(addemup, (1, 2), {}, resource_spec)
+    execute_task(buff)
+    for key, val in resource_spec.items():
+        assert os.environ[f"PARSL_{key.upper()}"] == str(val)

parsl/tests/test_flux.py

@@ -1,5 +1,5 @@
-import os
 import concurrent.futures as cf
+import os
 
 import pytest
 

parsl/tests/test_htex/test_basic.py

@@ -14,7 +14,6 @@ def local_setup():
 
 def local_teardown():
     parsl.dfk().cleanup()
-    parsl.clear()
 
 
 @python_app
@@ -23,6 +22,6 @@ def dummy():
 
 
 @pytest.mark.local
-def test_that_it_fails():
+def test_app():
     x = dummy()
-    x.result()
+    assert x.result() is None

parsl/tests/test_htex/test_block_manager_selector_unit.py

@@ -0,0 +1,20 @@
+import pytest
+
+from parsl.executors.high_throughput.manager_record import ManagerRecord
+from parsl.executors.high_throughput.manager_selector import BlockIdManagerSelector
+
+
+@pytest.mark.local
+def test_sort_managers():
+    ready_managers = {
+        b'manager1': {'block_id': 1},
+        b'manager2': {'block_id': None},
+        b'manager3': {'block_id': 3},
+        b'manager4': {'block_id': 2}
+    }
+
+    manager_list = {b'manager1', b'manager2', b'manager3', b'manager4'}
+    expected_sorted_list = [b'manager2', b'manager1', b'manager4', b'manager3']
+    manager_selector = BlockIdManagerSelector()
+    sorted_managers = manager_selector.sort_managers(ready_managers, manager_list)
+    assert sorted_managers == expected_sorted_list

parsl/tests/test_htex/test_command_client_timeout.py

@@ -0,0 +1,66 @@
+import threading
+import time
+
+import pytest
+import zmq
+
+from parsl import curvezmq
+from parsl.executors.high_throughput.errors import (
+    CommandClientBadError,
+    CommandClientTimeoutError,
+)
+from parsl.executors.high_throughput.zmq_pipes import CommandClient
+
+# Time constant used for timeout tests: various delays and
+# timeouts will be appropriate multiples of this, but the
+# value of T itself should not matter too much as long as
+# it is big enough for zmq connections to happen successfully.
+T = 0.25
+
+
+@pytest.mark.local
+def test_command_not_sent() -> None:
+    """Tests timeout on command send.
+    """
+    # RFC6335 ephemeral port range
+    cc = CommandClient("127.0.0.1", (49152, 65535))
+
+    # cc will now wait for a connection, but we won't do anything to make the
+    # other side of the connection exist, so any command given to cc should
+    # timeout.
+
+    with pytest.raises(CommandClientTimeoutError):
+        cc.run("SOMECOMMAND", timeout_s=T)
+
+    cc.close()
+
+
+@pytest.mark.local
+def test_command_ignored() -> None:
+    """Tests timeout on command response.
+    Tests that we timeout after a response and that the command client
+    sets itself into a bad state.
+
+    This only tests sequential access to the command client, even though
+    htex makes multithreaded use of the command client: see issue #3376 about
+    that lack of thread safety.
+    """
+    # RFC6335 ephemeral port range
+    cc = CommandClient("127.0.0.1", (49152, 65535))
+
+    ic_ctx = curvezmq.ServerContext(None)
+    ic_channel = ic_ctx.socket(zmq.REP)
+    ic_channel.connect(f"tcp://127.0.0.1:{cc.port}")
+
+    with pytest.raises(CommandClientTimeoutError):
+        cc.run("SLOW_COMMAND", timeout_s=T)
+
+    req = ic_channel.recv_pyobj()
+    assert req == "SLOW_COMMAND", "Should have received command on interchange side"
+    assert not cc.ok, "CommandClient should have set itself to bad"
+
+    with pytest.raises(CommandClientBadError):
+        cc.run("ANOTHER_COMMAND")
+
+    cc.close()
+    ic_channel.close()

parsl/tests/test_htex/test_connected_blocks.py

@@ -1,7 +1,8 @@
-import parsl
 import pytest
-from parsl.executors import HighThroughputExecutor
+
+import parsl
 from parsl import Config
+from parsl.executors import HighThroughputExecutor
 from parsl.providers import LocalProvider
 
 

parsl/tests/test_htex/test_cpu_affinity_explicit.py

@@ -1,8 +1,10 @@
 import logging
 import os
-import parsl
-import pytest
 import random
+
+import pytest
+
+import parsl
 from parsl.tests.configs.htex_local import fresh_config
 
 logger = logging.getLogger(__name__)
@@ -18,6 +20,7 @@ def my_affinity():
 
 @pytest.mark.local
 @pytest.mark.multiple_cores_required
+@pytest.mark.skipif('sched_getaffinity' not in dir(os), reason='System does not support sched_setaffinity')
 def test_cpu_affinity_explicit():
     available_cores = os.sched_getaffinity(0)
 
@@ -37,16 +40,9 @@ def test_cpu_affinity_explicit():
     config.executors[0].max_workers_per_node = 1
 
     logger.debug(f"config: {config}")
-    # TODO: is there a `with` style for this, to properly deal with exceptions?
-
-    parsl.load(config)
-    try:
 
+    with parsl.load(config):
         worker_affinity = my_affinity().result()
         logger.debug(f"worker reported this affinity: {worker_affinity}")
         assert len(worker_affinity) == 1
         assert worker_affinity == set((single_core,))
-
-    finally:
-        parsl.dfk().cleanup()
-        parsl.clear()

parsl/tests/test_htex/test_disconnected_blocks.py

@@ -1,11 +1,13 @@
 import logging
-import parsl
+
 import pytest
-from parsl.executors import HighThroughputExecutor
+
+import parsl
 from parsl import Config
-from parsl.providers import LocalProvider
+from parsl.executors import HighThroughputExecutor
 from parsl.executors.errors import BadStateException
-from parsl.jobs.states import JobStatus, JobState
+from parsl.jobs.states import JobState, JobStatus
+from parsl.providers import LocalProvider
 
 
 def local_config():
@@ -26,7 +28,6 @@ def local_config():
                 ),
             )
         ],
-        run_dir="/tmp/test_htex",
         max_idletime=0.5,
         strategy='htex_auto_scale',
     )

parsl/tests/test_htex/test_disconnected_blocks_failing_provider.py

@@ -0,0 +1,71 @@
+import logging
+
+import pytest
+
+import parsl
+from parsl import Config
+from parsl.executors import HighThroughputExecutor
+from parsl.executors.errors import BadStateException
+from parsl.jobs.states import JobState, JobStatus
+from parsl.providers import LocalProvider
+
+
+class FailingProvider(LocalProvider):
+    def submit(*args, **kwargs):
+        raise RuntimeError("Deliberate failure of provider.submit")
+
+
+def local_config():
+    """Config to simulate failing blocks without connecting"""
+    return Config(
+        executors=[
+            HighThroughputExecutor(
+                label="HTEX",
+                heartbeat_period=1,
+                heartbeat_threshold=2,
+                poll_period=100,
+                max_workers_per_node=1,
+                provider=FailingProvider(
+                    init_blocks=0,
+                    max_blocks=2,
+                    min_blocks=0,
+                ),
+            )
+        ],
+        max_idletime=0.5,
+        strategy='htex_auto_scale',
+        strategy_period=0.1
+        # this strategy period needs to be a few times smaller than the
+        # status_polling_interval of FailingProvider, which is 5s at
+        # time of writing
+    )
+
+
+@parsl.python_app
+def double(x):
+    return x * 2
+
+
+@pytest.mark.local
+def test_disconnected_blocks():
+    """Test reporting of blocks that fail to connect from HTEX"""
+    dfk = parsl.dfk()
+    executor = dfk.executors["HTEX"]
+
+    connected_blocks = executor.connected_blocks()
+    assert not connected_blocks, "Expected 0 blocks"
+
+    future = double(5)
+    with pytest.raises(BadStateException):
+        future.result()
+
+    assert isinstance(future.exception(), BadStateException)
+
+    status_dict = executor.status()
+    assert len(status_dict) == 1, "Expected exactly 1 block"
+    for status in status_dict.values():
+        assert isinstance(status, JobStatus)
+        assert status.state == JobState.MISSING
+
+    connected_blocks = executor.connected_blocks()
+    assert connected_blocks == [], "Expected exactly 0 connected blocks"