parsl 2023.7.3__py3-none-any.whl → 2023.7.17__py3-none-any.whl

parsl/tests/test_python_apps/test_pipeline.py

@@ -1,5 +1,7 @@
 import argparse
 
+import pytest
+
 import parsl
 from parsl.app.app import python_app
 from parsl.tests.configs.local_threads import config
@@ -11,28 +13,23 @@ def increment(x):
 
 
 @python_app
-def slow_increment(x, dur):
+def slow_increment(x):
     import time
-    time.sleep(dur)
+    time.sleep(0.001)
     return x + 1
 
 
-def test_increment(depth=2):
-    """Test simple pipeline A->B...->N
-    """
-    futs = {0: 0}
-    for i in range(1, depth):
-        futs[i] = increment(futs[i - 1])
-
-    print([futs[i].result() for i in futs if not isinstance(futs[i], int)])
-
+@pytest.mark.parametrize("depth", (2, 3))
+def test_increment(depth):
+    """Test simple pipeline A->B...->N"""
+    futs = [increment(0)]
+    futs.extend(increment(futs[i - 1]) for i in range(1, depth))
+    assert sum(f.result() for f in futs) == sum(range(1, depth + 1))
 
-def test_increment_slow(depth=2):
-    """Test simple pipeline A->B...->N with delay
-    """
-    futs = {0: 0}
-    for i in range(1, depth):
-        futs[i] = slow_increment(futs[i - 1], 0.5)
 
-    print(futs[i])
-    print([futs[i].result() for i in futs if not isinstance(futs[i], int)])
+@pytest.mark.parametrize("depth", (2, 3))
+def test_increment_slow(depth):
+    """Test simple pipeline A->B...->N with delay"""
+    futs = [slow_increment(0)]
+    futs.extend(slow_increment(futs[i - 1]) for i in range(1, depth))
+    assert sum(f.result() for f in futs) == sum(range(1, depth + 1))

parsl/tests/test_regression/test_1606_wait_for_current_tasks.py

@@ -1,17 +1,42 @@
+import threading
+import time
+
+import pytest
+
 import parsl
+from parsl.tests.configs.local_threads import fresh_config as local_config  # noqa
 
 
 @parsl.python_app
-def slow_app(delay):
-    import time
-    time.sleep(delay)
+def slow_app(evt: threading.Event):
+    evt.wait()
 
 
+@pytest.mark.local
 def test_wait_for_tasks():
-    slow_app(5)
-    slow_app(10)  # This test has a higher task ID, and runs for a longer period
-    slow_app(3)  # This test has a higher task ID, but runs for a shorter period
-    parsl.dfk().wait_for_current_tasks()
-    # the regression reported in #1606 is that wait_for_current_tasks
-    # fails due to tasks being removed from the DFK tasks dict as they
-    # complete, introduced in #1543.
+    """
+    gh#1606 reported that wait_for_current_tasks fails due to tasks being removed
+    from the DFK tasks dict as they complete; bug introduced in #1543.
+    """
+    def test_kernel(may_wait: threading.Event):
+        e1, e2 = threading.Event(), threading.Event()
+
+        # app_slow is in *middle* of internal DFK data structure
+        app_fast1, app_slow, app_fast2 = slow_app(e1), slow_app(e2), slow_app(e1)
+
+        may_wait.set()  # initiated wait in outer test
+        time.sleep(0.01)
+
+        e1.set()
+
+        while not all(f.done() for f in (app_fast1, app_fast2)):
+            time.sleep(0.01)
+
+        e2.set()
+        app_slow.result()
+
+    may_continue = threading.Event()
+    threading.Thread(target=test_kernel, daemon=True, args=(may_continue,)).start()
+
+    may_continue.wait()
+    parsl.dfk().wait_for_current_tasks()  # per sleeps, waits for all 3 tasks

parsl/tests/test_scaling/test_regression_1621.py

@@ -1,33 +1,27 @@
-# this test is intended to ensure that only one block is launched when only
-# one app is invoked. this is a regression test.
+import threading
+
+import pytest
 
-import logging
 import parsl
 from parsl.channels import LocalChannel
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
 from parsl.launchers import SimpleLauncher
 from parsl.providers import LocalProvider
-import pytest
-
-
-logger = logging.getLogger(__name__)
 
 
 @parsl.python_app
 def app():
     import time
-    time.sleep(45)
+    time.sleep(1)
 
 
 class OneShotLocalProvider(LocalProvider):
     def __init__(self, *args, **kwargs):
-        logger.info("OneShotLocalProvider __init__ with MRO: {}".format(type(self).mro()))
         self.recorded_submits = 0
         super().__init__(*args, **kwargs)
 
     def submit(self, *args, **kwargs):
-        logger.info("OneShotLocalProvider submit")
         self.recorded_submits += 1
         return super().submit(*args, **kwargs)
 
@@ -35,32 +29,44 @@ class OneShotLocalProvider(LocalProvider):
 
 
 @pytest.mark.local
-def test_one_block():
-
+def test_one_block(tmpd_cwd):
+    """
+    this test is intended to ensure that only one block is launched when only
+    one app is invoked. this is a regression test.
+    """
     oneshot_provider = OneShotLocalProvider(
-                    channel=LocalChannel(),
-                    init_blocks=0,
-                    min_blocks=0,
-                    max_blocks=10,
-                    launcher=SimpleLauncher(),
-                )
+        channel=LocalChannel(),
+        init_blocks=0,
+        min_blocks=0,
+        max_blocks=10,
+        launcher=SimpleLauncher(),
+    )
 
     config = Config(
         executors=[
             HighThroughputExecutor(
                 label="htex_local",
+                address="127.0.0.1",
                 worker_debug=True,
                 cores_per_worker=1,
                 provider=oneshot_provider,
+                worker_logdir_root=str(tmpd_cwd)
             )
         ],
         strategy='simple',
     )
 
     parsl.load(config)
+    dfk = parsl.dfk()
+
+    def poller():
+        import time
+        while True:
+            dfk.job_status_poller.poll()
+            time.sleep(0.1)
 
-    f = app()
-    f.result()
+    threading.Thread(target=poller, daemon=True).start()
+    app().result()
     parsl.dfk().cleanup()
     parsl.clear()
 

parsl/tests/test_scaling/test_scale_down.py

@@ -1,80 +1,97 @@
 import logging
-import parsl
-import pytest
 import time
-from parsl import python_app
 
+import pytest
+
+import parsl
+
+from parsl import File, python_app
 from parsl.providers import LocalProvider
 from parsl.channels import LocalChannel
-# from parsl.launchers import SimpleLauncher
 from parsl.launchers import SingleNodeLauncher
-
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
 
 logger = logging.getLogger(__name__)
 
+_max_blocks = 5
+_min_blocks = 2
+
 
 def local_config():
     return Config(
         executors=[
             HighThroughputExecutor(
-                heartbeat_period=2,
-                heartbeat_threshold=6,
-                poll_period=1,
+                heartbeat_period=1,
+                heartbeat_threshold=2,
+                poll_period=100,
                 label="htex_local",
+                address="127.0.0.1",
                 max_workers=1,
                 provider=LocalProvider(
                     channel=LocalChannel(),
                     init_blocks=0,
-                    max_blocks=5,
-                    min_blocks=2,
+                    max_blocks=_max_blocks,
+                    min_blocks=_min_blocks,
                     launcher=SingleNodeLauncher(),
                 ),
             )
         ],
-        max_idletime=5,
+        max_idletime=0.5,
         strategy='htex_auto_scale',
     )
 
 
 @python_app
-def sleeper(t):
+def waiting_app(ident: int, inputs=()):
+    import pathlib
     import time
-    time.sleep(t)
+
+    # Approximate an Event by writing to files; the test logic will poll this file
+    with open(inputs[0], "a") as f:
+        f.write(f"Ready: {ident}\n")
+
+    # Similarly, use Event approximation (file check!) by polling.
+    may_finish_file = pathlib.Path(inputs[1])
+    while not may_finish_file.exists():
+        time.sleep(0.01)
 
 
 # see issue #1885 for details of failures of this test.
 # at the time of issue #1885 this test was failing frequently
 # in CI.
 @pytest.mark.local
-def test_scale_out():
-    logger.info("start")
+def test_scale_out(tmpd_cwd, try_assert):
     dfk = parsl.dfk()
 
-    logger.info("initial asserts")
-    assert len(dfk.executors['htex_local'].connected_managers()) == 0, "Expected 0 managers at start"
+    num_managers = len(dfk.executors['htex_local'].connected_managers())
+
+    assert num_managers == 0, "Expected 0 managers at start"
     assert dfk.executors['htex_local'].outstanding == 0, "Expected 0 tasks at start"
 
-    logger.info("launching tasks")
-    fus = [sleeper(i) for i in [15 for x in range(0, 10)]]
+    ntasks = 10
+    ready_path = tmpd_cwd / "workers_ready"
+    finish_path = tmpd_cwd / "workers_may_continue"
+    ready_path.touch()
+    inputs = [File(str(ready_path)), File(str(finish_path))]
 
-    logger.info("waiting for warm up")
-    time.sleep(15)
+    futs = [waiting_app(i, inputs=inputs) for i in range(ntasks)]
 
-    logger.info("asserting 5 managers")
-    assert len(dfk.executors['htex_local'].connected_managers()) == 5, "Expected 5 managers after some time"
+    while ready_path.read_text().count("\n") < _max_blocks:
+        time.sleep(0.5)
 
-    logger.info("waiting for all futures to complete")
-    [x.result() for x in fus]
+    assert len(dfk.executors['htex_local'].connected_managers()) == _max_blocks
 
-    logger.info("asserting 0 outstanding tasks after completion")
-    assert dfk.executors['htex_local'].outstanding == 0, "Expected 0 outstanding tasks after future completion"
+    finish_path.touch()  # Approximation of Event, via files
+    [x.result() for x in futs]
 
-    logger.info("waiting a while for scale down")
-    time.sleep(25)
+    assert dfk.executors['htex_local'].outstanding == 0
 
-    logger.info("asserting 2 managers remain")
-    assert len(dfk.executors['htex_local'].connected_managers()) == 2, "Expected 2 managers when no tasks, lower bound by min_blocks"
+    def assert_kernel():
+        return len(dfk.executors['htex_local'].connected_managers()) == _min_blocks
 
-    logger.info("test passed")
+    try_assert(
+        assert_kernel,
+        fail_msg=f"Expected {_min_blocks} managers when no tasks (min_blocks)",
+        timeout_ms=15000,
+    )

parsl/tests/test_serialization/test_2555_caching_deserializer.py

@@ -0,0 +1,34 @@
+import parsl
+import pytest
+
+from parsl.tests.configs.htex_local import fresh_config as local_config
+
+
+@parsl.python_app
+def return_range(x):
+    return list(range(x))
+
+
+@pytest.mark.local
+def test_range_identities():
+    x = 3
+
+    fut1 = return_range(x)
+    res1 = fut1.result()
+
+    fut2 = return_range(x)
+    res2 = fut2.result()
+
+    # Check that the returned futures are different, by both usual
+    # Python equalities.
+    # This is not strictly part of the regression test for #2555
+    # but will detect related unexpected Future caching.
+
+    assert fut1 != fut2
+    assert id(fut1) != id(fut2)
+
+    # check that the two invocations returned the same value...
+    assert res1 == res2
+
+    # ... but in two different objects.
+    assert id(res1) != id(res2)

parsl/utils.py

@@ -5,12 +5,11 @@ import shlex
 import subprocess
 import threading
 import time
-from types import TracebackType
-
-import typeguard
 from contextlib import contextmanager
+from types import TracebackType
 from typing import Any, Callable, List, Tuple, Union, Generator, IO, AnyStr, Dict, Optional
 
+import typeguard
 from typing_extensions import Type
 
 import parsl
@@ -110,9 +109,12 @@ def get_last_checkpoint(rundir: str = "runinfo") -> List[str]:
 
 
 @typeguard.typechecked
-def get_std_fname_mode(fdname: str, stdfspec: Union[str, Tuple[str, str]]) -> Tuple[str, str]:
+def get_std_fname_mode(
+    fdname: str,
+    stdfspec: Union[os.PathLike, str, Tuple[str, str], Tuple[os.PathLike, str]]
+) -> Tuple[str, str]:
     import parsl.app.errors as pe
-    if isinstance(stdfspec, str):
+    if isinstance(stdfspec, (str, os.PathLike)):
         fname = stdfspec
         mode = 'a+'
     elif isinstance(stdfspec, tuple):
@@ -121,7 +123,7 @@ def get_std_fname_mode(fdname: str, stdfspec: Union[str, Tuple[str, str]]) -> Tu
                    f"{len(stdfspec)}")
             raise pe.BadStdStreamFile(msg, TypeError('Bad Tuple Length'))
         fname, mode = stdfspec
-    return fname, mode
+    return str(fname), mode
 
 
 @contextmanager

parsl/version.py

@@ -3,4 +3,4 @@
 Year.Month.Day[alpha/beta/..]
 Alphas will be numbered like this -> 2024.12.10a0
 """
-VERSION = '2023.07.03'
+VERSION = '2023.07.17'

{parsl-2023.7.3.data → parsl-2023.7.17.data}/scripts/process_worker_pool.py

@@ -370,7 +370,7 @@ class Manager:
         logger.critical("Exiting")
 
     @wrap_with_logs
-    def worker_watchdog(self, kill_event):
+    def worker_watchdog(self, kill_event: threading.Event):
         """Keeps workers alive.
 
         Parameters:
@@ -381,7 +381,7 @@ class Manager:
 
         logger.debug("Starting worker watchdog")
 
-        while not kill_event.is_set():
+        while not kill_event.wait(self.heartbeat_period):
             for worker_id, p in self.procs.items():
                 if not p.is_alive():
                     logger.error("Worker {} has died".format(worker_id))
@@ -409,7 +409,6 @@ class Manager:
                                        name="HTEX-Worker-{}".format(worker_id))
                     self.procs[worker_id] = p
                     logger.info("Worker {} has been restarted".format(worker_id))
-                time.sleep(self.heartbeat_period)
 
         logger.critical("Exiting")
 

{parsl-2023.7.3.dist-info → parsl-2023.7.17.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: parsl
-Version: 2023.7.3
+Version: 2023.7.17
 Summary: Simple data dependent workflows in Python
 Home-page: https://github.com/Parsl/parsl
-Download-URL: https://github.com/Parsl/parsl/archive/2023.07.03.tar.gz
+Download-URL: https://github.com/Parsl/parsl/archive/2023.07.17.tar.gz
 Author: The Parsl Team
 Author-email: parsl@googlegroups.com
 License: Apache 2.0