parsl 2023.7.3__tar.gz → 2023.7.17__tar.gz

{parsl-2023.7.3/parsl.egg-info → parsl-2023.7.17}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: parsl
-Version: 2023.7.3
+Version: 2023.7.17
 Summary: Simple data dependent workflows in Python
 Home-page: https://github.com/Parsl/parsl
-Download-URL: https://github.com/Parsl/parsl/archive/2023.07.03.tar.gz
+Download-URL: https://github.com/Parsl/parsl/archive/2023.07.17.tar.gz
 Author: The Parsl Team
 Author-email: parsl@googlegroups.com
 License: Apache 2.0

{parsl-2023.7.3 → parsl-2023.7.17}/parsl/data_provider/files.py

@@ -5,11 +5,11 @@ to transfer the file as well as to give the appropriate filepath depending
 on where (client-side, remote-side, intermediary-side) the File.filepath is
 being called from.
 """
-
 import os
+
 import typeguard
 import logging
-from typing import Optional
+from typing import Optional, Union
 from urllib.parse import urlparse
 
 logger = logging.getLogger(__name__)
@@ -28,17 +28,18 @@ class File:
     """
 
     @typeguard.typechecked
-    def __init__(self, url: str):
+    def __init__(self, url: Union[os.PathLike, str]):
         """Construct a File object from a url string.
 
         Args:
-           - url (string) : url string of the file e.g.
+           - url (string or PathLike) : url of the file e.g.
               - 'input.txt'
+              - pathlib.Path('input.txt')
               - 'file:///scratch/proj101/input.txt'
               - 'globus://go#ep1/~/data/input.txt'
               - 'globus://ddb59aef-6d04-11e5-ba46-22000b92c6ec/home/johndoe/data/input.txt'
         """
-        self.url = url
+        self.url = str(url)
         parsed_url = urlparse(self.url)
         self.scheme = parsed_url.scheme if parsed_url.scheme else 'file'
         self.netloc = parsed_url.netloc

{parsl-2023.7.3 → parsl-2023.7.17}/parsl/dataflow/dflow.py

@@ -380,6 +380,12 @@ class DataFlowKernel:
                         task_record['join_lock'] = threading.Lock()
                         self._send_task_log_info(task_record)
                         joinable.add_done_callback(partial(self.handle_join_update, task_record))
+                    elif joinable == []:  # got a list, but it had no entries, and specifically, no Futures.
+                        self.update_task_state(task_record, States.joining)
+                        task_record['joins'] = joinable
+                        task_record['join_lock'] = threading.Lock()
+                        self._send_task_log_info(task_record)
+                        self.handle_join_update(task_record, None)
                     elif isinstance(joinable, list) and [j for j in joinable if not isinstance(j, Future)] == []:
                         self.update_task_state(task_record, States.joining)
                         task_record['joins'] = joinable
@@ -403,7 +409,7 @@ class DataFlowKernel:
         if task_record['status'] == States.pending:
             self.launch_if_ready(task_record)
 
-    def handle_join_update(self, task_record: TaskRecord, inner_app_future: AppFuture) -> None:
+    def handle_join_update(self, task_record: TaskRecord, inner_app_future: Optional[AppFuture]) -> None:
         with task_record['join_lock']:
             # inner_app_future has completed, which is one (potentially of many)
             # futures the outer task is joining on.

{parsl-2023.7.3 → parsl-2023.7.17}/parsl/dataflow/memoization.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 import hashlib
 from functools import lru_cache, singledispatch
 import logging
+import pickle
 from parsl.dataflow.taskrecord import TaskRecord
 
 from typing import Dict, Any, List, Optional, TYPE_CHECKING
@@ -11,7 +12,6 @@ if TYPE_CHECKING:
 
 from concurrent.futures import Future
 
-from parsl.serialize import serialize
 import types
 
 logger = logging.getLogger(__name__)
@@ -54,8 +54,8 @@ def id_for_memo(obj: object, output_ref: bool = False) -> bytes:
 @id_for_memo.register(int)
 @id_for_memo.register(float)
 @id_for_memo.register(type(None))
-def id_for_memo_serialize(obj: object, output_ref: bool = False) -> bytes:
-    return serialize(obj)
+def id_for_memo_pickle(obj: object, output_ref: bool = False) -> bytes:
+    return pickle.dumps(obj)
 
 
 @id_for_memo.register(list)
@@ -68,7 +68,7 @@ def id_for_memo_list(denormalized_list: list, output_ref: bool = False) -> bytes
     for e in denormalized_list:
         normalized_list.append(id_for_memo(e, output_ref=output_ref))
 
-    return serialize(normalized_list)
+    return pickle.dumps(normalized_list)
 
 
 @id_for_memo.register(tuple)
@@ -81,7 +81,7 @@ def id_for_memo_tuple(denormalized_tuple: tuple, output_ref: bool = False) -> by
     for e in denormalized_tuple:
         normalized_list.append(id_for_memo(e, output_ref=output_ref))
 
-    return serialize(normalized_list)
+    return pickle.dumps(normalized_list)
 
 
 @id_for_memo.register(dict)
@@ -100,7 +100,7 @@ def id_for_memo_dict(denormalized_dict: dict, output_ref: bool = False) -> bytes
     for k in keys:
         normalized_list.append(id_for_memo(k))
         normalized_list.append(id_for_memo(denormalized_dict[k], output_ref=output_ref))
-    return serialize(normalized_list)
+    return pickle.dumps(normalized_list)
 
 
 # the LRU cache decorator must be applied closer to the id_for_memo_function call
@@ -112,7 +112,7 @@ def id_for_memo_function(f: types.FunctionType, output_ref: bool = False) -> byt
     This means that changing source code (other than the function name) will
     not cause a checkpoint invalidation.
     """
-    return serialize(["types.FunctionType", f.__name__, f.__module__])
+    return pickle.dumps(["types.FunctionType", f.__name__, f.__module__])
 
 
 class Memoizer:

{parsl-2023.7.3 → parsl-2023.7.17}/parsl/executors/high_throughput/executor.py

@@ -595,10 +595,9 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin):
         except TypeError:
             raise SerializationError(func.__name__)
 
-        msg = {"task_id": task_id,
-               "buffer": fn_buf}
+        msg = {"task_id": task_id, "buffer": fn_buf}
 
-        # Post task to the the outgoing queue
+        # Post task to the outgoing queue
         self.outgoing_q.put(msg)
 
         # Return the future

{parsl-2023.7.3 → parsl-2023.7.17}/parsl/executors/high_throughput/process_worker_pool.py

@@ -370,7 +370,7 @@ class Manager:
         logger.critical("Exiting")
 
     @wrap_with_logs
-    def worker_watchdog(self, kill_event):
+    def worker_watchdog(self, kill_event: threading.Event):
         """Keeps workers alive.
 
         Parameters:
@@ -381,7 +381,7 @@ class Manager:
 
         logger.debug("Starting worker watchdog")
 
-        while not kill_event.is_set():
+        while not kill_event.wait(self.heartbeat_period):
             for worker_id, p in self.procs.items():
                 if not p.is_alive():
                     logger.error("Worker {} has died".format(worker_id))
@@ -409,7 +409,6 @@ class Manager:
                                        name="HTEX-Worker-{}".format(worker_id))
                     self.procs[worker_id] = p
                     logger.info("Worker {} has been restarted".format(worker_id))
-                time.sleep(self.heartbeat_period)
 
         logger.critical("Exiting")
 

{parsl-2023.7.3 → parsl-2023.7.17}/parsl/launchers/errors.py

@@ -10,4 +10,4 @@ class BadLauncher(ExecutionProviderException, TypeError):
         self.launcher = launcher
 
     def __str__(self) -> str:
-        return f"Bad Launcher provided: {self.launcher}, expecting a parsl.launcher.launcher.Launcher or callable"
+        return f"Bad Launcher provided: {self.launcher}, expecting a parsl.launcher.launcher.Launcher"

{parsl-2023.7.3 → parsl-2023.7.17}/parsl/providers/cluster_provider.py

@@ -3,6 +3,7 @@ from abc import abstractmethod
 from string import Template
 
 from parsl.providers.errors import SchedulerMissingArgs, ScriptPathError
+from parsl.launchers.base import Launcher
 from parsl.launchers.errors import BadLauncher
 from parsl.providers.base import ExecutionProvider
 
@@ -66,7 +67,7 @@ class ClusterProvider(ExecutionProvider):
         self.launcher = launcher
         self.walltime = walltime
         self.cmd_timeout = cmd_timeout
-        if not callable(self.launcher):
+        if not isinstance(self.launcher, Launcher):
             raise BadLauncher(self.launcher)
 
         self.script_dir = None

{parsl-2023.7.3 → parsl-2023.7.17}/parsl/providers/local/local.py

@@ -214,7 +214,7 @@ class LocalProvider(ExecutionProvider, RepresentationMixin):
             logger.debug("Pushing start script")
             script_path = self.channel.push_file(script_path, self.channel.script_dir)
 
-        logger.debug("Launching in remote mode")
+        logger.debug("Launching")
         # We need to capture the exit code and the streams, so we put them in files. We also write
         # '-' to the exit code file to isolate potential problems with writing to files in the
         # script directory

{parsl-2023.7.3 → parsl-2023.7.17}/parsl/serialize/base.py

@@ -1,6 +1,5 @@
 from abc import abstractmethod
 import logging
-import functools
 
 from typing import Any
 
@@ -32,24 +31,15 @@ class SerializerBase:
 
     @property
     def identifier(self) -> bytes:
-        """ Get the identifier of the serialization method
+        """Get that identifier that will be used to indicate in byte streams
+        that this class should be used for deserialization.
 
         Returns
         -------
-        identifier : str
+        identifier : bytes
         """
         return self._identifier
 
-    def enable_caching(self, maxsize: int = 128) -> None:
-        """ Add functools.lru_cache onto the serialize, deserialize methods
-        """
-
-        # ignore types here because mypy at the moment is not fond of monkeypatching
-        self.serialize = functools.lru_cache(maxsize=maxsize)(self.serialize)  # type: ignore[method-assign]
-        self.deserialize = functools.lru_cache(maxsize=maxsize)(self.deserialize)  # type: ignore[method-assign]
-
-        return
-
     @abstractmethod
     def serialize(self, data: Any) -> bytes:
         pass

{parsl-2023.7.3 → parsl-2023.7.17}/parsl/serialize/concretes.py

@@ -1,4 +1,5 @@
 import dill
+import functools
 import pickle
 import logging
 
@@ -14,11 +15,10 @@ class PickleSerializer(SerializerBase):
     * functions defined in a interpreter/notebook
     * classes defined in local context and not importable using a fully qualified name
     * closures, generators and coroutines
-    * [sometimes] issues with wrapped/decorated functions
     """
 
     _identifier = b'01'
-    _for_code = True
+    _for_code = False
     _for_data = True
 
     def serialize(self, data: Any) -> bytes:
@@ -41,7 +41,7 @@ class DillSerializer(SerializerBase):
     """
 
     _identifier = b'02'
-    _for_code = True
+    _for_code = False
     _for_data = True
 
     def serialize(self, data: Any) -> bytes:
@@ -49,3 +49,22 @@ class DillSerializer(SerializerBase):
 
     def deserialize(self, body: bytes) -> Any:
         return dill.loads(body)
+
+
+class DillCallableSerializer(SerializerBase):
+    """This serializer is a variant of the DillSerializer that will
+    serialize and deserialize callables using an lru_cache, under the
+    assumption that callables are immutable and so can be cached.
+    """
+
+    _identifier = b'C2'
+    _for_code = True
+    _for_data = False
+
+    @functools.lru_cache
+    def serialize(self, data: Any) -> bytes:
+        return dill.dumps(data)
+
+    @functools.lru_cache
+    def deserialize(self, body: bytes) -> Any:
+        return dill.loads(body)

{parsl-2023.7.3 → parsl-2023.7.17}/parsl/serialize/facade.py

@@ -1,8 +1,8 @@
 from parsl.serialize.concretes import *  # noqa: F403,F401
-from parsl.serialize.base import METHODS_MAP_DATA, METHODS_MAP_CODE, SerializerBase
+from parsl.serialize.base import METHODS_MAP_DATA, METHODS_MAP_CODE
 import logging
 
-from typing import Any, Dict, List, Tuple, Union
+from typing import Any, List, Union
 
 logger = logging.getLogger(__name__)
 
@@ -14,16 +14,11 @@ methods_for_data = {}
 
 for key in METHODS_MAP_CODE:
     methods_for_code[key] = METHODS_MAP_CODE[key]()
-    methods_for_code[key].enable_caching(maxsize=128)
 
 for key in METHODS_MAP_DATA:
     methods_for_data[key] = METHODS_MAP_DATA[key]()
 
 
-def _list_methods() -> Tuple[Dict[bytes, SerializerBase], Dict[bytes, SerializerBase]]:
-    return methods_for_code, methods_for_data
-
-
 def pack_apply_message(func: Any, args: Any, kwargs: Any, buffer_threshold: int = int(128 * 1e6)) -> bytes:
     """Serialize and pack function and parameters
 
@@ -65,23 +60,18 @@ def serialize(obj: Any, buffer_threshold: int = int(1e6)) -> bytes:
     """
     result: Union[bytes, Exception]
     if callable(obj):
-        for method in methods_for_code.values():
-            try:
-                result = method._identifier + b'\n' + method.serialize(obj)
-            except Exception as e:
-                result = e
-                continue
-            else:
-                break
+        methods = methods_for_code
     else:
-        for method in methods_for_data.values():
-            try:
-                result = method._identifier + b'\n' + method.serialize(obj)
-            except Exception as e:
-                result = e
-                continue
-            else:
-                break
+        methods = methods_for_data
+
+    for method in methods.values():
+        try:
+            result = method._identifier + b'\n' + method.serialize(obj)
+        except Exception as e:
+            result = e
+            continue
+        else:
+            break
 
     if isinstance(result, BaseException):
         raise result

{parsl-2023.7.3 → parsl-2023.7.17}/parsl/tests/conftest.py

@@ -1,12 +1,19 @@
 import importlib.util
+import itertools
 import logging
 import os
-from glob import glob
-from itertools import chain
+import pathlib
+import time
+import types
 import signal
 import sys
+import tempfile
 import threading
 import traceback
+import typing as t
+from datetime import datetime
+from glob import glob
+from itertools import chain
 
 import pytest
 import _pytest.runner as runner
@@ -36,6 +43,20 @@ def pytest_sessionstart(session):
     signal.signal(signal.SIGUSR1, dumpstacks)
 
 
+@pytest.fixture(scope="session")
+def tmpd_cwd_session():
+    n = datetime.now().strftime('%Y%m%d.%H%I%S')
+    with tempfile.TemporaryDirectory(dir=os.getcwd(), prefix=f".pytest-{n}-") as tmpd:
+        yield pathlib.Path(tmpd)
+
+
+@pytest.fixture
+def tmpd_cwd(tmpd_cwd_session, request):
+    prefix = f"{request.node.name}-"
+    with tempfile.TemporaryDirectory(dir=tmpd_cwd_session, prefix=prefix) as tmpd:
+        yield pathlib.Path(tmpd)
+
+
 def pytest_addoption(parser):
     """Add parsl-specific command-line options to pytest.
     """
@@ -140,9 +161,9 @@ def load_dfk_local_module(request, pytestconfig):
     parsl.load. It should be a Callable that returns a parsl Config object.
 
     If local_setup and/or local_teardown are callables (such as functions) in
-    the test module, they they will be invoked before/after the tests. This
-    can be used to perform more interesting DFK initialisation not possible
-    with local_config.
+    the test module, they will be invoked before/after the tests. This can
+    be used to perform more interesting DFK initialisation not possible with
+    local_config.
     """
 
     config = pytestconfig.getoption('config')[0]
@@ -212,14 +233,12 @@ def apply_masks(request, pytestconfig):
 
 
 @pytest.fixture
-def setup_data(tmp_path):
-    data_dir = tmp_path / "data"
+def setup_data(tmpd_cwd):
+    data_dir = tmpd_cwd / "data"
     data_dir.mkdir()
 
-    with open(data_dir / "test1.txt", "w") as f:
-        f.write("1\n")
-    with open(data_dir / "test2.txt", "w") as f:
-        f.write("2\n")
+    (data_dir / "test1.txt").write_text("1\n")
+    (data_dir / "test2.txt").write_text("2\n")
     return data_dir
 
 
@@ -275,3 +294,67 @@ def pytest_ignore_collect(path):
         return True
     else:
         return False
+
+
+def create_traceback(start: int = 0) -> t.Optional[types.TracebackType]:
+    """
+    Dynamically create a traceback.
+
+    Builds a traceback from the top of the stack (the currently executing frame) on
+    down to the root frame.  Optionally, use start to build from an earlier stack
+    frame.
+
+    N.B. uses `sys._getframe`, which I only know to exist in CPython.
+    """
+    tb = None
+    for depth in itertools.count(start + 1, 1):
+        try:
+            frame = sys._getframe(depth)
+            tb = types.TracebackType(tb, frame, frame.f_lasti, frame.f_lineno)
+        except ValueError:
+            break
+    return tb
+
+
+@pytest.fixture
+def try_assert():
+    def _impl(
+        test_func: t.Callable[[], bool],
+        fail_msg: str = "",
+        timeout_ms: float = 5000,
+        attempts: int = 0,
+        check_period_ms: int = 20,
+    ):
+        tb = create_traceback(start=1)
+        timeout_s = abs(timeout_ms) / 1000.0
+        check_period_s = abs(check_period_ms) / 1000.0
+        if attempts > 0:
+            for _attempt_no in range(attempts):
+                if test_func():
+                    return
+                time.sleep(check_period_s)
+            else:
+                att_fail = (
+                    f"\n  (Still failing after attempt limit [{attempts}], testing"
+                    f" every {check_period_ms}ms)"
+                )
+                exc = AssertionError(f"{str(fail_msg)}{att_fail}".strip())
+                raise exc.with_traceback(tb)
+
+        elif timeout_s > 0:
+            end = time.monotonic() + timeout_s
+            while time.monotonic() < end:
+                if test_func():
+                    return
+                time.sleep(check_period_s)
+            att_fail = (
+                f"\n  (Still failing after timeout [{timeout_ms}ms], with attempts "
+                f"every {check_period_ms}ms)"
+            )
+            exc = AssertionError(f"{str(fail_msg)}{att_fail}".strip())
+            raise exc.with_traceback(tb)
+
+        else:
+            raise AssertionError("Bad assert call: no attempts or timeout period")
+
+    yield _impl

parsl-2023.7.17/parsl/tests/test_bash_apps/test_basic.py

@@ -0,0 +1,84 @@
+import os
+import random
+import re
+
+import pytest
+
+import parsl
+from parsl import File
+from parsl.app.app import bash_app
+
+
+@bash_app
+def echo_to_file(inputs=(), outputs=(), stderr=None, stdout=None):
+    res = ""
+    for o in outputs:
+        for i in inputs:
+            res += "echo {} >& {}".format(i, o)
+    return res
+
+
+@bash_app
+def foo(x, y, z=10, stdout=None, label=None):
+    return f"echo {x} {y} {z}"
+
+
+@pytest.mark.issue363
+def test_command_format_1(tmpd_cwd):
+    """Testing command format for BashApps"""
+
+    outdir = tmpd_cwd / "outputs"
+    outdir.mkdir()
+    stdout = outdir / "foo-std.out"
+
+    foo_future = foo(1, 4, stdout=str(stdout))
+    assert foo_future.result() == 0, "BashApp had non-zero exit code"
+
+    so_content = stdout.read_text().strip()
+    assert so_content == "1 4 10"
+
+
+@pytest.mark.issue363
+def test_auto_log_filename_format():
+    """Testing auto log filename format for BashApps
+    """
+    app_label = "label_test_auto_log_filename_format"
+    rand_int = random.randint(1000, 1000000000)
+
+    foo_future = foo(1, rand_int, stdout=parsl.AUTO_LOGNAME, label=app_label)
+
+    assert foo_future.result() == 0, "BashApp exited with an error code : {0}".format(
+        foo_future.result())
+
+    log_fpath = foo_future.stdout
+    log_pattern = fr".*/task_\d+_foo_{app_label}"
+    assert re.match(log_pattern, log_fpath), 'Output file "{0}" does not match pattern "{1}"'.format(
+        log_fpath, log_pattern)
+    assert os.path.exists(log_fpath), 'Output file does not exist "{0}"'.format(log_fpath)
+    with open(log_fpath, 'r') as stdout_f:
+        contents = stdout_f.read()
+
+    assert contents == '1 {0} 10\n'.format(rand_int), \
+        'Output does not match expected string "1 {0} 10", Got: "{1}"'.format(rand_int, contents)
+
+
+@pytest.mark.issue363
+def test_parallel_for(tmpd_cwd, n=3):
+    """Testing a simple parallel for loop"""
+    outdir = tmpd_cwd / "outputs/test_parallel"
+    outdir.mkdir(parents=True)
+    futs = [
+        echo_to_file(
+            inputs=[f"Hello World {i}"],
+            outputs=[File(str(outdir / f"out.{i}.txt"))],
+            stdout=str(outdir / f"std.{i}.out"),
+            stderr=str(outdir / f"std.{i}.err"),
+        )
+        for i in range(n)
+    ]
+
+    for f in futs:
+        f.result()
+
+    stdout_file_count = len(list(outdir.glob("*.out")))
+    assert stdout_file_count == n, sorted(outdir.iterdir())

parsl-2023.7.17/parsl/tests/test_bash_apps/test_kwarg_storage.py

@@ -0,0 +1,33 @@
+import pytest
+
+from parsl.app.app import bash_app
+
+
+@bash_app
+def foo(z=2, stdout=None):
+    return f"echo {z}"
+
+
+@pytest.mark.issue363
+def test_command_format_1(tmpd_cwd):
+    """Testing command format for BashApps
+    """
+
+    stdout = tmpd_cwd / "std.out"
+    for exp_value, z in (
+        ("3", 3),
+        ("4", 4),
+        ("5", 5),
+    ):
+        app_fu = foo(z=z, stdout=str(stdout))
+        assert app_fu.result() == 0, "BashApp had non-zero exit"
+
+        so_content = stdout.read_text().strip()
+        assert so_content == exp_value
+        stdout.unlink()
+
+    app_fu = foo(stdout=str(stdout))
+    assert app_fu.result() == 0, "BashApp had non-zero exit"
+
+    so_content = stdout.read_text().strip()
+    assert so_content == "2"

parsl-2023.7.17/parsl/tests/test_bash_apps/test_memoize.py

@@ -0,0 +1,48 @@
+import pytest
+
+from parsl import File
+from parsl.app.app import bash_app
+
+
+@bash_app(cache=True)
+def fail_on_presence(outputs=()):
+    return 'if [ -f {0} ] ; then exit 1 ; else touch {0}; fi'.format(outputs[0])
+
+
+# This test is an oddity that requires a shared-FS and simply
+# won't work if there's a staging provider.
+# @pytest.mark.sharedFS_required
+@pytest.mark.issue363
+def test_bash_memoization(tmpd_cwd, n=2):
+    """Testing bash memoization
+    """
+    mpath = tmpd_cwd / "test.memoization.tmp"
+    temp_file = File(str(mpath))
+    fail_on_presence(outputs=[temp_file]).result()
+
+    futs = [fail_on_presence(outputs=[temp_file]) for _ in range(n)]
+    for f in futs:
+        assert f.exception() is None
+
+
+@bash_app(cache=True)
+def fail_on_presence_kw(outputs=(), foo=None):
+    return 'if [ -f {0} ] ; then exit 1 ; else touch {0}; fi'.format(outputs[0])
+
+
+# This test is an oddity that requires a shared-FS and simply
+# won't work if there's a staging provider.
+# @pytest.mark.sharedFS_required
+@pytest.mark.issue363
+def test_bash_memoization_keywords(tmpd_cwd, n=2):
+    """Testing bash memoization
+    """
+    mpath = tmpd_cwd / "test.memoization.tmp"
+    temp_file = File(str(mpath))
+
+    foo = {"a": 1, "b": 2}
+    fail_on_presence_kw(outputs=[temp_file], foo=foo).result()
+
+    futs = [fail_on_presence_kw(outputs=[temp_file], foo=foo) for _ in range(n)]
+    for f in futs:
+        assert f.exception() is None