parsl 2025.9.29__tar.gz → 2025.10.6__tar.gz

{parsl-2025.9.29/parsl.egg-info → parsl-2025.10.6}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: parsl
-Version: 2025.9.29
+Version: 2025.10.6
 Summary: Simple data dependent workflows in Python
 Home-page: https://github.com/Parsl/parsl
-Download-URL: https://github.com/Parsl/parsl/archive/2025.09.29.tar.gz
+Download-URL: https://github.com/Parsl/parsl/archive/2025.10.06.tar.gz
 Author: The Parsl Team
 Author-email: parsl@googlegroups.com
 License: Apache 2.0

{parsl-2025.9.29 → parsl-2025.10.6}/parsl/dataflow/dflow.py

@@ -375,6 +375,7 @@ class DataFlowKernel:
                 logger.info("Task {} failed due to dependency failure so skipping retries".format(task_id))
                 task_record['time_returned'] = datetime.datetime.now()
                 self._send_task_log_info(task_record)
+                self.memoizer.update_memo(task_record)
                 with task_record['app_fu']._update_lock:
                     task_record['app_fu'].set_exception(e)
 
@@ -400,6 +401,7 @@ class DataFlowKernel:
                 self.update_task_state(task_record, States.failed)
                 task_record['time_returned'] = datetime.datetime.now()
                 self._send_task_log_info(task_record)
+                self.memoizer.update_memo(task_record)
                 with task_record['app_fu']._update_lock:
                     task_record['app_fu'].set_exception(e)
 
@@ -446,6 +448,7 @@ class DataFlowKernel:
                         self.update_task_state(task_record, States.failed)
                         task_record['time_returned'] = datetime.datetime.now()
                         self._send_task_log_info(task_record)
+                        self.memoizer.update_memo(task_record)
                         with task_record['app_fu']._update_lock:
                             task_record['app_fu'].set_exception(
                                 TypeError(f"join_app body must return a Future or list of Futures, got {joinable} of type {type(joinable)}"))
@@ -521,6 +524,7 @@ class DataFlowKernel:
 
                 self.update_task_state(task_record, States.failed)
                 task_record['time_returned'] = datetime.datetime.now()
+                self.memoizer.update_memo(task_record)
                 with task_record['app_fu']._update_lock:
                     task_record['app_fu'].set_exception(e)
 
@@ -561,8 +565,6 @@ class DataFlowKernel:
         if not task_record['app_fu'] == future:
             logger.error("Internal consistency error: callback future is not the app_fu in task structure, for task {}".format(task_id))
 
-        self.memoizer.update_memo(task_record)
-
         # Cover all checkpointing cases here:
         # Do we need to checkpoint now, or queue for later,
         # or do nothing?
@@ -591,6 +593,7 @@ class DataFlowKernel:
         logger.info(f"Task {task_record['id']} completed ({old_state.name} -> {new_state.name})")
         task_record['time_returned'] = datetime.datetime.now()
 
+        self.memoizer.update_memo(task_record)
         with task_record['app_fu']._update_lock:
             task_record['app_fu'].set_result(result)
 

{parsl-2025.9.29 → parsl-2025.10.6}/parsl/executors/high_throughput/executor.py

@@ -160,6 +160,12 @@ GENERAL_HTEX_PARAM_DOCS = """provider : :class:`~parsl.providers.base.ExecutionP
 """  # Documentation for params used by both HTEx and MPIEx
 
 
+class HTEXFuture(Future):
+    def __init__(self, task_id) -> None:
+        super().__init__()
+        self.parsl_executor_task_id = task_id
+
+
 class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin, UsageInformation):
     __doc__ = f"""Executor designed for cluster-scale
 
@@ -670,7 +676,7 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin, UsageIn
                 logger.debug("Sending hold to manager: {}".format(manager['manager']))
                 self._hold_manager(manager['manager'])
 
-    def submit(self, func, resource_specification, *args, **kwargs):
+    def submit(self, func: Callable, resource_specification: dict, *args, **kwargs) -> HTEXFuture:
         """Submits work to the outgoing_q.
 
         The outgoing_q is an external process listens on this
@@ -691,21 +697,11 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin, UsageIn
 
         self.validate_resource_spec(resource_specification)
 
-        if self.bad_state_is_set:
-            raise self.executor_exception
-
-        self._task_counter += 1
-        task_id = self._task_counter
-
         # handle people sending blobs gracefully
         if logger.getEffectiveLevel() <= logging.DEBUG:
             args_to_print = tuple([ar if len(ar := repr(arg)) < 100 else (ar[:100] + '...') for arg in args])
             logger.debug("Pushing function {} to queue with args {}".format(func, args_to_print))
 
-        fut = Future()
-        fut.parsl_executor_task_id = task_id
-        self.tasks[task_id] = fut
-
         try:
             fn_buf = pack_apply_message(func, args, kwargs, buffer_threshold=1 << 20)
         except TypeError:
@@ -715,12 +711,69 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin, UsageIn
         if resource_specification:
             context["resource_spec"] = resource_specification
 
-        msg = {"task_id": task_id, "context": context, "buffer": fn_buf}
+        return self.submit_payload(context, fn_buf)
+
+    def submit_payload(self, context: dict, buffer: bytes) -> HTEXFuture:
+        """
+        Submit specially crafted payloads.
+
+        For use-cases where the ``HighThroughputExecutor`` consumer needs the payload
+        handled by the worker in a special way.  For example, if the function is
+        serialized differently than Parsl's default approach, or if the task must
+        be setup more precisely than Parsl's default ``execute_task`` allows.
+
+        An example interaction:
+
+        .. code-block: python
+
+            >>> htex: HighThroughputExecutor  # setup prior to this example
+            >>> ctxt = {
+            ...   "task_executor": {
+            ...     "f": "full.import.path.of.custom_execute_task",
+            ...     "a": ("additional", "arguments"),
+            ...     "k": {"some": "keyword", "args": "here"}
+            ...   }
+            ... }
+            >>> fn_buf = custom_serialize(task_func, *task_args, **task_kwargs)
+            >>> fut = htex.submit_payload(ctxt, fn_buf)
+
+        The custom ``custom_execute_task`` would be dynamically imported, and
+        invoked as:
+
+        .. code-block: python
+
+            args = ("additional", "arguments")
+            kwargs = {"some": "keyword", "args": "here"}
+            result = custom_execute_task(fn_buf, *args, **kwargs)
+
+        Parameters
+        ----------
+        context:
+            A task-specific context associated with the function buffer.  Parsl
+            currently implements the keys ``task_executor`` and ``resource_spec``
+
+        buffer:
+            A serialized function, that will be deserialized and executed by
+            ``execute_task`` (or custom function, if ``task_executor`` is specified)
+
+        Returns
+        -------
+        An HTEXFuture (a normal Future, with the attribute ``.parsl_executor_task_id``
+        set).  The future will be set to done when the associated function buffer has
+        been invoked and completed.
+        """
+        if self.bad_state_is_set:
+            raise self.executor_exception
+
+        self._task_counter += 1
+        task_id = self._task_counter
+
+        fut = HTEXFuture(task_id)
+        self.tasks[task_id] = fut
 
-        # Post task to the outgoing queue
+        msg = {"task_id": task_id, "context": context, "buffer": buffer}
         self.outgoing_q.put(msg)
 
-        # Return the future
         return fut
 
     @property

{parsl-2025.9.29 → parsl-2025.10.6}/parsl/executors/high_throughput/process_worker_pool.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
 import argparse
+import importlib
 import logging
 import math
 import multiprocessing
@@ -17,7 +18,7 @@ from importlib.metadata import distributions
 from multiprocessing.context import SpawnProcess
 from multiprocessing.managers import DictProxy
 from multiprocessing.sharedctypes import Synchronized
-from typing import Dict, List, Optional, Sequence
+from typing import Callable, Dict, List, Optional, Sequence
 
 import psutil
 import zmq
@@ -778,8 +779,20 @@ def worker(
 
         _init_mpi_env(mpi_launcher=mpi_launcher, resource_spec=res_spec)
 
+        exec_func: Callable = execute_task
+        exec_args = ()
+        exec_kwargs = {}
+
         try:
-            result = execute_task(req['buffer'])
+            if task_executor := ctxt.get("task_executor", None):
+                mod_name, _, fn_name = task_executor["f"].rpartition(".")
+                exec_mod = importlib.import_module(mod_name)
+                exec_func = getattr(exec_mod, fn_name)
+
+                exec_args = task_executor.get("a", ())
+                exec_kwargs = task_executor.get("k", {})
+
+            result = exec_func(req['buffer'], *exec_args, **exec_kwargs)
             serialized_result = serialize(result, buffer_threshold=1000000)
         except Exception as e:
             logger.info('Caught an exception: {}'.format(e))

{parsl-2025.9.29 → parsl-2025.10.6}/parsl/tests/test_checkpointing/test_regression_233.py

@@ -5,7 +5,6 @@ from parsl.dataflow.dflow import DataFlowKernel
 
 
 def run_checkpointed(checkpoints):
-    # set_stream_logger()
     from parsl.tests.configs.local_threads_checkpoint_task_exit import config
     config.checkpoint_files = checkpoints
     dfk = DataFlowKernel(config=config)

{parsl-2025.9.29 → parsl-2025.10.6}/parsl/tests/test_htex/test_htex.py

@@ -7,6 +7,7 @@ from unittest import mock
 import pytest
 
 from parsl import HighThroughputExecutor, curvezmq
+from parsl.serialize.facade import pack_apply_message, unpack_apply_message
 
 _MOCK_BASE = "parsl.executors.high_throughput.executor"
 
@@ -19,11 +20,16 @@ def encrypted(request: pytest.FixtureRequest):
 
 
 @pytest.fixture
-def htex(encrypted: bool):
+def htex(encrypted: bool, tmpd_cwd):
     htex = HighThroughputExecutor(encrypted=encrypted)
+    htex.max_workers_per_node = 1
+    htex.run_dir = tmpd_cwd
+    htex.provider.script_dir = tmpd_cwd
 
     yield htex
 
+    if hasattr(htex, "outgoing_q"):
+        htex.scale_in(blocks=1000)
     htex.shutdown()
 
 
@@ -146,3 +152,32 @@ def test_htex_interchange_launch_cmd(cmd: Optional[Sequence[str]]):
     else:
         htex = HighThroughputExecutor()
         assert htex.interchange_launch_cmd == ["interchange.py"]
+
+
+def dyn_exec(buf, *vec_y):
+    f, a, _ = unpack_apply_message(buf)
+    custom_args = [a, vec_y]
+    return f(*custom_args)
+
+
+@pytest.mark.local
+def test_worker_dynamic_import(htex: HighThroughputExecutor):
+    def _dot_prod(vec_x, vec_y):
+        return sum(x * y for x, y in zip(vec_x, vec_y))
+
+    htex.start()
+    htex.scale_out_facade(1)
+
+    num_array = tuple(range(10))
+
+    fn_buf = pack_apply_message(_dot_prod, num_array, {})
+    ctxt = {
+        "task_executor": {
+            "f": f"{dyn_exec.__module__}.{dyn_exec.__name__}",
+            "a": num_array,  # prove "custom" dyn_exec
+        }
+    }
+    val = htex.submit_payload(ctxt, fn_buf).result()
+    exp_val = _dot_prod(num_array, num_array)
+
+    assert val == exp_val

{parsl-2025.9.29 → parsl-2025.10.6}/parsl/tests/test_python_apps/test_garbage_collect.py

@@ -27,10 +27,5 @@ def test_garbage_collect():
 
     evt.set()
     assert x.result() == 10 * 4
-    if parsl.dfk().checkpoint_mode is not None:
-        # We explicit call checkpoint if checkpoint_mode is enabled covering
-        # cases like manual/periodic where checkpointing may be deferred.
-        parsl.dfk().checkpoint()
-
-    time.sleep(0.01)  # Give enough time for task wipes to work
+    time.sleep(0.01)  # Give enough time for task wipes to work - see issue #1279
     assert x.tid not in parsl.dfk().tasks, "Task record should be wiped after task completion"

{parsl-2025.9.29 → parsl-2025.10.6}/parsl/tests/test_python_apps/test_memoize_2.py

@@ -4,7 +4,17 @@ import pytest
 
 import parsl
 from parsl.app.app import python_app
-from parsl.tests.configs.local_threads_no_cache import fresh_config as local_config
+from parsl.config import Config
+from parsl.executors.threads import ThreadPoolExecutor
+
+
+def local_config():
+    return Config(
+        executors=[
+            ThreadPoolExecutor(max_threads=4),
+        ],
+        app_cache=False
+    )
 
 
 @python_app

{parsl-2025.9.29 → parsl-2025.10.6}/parsl/version.py

@@ -3,4 +3,4 @@
 Year.Month.Day[alpha/beta/..]
 Alphas will be numbered like this -> 2024.12.10a0
 """
-VERSION = '2025.09.29'
+VERSION = '2025.10.06'

{parsl-2025.9.29 → parsl-2025.10.6/parsl.egg-info}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: parsl
-Version: 2025.9.29
+Version: 2025.10.6
 Summary: Simple data dependent workflows in Python
 Home-page: https://github.com/Parsl/parsl
-Download-URL: https://github.com/Parsl/parsl/archive/2025.09.29.tar.gz
+Download-URL: https://github.com/Parsl/parsl/archive/2025.10.06.tar.gz
 Author: The Parsl Team
 Author-email: parsl@googlegroups.com
 License: Apache 2.0

{parsl-2025.9.29 → parsl-2025.10.6}/parsl.egg-info/SOURCES.txt

@@ -241,7 +241,6 @@ parsl/tests/configs/local_threads_checkpoint_task_exit.py
 parsl/tests/configs/local_threads_ftp_in_task.py
 parsl/tests/configs/local_threads_globus.py
 parsl/tests/configs/local_threads_http_in_task.py
-parsl/tests/configs/local_threads_no_cache.py
 parsl/tests/configs/midway.py
 parsl/tests/configs/nscc_singapore.py
 parsl/tests/configs/osg_htex.py

parsl-2025.9.29/parsl/tests/configs/local_threads_no_cache.py

@@ -1,11 +0,0 @@
-from parsl.config import Config
-from parsl.executors.threads import ThreadPoolExecutor
-
-
-def fresh_config():
-    return Config(
-        executors=[
-            ThreadPoolExecutor(max_threads=4),
-        ],
-        app_cache=False
-    )