parsl 2025.9.29__tar.gz → 2025.10.13__tar.gz

{parsl-2025.9.29/parsl.egg-info → parsl-2025.10.13}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: parsl
-Version: 2025.9.29
+Version: 2025.10.13
 Summary: Simple data dependent workflows in Python
 Home-page: https://github.com/Parsl/parsl
-Download-URL: https://github.com/Parsl/parsl/archive/2025.09.29.tar.gz
+Download-URL: https://github.com/Parsl/parsl/archive/2025.10.13.tar.gz
 Author: The Parsl Team
 Author-email: parsl@googlegroups.com
 License: Apache 2.0

{parsl-2025.9.29 → parsl-2025.10.13}/parsl/benchmark/perf.py

@@ -6,6 +6,7 @@ from typing import Any, Dict
 
 import parsl
 from parsl.dataflow.dflow import DataFlowKernel
+from parsl.errors import InternalConsistencyError
 
 min_iterations = 2
 
@@ -40,19 +41,18 @@ def app(extra_payload: Any, parsl_resource_specification: Dict = {}) -> int:
     return 7
 
 
-def performance(*, resources: dict, target_t: float, args_extra_size: int) -> None:
+def performance(*, resources: dict, target_t: float, args_extra_size: int, iterate_mode: str) -> None:
     n = 10
 
     delta_t: float
-    delta_t = 0
-
-    threshold_t = int(0.75 * target_t)
 
     iteration = 1
 
     args_extra_payload = "x" * args_extra_size
 
-    while delta_t < threshold_t or iteration <= min_iterations:
+    iterate = True
+
+    while iterate:
         print(f"==== Iteration {iteration} ====")
         print(f"Will run {n} tasks to target {target_t} seconds runtime")
         start_t = time.time()
@@ -78,10 +78,20 @@ def performance(*, resources: dict, target_t: float, args_extra_size: int) -> No
         print(f"Runtime: actual {delta_t:.3f}s vs target {target_t}s")
         print(f"Tasks per second: {rate:.3f}")
 
-        n = max(1, int(target_t * rate))
-
         iteration += 1
 
+        # decide upon next iteration
+
+        match iterate_mode:
+            case "estimate":
+                n = max(1, int(target_t * rate))
+                iterate = delta_t < (0.75 * target_t) or iteration <= min_iterations
+            case "exponential":
+                n = int(n * 2)
+                iterate = delta_t < target_t or iteration <= min_iterations
+            case _:
+                raise InternalConsistencyError(f"Bad iterate mode {iterate_mode} - should have been validated at arg parse time")
+
 
 def cli_run() -> None:
     parser = argparse.ArgumentParser(
@@ -96,6 +106,12 @@ Example usage: python -m parsl.benchmark.perf --config parsl/tests/configs/workq
     parser.add_argument("--time", metavar="SECONDS", help="target number of seconds for an iteration", default=120, type=float)
     parser.add_argument("--argsize", metavar="BYTES", help="extra bytes to add into app invocation arguments", default=0, type=int)
     parser.add_argument("--version", action="version", version=f"parsl-perf from Parsl {parsl.__version__}")
+    parser.add_argument("--iterate",
+                        metavar="MODE",
+                        help="Iteration mode: estimate, exponential",
+                        type=str,
+                        default="estimate",
+                        choices=("estimate", "exponential"))
 
     args = parser.parse_args()
 
@@ -105,7 +121,7 @@ Example usage: python -m parsl.benchmark.perf --config parsl/tests/configs/workq
         resources = {}
 
     with load_dfk_from_config(args.config):
-        performance(resources=resources, target_t=args.time, args_extra_size=args.argsize)
+        performance(resources=resources, target_t=args.time, args_extra_size=args.argsize, iterate_mode=args.iterate)
         print("Tests complete - leaving DFK block")
     print("The end")
 

{parsl-2025.9.29 → parsl-2025.10.13}/parsl/dataflow/dflow.py

@@ -375,6 +375,7 @@ class DataFlowKernel:
                 logger.info("Task {} failed due to dependency failure so skipping retries".format(task_id))
                 task_record['time_returned'] = datetime.datetime.now()
                 self._send_task_log_info(task_record)
+                self.memoizer.update_memo(task_record)
                 with task_record['app_fu']._update_lock:
                     task_record['app_fu'].set_exception(e)
 
@@ -400,6 +401,7 @@ class DataFlowKernel:
                 self.update_task_state(task_record, States.failed)
                 task_record['time_returned'] = datetime.datetime.now()
                 self._send_task_log_info(task_record)
+                self.memoizer.update_memo(task_record)
                 with task_record['app_fu']._update_lock:
                     task_record['app_fu'].set_exception(e)
 
@@ -446,6 +448,7 @@ class DataFlowKernel:
                         self.update_task_state(task_record, States.failed)
                         task_record['time_returned'] = datetime.datetime.now()
                         self._send_task_log_info(task_record)
+                        self.memoizer.update_memo(task_record)
                         with task_record['app_fu']._update_lock:
                             task_record['app_fu'].set_exception(
                                 TypeError(f"join_app body must return a Future or list of Futures, got {joinable} of type {type(joinable)}"))
@@ -521,6 +524,7 @@ class DataFlowKernel:
 
                 self.update_task_state(task_record, States.failed)
                 task_record['time_returned'] = datetime.datetime.now()
+                self.memoizer.update_memo(task_record)
                 with task_record['app_fu']._update_lock:
                     task_record['app_fu'].set_exception(e)
 
@@ -561,8 +565,6 @@ class DataFlowKernel:
         if not task_record['app_fu'] == future:
             logger.error("Internal consistency error: callback future is not the app_fu in task structure, for task {}".format(task_id))
 
-        self.memoizer.update_memo(task_record)
-
         # Cover all checkpointing cases here:
         # Do we need to checkpoint now, or queue for later,
         # or do nothing?
@@ -591,6 +593,7 @@ class DataFlowKernel:
         logger.info(f"Task {task_record['id']} completed ({old_state.name} -> {new_state.name})")
         task_record['time_returned'] = datetime.datetime.now()
 
+        self.memoizer.update_memo(task_record)
         with task_record['app_fu']._update_lock:
             task_record['app_fu'].set_result(result)
 

{parsl-2025.9.29 → parsl-2025.10.13}/parsl/executors/base.py

@@ -80,11 +80,11 @@ class ParslExecutor(metaclass=ABCMeta):
         self.shutdown()
         return False
 
-    @abstractmethod
     def start(self) -> None:
         """Start the executor.
 
-        Any spin-up operations (for example: starting thread pools) should be performed here.
+        By default, this does nothing, but this method should be overridden to
+        perform any spin-up operations (for example: starting thread pools).
         """
         pass
 

{parsl-2025.9.29 → parsl-2025.10.13}/parsl/executors/globus_compute.py

@@ -76,10 +76,6 @@ class GlobusComputeExecutor(ParslExecutor, RepresentationMixin):
         self.storage_access = storage_access
         self.working_dir = working_dir
 
-    def start(self) -> None:
-        """ Start the Globus Compute Executor """
-        super().start()
-
     def submit(self, func: Callable, resource_specification: Dict[str, Any], *args: Any, **kwargs: Any) -> Future:
         """ Submit func to globus-compute
 

{parsl-2025.9.29 → parsl-2025.10.13}/parsl/executors/high_throughput/executor.py

@@ -160,6 +160,12 @@ GENERAL_HTEX_PARAM_DOCS = """provider : :class:`~parsl.providers.base.ExecutionP
 """  # Documentation for params used by both HTEx and MPIEx
 
 
+class HTEXFuture(Future):
+    def __init__(self, task_id) -> None:
+        super().__init__()
+        self.parsl_executor_task_id = task_id
+
+
 class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin, UsageInformation):
     __doc__ = f"""Executor designed for cluster-scale
 
@@ -670,7 +676,7 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin, UsageIn
                 logger.debug("Sending hold to manager: {}".format(manager['manager']))
                 self._hold_manager(manager['manager'])
 
-    def submit(self, func, resource_specification, *args, **kwargs):
+    def submit(self, func: Callable, resource_specification: dict, *args, **kwargs) -> HTEXFuture:
         """Submits work to the outgoing_q.
 
         The outgoing_q is an external process listens on this
@@ -691,21 +697,11 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin, UsageIn
 
         self.validate_resource_spec(resource_specification)
 
-        if self.bad_state_is_set:
-            raise self.executor_exception
-
-        self._task_counter += 1
-        task_id = self._task_counter
-
         # handle people sending blobs gracefully
         if logger.getEffectiveLevel() <= logging.DEBUG:
             args_to_print = tuple([ar if len(ar := repr(arg)) < 100 else (ar[:100] + '...') for arg in args])
             logger.debug("Pushing function {} to queue with args {}".format(func, args_to_print))
 
-        fut = Future()
-        fut.parsl_executor_task_id = task_id
-        self.tasks[task_id] = fut
-
         try:
             fn_buf = pack_apply_message(func, args, kwargs, buffer_threshold=1 << 20)
         except TypeError:
@@ -715,12 +711,69 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin, UsageIn
         if resource_specification:
             context["resource_spec"] = resource_specification
 
-        msg = {"task_id": task_id, "context": context, "buffer": fn_buf}
+        return self.submit_payload(context, fn_buf)
+
+    def submit_payload(self, context: dict, buffer: bytes) -> HTEXFuture:
+        """
+        Submit specially crafted payloads.
+
+        For use-cases where the ``HighThroughputExecutor`` consumer needs the payload
+        handled by the worker in a special way.  For example, if the function is
+        serialized differently than Parsl's default approach, or if the task must
+        be setup more precisely than Parsl's default ``execute_task`` allows.
+
+        An example interaction:
+
+        .. code-block: python
+
+            >>> htex: HighThroughputExecutor  # setup prior to this example
+            >>> ctxt = {
+            ...   "task_executor": {
+            ...     "f": "full.import.path.of.custom_execute_task",
+            ...     "a": ("additional", "arguments"),
+            ...     "k": {"some": "keyword", "args": "here"}
+            ...   }
+            ... }
+            >>> fn_buf = custom_serialize(task_func, *task_args, **task_kwargs)
+            >>> fut = htex.submit_payload(ctxt, fn_buf)
+
+        The custom ``custom_execute_task`` would be dynamically imported, and
+        invoked as:
+
+        .. code-block: python
+
+            args = ("additional", "arguments")
+            kwargs = {"some": "keyword", "args": "here"}
+            result = custom_execute_task(fn_buf, *args, **kwargs)
+
+        Parameters
+        ----------
+        context:
+            A task-specific context associated with the function buffer.  Parsl
+            currently implements the keys ``task_executor`` and ``resource_spec``
+
+        buffer:
+            A serialized function, that will be deserialized and executed by
+            ``execute_task`` (or custom function, if ``task_executor`` is specified)
+
+        Returns
+        -------
+        An HTEXFuture (a normal Future, with the attribute ``.parsl_executor_task_id``
+        set).  The future will be set to done when the associated function buffer has
+        been invoked and completed.
+        """
+        if self.bad_state_is_set:
+            raise self.executor_exception
+
+        self._task_counter += 1
+        task_id = self._task_counter
+
+        fut = HTEXFuture(task_id)
+        self.tasks[task_id] = fut
 
-        # Post task to the outgoing queue
+        msg = {"task_id": task_id, "context": context, "buffer": buffer}
         self.outgoing_q.put(msg)
 
-        # Return the future
         return fut
 
     @property

{parsl-2025.9.29 → parsl-2025.10.13}/parsl/executors/high_throughput/process_worker_pool.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
 import argparse
+import importlib
 import logging
 import math
 import multiprocessing
@@ -17,7 +18,7 @@ from importlib.metadata import distributions
 from multiprocessing.context import SpawnProcess
 from multiprocessing.managers import DictProxy
 from multiprocessing.sharedctypes import Synchronized
-from typing import Dict, List, Optional, Sequence
+from typing import Callable, Dict, List, Optional, Sequence
 
 import psutil
 import zmq
@@ -378,7 +379,7 @@ class Manager:
                 last_interchange_contact = time.time()
 
                 if tasks == HEARTBEAT_CODE:
-                    logger.debug("Got heartbeat from interchange")
+                    logger.debug("Got heartbeat response from interchange")
                 elif tasks == DRAINED_CODE:
                     logger.info("Got fully drained message from interchange - setting kill flag")
                     self._stop_event.set()
@@ -778,8 +779,20 @@ def worker(
 
         _init_mpi_env(mpi_launcher=mpi_launcher, resource_spec=res_spec)
 
+        exec_func: Callable = execute_task
+        exec_args = ()
+        exec_kwargs = {}
+
         try:
-            result = execute_task(req['buffer'])
+            if task_executor := ctxt.get("task_executor", None):
+                mod_name, _, fn_name = task_executor["f"].rpartition(".")
+                exec_mod = importlib.import_module(mod_name)
+                exec_func = getattr(exec_mod, fn_name)
+
+                exec_args = task_executor.get("a", ())
+                exec_kwargs = task_executor.get("k", {})
+
+            result = exec_func(req['buffer'], *exec_args, **exec_kwargs)
             serialized_result = serialize(result, buffer_threshold=1000000)
         except Exception as e:
             logger.info('Caught an exception: {}'.format(e))

{parsl-2025.9.29 → parsl-2025.10.13}/parsl/monitoring/radios/filesystem.py

@@ -10,11 +10,12 @@ from parsl.monitoring.radios.base import (
     RadioConfig,
 )
 from parsl.monitoring.radios.filesystem_router import FilesystemRadioReceiver
+from parsl.utils import RepresentationMixin
 
 logger = logging.getLogger(__name__)
 
 
-class FilesystemRadio(RadioConfig):
+class FilesystemRadio(RadioConfig, RepresentationMixin):
     """A MonitoringRadioSender that sends messages over a shared filesystem.
 
     The messsage directory structure is based on maildir,

{parsl-2025.9.29 → parsl-2025.10.13}/parsl/monitoring/radios/htex.py

@@ -7,11 +7,12 @@ from parsl.monitoring.radios.base import (
     MonitoringRadioSender,
     RadioConfig,
 )
+from parsl.utils import RepresentationMixin
 
 logger = logging.getLogger(__name__)
 
 
-class HTEXRadio(RadioConfig):
+class HTEXRadio(RadioConfig, RepresentationMixin):
     def create_sender(self) -> MonitoringRadioSender:
         return HTEXRadioSender()
 

{parsl-2025.9.29 → parsl-2025.10.13}/parsl/monitoring/radios/multiprocessing.py

@@ -5,9 +5,10 @@ from parsl.monitoring.radios.base import (
     MonitoringRadioSender,
     RadioConfig,
 )
+from parsl.utils import RepresentationMixin
 
 
-class MultiprocessingQueueRadioSender(MonitoringRadioSender):
+class MultiprocessingQueueRadioSender(MonitoringRadioSender, RepresentationMixin):
     """A monitoring radio which connects over a multiprocessing Queue.
     This radio is intended to be used on the submit side, where components
     in the submit process, or processes launched by multiprocessing, will have

{parsl-2025.9.29 → parsl-2025.10.13}/parsl/monitoring/radios/udp.py

@@ -13,11 +13,12 @@ from parsl.monitoring.radios.base import (
     RadioConfig,
 )
 from parsl.monitoring.radios.udp_router import start_udp_receiver
+from parsl.utils import RepresentationMixin
 
 logger = logging.getLogger(__name__)
 
 
-class UDPRadio(RadioConfig):
+class UDPRadio(RadioConfig, RepresentationMixin):
     def __init__(self, *, port: Optional[int] = None, atexit_timeout: int = 3, address: str, debug: bool = False, hmac_digest: str = 'sha512'):
         self.port = port
         self.atexit_timeout = atexit_timeout

{parsl-2025.9.29 → parsl-2025.10.13}/parsl/providers/base.py

@@ -33,7 +33,28 @@ class ExecutionProvider(metaclass=ABCMeta):
           [cancel]     <--------|----+
                                 |
                                 +-------------------
-     """
+
+    In addition to the listed methods, an ExecutionProvider instance must always
+    have these attributes, which both default to `None`:
+
+      mem_per_node: Real memory to provision per node in GB.
+
+                    Providers which set this attribute should ask for mem_per_node of memory
+                    when provisioning resources, and set the corresponding environment
+                    variable PARSL_MEMORY_GB before executing submitted commands.
+
+                    If this attribute is set, executors may use it to calculate how many tasks can
+                    run concurrently per node.
+
+      cores_per_node: Number of cores to provision per node.
+
+                    Providers which set this attribute should ask for cores_per_node cores
+                    when provisioning resources, and set the corresponding environment
+                    variable PARSL_CORES before executing submitted commands.
+
+                    If this attribute is set, executors may use it to calculate how many tasks can
+                    run concurrently per node.
+    """
 
     @abstractmethod
     def __init__(self) -> None:
@@ -44,8 +65,8 @@ class ExecutionProvider(metaclass=ABCMeta):
         self.script_dir: Optional[str]
         self.parallelism: float
         self.resources: Dict[object, Any]
-        self._cores_per_node: Optional[int] = None
-        self._mem_per_node: Optional[float] = None
+        self.cores_per_node: Optional[int] = None
+        self.mem_per_node: Optional[float] = None
         pass
 
     @abstractmethod
@@ -111,40 +132,6 @@ class ExecutionProvider(metaclass=ABCMeta):
         ''' Provides the label for this provider '''
         pass
 
-    @property
-    def mem_per_node(self) -> Optional[float]:
-        """Real memory to provision per node in GB.
-
-        Providers which set this property should ask for mem_per_node of memory
-        when provisioning resources, and set the corresponding environment
-        variable PARSL_MEMORY_GB before executing submitted commands.
-
-        If this property is set, executors may use it to calculate how many tasks can
-        run concurrently per node.
-        """
-        return self._mem_per_node
-
-    @mem_per_node.setter
-    def mem_per_node(self, value: float) -> None:
-        self._mem_per_node = value
-
-    @property
-    def cores_per_node(self) -> Optional[int]:
-        """Number of cores to provision per node.
-
-        Providers which set this property should ask for cores_per_node cores
-        when provisioning resources, and set the corresponding environment
-        variable PARSL_CORES before executing submitted commands.
-
-        If this property is set, executors may use it to calculate how many tasks can
-        run concurrently per node.
-        """
-        return self._cores_per_node
-
-    @cores_per_node.setter
-    def cores_per_node(self, value: int) -> None:
-        self._cores_per_node = value
-
     @property
     @abstractmethod
     def status_polling_interval(self) -> int:

{parsl-2025.9.29 → parsl-2025.10.13}/parsl/tests/test_checkpointing/test_regression_233.py

@@ -5,7 +5,6 @@ from parsl.dataflow.dflow import DataFlowKernel
 
 
 def run_checkpointed(checkpoints):
-    # set_stream_logger()
     from parsl.tests.configs.local_threads_checkpoint_task_exit import config
     config.checkpoint_files = checkpoints
     dfk = DataFlowKernel(config=config)

{parsl-2025.9.29 → parsl-2025.10.13}/parsl/tests/test_htex/test_htex.py

@@ -7,6 +7,7 @@ from unittest import mock
 import pytest
 
 from parsl import HighThroughputExecutor, curvezmq
+from parsl.serialize.facade import pack_apply_message, unpack_apply_message
 
 _MOCK_BASE = "parsl.executors.high_throughput.executor"
 
@@ -19,11 +20,16 @@ def encrypted(request: pytest.FixtureRequest):
 
 
 @pytest.fixture
-def htex(encrypted: bool):
+def htex(encrypted: bool, tmpd_cwd):
     htex = HighThroughputExecutor(encrypted=encrypted)
+    htex.max_workers_per_node = 1
+    htex.run_dir = tmpd_cwd
+    htex.provider.script_dir = tmpd_cwd
 
     yield htex
 
+    if hasattr(htex, "outgoing_q"):
+        htex.scale_in(blocks=1000)
     htex.shutdown()
 
 
@@ -146,3 +152,32 @@ def test_htex_interchange_launch_cmd(cmd: Optional[Sequence[str]]):
     else:
         htex = HighThroughputExecutor()
         assert htex.interchange_launch_cmd == ["interchange.py"]
+
+
+def dyn_exec(buf, *vec_y):
+    f, a, _ = unpack_apply_message(buf)
+    custom_args = [a, vec_y]
+    return f(*custom_args)
+
+
+@pytest.mark.local
+def test_worker_dynamic_import(htex: HighThroughputExecutor):
+    def _dot_prod(vec_x, vec_y):
+        return sum(x * y for x, y in zip(vec_x, vec_y))
+
+    htex.start()
+    htex.scale_out_facade(1)
+
+    num_array = tuple(range(10))
+
+    fn_buf = pack_apply_message(_dot_prod, num_array, {})
+    ctxt = {
+        "task_executor": {
+            "f": f"{dyn_exec.__module__}.{dyn_exec.__name__}",
+            "a": num_array,  # prove "custom" dyn_exec
+        }
+    }
+    val = htex.submit_payload(ctxt, fn_buf).result()
+    exp_val = _dot_prod(num_array, num_array)
+
+    assert val == exp_val

{parsl-2025.9.29 → parsl-2025.10.13}/parsl/tests/test_python_apps/test_garbage_collect.py

@@ -27,10 +27,5 @@ def test_garbage_collect():
 
     evt.set()
     assert x.result() == 10 * 4
-    if parsl.dfk().checkpoint_mode is not None:
-        # We explicit call checkpoint if checkpoint_mode is enabled covering
-        # cases like manual/periodic where checkpointing may be deferred.
-        parsl.dfk().checkpoint()
-
-    time.sleep(0.01)  # Give enough time for task wipes to work
+    time.sleep(0.01)  # Give enough time for task wipes to work - see issue #1279
     assert x.tid not in parsl.dfk().tasks, "Task record should be wiped after task completion"

{parsl-2025.9.29 → parsl-2025.10.13}/parsl/tests/test_python_apps/test_memoize_2.py

@@ -4,7 +4,17 @@ import pytest
 
 import parsl
 from parsl.app.app import python_app
-from parsl.tests.configs.local_threads_no_cache import fresh_config as local_config
+from parsl.config import Config
+from parsl.executors.threads import ThreadPoolExecutor
+
+
+def local_config():
+    return Config(
+        executors=[
+            ThreadPoolExecutor(max_threads=4),
+        ],
+        app_cache=False
+    )
 
 
 @python_app

{parsl-2025.9.29 → parsl-2025.10.13}/parsl/version.py

@@ -3,4 +3,4 @@
 Year.Month.Day[alpha/beta/..]
 Alphas will be numbered like this -> 2024.12.10a0
 """
-VERSION = '2025.09.29'
+VERSION = '2025.10.13'

{parsl-2025.9.29 → parsl-2025.10.13/parsl.egg-info}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: parsl
-Version: 2025.9.29
+Version: 2025.10.13
 Summary: Simple data dependent workflows in Python
 Home-page: https://github.com/Parsl/parsl
-Download-URL: https://github.com/Parsl/parsl/archive/2025.09.29.tar.gz
+Download-URL: https://github.com/Parsl/parsl/archive/2025.10.13.tar.gz
 Author: The Parsl Team
 Author-email: parsl@googlegroups.com
 License: Apache 2.0

{parsl-2025.9.29 → parsl-2025.10.13}/parsl.egg-info/SOURCES.txt

@@ -241,7 +241,6 @@ parsl/tests/configs/local_threads_checkpoint_task_exit.py
 parsl/tests/configs/local_threads_ftp_in_task.py
 parsl/tests/configs/local_threads_globus.py
 parsl/tests/configs/local_threads_http_in_task.py
-parsl/tests/configs/local_threads_no_cache.py
 parsl/tests/configs/midway.py
 parsl/tests/configs/nscc_singapore.py
 parsl/tests/configs/osg_htex.py

parsl-2025.9.29/parsl/tests/configs/local_threads_no_cache.py

@@ -1,11 +0,0 @@
-from parsl.config import Config
-from parsl.executors.threads import ThreadPoolExecutor
-
-
-def fresh_config():
-    return Config(
-        executors=[
-            ThreadPoolExecutor(max_threads=4),
-        ],
-        app_cache=False
-    )