parsl 2025.1.20__py3-none-any.whl → 2025.2.3__py3-none-any.whl

parsl/dataflow/dflow.py

@@ -28,7 +28,7 @@ from parsl.config import Config
 from parsl.data_provider.data_manager import DataManager
 from parsl.data_provider.files import File
 from parsl.dataflow.dependency_resolvers import SHALLOW_DEPENDENCY_RESOLVER
-from parsl.dataflow.errors import BadCheckpoint, DependencyError, JoinError
+from parsl.dataflow.errors import DependencyError, JoinError
 from parsl.dataflow.futures import AppFuture
 from parsl.dataflow.memoization import Memoizer
 from parsl.dataflow.rundirs import make_rundir
@@ -161,13 +161,13 @@ class DataFlowKernel:
                                  workflow_info))
 
         if config.checkpoint_files is not None:
-            checkpoints = self.load_checkpoints(config.checkpoint_files)
+            checkpoint_files = config.checkpoint_files
         elif config.checkpoint_files is None and config.checkpoint_mode is not None:
-            checkpoints = self.load_checkpoints(get_all_checkpoints(self.run_dir))
+            checkpoint_files = get_all_checkpoints(self.run_dir)
         else:
-            checkpoints = {}
+            checkpoint_files = []
 
-        self.memoizer = Memoizer(self, memoize=config.app_cache, checkpoint=checkpoints)
+        self.memoizer = Memoizer(self, memoize=config.app_cache, checkpoint_files=checkpoint_files)
         self.checkpointed_tasks = 0
         self._checkpoint_timer = None
         self.checkpoint_mode = config.checkpoint_mode
@@ -1263,7 +1263,7 @@ class DataFlowKernel:
         Returns:
             Checkpoint dir if checkpoints were written successfully.
             By default the checkpoints are written to the RUNDIR of the current
-            run under RUNDIR/checkpoints/{tasks.pkl, dfk.pkl}
+            run under RUNDIR/checkpoints/tasks.pkl
         """
         with self.checkpoint_lock:
             if tasks:
@@ -1273,18 +1273,11 @@ class DataFlowKernel:
                 self.checkpointable_tasks = []
 
             checkpoint_dir = '{0}/checkpoint'.format(self.run_dir)
-            checkpoint_dfk = checkpoint_dir + '/dfk.pkl'
             checkpoint_tasks = checkpoint_dir + '/tasks.pkl'
 
             if not os.path.exists(checkpoint_dir):
                 os.makedirs(checkpoint_dir, exist_ok=True)
 
-            with open(checkpoint_dfk, 'wb') as f:
-                state = {'rundir': self.run_dir,
-                         'task_count': self.task_count
-                         }
-                pickle.dump(state, f)
-
             count = 0
 
             with open(checkpoint_tasks, 'ab') as f:
@@ -1317,74 +1310,6 @@ class DataFlowKernel:
 
             return checkpoint_dir
 
-    def _load_checkpoints(self, checkpointDirs: Sequence[str]) -> Dict[str, Future[Any]]:
-        """Load a checkpoint file into a lookup table.
-
-        The data being loaded from the pickle file mostly contains input
-        attributes of the task: func, args, kwargs, env...
-        To simplify the check of whether the exact task has been completed
-        in the checkpoint, we hash these input params and use it as the key
-        for the memoized lookup table.
-
-        Args:
-            - checkpointDirs (list) : List of filepaths to checkpoints
-              Eg. ['runinfo/001', 'runinfo/002']
-
-        Returns:
-            - memoized_lookup_table (dict)
-        """
-        memo_lookup_table = {}
-
-        for checkpoint_dir in checkpointDirs:
-            logger.info("Loading checkpoints from {}".format(checkpoint_dir))
-            checkpoint_file = os.path.join(checkpoint_dir, 'tasks.pkl')
-            try:
-                with open(checkpoint_file, 'rb') as f:
-                    while True:
-                        try:
-                            data = pickle.load(f)
-                            # Copy and hash only the input attributes
-                            memo_fu: Future = Future()
-                            assert data['exception'] is None
-                            memo_fu.set_result(data['result'])
-                            memo_lookup_table[data['hash']] = memo_fu
-
-                        except EOFError:
-                            # Done with the checkpoint file
-                            break
-            except FileNotFoundError:
-                reason = "Checkpoint file was not found: {}".format(
-                    checkpoint_file)
-                logger.error(reason)
-                raise BadCheckpoint(reason)
-            except Exception:
-                reason = "Failed to load checkpoint: {}".format(
-                    checkpoint_file)
-                logger.error(reason)
-                raise BadCheckpoint(reason)
-
-            logger.info("Completed loading checkpoint: {0} with {1} tasks".format(checkpoint_file,
-                                                                                  len(memo_lookup_table.keys())))
-        return memo_lookup_table
-
-    @typeguard.typechecked
-    def load_checkpoints(self, checkpointDirs: Optional[Sequence[str]]) -> Dict[str, Future]:
-        """Load checkpoints from the checkpoint files into a dictionary.
-
-        The results are used to pre-populate the memoizer's lookup_table
-
-        Kwargs:
-             - checkpointDirs (list) : List of run folder to use as checkpoints
-               Eg. ['runinfo/001', 'runinfo/002']
-
-        Returns:
-             - dict containing, hashed -> future mappings
-        """
-        if checkpointDirs:
-            return self._load_checkpoints(checkpointDirs)
-        else:
-            return {}
-
     @staticmethod
     def _log_std_streams(task_record: TaskRecord) -> None:
         tid = task_record['id']

parsl/dataflow/errors.py

@@ -1,4 +1,4 @@
-from typing import Optional, Sequence, Tuple
+from typing import List, Sequence, Tuple
 
 from parsl.errors import ParslError
 
@@ -29,35 +29,77 @@ class BadCheckpoint(DataFlowException):
         return self.reason
 
 
-class DependencyError(DataFlowException):
-    """Error raised if an app cannot run because there was an error
-       in a dependency.
+class PropagatedException(DataFlowException):
+    """Error raised if an app fails because there was an error
+       in a related task. This is intended to be subclassed for
+       dependency and join_app errors.
 
     Args:
-         - dependent_exceptions_tids: List of exceptions and identifiers for
-           dependencies which failed. The identifier might be a task ID or
-           the repr of a non-DFK Future.
+         - dependent_exceptions_tids: List of exceptions and brief descriptions
+           for dependencies which failed. The description might be a task ID or
+           the repr of a non-AppFuture.
          - task_id: Task ID of the task that failed because of the dependency error
     """
 
-    def __init__(self, dependent_exceptions_tids: Sequence[Tuple[Exception, str]], task_id: int) -> None:
+    def __init__(self,
+                 dependent_exceptions_tids: Sequence[Tuple[BaseException, str]],
+                 task_id: int,
+                 *,
+                 failure_description: str) -> None:
         self.dependent_exceptions_tids = dependent_exceptions_tids
         self.task_id = task_id
+        self._failure_description = failure_description
+
+        (cause, cause_sequence) = self._find_any_root_cause()
+        self.__cause__ = cause
+        self._cause_sequence = cause_sequence
 
     def __str__(self) -> str:
-        deps = ", ".join(tid for _exc, tid in self.dependent_exceptions_tids)
-        return f"Dependency failure for task {self.task_id} with failed dependencies from {deps}"
+        sequence_text = " <- ".join(self._cause_sequence)
+        return f"{self._failure_description} for task {self.task_id}. " \
+               f"The representative cause is via {sequence_text}"
+
+    def _find_any_root_cause(self) -> Tuple[BaseException, List[str]]:
+        """Looks recursively through self.dependent_exceptions_tids to find
+        an exception that caused this propagated error, that is not itself
+        a propagated error.
+        """
+        e: BaseException = self
+        dep_ids = []
+        while isinstance(e, PropagatedException) and len(e.dependent_exceptions_tids) >= 1:
+            id_txt = e.dependent_exceptions_tids[0][1]
+            assert isinstance(id_txt, str)
+            # if there are several causes for this exception, label that
+            # there are more so that we know that the representative fail
+            # sequence is not the full story.
+            if len(e.dependent_exceptions_tids) > 1:
+                id_txt += " (+ others)"
+            dep_ids.append(id_txt)
+            e = e.dependent_exceptions_tids[0][0]
+        return e, dep_ids
+
+
+class DependencyError(PropagatedException):
+    """Error raised if an app cannot run because there was an error
+       in a dependency. There can be several exceptions (one from each
+       dependency) and DependencyError collects them all together.
 
+    Args:
+         - dependent_exceptions_tids: List of exceptions and brief descriptions
+           for dependencies which failed. The description might be a task ID or
+           the repr of a non-AppFuture.
+         - task_id: Task ID of the task that failed because of the dependency error
+    """
+    def __init__(self, dependent_exceptions_tids: Sequence[Tuple[BaseException, str]], task_id: int) -> None:
+        super().__init__(dependent_exceptions_tids, task_id,
+                         failure_description="Dependency failure")
 
-class JoinError(DataFlowException):
+
+class JoinError(PropagatedException):
     """Error raised if apps joining into a join_app raise exceptions.
        There can be several exceptions (one from each joining app),
        and JoinError collects them all together.
     """
-    def __init__(self, dependent_exceptions_tids: Sequence[Tuple[BaseException, Optional[str]]], task_id: int) -> None:
-        self.dependent_exceptions_tids = dependent_exceptions_tids
-        self.task_id = task_id
-
-    def __str__(self) -> str:
-        dep_tids = [tid for (exception, tid) in self.dependent_exceptions_tids]
-        return "Join failure for task {} with failed join dependencies from tasks {}".format(self.task_id, dep_tids)
+    def __init__(self, dependent_exceptions_tids: Sequence[Tuple[BaseException, str]], task_id: int) -> None:
+        super().__init__(dependent_exceptions_tids, task_id,
+                         failure_description="Join failure")

parsl/dataflow/memoization.py

@@ -2,10 +2,14 @@ from __future__ import annotations
 
 import hashlib
 import logging
+import os
 import pickle
 from functools import lru_cache, singledispatch
-from typing import TYPE_CHECKING, Any, Dict, List, Optional
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Sequence
 
+import typeguard
+
+from parsl.dataflow.errors import BadCheckpoint
 from parsl.dataflow.taskrecord import TaskRecord
 
 if TYPE_CHECKING:
@@ -146,7 +150,7 @@ class Memoizer:
 
     """
 
-    def __init__(self, dfk: DataFlowKernel, memoize: bool = True, checkpoint: Dict[str, Future[Any]] = {}):
+    def __init__(self, dfk: DataFlowKernel, *, memoize: bool = True, checkpoint_files: Sequence[str]):
         """Initialize the memoizer.
 
         Args:
@@ -159,6 +163,8 @@ class Memoizer:
         self.dfk = dfk
         self.memoize = memoize
 
+        checkpoint = self.load_checkpoints(checkpoint_files)
+
         if self.memoize:
             logger.info("App caching initialized")
             self.memo_lookup_table = checkpoint
@@ -274,3 +280,71 @@ class Memoizer:
         else:
             logger.debug(f"Storing app cache entry {task['hashsum']} with result from task {task_id}")
         self.memo_lookup_table[task['hashsum']] = r
+
+    def _load_checkpoints(self, checkpointDirs: Sequence[str]) -> Dict[str, Future[Any]]:
+        """Load a checkpoint file into a lookup table.
+
+        The data being loaded from the pickle file mostly contains input
+        attributes of the task: func, args, kwargs, env...
+        To simplify the check of whether the exact task has been completed
+        in the checkpoint, we hash these input params and use it as the key
+        for the memoized lookup table.
+
+        Args:
+            - checkpointDirs (list) : List of filepaths to checkpoints
+              Eg. ['runinfo/001', 'runinfo/002']
+
+        Returns:
+            - memoized_lookup_table (dict)
+        """
+        memo_lookup_table = {}
+
+        for checkpoint_dir in checkpointDirs:
+            logger.info("Loading checkpoints from {}".format(checkpoint_dir))
+            checkpoint_file = os.path.join(checkpoint_dir, 'tasks.pkl')
+            try:
+                with open(checkpoint_file, 'rb') as f:
+                    while True:
+                        try:
+                            data = pickle.load(f)
+                            # Copy and hash only the input attributes
+                            memo_fu: Future = Future()
+                            assert data['exception'] is None
+                            memo_fu.set_result(data['result'])
+                            memo_lookup_table[data['hash']] = memo_fu
+
+                        except EOFError:
+                            # Done with the checkpoint file
+                            break
+            except FileNotFoundError:
+                reason = "Checkpoint file was not found: {}".format(
+                    checkpoint_file)
+                logger.error(reason)
+                raise BadCheckpoint(reason)
+            except Exception:
+                reason = "Failed to load checkpoint: {}".format(
+                    checkpoint_file)
+                logger.error(reason)
+                raise BadCheckpoint(reason)
+
+            logger.info("Completed loading checkpoint: {0} with {1} tasks".format(checkpoint_file,
+                                                                                  len(memo_lookup_table.keys())))
+        return memo_lookup_table
+
+    @typeguard.typechecked
+    def load_checkpoints(self, checkpointDirs: Optional[Sequence[str]]) -> Dict[str, Future]:
+        """Load checkpoints from the checkpoint files into a dictionary.
+
+        The results are used to pre-populate the memoizer's lookup_table
+
+        Kwargs:
+             - checkpointDirs (list) : List of run folder to use as checkpoints
+               Eg. ['runinfo/001', 'runinfo/002']
+
+        Returns:
+             - dict containing, hashed -> future mappings
+        """
+        if checkpointDirs:
+            return self._load_checkpoints(checkpointDirs)
+        else:
+            return {}

parsl/executors/high_throughput/executor.py

@@ -431,8 +431,6 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin, UsageIn
         self._start_result_queue_thread()
         self._start_local_interchange_process()
 
-        logger.debug("Created result queue thread: %s", self._result_queue_thread)
-
         self.initialize_scaling()
 
     @wrap_with_logs
@@ -529,6 +527,8 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin, UsageIn
         get the worker task and result ports that the interchange has bound to.
         """
 
+        assert self.interchange_proc is None, f"Already exists! {self.interchange_proc!r}"
+
         interchange_config = {"client_address": self.loopback_address,
                               "client_ports": (self.outgoing_q.port,
                                                self.incoming_q.port,
@@ -563,7 +563,12 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin, UsageIn
         except CommandClientTimeoutError:
             logger.error("Interchange has not completed initialization. Aborting")
             raise Exception("Interchange failed to start")
-        logger.debug("Got worker ports")
+        logger.debug(
+            "Interchange process started (%r).  Worker ports: %d, %d",
+            self.interchange_proc,
+            self.worker_task_port,
+            self.worker_result_port
+        )
 
     def _start_result_queue_thread(self):
         """Method to start the result queue thread as a daemon.
@@ -571,15 +576,13 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin, UsageIn
         Checks if a thread already exists, then starts it.
         Could be used later as a restart if the result queue thread dies.
         """
-        if self._result_queue_thread is None:
-            logger.debug("Starting result queue thread")
-            self._result_queue_thread = threading.Thread(target=self._result_queue_worker, name="HTEX-Result-Queue-Thread")
-            self._result_queue_thread.daemon = True
-            self._result_queue_thread.start()
-            logger.debug("Started result queue thread")
+        assert self._result_queue_thread is None, f"Already exists! {self._result_queue_thread!r}"
 
-        else:
-            logger.error("Result queue thread already exists, returning")
+        logger.debug("Starting result queue thread")
+        self._result_queue_thread = threading.Thread(target=self._result_queue_worker, name="HTEX-Result-Queue-Thread")
+        self._result_queue_thread.daemon = True
+        self._result_queue_thread.start()
+        logger.debug("Started result queue thread: %r", self._result_queue_thread)
 
     def hold_worker(self, worker_id: str) -> None:
         """Puts a worker on hold, preventing scheduling of additional tasks to it.

parsl/executors/high_throughput/interchange.py

@@ -9,7 +9,7 @@ import queue
 import sys
 import threading
 import time
-from typing import Any, Dict, List, NoReturn, Optional, Sequence, Set, Tuple, cast
+from typing import Any, Dict, List, Optional, Sequence, Set, Tuple, cast
 
 import zmq
 
@@ -132,6 +132,11 @@ class Interchange:
         self.hub_zmq_port = hub_zmq_port
 
         self.pending_task_queue: queue.Queue[Any] = queue.Queue(maxsize=10 ** 6)
+
+        # count of tasks that have been received from the submit side
+        self.task_counter = 0
+
+        # count of tasks that have been sent out to worker pools
         self.count = 0
 
         self.worker_ports = worker_ports
@@ -201,28 +206,6 @@ class Interchange:
 
         return tasks
 
-    @wrap_with_logs(target="interchange")
-    def task_puller(self) -> NoReturn:
-        """Pull tasks from the incoming tasks zmq pipe onto the internal
-        pending task queue
-        """
-        logger.info("Starting")
-        task_counter = 0
-
-        while True:
-            logger.debug("launching recv_pyobj")
-            try:
-                msg = self.task_incoming.recv_pyobj()
-            except zmq.Again:
-                # We just timed out while attempting to receive
-                logger.debug("zmq.Again with {} tasks in internal queue".format(self.pending_task_queue.qsize()))
-                continue
-
-            logger.debug("putting message onto pending_task_queue")
-            self.pending_task_queue.put(msg)
-            task_counter += 1
-            logger.debug(f"Fetched {task_counter} tasks so far")
-
     def _send_monitoring_info(self, monitoring_radio: Optional[MonitoringRadioSender], manager: ManagerRecord) -> None:
         if monitoring_radio:
             logger.info("Sending message {} to MonitoringHub".format(manager))
@@ -234,79 +217,68 @@ class Interchange:
 
             monitoring_radio.send((MessageType.NODE_INFO, d))
 
-    @wrap_with_logs(target="interchange")
-    def _command_server(self) -> NoReturn:
+    def process_command(self, monitoring_radio: Optional[MonitoringRadioSender]) -> None:
         """ Command server to run async command to the interchange
         """
-        logger.debug("Command Server Starting")
-
-        if self.hub_address is not None and self.hub_zmq_port is not None:
-            logger.debug("Creating monitoring radio to %s:%s", self.hub_address, self.hub_zmq_port)
-            monitoring_radio = ZMQRadioSender(self.hub_address, self.hub_zmq_port)
-        else:
-            monitoring_radio = None
+        logger.debug("entering command_server section")
 
         reply: Any  # the type of reply depends on the command_req received (aka this needs dependent types...)
 
-        while True:
-            try:
-                command_req = self.command_channel.recv_pyobj()
-                logger.debug("Received command request: {}".format(command_req))
-                if command_req == "CONNECTED_BLOCKS":
-                    reply = self.connected_block_history
-
-                elif command_req == "WORKERS":
-                    num_workers = 0
-                    for manager in self._ready_managers.values():
-                        num_workers += manager['worker_count']
-                    reply = num_workers
-
-                elif command_req == "MANAGERS":
-                    reply = []
-                    for manager_id in self._ready_managers:
-                        m = self._ready_managers[manager_id]
-                        idle_since = m['idle_since']
-                        if idle_since is not None:
-                            idle_duration = time.time() - idle_since
-                        else:
-                            idle_duration = 0.0
-                        resp = {'manager': manager_id.decode('utf-8'),
-                                'block_id': m['block_id'],
-                                'worker_count': m['worker_count'],
-                                'tasks': len(m['tasks']),
-                                'idle_duration': idle_duration,
-                                'active': m['active'],
-                                'parsl_version': m['parsl_version'],
-                                'python_version': m['python_version'],
-                                'draining': m['draining']}
-                        reply.append(resp)
-
-                elif command_req.startswith("HOLD_WORKER"):
-                    cmd, s_manager = command_req.split(';')
-                    manager_id = s_manager.encode('utf-8')
-                    logger.info("Received HOLD_WORKER for {!r}".format(manager_id))
-                    if manager_id in self._ready_managers:
-                        m = self._ready_managers[manager_id]
-                        m['active'] = False
-                        self._send_monitoring_info(monitoring_radio, m)
+        if self.command_channel in self.socks and self.socks[self.command_channel] == zmq.POLLIN:
+
+            command_req = self.command_channel.recv_pyobj()
+            logger.debug("Received command request: {}".format(command_req))
+            if command_req == "CONNECTED_BLOCKS":
+                reply = self.connected_block_history
+
+            elif command_req == "WORKERS":
+                num_workers = 0
+                for manager in self._ready_managers.values():
+                    num_workers += manager['worker_count']
+                reply = num_workers
+
+            elif command_req == "MANAGERS":
+                reply = []
+                for manager_id in self._ready_managers:
+                    m = self._ready_managers[manager_id]
+                    idle_since = m['idle_since']
+                    if idle_since is not None:
+                        idle_duration = time.time() - idle_since
                     else:
-                        logger.warning("Worker to hold was not in ready managers list")
-
-                    reply = None
+                        idle_duration = 0.0
+                    resp = {'manager': manager_id.decode('utf-8'),
+                            'block_id': m['block_id'],
+                            'worker_count': m['worker_count'],
+                            'tasks': len(m['tasks']),
+                            'idle_duration': idle_duration,
+                            'active': m['active'],
+                            'parsl_version': m['parsl_version'],
+                            'python_version': m['python_version'],
+                            'draining': m['draining']}
+                    reply.append(resp)
+
+            elif command_req.startswith("HOLD_WORKER"):
+                cmd, s_manager = command_req.split(';')
+                manager_id = s_manager.encode('utf-8')
+                logger.info("Received HOLD_WORKER for {!r}".format(manager_id))
+                if manager_id in self._ready_managers:
+                    m = self._ready_managers[manager_id]
+                    m['active'] = False
+                    self._send_monitoring_info(monitoring_radio, m)
+                else:
+                    logger.warning("Worker to hold was not in ready managers list")
 
-                elif command_req == "WORKER_PORTS":
-                    reply = (self.worker_task_port, self.worker_result_port)
+                reply = None
 
-                else:
-                    logger.error(f"Received unknown command: {command_req}")
-                    reply = None
+            elif command_req == "WORKER_PORTS":
+                reply = (self.worker_task_port, self.worker_result_port)
 
-                logger.debug("Reply: {}".format(reply))
-                self.command_channel.send_pyobj(reply)
+            else:
+                logger.error(f"Received unknown command: {command_req}")
+                reply = None
 
-            except zmq.Again:
-                logger.debug("Command thread is alive")
-                continue
+            logger.debug("Reply: {}".format(reply))
+            self.command_channel.send_pyobj(reply)
 
     @wrap_with_logs
     def start(self) -> None:
@@ -326,21 +298,13 @@ class Interchange:
 
         start = time.time()
 
-        self._task_puller_thread = threading.Thread(target=self.task_puller,
-                                                    name="Interchange-Task-Puller",
-                                                    daemon=True)
-        self._task_puller_thread.start()
-
-        self._command_thread = threading.Thread(target=self._command_server,
-                                                name="Interchange-Command",
-                                                daemon=True)
-        self._command_thread.start()
-
         kill_event = threading.Event()
 
         poller = zmq.Poller()
         poller.register(self.task_outgoing, zmq.POLLIN)
         poller.register(self.results_incoming, zmq.POLLIN)
+        poller.register(self.task_incoming, zmq.POLLIN)
+        poller.register(self.command_channel, zmq.POLLIN)
 
         # These are managers which we should examine in an iteration
         # for scheduling a job (or maybe any other attention?).
@@ -351,6 +315,8 @@ class Interchange:
         while not kill_event.is_set():
             self.socks = dict(poller.poll(timeout=poll_period))
 
+            self.process_command(monitoring_radio)
+            self.process_task_incoming()
             self.process_task_outgoing_incoming(interesting_managers, monitoring_radio, kill_event)
             self.process_results_incoming(interesting_managers, monitoring_radio)
             self.expire_bad_managers(interesting_managers, monitoring_radio)
@@ -362,6 +328,18 @@ class Interchange:
         logger.info(f"Processed {self.count} tasks in {delta} seconds")
         logger.warning("Exiting")
 
+    def process_task_incoming(self) -> None:
+        """Process incoming task message(s).
+        """
+
+        if self.task_incoming in self.socks and self.socks[self.task_incoming] == zmq.POLLIN:
+            logger.debug("start task_incoming section")
+            msg = self.task_incoming.recv_pyobj()
+            logger.debug("putting message onto pending_task_queue")
+            self.pending_task_queue.put(msg)
+            self.task_counter += 1
+            logger.debug(f"Fetched {self.task_counter} tasks so far")
+
     def process_task_outgoing_incoming(
             self,
             interesting_managers: Set[bytes],

parsl/tests/test_checkpointing/test_python_checkpoint_1.py

@@ -27,8 +27,5 @@ def test_initial_checkpoint_write():
 
     cpt_dir = parsl.dfk().checkpoint()
 
-    cptpath = cpt_dir + '/dfk.pkl'
-    assert os.path.exists(cptpath), f"DFK checkpoint missing: {cptpath}"
-
     cptpath = cpt_dir + '/tasks.pkl'
     assert os.path.exists(cptpath), f"Tasks checkpoint missing: {cptpath}"

parsl/tests/test_python_apps/test_fail.py

@@ -39,6 +39,9 @@ def test_fail_sequence_first():
     assert isinstance(t_final.exception().dependent_exceptions_tids[0][0], DependencyError)
     assert t_final.exception().dependent_exceptions_tids[0][1].startswith("task ")
 
+    assert hasattr(t_final.exception(), '__cause__')
+    assert t_final.exception().__cause__ == t1.exception()
+
 
 def test_fail_sequence_middle():
     t1 = random_fail(fail_prob=0)
@@ -50,3 +53,6 @@ def test_fail_sequence_middle():
 
     assert len(t_final.exception().dependent_exceptions_tids) == 1
     assert isinstance(t_final.exception().dependent_exceptions_tids[0][0], ManufacturedTestFailure)
+
+    assert hasattr(t_final.exception(), '__cause__')
+    assert t_final.exception().__cause__ == t2.exception()

parsl/tests/test_python_apps/test_memoize_1.py

@@ -2,7 +2,6 @@ import argparse
 
 import parsl
 from parsl.app.app import python_app
-from parsl.tests.configs.local_threads import config
 
 
 @python_app(cache=True)
@@ -12,8 +11,7 @@ def random_uuid(x, cache=True):
 
 
 def test_python_memoization(n=2):
-    """Testing python memoization disable
-    """
+    """Testing python memoization."""
     x = random_uuid(0)
     print(x.result())
 

parsl/version.py

@@ -3,4 +3,4 @@
 Year.Month.Day[alpha/beta/..]
 Alphas will be numbered like this -> 2024.12.10a0
 """
-VERSION = '2025.01.20'
+VERSION = '2025.02.03'

{parsl-2025.1.20.data → parsl-2025.2.3.data}/scripts/interchange.py

@@ -9,7 +9,7 @@ import queue
 import sys
 import threading
 import time
-from typing import Any, Dict, List, NoReturn, Optional, Sequence, Set, Tuple, cast
+from typing import Any, Dict, List, Optional, Sequence, Set, Tuple, cast
 
 import zmq
 
@@ -132,6 +132,11 @@ class Interchange:
         self.hub_zmq_port = hub_zmq_port
 
         self.pending_task_queue: queue.Queue[Any] = queue.Queue(maxsize=10 ** 6)
+
+        # count of tasks that have been received from the submit side
+        self.task_counter = 0
+
+        # count of tasks that have been sent out to worker pools
         self.count = 0
 
         self.worker_ports = worker_ports
@@ -201,28 +206,6 @@ class Interchange:
 
         return tasks
 
-    @wrap_with_logs(target="interchange")
-    def task_puller(self) -> NoReturn:
-        """Pull tasks from the incoming tasks zmq pipe onto the internal
-        pending task queue
-        """
-        logger.info("Starting")
-        task_counter = 0
-
-        while True:
-            logger.debug("launching recv_pyobj")
-            try:
-                msg = self.task_incoming.recv_pyobj()
-            except zmq.Again:
-                # We just timed out while attempting to receive
-                logger.debug("zmq.Again with {} tasks in internal queue".format(self.pending_task_queue.qsize()))
-                continue
-
-            logger.debug("putting message onto pending_task_queue")
-            self.pending_task_queue.put(msg)
-            task_counter += 1
-            logger.debug(f"Fetched {task_counter} tasks so far")
-
     def _send_monitoring_info(self, monitoring_radio: Optional[MonitoringRadioSender], manager: ManagerRecord) -> None:
         if monitoring_radio:
             logger.info("Sending message {} to MonitoringHub".format(manager))
@@ -234,79 +217,68 @@ class Interchange:
 
             monitoring_radio.send((MessageType.NODE_INFO, d))
 
-    @wrap_with_logs(target="interchange")
-    def _command_server(self) -> NoReturn:
+    def process_command(self, monitoring_radio: Optional[MonitoringRadioSender]) -> None:
         """ Command server to run async command to the interchange
         """
-        logger.debug("Command Server Starting")
-
-        if self.hub_address is not None and self.hub_zmq_port is not None:
-            logger.debug("Creating monitoring radio to %s:%s", self.hub_address, self.hub_zmq_port)
-            monitoring_radio = ZMQRadioSender(self.hub_address, self.hub_zmq_port)
-        else:
-            monitoring_radio = None
+        logger.debug("entering command_server section")
 
         reply: Any  # the type of reply depends on the command_req received (aka this needs dependent types...)
 
-        while True:
-            try:
-                command_req = self.command_channel.recv_pyobj()
-                logger.debug("Received command request: {}".format(command_req))
-                if command_req == "CONNECTED_BLOCKS":
-                    reply = self.connected_block_history
-
-                elif command_req == "WORKERS":
-                    num_workers = 0
-                    for manager in self._ready_managers.values():
-                        num_workers += manager['worker_count']
-                    reply = num_workers
-
-                elif command_req == "MANAGERS":
-                    reply = []
-                    for manager_id in self._ready_managers:
-                        m = self._ready_managers[manager_id]
-                        idle_since = m['idle_since']
-                        if idle_since is not None:
-                            idle_duration = time.time() - idle_since
-                        else:
-                            idle_duration = 0.0
-                        resp = {'manager': manager_id.decode('utf-8'),
-                                'block_id': m['block_id'],
-                                'worker_count': m['worker_count'],
-                                'tasks': len(m['tasks']),
-                                'idle_duration': idle_duration,
-                                'active': m['active'],
-                                'parsl_version': m['parsl_version'],
-                                'python_version': m['python_version'],
-                                'draining': m['draining']}
-                        reply.append(resp)
-
-                elif command_req.startswith("HOLD_WORKER"):
-                    cmd, s_manager = command_req.split(';')
-                    manager_id = s_manager.encode('utf-8')
-                    logger.info("Received HOLD_WORKER for {!r}".format(manager_id))
-                    if manager_id in self._ready_managers:
-                        m = self._ready_managers[manager_id]
-                        m['active'] = False
-                        self._send_monitoring_info(monitoring_radio, m)
+        if self.command_channel in self.socks and self.socks[self.command_channel] == zmq.POLLIN:
+
+            command_req = self.command_channel.recv_pyobj()
+            logger.debug("Received command request: {}".format(command_req))
+            if command_req == "CONNECTED_BLOCKS":
+                reply = self.connected_block_history
+
+            elif command_req == "WORKERS":
+                num_workers = 0
+                for manager in self._ready_managers.values():
+                    num_workers += manager['worker_count']
+                reply = num_workers
+
+            elif command_req == "MANAGERS":
+                reply = []
+                for manager_id in self._ready_managers:
+                    m = self._ready_managers[manager_id]
+                    idle_since = m['idle_since']
+                    if idle_since is not None:
+                        idle_duration = time.time() - idle_since
                     else:
-                        logger.warning("Worker to hold was not in ready managers list")
-
-                    reply = None
+                        idle_duration = 0.0
+                    resp = {'manager': manager_id.decode('utf-8'),
+                            'block_id': m['block_id'],
+                            'worker_count': m['worker_count'],
+                            'tasks': len(m['tasks']),
+                            'idle_duration': idle_duration,
+                            'active': m['active'],
+                            'parsl_version': m['parsl_version'],
+                            'python_version': m['python_version'],
+                            'draining': m['draining']}
+                    reply.append(resp)
+
+            elif command_req.startswith("HOLD_WORKER"):
+                cmd, s_manager = command_req.split(';')
+                manager_id = s_manager.encode('utf-8')
+                logger.info("Received HOLD_WORKER for {!r}".format(manager_id))
+                if manager_id in self._ready_managers:
+                    m = self._ready_managers[manager_id]
+                    m['active'] = False
+                    self._send_monitoring_info(monitoring_radio, m)
+                else:
+                    logger.warning("Worker to hold was not in ready managers list")
 
-                elif command_req == "WORKER_PORTS":
-                    reply = (self.worker_task_port, self.worker_result_port)
+                reply = None
 
-                else:
-                    logger.error(f"Received unknown command: {command_req}")
-                    reply = None
+            elif command_req == "WORKER_PORTS":
+                reply = (self.worker_task_port, self.worker_result_port)
 
-                logger.debug("Reply: {}".format(reply))
-                self.command_channel.send_pyobj(reply)
+            else:
+                logger.error(f"Received unknown command: {command_req}")
+                reply = None
 
-            except zmq.Again:
-                logger.debug("Command thread is alive")
-                continue
+            logger.debug("Reply: {}".format(reply))
+            self.command_channel.send_pyobj(reply)
 
     @wrap_with_logs
     def start(self) -> None:
@@ -326,21 +298,13 @@ class Interchange:
 
         start = time.time()
 
-        self._task_puller_thread = threading.Thread(target=self.task_puller,
-                                                    name="Interchange-Task-Puller",
-                                                    daemon=True)
-        self._task_puller_thread.start()
-
-        self._command_thread = threading.Thread(target=self._command_server,
-                                                name="Interchange-Command",
-                                                daemon=True)
-        self._command_thread.start()
-
         kill_event = threading.Event()
 
         poller = zmq.Poller()
         poller.register(self.task_outgoing, zmq.POLLIN)
         poller.register(self.results_incoming, zmq.POLLIN)
+        poller.register(self.task_incoming, zmq.POLLIN)
+        poller.register(self.command_channel, zmq.POLLIN)
 
         # These are managers which we should examine in an iteration
         # for scheduling a job (or maybe any other attention?).
@@ -351,6 +315,8 @@ class Interchange:
         while not kill_event.is_set():
             self.socks = dict(poller.poll(timeout=poll_period))
 
+            self.process_command(monitoring_radio)
+            self.process_task_incoming()
             self.process_task_outgoing_incoming(interesting_managers, monitoring_radio, kill_event)
             self.process_results_incoming(interesting_managers, monitoring_radio)
             self.expire_bad_managers(interesting_managers, monitoring_radio)
@@ -362,6 +328,18 @@ class Interchange:
         logger.info(f"Processed {self.count} tasks in {delta} seconds")
         logger.warning("Exiting")
 
+    def process_task_incoming(self) -> None:
+        """Process incoming task message(s).
+        """
+
+        if self.task_incoming in self.socks and self.socks[self.task_incoming] == zmq.POLLIN:
+            logger.debug("start task_incoming section")
+            msg = self.task_incoming.recv_pyobj()
+            logger.debug("putting message onto pending_task_queue")
+            self.pending_task_queue.put(msg)
+            self.task_counter += 1
+            logger.debug(f"Fetched {self.task_counter} tasks so far")
+
     def process_task_outgoing_incoming(
             self,
             interesting_managers: Set[bytes],

{parsl-2025.1.20.dist-info → parsl-2025.2.3.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: parsl
-Version: 2025.1.20
+Version: 2025.2.3
 Summary: Simple data dependent workflows in Python
 Home-page: https://github.com/Parsl/parsl
-Download-URL: https://github.com/Parsl/parsl/archive/2025.01.20.tar.gz
+Download-URL: https://github.com/Parsl/parsl/archive/2025.02.03.tar.gz
 Author: The Parsl Team
 Author-email: parsl@googlegroups.com
 License: Apache 2.0

{parsl-2025.1.20.dist-info → parsl-2025.2.3.dist-info}/RECORD

@@ -8,7 +8,7 @@ parsl/multiprocessing.py,sha256=MyaEcEq-Qf860u7V98u-PZrPNdtzOZL_NW6EhIJnmfQ,1937
 parsl/process_loggers.py,sha256=uQ7Gd0W72Jz7rrcYlOMfLsAEhkRltxXJL2MgdduJjEw,1136
 parsl/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 parsl/utils.py,sha256=5FvHIMao3Ik0Rm2p2ieL1KQcQcYXc5K83Jrx5csi-B4,14301
-parsl/version.py,sha256=B68kDKT369JEJ5yXX5Ue1dD455x8WPmdaddBARy1bBc,131
+parsl/version.py,sha256=Zl1Frad6LRWH7vPgll3ZZU1ODYvo9dMwEa4jBjGYQkg,131
 parsl/app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 parsl/app/app.py,sha256=0gbM4AH2OtFOLsv07I5nglpElcwMSOi-FzdZZfrk7So,8532
 parsl/app/bash.py,sha256=jm2AvePlCT9DZR7H_4ANDWxatp5dN_22FUlT_gWhZ-g,5528
@@ -53,10 +53,10 @@ parsl/data_provider/staging.py,sha256=ZDZuuFg38pjUStegKPcvPsfGp3iMeReMzfU6DSwtJj
 parsl/data_provider/zip.py,sha256=S4kVuH9lxAegRURYbvIUR7EYYBOccyslaqyCrVWUBhw,4497
 parsl/dataflow/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 parsl/dataflow/dependency_resolvers.py,sha256=Om8Dgh7a0ZwgXAc6TlhxLSzvxXHDlNNV1aBNiD3JTNY,3325
-parsl/dataflow/dflow.py,sha256=tbAFEcLgKQp4IUwMhH1fYVxYU_e3_jw8ENElEA9kdeE,64928
-parsl/dataflow/errors.py,sha256=vzgEEFqIfIQ3_QGvqDCcsACZZlUKNAKaMchM30TGHyY,2114
+parsl/dataflow/dflow.py,sha256=jNxrAd2xmxesS3fR6eZyDN9f6I0BIBQbhL63zv51lkk,61752
+parsl/dataflow/errors.py,sha256=daVfr2BWs1zRsGD6JtosEMttWHvK1df1Npiu_MUvFKg,3998
 parsl/dataflow/futures.py,sha256=08LuP-HFiHBIZmeKCjlsazw_WpQ5fwevrU2_WbidkYw,6080
-parsl/dataflow/memoization.py,sha256=l9uw1Bu50GucBF70M5relpGKFkE4dIM9T3R1KrxW0v0,9583
+parsl/dataflow/memoization.py,sha256=QUkTduZ_gdr8i08VWNWrqhfEvoMGsPDZegWUE2_7sGQ,12579
 parsl/dataflow/rundirs.py,sha256=JZdzybVGubY35jL2YiKcDo65ZmRl1WyOApc8ajYxztc,1087
 parsl/dataflow/states.py,sha256=hV6mfv-y4A6xrujeQglcomnfEs7y3Xm2g6JFwC6dvgQ,2612
 parsl/dataflow/taskrecord.py,sha256=qIW7T6hn9dYTuNPdUura3HQwwUpUJACwPP5REm5COf4,3042
@@ -73,8 +73,8 @@ parsl/executors/flux/executor.py,sha256=8_xakLUu5zNJAHL0LbeTCFEWqWzRK1eE-3ep4GII
 parsl/executors/flux/flux_instance_manager.py,sha256=5T3Rp7ZM-mlT0Pf0Gxgs5_YmnaPrSF9ec7zvRfLfYJw,2129
 parsl/executors/high_throughput/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 parsl/executors/high_throughput/errors.py,sha256=k2XuvvFdUfNs2foHFnxmS-BToRMfdXpYEa4EF3ELKq4,1554
-parsl/executors/high_throughput/executor.py,sha256=xzNW4X9Zn10o0FsY8rnIuHCuaBu0vaKEXqMpe-2jlaA,38341
-parsl/executors/high_throughput/interchange.py,sha256=i0TYgAQzcnzRnSdKal9M9d6TcWRoIWK77kuxMsiYxXE,30142
+parsl/executors/high_throughput/executor.py,sha256=4WKp0ZqAz036WcaXq-4DDwLyu8W6xGPunnvLAVSeaiQ,38493
+parsl/executors/high_throughput/interchange.py,sha256=kVz7LTx6ThcYrQOSq-TsWHOlDwgyQk4w9Qyd3egQTuY,29143
 parsl/executors/high_throughput/manager_record.py,sha256=yn3L8TUJFkgm2lX1x0SeS9mkvJowC0s2VIMCFiU7ThM,455
 parsl/executors/high_throughput/manager_selector.py,sha256=UKcUE6v0tO7PDMTThpKSKxVpOpOUilxDL7UbNgpZCxo,2116
 parsl/executors/high_throughput/monitoring_info.py,sha256=HC0drp6nlXQpAop5PTUKNjdXMgtZVvrBL0JzZJebPP4,298
@@ -290,9 +290,8 @@ parsl/tests/test_bash_apps/test_std_uri.py,sha256=CvAt8BUhNl2pA5chq9YyhkD6eo2IUH
 parsl/tests/test_bash_apps/test_stdout.py,sha256=lNBzCJGst0IhKaSl8CM8-mTJ5eaK7hTlZ8gY-M2TDBU,3244
 parsl/tests/test_checkpointing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 parsl/tests/test_checkpointing/test_periodic.py,sha256=nfMgrG7sZ8rkMu6iOHS6lp_iTU4IsOyQLQ2Gur_FMmE,1509
-parsl/tests/test_checkpointing/test_python_checkpoint_1.py,sha256=k7_Zy4CV9OQt4ORYFCdyX53c4B0YPiwEIi35LhvLB2w,746
+parsl/tests/test_checkpointing/test_python_checkpoint_1.py,sha256=TP6kSK_0qpCqecpp7O50AbTsnLKU6wvTvNG89hh4LQw,637
 parsl/tests/test_checkpointing/test_python_checkpoint_2.py,sha256=Q_cXeAVz_dJuDDeiemUIGd-wmb7aCY3ggpqYjRRhHRc,1089
-parsl/tests/test_checkpointing/test_python_checkpoint_3.py,sha256=y4esbbp1h9FP-_rBfnFhohB6a7o3VOpjqKdTovV8HbA,805
 parsl/tests/test_checkpointing/test_regression_232.py,sha256=AsI6AJ0DcFaefAbEY9qWa41ER0VX-4yLuIdlgvBw360,2637
 parsl/tests/test_checkpointing/test_regression_233.py,sha256=jii7BKuygK6KMIGtg4IeBjix7Z28cYhv57rE9ixoXMU,1774
 parsl/tests/test_checkpointing/test_regression_239.py,sha256=xycW1_IwVC55L25oMES_OzJU58TN5BoMvRUZ_xB69jU,2441
@@ -366,7 +365,7 @@ parsl/tests/test_python_apps/test_dep_standard_futures.py,sha256=kMOMZLaxJMmpABC
 parsl/tests/test_python_apps/test_dependencies.py,sha256=IRiTI_lPoWBSFSFnaBlE6Bv08PKEaf-qj5dfqO2RjT0,272
 parsl/tests/test_python_apps/test_dependencies_deep.py,sha256=Cuow2LLGY7zffPFj89AOIwKlXxHtsin3v_UIhfdwV_w,1542
 parsl/tests/test_python_apps/test_depfail_propagation.py,sha256=3q3HlVWrOixFtXWBvR_ypKtbdAHAJcKndXQ5drwrBQU,1488
-parsl/tests/test_python_apps/test_fail.py,sha256=CgZq_ByzX6YLhBg71nWGXwaOaesYXwE6TWwslwrVuq4,1466
+parsl/tests/test_python_apps/test_fail.py,sha256=gMuZwxZNaUCaonlUX-7SOBvXg8kidkBcEeqKLEvqpYM,1692
 parsl/tests/test_python_apps/test_fibonacci_iterative.py,sha256=ly2s5HuB9R53Z2FM_zy0WWdOk01iVhgcwSpQyK6ErIY,573
 parsl/tests/test_python_apps/test_fibonacci_recursive.py,sha256=q7LMFcu_pJSNPdz8iY0UiRoIweEWIBGwMjQffHWAuDc,592
 parsl/tests/test_python_apps/test_futures.py,sha256=EWnzmPn5sVCgeMxc0Uz2ieaaVYr98tFZ7g8YJFqYuC8,2355
@@ -376,7 +375,7 @@ parsl/tests/test_python_apps/test_inputs_default.py,sha256=J2GR1NgdvEucNSJkfO6GC
 parsl/tests/test_python_apps/test_join.py,sha256=OWd6_A0Cf-1Xpjr0OT3HaJ1IMYcJ0LFL1VnmL0cZkL8,2988
 parsl/tests/test_python_apps/test_lifted.py,sha256=Na6qC_dZSeYJcZdkGn-dCjgYkQV267HmGFfaqFcRVcQ,3408
 parsl/tests/test_python_apps/test_mapred.py,sha256=C7nTl0NsP_2TCtcmZXWFMpvAG4pwGswrIJKr-5sRUNY,786
-parsl/tests/test_python_apps/test_memoize_1.py,sha256=PXazRnekBe_KScUdbh8P3I7Vu_1Tc-nGssWBpfcic7M,514
+parsl/tests/test_python_apps/test_memoize_1.py,sha256=E_VQAaykFKT_G7yRUWOhXxfOICj07qLq2R7onZ4oY9g,449
 parsl/tests/test_python_apps/test_memoize_2.py,sha256=uG9zG9j3ap1FqeJ8aB0Gj_dX191pN3dxWXeQ-asxPgU,553
 parsl/tests/test_python_apps/test_memoize_4.py,sha256=CdK_vHW5s-phi5KPqcAQm_BRh8xek91GVGeQRjfJ4Bk,569
 parsl/tests/test_python_apps/test_memoize_bad_id_for_memo.py,sha256=5v25zdU6koXexRTkccj_3sSSdXqHdsU8ZdNrnZ3ONZU,1436
@@ -456,13 +455,13 @@ parsl/usage_tracking/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hS
 parsl/usage_tracking/api.py,sha256=iaCY58Dc5J4UM7_dJzEEs871P1p1HdxBMtNGyVdzc9g,1821
 parsl/usage_tracking/levels.py,sha256=xbfzYEsd55KiZJ-mzNgPebvOH4rRHum04hROzEf41tU,291
 parsl/usage_tracking/usage.py,sha256=f9k6QcpbQxkGyP5WTC9PVyv0CA05s9NDpRe5wwRdBTM,9163
-parsl-2025.1.20.data/scripts/exec_parsl_function.py,sha256=YXKVVIa4zXmOtz-0Ca4E_5nQfN_3S2bh2tB75uZZB4w,7774
-parsl-2025.1.20.data/scripts/interchange.py,sha256=rUhF_Bwk5NOqLhh-HgP-ei_gclKnPIJJ7uS32p0j-XI,30129
-parsl-2025.1.20.data/scripts/parsl_coprocess.py,sha256=zrVjEqQvFOHxsLufPi00xzMONagjVwLZbavPM7bbjK4,5722
-parsl-2025.1.20.data/scripts/process_worker_pool.py,sha256=82FoJTye2SysJzPg-N8BpenuHGU7hOI8-Bedq8HV9C0,41851
-parsl-2025.1.20.dist-info/LICENSE,sha256=tAkwu8-AdEyGxGoSvJ2gVmQdcicWw3j1ZZueVV74M-E,11357
-parsl-2025.1.20.dist-info/METADATA,sha256=7DsaLRVRoBAcLHQbTggqANCocVk-5c4G6T2b-1Hd3go,4027
-parsl-2025.1.20.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-parsl-2025.1.20.dist-info/entry_points.txt,sha256=XqnsWDYoEcLbsMcpnYGKLEnSBmaIe1YoM5YsBdJG2tI,176
-parsl-2025.1.20.dist-info/top_level.txt,sha256=PIheYoUFQtF2icLsgOykgU-Cjuwr2Oi6On2jo5RYgRM,6
-parsl-2025.1.20.dist-info/RECORD,,
+parsl-2025.2.3.data/scripts/exec_parsl_function.py,sha256=YXKVVIa4zXmOtz-0Ca4E_5nQfN_3S2bh2tB75uZZB4w,7774
+parsl-2025.2.3.data/scripts/interchange.py,sha256=dAo8zl82gUFaeu_vNz-OXjkbeORZmwnqj4CMv8DahKY,29130
+parsl-2025.2.3.data/scripts/parsl_coprocess.py,sha256=zrVjEqQvFOHxsLufPi00xzMONagjVwLZbavPM7bbjK4,5722
+parsl-2025.2.3.data/scripts/process_worker_pool.py,sha256=82FoJTye2SysJzPg-N8BpenuHGU7hOI8-Bedq8HV9C0,41851
+parsl-2025.2.3.dist-info/LICENSE,sha256=tAkwu8-AdEyGxGoSvJ2gVmQdcicWw3j1ZZueVV74M-E,11357
+parsl-2025.2.3.dist-info/METADATA,sha256=nqLV9gMKl_5N-QYsDt-PJZ2W-9W6pqJEzWKvJ3LNf8g,4026
+parsl-2025.2.3.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+parsl-2025.2.3.dist-info/entry_points.txt,sha256=XqnsWDYoEcLbsMcpnYGKLEnSBmaIe1YoM5YsBdJG2tI,176
+parsl-2025.2.3.dist-info/top_level.txt,sha256=PIheYoUFQtF2icLsgOykgU-Cjuwr2Oi6On2jo5RYgRM,6
+parsl-2025.2.3.dist-info/RECORD,,

parsl/tests/test_checkpointing/test_python_checkpoint_3.py

@@ -1,42 +0,0 @@
-import os
-
-import pytest
-
-import parsl
-from parsl.app.app import python_app
-from parsl.tests.configs.local_threads import config
-
-
-def local_setup():
-    global dfk
-    dfk = parsl.load(config)
-
-
-def local_teardown():
-    parsl.dfk().cleanup()
-
-
-@python_app
-def slow_double(x, sleep_dur=1, cache=True):
-    import time
-    time.sleep(sleep_dur)
-    return x * 2
-
-
-@pytest.mark.local
-def test_checkpointing():
-    """Testing code snippet from documentation
-    """
-
-    N = 5  # Number of calls to slow_double
-    d = []  # List to store the futures
-    for i in range(0, N):
-        d.append(slow_double(i))
-
-    # Wait for the results
-    [i.result() for i in d]
-
-    checkpoint_dir = dfk.checkpoint()
-    print(checkpoint_dir)
-
-    assert os.path.exists(checkpoint_dir), "Checkpoint dir does not exist"