parsl 2024.5.20__py3-none-any.whl → 2024.5.27__py3-none-any.whl

parsl/config.py

@@ -5,6 +5,7 @@ from typing import Callable, Iterable, Optional, Sequence, Union
 from typing_extensions import Literal
 
 from parsl.utils import RepresentationMixin
+from parsl.dataflow.dependency_resolvers import DependencyResolver
 from parsl.executors.base import ParslExecutor
 from parsl.executors.threads import ThreadPoolExecutor
 from parsl.errors import ConfigurationError
@@ -35,6 +36,8 @@ class Config(RepresentationMixin, UsageInformation):
     checkpoint_period : str, optional
         Time interval (in "HH:MM:SS") at which to checkpoint completed tasks. Only has an effect if
         ``checkpoint_mode='periodic'``.
+    dependency_resolver: plugin point for custom dependency resolvers. Default: only resolve Futures,
+        using the `SHALLOW_DEPENDENCY_RESOLVER`.
     garbage_collect : bool. optional.
         Delete task records from DFK when tasks have completed. Default: True
     internal_tasks_max_threads : int, optional
@@ -88,6 +91,7 @@ class Config(RepresentationMixin, UsageInformation):
                                         Literal['dfk_exit'],
                                         Literal['manual']] = None,
                  checkpoint_period: Optional[str] = None,
+                 dependency_resolver: Optional[DependencyResolver] = None,
                  garbage_collect: bool = True,
                  internal_tasks_max_threads: int = 10,
                  retries: int = 0,
@@ -123,6 +127,7 @@ class Config(RepresentationMixin, UsageInformation):
         if checkpoint_mode == 'periodic' and checkpoint_period is None:
             checkpoint_period = "00:30:00"
         self.checkpoint_period = checkpoint_period
+        self.dependency_resolver = dependency_resolver
         self.garbage_collect = garbage_collect
         self.internal_tasks_max_threads = internal_tasks_max_threads
         self.retries = retries
@@ -152,4 +157,5 @@ class Config(RepresentationMixin, UsageInformation):
                 ', '.join(['label={}'.format(repr(d)) for d in duplicates])))
 
     def get_usage_information(self):
-        return {"executors_len": len(self.executors)}
+        return {"executors_len": len(self.executors),
+                "dependency_resolver": self.dependency_resolver is not None}

parsl/dataflow/dependency_resolvers.py

@@ -0,0 +1,115 @@
+from concurrent.futures import Future
+from dataclasses import dataclass
+from functools import singledispatch
+from typing import Callable, Sequence
+
+
+@dataclass
+class DependencyResolver:
+    """A DependencyResolver describes how app dependencies can be resolved.
+    It is specified as two functions: `traverse_to_gather` which turns an
+    app parameter into a sequence of futures which must be waited for before
+    the task can be executed (for example, in the case of
+    `DEEP_DEPENDENCY_RESOLVER` this traverses structures such as lists to
+    find every contained ``Future``), and `traverse_to_unwrap` which turns an
+    app parameter into its value to be passed to the app on execution
+    (for example in the case of `DEEP_DEPENDENCY_RESOLVER` this replaces a
+    list containing futures with a new list containing the values of those
+    resolved futures).
+
+    By default, Parsl will use `SHALLOW_DEPENDENCY_RESOLVER` which only
+    resolves Futures passed directly as arguments.
+    """
+    traverse_to_gather: Callable[[object], Sequence[Future]]
+    traverse_to_unwrap: Callable[[object], object]
+
+
+@singledispatch
+def shallow_traverse_to_gather(o):
+    # objects in general do not expose futures that we can see
+    return []
+
+
+@singledispatch
+def shallow_traverse_to_unwrap(o):
+    # objects in general unwrap to themselves
+    return o
+
+
+@shallow_traverse_to_gather.register
+def _(fut: Future):
+    return [fut]
+
+
+@shallow_traverse_to_unwrap.register
+@singledispatch
+def _(fut: Future):
+    assert fut.done()
+    return fut.result()
+
+
+@singledispatch
+def deep_traverse_to_gather(o):
+    # objects in general do not expose futures that we can see
+    return []
+
+
+@singledispatch
+def deep_traverse_to_unwrap(o):
+    # objects in general unwrap to themselves
+    return o
+
+
+@deep_traverse_to_gather.register
+def _(fut: Future):
+    return [fut]
+
+
+@deep_traverse_to_unwrap.register
+@singledispatch
+def _(fut: Future):
+    assert fut.done()
+    return fut.result()
+
+
+@deep_traverse_to_gather.register(tuple)
+@deep_traverse_to_gather.register(list)
+@deep_traverse_to_gather.register(set)
+def _(iterable):
+    return [e for v in iterable for e in deep_traverse_to_gather(v)]
+
+
+@deep_traverse_to_unwrap.register(tuple)
+@deep_traverse_to_unwrap.register(list)
+@deep_traverse_to_unwrap.register(set)
+@singledispatch
+def _(iterable):
+
+    type_ = type(iterable)
+    return type_(map(deep_traverse_to_unwrap, iterable))
+
+
+@deep_traverse_to_gather.register(dict)
+def _(dictionary):
+    futures = []
+    for key, value in dictionary.items():
+        futures.extend(deep_traverse_to_gather(key))
+        futures.extend(deep_traverse_to_gather(value))
+    return futures
+
+
+@deep_traverse_to_unwrap.register(dict)
+def _(dictionary):
+    unwrapped_dict = {}
+    for key, value in dictionary.items():
+        key = deep_traverse_to_unwrap(key)
+        value = deep_traverse_to_unwrap(value)
+        unwrapped_dict[key] = value
+    return unwrapped_dict
+
+
+DEEP_DEPENDENCY_RESOLVER = DependencyResolver(traverse_to_gather=deep_traverse_to_gather,
+                                              traverse_to_unwrap=deep_traverse_to_unwrap)
+
+SHALLOW_DEPENDENCY_RESOLVER = DependencyResolver(traverse_to_gather=shallow_traverse_to_gather,
+                                                 traverse_to_unwrap=shallow_traverse_to_unwrap)

parsl/dataflow/dflow.py

@@ -26,6 +26,7 @@ from parsl.channels import Channel
 from parsl.config import Config
 from parsl.data_provider.data_manager import DataManager
 from parsl.data_provider.files import File
+from parsl.dataflow.dependency_resolvers import SHALLOW_DEPENDENCY_RESOLVER
 from parsl.dataflow.errors import BadCheckpoint, DependencyError, JoinError
 from parsl.dataflow.futures import AppFuture
 from parsl.dataflow.memoization import Memoizer
@@ -203,6 +204,9 @@ class DataFlowKernel:
         self.tasks: Dict[int, TaskRecord] = {}
         self.submitter_lock = threading.Lock()
 
+        self.dependency_resolver = self.config.dependency_resolver if self.config.dependency_resolver is not None \
+            else SHALLOW_DEPENDENCY_RESOLVER
+
         atexit.register(self.atexit_cleanup)
 
     def __enter__(self):
@@ -852,8 +856,11 @@ class DataFlowKernel:
         depends: List[Future] = []
 
         def check_dep(d: Any) -> None:
-            if isinstance(d, Future):
-                depends.extend([d])
+            try:
+                depends.extend(self.dependency_resolver.traverse_to_gather(d))
+            except Exception:
+                logger.exception("Exception in dependency_resolver.traverse_to_gather")
+                raise
 
         # Check the positional args
         for dep in args:
@@ -870,7 +877,8 @@ class DataFlowKernel:
 
         return depends
 
-    def _unwrap_futures(self, args, kwargs):
+    def _unwrap_futures(self, args: Sequence[Any], kwargs: Dict[str, Any]) \
+            -> Tuple[Sequence[Any], Dict[str, Any], Sequence[Tuple[Exception, str]]]:
         """This function should be called when all dependencies have completed.
 
         It will rewrite the arguments for that task, replacing each Future
@@ -891,53 +899,40 @@ class DataFlowKernel:
         """
         dep_failures = []
 
+        def append_failure(e: Exception, dep: Future) -> None:
+            # If this Future is associated with a task inside this DFK,
+            # then refer to the task ID.
+            # Otherwise make a repr of the Future object.
+            if hasattr(dep, 'task_record') and dep.task_record['dfk'] == self:
+                tid = "task " + repr(dep.task_record['id'])
+            else:
+                tid = repr(dep)
+            dep_failures.extend([(e, tid)])
+
         # Replace item in args
         new_args = []
         for dep in args:
-            if isinstance(dep, Future):
-                try:
-                    new_args.extend([dep.result()])
-                except Exception as e:
-                    # If this Future is associated with a task inside this DFK,
-                    # then refer to the task ID.
-                    # Otherwise make a repr of the Future object.
-                    if hasattr(dep, 'task_record') and dep.task_record['dfk'] == self:
-                        tid = "task " + repr(dep.task_record['id'])
-                    else:
-                        tid = repr(dep)
-                    dep_failures.extend([(e, tid)])
-            else:
-                new_args.extend([dep])
+            try:
+                new_args.extend([self.dependency_resolver.traverse_to_unwrap(dep)])
+            except Exception as e:
+                append_failure(e, dep)
 
         # Check for explicit kwargs ex, fu_1=<fut>
         for key in kwargs:
             dep = kwargs[key]
-            if isinstance(dep, Future):
-                try:
-                    kwargs[key] = dep.result()
-                except Exception as e:
-                    if hasattr(dep, 'task_record'):
-                        tid = dep.task_record['id']
-                    else:
-                        tid = None
-                    dep_failures.extend([(e, tid)])
+            try:
+                kwargs[key] = self.dependency_resolver.traverse_to_unwrap(dep)
+            except Exception as e:
+                append_failure(e, dep)
 
         # Check for futures in inputs=[<fut>...]
         if 'inputs' in kwargs:
             new_inputs = []
             for dep in kwargs['inputs']:
-                if isinstance(dep, Future):
-                    try:
-                        new_inputs.extend([dep.result()])
-                    except Exception as e:
-                        if hasattr(dep, 'task_record'):
-                            tid = dep.task_record['id']
-                        else:
-                            tid = None
-                        dep_failures.extend([(e, tid)])
-
-                else:
-                    new_inputs.extend([dep])
+                try:
+                    new_inputs.extend([self.dependency_resolver.traverse_to_unwrap(dep)])
+                except Exception as e:
+                    append_failure(e, dep)
             kwargs['inputs'] = new_inputs
 
         return new_args, kwargs, dep_failures
@@ -1042,6 +1037,8 @@ class DataFlowKernel:
 
         func = self._add_output_deps(executor, app_args, app_kwargs, app_fu, func)
 
+        logger.debug("Added output dependencies")
+
         # Replace the function invocation in the TaskRecord with whatever file-staging
         # substitutions have been made.
         task_record.update({
@@ -1053,8 +1050,10 @@ class DataFlowKernel:
 
         self.tasks[task_id] = task_record
 
+        logger.debug("Gathering dependencies")
         # Get the list of dependencies for the task
         depends = self._gather_all_deps(app_args, app_kwargs)
+        logger.debug("Gathered dependencies")
         task_record['depends'] = depends
 
         depend_descs = []
@@ -1271,6 +1270,13 @@ class DataFlowKernel:
         atexit.unregister(self.atexit_cleanup)
         logger.info("Unregistered atexit hook")
 
+        if DataFlowKernelLoader._dfk is self:
+            logger.info("Unregistering default DFK")
+            parsl.clear()
+            logger.info("Unregistered default DFK")
+        else:
+            logger.debug("Cleaning up non-default DFK - not unregistering")
+
         logger.info("DFK cleanup complete")
 
     def checkpoint(self, tasks: Optional[Sequence[TaskRecord]] = None) -> str:

parsl/executors/high_throughput/errors.py

@@ -10,3 +10,13 @@ class WorkerLost(Exception):
 
     def __str__(self):
         return self.__repr__()
+
+
+class CommandClientTimeoutError(Exception):
+    """Raised when the command client times out waiting for a response.
+    """
+
+
+class CommandClientBadError(Exception):
+    """Raised when the command client is bad from an earlier timeout.
+    """

parsl/executors/high_throughput/executor.py

@@ -645,7 +645,8 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin, UsageIn
         Returns:
               Future
         """
-        validate_resource_spec(resource_specification)
+
+        validate_resource_spec(resource_specification, self.enable_mpi_mode)
 
         if self.bad_state_is_set:
             raise self.executor_exception

parsl/executors/high_throughput/mpi_executor.py

@@ -20,7 +20,7 @@ class MPIExecutor(HighThroughputExecutor):
     to spawn multi-node tasks.
 
     Specify the maximum number of multi-node tasks to run at once using ``max_workers_per_block``.
-    The maximum number should be smaller than the ``nodes_per_block`` in the Provider.
+    The value should be less than or equal to the ``nodes_per_block`` in the Provider.
 
     Parameters
     ----------

parsl/executors/high_throughput/mpi_prefix_composer.py

@@ -8,8 +8,18 @@ VALID_LAUNCHERS = ('srun',
                    'mpiexec')
 
 
+class MissingResourceSpecification(Exception):
+    """Exception raised when input is  not supplied a resource specification"""
+
+    def __init__(self, reason: str):
+        self.reason = reason
+
+    def __str__(self):
+        return f"Missing resource specification: {self.reason}"
+
+
 class InvalidResourceSpecification(Exception):
-    """Exception raised when Invalid keys are supplied via resource specification"""
+    """Exception raised when Invalid input is supplied via resource specification"""
 
     def __init__(self, invalid_keys: Set[str]):
         self.invalid_keys = invalid_keys
@@ -18,13 +28,19 @@ class InvalidResourceSpecification(Exception):
         return f"Invalid resource specification options supplied: {self.invalid_keys}"
 
 
-def validate_resource_spec(resource_spec: Dict[str, str]):
+def validate_resource_spec(resource_spec: Dict[str, str], is_mpi_enabled: bool):
     """Basic validation of keys in the resource_spec
 
     Raises: InvalidResourceSpecification if the resource_spec
         is invalid (e.g, contains invalid keys)
     """
     user_keys = set(resource_spec.keys())
+
+    # empty resource_spec when mpi_mode is set causes parsl to hang
+    # ref issue #3427
+    if is_mpi_enabled and len(user_keys) == 0:
+        raise MissingResourceSpecification('MPI mode requires optional parsl_resource_specification keyword argument to be configured')
+
     legal_keys = set(("ranks_per_node",
                       "num_nodes",
                       "num_ranks",

parsl/executors/high_throughput/zmq_pipes.py

@@ -3,8 +3,11 @@
 import zmq
 import logging
 import threading
+import time
 
 from parsl import curvezmq
+from parsl.errors import InternalConsistencyError
+from parsl.executors.high_throughput.errors import CommandClientBadError, CommandClientTimeoutError
 
 logger = logging.getLogger(__name__)
 
@@ -31,6 +34,7 @@ class CommandClient:
         self.port = None
         self.create_socket_and_bind()
         self._lock = threading.Lock()
+        self.ok = True
 
     def create_socket_and_bind(self):
         """ Creates socket and binds to a port.
@@ -46,7 +50,7 @@ class CommandClient:
         else:
             self.zmq_socket.bind("tcp://{}:{}".format(self.ip_address, self.port))
 
-    def run(self, message, max_retries=3):
+    def run(self, message, max_retries=3, timeout_s=None):
         """ This function needs to be fast at the same time aware of the possibility of
         ZMQ pipes overflowing.
 
@@ -54,13 +58,43 @@ class CommandClient:
         in ZMQ sockets reaching a broken state once there are ~10k tasks in flight.
         This issue can be magnified if each the serialized buffer itself is larger.
         """
+        if not self.ok:
+            raise CommandClientBadError()
+
+        start_time_s = time.monotonic()
+
         reply = '__PARSL_ZMQ_PIPES_MAGIC__'
         with self._lock:
             for _ in range(max_retries):
                 try:
                     logger.debug("Sending command client command")
+
+                    if timeout_s is not None:
+                        remaining_time_s = start_time_s + timeout_s - time.monotonic()
+                        poll_result = self.zmq_socket.poll(timeout=remaining_time_s * 1000, flags=zmq.POLLOUT)
+                        if poll_result == zmq.POLLOUT:
+                            pass  # this is OK, so continue
+                        elif poll_result == 0:
+                            raise CommandClientTimeoutError("Waiting for command channel to be ready for a command")
+                        else:
+                            raise InternalConsistencyError(f"ZMQ poll returned unexpected value: {poll_result}")
+
                     self.zmq_socket.send_pyobj(message, copy=True)
-                    logger.debug("Waiting for command client response")
+
+                    if timeout_s is not None:
+                        logger.debug("Polling for command client response or timeout")
+                        remaining_time_s = start_time_s + timeout_s - time.monotonic()
+                        poll_result = self.zmq_socket.poll(timeout=remaining_time_s * 1000, flags=zmq.POLLIN)
+                        if poll_result == zmq.POLLIN:
+                            pass  # this is OK, so continue
+                        elif poll_result == 0:
+                            logger.error("Command timed-out - command client is now bad forever")
+                            self.ok = False
+                            raise CommandClientTimeoutError("Waiting for a reply from command channel")
+                        else:
+                            raise InternalConsistencyError(f"ZMQ poll returned unexpected value: {poll_result}")
+
+                    logger.debug("Receiving command client response")
                     reply = self.zmq_socket.recv_pyobj()
                     logger.debug("Received command client response")
                 except zmq.ZMQError:

parsl/executors/radical/rpex_resources.py

@@ -3,15 +3,11 @@ import json
 
 from typing import List
 
-_setup_paths: List[str]
+_setup_paths: List[str] = []
 try:
     import radical.pilot as rp
-    import radical.utils as ru
 except ImportError:
-    _setup_paths = []
-else:
-    _setup_paths = [rp.sdist_path,
-                    ru.sdist_path]
+    pass
 
 
 MPI = "mpi"
@@ -77,7 +73,7 @@ class ResourceConfig:
 
     pilot_env_setup : list
         List of setup commands/packages for the pilot environment.
-        Default setup includes "parsl", rp.sdist_path, and ru.sdist_path.
+        Default is an empty list.
 
     python_v : str
         The Python version to be used in the pilot environment.

parsl/tests/conftest.py

@@ -201,7 +201,7 @@ def load_dfk_session(request, pytestconfig, tmpd_cwd_session):
         if parsl.dfk() != dfk:
             raise RuntimeError("DFK changed unexpectedly during test")
         dfk.cleanup()
-        parsl.clear()
+        assert DataFlowKernelLoader._dfk is None
     else:
         yield
 
@@ -253,7 +253,7 @@ def load_dfk_local_module(request, pytestconfig, tmpd_cwd_session):
             if parsl.dfk() != dfk:
                 raise RuntimeError("DFK changed unexpectedly during test")
             dfk.cleanup()
-            parsl.clear()
+            assert DataFlowKernelLoader._dfk is None
 
     else:
         yield

parsl/tests/sites/test_dynamic_executor.py

@@ -75,4 +75,3 @@ def test_dynamic_executor():
     print("Done testing")
 
     dfk.cleanup()
-    parsl.clear()

parsl/tests/test_bash_apps/test_std_uri.py

@@ -35,8 +35,6 @@ def const_with_cpath(autopath_specifier, content_path, caplog):
     for record in caplog.records:
         assert record.levelno < logging.ERROR
 
-    parsl.clear()
-
 
 @pytest.mark.local
 def test_std_autopath_const_str(caplog, tmpd_cwd):
@@ -74,8 +72,6 @@ def test_std_autopath_fail(caplog):
         with pytest.raises(URIFailError):
             app_stdout()
 
-    parsl.clear()
-
 
 @parsl.bash_app
 def app_both(stdout=parsl.AUTO_LOGNAME, stderr=parsl.AUTO_LOGNAME):
@@ -124,5 +120,3 @@ def test_std_autopath_zip(caplog, tmpd_cwd):
 
     for record in caplog.records:
         assert record.levelno < logging.ERROR
-
-    parsl.clear()

parsl/tests/test_checkpointing/test_periodic.py

@@ -9,12 +9,6 @@ def local_setup():
     parsl.load(fresh_config())
 
 
-def local_teardown():
-    # explicit clear without dfk.cleanup here, because the
-    # test does that already
-    parsl.clear()
-
-
 @python_app(cache=True)
 def slow_double(x, sleep_dur=1):
     import time
@@ -39,9 +33,10 @@ def test_periodic():
     with parsl.dfk():
         futs = [slow_double(sleep_for) for _ in range(4)]
         [f.result() for f in futs]
+        run_dir = parsl.dfk().run_dir
 
     # Here we will check if the loglines came back with 5 seconds deltas
-    with open("{}/parsl.log".format(parsl.dfk().run_dir)) as f:
+    with open("{}/parsl.log".format(run_dir)) as f:
         log_lines = f.readlines()
     expected_msg = " Done checkpointing"
     expected_msg2 = " No tasks checkpointed in this pass"

parsl/tests/test_checkpointing/test_python_checkpoint_2.py

@@ -19,7 +19,6 @@ def parsl_configured(run_dir, **kw):
     yield dfk
 
     parsl.dfk().cleanup()
-    parsl.clear()
 
 
 @python_app(cache=True)

parsl/tests/test_checkpointing/test_python_checkpoint_3.py

@@ -14,7 +14,6 @@ def local_setup():
 
 def local_teardown():
     parsl.dfk().cleanup()
-    parsl.clear()
 
 
 @python_app

parsl/tests/test_checkpointing/test_task_exit.py

@@ -16,7 +16,6 @@ def local_setup():
 
 def local_teardown():
     parsl.dfk().cleanup()
-    parsl.clear()
 
 
 @python_app(cache=True)

parsl/tests/test_htex/test_basic.py

@@ -14,7 +14,6 @@ def local_setup():
 
 def local_teardown():
     parsl.dfk().cleanup()
-    parsl.clear()
 
 
 @python_app

parsl/tests/test_htex/test_command_client_timeout.py

@@ -0,0 +1,69 @@
+import pytest
+import threading
+import time
+import zmq
+from parsl import curvezmq
+from parsl.executors.high_throughput.zmq_pipes import CommandClient
+from parsl.executors.high_throughput.errors import CommandClientTimeoutError, CommandClientBadError
+
+
+# Time constant used for timeout tests: various delays and
+# timeouts will be appropriate multiples of this, but the
+# value of T itself should not matter too much as long as
+# it is big enough for zmq connections to happen successfully.
+T = 0.25
+
+
+@pytest.mark.local
+def test_command_not_sent() -> None:
+    """Tests timeout on command send.
+    """
+    ctx = curvezmq.ClientContext(None)
+
+    # RFC6335 ephemeral port range
+    cc = CommandClient(ctx, "127.0.0.1", (49152, 65535))
+
+    # cc will now wait for a connection, but we won't do anything to make the
+    # other side of the connection exist, so any command given to cc should
+    # timeout.
+
+    with pytest.raises(CommandClientTimeoutError):
+        cc.run("SOMECOMMAND", timeout_s=T)
+
+    cc.close()
+
+
+@pytest.mark.local
+def test_command_ignored() -> None:
+    """Tests timeout on command response.
+    Tests that we timeout after a response and that the command client
+    sets itself into a bad state.
+
+    This only tests sequential access to the command client, even though
+    htex makes multithreaded use of the command client: see issue #3376 about
+    that lack of thread safety.
+    """
+    ctx = curvezmq.ClientContext(None)
+
+    # RFC6335 ephemeral port range
+    cc = CommandClient(ctx, "127.0.0.1", (49152, 65535))
+
+    ic_ctx = curvezmq.ServerContext(None)
+    ic_channel = ic_ctx.socket(zmq.REP)
+    ic_channel.connect(f"tcp://127.0.0.1:{cc.port}")
+
+    with pytest.raises(CommandClientTimeoutError):
+        cc.run("SLOW_COMMAND", timeout_s=T)
+
+    req = ic_channel.recv_pyobj()
+    assert req == "SLOW_COMMAND", "Should have received command on interchange side"
+    assert not cc.ok, "CommandClient should have set itself to bad"
+
+    with pytest.raises(CommandClientBadError):
+        cc.run("ANOTHER_COMMAND")
+
+    cc.close()
+    ctx.term()
+
+    ic_channel.close()
+    ic_ctx.term()

parsl/tests/test_htex/test_cpu_affinity_explicit.py

@@ -37,16 +37,9 @@ def test_cpu_affinity_explicit():
     config.executors[0].max_workers_per_node = 1
 
     logger.debug(f"config: {config}")
-    # TODO: is there a `with` style for this, to properly deal with exceptions?
-
-    parsl.load(config)
-    try:
 
+    with parsl.load(config):
         worker_affinity = my_affinity().result()
         logger.debug(f"worker reported this affinity: {worker_affinity}")
         assert len(worker_affinity) == 1
         assert worker_affinity == set((single_core,))
-
-    finally:
-        parsl.dfk().cleanup()
-        parsl.clear()