parsl 2023.10.23__py3-none-any.whl → 2023.11.13__py3-none-any.whl

parsl/executors/workqueue/executor.py

@@ -21,7 +21,7 @@ import inspect
 import shutil
 import itertools
 
-from parsl.serialize import pack_apply_message
+from parsl.serialize import pack_apply_message, deserialize
 import parsl.utils as putils
 from parsl.executors.errors import ExecutorError
 from parsl.data_provider.files import File
@@ -66,11 +66,11 @@ ParslTaskToWq = namedtuple('ParslTaskToWq',
 
 # Support structure to communicate final status of work queue tasks to parsl
 # if result_received is True:
-#   result is the result
+#   result_file is the path to the file containing the result.
 # if result_received is False:
 #   reason and status are only valid if result_received is False
-#   result is either None or an exception raised while looking for a result
-WqTaskToParsl = namedtuple('WqTaskToParsl', 'id result_received result reason status')
+#   result_file is None
+WqTaskToParsl = namedtuple('WqTaskToParsl', 'id result_received result_file reason status')
 
 # Support structure to report parsl filenames to work queue.
 # parsl_name is the local_name or filepath attribute of a parsl file object.
@@ -449,7 +449,7 @@ class WorkQueueExecutor(BlockProviderExecutor, putils.RepresentationMixin):
         input_files = []
         output_files = []
 
-        # Determine the input and output files that will exist at the workes:
+        # Determine the input and output files that will exist at the workers:
         input_files += [self._register_file(f) for f in kwargs.get("inputs", []) if isinstance(f, File)]
         output_files += [self._register_file(f) for f in kwargs.get("outputs", []) if isinstance(f, File)]
 
@@ -707,7 +707,6 @@ class WorkQueueExecutor(BlockProviderExecutor, putils.RepresentationMixin):
         self.collector_thread.join()
 
         logger.debug("Work Queue shutdown completed")
-        return True
 
     @wrap_with_logs
     def _collect_work_queue_results(self):
@@ -729,14 +728,29 @@ class WorkQueueExecutor(BlockProviderExecutor, putils.RepresentationMixin):
                 with self.tasks_lock:
                     future = self.tasks.pop(task_report.id)
                 logger.debug("Updating Future for executor task {}".format(task_report.id))
+                # If result_received, then there's a result file. The object inside the file
+                # may be a valid result or an exception caused within the function invocation.
+                # Otherwise there's no result file, implying errors from WorkQueue.
                 if task_report.result_received:
-                    future.set_result(task_report.result)
+                    try:
+                        with open(task_report.result_file, 'rb') as f_in:
+                            result = deserialize(f_in.read())
+                    except Exception as e:
+                        logger.error(f'Cannot load result from result file {task_report.result_file}. Exception: {e}')
+                        ex = WorkQueueTaskFailure('Cannot load result from result file', None)
+                        ex.__cause__ = e
+                        future.set_exception(ex)
+                    else:
+                        if isinstance(result, Exception):
+                            ex = WorkQueueTaskFailure('Task execution raises an exception', result)
+                            ex.__cause__ = result
+                            future.set_exception(ex)
+                        else:
+                            future.set_result(result)
                 else:
                     # If there are no results, then the task failed according to one of
                     # work queue modes, such as resource exhaustion.
-                    ex = WorkQueueTaskFailure(task_report.reason, task_report.result)
-                    if task_report.result is not None:
-                        ex.__cause__ = task_report.result
+                    ex = WorkQueueTaskFailure(task_report.reason, None)
                     future.set_exception(ex)
         finally:
             logger.debug("Marking all outstanding tasks as failed")
@@ -876,7 +890,7 @@ def _work_queue_submit_wait(*,
                 logger.error("Unable to create task: {}".format(e))
                 collector_queue.put_nowait(WqTaskToParsl(id=task.id,
                                                          result_received=False,
-                                                         result=None,
+                                                         result_file=None,
                                                          reason="task could not be created by work queue",
                                                          status=-1))
                 continue
@@ -937,7 +951,7 @@ def _work_queue_submit_wait(*,
                 logger.error("Unable to submit task to work queue: {}".format(e))
                 collector_queue.put_nowait(WqTaskToParsl(id=task.id,
                                                          result_received=False,
-                                                         result=None,
+                                                         result_file=None,
                                                          reason="task could not be submited to work queue",
                                                          status=-1))
                 continue
@@ -957,24 +971,20 @@ def _work_queue_submit_wait(*,
                 logger.debug("Completed Work Queue task {}, executor task {}".format(t.id, t.tag))
                 result_file = result_file_of_task_id.pop(t.tag)
 
-                # A tasks completes 'succesfully' if it has result file,
-                # and it can be loaded. This may mean that the 'success' is
-                # an exception.
+                # A tasks completes 'succesfully' if it has result file.
+                # The check whether this file can load a serialized Python object
+                # happens later in the collector thread of the executor process.
                 logger.debug("Looking for result in {}".format(result_file))
-                try:
-                    with open(result_file, "rb") as f_in:
-                        result = pickle.load(f_in)
+                if os.path.exists(result_file):
                     logger.debug("Found result in {}".format(result_file))
                     collector_queue.put_nowait(WqTaskToParsl(id=executor_task_id,
                                                              result_received=True,
-                                                             result=result,
+                                                             result_file=result_file,
                                                              reason=None,
                                                              status=t.return_status))
                 # If a result file could not be generated, explain the
-                # failure according to work queue error codes. We generate
-                # an exception and wrap it with RemoteExceptionWrapper, to
-                # match the positive case.
-                except Exception as e:
+                # failure according to work queue error codes.
+                else:
                     reason = _explain_work_queue_result(t)
                     logger.debug("Did not find result in {}".format(result_file))
                     logger.debug("Wrapper Script status: {}\nWorkQueue Status: {}"
@@ -983,7 +993,7 @@ def _work_queue_submit_wait(*,
                                  .format(executor_task_id, t.id, reason))
                     collector_queue.put_nowait(WqTaskToParsl(id=executor_task_id,
                                                              result_received=False,
-                                                             result=e,
+                                                             result_file=None,
                                                              reason=reason,
                                                              status=t.return_status))
     logger.debug("Exiting WorkQueue Monitoring Process")

parsl/monitoring/monitoring.py

@@ -194,10 +194,10 @@ class MonitoringHub(RepresentationMixin):
                                                "logdir": self.logdir,
                                                "logging_level": logging.DEBUG if self.monitoring_debug else logging.INFO,
                                                "run_id": run_id
-                                       },
+                                               },
                                        name="Monitoring-Router-Process",
                                        daemon=True,
-        )
+                                       )
         self.router_proc.start()
 
         self.dbm_proc = ForkProcess(target=dbm_starter,
@@ -205,10 +205,10 @@ class MonitoringHub(RepresentationMixin):
                                     kwargs={"logdir": self.logdir,
                                             "logging_level": logging.DEBUG if self.monitoring_debug else logging.INFO,
                                             "db_url": self.logging_endpoint,
-                                    },
+                                            },
                                     name="Monitoring-DBM-Process",
                                     daemon=True,
-        )
+                                    )
         self.dbm_proc.start()
         self.logger.info("Started the router process {} and DBM process {}".format(self.router_proc.pid, self.dbm_proc.pid))
 
@@ -216,7 +216,7 @@ class MonitoringHub(RepresentationMixin):
                                        args=(self.logdir, self.resource_msgs, run_dir),
                                        name="Monitoring-Filesystem-Process",
                                        daemon=True
-        )
+                                       )
         self.filesystem_proc.start()
         self.logger.info(f"Started filesystem radio receiver process {self.filesystem_proc.pid}")
 
@@ -359,7 +359,7 @@ class MonitoringRouter:
                  run_id: str,
                  logging_level: int = logging.INFO,
                  atexit_timeout: int = 3    # in seconds
-                ):
+                 ):
         """ Initializes a monitoring configuration class.
 
         Parameters

parsl/monitoring/remote.py

@@ -143,7 +143,7 @@ def send_first_last_message(try_id: int,
             'first_msg': not is_last,
             'last_msg': is_last,
             'timestamp': datetime.datetime.now()
-    })
+            })
     radio.send(msg)
     return
 

parsl/monitoring/visualization/plots/default/workflow_plots.py

@@ -22,7 +22,7 @@ gantt_colors = {'unsched': 'rgb(240, 240, 240)',
                 'exec_done': 'rgb(0, 200, 0)',
                 'memo_done': 'rgb(64, 200, 64)',
                 'fail_retryable': 'rgb(200, 128,128)'
-               }
+                }
 
 
 def task_gantt_plot(df_task, df_status, time_completed=None):
@@ -50,7 +50,7 @@ def task_gantt_plot(df_task, df_status, time_completed=None):
                                    'Start': last_status['timestamp'],
                                    'Finish': status['timestamp'],
                                    'Resource': last_status['task_status_name']
-                                  }
+                                   }
                 parsl_tasks.extend([last_status_bar])
             last_status = status
 
@@ -60,7 +60,7 @@ def task_gantt_plot(df_task, df_status, time_completed=None):
                                'Start': last_status['timestamp'],
                                'Finish': time_completed,
                                'Resource': last_status['task_status_name']
-                              }
+                               }
             parsl_tasks.extend([last_status_bar])
 
     fig = ff.create_gantt(parsl_tasks,
@@ -205,7 +205,7 @@ dag_state_colors = {"unsched": (0, 'rgb(240, 240, 240)'),
                     "fail_retryable": (8, 'rgb(200, 128,128)'),
                     "joining": (9, 'rgb(128, 128, 255)'),
                     "running_ended": (10, 'rgb(64, 64, 255)')
-                   }
+                    }
 
 
 def workflow_dag_plot(df_tasks, group_by_apps=True):

parsl/monitoring/visualization/plots/default/workflow_resource_plots.py

@@ -164,7 +164,7 @@ def worker_efficiency(task, node):
                              y=[total_workers] * (end - start + 1),
                              name='Total of workers in whole run',
                              )
-                 ],
+                  ],
             layout=go.Layout(xaxis=dict(autorange=True,
                                         title='Time (seconds)'),
                              yaxis=dict(title='Number of workers'),
@@ -230,7 +230,7 @@ def resource_efficiency(resource, node, label):
                              y=[total] * (end - start + 1),
                              name=name2,
                              )
-                 ],
+                  ],
             layout=go.Layout(xaxis=dict(autorange=True,
                                         title='Time (seconds)'),
                              yaxis=dict(title=yaxis),

parsl/providers/slurm/slurm.py

@@ -250,7 +250,7 @@ class SlurmProvider(ClusterProvider, RepresentationMixin):
                     self.resources[job_id] = {'job_id': job_id, 'status': JobStatus(JobState.PENDING)}
                     break
             else:
-                logger.error("Could not read job ID from sumbit command standard output.")
+                logger.error("Could not read job ID from submit command standard output.")
                 logger.error("Retcode:%s STDOUT:%s STDERR:%s", retcode, stdout.strip(), stderr.strip())
         else:
             logger.error("Submit command failed")

parsl/tests/configs/ad_hoc_cluster_htex.py

@@ -9,8 +9,8 @@ user_opts = {'adhoc':
              {'username': 'YOUR_USERNAME',
               'script_dir': 'YOUR_SCRIPT_DIR',
               'remote_hostnames': ['REMOTE_HOST_URL_1', 'REMOTE_HOST_URL_2']
-             }
-}  # type: Dict[str, Dict[str, Any]]
+              }
+             }  # type: Dict[str, Dict[str, Any]]
 
 config = Config(
     executors=[
@@ -25,7 +25,7 @@ config = Config(
                 channels=[SSHChannel(hostname=m,
                                      username=user_opts['adhoc']['username'],
                                      script_dir=user_opts['adhoc']['script_dir'],
-                ) for m in user_opts['adhoc']['remote_hostnames']]
+                                     ) for m in user_opts['adhoc']['remote_hostnames']]
             )
         )
     ],

parsl/tests/configs/htex_ad_hoc_cluster.py

@@ -20,7 +20,7 @@ config = Config(
                 channels=[SSHChannel(hostname=m,
                                      username=user_opts['adhoc']['username'],
                                      script_dir=user_opts['adhoc']['script_dir'],
-                ) for m in user_opts['adhoc']['remote_hostnames']]
+                                     ) for m in user_opts['adhoc']['remote_hostnames']]
             )
         )
     ],

parsl/tests/configs/local_threads_monitoring.py

@@ -8,4 +8,4 @@ config = Config(executors=[ThreadPoolExecutor(label='threads', max_threads=4)],
                     hub_port=55055,
                     resource_monitoring_interval=3,
                 )
-         )
+                )

parsl/tests/conftest.py

@@ -243,7 +243,7 @@ def setup_data(tmpd_cwd):
 
 
 @pytest.fixture(autouse=True, scope='function')
-def wait_for_task_completion(pytestconfig):
+def assert_no_outstanding_tasks(pytestconfig):
     """If we're in a config-file based mode, wait for task completion between
        each test. This will detect early on (by hanging) if particular test
        tasks are not finishing, rather than silently falling off the end of
@@ -254,7 +254,11 @@ def wait_for_task_completion(pytestconfig):
     config = pytestconfig.getoption('config')[0]
     yield
     if config != 'local':
-        parsl.dfk().wait_for_current_tasks()
+        logger.info("Checking no outstanding tasks")
+        for task_record in parsl.dfk().tasks.values():
+            fut = task_record['app_fu']
+            assert fut.done(), f"Incomplete task found, task id {task_record['id']}"
+        logger.info("No outstanding tasks found")
 
 
 def pytest_make_collect_report(collector):

parsl/tests/scaling_tests/vineex_condor.py

@@ -6,5 +6,5 @@ from parsl.providers import CondorProvider
 config = Config(
     executors=[TaskVineExecutor(manager_config=TaskVineManagerConfig(port=50055),
                                 provider=CondorProvider(),
-        )]
+                                )]
 )

parsl/tests/scaling_tests/vineex_local.py

@@ -7,5 +7,5 @@ config = Config(
     executors=[TaskVineExecutor(label='VineExec',
                                 worker_launch_method='factory',
                                 manager_config=TaskVineManagerConfig(port=50055),
-        )]
+                                )]
 )

parsl/tests/scaling_tests/wqex_condor.py

@@ -8,5 +8,5 @@ config = Config(
                                  provider=CondorProvider(),
                                  # init_command='source /home/yadu/src/wq_parsl/setup_parsl_env.sh;
                                  # echo "Ran at $date" > /home/yadu/src/wq_parsl/parsl/tests/workqueue_tests/ran.log',
-        )]
+                                 )]
 )

parsl/tests/scaling_tests/wqex_local.py

@@ -8,5 +8,5 @@ config = Config(
                                  provider=LocalProvider(),
                                  # init_command='source /home/yadu/src/wq_parsl/setup_parsl_env.sh;
                                  # echo "Ran at $date" > /home/yadu/src/wq_parsl/parsl/tests/workqueue_tests/ran.log',
-        )]
+                                 )]
 )

parsl/tests/test_docs/test_kwargs.py

@@ -0,0 +1,37 @@
+"""Functions used to explain kwargs"""
+from pathlib import Path
+
+from parsl import python_app, File
+
+
+def test_inputs():
+    @python_app()
+    def map_app(x):
+        return x * 2
+
+    @python_app()
+    def reduce_app(inputs=()):
+        return sum(inputs)
+
+    map_futures = [map_app(x) for x in range(3)]
+    reduce_future = reduce_app(inputs=map_futures)
+
+    assert reduce_future.result() == 6
+
+
+def test_outputs(tmpdir):
+    @python_app()
+    def write_app(message, outputs=()):
+        """Write a single message to every file in outputs"""
+        for path in outputs:
+            with open(path, 'w') as fp:
+                print(message, file=fp)
+
+    to_write = [
+        File(Path(tmpdir) / 'output-0.txt'),
+        File(Path(tmpdir) / 'output-1.txt')
+    ]
+    write_app('Hello!', outputs=to_write).result()
+    for path in to_write:
+        with open(path) as fp:
+            assert fp.read() == 'Hello!\n'

parsl/tests/test_python_apps/test_lifted.py

@@ -89,11 +89,12 @@ def test_returns_a_class_instance():
 
 def test_returns_a_class_instance_no_underscores():
     # test that _underscore attribute references are not lifted
+    f = returns_a_class_instance()
     with pytest.raises(AttributeError):
-        returns_a_class_instance()._nosuchattribute.result()
+        f._nosuchattribute.result()
+    f.exception()  # wait for f to complete before the test ends
 
 
-@pytest.mark.skip("returning classes is not supported in WorkQueue or Task Vine - see issue #2908")
 def test_returns_a_class():
 
     # precondition that returns_a_class behaves

parsl/utils.py

@@ -7,7 +7,7 @@ import threading
 import time
 from contextlib import contextmanager
 from types import TracebackType
-from typing import Any, Callable, List, Tuple, Union, Generator, IO, AnyStr, Dict, Optional
+from typing import Any, Callable, List, Sequence, Tuple, Union, Generator, IO, AnyStr, Dict, Optional
 
 import typeguard
 from typing_extensions import Type
@@ -47,7 +47,7 @@ def get_version() -> str:
 
 
 @typeguard.typechecked
-def get_all_checkpoints(rundir: str = "runinfo") -> List[str]:
+def get_all_checkpoints(rundir: str = "runinfo") -> Sequence[str]:
     """Finds the checkpoints from all runs in the rundir.
 
     Kwargs:
@@ -76,7 +76,7 @@ def get_all_checkpoints(rundir: str = "runinfo") -> List[str]:
 
 
 @typeguard.typechecked
-def get_last_checkpoint(rundir: str = "runinfo") -> List[str]:
+def get_last_checkpoint(rundir: str = "runinfo") -> Sequence[str]:
     """Finds the checkpoint from the last run, if one exists.
 
     Note that checkpoints are incremental, and this helper will not find
@@ -128,7 +128,7 @@ def get_std_fname_mode(
 
 @contextmanager
 def wait_for_file(path: str, seconds: int = 10) -> Generator[None, None, None]:
-    for i in range(0, int(seconds * 100)):
+    for _ in range(0, int(seconds * 100)):
         time.sleep(seconds / 100.)
         if os.path.exists(path):
             break

parsl/version.py

@@ -3,4 +3,4 @@
 Year.Month.Day[alpha/beta/..]
 Alphas will be numbered like this -> 2024.12.10a0
 """
-VERSION = '2023.10.23'
+VERSION = '2023.11.13'

{parsl-2023.10.23.data → parsl-2023.11.13.data}/scripts/exec_parsl_function.py

@@ -4,6 +4,7 @@ from parsl.utils import get_std_fname_mode
 import traceback
 import sys
 import pickle
+from parsl.serialize import serialize
 
 # This scripts executes a parsl function which is pickled in a file:
 #
@@ -32,7 +33,7 @@ def load_pickled_file(filename):
 
 def dump_result_to_file(result_file, result_package):
     with open(result_file, "wb") as f_out:
-        pickle.dump(result_package, f_out)
+        f_out.write(serialize(result_package))
 
 
 def remap_location(mapping, parsl_file):

{parsl-2023.10.23.data → parsl-2023.11.13.data}/scripts/process_worker_pool.py

@@ -234,7 +234,7 @@ class Manager:
                'dir': os.getcwd(),
                'cpu_count': psutil.cpu_count(logical=False),
                'total_memory': psutil.virtual_memory().total,
-        }
+               }
         b_msg = json.dumps(msg).encode('utf-8')
         return b_msg
 
@@ -608,7 +608,7 @@ def worker(worker_id, pool_id, pool_size, task_queue, result_queue, worker_queue
             logger.exception("Caught exception while trying to pickle the result package")
             pkl_package = pickle.dumps({'type': 'result', 'task_id': tid,
                                         'exception': serialize(RemoteExceptionWrapper(*sys.exc_info()))
-            })
+                                        })
 
         result_queue.put(pkl_package)
         tasks_in_progress.pop(worker_id)

{parsl-2023.10.23.dist-info → parsl-2023.11.13.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: parsl
-Version: 2023.10.23
+Version: 2023.11.13
 Summary: Simple data dependent workflows in Python
 Home-page: https://github.com/Parsl/parsl
-Download-URL: https://github.com/Parsl/parsl/archive/2023.10.23.tar.gz
+Download-URL: https://github.com/Parsl/parsl/archive/2023.11.13.tar.gz
 Author: The Parsl Team
 Author-email: parsl@googlegroups.com
 License: Apache 2.0