parsl 2023.10.23__py3-none-any.whl → 2023.11.20__py3-none-any.whl

parsl/monitoring/visualization/plots/default/workflow_plots.py

@@ -22,7 +22,7 @@ gantt_colors = {'unsched': 'rgb(240, 240, 240)',
                 'exec_done': 'rgb(0, 200, 0)',
                 'memo_done': 'rgb(64, 200, 64)',
                 'fail_retryable': 'rgb(200, 128,128)'
-               }
+                }
 
 
 def task_gantt_plot(df_task, df_status, time_completed=None):
@@ -50,7 +50,7 @@ def task_gantt_plot(df_task, df_status, time_completed=None):
                                    'Start': last_status['timestamp'],
                                    'Finish': status['timestamp'],
                                    'Resource': last_status['task_status_name']
-                                  }
+                                   }
                 parsl_tasks.extend([last_status_bar])
             last_status = status
 
@@ -60,7 +60,7 @@ def task_gantt_plot(df_task, df_status, time_completed=None):
                                'Start': last_status['timestamp'],
                                'Finish': time_completed,
                                'Resource': last_status['task_status_name']
-                              }
+                               }
             parsl_tasks.extend([last_status_bar])
 
     fig = ff.create_gantt(parsl_tasks,
@@ -205,7 +205,7 @@ dag_state_colors = {"unsched": (0, 'rgb(240, 240, 240)'),
                     "fail_retryable": (8, 'rgb(200, 128,128)'),
                     "joining": (9, 'rgb(128, 128, 255)'),
                     "running_ended": (10, 'rgb(64, 64, 255)')
-                   }
+                    }
 
 
 def workflow_dag_plot(df_tasks, group_by_apps=True):

parsl/monitoring/visualization/plots/default/workflow_resource_plots.py

@@ -164,7 +164,7 @@ def worker_efficiency(task, node):
                              y=[total_workers] * (end - start + 1),
                              name='Total of workers in whole run',
                              )
-                 ],
+                  ],
             layout=go.Layout(xaxis=dict(autorange=True,
                                         title='Time (seconds)'),
                              yaxis=dict(title='Number of workers'),
@@ -230,7 +230,7 @@ def resource_efficiency(resource, node, label):
                              y=[total] * (end - start + 1),
                              name=name2,
                              )
-                 ],
+                  ],
             layout=go.Layout(xaxis=dict(autorange=True,
                                         title='Time (seconds)'),
                              yaxis=dict(title=yaxis),

parsl/providers/slurm/slurm.py

@@ -250,7 +250,7 @@ class SlurmProvider(ClusterProvider, RepresentationMixin):
                     self.resources[job_id] = {'job_id': job_id, 'status': JobStatus(JobState.PENDING)}
                     break
             else:
-                logger.error("Could not read job ID from sumbit command standard output.")
+                logger.error("Could not read job ID from submit command standard output.")
                 logger.error("Retcode:%s STDOUT:%s STDERR:%s", retcode, stdout.strip(), stderr.strip())
         else:
             logger.error("Submit command failed")

parsl/tests/configs/ad_hoc_cluster_htex.py

@@ -9,8 +9,8 @@ user_opts = {'adhoc':
              {'username': 'YOUR_USERNAME',
               'script_dir': 'YOUR_SCRIPT_DIR',
               'remote_hostnames': ['REMOTE_HOST_URL_1', 'REMOTE_HOST_URL_2']
-             }
-}  # type: Dict[str, Dict[str, Any]]
+              }
+             }  # type: Dict[str, Dict[str, Any]]
 
 config = Config(
     executors=[
@@ -25,7 +25,7 @@ config = Config(
                 channels=[SSHChannel(hostname=m,
                                      username=user_opts['adhoc']['username'],
                                      script_dir=user_opts['adhoc']['script_dir'],
-                ) for m in user_opts['adhoc']['remote_hostnames']]
+                                     ) for m in user_opts['adhoc']['remote_hostnames']]
             )
         )
     ],

parsl/tests/configs/htex_ad_hoc_cluster.py

@@ -20,7 +20,7 @@ config = Config(
                 channels=[SSHChannel(hostname=m,
                                      username=user_opts['adhoc']['username'],
                                      script_dir=user_opts['adhoc']['script_dir'],
-                ) for m in user_opts['adhoc']['remote_hostnames']]
+                                     ) for m in user_opts['adhoc']['remote_hostnames']]
             )
         )
     ],

parsl/tests/configs/local_radical.py

@@ -0,0 +1,20 @@
+import os
+
+from parsl.config import Config
+from parsl.executors.radical import RadicalPilotExecutor
+from parsl.executors.radical import ResourceConfig
+
+
+rpex_cfg = ResourceConfig()
+
+
+def fresh_config():
+
+    return Config(
+            executors=[
+                RadicalPilotExecutor(
+                    label='RPEXBulk',
+                    rpex_cfg=rpex_cfg,
+                    bulk_mode=True,
+                    resource='local.localhost',
+                    runtime=30, cores=4)])

parsl/tests/configs/local_radical_mpi.py

@@ -0,0 +1,20 @@
+import os
+from parsl.config import Config
+
+
+def fresh_config():
+    from parsl.executors.radical import ResourceConfig
+    from parsl.executors.radical import RadicalPilotExecutor
+
+    rpex_cfg = ResourceConfig()
+    rpex_cfg.worker_type = "MPI"
+    rpex_cfg.worker_cores_per_node = 7
+
+    return Config(
+            executors=[
+                RadicalPilotExecutor(
+                    label='RPEXMPI',
+                    rpex_cfg=rpex_cfg,
+                    bulk_mode=True,
+                    resource='local.localhost',
+                    runtime=30, cores=8)])

parsl/tests/configs/local_threads_monitoring.py

@@ -8,4 +8,4 @@ config = Config(executors=[ThreadPoolExecutor(label='threads', max_threads=4)],
                     hub_port=55055,
                     resource_monitoring_interval=3,
                 )
-         )
+                )

parsl/tests/conftest.py

@@ -243,7 +243,7 @@ def setup_data(tmpd_cwd):
 
 
 @pytest.fixture(autouse=True, scope='function')
-def wait_for_task_completion(pytestconfig):
+def assert_no_outstanding_tasks(pytestconfig):
     """If we're in a config-file based mode, wait for task completion between
        each test. This will detect early on (by hanging) if particular test
        tasks are not finishing, rather than silently falling off the end of
@@ -254,7 +254,11 @@ def wait_for_task_completion(pytestconfig):
     config = pytestconfig.getoption('config')[0]
     yield
     if config != 'local':
-        parsl.dfk().wait_for_current_tasks()
+        logger.info("Checking no outstanding tasks")
+        for task_record in parsl.dfk().tasks.values():
+            fut = task_record['app_fu']
+            assert fut.done(), f"Incomplete task found, task id {task_record['id']}"
+        logger.info("No outstanding tasks found")
 
 
 def pytest_make_collect_report(collector):

parsl/tests/scaling_tests/vineex_condor.py

@@ -6,5 +6,5 @@ from parsl.providers import CondorProvider
 config = Config(
     executors=[TaskVineExecutor(manager_config=TaskVineManagerConfig(port=50055),
                                 provider=CondorProvider(),
-        )]
+                                )]
 )

parsl/tests/scaling_tests/vineex_local.py

@@ -7,5 +7,5 @@ config = Config(
     executors=[TaskVineExecutor(label='VineExec',
                                 worker_launch_method='factory',
                                 manager_config=TaskVineManagerConfig(port=50055),
-        )]
+                                )]
 )

parsl/tests/scaling_tests/wqex_condor.py

@@ -8,5 +8,5 @@ config = Config(
                                  provider=CondorProvider(),
                                  # init_command='source /home/yadu/src/wq_parsl/setup_parsl_env.sh;
                                  # echo "Ran at $date" > /home/yadu/src/wq_parsl/parsl/tests/workqueue_tests/ran.log',
-        )]
+                                 )]
 )

parsl/tests/scaling_tests/wqex_local.py

@@ -8,5 +8,5 @@ config = Config(
                                  provider=LocalProvider(),
                                  # init_command='source /home/yadu/src/wq_parsl/setup_parsl_env.sh;
                                  # echo "Ran at $date" > /home/yadu/src/wq_parsl/parsl/tests/workqueue_tests/ran.log',
-        )]
+                                 )]
 )

parsl/tests/test_docs/test_kwargs.py

@@ -0,0 +1,37 @@
+"""Functions used to explain kwargs"""
+from pathlib import Path
+
+from parsl import python_app, File
+
+
+def test_inputs():
+    @python_app()
+    def map_app(x):
+        return x * 2
+
+    @python_app()
+    def reduce_app(inputs=()):
+        return sum(inputs)
+
+    map_futures = [map_app(x) for x in range(3)]
+    reduce_future = reduce_app(inputs=map_futures)
+
+    assert reduce_future.result() == 6
+
+
+def test_outputs(tmpdir):
+    @python_app()
+    def write_app(message, outputs=()):
+        """Write a single message to every file in outputs"""
+        for path in outputs:
+            with open(path, 'w') as fp:
+                print(message, file=fp)
+
+    to_write = [
+        File(Path(tmpdir) / 'output-0.txt'),
+        File(Path(tmpdir) / 'output-1.txt')
+    ]
+    write_app('Hello!', outputs=to_write).result()
+    for path in to_write:
+        with open(path) as fp:
+            assert fp.read() == 'Hello!\n'

parsl/tests/test_python_apps/test_garbage_collect.py

@@ -5,7 +5,7 @@ import pytest
 
 import parsl
 from parsl.app.app import python_app
-from parsl.tests.configs.local_threads import fresh_config as local_config  # noqa
+from parsl.tests.configs.local_threads import fresh_config as local_config
 
 
 @python_app

parsl/tests/test_python_apps/test_lifted.py

@@ -89,11 +89,12 @@ def test_returns_a_class_instance():
 
 def test_returns_a_class_instance_no_underscores():
     # test that _underscore attribute references are not lifted
+    f = returns_a_class_instance()
     with pytest.raises(AttributeError):
-        returns_a_class_instance()._nosuchattribute.result()
+        f._nosuchattribute.result()
+    f.exception()  # wait for f to complete before the test ends
 
 
-@pytest.mark.skip("returning classes is not supported in WorkQueue or Task Vine - see issue #2908")
 def test_returns_a_class():
 
     # precondition that returns_a_class behaves

parsl/tests/test_radical/test_mpi_funcs.py

@@ -0,0 +1,27 @@
+import parsl
+import pytest
+
+from parsl.tests.configs.local_radical_mpi import fresh_config as local_config
+
+
+@parsl.python_app
+def test_mpi_func(msg, sleep, comm=None, parsl_resource_specification={}):
+    import time
+    msg = 'hello %d/%d: %s' % (comm.rank, comm.size, msg)
+    time.sleep(sleep)
+    print(msg)
+    return comm.size
+
+
+apps = []
+
+
+@pytest.mark.local
+def test_radical_mpi(n=7):
+    # rank size should be > 1 for the
+    # radical runtime system to run this function in MPI env
+    for i in range(2, n):
+        spec = {'ranks': i}
+        t = test_mpi_func(msg='mpi.func.%06d' % i, sleep=1, comm=None, parsl_resource_specification=spec)
+        apps.append(t)
+    assert [len(app.result()) for app in apps] == list(range(2, n))

parsl/tests/test_regression/test_1606_wait_for_current_tasks.py

@@ -4,7 +4,7 @@ import time
 import pytest
 
 import parsl
-from parsl.tests.configs.local_threads import fresh_config as local_config  # noqa
+from parsl.tests.configs.local_threads import fresh_config as local_config
 
 
 @parsl.python_app

parsl/utils.py

@@ -7,7 +7,7 @@ import threading
 import time
 from contextlib import contextmanager
 from types import TracebackType
-from typing import Any, Callable, List, Tuple, Union, Generator, IO, AnyStr, Dict, Optional
+from typing import Any, Callable, List, Sequence, Tuple, Union, Generator, IO, AnyStr, Dict, Optional
 
 import typeguard
 from typing_extensions import Type
@@ -47,7 +47,7 @@ def get_version() -> str:
 
 
 @typeguard.typechecked
-def get_all_checkpoints(rundir: str = "runinfo") -> List[str]:
+def get_all_checkpoints(rundir: str = "runinfo") -> Sequence[str]:
     """Finds the checkpoints from all runs in the rundir.
 
     Kwargs:
@@ -76,7 +76,7 @@ def get_all_checkpoints(rundir: str = "runinfo") -> List[str]:
 
 
 @typeguard.typechecked
-def get_last_checkpoint(rundir: str = "runinfo") -> List[str]:
+def get_last_checkpoint(rundir: str = "runinfo") -> Sequence[str]:
     """Finds the checkpoint from the last run, if one exists.
 
     Note that checkpoints are incremental, and this helper will not find
@@ -128,7 +128,7 @@ def get_std_fname_mode(
 
 @contextmanager
 def wait_for_file(path: str, seconds: int = 10) -> Generator[None, None, None]:
-    for i in range(0, int(seconds * 100)):
+    for _ in range(0, int(seconds * 100)):
         time.sleep(seconds / 100.)
         if os.path.exists(path):
             break

parsl/version.py

@@ -3,4 +3,4 @@
 Year.Month.Day[alpha/beta/..]
 Alphas will be numbered like this -> 2024.12.10a0
 """
-VERSION = '2023.10.23'
+VERSION = '2023.11.20'

{parsl-2023.10.23.data → parsl-2023.11.20.data}/scripts/exec_parsl_function.py

@@ -4,6 +4,7 @@ from parsl.utils import get_std_fname_mode
 import traceback
 import sys
 import pickle
+from parsl.serialize import serialize
 
 # This scripts executes a parsl function which is pickled in a file:
 #
@@ -32,7 +33,7 @@ def load_pickled_file(filename):
 
 def dump_result_to_file(result_file, result_package):
     with open(result_file, "wb") as f_out:
-        pickle.dump(result_package, f_out)
+        f_out.write(serialize(result_package))
 
 
 def remap_location(mapping, parsl_file):

{parsl-2023.10.23.data → parsl-2023.11.20.data}/scripts/process_worker_pool.py

@@ -234,7 +234,7 @@ class Manager:
                'dir': os.getcwd(),
                'cpu_count': psutil.cpu_count(logical=False),
                'total_memory': psutil.virtual_memory().total,
-        }
+               }
         b_msg = json.dumps(msg).encode('utf-8')
         return b_msg
 
@@ -608,7 +608,7 @@ def worker(worker_id, pool_id, pool_size, task_queue, result_queue, worker_queue
             logger.exception("Caught exception while trying to pickle the result package")
             pkl_package = pickle.dumps({'type': 'result', 'task_id': tid,
                                         'exception': serialize(RemoteExceptionWrapper(*sys.exc_info()))
-            })
+                                        })
 
         result_queue.put(pkl_package)
         tasks_in_progress.pop(worker_id)

{parsl-2023.10.23.dist-info → parsl-2023.11.20.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: parsl
-Version: 2023.10.23
+Version: 2023.11.20
 Summary: Simple data dependent workflows in Python
 Home-page: https://github.com/Parsl/parsl
-Download-URL: https://github.com/Parsl/parsl/archive/2023.10.23.tar.gz
+Download-URL: https://github.com/Parsl/parsl/archive/2023.11.20.tar.gz
 Author: The Parsl Team
 Author-email: parsl@googlegroups.com
 License: Apache 2.0
@@ -53,6 +53,7 @@ Requires-Dist: pyyaml ; extra == 'all'
 Requires-Dist: cffi ; extra == 'all'
 Requires-Dist: jsonschema ; extra == 'all'
 Requires-Dist: proxystore ; extra == 'all'
+Requires-Dist: radical.pilot ; extra == 'all'
 Provides-Extra: aws
 Requires-Dist: boto3 ; extra == 'aws'
 Provides-Extra: azure
@@ -79,6 +80,8 @@ Provides-Extra: oauth_ssh
 Requires-Dist: oauth-ssh >=0.9 ; extra == 'oauth_ssh'
 Provides-Extra: proxystore
 Requires-Dist: proxystore ; extra == 'proxystore'
+Provides-Extra: radical-pilot
+Requires-Dist: radical.pilot ; extra == 'radical-pilot'
 Provides-Extra: visualization
 Requires-Dist: pydot ; extra == 'visualization'
 Requires-Dist: networkx <2.6,>=2.5 ; extra == 'visualization'