parsl 2023.12.25__tar.gz → 2024.1.8__tar.gz

{parsl-2023.12.25/parsl.egg-info → parsl-2024.1.8}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: parsl
-Version: 2023.12.25
+Version: 2024.1.8
 Summary: Simple data dependent workflows in Python
 Home-page: https://github.com/Parsl/parsl
-Download-URL: https://github.com/Parsl/parsl/archive/2023.12.25.tar.gz
+Download-URL: https://github.com/Parsl/parsl/archive/2024.01.08.tar.gz
 Author: The Parsl Team
 Author-email: parsl@googlegroups.com
 License: Apache 2.0
@@ -15,6 +15,7 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
 Requires-Python: >=3.8.0
 Provides-Extra: monitoring
 Provides-Extra: visualization

{parsl-2023.12.25 → parsl-2024.1.8}/parsl/addresses.py

@@ -72,8 +72,9 @@ def address_by_hostname() -> str:
     """
     logger.debug("Finding address by using local hostname")
     addr = platform.node()
-    logger.debug("Address found: {}".format(addr))
-    return addr
+    ip_addr = socket.gethostbyname(addr)
+    logger.debug("Address found: {}".format(ip_addr))
+    return ip_addr
 
 
 @typeguard.typechecked

{parsl-2023.12.25 → parsl-2024.1.8}/parsl/dataflow/dflow.py

@@ -262,9 +262,8 @@ class DataFlowKernel:
         """
         count = 0
         for dep in depends:
-            if isinstance(dep, Future):
-                if not dep.done():
-                    count += 1
+            if not dep.done():
+                count += 1
 
         return count
 
@@ -527,9 +526,7 @@ class DataFlowKernel:
         # or do nothing?
         if self.checkpoint_mode == 'task_exit':
             self.checkpoint(tasks=[task_record])
-        elif self.checkpoint_mode == 'manual' or \
-                self.checkpoint_mode == 'periodic' or \
-                self.checkpoint_mode == 'dfk_exit':
+        elif self.checkpoint_mode in ('manual', 'periodic', 'dfk_exit'):
             with self.checkpoint_lock:
                 self.checkpointable_tasks.append(task_record)
         elif self.checkpoint_mode is None:
@@ -1299,11 +1296,7 @@ class DataFlowKernel:
                         hashsum = task_record['hashsum']
                         if not hashsum:
                             continue
-                        t = {'hash': hashsum,
-                             'exception': None,
-                             'result': None}
-
-                        t['result'] = app_fu.result()
+                        t = {'hash': hashsum, 'exception': None, 'result': app_fu.result()}
 
                         # We are using pickle here since pickle dumps to a file in 'ab'
                         # mode behave like a incremental log.

{parsl-2023.12.25 → parsl-2024.1.8}/parsl/executors/radical/executor.py

@@ -53,12 +53,13 @@ class RadicalPilotExecutor(ParslExecutor, RepresentationMixin):
     """Executor is designed for executing heterogeneous tasks
        in terms of type/resource.
 
-    The RadicalPilotExecutor system has the following components:
+    The RadicalPilotExecutor system has the following main components:
 
-      1. "start"    :creating the RADICAL-executor session and pilot.
-      2. "translate":unwrap/identify/ out of parsl task and construct RP task.
-      3. "submit"   :translating and submitting Parsl tasks to Radical Pilot.
-      4. "shut_down":shutting down the RADICAL-executor components.
+      1. "start": Create and start the RADICAL-Pilot runtime components ``rp.Session``,
+          ``rp.PilotManager`` and ``rp.TaskManager``.
+      2. "translate": Unwrap, identify, and parse Parsl ``apps`` into ``rp.TaskDescription``.
+      3. "submit": Submit Parsl apps to ``rp.TaskManager``.
+      4. "shut_down": Shut down the RADICAL-Pilot runtime and all associated components.
 
     Here is a diagram
 
@@ -75,7 +76,7 @@ class RadicalPilotExecutor(ParslExecutor, RepresentationMixin):
       ----------------------------------------------------------------------------
 
     The RadicalPilotExecutor creates a ``rp.Session``, ``rp.TaskManager``,
-    and ``rp.PilotManager``. The executor receives the parsl apps from the
+    and ``rp.PilotManager``. The executor receives the Parsl apps from the
     DFK and translates these apps (in-memory) into ``rp.TaskDescription``
     object to be passed to the ``rp.TaskManager``. This executor has two
     submission mechanisms:
@@ -146,7 +147,7 @@ class RadicalPilotExecutor(ParslExecutor, RepresentationMixin):
 
     def task_state_cb(self, task, state):
         """
-        Update the state of Parsl Future tasks
+        Update the state of Parsl Future apps
         Based on RP task state callbacks.
         """
         if not task.uid.startswith('master'):
@@ -275,7 +276,7 @@ class RadicalPilotExecutor(ParslExecutor, RepresentationMixin):
 
     def unwrap(self, func, args):
         """
-        Unwrap a parsl app and its args for further processing.
+        Unwrap a Parsl app and its args for further processing.
 
         Parameters
         ----------
@@ -325,14 +326,14 @@ class RadicalPilotExecutor(ParslExecutor, RepresentationMixin):
 
     def task_translate(self, tid, func, parsl_resource_specification, args, kwargs):
         """
-        Convert parsl function to RADICAL-Pilot rp.TaskDescription
+        Convert Parsl function to RADICAL-Pilot rp.TaskDescription
         """
 
         task = rp.TaskDescription()
         task.name = func.__name__
 
         if parsl_resource_specification and isinstance(parsl_resource_specification, dict):
-            logger.debug('mapping parsl resource specifications >> rp resource specifications')
+            logger.debug('mapping Parsl resource specifications >> RP resource specifications')
             for key, val in parsl_resource_specification.items():
                 if key not in task.as_dict():
                     key = PARSL_RP_RESOURCE_MAP.get(key, None)
@@ -523,7 +524,7 @@ class RadicalPilotExecutor(ParslExecutor, RepresentationMixin):
                                 ns=self.session.uid)
         parsl_tid = int(rp_tid.split('task.')[1])
 
-        logger.debug("got Task {0} from parsl-dfk".format(parsl_tid))
+        logger.debug("got Task {0} from Parsl-dfk".format(parsl_tid))
         task = self.task_translate(parsl_tid, func, resource_specification, args, kwargs)
 
         # assign task id for rp task

{parsl-2023.12.25 → parsl-2024.1.8}/parsl/executors/workqueue/executor.py

@@ -959,43 +959,42 @@ def _work_queue_submit_wait(*,
 
         # If the queue is not empty wait on the WorkQueue queue for a task
         task_found = True
-        if not q.empty():
-            while task_found and not should_stop.value:
-                # Obtain the task from the queue
-                t = q.wait(1)
-                if t is None:
-                    task_found = False
-                    continue
-                # When a task is found:
-                executor_task_id = t.tag
-                logger.debug("Completed Work Queue task {}, executor task {}".format(t.id, t.tag))
-                result_file = result_file_of_task_id.pop(t.tag)
-
-                # A tasks completes 'succesfully' if it has result file.
-                # The check whether this file can load a serialized Python object
-                # happens later in the collector thread of the executor process.
-                logger.debug("Looking for result in {}".format(result_file))
-                if os.path.exists(result_file):
-                    logger.debug("Found result in {}".format(result_file))
-                    collector_queue.put_nowait(WqTaskToParsl(id=executor_task_id,
-                                                             result_received=True,
-                                                             result_file=result_file,
-                                                             reason=None,
-                                                             status=t.return_status))
-                # If a result file could not be generated, explain the
-                # failure according to work queue error codes.
-                else:
-                    reason = _explain_work_queue_result(t)
-                    logger.debug("Did not find result in {}".format(result_file))
-                    logger.debug("Wrapper Script status: {}\nWorkQueue Status: {}"
-                                 .format(t.return_status, t.result))
-                    logger.debug("Task with executor id {} / Work Queue id {} failed because:\n{}"
-                                 .format(executor_task_id, t.id, reason))
-                    collector_queue.put_nowait(WqTaskToParsl(id=executor_task_id,
-                                                             result_received=False,
-                                                             result_file=None,
-                                                             reason=reason,
-                                                             status=t.return_status))
+        while not q.empty() and task_found and not should_stop.value:
+            # Obtain the task from the queue
+            t = q.wait(1)
+            if t is None:
+                task_found = False
+                continue
+            # When a task is found:
+            executor_task_id = t.tag
+            logger.debug("Completed Work Queue task {}, executor task {}".format(t.id, t.tag))
+            result_file = result_file_of_task_id.pop(t.tag)
+
+            # A tasks completes 'succesfully' if it has result file.
+            # The check whether this file can load a serialized Python object
+            # happens later in the collector thread of the executor process.
+            logger.debug("Looking for result in {}".format(result_file))
+            if os.path.exists(result_file):
+                logger.debug("Found result in {}".format(result_file))
+                collector_queue.put_nowait(WqTaskToParsl(id=executor_task_id,
+                                                         result_received=True,
+                                                         result_file=result_file,
+                                                         reason=None,
+                                                         status=t.return_status))
+            # If a result file could not be generated, explain the
+            # failure according to work queue error codes.
+            else:
+                reason = _explain_work_queue_result(t)
+                logger.debug("Did not find result in {}".format(result_file))
+                logger.debug("Wrapper Script status: {}\nWorkQueue Status: {}"
+                             .format(t.return_status, t.result))
+                logger.debug("Task with executor id {} / Work Queue id {} failed because:\n{}"
+                             .format(executor_task_id, t.id, reason))
+                collector_queue.put_nowait(WqTaskToParsl(id=executor_task_id,
+                                                         result_received=False,
+                                                         result_file=None,
+                                                         reason=reason,
+                                                         status=t.return_status))
     logger.debug("Exiting WorkQueue Monitoring Process")
     return 0
 

{parsl-2023.12.25 → parsl-2024.1.8}/parsl/tests/configs/local_radical.py

@@ -15,6 +15,5 @@ def fresh_config():
                 RadicalPilotExecutor(
                     label='RPEXBulk',
                     rpex_cfg=rpex_cfg,
-                    bulk_mode=True,
                     resource='local.localhost',
                     runtime=30, cores=4)])

{parsl-2023.12.25 → parsl-2024.1.8}/parsl/tests/conftest.py

@@ -3,6 +3,8 @@ import itertools
 import logging
 import os
 import pathlib
+import re
+import shutil
 import time
 import types
 import signal
@@ -44,17 +46,46 @@ def pytest_sessionstart(session):
 
 
 @pytest.fixture(scope="session")
-def tmpd_cwd_session():
-    n = datetime.now().strftime('%Y%m%d.%H%I%S')
-    with tempfile.TemporaryDirectory(dir=os.getcwd(), prefix=f".pytest-{n}-") as tmpd:
-        yield pathlib.Path(tmpd)
+def tmpd_cwd_session(pytestconfig):
+    config = re.sub(r"[^A-z0-9_-]+", "_", pytestconfig.getoption('config')[0])
+    cwd = pathlib.Path(os.getcwd())
+    pytest_dir = cwd / ".pytest"
+    pytest_dir.mkdir(mode=0o700, parents=True, exist_ok=True)
+
+    test_dir_prefix = "parsltest-"
+    link = pytest_dir / f"{test_dir_prefix}current"
+    link.unlink(missing_ok=True)
+    n = datetime.now().strftime('%Y%m%d.%H%M%S')
+    tmpd = tempfile.mkdtemp(dir=pytest_dir, prefix=f"{test_dir_prefix}{n}-{config}-")
+    tmpd = pathlib.Path(tmpd)
+    link.symlink_to(tmpd.name)
+    yield link
+
+    try:
+        preserve = int(os.getenv("PARSL_TEST_PRESERVE_NUM_RUNS", "3"))
+    except Exception:
+        preserve = 3
+
+    test_runs = sorted(
+        d for d in pytest_dir.glob(f"{test_dir_prefix}*")
+        if d.is_dir() and not d.is_symlink()
+    )
+    for run_to_remove in test_runs[:-preserve]:
+        run_to_remove.chmod(0o700)
+        for root, subdirnames, fnames in os.walk(run_to_remove):
+            rpath = pathlib.Path(root)
+            for d in subdirnames:
+                (rpath / d).lchmod(0o700)
+            for f in fnames:
+                (rpath / f).lchmod(0o600)
+        shutil.rmtree(run_to_remove)
 
 
 @pytest.fixture
 def tmpd_cwd(tmpd_cwd_session, request):
     prefix = f"{request.node.name}-"
-    with tempfile.TemporaryDirectory(dir=tmpd_cwd_session, prefix=prefix) as tmpd:
-        yield pathlib.Path(tmpd)
+    tmpd = tempfile.mkdtemp(dir=tmpd_cwd_session, prefix=prefix)
+    yield pathlib.Path(tmpd)
 
 
 def pytest_addoption(parser):
@@ -118,7 +149,7 @@ def pytest_configure(config):
 
 
 @pytest.fixture(autouse=True, scope='session')
-def load_dfk_session(request, pytestconfig):
+def load_dfk_session(request, pytestconfig, tmpd_cwd_session):
     """Load a dfk around entire test suite, except in local mode.
 
     The special path `local` indicates that configuration will not come
@@ -137,12 +168,19 @@ def load_dfk_session(request, pytestconfig):
             raise RuntimeError("DFK didn't start as None - there was a DFK from somewhere already")
 
         if hasattr(module, 'config'):
-            dfk = parsl.load(module.config)
+            parsl_conf = module.config
         elif hasattr(module, 'fresh_config'):
-            dfk = parsl.load(module.fresh_config())
+            parsl_conf = module.fresh_config()
         else:
             raise RuntimeError("Config module does not define config or fresh_config")
 
+        if parsl_conf.run_dir == "runinfo":  # the default
+            parsl_conf.run_dir = tmpd_cwd_session / parsl_conf.run_dir
+        dfk = parsl.load(parsl_conf)
+
+        for ex in dfk.executors.values():
+            ex.working_dir = tmpd_cwd_session
+
         yield
 
         if parsl.dfk() != dfk:
@@ -154,7 +192,7 @@ def load_dfk_session(request, pytestconfig):
 
 
 @pytest.fixture(autouse=True, scope='module')
-def load_dfk_local_module(request, pytestconfig):
+def load_dfk_local_module(request, pytestconfig, tmpd_cwd_session):
     """Load the dfk around test modules, in local mode.
 
     If local_config is specified in the test module, it will be loaded using
@@ -177,8 +215,15 @@ def load_dfk_local_module(request, pytestconfig):
             assert callable(local_config)
             c = local_config()
             assert isinstance(c, parsl.Config)
+
+            if c.run_dir == "runinfo":  # the default
+                c.run_dir = tmpd_cwd_session / c.run_dir
+
             dfk = parsl.load(c)
 
+            for ex in dfk.executors.values():
+                ex.working_dir = tmpd_cwd_session
+
         if callable(local_setup):
             local_setup()
 

parsl-2024.1.8/parsl/tests/test_checkpointing/test_python_checkpoint_1.py

@@ -0,0 +1,33 @@
+import os
+import pytest
+
+import parsl
+from parsl import python_app
+from parsl.tests.configs.local_threads import fresh_config
+
+
+def local_config():
+    config = fresh_config()
+    config.checkpoint_mode = "manual"
+    return config
+
+
+@python_app(cache=True)
+def uuid_app():
+    import uuid
+    return uuid.uuid4()
+
+
+@pytest.mark.local
+def test_initial_checkpoint_write():
+    """1. Launch a few apps and write the checkpoint once a few have completed
+    """
+    uuid_app().result()
+
+    cpt_dir = parsl.dfk().checkpoint()
+
+    cptpath = cpt_dir + '/dfk.pkl'
+    assert os.path.exists(cptpath), f"DFK checkpoint missing: {cptpath}"
+
+    cptpath = cpt_dir + '/tasks.pkl'
+    assert os.path.exists(cptpath), f"Tasks checkpoint missing: {cptpath}"

parsl-2024.1.8/parsl/tests/test_checkpointing/test_python_checkpoint_2.py

@@ -0,0 +1,42 @@
+import contextlib
+import os
+import pytest
+import parsl
+from parsl import python_app
+
+from parsl.tests.configs.local_threads_checkpoint import fresh_config
+
+
+@contextlib.contextmanager
+def parsl_configured(run_dir, **kw):
+    c = fresh_config()
+    c.run_dir = run_dir
+    for config_attr, config_val in kw.items():
+        setattr(c, config_attr, config_val)
+    dfk = parsl.load(c)
+    for ex in dfk.executors.values():
+        ex.working_dir = run_dir
+    yield dfk
+
+    parsl.dfk().cleanup()
+    parsl.clear()
+
+
+@python_app(cache=True)
+def uuid_app():
+    import uuid
+    return uuid.uuid4()
+
+
+@pytest.mark.local
+def test_loading_checkpoint(tmpd_cwd):
+    """Load memoization table from previous checkpoint
+    """
+    with parsl_configured(tmpd_cwd, checkpoint_mode="task_exit"):
+        checkpoint_files = [os.path.join(parsl.dfk().run_dir, "checkpoint")]
+        result = uuid_app().result()
+
+    with parsl_configured(tmpd_cwd, checkpoint_files=checkpoint_files):
+        relaunched = uuid_app().result()
+
+    assert result == relaunched, "Expected following call to uuid_app to return cached uuid"

parsl-2024.1.8/parsl/tests/test_providers/test_slurm_instantiate.py

@@ -0,0 +1,16 @@
+import pytest
+
+from parsl.providers import SlurmProvider
+
+
+@pytest.mark.local
+def test_slurm_instantiate_regression_2994():
+    """This test checks that repr can be executed on SlurmProvider.
+    This does not need a SLURM installation around.
+
+    Because of the behaviour of RepresentationMixin, it's a relatively
+    common problem to break repr when adding new parameters to the
+    SlurmProvider, and this tests that repr does not raise an exception.
+    """
+    p = SlurmProvider()
+    repr(p)

{parsl-2023.12.25 → parsl-2024.1.8}/parsl/tests/test_radical/test_mpi_funcs.py

@@ -5,7 +5,7 @@ from parsl.tests.configs.local_radical_mpi import fresh_config as local_config
 
 
 @parsl.python_app
-def test_mpi_func(msg, sleep, comm=None, parsl_resource_specification={}):
+def some_mpi_func(msg, sleep, comm=None, parsl_resource_specification={}):
     import time
     msg = 'hello %d/%d: %s' % (comm.rank, comm.size, msg)
     time.sleep(sleep)
@@ -17,11 +17,12 @@ apps = []
 
 
 @pytest.mark.local
+@pytest.mark.radical
 def test_radical_mpi(n=7):
     # rank size should be > 1 for the
     # radical runtime system to run this function in MPI env
     for i in range(2, n):
         spec = {'ranks': i}
-        t = test_mpi_func(msg='mpi.func.%06d' % i, sleep=1, comm=None, parsl_resource_specification=spec)
+        t = some_mpi_func(msg='mpi.func.%06d' % i, sleep=1, comm=None, parsl_resource_specification=spec)
         apps.append(t)
     assert [len(app.result()) for app in apps] == list(range(2, n))

parsl-2024.1.8/parsl/tests/test_staging/__init__.py

@@ -0,0 +1,7 @@
+def read_sort_write(in_path, out_path):
+    with open(in_path) as u:
+        strs = u.read().split()
+        strs.sort()
+    with open(out_path, 'w') as s:
+        for e in strs:
+            print(e, file=s)

{parsl-2023.12.25 → parsl-2024.1.8}/parsl/tests/test_staging/test_elaborate_noop_file.py

@@ -20,8 +20,8 @@ logger = logging.getLogger(__name__)
 
 
 @bash_app
-def touch(filename, inputs=[], outputs=[]):
-    return "touch {}".format(filename)
+def touch(filename, outputs=()):
+    return f"touch {filename}"
 
 
 @python_app
@@ -31,52 +31,47 @@ def app_test_in(file):
     pass
 
 
+@pytest.fixture
+def storage_access_parsl():
+    def _setup_config(*args, **kwargs):
+        tpe = ThreadPoolExecutor(
+            label='local_threads',
+            storage_access=[NoOpTestingFileStaging(*args, **kwargs)]
+        )
+        config = Config(executors=[tpe])
+        parsl.load(config)
+
+    yield _setup_config
+
+    parsl.dfk().cleanup()
+    parsl.clear()
+
+
 @pytest.mark.local
-def test_regression_stage_out_does_not_stage_in():
-    no_stageout_config = Config(
-        executors=[
-            ThreadPoolExecutor(
-                label='local_threads',
-                storage_access=[NoOpTestingFileStaging(allow_stage_in=False)]
-            )
-        ]
-    )
-
-    parsl.load(no_stageout_config)
+def test_regression_stage_out_does_not_stage_in(storage_access_parsl, tmpd_cwd):
+    storage_access_parsl(allow_stage_in=False)
 
     # Test that the helper app runs with no staging
-    touch("test.1", outputs=[]).result()
+    touch(str(tmpd_cwd / "test.1"), outputs=[]).result()
 
-    # Test with stage-out, checking that provider stage in is never
-    # invoked. If stage-in is invoked, the the NoOpTestingFileStaging
+    # Test with stage-out, checking that provider stage-in is never
+    # invoked. If stage-in is invoked, then the NoOpTestingFileStaging
     # provider will raise an exception, which should propagate to
     # .result() here.
-    touch("test.2", outputs=[File("test.2")]).result()
+    fpath = tmpd_cwd / "test.2"
+    touch(str(fpath), outputs=[File(fpath)]).result()
 
     # Test that stage-in exceptions propagate out to user code.
     with pytest.raises(NoOpError):
         touch("test.3", inputs=[File("test.3")]).result()
 
-    parsl.dfk().cleanup()
-    parsl.clear()
-
 
 @pytest.mark.local
-def test_regression_stage_in_does_not_stage_out():
-    no_stageout_config = Config(
-        executors=[
-            ThreadPoolExecutor(
-                label='local_threads',
-                storage_access=[NoOpTestingFileStaging(allow_stage_out=False)]
-            )
-        ],
-    )
-
-    parsl.load(no_stageout_config)
-
-    f = open("test.4", "a")
-    f.write("test")
-    f.close()
+def test_regression_stage_in_does_not_stage_out(storage_access_parsl, tmpd_cwd):
+    storage_access_parsl(allow_stage_out=False)
+
+    fpath = tmpd_cwd / "test.4"
+    fpath.write_text("test")
 
     # Test that stage in does not invoke stage out. If stage out is
     # attempted, then the NoOpTestingFileStaging provider will raise
@@ -86,6 +81,3 @@ def test_regression_stage_in_does_not_stage_out():
     # Test that stage out exceptions propagate to user code.
     with pytest.raises(NoOpError):
         touch("test.5", outputs=[File("test.5")]).result()
-
-    parsl.dfk().cleanup()
-    parsl.clear()